1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015-2019 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_common.h> 14 #include <rte_branch_prediction.h> 15 #include <rte_ether.h> 16 #include <rte_cycles.h> 17 #include <rte_flow.h> 18 19 #include <mlx5_prm.h> 20 #include <mlx5_common.h> 21 22 #include "mlx5_autoconf.h" 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_mr.h" 26 #include "mlx5_utils.h" 27 #include "mlx5_rxtx.h" 28 29 /* TX burst subroutines return codes. */ 30 enum mlx5_txcmp_code { 31 MLX5_TXCMP_CODE_EXIT = 0, 32 MLX5_TXCMP_CODE_ERROR, 33 MLX5_TXCMP_CODE_SINGLE, 34 MLX5_TXCMP_CODE_MULTI, 35 MLX5_TXCMP_CODE_TSO, 36 MLX5_TXCMP_CODE_EMPW, 37 }; 38 39 /* 40 * These defines are used to configure Tx burst routine option set 41 * supported at compile time. The not specified options are optimized out 42 * out due to if conditions can be explicitly calculated at compile time. 43 * The offloads with bigger runtime check (require more CPU cycles to 44 * skip) overhead should have the bigger index - this is needed to 45 * select the better matching routine function if no exact match and 46 * some offloads are not actually requested. 47 */ 48 #define MLX5_TXOFF_CONFIG_MULTI (1u << 0) /* Multi-segment packets.*/ 49 #define MLX5_TXOFF_CONFIG_TSO (1u << 1) /* TCP send offload supported.*/ 50 #define MLX5_TXOFF_CONFIG_SWP (1u << 2) /* Tunnels/SW Parser offloads.*/ 51 #define MLX5_TXOFF_CONFIG_CSUM (1u << 3) /* Check Sums offloaded. */ 52 #define MLX5_TXOFF_CONFIG_INLINE (1u << 4) /* Data inlining supported. */ 53 #define MLX5_TXOFF_CONFIG_VLAN (1u << 5) /* VLAN insertion supported.*/ 54 #define MLX5_TXOFF_CONFIG_METADATA (1u << 6) /* Flow metadata. */ 55 #define MLX5_TXOFF_CONFIG_EMPW (1u << 8) /* Enhanced MPW supported.*/ 56 #define MLX5_TXOFF_CONFIG_MPW (1u << 9) /* Legacy MPW supported.*/ 57 #define MLX5_TXOFF_CONFIG_TXPP (1u << 10) /* Scheduling on timestamp.*/ 58 59 /* The most common offloads groups. */ 60 #define MLX5_TXOFF_CONFIG_NONE 0 61 #define MLX5_TXOFF_CONFIG_FULL (MLX5_TXOFF_CONFIG_MULTI | \ 62 MLX5_TXOFF_CONFIG_TSO | \ 63 MLX5_TXOFF_CONFIG_SWP | \ 64 MLX5_TXOFF_CONFIG_CSUM | \ 65 MLX5_TXOFF_CONFIG_INLINE | \ 66 MLX5_TXOFF_CONFIG_VLAN | \ 67 MLX5_TXOFF_CONFIG_METADATA) 68 69 #define MLX5_TXOFF_CONFIG(mask) (olx & MLX5_TXOFF_CONFIG_##mask) 70 71 #define MLX5_TXOFF_DECL(func, olx) \ 72 static uint16_t mlx5_tx_burst_##func(void *txq, \ 73 struct rte_mbuf **pkts, \ 74 uint16_t pkts_n) \ 75 { \ 76 return mlx5_tx_burst_tmpl((struct mlx5_txq_data *)txq, \ 77 pkts, pkts_n, (olx)); \ 78 } 79 80 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx}, 81 82 /* static asserts */ 83 static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value"); 84 static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value"); 85 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 86 (sizeof(uint16_t) + 87 sizeof(rte_v128u32_t)), 88 "invalid Ethernet Segment data size"); 89 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 90 (sizeof(uint16_t) + 91 sizeof(struct rte_vlan_hdr) + 92 2 * RTE_ETHER_ADDR_LEN), 93 "invalid Ethernet Segment data size"); 94 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 95 (sizeof(uint16_t) + 96 sizeof(rte_v128u32_t)), 97 "invalid Ethernet Segment data size"); 98 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 99 (sizeof(uint16_t) + 100 sizeof(struct rte_vlan_hdr) + 101 2 * RTE_ETHER_ADDR_LEN), 102 "invalid Ethernet Segment data size"); 103 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 104 (sizeof(uint16_t) + 105 sizeof(rte_v128u32_t)), 106 "invalid Ethernet Segment data size"); 107 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 108 (sizeof(uint16_t) + 109 sizeof(struct rte_vlan_hdr) + 110 2 * RTE_ETHER_ADDR_LEN), 111 "invalid Ethernet Segment data size"); 112 static_assert(MLX5_DSEG_MIN_INLINE_SIZE == 113 (2 * RTE_ETHER_ADDR_LEN), 114 "invalid Data Segment data size"); 115 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size"); 116 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size"); 117 static_assert((sizeof(struct rte_vlan_hdr) + 118 sizeof(struct rte_ether_hdr)) == 119 MLX5_ESEG_MIN_INLINE_SIZE, 120 "invalid min inline data size"); 121 static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <= 122 MLX5_DSEG_MAX, "invalid WQE max size"); 123 static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE, 124 "invalid WQE Control Segment size"); 125 static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE, 126 "invalid WQE Ethernet Segment size"); 127 static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE, 128 "invalid WQE Data Segment size"); 129 static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE, 130 "invalid WQE size"); 131 132 static __rte_always_inline uint32_t 133 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 134 volatile struct mlx5_mini_cqe8 *mcqe); 135 136 static __rte_always_inline int 137 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 138 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe); 139 140 static __rte_always_inline uint32_t 141 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe); 142 143 static __rte_always_inline void 144 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 145 volatile struct mlx5_cqe *cqe, 146 volatile struct mlx5_mini_cqe8 *mcqe); 147 148 static int 149 mlx5_queue_state_modify(struct rte_eth_dev *dev, 150 struct mlx5_mp_arg_queue_state_modify *sm); 151 152 static inline void 153 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp, 154 volatile struct mlx5_cqe *__rte_restrict cqe, 155 uint32_t phcsum, uint8_t l4_type); 156 157 static inline void 158 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd, 159 volatile struct mlx5_cqe *__rte_restrict cqe, 160 volatile struct mlx5_mini_cqe8 *mcqe, 161 struct mlx5_rxq_data *rxq, uint32_t len); 162 163 uint32_t mlx5_ptype_table[] __rte_cache_aligned = { 164 [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */ 165 }; 166 167 uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned; 168 uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned; 169 170 uint64_t rte_net_mlx5_dynf_inline_mask; 171 #define PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask 172 173 /** 174 * Build a table to translate Rx completion flags to packet type. 175 * 176 * @note: fix mlx5_dev_supported_ptypes_get() if any change here. 177 */ 178 void 179 mlx5_set_ptype_table(void) 180 { 181 unsigned int i; 182 uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table; 183 184 /* Last entry must not be overwritten, reserved for errored packet. */ 185 for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i) 186 (*p)[i] = RTE_PTYPE_UNKNOWN; 187 /* 188 * The index to the array should have: 189 * bit[1:0] = l3_hdr_type 190 * bit[4:2] = l4_hdr_type 191 * bit[5] = ip_frag 192 * bit[6] = tunneled 193 * bit[7] = outer_l3_type 194 */ 195 /* L2 */ 196 (*p)[0x00] = RTE_PTYPE_L2_ETHER; 197 /* L3 */ 198 (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 199 RTE_PTYPE_L4_NONFRAG; 200 (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 201 RTE_PTYPE_L4_NONFRAG; 202 /* Fragmented */ 203 (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 204 RTE_PTYPE_L4_FRAG; 205 (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 206 RTE_PTYPE_L4_FRAG; 207 /* TCP */ 208 (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 209 RTE_PTYPE_L4_TCP; 210 (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 211 RTE_PTYPE_L4_TCP; 212 (*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 213 RTE_PTYPE_L4_TCP; 214 (*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 215 RTE_PTYPE_L4_TCP; 216 (*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 217 RTE_PTYPE_L4_TCP; 218 (*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 219 RTE_PTYPE_L4_TCP; 220 /* UDP */ 221 (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 222 RTE_PTYPE_L4_UDP; 223 (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 224 RTE_PTYPE_L4_UDP; 225 /* Repeat with outer_l3_type being set. Just in case. */ 226 (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 227 RTE_PTYPE_L4_NONFRAG; 228 (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 229 RTE_PTYPE_L4_NONFRAG; 230 (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 231 RTE_PTYPE_L4_FRAG; 232 (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 233 RTE_PTYPE_L4_FRAG; 234 (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 235 RTE_PTYPE_L4_TCP; 236 (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 237 RTE_PTYPE_L4_TCP; 238 (*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 239 RTE_PTYPE_L4_TCP; 240 (*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 241 RTE_PTYPE_L4_TCP; 242 (*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 243 RTE_PTYPE_L4_TCP; 244 (*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 245 RTE_PTYPE_L4_TCP; 246 (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 247 RTE_PTYPE_L4_UDP; 248 (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 249 RTE_PTYPE_L4_UDP; 250 /* Tunneled - L3 */ 251 (*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 252 (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 253 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 254 RTE_PTYPE_INNER_L4_NONFRAG; 255 (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 256 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 257 RTE_PTYPE_INNER_L4_NONFRAG; 258 (*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 259 (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 260 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 261 RTE_PTYPE_INNER_L4_NONFRAG; 262 (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 263 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 264 RTE_PTYPE_INNER_L4_NONFRAG; 265 /* Tunneled - Fragmented */ 266 (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 267 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 268 RTE_PTYPE_INNER_L4_FRAG; 269 (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 270 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 271 RTE_PTYPE_INNER_L4_FRAG; 272 (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 273 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 274 RTE_PTYPE_INNER_L4_FRAG; 275 (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 276 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 277 RTE_PTYPE_INNER_L4_FRAG; 278 /* Tunneled - TCP */ 279 (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 280 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 281 RTE_PTYPE_INNER_L4_TCP; 282 (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 283 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 284 RTE_PTYPE_INNER_L4_TCP; 285 (*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 286 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 287 RTE_PTYPE_INNER_L4_TCP; 288 (*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 289 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 290 RTE_PTYPE_INNER_L4_TCP; 291 (*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 292 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 293 RTE_PTYPE_INNER_L4_TCP; 294 (*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 295 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 296 RTE_PTYPE_INNER_L4_TCP; 297 (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 298 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 299 RTE_PTYPE_INNER_L4_TCP; 300 (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 301 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 302 RTE_PTYPE_INNER_L4_TCP; 303 (*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 304 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 305 RTE_PTYPE_INNER_L4_TCP; 306 (*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 307 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 308 RTE_PTYPE_INNER_L4_TCP; 309 (*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 310 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 311 RTE_PTYPE_INNER_L4_TCP; 312 (*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 313 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 314 RTE_PTYPE_INNER_L4_TCP; 315 /* Tunneled - UDP */ 316 (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 317 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 318 RTE_PTYPE_INNER_L4_UDP; 319 (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 320 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 321 RTE_PTYPE_INNER_L4_UDP; 322 (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 323 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 324 RTE_PTYPE_INNER_L4_UDP; 325 (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 326 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 327 RTE_PTYPE_INNER_L4_UDP; 328 } 329 330 /** 331 * Build a table to translate packet to checksum type of Verbs. 332 */ 333 void 334 mlx5_set_cksum_table(void) 335 { 336 unsigned int i; 337 uint8_t v; 338 339 /* 340 * The index should have: 341 * bit[0] = PKT_TX_TCP_SEG 342 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM 343 * bit[4] = PKT_TX_IP_CKSUM 344 * bit[8] = PKT_TX_OUTER_IP_CKSUM 345 * bit[9] = tunnel 346 */ 347 for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) { 348 v = 0; 349 if (i & (1 << 9)) { 350 /* Tunneled packet. */ 351 if (i & (1 << 8)) /* Outer IP. */ 352 v |= MLX5_ETH_WQE_L3_CSUM; 353 if (i & (1 << 4)) /* Inner IP. */ 354 v |= MLX5_ETH_WQE_L3_INNER_CSUM; 355 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */ 356 v |= MLX5_ETH_WQE_L4_INNER_CSUM; 357 } else { 358 /* No tunnel. */ 359 if (i & (1 << 4)) /* IP. */ 360 v |= MLX5_ETH_WQE_L3_CSUM; 361 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */ 362 v |= MLX5_ETH_WQE_L4_CSUM; 363 } 364 mlx5_cksum_table[i] = v; 365 } 366 } 367 368 /** 369 * Build a table to translate packet type of mbuf to SWP type of Verbs. 370 */ 371 void 372 mlx5_set_swp_types_table(void) 373 { 374 unsigned int i; 375 uint8_t v; 376 377 /* 378 * The index should have: 379 * bit[0:1] = PKT_TX_L4_MASK 380 * bit[4] = PKT_TX_IPV6 381 * bit[8] = PKT_TX_OUTER_IPV6 382 * bit[9] = PKT_TX_OUTER_UDP 383 */ 384 for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) { 385 v = 0; 386 if (i & (1 << 8)) 387 v |= MLX5_ETH_WQE_L3_OUTER_IPV6; 388 if (i & (1 << 9)) 389 v |= MLX5_ETH_WQE_L4_OUTER_UDP; 390 if (i & (1 << 4)) 391 v |= MLX5_ETH_WQE_L3_INNER_IPV6; 392 if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52)) 393 v |= MLX5_ETH_WQE_L4_INNER_UDP; 394 mlx5_swp_types_table[i] = v; 395 } 396 } 397 398 /** 399 * Set Software Parser flags and offsets in Ethernet Segment of WQE. 400 * Flags must be preliminary initialized to zero. 401 * 402 * @param loc 403 * Pointer to burst routine local context. 404 * @param swp_flags 405 * Pointer to store Software Parser flags 406 * @param olx 407 * Configured Tx offloads mask. It is fully defined at 408 * compile time and may be used for optimization. 409 * 410 * @return 411 * Software Parser offsets packed in dword. 412 * Software Parser flags are set by pointer. 413 */ 414 static __rte_always_inline uint32_t 415 txq_mbuf_to_swp(struct mlx5_txq_local *__rte_restrict loc, 416 uint8_t *swp_flags, 417 unsigned int olx) 418 { 419 uint64_t ol, tunnel; 420 unsigned int idx, off; 421 uint32_t set; 422 423 if (!MLX5_TXOFF_CONFIG(SWP)) 424 return 0; 425 ol = loc->mbuf->ol_flags; 426 tunnel = ol & PKT_TX_TUNNEL_MASK; 427 /* 428 * Check whether Software Parser is required. 429 * Only customized tunnels may ask for. 430 */ 431 if (likely(tunnel != PKT_TX_TUNNEL_UDP && tunnel != PKT_TX_TUNNEL_IP)) 432 return 0; 433 /* 434 * The index should have: 435 * bit[0:1] = PKT_TX_L4_MASK 436 * bit[4] = PKT_TX_IPV6 437 * bit[8] = PKT_TX_OUTER_IPV6 438 * bit[9] = PKT_TX_OUTER_UDP 439 */ 440 idx = (ol & (PKT_TX_L4_MASK | PKT_TX_IPV6 | PKT_TX_OUTER_IPV6)) >> 52; 441 idx |= (tunnel == PKT_TX_TUNNEL_UDP) ? (1 << 9) : 0; 442 *swp_flags = mlx5_swp_types_table[idx]; 443 /* 444 * Set offsets for SW parser. Since ConnectX-5, SW parser just 445 * complements HW parser. SW parser starts to engage only if HW parser 446 * can't reach a header. For the older devices, HW parser will not kick 447 * in if any of SWP offsets is set. Therefore, all of the L3 offsets 448 * should be set regardless of HW offload. 449 */ 450 off = loc->mbuf->outer_l2_len; 451 if (MLX5_TXOFF_CONFIG(VLAN) && ol & PKT_TX_VLAN_PKT) 452 off += sizeof(struct rte_vlan_hdr); 453 set = (off >> 1) << 8; /* Outer L3 offset. */ 454 off += loc->mbuf->outer_l3_len; 455 if (tunnel == PKT_TX_TUNNEL_UDP) 456 set |= off >> 1; /* Outer L4 offset. */ 457 if (ol & (PKT_TX_IPV4 | PKT_TX_IPV6)) { /* Inner IP. */ 458 const uint64_t csum = ol & PKT_TX_L4_MASK; 459 off += loc->mbuf->l2_len; 460 set |= (off >> 1) << 24; /* Inner L3 offset. */ 461 if (csum == PKT_TX_TCP_CKSUM || 462 csum == PKT_TX_UDP_CKSUM || 463 (MLX5_TXOFF_CONFIG(TSO) && ol & PKT_TX_TCP_SEG)) { 464 off += loc->mbuf->l3_len; 465 set |= (off >> 1) << 16; /* Inner L4 offset. */ 466 } 467 } 468 set = rte_cpu_to_le_32(set); 469 return set; 470 } 471 472 /** 473 * Convert the Checksum offloads to Verbs. 474 * 475 * @param buf 476 * Pointer to the mbuf. 477 * 478 * @return 479 * Converted checksum flags. 480 */ 481 static __rte_always_inline uint8_t 482 txq_ol_cksum_to_cs(struct rte_mbuf *buf) 483 { 484 uint32_t idx; 485 uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK); 486 const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK | 487 PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM; 488 489 /* 490 * The index should have: 491 * bit[0] = PKT_TX_TCP_SEG 492 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM 493 * bit[4] = PKT_TX_IP_CKSUM 494 * bit[8] = PKT_TX_OUTER_IP_CKSUM 495 * bit[9] = tunnel 496 */ 497 idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9); 498 return mlx5_cksum_table[idx]; 499 } 500 501 /** 502 * Internal function to compute the number of used descriptors in an RX queue 503 * 504 * @param rxq 505 * The Rx queue. 506 * 507 * @return 508 * The number of used rx descriptor. 509 */ 510 static uint32_t 511 rx_queue_count(struct mlx5_rxq_data *rxq) 512 { 513 struct rxq_zip *zip = &rxq->zip; 514 volatile struct mlx5_cqe *cqe; 515 const unsigned int cqe_n = (1 << rxq->cqe_n); 516 const unsigned int sges_n = (1 << rxq->sges_n); 517 const unsigned int elts_n = (1 << rxq->elts_n); 518 const unsigned int strd_n = (1 << rxq->strd_num_n); 519 const unsigned int cqe_cnt = cqe_n - 1; 520 unsigned int cq_ci, used; 521 522 /* if we are processing a compressed cqe */ 523 if (zip->ai) { 524 used = zip->cqe_cnt - zip->ai; 525 cq_ci = zip->cq_ci; 526 } else { 527 used = 0; 528 cq_ci = rxq->cq_ci; 529 } 530 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 531 while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) { 532 int8_t op_own; 533 unsigned int n; 534 535 op_own = cqe->op_own; 536 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) 537 n = rte_be_to_cpu_32(cqe->byte_cnt); 538 else 539 n = 1; 540 cq_ci += n; 541 used += n; 542 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 543 } 544 used = RTE_MIN(used * sges_n, elts_n * strd_n); 545 return used; 546 } 547 548 /** 549 * DPDK callback to check the status of a rx descriptor. 550 * 551 * @param rx_queue 552 * The Rx queue. 553 * @param[in] offset 554 * The index of the descriptor in the ring. 555 * 556 * @return 557 * The status of the tx descriptor. 558 */ 559 int 560 mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset) 561 { 562 struct mlx5_rxq_data *rxq = rx_queue; 563 struct mlx5_rxq_ctrl *rxq_ctrl = 564 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 565 struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv); 566 567 if (dev->rx_pkt_burst == NULL || 568 dev->rx_pkt_burst == removed_rx_burst) { 569 rte_errno = ENOTSUP; 570 return -rte_errno; 571 } 572 if (offset >= (1 << rxq->cqe_n)) { 573 rte_errno = EINVAL; 574 return -rte_errno; 575 } 576 if (offset < rx_queue_count(rxq)) 577 return RTE_ETH_RX_DESC_DONE; 578 return RTE_ETH_RX_DESC_AVAIL; 579 } 580 581 /** 582 * DPDK callback to get the RX queue information 583 * 584 * @param dev 585 * Pointer to the device structure. 586 * 587 * @param rx_queue_id 588 * Rx queue identificator. 589 * 590 * @param qinfo 591 * Pointer to the RX queue information structure. 592 * 593 * @return 594 * None. 595 */ 596 597 void 598 mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, 599 struct rte_eth_rxq_info *qinfo) 600 { 601 struct mlx5_priv *priv = dev->data->dev_private; 602 struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id]; 603 struct mlx5_rxq_ctrl *rxq_ctrl = 604 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 605 606 if (!rxq) 607 return; 608 qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ? 609 rxq->mprq_mp : rxq->mp; 610 qinfo->conf.rx_thresh.pthresh = 0; 611 qinfo->conf.rx_thresh.hthresh = 0; 612 qinfo->conf.rx_thresh.wthresh = 0; 613 qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh; 614 qinfo->conf.rx_drop_en = 1; 615 qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1; 616 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 617 qinfo->scattered_rx = dev->data->scattered_rx; 618 qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ? 619 (1 << rxq->elts_n) * (1 << rxq->strd_num_n) : 620 (1 << rxq->elts_n); 621 } 622 623 /** 624 * DPDK callback to get the RX packet burst mode information 625 * 626 * @param dev 627 * Pointer to the device structure. 628 * 629 * @param rx_queue_id 630 * Rx queue identificatior. 631 * 632 * @param mode 633 * Pointer to the burts mode information. 634 * 635 * @return 636 * 0 as success, -EINVAL as failure. 637 */ 638 639 int 640 mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, 641 uint16_t rx_queue_id __rte_unused, 642 struct rte_eth_burst_mode *mode) 643 { 644 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 645 struct mlx5_priv *priv = dev->data->dev_private; 646 struct mlx5_rxq_data *rxq; 647 648 rxq = (*priv->rxqs)[rx_queue_id]; 649 if (!rxq) { 650 rte_errno = EINVAL; 651 return -rte_errno; 652 } 653 if (pkt_burst == mlx5_rx_burst) { 654 snprintf(mode->info, sizeof(mode->info), "%s", "Scalar"); 655 } else if (pkt_burst == mlx5_rx_burst_mprq) { 656 snprintf(mode->info, sizeof(mode->info), "%s", "Multi-Packet RQ"); 657 } else if (pkt_burst == mlx5_rx_burst_vec) { 658 #if defined RTE_ARCH_X86_64 659 snprintf(mode->info, sizeof(mode->info), "%s", "Vector SSE"); 660 #elif defined RTE_ARCH_ARM64 661 snprintf(mode->info, sizeof(mode->info), "%s", "Vector Neon"); 662 #elif defined RTE_ARCH_PPC_64 663 snprintf(mode->info, sizeof(mode->info), "%s", "Vector AltiVec"); 664 #else 665 return -EINVAL; 666 #endif 667 } else if (pkt_burst == mlx5_rx_burst_mprq_vec) { 668 #if defined RTE_ARCH_X86_64 669 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector SSE"); 670 #elif defined RTE_ARCH_ARM64 671 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector Neon"); 672 #elif defined RTE_ARCH_PPC_64 673 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector AltiVec"); 674 #else 675 return -EINVAL; 676 #endif 677 } else { 678 return -EINVAL; 679 } 680 return 0; 681 } 682 683 /** 684 * DPDK callback to get the number of used descriptors in a RX queue 685 * 686 * @param dev 687 * Pointer to the device structure. 688 * 689 * @param rx_queue_id 690 * The Rx queue. 691 * 692 * @return 693 * The number of used rx descriptor. 694 * -EINVAL if the queue is invalid 695 */ 696 uint32_t 697 mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 698 { 699 struct mlx5_priv *priv = dev->data->dev_private; 700 struct mlx5_rxq_data *rxq; 701 702 if (dev->rx_pkt_burst == NULL || 703 dev->rx_pkt_burst == removed_rx_burst) { 704 rte_errno = ENOTSUP; 705 return -rte_errno; 706 } 707 rxq = (*priv->rxqs)[rx_queue_id]; 708 if (!rxq) { 709 rte_errno = EINVAL; 710 return -rte_errno; 711 } 712 return rx_queue_count(rxq); 713 } 714 715 #define MLX5_SYSTEM_LOG_DIR "/var/log" 716 /** 717 * Dump debug information to log file. 718 * 719 * @param fname 720 * The file name. 721 * @param hex_title 722 * If not NULL this string is printed as a header to the output 723 * and the output will be in hexadecimal view. 724 * @param buf 725 * This is the buffer address to print out. 726 * @param len 727 * The number of bytes to dump out. 728 */ 729 void 730 mlx5_dump_debug_information(const char *fname, const char *hex_title, 731 const void *buf, unsigned int hex_len) 732 { 733 FILE *fd; 734 735 MKSTR(path, "%s/%s", MLX5_SYSTEM_LOG_DIR, fname); 736 fd = fopen(path, "a+"); 737 if (!fd) { 738 DRV_LOG(WARNING, "cannot open %s for debug dump", path); 739 MKSTR(path2, "./%s", fname); 740 fd = fopen(path2, "a+"); 741 if (!fd) { 742 DRV_LOG(ERR, "cannot open %s for debug dump", path2); 743 return; 744 } 745 DRV_LOG(INFO, "New debug dump in file %s", path2); 746 } else { 747 DRV_LOG(INFO, "New debug dump in file %s", path); 748 } 749 if (hex_title) 750 rte_hexdump(fd, hex_title, buf, hex_len); 751 else 752 fprintf(fd, "%s", (const char *)buf); 753 fprintf(fd, "\n\n\n"); 754 fclose(fd); 755 } 756 757 /** 758 * Move QP from error state to running state and initialize indexes. 759 * 760 * @param txq_ctrl 761 * Pointer to TX queue control structure. 762 * 763 * @return 764 * 0 on success, else -1. 765 */ 766 static int 767 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl) 768 { 769 struct mlx5_mp_arg_queue_state_modify sm = { 770 .is_wq = 0, 771 .queue_id = txq_ctrl->txq.idx, 772 }; 773 774 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm)) 775 return -1; 776 txq_ctrl->txq.wqe_ci = 0; 777 txq_ctrl->txq.wqe_pi = 0; 778 txq_ctrl->txq.elts_comp = 0; 779 return 0; 780 } 781 782 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */ 783 static int 784 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe) 785 { 786 static const uint8_t magic[] = "seen"; 787 int ret = 1; 788 unsigned int i; 789 790 for (i = 0; i < sizeof(magic); ++i) 791 if (!ret || err_cqe->rsvd1[i] != magic[i]) { 792 ret = 0; 793 err_cqe->rsvd1[i] = magic[i]; 794 } 795 return ret; 796 } 797 798 /** 799 * Handle error CQE. 800 * 801 * @param txq 802 * Pointer to TX queue structure. 803 * @param error_cqe 804 * Pointer to the error CQE. 805 * 806 * @return 807 * Negative value if queue recovery failed, otherwise 808 * the error completion entry is handled successfully. 809 */ 810 static int 811 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq, 812 volatile struct mlx5_err_cqe *err_cqe) 813 { 814 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) { 815 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1); 816 struct mlx5_txq_ctrl *txq_ctrl = 817 container_of(txq, struct mlx5_txq_ctrl, txq); 818 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter); 819 int seen = check_err_cqe_seen(err_cqe); 820 821 if (!seen && txq_ctrl->dump_file_n < 822 txq_ctrl->priv->config.max_dump_files_num) { 823 MKSTR(err_str, "Unexpected CQE error syndrome " 824 "0x%02x CQN = %u SQN = %u wqe_counter = %u " 825 "wq_ci = %u cq_ci = %u", err_cqe->syndrome, 826 txq->cqe_s, txq->qp_num_8s >> 8, 827 rte_be_to_cpu_16(err_cqe->wqe_counter), 828 txq->wqe_ci, txq->cq_ci); 829 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u", 830 PORT_ID(txq_ctrl->priv), txq->idx, 831 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc()); 832 mlx5_dump_debug_information(name, NULL, err_str, 0); 833 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 834 (const void *)((uintptr_t) 835 txq->cqes), 836 sizeof(*err_cqe) * 837 (1 << txq->cqe_n)); 838 mlx5_dump_debug_information(name, "MLX5 Error SQ:", 839 (const void *)((uintptr_t) 840 txq->wqes), 841 MLX5_WQE_SIZE * 842 (1 << txq->wqe_n)); 843 txq_ctrl->dump_file_n++; 844 } 845 if (!seen) 846 /* 847 * Count errors in WQEs units. 848 * Later it can be improved to count error packets, 849 * for example, by SQ parsing to find how much packets 850 * should be counted for each WQE. 851 */ 852 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) - 853 new_wqe_pi) & wqe_m; 854 if (tx_recover_qp(txq_ctrl)) { 855 /* Recovering failed - retry later on the same WQE. */ 856 return -1; 857 } 858 /* Release all the remaining buffers. */ 859 txq_free_elts(txq_ctrl); 860 } 861 return 0; 862 } 863 864 /** 865 * Translate RX completion flags to packet type. 866 * 867 * @param[in] rxq 868 * Pointer to RX queue structure. 869 * @param[in] cqe 870 * Pointer to CQE. 871 * 872 * @note: fix mlx5_dev_supported_ptypes_get() if any change here. 873 * 874 * @return 875 * Packet type for struct rte_mbuf. 876 */ 877 static inline uint32_t 878 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 879 volatile struct mlx5_mini_cqe8 *mcqe) 880 { 881 uint8_t idx; 882 uint8_t ptype; 883 uint8_t pinfo = (cqe->pkt_info & 0x3) << 6; 884 885 /* Get l3/l4 header from mini-CQE in case L3/L4 format*/ 886 if (mcqe == NULL || 887 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 888 ptype = (cqe->hdr_type_etc & 0xfc00) >> 10; 889 else 890 ptype = mcqe->hdr_type >> 2; 891 /* 892 * The index to the array should have: 893 * bit[1:0] = l3_hdr_type 894 * bit[4:2] = l4_hdr_type 895 * bit[5] = ip_frag 896 * bit[6] = tunneled 897 * bit[7] = outer_l3_type 898 */ 899 idx = pinfo | ptype; 900 return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6)); 901 } 902 903 /** 904 * Initialize Rx WQ and indexes. 905 * 906 * @param[in] rxq 907 * Pointer to RX queue structure. 908 */ 909 void 910 mlx5_rxq_initialize(struct mlx5_rxq_data *rxq) 911 { 912 const unsigned int wqe_n = 1 << rxq->elts_n; 913 unsigned int i; 914 915 for (i = 0; (i != wqe_n); ++i) { 916 volatile struct mlx5_wqe_data_seg *scat; 917 uintptr_t addr; 918 uint32_t byte_count; 919 920 if (mlx5_rxq_mprq_enabled(rxq)) { 921 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i]; 922 923 scat = &((volatile struct mlx5_wqe_mprq *) 924 rxq->wqes)[i].dseg; 925 addr = (uintptr_t)mlx5_mprq_buf_addr(buf, 926 1 << rxq->strd_num_n); 927 byte_count = (1 << rxq->strd_sz_n) * 928 (1 << rxq->strd_num_n); 929 } else { 930 struct rte_mbuf *buf = (*rxq->elts)[i]; 931 932 scat = &((volatile struct mlx5_wqe_data_seg *) 933 rxq->wqes)[i]; 934 addr = rte_pktmbuf_mtod(buf, uintptr_t); 935 byte_count = DATA_LEN(buf); 936 } 937 /* scat->addr must be able to store a pointer. */ 938 MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t)); 939 *scat = (struct mlx5_wqe_data_seg){ 940 .addr = rte_cpu_to_be_64(addr), 941 .byte_count = rte_cpu_to_be_32(byte_count), 942 .lkey = mlx5_rx_addr2mr(rxq, addr), 943 }; 944 } 945 rxq->consumed_strd = 0; 946 rxq->decompressed = 0; 947 rxq->rq_pi = 0; 948 rxq->zip = (struct rxq_zip){ 949 .ai = 0, 950 }; 951 rxq->elts_ci = mlx5_rxq_mprq_enabled(rxq) ? 952 (wqe_n >> rxq->sges_n) * (1 << rxq->strd_num_n) : 0; 953 /* Update doorbell counter. */ 954 rxq->rq_ci = wqe_n >> rxq->sges_n; 955 rte_io_wmb(); 956 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 957 } 958 959 /** 960 * Modify a Verbs/DevX queue state. 961 * This must be called from the primary process. 962 * 963 * @param dev 964 * Pointer to Ethernet device. 965 * @param sm 966 * State modify request parameters. 967 * 968 * @return 969 * 0 in case of success else non-zero value and rte_errno is set. 970 */ 971 int 972 mlx5_queue_state_modify_primary(struct rte_eth_dev *dev, 973 const struct mlx5_mp_arg_queue_state_modify *sm) 974 { 975 int ret; 976 struct mlx5_priv *priv = dev->data->dev_private; 977 978 if (sm->is_wq) { 979 struct mlx5_rxq_data *rxq = (*priv->rxqs)[sm->queue_id]; 980 struct mlx5_rxq_ctrl *rxq_ctrl = 981 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 982 983 ret = priv->obj_ops.rxq_obj_modify(rxq_ctrl->obj, sm->state); 984 if (ret) { 985 DRV_LOG(ERR, "Cannot change Rx WQ state to %u - %s", 986 sm->state, strerror(errno)); 987 rte_errno = errno; 988 return ret; 989 } 990 } else { 991 struct mlx5_txq_data *txq = (*priv->txqs)[sm->queue_id]; 992 struct mlx5_txq_ctrl *txq_ctrl = 993 container_of(txq, struct mlx5_txq_ctrl, txq); 994 995 ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj, 996 MLX5_TXQ_MOD_ERR2RDY, 997 (uint8_t)priv->dev_port); 998 if (ret) 999 return ret; 1000 } 1001 return 0; 1002 } 1003 1004 /** 1005 * Modify a Verbs queue state. 1006 * 1007 * @param dev 1008 * Pointer to Ethernet device. 1009 * @param sm 1010 * State modify request parameters. 1011 * 1012 * @return 1013 * 0 in case of success else non-zero value. 1014 */ 1015 static int 1016 mlx5_queue_state_modify(struct rte_eth_dev *dev, 1017 struct mlx5_mp_arg_queue_state_modify *sm) 1018 { 1019 struct mlx5_priv *priv = dev->data->dev_private; 1020 int ret = 0; 1021 1022 switch (rte_eal_process_type()) { 1023 case RTE_PROC_PRIMARY: 1024 ret = mlx5_queue_state_modify_primary(dev, sm); 1025 break; 1026 case RTE_PROC_SECONDARY: 1027 ret = mlx5_mp_req_queue_state_modify(&priv->mp_id, sm); 1028 break; 1029 default: 1030 break; 1031 } 1032 return ret; 1033 } 1034 1035 /** 1036 * Handle a Rx error. 1037 * The function inserts the RQ state to reset when the first error CQE is 1038 * shown, then drains the CQ by the caller function loop. When the CQ is empty, 1039 * it moves the RQ state to ready and initializes the RQ. 1040 * Next CQE identification and error counting are in the caller responsibility. 1041 * 1042 * @param[in] rxq 1043 * Pointer to RX queue structure. 1044 * @param[in] vec 1045 * 1 when called from vectorized Rx burst, need to prepare mbufs for the RQ. 1046 * 0 when called from non-vectorized Rx burst. 1047 * 1048 * @return 1049 * -1 in case of recovery error, otherwise the CQE status. 1050 */ 1051 int 1052 mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) 1053 { 1054 const uint16_t cqe_n = 1 << rxq->cqe_n; 1055 const uint16_t cqe_mask = cqe_n - 1; 1056 const uint16_t wqe_n = 1 << rxq->elts_n; 1057 const uint16_t strd_n = 1 << rxq->strd_num_n; 1058 struct mlx5_rxq_ctrl *rxq_ctrl = 1059 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 1060 union { 1061 volatile struct mlx5_cqe *cqe; 1062 volatile struct mlx5_err_cqe *err_cqe; 1063 } u = { 1064 .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask], 1065 }; 1066 struct mlx5_mp_arg_queue_state_modify sm; 1067 int ret; 1068 1069 switch (rxq->err_state) { 1070 case MLX5_RXQ_ERR_STATE_NO_ERROR: 1071 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET; 1072 /* Fall-through */ 1073 case MLX5_RXQ_ERR_STATE_NEED_RESET: 1074 sm.is_wq = 1; 1075 sm.queue_id = rxq->idx; 1076 sm.state = IBV_WQS_RESET; 1077 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), &sm)) 1078 return -1; 1079 if (rxq_ctrl->dump_file_n < 1080 rxq_ctrl->priv->config.max_dump_files_num) { 1081 MKSTR(err_str, "Unexpected CQE error syndrome " 1082 "0x%02x CQN = %u RQN = %u wqe_counter = %u" 1083 " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome, 1084 rxq->cqn, rxq_ctrl->wqn, 1085 rte_be_to_cpu_16(u.err_cqe->wqe_counter), 1086 rxq->rq_ci << rxq->sges_n, rxq->cq_ci); 1087 MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u", 1088 rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc()); 1089 mlx5_dump_debug_information(name, NULL, err_str, 0); 1090 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 1091 (const void *)((uintptr_t) 1092 rxq->cqes), 1093 sizeof(*u.cqe) * cqe_n); 1094 mlx5_dump_debug_information(name, "MLX5 Error RQ:", 1095 (const void *)((uintptr_t) 1096 rxq->wqes), 1097 16 * wqe_n); 1098 rxq_ctrl->dump_file_n++; 1099 } 1100 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY; 1101 /* Fall-through */ 1102 case MLX5_RXQ_ERR_STATE_NEED_READY: 1103 ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci); 1104 if (ret == MLX5_CQE_STATUS_HW_OWN) { 1105 rte_io_wmb(); 1106 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1107 rte_io_wmb(); 1108 /* 1109 * The RQ consumer index must be zeroed while moving 1110 * from RESET state to RDY state. 1111 */ 1112 *rxq->rq_db = rte_cpu_to_be_32(0); 1113 rte_io_wmb(); 1114 sm.is_wq = 1; 1115 sm.queue_id = rxq->idx; 1116 sm.state = IBV_WQS_RDY; 1117 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), 1118 &sm)) 1119 return -1; 1120 if (vec) { 1121 const uint32_t elts_n = 1122 mlx5_rxq_mprq_enabled(rxq) ? 1123 wqe_n * strd_n : wqe_n; 1124 const uint32_t e_mask = elts_n - 1; 1125 uint32_t elts_ci = 1126 mlx5_rxq_mprq_enabled(rxq) ? 1127 rxq->elts_ci : rxq->rq_ci; 1128 uint32_t elt_idx; 1129 struct rte_mbuf **elt; 1130 int i; 1131 unsigned int n = elts_n - (elts_ci - 1132 rxq->rq_pi); 1133 1134 for (i = 0; i < (int)n; ++i) { 1135 elt_idx = (elts_ci + i) & e_mask; 1136 elt = &(*rxq->elts)[elt_idx]; 1137 *elt = rte_mbuf_raw_alloc(rxq->mp); 1138 if (!*elt) { 1139 for (i--; i >= 0; --i) { 1140 elt_idx = (elts_ci + 1141 i) & elts_n; 1142 elt = &(*rxq->elts) 1143 [elt_idx]; 1144 rte_pktmbuf_free_seg 1145 (*elt); 1146 } 1147 return -1; 1148 } 1149 } 1150 for (i = 0; i < (int)elts_n; ++i) { 1151 elt = &(*rxq->elts)[i]; 1152 DATA_LEN(*elt) = 1153 (uint16_t)((*elt)->buf_len - 1154 rte_pktmbuf_headroom(*elt)); 1155 } 1156 /* Padding with a fake mbuf for vec Rx. */ 1157 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 1158 (*rxq->elts)[elts_n + i] = 1159 &rxq->fake_mbuf; 1160 } 1161 mlx5_rxq_initialize(rxq); 1162 rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; 1163 } 1164 return ret; 1165 default: 1166 return -1; 1167 } 1168 } 1169 1170 /** 1171 * Get size of the next packet for a given CQE. For compressed CQEs, the 1172 * consumer index is updated only once all packets of the current one have 1173 * been processed. 1174 * 1175 * @param rxq 1176 * Pointer to RX queue. 1177 * @param cqe 1178 * CQE to process. 1179 * @param[out] mcqe 1180 * Store pointer to mini-CQE if compressed. Otherwise, the pointer is not 1181 * written. 1182 * 1183 * @return 1184 * 0 in case of empty CQE, otherwise the packet size in bytes. 1185 */ 1186 static inline int 1187 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 1188 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe) 1189 { 1190 struct rxq_zip *zip = &rxq->zip; 1191 uint16_t cqe_n = cqe_cnt + 1; 1192 int len; 1193 uint16_t idx, end; 1194 1195 do { 1196 len = 0; 1197 /* Process compressed data in the CQE and mini arrays. */ 1198 if (zip->ai) { 1199 volatile struct mlx5_mini_cqe8 (*mc)[8] = 1200 (volatile struct mlx5_mini_cqe8 (*)[8]) 1201 (uintptr_t)(&(*rxq->cqes)[zip->ca & 1202 cqe_cnt].pkt_info); 1203 len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt & 1204 rxq->byte_mask); 1205 *mcqe = &(*mc)[zip->ai & 7]; 1206 if ((++zip->ai & 7) == 0) { 1207 /* Invalidate consumed CQEs */ 1208 idx = zip->ca; 1209 end = zip->na; 1210 while (idx != end) { 1211 (*rxq->cqes)[idx & cqe_cnt].op_own = 1212 MLX5_CQE_INVALIDATE; 1213 ++idx; 1214 } 1215 /* 1216 * Increment consumer index to skip the number 1217 * of CQEs consumed. Hardware leaves holes in 1218 * the CQ ring for software use. 1219 */ 1220 zip->ca = zip->na; 1221 zip->na += 8; 1222 } 1223 if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) { 1224 /* Invalidate the rest */ 1225 idx = zip->ca; 1226 end = zip->cq_ci; 1227 1228 while (idx != end) { 1229 (*rxq->cqes)[idx & cqe_cnt].op_own = 1230 MLX5_CQE_INVALIDATE; 1231 ++idx; 1232 } 1233 rxq->cq_ci = zip->cq_ci; 1234 zip->ai = 0; 1235 } 1236 /* 1237 * No compressed data, get next CQE and verify if it is 1238 * compressed. 1239 */ 1240 } else { 1241 int ret; 1242 int8_t op_own; 1243 uint32_t cq_ci; 1244 1245 ret = check_cqe(cqe, cqe_n, rxq->cq_ci); 1246 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 1247 if (unlikely(ret == MLX5_CQE_STATUS_ERR || 1248 rxq->err_state)) { 1249 ret = mlx5_rx_err_handle(rxq, 0); 1250 if (ret == MLX5_CQE_STATUS_HW_OWN || 1251 ret == -1) 1252 return 0; 1253 } else { 1254 return 0; 1255 } 1256 } 1257 /* 1258 * Introduce the local variable to have queue cq_ci 1259 * index in queue structure always consistent with 1260 * actual CQE boundary (not pointing to the middle 1261 * of compressed CQE session). 1262 */ 1263 cq_ci = rxq->cq_ci + 1; 1264 op_own = cqe->op_own; 1265 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) { 1266 volatile struct mlx5_mini_cqe8 (*mc)[8] = 1267 (volatile struct mlx5_mini_cqe8 (*)[8]) 1268 (uintptr_t)(&(*rxq->cqes) 1269 [cq_ci & cqe_cnt].pkt_info); 1270 1271 /* Fix endianness. */ 1272 zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt); 1273 /* 1274 * Current mini array position is the one 1275 * returned by check_cqe64(). 1276 * 1277 * If completion comprises several mini arrays, 1278 * as a special case the second one is located 1279 * 7 CQEs after the initial CQE instead of 8 1280 * for subsequent ones. 1281 */ 1282 zip->ca = cq_ci; 1283 zip->na = zip->ca + 7; 1284 /* Compute the next non compressed CQE. */ 1285 zip->cq_ci = rxq->cq_ci + zip->cqe_cnt; 1286 /* Get packet size to return. */ 1287 len = rte_be_to_cpu_32((*mc)[0].byte_cnt & 1288 rxq->byte_mask); 1289 *mcqe = &(*mc)[0]; 1290 zip->ai = 1; 1291 /* Prefetch all to be invalidated */ 1292 idx = zip->ca; 1293 end = zip->cq_ci; 1294 while (idx != end) { 1295 rte_prefetch0(&(*rxq->cqes)[(idx) & 1296 cqe_cnt]); 1297 ++idx; 1298 } 1299 } else { 1300 rxq->cq_ci = cq_ci; 1301 len = rte_be_to_cpu_32(cqe->byte_cnt); 1302 } 1303 } 1304 if (unlikely(rxq->err_state)) { 1305 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1306 ++rxq->stats.idropped; 1307 } else { 1308 return len; 1309 } 1310 } while (1); 1311 } 1312 1313 /** 1314 * Translate RX completion flags to offload flags. 1315 * 1316 * @param[in] cqe 1317 * Pointer to CQE. 1318 * 1319 * @return 1320 * Offload flags (ol_flags) for struct rte_mbuf. 1321 */ 1322 static inline uint32_t 1323 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe) 1324 { 1325 uint32_t ol_flags = 0; 1326 uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc); 1327 1328 ol_flags = 1329 TRANSPOSE(flags, 1330 MLX5_CQE_RX_L3_HDR_VALID, 1331 PKT_RX_IP_CKSUM_GOOD) | 1332 TRANSPOSE(flags, 1333 MLX5_CQE_RX_L4_HDR_VALID, 1334 PKT_RX_L4_CKSUM_GOOD); 1335 return ol_flags; 1336 } 1337 1338 /** 1339 * Fill in mbuf fields from RX completion flags. 1340 * Note that pkt->ol_flags should be initialized outside of this function. 1341 * 1342 * @param rxq 1343 * Pointer to RX queue. 1344 * @param pkt 1345 * mbuf to fill. 1346 * @param cqe 1347 * CQE to process. 1348 * @param rss_hash_res 1349 * Packet RSS Hash result. 1350 */ 1351 static inline void 1352 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 1353 volatile struct mlx5_cqe *cqe, 1354 volatile struct mlx5_mini_cqe8 *mcqe) 1355 { 1356 /* Update packet information. */ 1357 pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe); 1358 1359 if (rxq->rss_hash) { 1360 uint32_t rss_hash_res = 0; 1361 1362 /* If compressed, take hash result from mini-CQE. */ 1363 if (mcqe == NULL || 1364 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH) 1365 rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res); 1366 else 1367 rss_hash_res = rte_be_to_cpu_32(mcqe->rx_hash_result); 1368 if (rss_hash_res) { 1369 pkt->hash.rss = rss_hash_res; 1370 pkt->ol_flags |= PKT_RX_RSS_HASH; 1371 } 1372 } 1373 if (rxq->mark) { 1374 uint32_t mark = 0; 1375 1376 /* If compressed, take flow tag from mini-CQE. */ 1377 if (mcqe == NULL || 1378 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_FTAG_STRIDX) 1379 mark = cqe->sop_drop_qpn; 1380 else 1381 mark = ((mcqe->byte_cnt_flow & 0xff) << 8) | 1382 (mcqe->flow_tag_high << 16); 1383 if (MLX5_FLOW_MARK_IS_VALID(mark)) { 1384 pkt->ol_flags |= PKT_RX_FDIR; 1385 if (mark != RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) { 1386 pkt->ol_flags |= PKT_RX_FDIR_ID; 1387 pkt->hash.fdir.hi = mlx5_flow_mark_get(mark); 1388 } 1389 } 1390 } 1391 if (rxq->dynf_meta && cqe->flow_table_metadata) { 1392 pkt->ol_flags |= rxq->flow_meta_mask; 1393 *RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset, uint32_t *) = 1394 cqe->flow_table_metadata; 1395 } 1396 if (rxq->csum) 1397 pkt->ol_flags |= rxq_cq_to_ol_flags(cqe); 1398 if (rxq->vlan_strip) { 1399 bool vlan_strip; 1400 1401 if (mcqe == NULL || 1402 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 1403 vlan_strip = cqe->hdr_type_etc & 1404 RTE_BE16(MLX5_CQE_VLAN_STRIPPED); 1405 else 1406 vlan_strip = mcqe->hdr_type & 1407 RTE_BE16(MLX5_CQE_VLAN_STRIPPED); 1408 if (vlan_strip) { 1409 pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1410 pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info); 1411 } 1412 } 1413 if (rxq->hw_timestamp) { 1414 uint64_t ts = rte_be_to_cpu_64(cqe->timestamp); 1415 1416 if (rxq->rt_timestamp) 1417 ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts); 1418 mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts); 1419 pkt->ol_flags |= rxq->timestamp_rx_flag; 1420 } 1421 } 1422 1423 /** 1424 * DPDK callback for RX. 1425 * 1426 * @param dpdk_rxq 1427 * Generic pointer to RX queue structure. 1428 * @param[out] pkts 1429 * Array to store received packets. 1430 * @param pkts_n 1431 * Maximum number of packets in array. 1432 * 1433 * @return 1434 * Number of packets successfully received (<= pkts_n). 1435 */ 1436 uint16_t 1437 mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 1438 { 1439 struct mlx5_rxq_data *rxq = dpdk_rxq; 1440 const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1; 1441 const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1; 1442 const unsigned int sges_n = rxq->sges_n; 1443 struct rte_mbuf *pkt = NULL; 1444 struct rte_mbuf *seg = NULL; 1445 volatile struct mlx5_cqe *cqe = 1446 &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1447 unsigned int i = 0; 1448 unsigned int rq_ci = rxq->rq_ci << sges_n; 1449 int len = 0; /* keep its value across iterations. */ 1450 1451 while (pkts_n) { 1452 unsigned int idx = rq_ci & wqe_cnt; 1453 volatile struct mlx5_wqe_data_seg *wqe = 1454 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; 1455 struct rte_mbuf *rep = (*rxq->elts)[idx]; 1456 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 1457 1458 if (pkt) 1459 NEXT(seg) = rep; 1460 seg = rep; 1461 rte_prefetch0(seg); 1462 rte_prefetch0(cqe); 1463 rte_prefetch0(wqe); 1464 /* Allocate the buf from the same pool. */ 1465 rep = rte_mbuf_raw_alloc(seg->pool); 1466 if (unlikely(rep == NULL)) { 1467 ++rxq->stats.rx_nombuf; 1468 if (!pkt) { 1469 /* 1470 * no buffers before we even started, 1471 * bail out silently. 1472 */ 1473 break; 1474 } 1475 while (pkt != seg) { 1476 MLX5_ASSERT(pkt != (*rxq->elts)[idx]); 1477 rep = NEXT(pkt); 1478 NEXT(pkt) = NULL; 1479 NB_SEGS(pkt) = 1; 1480 rte_mbuf_raw_free(pkt); 1481 pkt = rep; 1482 } 1483 rq_ci >>= sges_n; 1484 ++rq_ci; 1485 rq_ci <<= sges_n; 1486 break; 1487 } 1488 if (!pkt) { 1489 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1490 len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe); 1491 if (!len) { 1492 rte_mbuf_raw_free(rep); 1493 break; 1494 } 1495 pkt = seg; 1496 MLX5_ASSERT(len >= (rxq->crc_present << 2)); 1497 pkt->ol_flags &= EXT_ATTACHED_MBUF; 1498 rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe); 1499 if (rxq->crc_present) 1500 len -= RTE_ETHER_CRC_LEN; 1501 PKT_LEN(pkt) = len; 1502 if (cqe->lro_num_seg > 1) { 1503 mlx5_lro_update_hdr 1504 (rte_pktmbuf_mtod(pkt, uint8_t *), cqe, 1505 mcqe, rxq, len); 1506 pkt->ol_flags |= PKT_RX_LRO; 1507 pkt->tso_segsz = len / cqe->lro_num_seg; 1508 } 1509 } 1510 DATA_LEN(rep) = DATA_LEN(seg); 1511 PKT_LEN(rep) = PKT_LEN(seg); 1512 SET_DATA_OFF(rep, DATA_OFF(seg)); 1513 PORT(rep) = PORT(seg); 1514 (*rxq->elts)[idx] = rep; 1515 /* 1516 * Fill NIC descriptor with the new buffer. The lkey and size 1517 * of the buffers are already known, only the buffer address 1518 * changes. 1519 */ 1520 wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t)); 1521 /* If there's only one MR, no need to replace LKey in WQE. */ 1522 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) 1523 wqe->lkey = mlx5_rx_mb2mr(rxq, rep); 1524 if (len > DATA_LEN(seg)) { 1525 len -= DATA_LEN(seg); 1526 ++NB_SEGS(pkt); 1527 ++rq_ci; 1528 continue; 1529 } 1530 DATA_LEN(seg) = len; 1531 #ifdef MLX5_PMD_SOFT_COUNTERS 1532 /* Increment bytes counter. */ 1533 rxq->stats.ibytes += PKT_LEN(pkt); 1534 #endif 1535 /* Return packet. */ 1536 *(pkts++) = pkt; 1537 pkt = NULL; 1538 --pkts_n; 1539 ++i; 1540 /* Align consumer index to the next stride. */ 1541 rq_ci >>= sges_n; 1542 ++rq_ci; 1543 rq_ci <<= sges_n; 1544 } 1545 if (unlikely((i == 0) && ((rq_ci >> sges_n) == rxq->rq_ci))) 1546 return 0; 1547 /* Update the consumer index. */ 1548 rxq->rq_ci = rq_ci >> sges_n; 1549 rte_io_wmb(); 1550 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1551 rte_io_wmb(); 1552 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 1553 #ifdef MLX5_PMD_SOFT_COUNTERS 1554 /* Increment packets counter. */ 1555 rxq->stats.ipackets += i; 1556 #endif 1557 return i; 1558 } 1559 1560 /** 1561 * Update LRO packet TCP header. 1562 * The HW LRO feature doesn't update the TCP header after coalescing the 1563 * TCP segments but supplies information in CQE to fill it by SW. 1564 * 1565 * @param tcp 1566 * Pointer to the TCP header. 1567 * @param cqe 1568 * Pointer to the completion entry.. 1569 * @param phcsum 1570 * The L3 pseudo-header checksum. 1571 */ 1572 static inline void 1573 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp, 1574 volatile struct mlx5_cqe *__rte_restrict cqe, 1575 uint32_t phcsum, uint8_t l4_type) 1576 { 1577 /* 1578 * The HW calculates only the TCP payload checksum, need to complete 1579 * the TCP header checksum and the L3 pseudo-header checksum. 1580 */ 1581 uint32_t csum = phcsum + cqe->csum; 1582 1583 if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK || 1584 l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) { 1585 tcp->tcp_flags |= RTE_TCP_ACK_FLAG; 1586 tcp->recv_ack = cqe->lro_ack_seq_num; 1587 tcp->rx_win = cqe->lro_tcp_win; 1588 } 1589 if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK) 1590 tcp->tcp_flags |= RTE_TCP_PSH_FLAG; 1591 tcp->cksum = 0; 1592 csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4); 1593 csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff); 1594 csum = (~csum) & 0xffff; 1595 if (csum == 0) 1596 csum = 0xffff; 1597 tcp->cksum = csum; 1598 } 1599 1600 /** 1601 * Update LRO packet headers. 1602 * The HW LRO feature doesn't update the L3/TCP headers after coalescing the 1603 * TCP segments but supply information in CQE to fill it by SW. 1604 * 1605 * @param padd 1606 * The packet address. 1607 * @param cqe 1608 * Pointer to the completion entry.. 1609 * @param len 1610 * The packet length. 1611 */ 1612 static inline void 1613 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd, 1614 volatile struct mlx5_cqe *__rte_restrict cqe, 1615 volatile struct mlx5_mini_cqe8 *mcqe, 1616 struct mlx5_rxq_data *rxq, uint32_t len) 1617 { 1618 union { 1619 struct rte_ether_hdr *eth; 1620 struct rte_vlan_hdr *vlan; 1621 struct rte_ipv4_hdr *ipv4; 1622 struct rte_ipv6_hdr *ipv6; 1623 struct rte_tcp_hdr *tcp; 1624 uint8_t *hdr; 1625 } h = { 1626 .hdr = padd, 1627 }; 1628 uint16_t proto = h.eth->ether_type; 1629 uint32_t phcsum; 1630 uint8_t l4_type; 1631 1632 h.eth++; 1633 while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 1634 proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 1635 proto = h.vlan->eth_proto; 1636 h.vlan++; 1637 } 1638 if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) { 1639 h.ipv4->time_to_live = cqe->lro_min_ttl; 1640 h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr - padd)); 1641 h.ipv4->hdr_checksum = 0; 1642 h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4); 1643 phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0); 1644 h.ipv4++; 1645 } else { 1646 h.ipv6->hop_limits = cqe->lro_min_ttl; 1647 h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd) - 1648 sizeof(*h.ipv6)); 1649 phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0); 1650 h.ipv6++; 1651 } 1652 if (mcqe == NULL || 1653 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 1654 l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) & 1655 MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT; 1656 else 1657 l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) & 1658 MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT; 1659 mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type); 1660 } 1661 1662 void 1663 mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque) 1664 { 1665 struct mlx5_mprq_buf *buf = opaque; 1666 1667 if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) { 1668 rte_mempool_put(buf->mp, buf); 1669 } else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1, 1670 __ATOMIC_RELAXED) == 0)) { 1671 __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED); 1672 rte_mempool_put(buf->mp, buf); 1673 } 1674 } 1675 1676 void 1677 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf) 1678 { 1679 mlx5_mprq_buf_free_cb(NULL, buf); 1680 } 1681 1682 /** 1683 * DPDK callback for RX with Multi-Packet RQ support. 1684 * 1685 * @param dpdk_rxq 1686 * Generic pointer to RX queue structure. 1687 * @param[out] pkts 1688 * Array to store received packets. 1689 * @param pkts_n 1690 * Maximum number of packets in array. 1691 * 1692 * @return 1693 * Number of packets successfully received (<= pkts_n). 1694 */ 1695 uint16_t 1696 mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 1697 { 1698 struct mlx5_rxq_data *rxq = dpdk_rxq; 1699 const uint32_t strd_n = 1 << rxq->strd_num_n; 1700 const uint32_t strd_sz = 1 << rxq->strd_sz_n; 1701 const uint32_t cq_mask = (1 << rxq->cqe_n) - 1; 1702 const uint32_t wq_mask = (1 << rxq->elts_n) - 1; 1703 volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1704 unsigned int i = 0; 1705 uint32_t rq_ci = rxq->rq_ci; 1706 uint16_t consumed_strd = rxq->consumed_strd; 1707 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1708 1709 while (i < pkts_n) { 1710 struct rte_mbuf *pkt; 1711 int ret; 1712 uint32_t len; 1713 uint16_t strd_cnt; 1714 uint16_t strd_idx; 1715 uint32_t byte_cnt; 1716 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 1717 enum mlx5_rqx_code rxq_code; 1718 1719 if (consumed_strd == strd_n) { 1720 /* Replace WQE if the buffer is still in use. */ 1721 mprq_buf_replace(rxq, rq_ci & wq_mask); 1722 /* Advance to the next WQE. */ 1723 consumed_strd = 0; 1724 ++rq_ci; 1725 buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1726 } 1727 cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1728 ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe); 1729 if (!ret) 1730 break; 1731 byte_cnt = ret; 1732 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; 1733 MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); 1734 if (rxq->crc_present) 1735 len -= RTE_ETHER_CRC_LEN; 1736 if (mcqe && 1737 rxq->mcqe_format == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX) 1738 strd_cnt = (len / strd_sz) + !!(len % strd_sz); 1739 else 1740 strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >> 1741 MLX5_MPRQ_STRIDE_NUM_SHIFT; 1742 MLX5_ASSERT(strd_cnt); 1743 consumed_strd += strd_cnt; 1744 if (byte_cnt & MLX5_MPRQ_FILLER_MASK) 1745 continue; 1746 strd_idx = rte_be_to_cpu_16(mcqe == NULL ? 1747 cqe->wqe_counter : 1748 mcqe->stride_idx); 1749 MLX5_ASSERT(strd_idx < strd_n); 1750 MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & 1751 wq_mask)); 1752 pkt = rte_pktmbuf_alloc(rxq->mp); 1753 if (unlikely(pkt == NULL)) { 1754 ++rxq->stats.rx_nombuf; 1755 break; 1756 } 1757 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; 1758 MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); 1759 if (rxq->crc_present) 1760 len -= RTE_ETHER_CRC_LEN; 1761 rxq_code = mprq_buf_to_pkt(rxq, pkt, len, buf, 1762 strd_idx, strd_cnt); 1763 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) { 1764 rte_pktmbuf_free_seg(pkt); 1765 if (rxq_code == MLX5_RXQ_CODE_DROPPED) { 1766 ++rxq->stats.idropped; 1767 continue; 1768 } 1769 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) { 1770 ++rxq->stats.rx_nombuf; 1771 break; 1772 } 1773 } 1774 rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe); 1775 if (cqe->lro_num_seg > 1) { 1776 mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt, uint8_t *), 1777 cqe, mcqe, rxq, len); 1778 pkt->ol_flags |= PKT_RX_LRO; 1779 pkt->tso_segsz = len / cqe->lro_num_seg; 1780 } 1781 PKT_LEN(pkt) = len; 1782 PORT(pkt) = rxq->port_id; 1783 #ifdef MLX5_PMD_SOFT_COUNTERS 1784 /* Increment bytes counter. */ 1785 rxq->stats.ibytes += PKT_LEN(pkt); 1786 #endif 1787 /* Return packet. */ 1788 *(pkts++) = pkt; 1789 ++i; 1790 } 1791 /* Update the consumer indexes. */ 1792 rxq->consumed_strd = consumed_strd; 1793 rte_io_wmb(); 1794 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1795 if (rq_ci != rxq->rq_ci) { 1796 rxq->rq_ci = rq_ci; 1797 rte_io_wmb(); 1798 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 1799 } 1800 #ifdef MLX5_PMD_SOFT_COUNTERS 1801 /* Increment packets counter. */ 1802 rxq->stats.ipackets += i; 1803 #endif 1804 return i; 1805 } 1806 1807 /** 1808 * Dummy DPDK callback for TX. 1809 * 1810 * This function is used to temporarily replace the real callback during 1811 * unsafe control operations on the queue, or in case of error. 1812 * 1813 * @param dpdk_txq 1814 * Generic pointer to TX queue structure. 1815 * @param[in] pkts 1816 * Packets to transmit. 1817 * @param pkts_n 1818 * Number of packets in array. 1819 * 1820 * @return 1821 * Number of packets successfully transmitted (<= pkts_n). 1822 */ 1823 uint16_t 1824 removed_tx_burst(void *dpdk_txq __rte_unused, 1825 struct rte_mbuf **pkts __rte_unused, 1826 uint16_t pkts_n __rte_unused) 1827 { 1828 rte_mb(); 1829 return 0; 1830 } 1831 1832 /** 1833 * Dummy DPDK callback for RX. 1834 * 1835 * This function is used to temporarily replace the real callback during 1836 * unsafe control operations on the queue, or in case of error. 1837 * 1838 * @param dpdk_rxq 1839 * Generic pointer to RX queue structure. 1840 * @param[out] pkts 1841 * Array to store received packets. 1842 * @param pkts_n 1843 * Maximum number of packets in array. 1844 * 1845 * @return 1846 * Number of packets successfully received (<= pkts_n). 1847 */ 1848 uint16_t 1849 removed_rx_burst(void *dpdk_txq __rte_unused, 1850 struct rte_mbuf **pkts __rte_unused, 1851 uint16_t pkts_n __rte_unused) 1852 { 1853 rte_mb(); 1854 return 0; 1855 } 1856 1857 /* 1858 * Vectorized Rx/Tx routines are not compiled in when required vector 1859 * instructions are not supported on a target architecture. The following null 1860 * stubs are needed for linkage when those are not included outside of this file 1861 * (e.g. mlx5_rxtx_vec_sse.c for x86). 1862 */ 1863 1864 __rte_weak uint16_t 1865 mlx5_rx_burst_vec(void *dpdk_txq __rte_unused, 1866 struct rte_mbuf **pkts __rte_unused, 1867 uint16_t pkts_n __rte_unused) 1868 { 1869 return 0; 1870 } 1871 1872 __rte_weak uint16_t 1873 mlx5_rx_burst_mprq_vec(void *dpdk_txq __rte_unused, 1874 struct rte_mbuf **pkts __rte_unused, 1875 uint16_t pkts_n __rte_unused) 1876 { 1877 return 0; 1878 } 1879 1880 __rte_weak int 1881 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) 1882 { 1883 return -ENOTSUP; 1884 } 1885 1886 __rte_weak int 1887 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused) 1888 { 1889 return -ENOTSUP; 1890 } 1891 1892 /** 1893 * Free the mbufs from the linear array of pointers. 1894 * 1895 * @param txq 1896 * Pointer to Tx queue structure. 1897 * @param pkts 1898 * Pointer to array of packets to be free. 1899 * @param pkts_n 1900 * Number of packets to be freed. 1901 * @param olx 1902 * Configured Tx offloads mask. It is fully defined at 1903 * compile time and may be used for optimization. 1904 */ 1905 static __rte_always_inline void 1906 mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq, 1907 struct rte_mbuf **__rte_restrict pkts, 1908 unsigned int pkts_n, 1909 unsigned int olx __rte_unused) 1910 { 1911 struct rte_mempool *pool = NULL; 1912 struct rte_mbuf **p_free = NULL; 1913 struct rte_mbuf *mbuf; 1914 unsigned int n_free = 0; 1915 1916 /* 1917 * The implemented algorithm eliminates 1918 * copying pointers to temporary array 1919 * for rte_mempool_put_bulk() calls. 1920 */ 1921 MLX5_ASSERT(pkts); 1922 MLX5_ASSERT(pkts_n); 1923 /* 1924 * Free mbufs directly to the pool in bulk 1925 * if fast free offload is engaged 1926 */ 1927 if (!MLX5_TXOFF_CONFIG(MULTI) && txq->fast_free) { 1928 mbuf = *pkts; 1929 pool = mbuf->pool; 1930 rte_mempool_put_bulk(pool, (void *)pkts, pkts_n); 1931 return; 1932 } 1933 for (;;) { 1934 for (;;) { 1935 /* 1936 * Decrement mbuf reference counter, detach 1937 * indirect and external buffers if needed. 1938 */ 1939 mbuf = rte_pktmbuf_prefree_seg(*pkts); 1940 if (likely(mbuf != NULL)) { 1941 MLX5_ASSERT(mbuf == *pkts); 1942 if (likely(n_free != 0)) { 1943 if (unlikely(pool != mbuf->pool)) 1944 /* From different pool. */ 1945 break; 1946 } else { 1947 /* Start new scan array. */ 1948 pool = mbuf->pool; 1949 p_free = pkts; 1950 } 1951 ++n_free; 1952 ++pkts; 1953 --pkts_n; 1954 if (unlikely(pkts_n == 0)) { 1955 mbuf = NULL; 1956 break; 1957 } 1958 } else { 1959 /* 1960 * This happens if mbuf is still referenced. 1961 * We can't put it back to the pool, skip. 1962 */ 1963 ++pkts; 1964 --pkts_n; 1965 if (unlikely(n_free != 0)) 1966 /* There is some array to free.*/ 1967 break; 1968 if (unlikely(pkts_n == 0)) 1969 /* Last mbuf, nothing to free. */ 1970 return; 1971 } 1972 } 1973 for (;;) { 1974 /* 1975 * This loop is implemented to avoid multiple 1976 * inlining of rte_mempool_put_bulk(). 1977 */ 1978 MLX5_ASSERT(pool); 1979 MLX5_ASSERT(p_free); 1980 MLX5_ASSERT(n_free); 1981 /* 1982 * Free the array of pre-freed mbufs 1983 * belonging to the same memory pool. 1984 */ 1985 rte_mempool_put_bulk(pool, (void *)p_free, n_free); 1986 if (unlikely(mbuf != NULL)) { 1987 /* There is the request to start new scan. */ 1988 pool = mbuf->pool; 1989 p_free = pkts++; 1990 n_free = 1; 1991 --pkts_n; 1992 if (likely(pkts_n != 0)) 1993 break; 1994 /* 1995 * This is the last mbuf to be freed. 1996 * Do one more loop iteration to complete. 1997 * This is rare case of the last unique mbuf. 1998 */ 1999 mbuf = NULL; 2000 continue; 2001 } 2002 if (likely(pkts_n == 0)) 2003 return; 2004 n_free = 0; 2005 break; 2006 } 2007 } 2008 } 2009 /* 2010 * No inline version to free buffers for optimal call 2011 * on the tx_burst completion. 2012 */ 2013 static __rte_noinline void 2014 __mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq, 2015 struct rte_mbuf **__rte_restrict pkts, 2016 unsigned int pkts_n, 2017 unsigned int olx __rte_unused) 2018 { 2019 mlx5_tx_free_mbuf(txq, pkts, pkts_n, olx); 2020 } 2021 2022 /** 2023 * Free the mbuf from the elts ring buffer till new tail. 2024 * 2025 * @param txq 2026 * Pointer to Tx queue structure. 2027 * @param tail 2028 * Index in elts to free up to, becomes new elts tail. 2029 * @param olx 2030 * Configured Tx offloads mask. It is fully defined at 2031 * compile time and may be used for optimization. 2032 */ 2033 static __rte_always_inline void 2034 mlx5_tx_free_elts(struct mlx5_txq_data *__rte_restrict txq, 2035 uint16_t tail, 2036 unsigned int olx __rte_unused) 2037 { 2038 uint16_t n_elts = tail - txq->elts_tail; 2039 2040 MLX5_ASSERT(n_elts); 2041 MLX5_ASSERT(n_elts <= txq->elts_s); 2042 /* 2043 * Implement a loop to support ring buffer wraparound 2044 * with single inlining of mlx5_tx_free_mbuf(). 2045 */ 2046 do { 2047 unsigned int part; 2048 2049 part = txq->elts_s - (txq->elts_tail & txq->elts_m); 2050 part = RTE_MIN(part, n_elts); 2051 MLX5_ASSERT(part); 2052 MLX5_ASSERT(part <= txq->elts_s); 2053 mlx5_tx_free_mbuf(txq, 2054 &txq->elts[txq->elts_tail & txq->elts_m], 2055 part, olx); 2056 txq->elts_tail += part; 2057 n_elts -= part; 2058 } while (n_elts); 2059 } 2060 2061 /** 2062 * Store the mbuf being sent into elts ring buffer. 2063 * On Tx completion these mbufs will be freed. 2064 * 2065 * @param txq 2066 * Pointer to Tx queue structure. 2067 * @param pkts 2068 * Pointer to array of packets to be stored. 2069 * @param pkts_n 2070 * Number of packets to be stored. 2071 * @param olx 2072 * Configured Tx offloads mask. It is fully defined at 2073 * compile time and may be used for optimization. 2074 */ 2075 static __rte_always_inline void 2076 mlx5_tx_copy_elts(struct mlx5_txq_data *__rte_restrict txq, 2077 struct rte_mbuf **__rte_restrict pkts, 2078 unsigned int pkts_n, 2079 unsigned int olx __rte_unused) 2080 { 2081 unsigned int part; 2082 struct rte_mbuf **elts = (struct rte_mbuf **)txq->elts; 2083 2084 MLX5_ASSERT(pkts); 2085 MLX5_ASSERT(pkts_n); 2086 part = txq->elts_s - (txq->elts_head & txq->elts_m); 2087 MLX5_ASSERT(part); 2088 MLX5_ASSERT(part <= txq->elts_s); 2089 /* This code is a good candidate for vectorizing with SIMD. */ 2090 rte_memcpy((void *)(elts + (txq->elts_head & txq->elts_m)), 2091 (void *)pkts, 2092 RTE_MIN(part, pkts_n) * sizeof(struct rte_mbuf *)); 2093 txq->elts_head += pkts_n; 2094 if (unlikely(part < pkts_n)) 2095 /* The copy is wrapping around the elts array. */ 2096 rte_memcpy((void *)elts, (void *)(pkts + part), 2097 (pkts_n - part) * sizeof(struct rte_mbuf *)); 2098 } 2099 2100 /** 2101 * Update completion queue consuming index via doorbell 2102 * and flush the completed data buffers. 2103 * 2104 * @param txq 2105 * Pointer to TX queue structure. 2106 * @param valid CQE pointer 2107 * if not NULL update txq->wqe_pi and flush the buffers 2108 * @param olx 2109 * Configured Tx offloads mask. It is fully defined at 2110 * compile time and may be used for optimization. 2111 */ 2112 static __rte_always_inline void 2113 mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq, 2114 volatile struct mlx5_cqe *last_cqe, 2115 unsigned int olx __rte_unused) 2116 { 2117 if (likely(last_cqe != NULL)) { 2118 uint16_t tail; 2119 2120 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter); 2121 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m]; 2122 if (likely(tail != txq->elts_tail)) { 2123 mlx5_tx_free_elts(txq, tail, olx); 2124 MLX5_ASSERT(tail == txq->elts_tail); 2125 } 2126 } 2127 } 2128 2129 /** 2130 * Manage TX completions. This routine checks the CQ for 2131 * arrived CQEs, deduces the last accomplished WQE in SQ, 2132 * updates SQ producing index and frees all completed mbufs. 2133 * 2134 * @param txq 2135 * Pointer to TX queue structure. 2136 * @param olx 2137 * Configured Tx offloads mask. It is fully defined at 2138 * compile time and may be used for optimization. 2139 * 2140 * NOTE: not inlined intentionally, it makes tx_burst 2141 * routine smaller, simple and faster - from experiments. 2142 */ 2143 static void 2144 mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq, 2145 unsigned int olx __rte_unused) 2146 { 2147 unsigned int count = MLX5_TX_COMP_MAX_CQE; 2148 volatile struct mlx5_cqe *last_cqe = NULL; 2149 bool ring_doorbell = false; 2150 int ret; 2151 2152 do { 2153 volatile struct mlx5_cqe *cqe; 2154 2155 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m]; 2156 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci); 2157 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 2158 if (likely(ret != MLX5_CQE_STATUS_ERR)) { 2159 /* No new CQEs in completion queue. */ 2160 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN); 2161 break; 2162 } 2163 /* 2164 * Some error occurred, try to restart. 2165 * We have no barrier after WQE related Doorbell 2166 * written, make sure all writes are completed 2167 * here, before we might perform SQ reset. 2168 */ 2169 rte_wmb(); 2170 ret = mlx5_tx_error_cqe_handle 2171 (txq, (volatile struct mlx5_err_cqe *)cqe); 2172 if (unlikely(ret < 0)) { 2173 /* 2174 * Some error occurred on queue error 2175 * handling, we do not advance the index 2176 * here, allowing to retry on next call. 2177 */ 2178 return; 2179 } 2180 /* 2181 * We are going to fetch all entries with 2182 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status. 2183 * The send queue is supposed to be empty. 2184 */ 2185 ring_doorbell = true; 2186 ++txq->cq_ci; 2187 txq->cq_pi = txq->cq_ci; 2188 last_cqe = NULL; 2189 continue; 2190 } 2191 /* Normal transmit completion. */ 2192 MLX5_ASSERT(txq->cq_ci != txq->cq_pi); 2193 #ifdef RTE_LIBRTE_MLX5_DEBUG 2194 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) == 2195 cqe->wqe_counter); 2196 #endif 2197 ring_doorbell = true; 2198 ++txq->cq_ci; 2199 last_cqe = cqe; 2200 /* 2201 * We have to restrict the amount of processed CQEs 2202 * in one tx_burst routine call. The CQ may be large 2203 * and many CQEs may be updated by the NIC in one 2204 * transaction. Buffers freeing is time consuming, 2205 * multiple iterations may introduce significant 2206 * latency. 2207 */ 2208 if (likely(--count == 0)) 2209 break; 2210 } while (true); 2211 if (likely(ring_doorbell)) { 2212 /* Ring doorbell to notify hardware. */ 2213 rte_compiler_barrier(); 2214 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci); 2215 mlx5_tx_comp_flush(txq, last_cqe, olx); 2216 } 2217 } 2218 2219 /** 2220 * Check if the completion request flag should be set in the last WQE. 2221 * Both pushed mbufs and WQEs are monitored and the completion request 2222 * flag is set if any of thresholds is reached. 2223 * 2224 * @param txq 2225 * Pointer to TX queue structure. 2226 * @param loc 2227 * Pointer to burst routine local context. 2228 * @param olx 2229 * Configured Tx offloads mask. It is fully defined at 2230 * compile time and may be used for optimization. 2231 */ 2232 static __rte_always_inline void 2233 mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq, 2234 struct mlx5_txq_local *__rte_restrict loc, 2235 unsigned int olx) 2236 { 2237 uint16_t head = txq->elts_head; 2238 unsigned int part; 2239 2240 part = MLX5_TXOFF_CONFIG(INLINE) ? 2241 0 : loc->pkts_sent - loc->pkts_copy; 2242 head += part; 2243 if ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH || 2244 (MLX5_TXOFF_CONFIG(INLINE) && 2245 (uint16_t)(txq->wqe_ci - txq->wqe_comp) >= txq->wqe_thres)) { 2246 volatile struct mlx5_wqe *last = loc->wqe_last; 2247 2248 MLX5_ASSERT(last); 2249 txq->elts_comp = head; 2250 if (MLX5_TXOFF_CONFIG(INLINE)) 2251 txq->wqe_comp = txq->wqe_ci; 2252 /* Request unconditional completion on last WQE. */ 2253 last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 2254 MLX5_COMP_MODE_OFFSET); 2255 /* Save elts_head in dedicated free on completion queue. */ 2256 #ifdef RTE_LIBRTE_MLX5_DEBUG 2257 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head | 2258 (last->cseg.opcode >> 8) << 16; 2259 #else 2260 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head; 2261 #endif 2262 /* A CQE slot must always be available. */ 2263 MLX5_ASSERT((txq->cq_pi - txq->cq_ci) <= txq->cqe_s); 2264 } 2265 } 2266 2267 /** 2268 * DPDK callback to check the status of a tx descriptor. 2269 * 2270 * @param tx_queue 2271 * The tx queue. 2272 * @param[in] offset 2273 * The index of the descriptor in the ring. 2274 * 2275 * @return 2276 * The status of the tx descriptor. 2277 */ 2278 int 2279 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset) 2280 { 2281 struct mlx5_txq_data *__rte_restrict txq = tx_queue; 2282 uint16_t used; 2283 2284 mlx5_tx_handle_completion(txq, 0); 2285 used = txq->elts_head - txq->elts_tail; 2286 if (offset < used) 2287 return RTE_ETH_TX_DESC_FULL; 2288 return RTE_ETH_TX_DESC_DONE; 2289 } 2290 2291 /** 2292 * Build the Control Segment with specified opcode: 2293 * - MLX5_OPCODE_SEND 2294 * - MLX5_OPCODE_ENHANCED_MPSW 2295 * - MLX5_OPCODE_TSO 2296 * 2297 * @param txq 2298 * Pointer to TX queue structure. 2299 * @param loc 2300 * Pointer to burst routine local context. 2301 * @param wqe 2302 * Pointer to WQE to fill with built Control Segment. 2303 * @param ds 2304 * Supposed length of WQE in segments. 2305 * @param opcode 2306 * SQ WQE opcode to put into Control Segment. 2307 * @param olx 2308 * Configured Tx offloads mask. It is fully defined at 2309 * compile time and may be used for optimization. 2310 */ 2311 static __rte_always_inline void 2312 mlx5_tx_cseg_init(struct mlx5_txq_data *__rte_restrict txq, 2313 struct mlx5_txq_local *__rte_restrict loc __rte_unused, 2314 struct mlx5_wqe *__rte_restrict wqe, 2315 unsigned int ds, 2316 unsigned int opcode, 2317 unsigned int olx __rte_unused) 2318 { 2319 struct mlx5_wqe_cseg *__rte_restrict cs = &wqe->cseg; 2320 2321 /* For legacy MPW replace the EMPW by TSO with modifier. */ 2322 if (MLX5_TXOFF_CONFIG(MPW) && opcode == MLX5_OPCODE_ENHANCED_MPSW) 2323 opcode = MLX5_OPCODE_TSO | MLX5_OPC_MOD_MPW << 24; 2324 cs->opcode = rte_cpu_to_be_32((txq->wqe_ci << 8) | opcode); 2325 cs->sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 2326 cs->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 2327 MLX5_COMP_MODE_OFFSET); 2328 cs->misc = RTE_BE32(0); 2329 } 2330 2331 /** 2332 * Build the Synchronize Queue Segment with specified completion index. 2333 * 2334 * @param txq 2335 * Pointer to TX queue structure. 2336 * @param loc 2337 * Pointer to burst routine local context. 2338 * @param wqe 2339 * Pointer to WQE to fill with built Control Segment. 2340 * @param wci 2341 * Completion index in Clock Queue to wait. 2342 * @param olx 2343 * Configured Tx offloads mask. It is fully defined at 2344 * compile time and may be used for optimization. 2345 */ 2346 static __rte_always_inline void 2347 mlx5_tx_wseg_init(struct mlx5_txq_data *restrict txq, 2348 struct mlx5_txq_local *restrict loc __rte_unused, 2349 struct mlx5_wqe *restrict wqe, 2350 unsigned int wci, 2351 unsigned int olx __rte_unused) 2352 { 2353 struct mlx5_wqe_qseg *qs; 2354 2355 qs = RTE_PTR_ADD(wqe, MLX5_WSEG_SIZE); 2356 qs->max_index = rte_cpu_to_be_32(wci); 2357 qs->qpn_cqn = rte_cpu_to_be_32(txq->sh->txpp.clock_queue.cq_obj.cq->id); 2358 qs->reserved0 = RTE_BE32(0); 2359 qs->reserved1 = RTE_BE32(0); 2360 } 2361 2362 /** 2363 * Build the Ethernet Segment without inlined data. 2364 * Supports Software Parser, Checksums and VLAN 2365 * insertion Tx offload features. 2366 * 2367 * @param txq 2368 * Pointer to TX queue structure. 2369 * @param loc 2370 * Pointer to burst routine local context. 2371 * @param wqe 2372 * Pointer to WQE to fill with built Ethernet Segment. 2373 * @param olx 2374 * Configured Tx offloads mask. It is fully defined at 2375 * compile time and may be used for optimization. 2376 */ 2377 static __rte_always_inline void 2378 mlx5_tx_eseg_none(struct mlx5_txq_data *__rte_restrict txq __rte_unused, 2379 struct mlx5_txq_local *__rte_restrict loc, 2380 struct mlx5_wqe *__rte_restrict wqe, 2381 unsigned int olx) 2382 { 2383 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg; 2384 uint32_t csum; 2385 2386 /* 2387 * Calculate and set check sum flags first, dword field 2388 * in segment may be shared with Software Parser flags. 2389 */ 2390 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2391 es->flags = rte_cpu_to_le_32(csum); 2392 /* 2393 * Calculate and set Software Parser offsets and flags. 2394 * These flags a set for custom UDP and IP tunnel packets. 2395 */ 2396 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2397 /* Fill metadata field if needed. */ 2398 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2399 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2400 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2401 /* Engage VLAN tag insertion feature if requested. */ 2402 if (MLX5_TXOFF_CONFIG(VLAN) && 2403 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 2404 /* 2405 * We should get here only if device support 2406 * this feature correctly. 2407 */ 2408 MLX5_ASSERT(txq->vlan_en); 2409 es->inline_hdr = rte_cpu_to_be_32(MLX5_ETH_WQE_VLAN_INSERT | 2410 loc->mbuf->vlan_tci); 2411 } else { 2412 es->inline_hdr = RTE_BE32(0); 2413 } 2414 } 2415 2416 /** 2417 * Build the Ethernet Segment with minimal inlined data 2418 * of MLX5_ESEG_MIN_INLINE_SIZE bytes length. This is 2419 * used to fill the gap in single WQEBB WQEs. 2420 * Supports Software Parser, Checksums and VLAN 2421 * insertion Tx offload features. 2422 * 2423 * @param txq 2424 * Pointer to TX queue structure. 2425 * @param loc 2426 * Pointer to burst routine local context. 2427 * @param wqe 2428 * Pointer to WQE to fill with built Ethernet Segment. 2429 * @param vlan 2430 * Length of VLAN tag insertion if any. 2431 * @param olx 2432 * Configured Tx offloads mask. It is fully defined at 2433 * compile time and may be used for optimization. 2434 */ 2435 static __rte_always_inline void 2436 mlx5_tx_eseg_dmin(struct mlx5_txq_data *__rte_restrict txq __rte_unused, 2437 struct mlx5_txq_local *__rte_restrict loc, 2438 struct mlx5_wqe *__rte_restrict wqe, 2439 unsigned int vlan, 2440 unsigned int olx) 2441 { 2442 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg; 2443 uint32_t csum; 2444 uint8_t *psrc, *pdst; 2445 2446 /* 2447 * Calculate and set check sum flags first, dword field 2448 * in segment may be shared with Software Parser flags. 2449 */ 2450 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2451 es->flags = rte_cpu_to_le_32(csum); 2452 /* 2453 * Calculate and set Software Parser offsets and flags. 2454 * These flags a set for custom UDP and IP tunnel packets. 2455 */ 2456 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2457 /* Fill metadata field if needed. */ 2458 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2459 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2460 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2461 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 2462 es->inline_hdr_sz = RTE_BE16(MLX5_ESEG_MIN_INLINE_SIZE); 2463 es->inline_data = *(unaligned_uint16_t *)psrc; 2464 psrc += sizeof(uint16_t); 2465 pdst = (uint8_t *)(es + 1); 2466 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2467 /* Implement VLAN tag insertion as part inline data. */ 2468 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t)); 2469 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2470 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2471 /* Insert VLAN ethertype + VLAN tag. */ 2472 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2473 ((RTE_ETHER_TYPE_VLAN << 16) | 2474 loc->mbuf->vlan_tci); 2475 pdst += sizeof(struct rte_vlan_hdr); 2476 /* Copy the rest two bytes from packet data. */ 2477 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t))); 2478 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc; 2479 } else { 2480 /* Fill the gap in the title WQEBB with inline data. */ 2481 rte_mov16(pdst, psrc); 2482 } 2483 } 2484 2485 /** 2486 * Build the Ethernet Segment with entire packet 2487 * data inlining. Checks the boundary of WQEBB and 2488 * ring buffer wrapping, supports Software Parser, 2489 * Checksums and VLAN insertion Tx offload features. 2490 * 2491 * @param txq 2492 * Pointer to TX queue structure. 2493 * @param loc 2494 * Pointer to burst routine local context. 2495 * @param wqe 2496 * Pointer to WQE to fill with built Ethernet Segment. 2497 * @param vlan 2498 * Length of VLAN tag insertion if any. 2499 * @param inlen 2500 * Length of data to inline (VLAN included, if any). 2501 * @param tso 2502 * TSO flag, set mss field from the packet. 2503 * @param olx 2504 * Configured Tx offloads mask. It is fully defined at 2505 * compile time and may be used for optimization. 2506 * 2507 * @return 2508 * Pointer to the next Data Segment (aligned and wrapped around). 2509 */ 2510 static __rte_always_inline struct mlx5_wqe_dseg * 2511 mlx5_tx_eseg_data(struct mlx5_txq_data *__rte_restrict txq, 2512 struct mlx5_txq_local *__rte_restrict loc, 2513 struct mlx5_wqe *__rte_restrict wqe, 2514 unsigned int vlan, 2515 unsigned int inlen, 2516 unsigned int tso, 2517 unsigned int olx) 2518 { 2519 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg; 2520 uint32_t csum; 2521 uint8_t *psrc, *pdst; 2522 unsigned int part; 2523 2524 /* 2525 * Calculate and set check sum flags first, dword field 2526 * in segment may be shared with Software Parser flags. 2527 */ 2528 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2529 if (tso) { 2530 csum <<= 24; 2531 csum |= loc->mbuf->tso_segsz; 2532 es->flags = rte_cpu_to_be_32(csum); 2533 } else { 2534 es->flags = rte_cpu_to_le_32(csum); 2535 } 2536 /* 2537 * Calculate and set Software Parser offsets and flags. 2538 * These flags a set for custom UDP and IP tunnel packets. 2539 */ 2540 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2541 /* Fill metadata field if needed. */ 2542 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2543 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2544 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2545 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 2546 es->inline_hdr_sz = rte_cpu_to_be_16(inlen); 2547 es->inline_data = *(unaligned_uint16_t *)psrc; 2548 psrc += sizeof(uint16_t); 2549 pdst = (uint8_t *)(es + 1); 2550 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2551 /* Implement VLAN tag insertion as part inline data. */ 2552 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t)); 2553 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2554 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2555 /* Insert VLAN ethertype + VLAN tag. */ 2556 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2557 ((RTE_ETHER_TYPE_VLAN << 16) | 2558 loc->mbuf->vlan_tci); 2559 pdst += sizeof(struct rte_vlan_hdr); 2560 /* Copy the rest two bytes from packet data. */ 2561 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t))); 2562 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc; 2563 psrc += sizeof(uint16_t); 2564 } else { 2565 /* Fill the gap in the title WQEBB with inline data. */ 2566 rte_mov16(pdst, psrc); 2567 psrc += sizeof(rte_v128u32_t); 2568 } 2569 pdst = (uint8_t *)(es + 2); 2570 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE); 2571 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end); 2572 inlen -= MLX5_ESEG_MIN_INLINE_SIZE; 2573 if (!inlen) { 2574 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE)); 2575 return (struct mlx5_wqe_dseg *)pdst; 2576 } 2577 /* 2578 * The WQEBB space availability is checked by caller. 2579 * Here we should be aware of WQE ring buffer wraparound only. 2580 */ 2581 part = (uint8_t *)txq->wqes_end - pdst; 2582 part = RTE_MIN(part, inlen); 2583 do { 2584 rte_memcpy(pdst, psrc, part); 2585 inlen -= part; 2586 if (likely(!inlen)) { 2587 /* 2588 * If return value is not used by the caller 2589 * the code below will be optimized out. 2590 */ 2591 pdst += part; 2592 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2593 if (unlikely(pdst >= (uint8_t *)txq->wqes_end)) 2594 pdst = (uint8_t *)txq->wqes; 2595 return (struct mlx5_wqe_dseg *)pdst; 2596 } 2597 pdst = (uint8_t *)txq->wqes; 2598 psrc += part; 2599 part = inlen; 2600 } while (true); 2601 } 2602 2603 /** 2604 * Copy data from chain of mbuf to the specified linear buffer. 2605 * Checksums and VLAN insertion Tx offload features. If data 2606 * from some mbuf copied completely this mbuf is freed. Local 2607 * structure is used to keep the byte stream state. 2608 * 2609 * @param pdst 2610 * Pointer to the destination linear buffer. 2611 * @param loc 2612 * Pointer to burst routine local context. 2613 * @param len 2614 * Length of data to be copied. 2615 * @param must 2616 * Length of data to be copied ignoring no inline hint. 2617 * @param olx 2618 * Configured Tx offloads mask. It is fully defined at 2619 * compile time and may be used for optimization. 2620 * 2621 * @return 2622 * Number of actual copied data bytes. This is always greater than or 2623 * equal to must parameter and might be lesser than len in no inline 2624 * hint flag is encountered. 2625 */ 2626 static __rte_always_inline unsigned int 2627 mlx5_tx_mseg_memcpy(uint8_t *pdst, 2628 struct mlx5_txq_local *__rte_restrict loc, 2629 unsigned int len, 2630 unsigned int must, 2631 unsigned int olx __rte_unused) 2632 { 2633 struct rte_mbuf *mbuf; 2634 unsigned int part, dlen, copy = 0; 2635 uint8_t *psrc; 2636 2637 MLX5_ASSERT(len); 2638 MLX5_ASSERT(must <= len); 2639 do { 2640 /* Allow zero length packets, must check first. */ 2641 dlen = rte_pktmbuf_data_len(loc->mbuf); 2642 if (dlen <= loc->mbuf_off) { 2643 /* Exhausted packet, just free. */ 2644 mbuf = loc->mbuf; 2645 loc->mbuf = mbuf->next; 2646 rte_pktmbuf_free_seg(mbuf); 2647 loc->mbuf_off = 0; 2648 MLX5_ASSERT(loc->mbuf_nseg > 1); 2649 MLX5_ASSERT(loc->mbuf); 2650 --loc->mbuf_nseg; 2651 if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) { 2652 unsigned int diff; 2653 2654 if (copy >= must) { 2655 /* 2656 * We already copied the minimal 2657 * requested amount of data. 2658 */ 2659 return copy; 2660 } 2661 diff = must - copy; 2662 if (diff <= rte_pktmbuf_data_len(loc->mbuf)) { 2663 /* 2664 * Copy only the minimal required 2665 * part of the data buffer. 2666 */ 2667 len = diff; 2668 } 2669 } 2670 continue; 2671 } 2672 dlen -= loc->mbuf_off; 2673 psrc = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *, 2674 loc->mbuf_off); 2675 part = RTE_MIN(len, dlen); 2676 rte_memcpy(pdst, psrc, part); 2677 copy += part; 2678 loc->mbuf_off += part; 2679 len -= part; 2680 if (!len) { 2681 if (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) { 2682 loc->mbuf_off = 0; 2683 /* Exhausted packet, just free. */ 2684 mbuf = loc->mbuf; 2685 loc->mbuf = mbuf->next; 2686 rte_pktmbuf_free_seg(mbuf); 2687 loc->mbuf_off = 0; 2688 MLX5_ASSERT(loc->mbuf_nseg >= 1); 2689 --loc->mbuf_nseg; 2690 } 2691 return copy; 2692 } 2693 pdst += part; 2694 } while (true); 2695 } 2696 2697 /** 2698 * Build the Ethernet Segment with inlined data from 2699 * multi-segment packet. Checks the boundary of WQEBB 2700 * and ring buffer wrapping, supports Software Parser, 2701 * Checksums and VLAN insertion Tx offload features. 2702 * 2703 * @param txq 2704 * Pointer to TX queue structure. 2705 * @param loc 2706 * Pointer to burst routine local context. 2707 * @param wqe 2708 * Pointer to WQE to fill with built Ethernet Segment. 2709 * @param vlan 2710 * Length of VLAN tag insertion if any. 2711 * @param inlen 2712 * Length of data to inline (VLAN included, if any). 2713 * @param tso 2714 * TSO flag, set mss field from the packet. 2715 * @param olx 2716 * Configured Tx offloads mask. It is fully defined at 2717 * compile time and may be used for optimization. 2718 * 2719 * @return 2720 * Pointer to the next Data Segment (aligned and 2721 * possible NOT wrapped around - caller should do 2722 * wrapping check on its own). 2723 */ 2724 static __rte_always_inline struct mlx5_wqe_dseg * 2725 mlx5_tx_eseg_mdat(struct mlx5_txq_data *__rte_restrict txq, 2726 struct mlx5_txq_local *__rte_restrict loc, 2727 struct mlx5_wqe *__rte_restrict wqe, 2728 unsigned int vlan, 2729 unsigned int inlen, 2730 unsigned int tso, 2731 unsigned int olx) 2732 { 2733 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg; 2734 uint32_t csum; 2735 uint8_t *pdst; 2736 unsigned int part, tlen = 0; 2737 2738 /* 2739 * Calculate and set check sum flags first, uint32_t field 2740 * in segment may be shared with Software Parser flags. 2741 */ 2742 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2743 if (tso) { 2744 csum <<= 24; 2745 csum |= loc->mbuf->tso_segsz; 2746 es->flags = rte_cpu_to_be_32(csum); 2747 } else { 2748 es->flags = rte_cpu_to_le_32(csum); 2749 } 2750 /* 2751 * Calculate and set Software Parser offsets and flags. 2752 * These flags a set for custom UDP and IP tunnel packets. 2753 */ 2754 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2755 /* Fill metadata field if needed. */ 2756 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2757 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2758 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2759 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE); 2760 pdst = (uint8_t *)&es->inline_data; 2761 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2762 /* Implement VLAN tag insertion as part inline data. */ 2763 mlx5_tx_mseg_memcpy(pdst, loc, 2764 2 * RTE_ETHER_ADDR_LEN, 2765 2 * RTE_ETHER_ADDR_LEN, olx); 2766 pdst += 2 * RTE_ETHER_ADDR_LEN; 2767 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2768 ((RTE_ETHER_TYPE_VLAN << 16) | 2769 loc->mbuf->vlan_tci); 2770 pdst += sizeof(struct rte_vlan_hdr); 2771 tlen += 2 * RTE_ETHER_ADDR_LEN + sizeof(struct rte_vlan_hdr); 2772 } 2773 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end); 2774 /* 2775 * The WQEBB space availability is checked by caller. 2776 * Here we should be aware of WQE ring buffer wraparound only. 2777 */ 2778 part = (uint8_t *)txq->wqes_end - pdst; 2779 part = RTE_MIN(part, inlen - tlen); 2780 MLX5_ASSERT(part); 2781 do { 2782 unsigned int copy; 2783 2784 /* 2785 * Copying may be interrupted inside the routine 2786 * if run into no inline hint flag. 2787 */ 2788 copy = tlen >= txq->inlen_mode ? 0 : (txq->inlen_mode - tlen); 2789 copy = mlx5_tx_mseg_memcpy(pdst, loc, part, copy, olx); 2790 tlen += copy; 2791 if (likely(inlen <= tlen) || copy < part) { 2792 es->inline_hdr_sz = rte_cpu_to_be_16(tlen); 2793 pdst += copy; 2794 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2795 return (struct mlx5_wqe_dseg *)pdst; 2796 } 2797 pdst = (uint8_t *)txq->wqes; 2798 part = inlen - tlen; 2799 } while (true); 2800 } 2801 2802 /** 2803 * Build the Data Segment of pointer type. 2804 * 2805 * @param txq 2806 * Pointer to TX queue structure. 2807 * @param loc 2808 * Pointer to burst routine local context. 2809 * @param dseg 2810 * Pointer to WQE to fill with built Data Segment. 2811 * @param buf 2812 * Data buffer to point. 2813 * @param len 2814 * Data buffer length. 2815 * @param olx 2816 * Configured Tx offloads mask. It is fully defined at 2817 * compile time and may be used for optimization. 2818 */ 2819 static __rte_always_inline void 2820 mlx5_tx_dseg_ptr(struct mlx5_txq_data *__rte_restrict txq, 2821 struct mlx5_txq_local *__rte_restrict loc, 2822 struct mlx5_wqe_dseg *__rte_restrict dseg, 2823 uint8_t *buf, 2824 unsigned int len, 2825 unsigned int olx __rte_unused) 2826 2827 { 2828 MLX5_ASSERT(len); 2829 dseg->bcount = rte_cpu_to_be_32(len); 2830 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf); 2831 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf); 2832 } 2833 2834 /** 2835 * Build the Data Segment of pointer type or inline 2836 * if data length is less than buffer in minimal 2837 * Data Segment size. 2838 * 2839 * @param txq 2840 * Pointer to TX queue structure. 2841 * @param loc 2842 * Pointer to burst routine local context. 2843 * @param dseg 2844 * Pointer to WQE to fill with built Data Segment. 2845 * @param buf 2846 * Data buffer to point. 2847 * @param len 2848 * Data buffer length. 2849 * @param olx 2850 * Configured Tx offloads mask. It is fully defined at 2851 * compile time and may be used for optimization. 2852 */ 2853 static __rte_always_inline void 2854 mlx5_tx_dseg_iptr(struct mlx5_txq_data *__rte_restrict txq, 2855 struct mlx5_txq_local *__rte_restrict loc, 2856 struct mlx5_wqe_dseg *__rte_restrict dseg, 2857 uint8_t *buf, 2858 unsigned int len, 2859 unsigned int olx __rte_unused) 2860 2861 { 2862 uintptr_t dst, src; 2863 2864 MLX5_ASSERT(len); 2865 if (len > MLX5_DSEG_MIN_INLINE_SIZE) { 2866 dseg->bcount = rte_cpu_to_be_32(len); 2867 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf); 2868 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf); 2869 2870 return; 2871 } 2872 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE); 2873 /* Unrolled implementation of generic rte_memcpy. */ 2874 dst = (uintptr_t)&dseg->inline_data[0]; 2875 src = (uintptr_t)buf; 2876 if (len & 0x08) { 2877 #ifdef RTE_ARCH_STRICT_ALIGN 2878 MLX5_ASSERT(dst == RTE_PTR_ALIGN(dst, sizeof(uint32_t))); 2879 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2880 dst += sizeof(uint32_t); 2881 src += sizeof(uint32_t); 2882 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2883 dst += sizeof(uint32_t); 2884 src += sizeof(uint32_t); 2885 #else 2886 *(uint64_t *)dst = *(unaligned_uint64_t *)src; 2887 dst += sizeof(uint64_t); 2888 src += sizeof(uint64_t); 2889 #endif 2890 } 2891 if (len & 0x04) { 2892 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2893 dst += sizeof(uint32_t); 2894 src += sizeof(uint32_t); 2895 } 2896 if (len & 0x02) { 2897 *(uint16_t *)dst = *(unaligned_uint16_t *)src; 2898 dst += sizeof(uint16_t); 2899 src += sizeof(uint16_t); 2900 } 2901 if (len & 0x01) 2902 *(uint8_t *)dst = *(uint8_t *)src; 2903 } 2904 2905 /** 2906 * Build the Data Segment of inlined data from single 2907 * segment packet, no VLAN insertion. 2908 * 2909 * @param txq 2910 * Pointer to TX queue structure. 2911 * @param loc 2912 * Pointer to burst routine local context. 2913 * @param dseg 2914 * Pointer to WQE to fill with built Data Segment. 2915 * @param buf 2916 * Data buffer to point. 2917 * @param len 2918 * Data buffer length. 2919 * @param olx 2920 * Configured Tx offloads mask. It is fully defined at 2921 * compile time and may be used for optimization. 2922 * 2923 * @return 2924 * Pointer to the next Data Segment after inlined data. 2925 * Ring buffer wraparound check is needed. We do not 2926 * do it here because it may not be needed for the 2927 * last packet in the eMPW session. 2928 */ 2929 static __rte_always_inline struct mlx5_wqe_dseg * 2930 mlx5_tx_dseg_empw(struct mlx5_txq_data *__rte_restrict txq, 2931 struct mlx5_txq_local *__rte_restrict loc __rte_unused, 2932 struct mlx5_wqe_dseg *__rte_restrict dseg, 2933 uint8_t *buf, 2934 unsigned int len, 2935 unsigned int olx __rte_unused) 2936 { 2937 unsigned int part; 2938 uint8_t *pdst; 2939 2940 if (!MLX5_TXOFF_CONFIG(MPW)) { 2941 /* Store the descriptor byte counter for eMPW sessions. */ 2942 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE); 2943 pdst = &dseg->inline_data[0]; 2944 } else { 2945 /* The entire legacy MPW session counter is stored on close. */ 2946 pdst = (uint8_t *)dseg; 2947 } 2948 /* 2949 * The WQEBB space availability is checked by caller. 2950 * Here we should be aware of WQE ring buffer wraparound only. 2951 */ 2952 part = (uint8_t *)txq->wqes_end - pdst; 2953 part = RTE_MIN(part, len); 2954 do { 2955 rte_memcpy(pdst, buf, part); 2956 len -= part; 2957 if (likely(!len)) { 2958 pdst += part; 2959 if (!MLX5_TXOFF_CONFIG(MPW)) 2960 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2961 /* Note: no final wraparound check here. */ 2962 return (struct mlx5_wqe_dseg *)pdst; 2963 } 2964 pdst = (uint8_t *)txq->wqes; 2965 buf += part; 2966 part = len; 2967 } while (true); 2968 } 2969 2970 /** 2971 * Build the Data Segment of inlined data from single 2972 * segment packet with VLAN insertion. 2973 * 2974 * @param txq 2975 * Pointer to TX queue structure. 2976 * @param loc 2977 * Pointer to burst routine local context. 2978 * @param dseg 2979 * Pointer to the dseg fill with built Data Segment. 2980 * @param buf 2981 * Data buffer to point. 2982 * @param len 2983 * Data buffer length. 2984 * @param olx 2985 * Configured Tx offloads mask. It is fully defined at 2986 * compile time and may be used for optimization. 2987 * 2988 * @return 2989 * Pointer to the next Data Segment after inlined data. 2990 * Ring buffer wraparound check is needed. 2991 */ 2992 static __rte_always_inline struct mlx5_wqe_dseg * 2993 mlx5_tx_dseg_vlan(struct mlx5_txq_data *__rte_restrict txq, 2994 struct mlx5_txq_local *__rte_restrict loc __rte_unused, 2995 struct mlx5_wqe_dseg *__rte_restrict dseg, 2996 uint8_t *buf, 2997 unsigned int len, 2998 unsigned int olx __rte_unused) 2999 3000 { 3001 unsigned int part; 3002 uint8_t *pdst; 3003 3004 MLX5_ASSERT(len > MLX5_ESEG_MIN_INLINE_SIZE); 3005 if (!MLX5_TXOFF_CONFIG(MPW)) { 3006 /* Store the descriptor byte counter for eMPW sessions. */ 3007 dseg->bcount = rte_cpu_to_be_32 3008 ((len + sizeof(struct rte_vlan_hdr)) | 3009 MLX5_ETH_WQE_DATA_INLINE); 3010 pdst = &dseg->inline_data[0]; 3011 } else { 3012 /* The entire legacy MPW session counter is stored on close. */ 3013 pdst = (uint8_t *)dseg; 3014 } 3015 memcpy(pdst, buf, MLX5_DSEG_MIN_INLINE_SIZE); 3016 buf += MLX5_DSEG_MIN_INLINE_SIZE; 3017 pdst += MLX5_DSEG_MIN_INLINE_SIZE; 3018 len -= MLX5_DSEG_MIN_INLINE_SIZE; 3019 /* Insert VLAN ethertype + VLAN tag. Pointer is aligned. */ 3020 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE)); 3021 if (unlikely(pdst >= (uint8_t *)txq->wqes_end)) 3022 pdst = (uint8_t *)txq->wqes; 3023 *(uint32_t *)pdst = rte_cpu_to_be_32((RTE_ETHER_TYPE_VLAN << 16) | 3024 loc->mbuf->vlan_tci); 3025 pdst += sizeof(struct rte_vlan_hdr); 3026 /* 3027 * The WQEBB space availability is checked by caller. 3028 * Here we should be aware of WQE ring buffer wraparound only. 3029 */ 3030 part = (uint8_t *)txq->wqes_end - pdst; 3031 part = RTE_MIN(part, len); 3032 do { 3033 rte_memcpy(pdst, buf, part); 3034 len -= part; 3035 if (likely(!len)) { 3036 pdst += part; 3037 if (!MLX5_TXOFF_CONFIG(MPW)) 3038 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 3039 /* Note: no final wraparound check here. */ 3040 return (struct mlx5_wqe_dseg *)pdst; 3041 } 3042 pdst = (uint8_t *)txq->wqes; 3043 buf += part; 3044 part = len; 3045 } while (true); 3046 } 3047 3048 /** 3049 * Build the Ethernet Segment with optionally inlined data with 3050 * VLAN insertion and following Data Segments (if any) from 3051 * multi-segment packet. Used by ordinary send and TSO. 3052 * 3053 * @param txq 3054 * Pointer to TX queue structure. 3055 * @param loc 3056 * Pointer to burst routine local context. 3057 * @param wqe 3058 * Pointer to WQE to fill with built Ethernet/Data Segments. 3059 * @param vlan 3060 * Length of VLAN header to insert, 0 means no VLAN insertion. 3061 * @param inlen 3062 * Data length to inline. For TSO this parameter specifies 3063 * exact value, for ordinary send routine can be aligned by 3064 * caller to provide better WQE space saving and data buffer 3065 * start address alignment. This length includes VLAN header 3066 * being inserted. 3067 * @param tso 3068 * Zero means ordinary send, inlined data can be extended, 3069 * otherwise this is TSO, inlined data length is fixed. 3070 * @param olx 3071 * Configured Tx offloads mask. It is fully defined at 3072 * compile time and may be used for optimization. 3073 * 3074 * @return 3075 * Actual size of built WQE in segments. 3076 */ 3077 static __rte_always_inline unsigned int 3078 mlx5_tx_mseg_build(struct mlx5_txq_data *__rte_restrict txq, 3079 struct mlx5_txq_local *__rte_restrict loc, 3080 struct mlx5_wqe *__rte_restrict wqe, 3081 unsigned int vlan, 3082 unsigned int inlen, 3083 unsigned int tso, 3084 unsigned int olx __rte_unused) 3085 { 3086 struct mlx5_wqe_dseg *__rte_restrict dseg; 3087 unsigned int ds; 3088 3089 MLX5_ASSERT((rte_pktmbuf_pkt_len(loc->mbuf) + vlan) >= inlen); 3090 loc->mbuf_nseg = NB_SEGS(loc->mbuf); 3091 loc->mbuf_off = 0; 3092 3093 dseg = mlx5_tx_eseg_mdat(txq, loc, wqe, vlan, inlen, tso, olx); 3094 if (!loc->mbuf_nseg) 3095 goto dseg_done; 3096 /* 3097 * There are still some mbuf remaining, not inlined. 3098 * The first mbuf may be partially inlined and we 3099 * must process the possible non-zero data offset. 3100 */ 3101 if (loc->mbuf_off) { 3102 unsigned int dlen; 3103 uint8_t *dptr; 3104 3105 /* 3106 * Exhausted packets must be dropped before. 3107 * Non-zero offset means there are some data 3108 * remained in the packet. 3109 */ 3110 MLX5_ASSERT(loc->mbuf_off < rte_pktmbuf_data_len(loc->mbuf)); 3111 MLX5_ASSERT(rte_pktmbuf_data_len(loc->mbuf)); 3112 dptr = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *, 3113 loc->mbuf_off); 3114 dlen = rte_pktmbuf_data_len(loc->mbuf) - loc->mbuf_off; 3115 /* 3116 * Build the pointer/minimal data Data Segment. 3117 * Do ring buffer wrapping check in advance. 3118 */ 3119 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3120 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3121 mlx5_tx_dseg_iptr(txq, loc, dseg, dptr, dlen, olx); 3122 /* Store the mbuf to be freed on completion. */ 3123 MLX5_ASSERT(loc->elts_free); 3124 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3125 --loc->elts_free; 3126 ++dseg; 3127 if (--loc->mbuf_nseg == 0) 3128 goto dseg_done; 3129 loc->mbuf = loc->mbuf->next; 3130 loc->mbuf_off = 0; 3131 } 3132 do { 3133 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) { 3134 struct rte_mbuf *mbuf; 3135 3136 /* Zero length segment found, just skip. */ 3137 mbuf = loc->mbuf; 3138 loc->mbuf = loc->mbuf->next; 3139 rte_pktmbuf_free_seg(mbuf); 3140 if (--loc->mbuf_nseg == 0) 3141 break; 3142 } else { 3143 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3144 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3145 mlx5_tx_dseg_iptr 3146 (txq, loc, dseg, 3147 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 3148 rte_pktmbuf_data_len(loc->mbuf), olx); 3149 MLX5_ASSERT(loc->elts_free); 3150 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3151 --loc->elts_free; 3152 ++dseg; 3153 if (--loc->mbuf_nseg == 0) 3154 break; 3155 loc->mbuf = loc->mbuf->next; 3156 } 3157 } while (true); 3158 3159 dseg_done: 3160 /* Calculate actual segments used from the dseg pointer. */ 3161 if ((uintptr_t)wqe < (uintptr_t)dseg) 3162 ds = ((uintptr_t)dseg - (uintptr_t)wqe) / MLX5_WSEG_SIZE; 3163 else 3164 ds = (((uintptr_t)dseg - (uintptr_t)wqe) + 3165 txq->wqe_s * MLX5_WQE_SIZE) / MLX5_WSEG_SIZE; 3166 return ds; 3167 } 3168 3169 /** 3170 * The routine checks timestamp flag in the current packet, 3171 * and push WAIT WQE into the queue if scheduling is required. 3172 * 3173 * @param txq 3174 * Pointer to TX queue structure. 3175 * @param loc 3176 * Pointer to burst routine local context. 3177 * @param olx 3178 * Configured Tx offloads mask. It is fully defined at 3179 * compile time and may be used for optimization. 3180 * 3181 * @return 3182 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3183 * MLX5_TXCMP_CODE_SINGLE - continue processing with the packet. 3184 * MLX5_TXCMP_CODE_MULTI - the WAIT inserted, continue processing. 3185 * Local context variables partially updated. 3186 */ 3187 static __rte_always_inline enum mlx5_txcmp_code 3188 mlx5_tx_schedule_send(struct mlx5_txq_data *restrict txq, 3189 struct mlx5_txq_local *restrict loc, 3190 unsigned int olx) 3191 { 3192 if (MLX5_TXOFF_CONFIG(TXPP) && 3193 loc->mbuf->ol_flags & txq->ts_mask) { 3194 struct mlx5_wqe *wqe; 3195 uint64_t ts; 3196 int32_t wci; 3197 3198 /* 3199 * Estimate the required space quickly and roughly. 3200 * We would like to ensure the packet can be pushed 3201 * to the queue and we won't get the orphan WAIT WQE. 3202 */ 3203 if (loc->wqe_free <= MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE || 3204 loc->elts_free < NB_SEGS(loc->mbuf)) 3205 return MLX5_TXCMP_CODE_EXIT; 3206 /* Convert the timestamp into completion to wait. */ 3207 ts = *RTE_MBUF_DYNFIELD(loc->mbuf, txq->ts_offset, uint64_t *); 3208 wci = mlx5_txpp_convert_tx_ts(txq->sh, ts); 3209 if (unlikely(wci < 0)) 3210 return MLX5_TXCMP_CODE_SINGLE; 3211 /* Build the WAIT WQE with specified completion. */ 3212 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3213 mlx5_tx_cseg_init(txq, loc, wqe, 2, MLX5_OPCODE_WAIT, olx); 3214 mlx5_tx_wseg_init(txq, loc, wqe, wci, olx); 3215 ++txq->wqe_ci; 3216 --loc->wqe_free; 3217 return MLX5_TXCMP_CODE_MULTI; 3218 } 3219 return MLX5_TXCMP_CODE_SINGLE; 3220 } 3221 3222 /** 3223 * Tx one packet function for multi-segment TSO. Supports all 3224 * types of Tx offloads, uses MLX5_OPCODE_TSO to build WQEs, 3225 * sends one packet per WQE. 3226 * 3227 * This routine is responsible for storing processed mbuf 3228 * into elts ring buffer and update elts_head. 3229 * 3230 * @param txq 3231 * Pointer to TX queue structure. 3232 * @param loc 3233 * Pointer to burst routine local context. 3234 * @param olx 3235 * Configured Tx offloads mask. It is fully defined at 3236 * compile time and may be used for optimization. 3237 * 3238 * @return 3239 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3240 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3241 * Local context variables partially updated. 3242 */ 3243 static __rte_always_inline enum mlx5_txcmp_code 3244 mlx5_tx_packet_multi_tso(struct mlx5_txq_data *__rte_restrict txq, 3245 struct mlx5_txq_local *__rte_restrict loc, 3246 unsigned int olx) 3247 { 3248 struct mlx5_wqe *__rte_restrict wqe; 3249 unsigned int ds, dlen, inlen, ntcp, vlan = 0; 3250 3251 if (MLX5_TXOFF_CONFIG(TXPP)) { 3252 enum mlx5_txcmp_code wret; 3253 3254 /* Generate WAIT for scheduling if requested. */ 3255 wret = mlx5_tx_schedule_send(txq, loc, olx); 3256 if (wret == MLX5_TXCMP_CODE_EXIT) 3257 return MLX5_TXCMP_CODE_EXIT; 3258 if (wret == MLX5_TXCMP_CODE_ERROR) 3259 return MLX5_TXCMP_CODE_ERROR; 3260 } 3261 /* 3262 * Calculate data length to be inlined to estimate 3263 * the required space in WQE ring buffer. 3264 */ 3265 dlen = rte_pktmbuf_pkt_len(loc->mbuf); 3266 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3267 vlan = sizeof(struct rte_vlan_hdr); 3268 inlen = loc->mbuf->l2_len + vlan + 3269 loc->mbuf->l3_len + loc->mbuf->l4_len; 3270 if (unlikely((!inlen || !loc->mbuf->tso_segsz))) 3271 return MLX5_TXCMP_CODE_ERROR; 3272 if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK) 3273 inlen += loc->mbuf->outer_l2_len + loc->mbuf->outer_l3_len; 3274 /* Packet must contain all TSO headers. */ 3275 if (unlikely(inlen > MLX5_MAX_TSO_HEADER || 3276 inlen <= MLX5_ESEG_MIN_INLINE_SIZE || 3277 inlen > (dlen + vlan))) 3278 return MLX5_TXCMP_CODE_ERROR; 3279 MLX5_ASSERT(inlen >= txq->inlen_mode); 3280 /* 3281 * Check whether there are enough free WQEBBs: 3282 * - Control Segment 3283 * - Ethernet Segment 3284 * - First Segment of inlined Ethernet data 3285 * - ... data continued ... 3286 * - Data Segments of pointer/min inline type 3287 */ 3288 ds = NB_SEGS(loc->mbuf) + 2 + (inlen - 3289 MLX5_ESEG_MIN_INLINE_SIZE + 3290 MLX5_WSEG_SIZE + 3291 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3292 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3293 return MLX5_TXCMP_CODE_EXIT; 3294 /* Check for maximal WQE size. */ 3295 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3296 return MLX5_TXCMP_CODE_ERROR; 3297 #ifdef MLX5_PMD_SOFT_COUNTERS 3298 /* Update sent data bytes/packets counters. */ 3299 ntcp = (dlen - (inlen - vlan) + loc->mbuf->tso_segsz - 1) / 3300 loc->mbuf->tso_segsz; 3301 /* 3302 * One will be added for mbuf itself 3303 * at the end of the mlx5_tx_burst from 3304 * loc->pkts_sent field. 3305 */ 3306 --ntcp; 3307 txq->stats.opackets += ntcp; 3308 txq->stats.obytes += dlen + vlan + ntcp * inlen; 3309 #endif 3310 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3311 loc->wqe_last = wqe; 3312 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_TSO, olx); 3313 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 1, olx); 3314 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3315 txq->wqe_ci += (ds + 3) / 4; 3316 loc->wqe_free -= (ds + 3) / 4; 3317 return MLX5_TXCMP_CODE_MULTI; 3318 } 3319 3320 /** 3321 * Tx one packet function for multi-segment SEND. Supports all 3322 * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs, 3323 * sends one packet per WQE, without any data inlining in 3324 * Ethernet Segment. 3325 * 3326 * This routine is responsible for storing processed mbuf 3327 * into elts ring buffer and update elts_head. 3328 * 3329 * @param txq 3330 * Pointer to TX queue structure. 3331 * @param loc 3332 * Pointer to burst routine local context. 3333 * @param olx 3334 * Configured Tx offloads mask. It is fully defined at 3335 * compile time and may be used for optimization. 3336 * 3337 * @return 3338 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3339 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3340 * Local context variables partially updated. 3341 */ 3342 static __rte_always_inline enum mlx5_txcmp_code 3343 mlx5_tx_packet_multi_send(struct mlx5_txq_data *__rte_restrict txq, 3344 struct mlx5_txq_local *__rte_restrict loc, 3345 unsigned int olx) 3346 { 3347 struct mlx5_wqe_dseg *__rte_restrict dseg; 3348 struct mlx5_wqe *__rte_restrict wqe; 3349 unsigned int ds, nseg; 3350 3351 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3352 if (MLX5_TXOFF_CONFIG(TXPP)) { 3353 enum mlx5_txcmp_code wret; 3354 3355 /* Generate WAIT for scheduling if requested. */ 3356 wret = mlx5_tx_schedule_send(txq, loc, olx); 3357 if (wret == MLX5_TXCMP_CODE_EXIT) 3358 return MLX5_TXCMP_CODE_EXIT; 3359 if (wret == MLX5_TXCMP_CODE_ERROR) 3360 return MLX5_TXCMP_CODE_ERROR; 3361 } 3362 /* 3363 * No inline at all, it means the CPU cycles saving 3364 * is prioritized at configuration, we should not 3365 * copy any packet data to WQE. 3366 */ 3367 nseg = NB_SEGS(loc->mbuf); 3368 ds = 2 + nseg; 3369 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3370 return MLX5_TXCMP_CODE_EXIT; 3371 /* Check for maximal WQE size. */ 3372 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3373 return MLX5_TXCMP_CODE_ERROR; 3374 /* 3375 * Some Tx offloads may cause an error if 3376 * packet is not long enough, check against 3377 * assumed minimal length. 3378 */ 3379 if (rte_pktmbuf_pkt_len(loc->mbuf) <= MLX5_ESEG_MIN_INLINE_SIZE) 3380 return MLX5_TXCMP_CODE_ERROR; 3381 #ifdef MLX5_PMD_SOFT_COUNTERS 3382 /* Update sent data bytes counter. */ 3383 txq->stats.obytes += rte_pktmbuf_pkt_len(loc->mbuf); 3384 if (MLX5_TXOFF_CONFIG(VLAN) && 3385 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3386 txq->stats.obytes += sizeof(struct rte_vlan_hdr); 3387 #endif 3388 /* 3389 * SEND WQE, one WQEBB: 3390 * - Control Segment, SEND opcode 3391 * - Ethernet Segment, optional VLAN, no inline 3392 * - Data Segments, pointer only type 3393 */ 3394 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3395 loc->wqe_last = wqe; 3396 mlx5_tx_cseg_init(txq, loc, wqe, ds, MLX5_OPCODE_SEND, olx); 3397 mlx5_tx_eseg_none(txq, loc, wqe, olx); 3398 dseg = &wqe->dseg[0]; 3399 do { 3400 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) { 3401 struct rte_mbuf *mbuf; 3402 3403 /* 3404 * Zero length segment found, have to 3405 * correct total size of WQE in segments. 3406 * It is supposed to be rare occasion, so 3407 * in normal case (no zero length segments) 3408 * we avoid extra writing to the Control 3409 * Segment. 3410 */ 3411 --ds; 3412 wqe->cseg.sq_ds -= RTE_BE32(1); 3413 mbuf = loc->mbuf; 3414 loc->mbuf = mbuf->next; 3415 rte_pktmbuf_free_seg(mbuf); 3416 if (--nseg == 0) 3417 break; 3418 } else { 3419 mlx5_tx_dseg_ptr 3420 (txq, loc, dseg, 3421 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 3422 rte_pktmbuf_data_len(loc->mbuf), olx); 3423 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3424 --loc->elts_free; 3425 if (--nseg == 0) 3426 break; 3427 ++dseg; 3428 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3429 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3430 loc->mbuf = loc->mbuf->next; 3431 } 3432 } while (true); 3433 txq->wqe_ci += (ds + 3) / 4; 3434 loc->wqe_free -= (ds + 3) / 4; 3435 return MLX5_TXCMP_CODE_MULTI; 3436 } 3437 3438 /** 3439 * Tx one packet function for multi-segment SEND. Supports all 3440 * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs, 3441 * sends one packet per WQE, with data inlining in 3442 * Ethernet Segment and minimal Data Segments. 3443 * 3444 * This routine is responsible for storing processed mbuf 3445 * into elts ring buffer and update elts_head. 3446 * 3447 * @param txq 3448 * Pointer to TX queue structure. 3449 * @param loc 3450 * Pointer to burst routine local context. 3451 * @param olx 3452 * Configured Tx offloads mask. It is fully defined at 3453 * compile time and may be used for optimization. 3454 * 3455 * @return 3456 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3457 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3458 * Local context variables partially updated. 3459 */ 3460 static __rte_always_inline enum mlx5_txcmp_code 3461 mlx5_tx_packet_multi_inline(struct mlx5_txq_data *__rte_restrict txq, 3462 struct mlx5_txq_local *__rte_restrict loc, 3463 unsigned int olx) 3464 { 3465 struct mlx5_wqe *__rte_restrict wqe; 3466 unsigned int ds, inlen, dlen, vlan = 0; 3467 3468 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 3469 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3470 if (MLX5_TXOFF_CONFIG(TXPP)) { 3471 enum mlx5_txcmp_code wret; 3472 3473 /* Generate WAIT for scheduling if requested. */ 3474 wret = mlx5_tx_schedule_send(txq, loc, olx); 3475 if (wret == MLX5_TXCMP_CODE_EXIT) 3476 return MLX5_TXCMP_CODE_EXIT; 3477 if (wret == MLX5_TXCMP_CODE_ERROR) 3478 return MLX5_TXCMP_CODE_ERROR; 3479 } 3480 /* 3481 * First calculate data length to be inlined 3482 * to estimate the required space for WQE. 3483 */ 3484 dlen = rte_pktmbuf_pkt_len(loc->mbuf); 3485 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3486 vlan = sizeof(struct rte_vlan_hdr); 3487 inlen = dlen + vlan; 3488 /* Check against minimal length. */ 3489 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE) 3490 return MLX5_TXCMP_CODE_ERROR; 3491 MLX5_ASSERT(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE); 3492 if (inlen > txq->inlen_send || 3493 loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) { 3494 struct rte_mbuf *mbuf; 3495 unsigned int nxlen; 3496 uintptr_t start; 3497 3498 /* 3499 * Packet length exceeds the allowed inline 3500 * data length, check whether the minimal 3501 * inlining is required. 3502 */ 3503 if (txq->inlen_mode) { 3504 MLX5_ASSERT(txq->inlen_mode >= 3505 MLX5_ESEG_MIN_INLINE_SIZE); 3506 MLX5_ASSERT(txq->inlen_mode <= txq->inlen_send); 3507 inlen = txq->inlen_mode; 3508 } else { 3509 if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE || 3510 !vlan || txq->vlan_en) { 3511 /* 3512 * VLAN insertion will be done inside by HW. 3513 * It is not utmost effective - VLAN flag is 3514 * checked twice, but we should proceed the 3515 * inlining length correctly and take into 3516 * account the VLAN header being inserted. 3517 */ 3518 return mlx5_tx_packet_multi_send 3519 (txq, loc, olx); 3520 } 3521 inlen = MLX5_ESEG_MIN_INLINE_SIZE; 3522 } 3523 /* 3524 * Now we know the minimal amount of data is requested 3525 * to inline. Check whether we should inline the buffers 3526 * from the chain beginning to eliminate some mbufs. 3527 */ 3528 mbuf = loc->mbuf; 3529 nxlen = rte_pktmbuf_data_len(mbuf); 3530 if (unlikely(nxlen <= txq->inlen_send)) { 3531 /* We can inline first mbuf at least. */ 3532 if (nxlen < inlen) { 3533 unsigned int smlen; 3534 3535 /* Scan mbufs till inlen filled. */ 3536 do { 3537 smlen = nxlen; 3538 mbuf = NEXT(mbuf); 3539 MLX5_ASSERT(mbuf); 3540 nxlen = rte_pktmbuf_data_len(mbuf); 3541 nxlen += smlen; 3542 } while (unlikely(nxlen < inlen)); 3543 if (unlikely(nxlen > txq->inlen_send)) { 3544 /* We cannot inline entire mbuf. */ 3545 smlen = inlen - smlen; 3546 start = rte_pktmbuf_mtod_offset 3547 (mbuf, uintptr_t, smlen); 3548 goto do_align; 3549 } 3550 } 3551 do { 3552 inlen = nxlen; 3553 mbuf = NEXT(mbuf); 3554 /* There should be not end of packet. */ 3555 MLX5_ASSERT(mbuf); 3556 nxlen = inlen + rte_pktmbuf_data_len(mbuf); 3557 } while (unlikely(nxlen < txq->inlen_send)); 3558 } 3559 start = rte_pktmbuf_mtod(mbuf, uintptr_t); 3560 /* 3561 * Check whether we can do inline to align start 3562 * address of data buffer to cacheline. 3563 */ 3564 do_align: 3565 start = (~start + 1) & (RTE_CACHE_LINE_SIZE - 1); 3566 if (unlikely(start)) { 3567 start += inlen; 3568 if (start <= txq->inlen_send) 3569 inlen = start; 3570 } 3571 } 3572 /* 3573 * Check whether there are enough free WQEBBs: 3574 * - Control Segment 3575 * - Ethernet Segment 3576 * - First Segment of inlined Ethernet data 3577 * - ... data continued ... 3578 * - Data Segments of pointer/min inline type 3579 * 3580 * Estimate the number of Data Segments conservatively, 3581 * supposing no any mbufs is being freed during inlining. 3582 */ 3583 MLX5_ASSERT(inlen <= txq->inlen_send); 3584 ds = NB_SEGS(loc->mbuf) + 2 + (inlen - 3585 MLX5_ESEG_MIN_INLINE_SIZE + 3586 MLX5_WSEG_SIZE + 3587 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3588 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3589 return MLX5_TXCMP_CODE_EXIT; 3590 /* Check for maximal WQE size. */ 3591 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3592 return MLX5_TXCMP_CODE_ERROR; 3593 #ifdef MLX5_PMD_SOFT_COUNTERS 3594 /* Update sent data bytes/packets counters. */ 3595 txq->stats.obytes += dlen + vlan; 3596 #endif 3597 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3598 loc->wqe_last = wqe; 3599 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx); 3600 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx); 3601 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3602 txq->wqe_ci += (ds + 3) / 4; 3603 loc->wqe_free -= (ds + 3) / 4; 3604 return MLX5_TXCMP_CODE_MULTI; 3605 } 3606 3607 /** 3608 * Tx burst function for multi-segment packets. Supports all 3609 * types of Tx offloads, uses MLX5_OPCODE_SEND/TSO to build WQEs, 3610 * sends one packet per WQE. Function stops sending if it 3611 * encounters the single-segment packet. 3612 * 3613 * This routine is responsible for storing processed mbuf 3614 * into elts ring buffer and update elts_head. 3615 * 3616 * @param txq 3617 * Pointer to TX queue structure. 3618 * @param[in] pkts 3619 * Packets to transmit. 3620 * @param pkts_n 3621 * Number of packets in array. 3622 * @param loc 3623 * Pointer to burst routine local context. 3624 * @param olx 3625 * Configured Tx offloads mask. It is fully defined at 3626 * compile time and may be used for optimization. 3627 * 3628 * @return 3629 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3630 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3631 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered. 3632 * MLX5_TXCMP_CODE_TSO - TSO single-segment packet encountered. 3633 * Local context variables updated. 3634 */ 3635 static __rte_always_inline enum mlx5_txcmp_code 3636 mlx5_tx_burst_mseg(struct mlx5_txq_data *__rte_restrict txq, 3637 struct rte_mbuf **__rte_restrict pkts, 3638 unsigned int pkts_n, 3639 struct mlx5_txq_local *__rte_restrict loc, 3640 unsigned int olx) 3641 { 3642 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 3643 MLX5_ASSERT(pkts_n > loc->pkts_sent); 3644 pkts += loc->pkts_sent + 1; 3645 pkts_n -= loc->pkts_sent; 3646 for (;;) { 3647 enum mlx5_txcmp_code ret; 3648 3649 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3650 /* 3651 * Estimate the number of free elts quickly but 3652 * conservatively. Some segment may be fully inlined 3653 * and freed, ignore this here - precise estimation 3654 * is costly. 3655 */ 3656 if (loc->elts_free < NB_SEGS(loc->mbuf)) 3657 return MLX5_TXCMP_CODE_EXIT; 3658 if (MLX5_TXOFF_CONFIG(TSO) && 3659 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) { 3660 /* Proceed with multi-segment TSO. */ 3661 ret = mlx5_tx_packet_multi_tso(txq, loc, olx); 3662 } else if (MLX5_TXOFF_CONFIG(INLINE)) { 3663 /* Proceed with multi-segment SEND with inlining. */ 3664 ret = mlx5_tx_packet_multi_inline(txq, loc, olx); 3665 } else { 3666 /* Proceed with multi-segment SEND w/o inlining. */ 3667 ret = mlx5_tx_packet_multi_send(txq, loc, olx); 3668 } 3669 if (ret == MLX5_TXCMP_CODE_EXIT) 3670 return MLX5_TXCMP_CODE_EXIT; 3671 if (ret == MLX5_TXCMP_CODE_ERROR) 3672 return MLX5_TXCMP_CODE_ERROR; 3673 /* WQE is built, go to the next packet. */ 3674 ++loc->pkts_sent; 3675 --pkts_n; 3676 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 3677 return MLX5_TXCMP_CODE_EXIT; 3678 loc->mbuf = *pkts++; 3679 if (pkts_n > 1) 3680 rte_prefetch0(*pkts); 3681 if (likely(NB_SEGS(loc->mbuf) > 1)) 3682 continue; 3683 /* Here ends the series of multi-segment packets. */ 3684 if (MLX5_TXOFF_CONFIG(TSO) && 3685 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) 3686 return MLX5_TXCMP_CODE_TSO; 3687 return MLX5_TXCMP_CODE_SINGLE; 3688 } 3689 MLX5_ASSERT(false); 3690 } 3691 3692 /** 3693 * Tx burst function for single-segment packets with TSO. 3694 * Supports all types of Tx offloads, except multi-packets. 3695 * Uses MLX5_OPCODE_TSO to build WQEs, sends one packet per WQE. 3696 * Function stops sending if it encounters the multi-segment 3697 * packet or packet without TSO requested. 3698 * 3699 * The routine is responsible for storing processed mbuf 3700 * into elts ring buffer and update elts_head if inline 3701 * offloads is requested due to possible early freeing 3702 * of the inlined mbufs (can not store pkts array in elts 3703 * as a batch). 3704 * 3705 * @param txq 3706 * Pointer to TX queue structure. 3707 * @param[in] pkts 3708 * Packets to transmit. 3709 * @param pkts_n 3710 * Number of packets in array. 3711 * @param loc 3712 * Pointer to burst routine local context. 3713 * @param olx 3714 * Configured Tx offloads mask. It is fully defined at 3715 * compile time and may be used for optimization. 3716 * 3717 * @return 3718 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3719 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3720 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered. 3721 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 3722 * Local context variables updated. 3723 */ 3724 static __rte_always_inline enum mlx5_txcmp_code 3725 mlx5_tx_burst_tso(struct mlx5_txq_data *__rte_restrict txq, 3726 struct rte_mbuf **__rte_restrict pkts, 3727 unsigned int pkts_n, 3728 struct mlx5_txq_local *__rte_restrict loc, 3729 unsigned int olx) 3730 { 3731 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 3732 MLX5_ASSERT(pkts_n > loc->pkts_sent); 3733 pkts += loc->pkts_sent + 1; 3734 pkts_n -= loc->pkts_sent; 3735 for (;;) { 3736 struct mlx5_wqe_dseg *__rte_restrict dseg; 3737 struct mlx5_wqe *__rte_restrict wqe; 3738 unsigned int ds, dlen, hlen, ntcp, vlan = 0; 3739 uint8_t *dptr; 3740 3741 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 3742 if (MLX5_TXOFF_CONFIG(TXPP)) { 3743 enum mlx5_txcmp_code wret; 3744 3745 /* Generate WAIT for scheduling if requested. */ 3746 wret = mlx5_tx_schedule_send(txq, loc, olx); 3747 if (wret == MLX5_TXCMP_CODE_EXIT) 3748 return MLX5_TXCMP_CODE_EXIT; 3749 if (wret == MLX5_TXCMP_CODE_ERROR) 3750 return MLX5_TXCMP_CODE_ERROR; 3751 } 3752 dlen = rte_pktmbuf_data_len(loc->mbuf); 3753 if (MLX5_TXOFF_CONFIG(VLAN) && 3754 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 3755 vlan = sizeof(struct rte_vlan_hdr); 3756 } 3757 /* 3758 * First calculate the WQE size to check 3759 * whether we have enough space in ring buffer. 3760 */ 3761 hlen = loc->mbuf->l2_len + vlan + 3762 loc->mbuf->l3_len + loc->mbuf->l4_len; 3763 if (unlikely((!hlen || !loc->mbuf->tso_segsz))) 3764 return MLX5_TXCMP_CODE_ERROR; 3765 if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK) 3766 hlen += loc->mbuf->outer_l2_len + 3767 loc->mbuf->outer_l3_len; 3768 /* Segment must contain all TSO headers. */ 3769 if (unlikely(hlen > MLX5_MAX_TSO_HEADER || 3770 hlen <= MLX5_ESEG_MIN_INLINE_SIZE || 3771 hlen > (dlen + vlan))) 3772 return MLX5_TXCMP_CODE_ERROR; 3773 /* 3774 * Check whether there are enough free WQEBBs: 3775 * - Control Segment 3776 * - Ethernet Segment 3777 * - First Segment of inlined Ethernet data 3778 * - ... data continued ... 3779 * - Finishing Data Segment of pointer type 3780 */ 3781 ds = 4 + (hlen - MLX5_ESEG_MIN_INLINE_SIZE + 3782 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3783 if (loc->wqe_free < ((ds + 3) / 4)) 3784 return MLX5_TXCMP_CODE_EXIT; 3785 #ifdef MLX5_PMD_SOFT_COUNTERS 3786 /* Update sent data bytes/packets counters. */ 3787 ntcp = (dlen + vlan - hlen + 3788 loc->mbuf->tso_segsz - 1) / 3789 loc->mbuf->tso_segsz; 3790 /* 3791 * One will be added for mbuf itself at the end 3792 * of the mlx5_tx_burst from loc->pkts_sent field. 3793 */ 3794 --ntcp; 3795 txq->stats.opackets += ntcp; 3796 txq->stats.obytes += dlen + vlan + ntcp * hlen; 3797 #endif 3798 /* 3799 * Build the TSO WQE: 3800 * - Control Segment 3801 * - Ethernet Segment with hlen bytes inlined 3802 * - Data Segment of pointer type 3803 */ 3804 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3805 loc->wqe_last = wqe; 3806 mlx5_tx_cseg_init(txq, loc, wqe, ds, 3807 MLX5_OPCODE_TSO, olx); 3808 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, hlen, 1, olx); 3809 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + hlen - vlan; 3810 dlen -= hlen - vlan; 3811 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx); 3812 /* 3813 * WQE is built, update the loop parameters 3814 * and go to the next packet. 3815 */ 3816 txq->wqe_ci += (ds + 3) / 4; 3817 loc->wqe_free -= (ds + 3) / 4; 3818 if (MLX5_TXOFF_CONFIG(INLINE)) 3819 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3820 --loc->elts_free; 3821 ++loc->pkts_sent; 3822 --pkts_n; 3823 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 3824 return MLX5_TXCMP_CODE_EXIT; 3825 loc->mbuf = *pkts++; 3826 if (pkts_n > 1) 3827 rte_prefetch0(*pkts); 3828 if (MLX5_TXOFF_CONFIG(MULTI) && 3829 unlikely(NB_SEGS(loc->mbuf) > 1)) 3830 return MLX5_TXCMP_CODE_MULTI; 3831 if (likely(!(loc->mbuf->ol_flags & PKT_TX_TCP_SEG))) 3832 return MLX5_TXCMP_CODE_SINGLE; 3833 /* Continue with the next TSO packet. */ 3834 } 3835 MLX5_ASSERT(false); 3836 } 3837 3838 /** 3839 * Analyze the packet and select the best method to send. 3840 * 3841 * @param txq 3842 * Pointer to TX queue structure. 3843 * @param loc 3844 * Pointer to burst routine local context. 3845 * @param olx 3846 * Configured Tx offloads mask. It is fully defined at 3847 * compile time and may be used for optimization. 3848 * @param newp 3849 * The predefined flag whether do complete check for 3850 * multi-segment packets and TSO. 3851 * 3852 * @return 3853 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 3854 * MLX5_TXCMP_CODE_TSO - TSO required, use TSO/LSO. 3855 * MLX5_TXCMP_CODE_SINGLE - single-segment packet, use SEND. 3856 * MLX5_TXCMP_CODE_EMPW - single-segment packet, use MPW. 3857 */ 3858 static __rte_always_inline enum mlx5_txcmp_code 3859 mlx5_tx_able_to_empw(struct mlx5_txq_data *__rte_restrict txq, 3860 struct mlx5_txq_local *__rte_restrict loc, 3861 unsigned int olx, 3862 bool newp) 3863 { 3864 /* Check for multi-segment packet. */ 3865 if (newp && 3866 MLX5_TXOFF_CONFIG(MULTI) && 3867 unlikely(NB_SEGS(loc->mbuf) > 1)) 3868 return MLX5_TXCMP_CODE_MULTI; 3869 /* Check for TSO packet. */ 3870 if (newp && 3871 MLX5_TXOFF_CONFIG(TSO) && 3872 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) 3873 return MLX5_TXCMP_CODE_TSO; 3874 /* Check if eMPW is enabled at all. */ 3875 if (!MLX5_TXOFF_CONFIG(EMPW)) 3876 return MLX5_TXCMP_CODE_SINGLE; 3877 /* Check if eMPW can be engaged. */ 3878 if (MLX5_TXOFF_CONFIG(VLAN) && 3879 unlikely(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) && 3880 (!MLX5_TXOFF_CONFIG(INLINE) || 3881 unlikely((rte_pktmbuf_data_len(loc->mbuf) + 3882 sizeof(struct rte_vlan_hdr)) > txq->inlen_empw))) { 3883 /* 3884 * eMPW does not support VLAN insertion offload, 3885 * we have to inline the entire packet but 3886 * packet is too long for inlining. 3887 */ 3888 return MLX5_TXCMP_CODE_SINGLE; 3889 } 3890 return MLX5_TXCMP_CODE_EMPW; 3891 } 3892 3893 /** 3894 * Check the next packet attributes to match with the eMPW batch ones. 3895 * In addition, for legacy MPW the packet length is checked either. 3896 * 3897 * @param txq 3898 * Pointer to TX queue structure. 3899 * @param es 3900 * Pointer to Ethernet Segment of eMPW batch. 3901 * @param loc 3902 * Pointer to burst routine local context. 3903 * @param dlen 3904 * Length of previous packet in MPW descriptor. 3905 * @param olx 3906 * Configured Tx offloads mask. It is fully defined at 3907 * compile time and may be used for optimization. 3908 * 3909 * @return 3910 * true - packet match with eMPW batch attributes. 3911 * false - no match, eMPW should be restarted. 3912 */ 3913 static __rte_always_inline bool 3914 mlx5_tx_match_empw(struct mlx5_txq_data *__rte_restrict txq, 3915 struct mlx5_wqe_eseg *__rte_restrict es, 3916 struct mlx5_txq_local *__rte_restrict loc, 3917 uint32_t dlen, 3918 unsigned int olx) 3919 { 3920 uint8_t swp_flags = 0; 3921 3922 /* Compare the checksum flags, if any. */ 3923 if (MLX5_TXOFF_CONFIG(CSUM) && 3924 txq_ol_cksum_to_cs(loc->mbuf) != es->cs_flags) 3925 return false; 3926 /* Compare the Software Parser offsets and flags. */ 3927 if (MLX5_TXOFF_CONFIG(SWP) && 3928 (es->swp_offs != txq_mbuf_to_swp(loc, &swp_flags, olx) || 3929 es->swp_flags != swp_flags)) 3930 return false; 3931 /* Fill metadata field if needed. */ 3932 if (MLX5_TXOFF_CONFIG(METADATA) && 3933 es->metadata != (loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 3934 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0)) 3935 return false; 3936 /* Legacy MPW can send packets with the same lengt only. */ 3937 if (MLX5_TXOFF_CONFIG(MPW) && 3938 dlen != rte_pktmbuf_data_len(loc->mbuf)) 3939 return false; 3940 /* There must be no VLAN packets in eMPW loop. */ 3941 if (MLX5_TXOFF_CONFIG(VLAN)) 3942 MLX5_ASSERT(!(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)); 3943 /* Check if the scheduling is requested. */ 3944 if (MLX5_TXOFF_CONFIG(TXPP) && 3945 loc->mbuf->ol_flags & txq->ts_mask) 3946 return false; 3947 return true; 3948 } 3949 3950 /* 3951 * Update send loop variables and WQE for eMPW loop 3952 * without data inlining. Number of Data Segments is 3953 * equal to the number of sent packets. 3954 * 3955 * @param txq 3956 * Pointer to TX queue structure. 3957 * @param loc 3958 * Pointer to burst routine local context. 3959 * @param ds 3960 * Number of packets/Data Segments/Packets. 3961 * @param slen 3962 * Accumulated statistics, bytes sent 3963 * @param olx 3964 * Configured Tx offloads mask. It is fully defined at 3965 * compile time and may be used for optimization. 3966 * 3967 * @return 3968 * true - packet match with eMPW batch attributes. 3969 * false - no match, eMPW should be restarted. 3970 */ 3971 static __rte_always_inline void 3972 mlx5_tx_sdone_empw(struct mlx5_txq_data *__rte_restrict txq, 3973 struct mlx5_txq_local *__rte_restrict loc, 3974 unsigned int ds, 3975 unsigned int slen, 3976 unsigned int olx __rte_unused) 3977 { 3978 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 3979 #ifdef MLX5_PMD_SOFT_COUNTERS 3980 /* Update sent data bytes counter. */ 3981 txq->stats.obytes += slen; 3982 #else 3983 (void)slen; 3984 #endif 3985 loc->elts_free -= ds; 3986 loc->pkts_sent += ds; 3987 ds += 2; 3988 loc->wqe_last->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3989 txq->wqe_ci += (ds + 3) / 4; 3990 loc->wqe_free -= (ds + 3) / 4; 3991 } 3992 3993 /* 3994 * Update send loop variables and WQE for eMPW loop 3995 * with data inlining. Gets the size of pushed descriptors 3996 * and data to the WQE. 3997 * 3998 * @param txq 3999 * Pointer to TX queue structure. 4000 * @param loc 4001 * Pointer to burst routine local context. 4002 * @param len 4003 * Total size of descriptor/data in bytes. 4004 * @param slen 4005 * Accumulated statistics, data bytes sent. 4006 * @param wqem 4007 * The base WQE for the eMPW/MPW descriptor. 4008 * @param olx 4009 * Configured Tx offloads mask. It is fully defined at 4010 * compile time and may be used for optimization. 4011 * 4012 * @return 4013 * true - packet match with eMPW batch attributes. 4014 * false - no match, eMPW should be restarted. 4015 */ 4016 static __rte_always_inline void 4017 mlx5_tx_idone_empw(struct mlx5_txq_data *__rte_restrict txq, 4018 struct mlx5_txq_local *__rte_restrict loc, 4019 unsigned int len, 4020 unsigned int slen, 4021 struct mlx5_wqe *__rte_restrict wqem, 4022 unsigned int olx __rte_unused) 4023 { 4024 struct mlx5_wqe_dseg *dseg = &wqem->dseg[0]; 4025 4026 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4027 #ifdef MLX5_PMD_SOFT_COUNTERS 4028 /* Update sent data bytes counter. */ 4029 txq->stats.obytes += slen; 4030 #else 4031 (void)slen; 4032 #endif 4033 if (MLX5_TXOFF_CONFIG(MPW) && dseg->bcount == RTE_BE32(0)) { 4034 /* 4035 * If the legacy MPW session contains the inline packets 4036 * we should set the only inline data segment length 4037 * and align the total length to the segment size. 4038 */ 4039 MLX5_ASSERT(len > sizeof(dseg->bcount)); 4040 dseg->bcount = rte_cpu_to_be_32((len - sizeof(dseg->bcount)) | 4041 MLX5_ETH_WQE_DATA_INLINE); 4042 len = (len + MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE + 2; 4043 } else { 4044 /* 4045 * The session is not legacy MPW or contains the 4046 * data buffer pointer segments. 4047 */ 4048 MLX5_ASSERT((len % MLX5_WSEG_SIZE) == 0); 4049 len = len / MLX5_WSEG_SIZE + 2; 4050 } 4051 wqem->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | len); 4052 txq->wqe_ci += (len + 3) / 4; 4053 loc->wqe_free -= (len + 3) / 4; 4054 loc->wqe_last = wqem; 4055 } 4056 4057 /** 4058 * The set of Tx burst functions for single-segment packets 4059 * without TSO and with Multi-Packet Writing feature support. 4060 * Supports all types of Tx offloads, except multi-packets 4061 * and TSO. 4062 * 4063 * Uses MLX5_OPCODE_EMPW to build WQEs if possible and sends 4064 * as many packet per WQE as it can. If eMPW is not configured 4065 * or packet can not be sent with eMPW (VLAN insertion) the 4066 * ordinary SEND opcode is used and only one packet placed 4067 * in WQE. 4068 * 4069 * Functions stop sending if it encounters the multi-segment 4070 * packet or packet with TSO requested. 4071 * 4072 * The routines are responsible for storing processed mbuf 4073 * into elts ring buffer and update elts_head if inlining 4074 * offload is requested. Otherwise the copying mbufs to elts 4075 * can be postponed and completed at the end of burst routine. 4076 * 4077 * @param txq 4078 * Pointer to TX queue structure. 4079 * @param[in] pkts 4080 * Packets to transmit. 4081 * @param pkts_n 4082 * Number of packets in array. 4083 * @param loc 4084 * Pointer to burst routine local context. 4085 * @param olx 4086 * Configured Tx offloads mask. It is fully defined at 4087 * compile time and may be used for optimization. 4088 * 4089 * @return 4090 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 4091 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 4092 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 4093 * MLX5_TXCMP_CODE_TSO - TSO packet encountered. 4094 * MLX5_TXCMP_CODE_SINGLE - used inside functions set. 4095 * MLX5_TXCMP_CODE_EMPW - used inside functions set. 4096 * 4097 * Local context variables updated. 4098 * 4099 * 4100 * The routine sends packets with MLX5_OPCODE_EMPW 4101 * without inlining, this is dedicated optimized branch. 4102 * No VLAN insertion is supported. 4103 */ 4104 static __rte_always_inline enum mlx5_txcmp_code 4105 mlx5_tx_burst_empw_simple(struct mlx5_txq_data *__rte_restrict txq, 4106 struct rte_mbuf **__rte_restrict pkts, 4107 unsigned int pkts_n, 4108 struct mlx5_txq_local *__rte_restrict loc, 4109 unsigned int olx) 4110 { 4111 /* 4112 * Subroutine is the part of mlx5_tx_burst_single() 4113 * and sends single-segment packet with eMPW opcode 4114 * without data inlining. 4115 */ 4116 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 4117 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW)); 4118 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4119 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4120 pkts += loc->pkts_sent + 1; 4121 pkts_n -= loc->pkts_sent; 4122 for (;;) { 4123 struct mlx5_wqe_dseg *__rte_restrict dseg; 4124 struct mlx5_wqe_eseg *__rte_restrict eseg; 4125 enum mlx5_txcmp_code ret; 4126 unsigned int part, loop; 4127 unsigned int slen = 0; 4128 4129 next_empw: 4130 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4131 if (MLX5_TXOFF_CONFIG(TXPP)) { 4132 enum mlx5_txcmp_code wret; 4133 4134 /* Generate WAIT for scheduling if requested. */ 4135 wret = mlx5_tx_schedule_send(txq, loc, olx); 4136 if (wret == MLX5_TXCMP_CODE_EXIT) 4137 return MLX5_TXCMP_CODE_EXIT; 4138 if (wret == MLX5_TXCMP_CODE_ERROR) 4139 return MLX5_TXCMP_CODE_ERROR; 4140 } 4141 part = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ? 4142 MLX5_MPW_MAX_PACKETS : 4143 MLX5_EMPW_MAX_PACKETS); 4144 if (unlikely(loc->elts_free < part)) { 4145 /* We have no enough elts to save all mbufs. */ 4146 if (unlikely(loc->elts_free < MLX5_EMPW_MIN_PACKETS)) 4147 return MLX5_TXCMP_CODE_EXIT; 4148 /* But we still able to send at least minimal eMPW. */ 4149 part = loc->elts_free; 4150 } 4151 /* Check whether we have enough WQEs */ 4152 if (unlikely(loc->wqe_free < ((2 + part + 3) / 4))) { 4153 if (unlikely(loc->wqe_free < 4154 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4))) 4155 return MLX5_TXCMP_CODE_EXIT; 4156 part = (loc->wqe_free * 4) - 2; 4157 } 4158 if (likely(part > 1)) 4159 rte_prefetch0(*pkts); 4160 loc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4161 /* 4162 * Build eMPW title WQEBB: 4163 * - Control Segment, eMPW opcode 4164 * - Ethernet Segment, no inline 4165 */ 4166 mlx5_tx_cseg_init(txq, loc, loc->wqe_last, part + 2, 4167 MLX5_OPCODE_ENHANCED_MPSW, olx); 4168 mlx5_tx_eseg_none(txq, loc, loc->wqe_last, 4169 olx & ~MLX5_TXOFF_CONFIG_VLAN); 4170 eseg = &loc->wqe_last->eseg; 4171 dseg = &loc->wqe_last->dseg[0]; 4172 loop = part; 4173 /* Store the packet length for legacy MPW. */ 4174 if (MLX5_TXOFF_CONFIG(MPW)) 4175 eseg->mss = rte_cpu_to_be_16 4176 (rte_pktmbuf_data_len(loc->mbuf)); 4177 for (;;) { 4178 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf); 4179 #ifdef MLX5_PMD_SOFT_COUNTERS 4180 /* Update sent data bytes counter. */ 4181 slen += dlen; 4182 #endif 4183 mlx5_tx_dseg_ptr 4184 (txq, loc, dseg, 4185 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 4186 dlen, olx); 4187 if (unlikely(--loop == 0)) 4188 break; 4189 loc->mbuf = *pkts++; 4190 if (likely(loop > 1)) 4191 rte_prefetch0(*pkts); 4192 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4193 /* 4194 * Unroll the completion code to avoid 4195 * returning variable value - it results in 4196 * unoptimized sequent checking in caller. 4197 */ 4198 if (ret == MLX5_TXCMP_CODE_MULTI) { 4199 part -= loop; 4200 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4201 if (unlikely(!loc->elts_free || 4202 !loc->wqe_free)) 4203 return MLX5_TXCMP_CODE_EXIT; 4204 return MLX5_TXCMP_CODE_MULTI; 4205 } 4206 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4207 if (ret == MLX5_TXCMP_CODE_TSO) { 4208 part -= loop; 4209 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4210 if (unlikely(!loc->elts_free || 4211 !loc->wqe_free)) 4212 return MLX5_TXCMP_CODE_EXIT; 4213 return MLX5_TXCMP_CODE_TSO; 4214 } 4215 if (ret == MLX5_TXCMP_CODE_SINGLE) { 4216 part -= loop; 4217 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4218 if (unlikely(!loc->elts_free || 4219 !loc->wqe_free)) 4220 return MLX5_TXCMP_CODE_EXIT; 4221 return MLX5_TXCMP_CODE_SINGLE; 4222 } 4223 if (ret != MLX5_TXCMP_CODE_EMPW) { 4224 MLX5_ASSERT(false); 4225 part -= loop; 4226 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4227 return MLX5_TXCMP_CODE_ERROR; 4228 } 4229 /* 4230 * Check whether packet parameters coincide 4231 * within assumed eMPW batch: 4232 * - check sum settings 4233 * - metadata value 4234 * - software parser settings 4235 * - packets length (legacy MPW only) 4236 * - scheduling is not required 4237 */ 4238 if (!mlx5_tx_match_empw(txq, eseg, loc, dlen, olx)) { 4239 MLX5_ASSERT(loop); 4240 part -= loop; 4241 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4242 if (unlikely(!loc->elts_free || 4243 !loc->wqe_free)) 4244 return MLX5_TXCMP_CODE_EXIT; 4245 pkts_n -= part; 4246 goto next_empw; 4247 } 4248 /* Packet attributes match, continue the same eMPW. */ 4249 ++dseg; 4250 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 4251 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 4252 } 4253 /* eMPW is built successfully, update loop parameters. */ 4254 MLX5_ASSERT(!loop); 4255 MLX5_ASSERT(pkts_n >= part); 4256 #ifdef MLX5_PMD_SOFT_COUNTERS 4257 /* Update sent data bytes counter. */ 4258 txq->stats.obytes += slen; 4259 #endif 4260 loc->elts_free -= part; 4261 loc->pkts_sent += part; 4262 txq->wqe_ci += (2 + part + 3) / 4; 4263 loc->wqe_free -= (2 + part + 3) / 4; 4264 pkts_n -= part; 4265 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 4266 return MLX5_TXCMP_CODE_EXIT; 4267 loc->mbuf = *pkts++; 4268 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4269 if (unlikely(ret != MLX5_TXCMP_CODE_EMPW)) 4270 return ret; 4271 /* Continue sending eMPW batches. */ 4272 } 4273 MLX5_ASSERT(false); 4274 } 4275 4276 /** 4277 * The routine sends packets with MLX5_OPCODE_EMPW 4278 * with inlining, optionally supports VLAN insertion. 4279 */ 4280 static __rte_always_inline enum mlx5_txcmp_code 4281 mlx5_tx_burst_empw_inline(struct mlx5_txq_data *__rte_restrict txq, 4282 struct rte_mbuf **__rte_restrict pkts, 4283 unsigned int pkts_n, 4284 struct mlx5_txq_local *__rte_restrict loc, 4285 unsigned int olx) 4286 { 4287 /* 4288 * Subroutine is the part of mlx5_tx_burst_single() 4289 * and sends single-segment packet with eMPW opcode 4290 * with data inlining. 4291 */ 4292 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4293 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW)); 4294 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4295 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4296 pkts += loc->pkts_sent + 1; 4297 pkts_n -= loc->pkts_sent; 4298 for (;;) { 4299 struct mlx5_wqe_dseg *__rte_restrict dseg; 4300 struct mlx5_wqe *__rte_restrict wqem; 4301 enum mlx5_txcmp_code ret; 4302 unsigned int room, part, nlim; 4303 unsigned int slen = 0; 4304 4305 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4306 if (MLX5_TXOFF_CONFIG(TXPP)) { 4307 enum mlx5_txcmp_code wret; 4308 4309 /* Generate WAIT for scheduling if requested. */ 4310 wret = mlx5_tx_schedule_send(txq, loc, olx); 4311 if (wret == MLX5_TXCMP_CODE_EXIT) 4312 return MLX5_TXCMP_CODE_EXIT; 4313 if (wret == MLX5_TXCMP_CODE_ERROR) 4314 return MLX5_TXCMP_CODE_ERROR; 4315 } 4316 /* 4317 * Limits the amount of packets in one WQE 4318 * to improve CQE latency generation. 4319 */ 4320 nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ? 4321 MLX5_MPW_INLINE_MAX_PACKETS : 4322 MLX5_EMPW_MAX_PACKETS); 4323 /* Check whether we have minimal amount WQEs */ 4324 if (unlikely(loc->wqe_free < 4325 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4))) 4326 return MLX5_TXCMP_CODE_EXIT; 4327 if (likely(pkts_n > 1)) 4328 rte_prefetch0(*pkts); 4329 wqem = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4330 /* 4331 * Build eMPW title WQEBB: 4332 * - Control Segment, eMPW opcode, zero DS 4333 * - Ethernet Segment, no inline 4334 */ 4335 mlx5_tx_cseg_init(txq, loc, wqem, 0, 4336 MLX5_OPCODE_ENHANCED_MPSW, olx); 4337 mlx5_tx_eseg_none(txq, loc, wqem, 4338 olx & ~MLX5_TXOFF_CONFIG_VLAN); 4339 dseg = &wqem->dseg[0]; 4340 /* Store the packet length for legacy MPW. */ 4341 if (MLX5_TXOFF_CONFIG(MPW)) 4342 wqem->eseg.mss = rte_cpu_to_be_16 4343 (rte_pktmbuf_data_len(loc->mbuf)); 4344 room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE, 4345 loc->wqe_free) * MLX5_WQE_SIZE - 4346 MLX5_WQE_CSEG_SIZE - 4347 MLX5_WQE_ESEG_SIZE; 4348 /* Limit the room for legacy MPW sessions for performance. */ 4349 if (MLX5_TXOFF_CONFIG(MPW)) 4350 room = RTE_MIN(room, 4351 RTE_MAX(txq->inlen_empw + 4352 sizeof(dseg->bcount) + 4353 (MLX5_TXOFF_CONFIG(VLAN) ? 4354 sizeof(struct rte_vlan_hdr) : 0), 4355 MLX5_MPW_INLINE_MAX_PACKETS * 4356 MLX5_WQE_DSEG_SIZE)); 4357 /* Build WQE till we have space, packets and resources. */ 4358 part = room; 4359 for (;;) { 4360 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf); 4361 uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 4362 unsigned int tlen; 4363 4364 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE); 4365 MLX5_ASSERT((room % MLX5_WQE_DSEG_SIZE) == 0); 4366 MLX5_ASSERT((uintptr_t)dseg < (uintptr_t)txq->wqes_end); 4367 /* 4368 * Some Tx offloads may cause an error if 4369 * packet is not long enough, check against 4370 * assumed minimal length. 4371 */ 4372 if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) { 4373 part -= room; 4374 if (unlikely(!part)) 4375 return MLX5_TXCMP_CODE_ERROR; 4376 /* 4377 * We have some successfully built 4378 * packet Data Segments to send. 4379 */ 4380 mlx5_tx_idone_empw(txq, loc, part, 4381 slen, wqem, olx); 4382 return MLX5_TXCMP_CODE_ERROR; 4383 } 4384 /* Inline or not inline - that's the Question. */ 4385 if (dlen > txq->inlen_empw || 4386 loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) 4387 goto pointer_empw; 4388 if (MLX5_TXOFF_CONFIG(MPW)) { 4389 if (dlen > txq->inlen_send) 4390 goto pointer_empw; 4391 tlen = dlen; 4392 if (part == room) { 4393 /* Open new inline MPW session. */ 4394 tlen += sizeof(dseg->bcount); 4395 dseg->bcount = RTE_BE32(0); 4396 dseg = RTE_PTR_ADD 4397 (dseg, sizeof(dseg->bcount)); 4398 } else { 4399 /* 4400 * No pointer and inline descriptor 4401 * intermix for legacy MPW sessions. 4402 */ 4403 if (wqem->dseg[0].bcount) 4404 break; 4405 } 4406 } else { 4407 tlen = sizeof(dseg->bcount) + dlen; 4408 } 4409 /* Inline entire packet, optional VLAN insertion. */ 4410 if (MLX5_TXOFF_CONFIG(VLAN) && 4411 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 4412 /* 4413 * The packet length must be checked in 4414 * mlx5_tx_able_to_empw() and packet 4415 * fits into inline length guaranteed. 4416 */ 4417 MLX5_ASSERT((dlen + 4418 sizeof(struct rte_vlan_hdr)) <= 4419 txq->inlen_empw); 4420 tlen += sizeof(struct rte_vlan_hdr); 4421 if (room < tlen) 4422 break; 4423 dseg = mlx5_tx_dseg_vlan(txq, loc, dseg, 4424 dptr, dlen, olx); 4425 #ifdef MLX5_PMD_SOFT_COUNTERS 4426 /* Update sent data bytes counter. */ 4427 slen += sizeof(struct rte_vlan_hdr); 4428 #endif 4429 } else { 4430 if (room < tlen) 4431 break; 4432 dseg = mlx5_tx_dseg_empw(txq, loc, dseg, 4433 dptr, dlen, olx); 4434 } 4435 if (!MLX5_TXOFF_CONFIG(MPW)) 4436 tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE); 4437 MLX5_ASSERT(room >= tlen); 4438 room -= tlen; 4439 /* 4440 * Packet data are completely inline, 4441 * we can try to free the packet. 4442 */ 4443 if (likely(loc->pkts_sent == loc->mbuf_free)) { 4444 /* 4445 * All the packets from the burst beginning 4446 * are inline, we can free mbufs directly 4447 * from the origin array on tx_burst exit(). 4448 */ 4449 loc->mbuf_free++; 4450 goto next_mbuf; 4451 } 4452 /* 4453 * In order no to call rte_pktmbuf_free_seg() here, 4454 * in the most inner loop (that might be very 4455 * expensive) we just save the mbuf in elts. 4456 */ 4457 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 4458 loc->elts_free--; 4459 goto next_mbuf; 4460 pointer_empw: 4461 /* 4462 * No pointer and inline descriptor 4463 * intermix for legacy MPW sessions. 4464 */ 4465 if (MLX5_TXOFF_CONFIG(MPW) && 4466 part != room && 4467 wqem->dseg[0].bcount == RTE_BE32(0)) 4468 break; 4469 /* 4470 * Not inlinable VLAN packets are 4471 * proceeded outside of this routine. 4472 */ 4473 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE); 4474 if (MLX5_TXOFF_CONFIG(VLAN)) 4475 MLX5_ASSERT(!(loc->mbuf->ol_flags & 4476 PKT_TX_VLAN_PKT)); 4477 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx); 4478 /* We have to store mbuf in elts.*/ 4479 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 4480 loc->elts_free--; 4481 room -= MLX5_WQE_DSEG_SIZE; 4482 /* Ring buffer wraparound is checked at the loop end.*/ 4483 ++dseg; 4484 next_mbuf: 4485 #ifdef MLX5_PMD_SOFT_COUNTERS 4486 /* Update sent data bytes counter. */ 4487 slen += dlen; 4488 #endif 4489 loc->pkts_sent++; 4490 pkts_n--; 4491 if (unlikely(!pkts_n || !loc->elts_free)) { 4492 /* 4493 * We have no resources/packets to 4494 * continue build descriptors. 4495 */ 4496 part -= room; 4497 mlx5_tx_idone_empw(txq, loc, part, 4498 slen, wqem, olx); 4499 return MLX5_TXCMP_CODE_EXIT; 4500 } 4501 loc->mbuf = *pkts++; 4502 if (likely(pkts_n > 1)) 4503 rte_prefetch0(*pkts); 4504 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4505 /* 4506 * Unroll the completion code to avoid 4507 * returning variable value - it results in 4508 * unoptimized sequent checking in caller. 4509 */ 4510 if (ret == MLX5_TXCMP_CODE_MULTI) { 4511 part -= room; 4512 mlx5_tx_idone_empw(txq, loc, part, 4513 slen, wqem, olx); 4514 if (unlikely(!loc->elts_free || 4515 !loc->wqe_free)) 4516 return MLX5_TXCMP_CODE_EXIT; 4517 return MLX5_TXCMP_CODE_MULTI; 4518 } 4519 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4520 if (ret == MLX5_TXCMP_CODE_TSO) { 4521 part -= room; 4522 mlx5_tx_idone_empw(txq, loc, part, 4523 slen, wqem, olx); 4524 if (unlikely(!loc->elts_free || 4525 !loc->wqe_free)) 4526 return MLX5_TXCMP_CODE_EXIT; 4527 return MLX5_TXCMP_CODE_TSO; 4528 } 4529 if (ret == MLX5_TXCMP_CODE_SINGLE) { 4530 part -= room; 4531 mlx5_tx_idone_empw(txq, loc, part, 4532 slen, wqem, olx); 4533 if (unlikely(!loc->elts_free || 4534 !loc->wqe_free)) 4535 return MLX5_TXCMP_CODE_EXIT; 4536 return MLX5_TXCMP_CODE_SINGLE; 4537 } 4538 if (ret != MLX5_TXCMP_CODE_EMPW) { 4539 MLX5_ASSERT(false); 4540 part -= room; 4541 mlx5_tx_idone_empw(txq, loc, part, 4542 slen, wqem, olx); 4543 return MLX5_TXCMP_CODE_ERROR; 4544 } 4545 /* Check if we have minimal room left. */ 4546 nlim--; 4547 if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE)) 4548 break; 4549 /* 4550 * Check whether packet parameters coincide 4551 * within assumed eMPW batch: 4552 * - check sum settings 4553 * - metadata value 4554 * - software parser settings 4555 * - packets length (legacy MPW only) 4556 * - scheduling is not required 4557 */ 4558 if (!mlx5_tx_match_empw(txq, &wqem->eseg, 4559 loc, dlen, olx)) 4560 break; 4561 /* Packet attributes match, continue the same eMPW. */ 4562 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 4563 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 4564 } 4565 /* 4566 * We get here to close an existing eMPW 4567 * session and start the new one. 4568 */ 4569 MLX5_ASSERT(pkts_n); 4570 part -= room; 4571 if (unlikely(!part)) 4572 return MLX5_TXCMP_CODE_EXIT; 4573 mlx5_tx_idone_empw(txq, loc, part, slen, wqem, olx); 4574 if (unlikely(!loc->elts_free || 4575 !loc->wqe_free)) 4576 return MLX5_TXCMP_CODE_EXIT; 4577 /* Continue the loop with new eMPW session. */ 4578 } 4579 MLX5_ASSERT(false); 4580 } 4581 4582 /** 4583 * The routine sends packets with ordinary MLX5_OPCODE_SEND. 4584 * Data inlining and VLAN insertion are supported. 4585 */ 4586 static __rte_always_inline enum mlx5_txcmp_code 4587 mlx5_tx_burst_single_send(struct mlx5_txq_data *__rte_restrict txq, 4588 struct rte_mbuf **__rte_restrict pkts, 4589 unsigned int pkts_n, 4590 struct mlx5_txq_local *__rte_restrict loc, 4591 unsigned int olx) 4592 { 4593 /* 4594 * Subroutine is the part of mlx5_tx_burst_single() 4595 * and sends single-segment packet with SEND opcode. 4596 */ 4597 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4598 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4599 pkts += loc->pkts_sent + 1; 4600 pkts_n -= loc->pkts_sent; 4601 for (;;) { 4602 struct mlx5_wqe *__rte_restrict wqe; 4603 enum mlx5_txcmp_code ret; 4604 4605 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4606 if (MLX5_TXOFF_CONFIG(TXPP)) { 4607 enum mlx5_txcmp_code wret; 4608 4609 /* Generate WAIT for scheduling if requested. */ 4610 wret = mlx5_tx_schedule_send(txq, loc, olx); 4611 if (wret == MLX5_TXCMP_CODE_EXIT) 4612 return MLX5_TXCMP_CODE_EXIT; 4613 if (wret == MLX5_TXCMP_CODE_ERROR) 4614 return MLX5_TXCMP_CODE_ERROR; 4615 } 4616 if (MLX5_TXOFF_CONFIG(INLINE)) { 4617 unsigned int inlen, vlan = 0; 4618 4619 inlen = rte_pktmbuf_data_len(loc->mbuf); 4620 if (MLX5_TXOFF_CONFIG(VLAN) && 4621 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 4622 vlan = sizeof(struct rte_vlan_hdr); 4623 inlen += vlan; 4624 } 4625 /* 4626 * If inlining is enabled at configuration time 4627 * the limit must be not less than minimal size. 4628 * Otherwise we would do extra check for data 4629 * size to avoid crashes due to length overflow. 4630 */ 4631 MLX5_ASSERT(txq->inlen_send >= 4632 MLX5_ESEG_MIN_INLINE_SIZE); 4633 if (inlen <= txq->inlen_send) { 4634 unsigned int seg_n, wqe_n; 4635 4636 rte_prefetch0(rte_pktmbuf_mtod 4637 (loc->mbuf, uint8_t *)); 4638 /* Check against minimal length. */ 4639 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE) 4640 return MLX5_TXCMP_CODE_ERROR; 4641 if (loc->mbuf->ol_flags & 4642 PKT_TX_DYNF_NOINLINE) { 4643 /* 4644 * The hint flag not to inline packet 4645 * data is set. Check whether we can 4646 * follow the hint. 4647 */ 4648 if ((!MLX5_TXOFF_CONFIG(EMPW) && 4649 txq->inlen_mode) || 4650 (MLX5_TXOFF_CONFIG(MPW) && 4651 txq->inlen_mode)) { 4652 if (inlen <= txq->inlen_send) 4653 goto single_inline; 4654 /* 4655 * The hardware requires the 4656 * minimal inline data header. 4657 */ 4658 goto single_min_inline; 4659 } 4660 if (MLX5_TXOFF_CONFIG(VLAN) && 4661 vlan && !txq->vlan_en) { 4662 /* 4663 * We must insert VLAN tag 4664 * by software means. 4665 */ 4666 goto single_part_inline; 4667 } 4668 goto single_no_inline; 4669 } 4670 single_inline: 4671 /* 4672 * Completely inlined packet data WQE: 4673 * - Control Segment, SEND opcode 4674 * - Ethernet Segment, no VLAN insertion 4675 * - Data inlined, VLAN optionally inserted 4676 * - Alignment to MLX5_WSEG_SIZE 4677 * Have to estimate amount of WQEBBs 4678 */ 4679 seg_n = (inlen + 3 * MLX5_WSEG_SIZE - 4680 MLX5_ESEG_MIN_INLINE_SIZE + 4681 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 4682 /* Check if there are enough WQEBBs. */ 4683 wqe_n = (seg_n + 3) / 4; 4684 if (wqe_n > loc->wqe_free) 4685 return MLX5_TXCMP_CODE_EXIT; 4686 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4687 loc->wqe_last = wqe; 4688 mlx5_tx_cseg_init(txq, loc, wqe, seg_n, 4689 MLX5_OPCODE_SEND, olx); 4690 mlx5_tx_eseg_data(txq, loc, wqe, 4691 vlan, inlen, 0, olx); 4692 txq->wqe_ci += wqe_n; 4693 loc->wqe_free -= wqe_n; 4694 /* 4695 * Packet data are completely inlined, 4696 * free the packet immediately. 4697 */ 4698 rte_pktmbuf_free_seg(loc->mbuf); 4699 } else if ((!MLX5_TXOFF_CONFIG(EMPW) || 4700 MLX5_TXOFF_CONFIG(MPW)) && 4701 txq->inlen_mode) { 4702 /* 4703 * If minimal inlining is requested the eMPW 4704 * feature should be disabled due to data is 4705 * inlined into Ethernet Segment, which can 4706 * not contain inlined data for eMPW due to 4707 * segment shared for all packets. 4708 */ 4709 struct mlx5_wqe_dseg *__rte_restrict dseg; 4710 unsigned int ds; 4711 uint8_t *dptr; 4712 4713 /* 4714 * The inline-mode settings require 4715 * to inline the specified amount of 4716 * data bytes to the Ethernet Segment. 4717 * We should check the free space in 4718 * WQE ring buffer to inline partially. 4719 */ 4720 single_min_inline: 4721 MLX5_ASSERT(txq->inlen_send >= txq->inlen_mode); 4722 MLX5_ASSERT(inlen > txq->inlen_mode); 4723 MLX5_ASSERT(txq->inlen_mode >= 4724 MLX5_ESEG_MIN_INLINE_SIZE); 4725 /* 4726 * Check whether there are enough free WQEBBs: 4727 * - Control Segment 4728 * - Ethernet Segment 4729 * - First Segment of inlined Ethernet data 4730 * - ... data continued ... 4731 * - Finishing Data Segment of pointer type 4732 */ 4733 ds = (MLX5_WQE_CSEG_SIZE + 4734 MLX5_WQE_ESEG_SIZE + 4735 MLX5_WQE_DSEG_SIZE + 4736 txq->inlen_mode - 4737 MLX5_ESEG_MIN_INLINE_SIZE + 4738 MLX5_WQE_DSEG_SIZE + 4739 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 4740 if (loc->wqe_free < ((ds + 3) / 4)) 4741 return MLX5_TXCMP_CODE_EXIT; 4742 /* 4743 * Build the ordinary SEND WQE: 4744 * - Control Segment 4745 * - Ethernet Segment, inline inlen_mode bytes 4746 * - Data Segment of pointer type 4747 */ 4748 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4749 loc->wqe_last = wqe; 4750 mlx5_tx_cseg_init(txq, loc, wqe, ds, 4751 MLX5_OPCODE_SEND, olx); 4752 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, 4753 txq->inlen_mode, 4754 0, olx); 4755 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + 4756 txq->inlen_mode - vlan; 4757 inlen -= txq->inlen_mode; 4758 mlx5_tx_dseg_ptr(txq, loc, dseg, 4759 dptr, inlen, olx); 4760 /* 4761 * WQE is built, update the loop parameters 4762 * and got to the next packet. 4763 */ 4764 txq->wqe_ci += (ds + 3) / 4; 4765 loc->wqe_free -= (ds + 3) / 4; 4766 /* We have to store mbuf in elts.*/ 4767 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4768 txq->elts[txq->elts_head++ & txq->elts_m] = 4769 loc->mbuf; 4770 --loc->elts_free; 4771 } else { 4772 uint8_t *dptr; 4773 unsigned int dlen; 4774 4775 /* 4776 * Partially inlined packet data WQE, we have 4777 * some space in title WQEBB, we can fill it 4778 * with some packet data. It takes one WQEBB, 4779 * it is available, no extra space check: 4780 * - Control Segment, SEND opcode 4781 * - Ethernet Segment, no VLAN insertion 4782 * - MLX5_ESEG_MIN_INLINE_SIZE bytes of Data 4783 * - Data Segment, pointer type 4784 * 4785 * We also get here if VLAN insertion is not 4786 * supported by HW, the inline is enabled. 4787 */ 4788 single_part_inline: 4789 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4790 loc->wqe_last = wqe; 4791 mlx5_tx_cseg_init(txq, loc, wqe, 4, 4792 MLX5_OPCODE_SEND, olx); 4793 mlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx); 4794 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + 4795 MLX5_ESEG_MIN_INLINE_SIZE - vlan; 4796 /* 4797 * The length check is performed above, by 4798 * comparing with txq->inlen_send. We should 4799 * not get overflow here. 4800 */ 4801 MLX5_ASSERT(inlen > MLX5_ESEG_MIN_INLINE_SIZE); 4802 dlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE; 4803 mlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1], 4804 dptr, dlen, olx); 4805 ++txq->wqe_ci; 4806 --loc->wqe_free; 4807 /* We have to store mbuf in elts.*/ 4808 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4809 txq->elts[txq->elts_head++ & txq->elts_m] = 4810 loc->mbuf; 4811 --loc->elts_free; 4812 } 4813 #ifdef MLX5_PMD_SOFT_COUNTERS 4814 /* Update sent data bytes counter. */ 4815 txq->stats.obytes += vlan + 4816 rte_pktmbuf_data_len(loc->mbuf); 4817 #endif 4818 } else { 4819 /* 4820 * No inline at all, it means the CPU cycles saving 4821 * is prioritized at configuration, we should not 4822 * copy any packet data to WQE. 4823 * 4824 * SEND WQE, one WQEBB: 4825 * - Control Segment, SEND opcode 4826 * - Ethernet Segment, optional VLAN, no inline 4827 * - Data Segment, pointer type 4828 */ 4829 single_no_inline: 4830 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4831 loc->wqe_last = wqe; 4832 mlx5_tx_cseg_init(txq, loc, wqe, 3, 4833 MLX5_OPCODE_SEND, olx); 4834 mlx5_tx_eseg_none(txq, loc, wqe, olx); 4835 mlx5_tx_dseg_ptr 4836 (txq, loc, &wqe->dseg[0], 4837 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 4838 rte_pktmbuf_data_len(loc->mbuf), olx); 4839 ++txq->wqe_ci; 4840 --loc->wqe_free; 4841 /* 4842 * We should not store mbuf pointer in elts 4843 * if no inlining is configured, this is done 4844 * by calling routine in a batch copy. 4845 */ 4846 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 4847 --loc->elts_free; 4848 #ifdef MLX5_PMD_SOFT_COUNTERS 4849 /* Update sent data bytes counter. */ 4850 txq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf); 4851 if (MLX5_TXOFF_CONFIG(VLAN) && 4852 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 4853 txq->stats.obytes += 4854 sizeof(struct rte_vlan_hdr); 4855 #endif 4856 } 4857 ++loc->pkts_sent; 4858 --pkts_n; 4859 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 4860 return MLX5_TXCMP_CODE_EXIT; 4861 loc->mbuf = *pkts++; 4862 if (pkts_n > 1) 4863 rte_prefetch0(*pkts); 4864 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4865 if (unlikely(ret != MLX5_TXCMP_CODE_SINGLE)) 4866 return ret; 4867 } 4868 MLX5_ASSERT(false); 4869 } 4870 4871 static __rte_always_inline enum mlx5_txcmp_code 4872 mlx5_tx_burst_single(struct mlx5_txq_data *__rte_restrict txq, 4873 struct rte_mbuf **__rte_restrict pkts, 4874 unsigned int pkts_n, 4875 struct mlx5_txq_local *__rte_restrict loc, 4876 unsigned int olx) 4877 { 4878 enum mlx5_txcmp_code ret; 4879 4880 ret = mlx5_tx_able_to_empw(txq, loc, olx, false); 4881 if (ret == MLX5_TXCMP_CODE_SINGLE) 4882 goto ordinary_send; 4883 MLX5_ASSERT(ret == MLX5_TXCMP_CODE_EMPW); 4884 for (;;) { 4885 /* Optimize for inline/no inline eMPW send. */ 4886 ret = (MLX5_TXOFF_CONFIG(INLINE)) ? 4887 mlx5_tx_burst_empw_inline 4888 (txq, pkts, pkts_n, loc, olx) : 4889 mlx5_tx_burst_empw_simple 4890 (txq, pkts, pkts_n, loc, olx); 4891 if (ret != MLX5_TXCMP_CODE_SINGLE) 4892 return ret; 4893 /* The resources to send one packet should remain. */ 4894 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4895 ordinary_send: 4896 ret = mlx5_tx_burst_single_send(txq, pkts, pkts_n, loc, olx); 4897 MLX5_ASSERT(ret != MLX5_TXCMP_CODE_SINGLE); 4898 if (ret != MLX5_TXCMP_CODE_EMPW) 4899 return ret; 4900 /* The resources to send one packet should remain. */ 4901 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4902 } 4903 } 4904 4905 /** 4906 * DPDK Tx callback template. This is configured template 4907 * used to generate routines optimized for specified offload setup. 4908 * One of this generated functions is chosen at SQ configuration 4909 * time. 4910 * 4911 * @param txq 4912 * Generic pointer to TX queue structure. 4913 * @param[in] pkts 4914 * Packets to transmit. 4915 * @param pkts_n 4916 * Number of packets in array. 4917 * @param olx 4918 * Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx 4919 * values. Should be static to take compile time static configuration 4920 * advantages. 4921 * 4922 * @return 4923 * Number of packets successfully transmitted (<= pkts_n). 4924 */ 4925 static __rte_always_inline uint16_t 4926 mlx5_tx_burst_tmpl(struct mlx5_txq_data *__rte_restrict txq, 4927 struct rte_mbuf **__rte_restrict pkts, 4928 uint16_t pkts_n, 4929 unsigned int olx) 4930 { 4931 struct mlx5_txq_local loc; 4932 enum mlx5_txcmp_code ret; 4933 unsigned int part; 4934 4935 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 4936 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 4937 if (unlikely(!pkts_n)) 4938 return 0; 4939 if (MLX5_TXOFF_CONFIG(INLINE)) 4940 loc.mbuf_free = 0; 4941 loc.pkts_sent = 0; 4942 loc.pkts_copy = 0; 4943 loc.wqe_last = NULL; 4944 4945 send_loop: 4946 loc.pkts_loop = loc.pkts_sent; 4947 /* 4948 * Check if there are some CQEs, if any: 4949 * - process an encountered errors 4950 * - process the completed WQEs 4951 * - free related mbufs 4952 * - doorbell the NIC about processed CQEs 4953 */ 4954 rte_prefetch0(*(pkts + loc.pkts_sent)); 4955 mlx5_tx_handle_completion(txq, olx); 4956 /* 4957 * Calculate the number of available resources - elts and WQEs. 4958 * There are two possible different scenarios: 4959 * - no data inlining into WQEs, one WQEBB may contains up to 4960 * four packets, in this case elts become scarce resource 4961 * - data inlining into WQEs, one packet may require multiple 4962 * WQEBBs, the WQEs become the limiting factor. 4963 */ 4964 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 4965 loc.elts_free = txq->elts_s - 4966 (uint16_t)(txq->elts_head - txq->elts_tail); 4967 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 4968 loc.wqe_free = txq->wqe_s - 4969 (uint16_t)(txq->wqe_ci - txq->wqe_pi); 4970 if (unlikely(!loc.elts_free || !loc.wqe_free)) 4971 goto burst_exit; 4972 for (;;) { 4973 /* 4974 * Fetch the packet from array. Usually this is 4975 * the first packet in series of multi/single 4976 * segment packets. 4977 */ 4978 loc.mbuf = *(pkts + loc.pkts_sent); 4979 /* Dedicated branch for multi-segment packets. */ 4980 if (MLX5_TXOFF_CONFIG(MULTI) && 4981 unlikely(NB_SEGS(loc.mbuf) > 1)) { 4982 /* 4983 * Multi-segment packet encountered. 4984 * Hardware is able to process it only 4985 * with SEND/TSO opcodes, one packet 4986 * per WQE, do it in dedicated routine. 4987 */ 4988 enter_send_multi: 4989 MLX5_ASSERT(loc.pkts_sent >= loc.pkts_copy); 4990 part = loc.pkts_sent - loc.pkts_copy; 4991 if (!MLX5_TXOFF_CONFIG(INLINE) && part) { 4992 /* 4993 * There are some single-segment mbufs not 4994 * stored in elts. The mbufs must be in the 4995 * same order as WQEs, so we must copy the 4996 * mbufs to elts here, before the coming 4997 * multi-segment packet mbufs is appended. 4998 */ 4999 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, 5000 part, olx); 5001 loc.pkts_copy = loc.pkts_sent; 5002 } 5003 MLX5_ASSERT(pkts_n > loc.pkts_sent); 5004 ret = mlx5_tx_burst_mseg(txq, pkts, pkts_n, &loc, olx); 5005 if (!MLX5_TXOFF_CONFIG(INLINE)) 5006 loc.pkts_copy = loc.pkts_sent; 5007 /* 5008 * These returned code checks are supposed 5009 * to be optimized out due to routine inlining. 5010 */ 5011 if (ret == MLX5_TXCMP_CODE_EXIT) { 5012 /* 5013 * The routine returns this code when 5014 * all packets are sent or there is no 5015 * enough resources to complete request. 5016 */ 5017 break; 5018 } 5019 if (ret == MLX5_TXCMP_CODE_ERROR) { 5020 /* 5021 * The routine returns this code when 5022 * some error in the incoming packets 5023 * format occurred. 5024 */ 5025 txq->stats.oerrors++; 5026 break; 5027 } 5028 if (ret == MLX5_TXCMP_CODE_SINGLE) { 5029 /* 5030 * The single-segment packet was encountered 5031 * in the array, try to send it with the 5032 * best optimized way, possible engaging eMPW. 5033 */ 5034 goto enter_send_single; 5035 } 5036 if (MLX5_TXOFF_CONFIG(TSO) && 5037 ret == MLX5_TXCMP_CODE_TSO) { 5038 /* 5039 * The single-segment TSO packet was 5040 * encountered in the array. 5041 */ 5042 goto enter_send_tso; 5043 } 5044 /* We must not get here. Something is going wrong. */ 5045 MLX5_ASSERT(false); 5046 txq->stats.oerrors++; 5047 break; 5048 } 5049 /* Dedicated branch for single-segment TSO packets. */ 5050 if (MLX5_TXOFF_CONFIG(TSO) && 5051 unlikely(loc.mbuf->ol_flags & PKT_TX_TCP_SEG)) { 5052 /* 5053 * TSO might require special way for inlining 5054 * (dedicated parameters) and is sent with 5055 * MLX5_OPCODE_TSO opcode only, provide this 5056 * in dedicated branch. 5057 */ 5058 enter_send_tso: 5059 MLX5_ASSERT(NB_SEGS(loc.mbuf) == 1); 5060 MLX5_ASSERT(pkts_n > loc.pkts_sent); 5061 ret = mlx5_tx_burst_tso(txq, pkts, pkts_n, &loc, olx); 5062 /* 5063 * These returned code checks are supposed 5064 * to be optimized out due to routine inlining. 5065 */ 5066 if (ret == MLX5_TXCMP_CODE_EXIT) 5067 break; 5068 if (ret == MLX5_TXCMP_CODE_ERROR) { 5069 txq->stats.oerrors++; 5070 break; 5071 } 5072 if (ret == MLX5_TXCMP_CODE_SINGLE) 5073 goto enter_send_single; 5074 if (MLX5_TXOFF_CONFIG(MULTI) && 5075 ret == MLX5_TXCMP_CODE_MULTI) { 5076 /* 5077 * The multi-segment packet was 5078 * encountered in the array. 5079 */ 5080 goto enter_send_multi; 5081 } 5082 /* We must not get here. Something is going wrong. */ 5083 MLX5_ASSERT(false); 5084 txq->stats.oerrors++; 5085 break; 5086 } 5087 /* 5088 * The dedicated branch for the single-segment packets 5089 * without TSO. Often these ones can be sent using 5090 * MLX5_OPCODE_EMPW with multiple packets in one WQE. 5091 * The routine builds the WQEs till it encounters 5092 * the TSO or multi-segment packet (in case if these 5093 * offloads are requested at SQ configuration time). 5094 */ 5095 enter_send_single: 5096 MLX5_ASSERT(pkts_n > loc.pkts_sent); 5097 ret = mlx5_tx_burst_single(txq, pkts, pkts_n, &loc, olx); 5098 /* 5099 * These returned code checks are supposed 5100 * to be optimized out due to routine inlining. 5101 */ 5102 if (ret == MLX5_TXCMP_CODE_EXIT) 5103 break; 5104 if (ret == MLX5_TXCMP_CODE_ERROR) { 5105 txq->stats.oerrors++; 5106 break; 5107 } 5108 if (MLX5_TXOFF_CONFIG(MULTI) && 5109 ret == MLX5_TXCMP_CODE_MULTI) { 5110 /* 5111 * The multi-segment packet was 5112 * encountered in the array. 5113 */ 5114 goto enter_send_multi; 5115 } 5116 if (MLX5_TXOFF_CONFIG(TSO) && 5117 ret == MLX5_TXCMP_CODE_TSO) { 5118 /* 5119 * The single-segment TSO packet was 5120 * encountered in the array. 5121 */ 5122 goto enter_send_tso; 5123 } 5124 /* We must not get here. Something is going wrong. */ 5125 MLX5_ASSERT(false); 5126 txq->stats.oerrors++; 5127 break; 5128 } 5129 /* 5130 * Main Tx loop is completed, do the rest: 5131 * - set completion request if thresholds are reached 5132 * - doorbell the hardware 5133 * - copy the rest of mbufs to elts (if any) 5134 */ 5135 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE) || 5136 loc.pkts_sent >= loc.pkts_copy); 5137 /* Take a shortcut if nothing is sent. */ 5138 if (unlikely(loc.pkts_sent == loc.pkts_loop)) 5139 goto burst_exit; 5140 /* Request CQE generation if limits are reached. */ 5141 mlx5_tx_request_completion(txq, &loc, olx); 5142 /* 5143 * Ring QP doorbell immediately after WQE building completion 5144 * to improve latencies. The pure software related data treatment 5145 * can be completed after doorbell. Tx CQEs for this SQ are 5146 * processed in this thread only by the polling. 5147 * 5148 * The rdma core library can map doorbell register in two ways, 5149 * depending on the environment variable "MLX5_SHUT_UP_BF": 5150 * 5151 * - as regular cached memory, the variable is either missing or 5152 * set to zero. This type of mapping may cause the significant 5153 * doorbell register writing latency and requires explicit 5154 * memory write barrier to mitigate this issue and prevent 5155 * write combining. 5156 * 5157 * - as non-cached memory, the variable is present and set to 5158 * not "0" value. This type of mapping may cause performance 5159 * impact under heavy loading conditions but the explicit write 5160 * memory barrier is not required and it may improve core 5161 * performance. 5162 * 5163 * - the legacy behaviour (prior 19.08 release) was to use some 5164 * heuristics to decide whether write memory barrier should 5165 * be performed. This behavior is supported with specifying 5166 * tx_db_nc=2, write barrier is skipped if application 5167 * provides the full recommended burst of packets, it 5168 * supposes the next packets are coming and the write barrier 5169 * will be issued on the next burst (after descriptor writing, 5170 * at least). 5171 */ 5172 mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, !txq->db_nc && 5173 (!txq->db_heu || pkts_n % MLX5_TX_DEFAULT_BURST)); 5174 /* Not all of the mbufs may be stored into elts yet. */ 5175 part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy; 5176 if (!MLX5_TXOFF_CONFIG(INLINE) && part) { 5177 /* 5178 * There are some single-segment mbufs not stored in elts. 5179 * It can be only if the last packet was single-segment. 5180 * The copying is gathered into one place due to it is 5181 * a good opportunity to optimize that with SIMD. 5182 * Unfortunately if inlining is enabled the gaps in 5183 * pointer array may happen due to early freeing of the 5184 * inlined mbufs. 5185 */ 5186 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, part, olx); 5187 loc.pkts_copy = loc.pkts_sent; 5188 } 5189 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 5190 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 5191 if (pkts_n > loc.pkts_sent) { 5192 /* 5193 * If burst size is large there might be no enough CQE 5194 * fetched from completion queue and no enough resources 5195 * freed to send all the packets. 5196 */ 5197 goto send_loop; 5198 } 5199 burst_exit: 5200 #ifdef MLX5_PMD_SOFT_COUNTERS 5201 /* Increment sent packets counter. */ 5202 txq->stats.opackets += loc.pkts_sent; 5203 #endif 5204 if (MLX5_TXOFF_CONFIG(INLINE) && loc.mbuf_free) 5205 __mlx5_tx_free_mbuf(txq, pkts, loc.mbuf_free, olx); 5206 return loc.pkts_sent; 5207 } 5208 5209 /* Generate routines with Enhanced Multi-Packet Write support. */ 5210 MLX5_TXOFF_DECL(full_empw, 5211 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_EMPW) 5212 5213 MLX5_TXOFF_DECL(none_empw, 5214 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW) 5215 5216 MLX5_TXOFF_DECL(md_empw, 5217 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5218 5219 MLX5_TXOFF_DECL(mt_empw, 5220 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5221 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5222 5223 MLX5_TXOFF_DECL(mtsc_empw, 5224 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5225 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5226 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5227 5228 MLX5_TXOFF_DECL(mti_empw, 5229 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5230 MLX5_TXOFF_CONFIG_INLINE | 5231 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5232 5233 MLX5_TXOFF_DECL(mtv_empw, 5234 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5235 MLX5_TXOFF_CONFIG_VLAN | 5236 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5237 5238 MLX5_TXOFF_DECL(mtiv_empw, 5239 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5240 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5241 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5242 5243 MLX5_TXOFF_DECL(sc_empw, 5244 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5245 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5246 5247 MLX5_TXOFF_DECL(sci_empw, 5248 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5249 MLX5_TXOFF_CONFIG_INLINE | 5250 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5251 5252 MLX5_TXOFF_DECL(scv_empw, 5253 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5254 MLX5_TXOFF_CONFIG_VLAN | 5255 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5256 5257 MLX5_TXOFF_DECL(sciv_empw, 5258 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5259 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5260 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5261 5262 MLX5_TXOFF_DECL(i_empw, 5263 MLX5_TXOFF_CONFIG_INLINE | 5264 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5265 5266 MLX5_TXOFF_DECL(v_empw, 5267 MLX5_TXOFF_CONFIG_VLAN | 5268 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5269 5270 MLX5_TXOFF_DECL(iv_empw, 5271 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5272 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5273 5274 /* Generate routines without Enhanced Multi-Packet Write support. */ 5275 MLX5_TXOFF_DECL(full, 5276 MLX5_TXOFF_CONFIG_FULL) 5277 5278 MLX5_TXOFF_DECL(none, 5279 MLX5_TXOFF_CONFIG_NONE) 5280 5281 MLX5_TXOFF_DECL(md, 5282 MLX5_TXOFF_CONFIG_METADATA) 5283 5284 MLX5_TXOFF_DECL(mt, 5285 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5286 MLX5_TXOFF_CONFIG_METADATA) 5287 5288 MLX5_TXOFF_DECL(mtsc, 5289 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5290 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5291 MLX5_TXOFF_CONFIG_METADATA) 5292 5293 MLX5_TXOFF_DECL(mti, 5294 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5295 MLX5_TXOFF_CONFIG_INLINE | 5296 MLX5_TXOFF_CONFIG_METADATA) 5297 5298 5299 MLX5_TXOFF_DECL(mtv, 5300 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5301 MLX5_TXOFF_CONFIG_VLAN | 5302 MLX5_TXOFF_CONFIG_METADATA) 5303 5304 5305 MLX5_TXOFF_DECL(mtiv, 5306 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5307 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5308 MLX5_TXOFF_CONFIG_METADATA) 5309 5310 MLX5_TXOFF_DECL(sc, 5311 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5312 MLX5_TXOFF_CONFIG_METADATA) 5313 5314 MLX5_TXOFF_DECL(sci, 5315 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5316 MLX5_TXOFF_CONFIG_INLINE | 5317 MLX5_TXOFF_CONFIG_METADATA) 5318 5319 5320 MLX5_TXOFF_DECL(scv, 5321 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5322 MLX5_TXOFF_CONFIG_VLAN | 5323 MLX5_TXOFF_CONFIG_METADATA) 5324 5325 5326 MLX5_TXOFF_DECL(sciv, 5327 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5328 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5329 MLX5_TXOFF_CONFIG_METADATA) 5330 5331 MLX5_TXOFF_DECL(i, 5332 MLX5_TXOFF_CONFIG_INLINE | 5333 MLX5_TXOFF_CONFIG_METADATA) 5334 5335 MLX5_TXOFF_DECL(v, 5336 MLX5_TXOFF_CONFIG_VLAN | 5337 MLX5_TXOFF_CONFIG_METADATA) 5338 5339 MLX5_TXOFF_DECL(iv, 5340 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5341 MLX5_TXOFF_CONFIG_METADATA) 5342 5343 /* Generate routines with timestamp scheduling. */ 5344 MLX5_TXOFF_DECL(full_ts_nompw, 5345 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP) 5346 5347 MLX5_TXOFF_DECL(full_ts_nompwi, 5348 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5349 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5350 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 5351 MLX5_TXOFF_CONFIG_TXPP) 5352 5353 MLX5_TXOFF_DECL(full_ts, 5354 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP | 5355 MLX5_TXOFF_CONFIG_EMPW) 5356 5357 MLX5_TXOFF_DECL(full_ts_noi, 5358 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5359 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5360 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 5361 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5362 5363 MLX5_TXOFF_DECL(none_ts, 5364 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP | 5365 MLX5_TXOFF_CONFIG_EMPW) 5366 5367 MLX5_TXOFF_DECL(mdi_ts, 5368 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 5369 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5370 5371 MLX5_TXOFF_DECL(mti_ts, 5372 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5373 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 5374 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5375 5376 MLX5_TXOFF_DECL(mtiv_ts, 5377 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5378 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5379 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP | 5380 MLX5_TXOFF_CONFIG_EMPW) 5381 5382 /* 5383 * Generate routines with Legacy Multi-Packet Write support. 5384 * This mode is supported by ConnectX-4 Lx only and imposes 5385 * offload limitations, not supported: 5386 * - ACL/Flows (metadata are becoming meaningless) 5387 * - WQE Inline headers 5388 * - SRIOV (E-Switch offloads) 5389 * - VLAN insertion 5390 * - tunnel encapsulation/decapsulation 5391 * - TSO 5392 */ 5393 MLX5_TXOFF_DECL(none_mpw, 5394 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW | 5395 MLX5_TXOFF_CONFIG_MPW) 5396 5397 MLX5_TXOFF_DECL(mci_mpw, 5398 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5399 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5400 MLX5_TXOFF_CONFIG_MPW) 5401 5402 MLX5_TXOFF_DECL(mc_mpw, 5403 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5404 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW) 5405 5406 MLX5_TXOFF_DECL(i_mpw, 5407 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5408 MLX5_TXOFF_CONFIG_MPW) 5409 5410 /* 5411 * Array of declared and compiled Tx burst function and corresponding 5412 * supported offloads set. The array is used to select the Tx burst 5413 * function for specified offloads set at Tx queue configuration time. 5414 */ 5415 const struct { 5416 eth_tx_burst_t func; 5417 unsigned int olx; 5418 } txoff_func[] = { 5419 MLX5_TXOFF_INFO(full_empw, 5420 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5421 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5422 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5423 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5424 5425 MLX5_TXOFF_INFO(none_empw, 5426 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW) 5427 5428 MLX5_TXOFF_INFO(md_empw, 5429 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5430 5431 MLX5_TXOFF_INFO(mt_empw, 5432 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5433 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5434 5435 MLX5_TXOFF_INFO(mtsc_empw, 5436 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5437 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5438 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5439 5440 MLX5_TXOFF_INFO(mti_empw, 5441 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5442 MLX5_TXOFF_CONFIG_INLINE | 5443 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5444 5445 MLX5_TXOFF_INFO(mtv_empw, 5446 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5447 MLX5_TXOFF_CONFIG_VLAN | 5448 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5449 5450 MLX5_TXOFF_INFO(mtiv_empw, 5451 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5452 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5453 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5454 5455 MLX5_TXOFF_INFO(sc_empw, 5456 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5457 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5458 5459 MLX5_TXOFF_INFO(sci_empw, 5460 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5461 MLX5_TXOFF_CONFIG_INLINE | 5462 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5463 5464 MLX5_TXOFF_INFO(scv_empw, 5465 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5466 MLX5_TXOFF_CONFIG_VLAN | 5467 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5468 5469 MLX5_TXOFF_INFO(sciv_empw, 5470 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5471 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5472 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5473 5474 MLX5_TXOFF_INFO(i_empw, 5475 MLX5_TXOFF_CONFIG_INLINE | 5476 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5477 5478 MLX5_TXOFF_INFO(v_empw, 5479 MLX5_TXOFF_CONFIG_VLAN | 5480 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5481 5482 MLX5_TXOFF_INFO(iv_empw, 5483 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5484 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5485 5486 MLX5_TXOFF_INFO(full_ts_nompw, 5487 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP) 5488 5489 MLX5_TXOFF_INFO(full_ts_nompwi, 5490 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5491 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5492 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 5493 MLX5_TXOFF_CONFIG_TXPP) 5494 5495 MLX5_TXOFF_INFO(full_ts, 5496 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP | 5497 MLX5_TXOFF_CONFIG_EMPW) 5498 5499 MLX5_TXOFF_INFO(full_ts_noi, 5500 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5501 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5502 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 5503 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5504 5505 MLX5_TXOFF_INFO(none_ts, 5506 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP | 5507 MLX5_TXOFF_CONFIG_EMPW) 5508 5509 MLX5_TXOFF_INFO(mdi_ts, 5510 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 5511 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5512 5513 MLX5_TXOFF_INFO(mti_ts, 5514 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5515 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 5516 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 5517 5518 MLX5_TXOFF_INFO(mtiv_ts, 5519 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5520 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5521 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP | 5522 MLX5_TXOFF_CONFIG_EMPW) 5523 5524 MLX5_TXOFF_INFO(full, 5525 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5526 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5527 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5528 MLX5_TXOFF_CONFIG_METADATA) 5529 5530 MLX5_TXOFF_INFO(none, 5531 MLX5_TXOFF_CONFIG_NONE) 5532 5533 MLX5_TXOFF_INFO(md, 5534 MLX5_TXOFF_CONFIG_METADATA) 5535 5536 MLX5_TXOFF_INFO(mt, 5537 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5538 MLX5_TXOFF_CONFIG_METADATA) 5539 5540 MLX5_TXOFF_INFO(mtsc, 5541 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5542 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5543 MLX5_TXOFF_CONFIG_METADATA) 5544 5545 MLX5_TXOFF_INFO(mti, 5546 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5547 MLX5_TXOFF_CONFIG_INLINE | 5548 MLX5_TXOFF_CONFIG_METADATA) 5549 5550 MLX5_TXOFF_INFO(mtv, 5551 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5552 MLX5_TXOFF_CONFIG_VLAN | 5553 MLX5_TXOFF_CONFIG_METADATA) 5554 5555 MLX5_TXOFF_INFO(mtiv, 5556 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5557 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5558 MLX5_TXOFF_CONFIG_METADATA) 5559 5560 MLX5_TXOFF_INFO(sc, 5561 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5562 MLX5_TXOFF_CONFIG_METADATA) 5563 5564 MLX5_TXOFF_INFO(sci, 5565 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5566 MLX5_TXOFF_CONFIG_INLINE | 5567 MLX5_TXOFF_CONFIG_METADATA) 5568 5569 MLX5_TXOFF_INFO(scv, 5570 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5571 MLX5_TXOFF_CONFIG_VLAN | 5572 MLX5_TXOFF_CONFIG_METADATA) 5573 5574 MLX5_TXOFF_INFO(sciv, 5575 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5576 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5577 MLX5_TXOFF_CONFIG_METADATA) 5578 5579 MLX5_TXOFF_INFO(i, 5580 MLX5_TXOFF_CONFIG_INLINE | 5581 MLX5_TXOFF_CONFIG_METADATA) 5582 5583 MLX5_TXOFF_INFO(v, 5584 MLX5_TXOFF_CONFIG_VLAN | 5585 MLX5_TXOFF_CONFIG_METADATA) 5586 5587 MLX5_TXOFF_INFO(iv, 5588 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5589 MLX5_TXOFF_CONFIG_METADATA) 5590 5591 MLX5_TXOFF_INFO(none_mpw, 5592 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW | 5593 MLX5_TXOFF_CONFIG_MPW) 5594 5595 MLX5_TXOFF_INFO(mci_mpw, 5596 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5597 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5598 MLX5_TXOFF_CONFIG_MPW) 5599 5600 MLX5_TXOFF_INFO(mc_mpw, 5601 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5602 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW) 5603 5604 MLX5_TXOFF_INFO(i_mpw, 5605 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5606 MLX5_TXOFF_CONFIG_MPW) 5607 }; 5608 5609 /** 5610 * Configure the Tx function to use. The routine checks configured 5611 * Tx offloads for the device and selects appropriate Tx burst 5612 * routine. There are multiple Tx burst routines compiled from 5613 * the same template in the most optimal way for the dedicated 5614 * Tx offloads set. 5615 * 5616 * @param dev 5617 * Pointer to private data structure. 5618 * 5619 * @return 5620 * Pointer to selected Tx burst function. 5621 */ 5622 eth_tx_burst_t 5623 mlx5_select_tx_function(struct rte_eth_dev *dev) 5624 { 5625 struct mlx5_priv *priv = dev->data->dev_private; 5626 struct mlx5_dev_config *config = &priv->config; 5627 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; 5628 unsigned int diff = 0, olx = 0, i, m; 5629 5630 MLX5_ASSERT(priv); 5631 if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) { 5632 /* We should support Multi-Segment Packets. */ 5633 olx |= MLX5_TXOFF_CONFIG_MULTI; 5634 } 5635 if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO | 5636 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 5637 DEV_TX_OFFLOAD_GRE_TNL_TSO | 5638 DEV_TX_OFFLOAD_IP_TNL_TSO | 5639 DEV_TX_OFFLOAD_UDP_TNL_TSO)) { 5640 /* We should support TCP Send Offload. */ 5641 olx |= MLX5_TXOFF_CONFIG_TSO; 5642 } 5643 if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO | 5644 DEV_TX_OFFLOAD_UDP_TNL_TSO | 5645 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 5646 /* We should support Software Parser for Tunnels. */ 5647 olx |= MLX5_TXOFF_CONFIG_SWP; 5648 } 5649 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 5650 DEV_TX_OFFLOAD_UDP_CKSUM | 5651 DEV_TX_OFFLOAD_TCP_CKSUM | 5652 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 5653 /* We should support IP/TCP/UDP Checksums. */ 5654 olx |= MLX5_TXOFF_CONFIG_CSUM; 5655 } 5656 if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) { 5657 /* We should support VLAN insertion. */ 5658 olx |= MLX5_TXOFF_CONFIG_VLAN; 5659 } 5660 if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP && 5661 rte_mbuf_dynflag_lookup 5662 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 && 5663 rte_mbuf_dynfield_lookup 5664 (RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) { 5665 /* Offload configured, dynamic entities registered. */ 5666 olx |= MLX5_TXOFF_CONFIG_TXPP; 5667 } 5668 if (priv->txqs_n && (*priv->txqs)[0]) { 5669 struct mlx5_txq_data *txd = (*priv->txqs)[0]; 5670 5671 if (txd->inlen_send) { 5672 /* 5673 * Check the data inline requirements. Data inline 5674 * is enabled on per device basis, we can check 5675 * the first Tx queue only. 5676 * 5677 * If device does not support VLAN insertion in WQE 5678 * and some queues are requested to perform VLAN 5679 * insertion offload than inline must be enabled. 5680 */ 5681 olx |= MLX5_TXOFF_CONFIG_INLINE; 5682 } 5683 } 5684 if (config->mps == MLX5_MPW_ENHANCED && 5685 config->txq_inline_min <= 0) { 5686 /* 5687 * The NIC supports Enhanced Multi-Packet Write 5688 * and does not require minimal inline data. 5689 */ 5690 olx |= MLX5_TXOFF_CONFIG_EMPW; 5691 } 5692 if (rte_flow_dynf_metadata_avail()) { 5693 /* We should support Flow metadata. */ 5694 olx |= MLX5_TXOFF_CONFIG_METADATA; 5695 } 5696 if (config->mps == MLX5_MPW) { 5697 /* 5698 * The NIC supports Legacy Multi-Packet Write. 5699 * The MLX5_TXOFF_CONFIG_MPW controls the 5700 * descriptor building method in combination 5701 * with MLX5_TXOFF_CONFIG_EMPW. 5702 */ 5703 if (!(olx & (MLX5_TXOFF_CONFIG_TSO | 5704 MLX5_TXOFF_CONFIG_SWP | 5705 MLX5_TXOFF_CONFIG_VLAN | 5706 MLX5_TXOFF_CONFIG_METADATA))) 5707 olx |= MLX5_TXOFF_CONFIG_EMPW | 5708 MLX5_TXOFF_CONFIG_MPW; 5709 } 5710 /* 5711 * Scan the routines table to find the minimal 5712 * satisfying routine with requested offloads. 5713 */ 5714 m = RTE_DIM(txoff_func); 5715 for (i = 0; i < RTE_DIM(txoff_func); i++) { 5716 unsigned int tmp; 5717 5718 tmp = txoff_func[i].olx; 5719 if (tmp == olx) { 5720 /* Meets requested offloads exactly.*/ 5721 m = i; 5722 break; 5723 } 5724 if ((tmp & olx) != olx) { 5725 /* Does not meet requested offloads at all. */ 5726 continue; 5727 } 5728 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW) 5729 /* Do not enable legacy MPW if not configured. */ 5730 continue; 5731 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW) 5732 /* Do not enable eMPW if not configured. */ 5733 continue; 5734 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE) 5735 /* Do not enable inlining if not configured. */ 5736 continue; 5737 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP) 5738 /* Do not enable scheduling if not configured. */ 5739 continue; 5740 /* 5741 * Some routine meets the requirements. 5742 * Check whether it has minimal amount 5743 * of not requested offloads. 5744 */ 5745 tmp = __builtin_popcountl(tmp & ~olx); 5746 if (m >= RTE_DIM(txoff_func) || tmp < diff) { 5747 /* First or better match, save and continue. */ 5748 m = i; 5749 diff = tmp; 5750 continue; 5751 } 5752 if (tmp == diff) { 5753 tmp = txoff_func[i].olx ^ txoff_func[m].olx; 5754 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) < 5755 __builtin_ffsl(txoff_func[m].olx & ~tmp)) { 5756 /* Lighter not requested offload. */ 5757 m = i; 5758 } 5759 } 5760 } 5761 if (m >= RTE_DIM(txoff_func)) { 5762 DRV_LOG(DEBUG, "port %u has no selected Tx function" 5763 " for requested offloads %04X", 5764 dev->data->port_id, olx); 5765 return NULL; 5766 } 5767 DRV_LOG(DEBUG, "port %u has selected Tx function" 5768 " supporting offloads %04X/%04X", 5769 dev->data->port_id, olx, txoff_func[m].olx); 5770 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI) 5771 DRV_LOG(DEBUG, "\tMULTI (multi segment)"); 5772 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO) 5773 DRV_LOG(DEBUG, "\tTSO (TCP send offload)"); 5774 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP) 5775 DRV_LOG(DEBUG, "\tSWP (software parser)"); 5776 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM) 5777 DRV_LOG(DEBUG, "\tCSUM (checksum offload)"); 5778 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE) 5779 DRV_LOG(DEBUG, "\tINLIN (inline data)"); 5780 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN) 5781 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)"); 5782 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA) 5783 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)"); 5784 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP) 5785 DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)"); 5786 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) { 5787 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW) 5788 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)"); 5789 else 5790 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)"); 5791 } 5792 return txoff_func[m].func; 5793 } 5794 5795 /** 5796 * DPDK callback to get the TX queue information 5797 * 5798 * @param dev 5799 * Pointer to the device structure. 5800 * 5801 * @param tx_queue_id 5802 * Tx queue identificator. 5803 * 5804 * @param qinfo 5805 * Pointer to the TX queue information structure. 5806 * 5807 * @return 5808 * None. 5809 */ 5810 5811 void 5812 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, 5813 struct rte_eth_txq_info *qinfo) 5814 { 5815 struct mlx5_priv *priv = dev->data->dev_private; 5816 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id]; 5817 struct mlx5_txq_ctrl *txq_ctrl = 5818 container_of(txq, struct mlx5_txq_ctrl, txq); 5819 5820 if (!txq) 5821 return; 5822 qinfo->nb_desc = txq->elts_s; 5823 qinfo->conf.tx_thresh.pthresh = 0; 5824 qinfo->conf.tx_thresh.hthresh = 0; 5825 qinfo->conf.tx_thresh.wthresh = 0; 5826 qinfo->conf.tx_rs_thresh = 0; 5827 qinfo->conf.tx_free_thresh = 0; 5828 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1; 5829 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 5830 } 5831 5832 /** 5833 * DPDK callback to get the TX packet burst mode information 5834 * 5835 * @param dev 5836 * Pointer to the device structure. 5837 * 5838 * @param tx_queue_id 5839 * Tx queue identificatior. 5840 * 5841 * @param mode 5842 * Pointer to the burts mode information. 5843 * 5844 * @return 5845 * 0 as success, -EINVAL as failure. 5846 */ 5847 5848 int 5849 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, 5850 uint16_t tx_queue_id, 5851 struct rte_eth_burst_mode *mode) 5852 { 5853 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 5854 struct mlx5_priv *priv = dev->data->dev_private; 5855 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id]; 5856 unsigned int i, olx; 5857 5858 for (i = 0; i < RTE_DIM(txoff_func); i++) { 5859 if (pkt_burst == txoff_func[i].func) { 5860 olx = txoff_func[i].olx; 5861 snprintf(mode->info, sizeof(mode->info), 5862 "%s%s%s%s%s%s%s%s%s%s", 5863 (olx & MLX5_TXOFF_CONFIG_EMPW) ? 5864 ((olx & MLX5_TXOFF_CONFIG_MPW) ? 5865 "Legacy MPW" : "Enhanced MPW") : "No MPW", 5866 (olx & MLX5_TXOFF_CONFIG_MULTI) ? 5867 " + MULTI" : "", 5868 (olx & MLX5_TXOFF_CONFIG_TSO) ? 5869 " + TSO" : "", 5870 (olx & MLX5_TXOFF_CONFIG_SWP) ? 5871 " + SWP" : "", 5872 (olx & MLX5_TXOFF_CONFIG_CSUM) ? 5873 " + CSUM" : "", 5874 (olx & MLX5_TXOFF_CONFIG_INLINE) ? 5875 " + INLINE" : "", 5876 (olx & MLX5_TXOFF_CONFIG_VLAN) ? 5877 " + VLAN" : "", 5878 (olx & MLX5_TXOFF_CONFIG_METADATA) ? 5879 " + METADATA" : "", 5880 (olx & MLX5_TXOFF_CONFIG_TXPP) ? 5881 " + TXPP" : "", 5882 (txq && txq->fast_free) ? 5883 " + Fast Free" : ""); 5884 return 0; 5885 } 5886 } 5887 return -EINVAL; 5888 } 5889