1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <unistd.h> 10 11 #include <rte_common.h> 12 #include <rte_ether.h> 13 #include <rte_ethdev_driver.h> 14 #include <rte_flow.h> 15 #include <rte_flow_driver.h> 16 #include <rte_malloc.h> 17 #include <rte_cycles.h> 18 #include <rte_ip.h> 19 #include <rte_gre.h> 20 #include <rte_vxlan.h> 21 #include <rte_gtp.h> 22 #include <rte_eal_paging.h> 23 #include <rte_mpls.h> 24 25 #include <mlx5_glue.h> 26 #include <mlx5_devx_cmds.h> 27 #include <mlx5_prm.h> 28 #include <mlx5_malloc.h> 29 30 #include "mlx5_defs.h" 31 #include "mlx5.h" 32 #include "mlx5_common_os.h" 33 #include "mlx5_flow.h" 34 #include "mlx5_flow_os.h" 35 #include "mlx5_rxtx.h" 36 37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 38 39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS 40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 41 #endif 42 43 #ifndef HAVE_MLX5DV_DR_ESWITCH 44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB 45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0 46 #endif 47 #endif 48 49 #ifndef HAVE_MLX5DV_DR 50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 51 #endif 52 53 /* VLAN header definitions */ 54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13 55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT) 56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff 57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK) 58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK) 59 60 union flow_dv_attr { 61 struct { 62 uint32_t valid:1; 63 uint32_t ipv4:1; 64 uint32_t ipv6:1; 65 uint32_t tcp:1; 66 uint32_t udp:1; 67 uint32_t reserved:27; 68 }; 69 uint32_t attr; 70 }; 71 72 static int 73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 74 struct mlx5_flow_tbl_resource *tbl); 75 76 static int 77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev); 78 79 /** 80 * Initialize flow attributes structure according to flow items' types. 81 * 82 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel 83 * mode. For tunnel mode, the items to be modified are the outermost ones. 84 * 85 * @param[in] item 86 * Pointer to item specification. 87 * @param[out] attr 88 * Pointer to flow attributes structure. 89 * @param[in] dev_flow 90 * Pointer to the sub flow. 91 * @param[in] tunnel_decap 92 * Whether action is after tunnel decapsulation. 93 */ 94 static void 95 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, 96 struct mlx5_flow *dev_flow, bool tunnel_decap) 97 { 98 uint64_t layers = dev_flow->handle->layers; 99 100 /* 101 * If layers is already initialized, it means this dev_flow is the 102 * suffix flow, the layers flags is set by the prefix flow. Need to 103 * use the layer flags from prefix flow as the suffix flow may not 104 * have the user defined items as the flow is split. 105 */ 106 if (layers) { 107 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 108 attr->ipv4 = 1; 109 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6) 110 attr->ipv6 = 1; 111 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP) 112 attr->tcp = 1; 113 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP) 114 attr->udp = 1; 115 attr->valid = 1; 116 return; 117 } 118 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 119 uint8_t next_protocol = 0xff; 120 switch (item->type) { 121 case RTE_FLOW_ITEM_TYPE_GRE: 122 case RTE_FLOW_ITEM_TYPE_NVGRE: 123 case RTE_FLOW_ITEM_TYPE_VXLAN: 124 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 125 case RTE_FLOW_ITEM_TYPE_GENEVE: 126 case RTE_FLOW_ITEM_TYPE_MPLS: 127 if (tunnel_decap) 128 attr->attr = 0; 129 break; 130 case RTE_FLOW_ITEM_TYPE_IPV4: 131 if (!attr->ipv6) 132 attr->ipv4 = 1; 133 if (item->mask != NULL && 134 ((const struct rte_flow_item_ipv4 *) 135 item->mask)->hdr.next_proto_id) 136 next_protocol = 137 ((const struct rte_flow_item_ipv4 *) 138 (item->spec))->hdr.next_proto_id & 139 ((const struct rte_flow_item_ipv4 *) 140 (item->mask))->hdr.next_proto_id; 141 if ((next_protocol == IPPROTO_IPIP || 142 next_protocol == IPPROTO_IPV6) && tunnel_decap) 143 attr->attr = 0; 144 break; 145 case RTE_FLOW_ITEM_TYPE_IPV6: 146 if (!attr->ipv4) 147 attr->ipv6 = 1; 148 if (item->mask != NULL && 149 ((const struct rte_flow_item_ipv6 *) 150 item->mask)->hdr.proto) 151 next_protocol = 152 ((const struct rte_flow_item_ipv6 *) 153 (item->spec))->hdr.proto & 154 ((const struct rte_flow_item_ipv6 *) 155 (item->mask))->hdr.proto; 156 if ((next_protocol == IPPROTO_IPIP || 157 next_protocol == IPPROTO_IPV6) && tunnel_decap) 158 attr->attr = 0; 159 break; 160 case RTE_FLOW_ITEM_TYPE_UDP: 161 if (!attr->tcp) 162 attr->udp = 1; 163 break; 164 case RTE_FLOW_ITEM_TYPE_TCP: 165 if (!attr->udp) 166 attr->tcp = 1; 167 break; 168 default: 169 break; 170 } 171 } 172 attr->valid = 1; 173 } 174 175 /** 176 * Convert rte_mtr_color to mlx5 color. 177 * 178 * @param[in] rcol 179 * rte_mtr_color. 180 * 181 * @return 182 * mlx5 color. 183 */ 184 static int 185 rte_col_2_mlx5_col(enum rte_color rcol) 186 { 187 switch (rcol) { 188 case RTE_COLOR_GREEN: 189 return MLX5_FLOW_COLOR_GREEN; 190 case RTE_COLOR_YELLOW: 191 return MLX5_FLOW_COLOR_YELLOW; 192 case RTE_COLOR_RED: 193 return MLX5_FLOW_COLOR_RED; 194 default: 195 break; 196 } 197 return MLX5_FLOW_COLOR_UNDEFINED; 198 } 199 200 struct field_modify_info { 201 uint32_t size; /* Size of field in protocol header, in bytes. */ 202 uint32_t offset; /* Offset of field in protocol header, in bytes. */ 203 enum mlx5_modification_field id; 204 }; 205 206 struct field_modify_info modify_eth[] = { 207 {4, 0, MLX5_MODI_OUT_DMAC_47_16}, 208 {2, 4, MLX5_MODI_OUT_DMAC_15_0}, 209 {4, 6, MLX5_MODI_OUT_SMAC_47_16}, 210 {2, 10, MLX5_MODI_OUT_SMAC_15_0}, 211 {0, 0, 0}, 212 }; 213 214 struct field_modify_info modify_vlan_out_first_vid[] = { 215 /* Size in bits !!! */ 216 {12, 0, MLX5_MODI_OUT_FIRST_VID}, 217 {0, 0, 0}, 218 }; 219 220 struct field_modify_info modify_ipv4[] = { 221 {1, 1, MLX5_MODI_OUT_IP_DSCP}, 222 {1, 8, MLX5_MODI_OUT_IPV4_TTL}, 223 {4, 12, MLX5_MODI_OUT_SIPV4}, 224 {4, 16, MLX5_MODI_OUT_DIPV4}, 225 {0, 0, 0}, 226 }; 227 228 struct field_modify_info modify_ipv6[] = { 229 {1, 0, MLX5_MODI_OUT_IP_DSCP}, 230 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, 231 {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, 232 {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, 233 {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, 234 {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, 235 {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, 236 {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, 237 {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, 238 {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, 239 {0, 0, 0}, 240 }; 241 242 struct field_modify_info modify_udp[] = { 243 {2, 0, MLX5_MODI_OUT_UDP_SPORT}, 244 {2, 2, MLX5_MODI_OUT_UDP_DPORT}, 245 {0, 0, 0}, 246 }; 247 248 struct field_modify_info modify_tcp[] = { 249 {2, 0, MLX5_MODI_OUT_TCP_SPORT}, 250 {2, 2, MLX5_MODI_OUT_TCP_DPORT}, 251 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, 252 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, 253 {0, 0, 0}, 254 }; 255 256 static void 257 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, 258 uint8_t next_protocol, uint64_t *item_flags, 259 int *tunnel) 260 { 261 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || 262 item->type == RTE_FLOW_ITEM_TYPE_IPV6); 263 if (next_protocol == IPPROTO_IPIP) { 264 *item_flags |= MLX5_FLOW_LAYER_IPIP; 265 *tunnel = 1; 266 } 267 if (next_protocol == IPPROTO_IPV6) { 268 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; 269 *tunnel = 1; 270 } 271 } 272 273 /** 274 * Acquire the synchronizing object to protect multithreaded access 275 * to shared dv context. Lock occurs only if context is actually 276 * shared, i.e. we have multiport IB device and representors are 277 * created. 278 * 279 * @param[in] dev 280 * Pointer to the rte_eth_dev structure. 281 */ 282 static void 283 flow_dv_shared_lock(struct rte_eth_dev *dev) 284 { 285 struct mlx5_priv *priv = dev->data->dev_private; 286 struct mlx5_dev_ctx_shared *sh = priv->sh; 287 288 if (sh->dv_refcnt > 1) { 289 int ret; 290 291 ret = pthread_mutex_lock(&sh->dv_mutex); 292 MLX5_ASSERT(!ret); 293 (void)ret; 294 } 295 } 296 297 static void 298 flow_dv_shared_unlock(struct rte_eth_dev *dev) 299 { 300 struct mlx5_priv *priv = dev->data->dev_private; 301 struct mlx5_dev_ctx_shared *sh = priv->sh; 302 303 if (sh->dv_refcnt > 1) { 304 int ret; 305 306 ret = pthread_mutex_unlock(&sh->dv_mutex); 307 MLX5_ASSERT(!ret); 308 (void)ret; 309 } 310 } 311 312 /* Update VLAN's VID/PCP based on input rte_flow_action. 313 * 314 * @param[in] action 315 * Pointer to struct rte_flow_action. 316 * @param[out] vlan 317 * Pointer to struct rte_vlan_hdr. 318 */ 319 static void 320 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, 321 struct rte_vlan_hdr *vlan) 322 { 323 uint16_t vlan_tci; 324 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) { 325 vlan_tci = 326 ((const struct rte_flow_action_of_set_vlan_pcp *) 327 action->conf)->vlan_pcp; 328 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT; 329 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 330 vlan->vlan_tci |= vlan_tci; 331 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) { 332 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 333 vlan->vlan_tci |= rte_be_to_cpu_16 334 (((const struct rte_flow_action_of_set_vlan_vid *) 335 action->conf)->vlan_vid); 336 } 337 } 338 339 /** 340 * Fetch 1, 2, 3 or 4 byte field from the byte array 341 * and return as unsigned integer in host-endian format. 342 * 343 * @param[in] data 344 * Pointer to data array. 345 * @param[in] size 346 * Size of field to extract. 347 * 348 * @return 349 * converted field in host endian format. 350 */ 351 static inline uint32_t 352 flow_dv_fetch_field(const uint8_t *data, uint32_t size) 353 { 354 uint32_t ret; 355 356 switch (size) { 357 case 1: 358 ret = *data; 359 break; 360 case 2: 361 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 362 break; 363 case 3: 364 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 365 ret = (ret << 8) | *(data + sizeof(uint16_t)); 366 break; 367 case 4: 368 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data); 369 break; 370 default: 371 MLX5_ASSERT(false); 372 ret = 0; 373 break; 374 } 375 return ret; 376 } 377 378 /** 379 * Convert modify-header action to DV specification. 380 * 381 * Data length of each action is determined by provided field description 382 * and the item mask. Data bit offset and width of each action is determined 383 * by provided item mask. 384 * 385 * @param[in] item 386 * Pointer to item specification. 387 * @param[in] field 388 * Pointer to field modification information. 389 * For MLX5_MODIFICATION_TYPE_SET specifies destination field. 390 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field. 391 * For MLX5_MODIFICATION_TYPE_COPY specifies source field. 392 * @param[in] dcopy 393 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type. 394 * Negative offset value sets the same offset as source offset. 395 * size field is ignored, value is taken from source field. 396 * @param[in,out] resource 397 * Pointer to the modify-header resource. 398 * @param[in] type 399 * Type of modification. 400 * @param[out] error 401 * Pointer to the error structure. 402 * 403 * @return 404 * 0 on success, a negative errno value otherwise and rte_errno is set. 405 */ 406 static int 407 flow_dv_convert_modify_action(struct rte_flow_item *item, 408 struct field_modify_info *field, 409 struct field_modify_info *dcopy, 410 struct mlx5_flow_dv_modify_hdr_resource *resource, 411 uint32_t type, struct rte_flow_error *error) 412 { 413 uint32_t i = resource->actions_num; 414 struct mlx5_modification_cmd *actions = resource->actions; 415 416 /* 417 * The item and mask are provided in big-endian format. 418 * The fields should be presented as in big-endian format either. 419 * Mask must be always present, it defines the actual field width. 420 */ 421 MLX5_ASSERT(item->mask); 422 MLX5_ASSERT(field->size); 423 do { 424 unsigned int size_b; 425 unsigned int off_b; 426 uint32_t mask; 427 uint32_t data; 428 429 if (i >= MLX5_MAX_MODIFY_NUM) 430 return rte_flow_error_set(error, EINVAL, 431 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 432 "too many items to modify"); 433 /* Fetch variable byte size mask from the array. */ 434 mask = flow_dv_fetch_field((const uint8_t *)item->mask + 435 field->offset, field->size); 436 if (!mask) { 437 ++field; 438 continue; 439 } 440 /* Deduce actual data width in bits from mask value. */ 441 off_b = rte_bsf32(mask); 442 size_b = sizeof(uint32_t) * CHAR_BIT - 443 off_b - __builtin_clz(mask); 444 MLX5_ASSERT(size_b); 445 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b; 446 actions[i] = (struct mlx5_modification_cmd) { 447 .action_type = type, 448 .field = field->id, 449 .offset = off_b, 450 .length = size_b, 451 }; 452 /* Convert entire record to expected big-endian format. */ 453 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 454 if (type == MLX5_MODIFICATION_TYPE_COPY) { 455 MLX5_ASSERT(dcopy); 456 actions[i].dst_field = dcopy->id; 457 actions[i].dst_offset = 458 (int)dcopy->offset < 0 ? off_b : dcopy->offset; 459 /* Convert entire record to big-endian format. */ 460 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); 461 } else { 462 MLX5_ASSERT(item->spec); 463 data = flow_dv_fetch_field((const uint8_t *)item->spec + 464 field->offset, field->size); 465 /* Shift out the trailing masked bits from data. */ 466 data = (data & mask) >> off_b; 467 actions[i].data1 = rte_cpu_to_be_32(data); 468 } 469 ++i; 470 ++field; 471 } while (field->size); 472 if (resource->actions_num == i) 473 return rte_flow_error_set(error, EINVAL, 474 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 475 "invalid modification flow item"); 476 resource->actions_num = i; 477 return 0; 478 } 479 480 /** 481 * Convert modify-header set IPv4 address action to DV specification. 482 * 483 * @param[in,out] resource 484 * Pointer to the modify-header resource. 485 * @param[in] action 486 * Pointer to action specification. 487 * @param[out] error 488 * Pointer to the error structure. 489 * 490 * @return 491 * 0 on success, a negative errno value otherwise and rte_errno is set. 492 */ 493 static int 494 flow_dv_convert_action_modify_ipv4 495 (struct mlx5_flow_dv_modify_hdr_resource *resource, 496 const struct rte_flow_action *action, 497 struct rte_flow_error *error) 498 { 499 const struct rte_flow_action_set_ipv4 *conf = 500 (const struct rte_flow_action_set_ipv4 *)(action->conf); 501 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 502 struct rte_flow_item_ipv4 ipv4; 503 struct rte_flow_item_ipv4 ipv4_mask; 504 505 memset(&ipv4, 0, sizeof(ipv4)); 506 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 507 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { 508 ipv4.hdr.src_addr = conf->ipv4_addr; 509 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; 510 } else { 511 ipv4.hdr.dst_addr = conf->ipv4_addr; 512 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; 513 } 514 item.spec = &ipv4; 515 item.mask = &ipv4_mask; 516 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 517 MLX5_MODIFICATION_TYPE_SET, error); 518 } 519 520 /** 521 * Convert modify-header set IPv6 address action to DV specification. 522 * 523 * @param[in,out] resource 524 * Pointer to the modify-header resource. 525 * @param[in] action 526 * Pointer to action specification. 527 * @param[out] error 528 * Pointer to the error structure. 529 * 530 * @return 531 * 0 on success, a negative errno value otherwise and rte_errno is set. 532 */ 533 static int 534 flow_dv_convert_action_modify_ipv6 535 (struct mlx5_flow_dv_modify_hdr_resource *resource, 536 const struct rte_flow_action *action, 537 struct rte_flow_error *error) 538 { 539 const struct rte_flow_action_set_ipv6 *conf = 540 (const struct rte_flow_action_set_ipv6 *)(action->conf); 541 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 542 struct rte_flow_item_ipv6 ipv6; 543 struct rte_flow_item_ipv6 ipv6_mask; 544 545 memset(&ipv6, 0, sizeof(ipv6)); 546 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 547 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { 548 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, 549 sizeof(ipv6.hdr.src_addr)); 550 memcpy(&ipv6_mask.hdr.src_addr, 551 &rte_flow_item_ipv6_mask.hdr.src_addr, 552 sizeof(ipv6.hdr.src_addr)); 553 } else { 554 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, 555 sizeof(ipv6.hdr.dst_addr)); 556 memcpy(&ipv6_mask.hdr.dst_addr, 557 &rte_flow_item_ipv6_mask.hdr.dst_addr, 558 sizeof(ipv6.hdr.dst_addr)); 559 } 560 item.spec = &ipv6; 561 item.mask = &ipv6_mask; 562 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 563 MLX5_MODIFICATION_TYPE_SET, error); 564 } 565 566 /** 567 * Convert modify-header set MAC address action to DV specification. 568 * 569 * @param[in,out] resource 570 * Pointer to the modify-header resource. 571 * @param[in] action 572 * Pointer to action specification. 573 * @param[out] error 574 * Pointer to the error structure. 575 * 576 * @return 577 * 0 on success, a negative errno value otherwise and rte_errno is set. 578 */ 579 static int 580 flow_dv_convert_action_modify_mac 581 (struct mlx5_flow_dv_modify_hdr_resource *resource, 582 const struct rte_flow_action *action, 583 struct rte_flow_error *error) 584 { 585 const struct rte_flow_action_set_mac *conf = 586 (const struct rte_flow_action_set_mac *)(action->conf); 587 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; 588 struct rte_flow_item_eth eth; 589 struct rte_flow_item_eth eth_mask; 590 591 memset(ð, 0, sizeof(eth)); 592 memset(ð_mask, 0, sizeof(eth_mask)); 593 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { 594 memcpy(ð.src.addr_bytes, &conf->mac_addr, 595 sizeof(eth.src.addr_bytes)); 596 memcpy(ð_mask.src.addr_bytes, 597 &rte_flow_item_eth_mask.src.addr_bytes, 598 sizeof(eth_mask.src.addr_bytes)); 599 } else { 600 memcpy(ð.dst.addr_bytes, &conf->mac_addr, 601 sizeof(eth.dst.addr_bytes)); 602 memcpy(ð_mask.dst.addr_bytes, 603 &rte_flow_item_eth_mask.dst.addr_bytes, 604 sizeof(eth_mask.dst.addr_bytes)); 605 } 606 item.spec = ð 607 item.mask = ð_mask; 608 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource, 609 MLX5_MODIFICATION_TYPE_SET, error); 610 } 611 612 /** 613 * Convert modify-header set VLAN VID action to DV specification. 614 * 615 * @param[in,out] resource 616 * Pointer to the modify-header resource. 617 * @param[in] action 618 * Pointer to action specification. 619 * @param[out] error 620 * Pointer to the error structure. 621 * 622 * @return 623 * 0 on success, a negative errno value otherwise and rte_errno is set. 624 */ 625 static int 626 flow_dv_convert_action_modify_vlan_vid 627 (struct mlx5_flow_dv_modify_hdr_resource *resource, 628 const struct rte_flow_action *action, 629 struct rte_flow_error *error) 630 { 631 const struct rte_flow_action_of_set_vlan_vid *conf = 632 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf); 633 int i = resource->actions_num; 634 struct mlx5_modification_cmd *actions = resource->actions; 635 struct field_modify_info *field = modify_vlan_out_first_vid; 636 637 if (i >= MLX5_MAX_MODIFY_NUM) 638 return rte_flow_error_set(error, EINVAL, 639 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 640 "too many items to modify"); 641 actions[i] = (struct mlx5_modification_cmd) { 642 .action_type = MLX5_MODIFICATION_TYPE_SET, 643 .field = field->id, 644 .length = field->size, 645 .offset = field->offset, 646 }; 647 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 648 actions[i].data1 = conf->vlan_vid; 649 actions[i].data1 = actions[i].data1 << 16; 650 resource->actions_num = ++i; 651 return 0; 652 } 653 654 /** 655 * Convert modify-header set TP action to DV specification. 656 * 657 * @param[in,out] resource 658 * Pointer to the modify-header resource. 659 * @param[in] action 660 * Pointer to action specification. 661 * @param[in] items 662 * Pointer to rte_flow_item objects list. 663 * @param[in] attr 664 * Pointer to flow attributes structure. 665 * @param[in] dev_flow 666 * Pointer to the sub flow. 667 * @param[in] tunnel_decap 668 * Whether action is after tunnel decapsulation. 669 * @param[out] error 670 * Pointer to the error structure. 671 * 672 * @return 673 * 0 on success, a negative errno value otherwise and rte_errno is set. 674 */ 675 static int 676 flow_dv_convert_action_modify_tp 677 (struct mlx5_flow_dv_modify_hdr_resource *resource, 678 const struct rte_flow_action *action, 679 const struct rte_flow_item *items, 680 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 681 bool tunnel_decap, struct rte_flow_error *error) 682 { 683 const struct rte_flow_action_set_tp *conf = 684 (const struct rte_flow_action_set_tp *)(action->conf); 685 struct rte_flow_item item; 686 struct rte_flow_item_udp udp; 687 struct rte_flow_item_udp udp_mask; 688 struct rte_flow_item_tcp tcp; 689 struct rte_flow_item_tcp tcp_mask; 690 struct field_modify_info *field; 691 692 if (!attr->valid) 693 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 694 if (attr->udp) { 695 memset(&udp, 0, sizeof(udp)); 696 memset(&udp_mask, 0, sizeof(udp_mask)); 697 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 698 udp.hdr.src_port = conf->port; 699 udp_mask.hdr.src_port = 700 rte_flow_item_udp_mask.hdr.src_port; 701 } else { 702 udp.hdr.dst_port = conf->port; 703 udp_mask.hdr.dst_port = 704 rte_flow_item_udp_mask.hdr.dst_port; 705 } 706 item.type = RTE_FLOW_ITEM_TYPE_UDP; 707 item.spec = &udp; 708 item.mask = &udp_mask; 709 field = modify_udp; 710 } else { 711 MLX5_ASSERT(attr->tcp); 712 memset(&tcp, 0, sizeof(tcp)); 713 memset(&tcp_mask, 0, sizeof(tcp_mask)); 714 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 715 tcp.hdr.src_port = conf->port; 716 tcp_mask.hdr.src_port = 717 rte_flow_item_tcp_mask.hdr.src_port; 718 } else { 719 tcp.hdr.dst_port = conf->port; 720 tcp_mask.hdr.dst_port = 721 rte_flow_item_tcp_mask.hdr.dst_port; 722 } 723 item.type = RTE_FLOW_ITEM_TYPE_TCP; 724 item.spec = &tcp; 725 item.mask = &tcp_mask; 726 field = modify_tcp; 727 } 728 return flow_dv_convert_modify_action(&item, field, NULL, resource, 729 MLX5_MODIFICATION_TYPE_SET, error); 730 } 731 732 /** 733 * Convert modify-header set TTL action to DV specification. 734 * 735 * @param[in,out] resource 736 * Pointer to the modify-header resource. 737 * @param[in] action 738 * Pointer to action specification. 739 * @param[in] items 740 * Pointer to rte_flow_item objects list. 741 * @param[in] attr 742 * Pointer to flow attributes structure. 743 * @param[in] dev_flow 744 * Pointer to the sub flow. 745 * @param[in] tunnel_decap 746 * Whether action is after tunnel decapsulation. 747 * @param[out] error 748 * Pointer to the error structure. 749 * 750 * @return 751 * 0 on success, a negative errno value otherwise and rte_errno is set. 752 */ 753 static int 754 flow_dv_convert_action_modify_ttl 755 (struct mlx5_flow_dv_modify_hdr_resource *resource, 756 const struct rte_flow_action *action, 757 const struct rte_flow_item *items, 758 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 759 bool tunnel_decap, struct rte_flow_error *error) 760 { 761 const struct rte_flow_action_set_ttl *conf = 762 (const struct rte_flow_action_set_ttl *)(action->conf); 763 struct rte_flow_item item; 764 struct rte_flow_item_ipv4 ipv4; 765 struct rte_flow_item_ipv4 ipv4_mask; 766 struct rte_flow_item_ipv6 ipv6; 767 struct rte_flow_item_ipv6 ipv6_mask; 768 struct field_modify_info *field; 769 770 if (!attr->valid) 771 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 772 if (attr->ipv4) { 773 memset(&ipv4, 0, sizeof(ipv4)); 774 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 775 ipv4.hdr.time_to_live = conf->ttl_value; 776 ipv4_mask.hdr.time_to_live = 0xFF; 777 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 778 item.spec = &ipv4; 779 item.mask = &ipv4_mask; 780 field = modify_ipv4; 781 } else { 782 MLX5_ASSERT(attr->ipv6); 783 memset(&ipv6, 0, sizeof(ipv6)); 784 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 785 ipv6.hdr.hop_limits = conf->ttl_value; 786 ipv6_mask.hdr.hop_limits = 0xFF; 787 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 788 item.spec = &ipv6; 789 item.mask = &ipv6_mask; 790 field = modify_ipv6; 791 } 792 return flow_dv_convert_modify_action(&item, field, NULL, resource, 793 MLX5_MODIFICATION_TYPE_SET, error); 794 } 795 796 /** 797 * Convert modify-header decrement TTL action to DV specification. 798 * 799 * @param[in,out] resource 800 * Pointer to the modify-header resource. 801 * @param[in] action 802 * Pointer to action specification. 803 * @param[in] items 804 * Pointer to rte_flow_item objects list. 805 * @param[in] attr 806 * Pointer to flow attributes structure. 807 * @param[in] dev_flow 808 * Pointer to the sub flow. 809 * @param[in] tunnel_decap 810 * Whether action is after tunnel decapsulation. 811 * @param[out] error 812 * Pointer to the error structure. 813 * 814 * @return 815 * 0 on success, a negative errno value otherwise and rte_errno is set. 816 */ 817 static int 818 flow_dv_convert_action_modify_dec_ttl 819 (struct mlx5_flow_dv_modify_hdr_resource *resource, 820 const struct rte_flow_item *items, 821 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 822 bool tunnel_decap, struct rte_flow_error *error) 823 { 824 struct rte_flow_item item; 825 struct rte_flow_item_ipv4 ipv4; 826 struct rte_flow_item_ipv4 ipv4_mask; 827 struct rte_flow_item_ipv6 ipv6; 828 struct rte_flow_item_ipv6 ipv6_mask; 829 struct field_modify_info *field; 830 831 if (!attr->valid) 832 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 833 if (attr->ipv4) { 834 memset(&ipv4, 0, sizeof(ipv4)); 835 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 836 ipv4.hdr.time_to_live = 0xFF; 837 ipv4_mask.hdr.time_to_live = 0xFF; 838 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 839 item.spec = &ipv4; 840 item.mask = &ipv4_mask; 841 field = modify_ipv4; 842 } else { 843 MLX5_ASSERT(attr->ipv6); 844 memset(&ipv6, 0, sizeof(ipv6)); 845 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 846 ipv6.hdr.hop_limits = 0xFF; 847 ipv6_mask.hdr.hop_limits = 0xFF; 848 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 849 item.spec = &ipv6; 850 item.mask = &ipv6_mask; 851 field = modify_ipv6; 852 } 853 return flow_dv_convert_modify_action(&item, field, NULL, resource, 854 MLX5_MODIFICATION_TYPE_ADD, error); 855 } 856 857 /** 858 * Convert modify-header increment/decrement TCP Sequence number 859 * to DV specification. 860 * 861 * @param[in,out] resource 862 * Pointer to the modify-header resource. 863 * @param[in] action 864 * Pointer to action specification. 865 * @param[out] error 866 * Pointer to the error structure. 867 * 868 * @return 869 * 0 on success, a negative errno value otherwise and rte_errno is set. 870 */ 871 static int 872 flow_dv_convert_action_modify_tcp_seq 873 (struct mlx5_flow_dv_modify_hdr_resource *resource, 874 const struct rte_flow_action *action, 875 struct rte_flow_error *error) 876 { 877 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 878 uint64_t value = rte_be_to_cpu_32(*conf); 879 struct rte_flow_item item; 880 struct rte_flow_item_tcp tcp; 881 struct rte_flow_item_tcp tcp_mask; 882 883 memset(&tcp, 0, sizeof(tcp)); 884 memset(&tcp_mask, 0, sizeof(tcp_mask)); 885 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) 886 /* 887 * The HW has no decrement operation, only increment operation. 888 * To simulate decrement X from Y using increment operation 889 * we need to add UINT32_MAX X times to Y. 890 * Each adding of UINT32_MAX decrements Y by 1. 891 */ 892 value *= UINT32_MAX; 893 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); 894 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); 895 item.type = RTE_FLOW_ITEM_TYPE_TCP; 896 item.spec = &tcp; 897 item.mask = &tcp_mask; 898 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 899 MLX5_MODIFICATION_TYPE_ADD, error); 900 } 901 902 /** 903 * Convert modify-header increment/decrement TCP Acknowledgment number 904 * to DV specification. 905 * 906 * @param[in,out] resource 907 * Pointer to the modify-header resource. 908 * @param[in] action 909 * Pointer to action specification. 910 * @param[out] error 911 * Pointer to the error structure. 912 * 913 * @return 914 * 0 on success, a negative errno value otherwise and rte_errno is set. 915 */ 916 static int 917 flow_dv_convert_action_modify_tcp_ack 918 (struct mlx5_flow_dv_modify_hdr_resource *resource, 919 const struct rte_flow_action *action, 920 struct rte_flow_error *error) 921 { 922 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 923 uint64_t value = rte_be_to_cpu_32(*conf); 924 struct rte_flow_item item; 925 struct rte_flow_item_tcp tcp; 926 struct rte_flow_item_tcp tcp_mask; 927 928 memset(&tcp, 0, sizeof(tcp)); 929 memset(&tcp_mask, 0, sizeof(tcp_mask)); 930 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) 931 /* 932 * The HW has no decrement operation, only increment operation. 933 * To simulate decrement X from Y using increment operation 934 * we need to add UINT32_MAX X times to Y. 935 * Each adding of UINT32_MAX decrements Y by 1. 936 */ 937 value *= UINT32_MAX; 938 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); 939 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); 940 item.type = RTE_FLOW_ITEM_TYPE_TCP; 941 item.spec = &tcp; 942 item.mask = &tcp_mask; 943 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 944 MLX5_MODIFICATION_TYPE_ADD, error); 945 } 946 947 static enum mlx5_modification_field reg_to_field[] = { 948 [REG_NON] = MLX5_MODI_OUT_NONE, 949 [REG_A] = MLX5_MODI_META_DATA_REG_A, 950 [REG_B] = MLX5_MODI_META_DATA_REG_B, 951 [REG_C_0] = MLX5_MODI_META_REG_C_0, 952 [REG_C_1] = MLX5_MODI_META_REG_C_1, 953 [REG_C_2] = MLX5_MODI_META_REG_C_2, 954 [REG_C_3] = MLX5_MODI_META_REG_C_3, 955 [REG_C_4] = MLX5_MODI_META_REG_C_4, 956 [REG_C_5] = MLX5_MODI_META_REG_C_5, 957 [REG_C_6] = MLX5_MODI_META_REG_C_6, 958 [REG_C_7] = MLX5_MODI_META_REG_C_7, 959 }; 960 961 /** 962 * Convert register set to DV specification. 963 * 964 * @param[in,out] resource 965 * Pointer to the modify-header resource. 966 * @param[in] action 967 * Pointer to action specification. 968 * @param[out] error 969 * Pointer to the error structure. 970 * 971 * @return 972 * 0 on success, a negative errno value otherwise and rte_errno is set. 973 */ 974 static int 975 flow_dv_convert_action_set_reg 976 (struct mlx5_flow_dv_modify_hdr_resource *resource, 977 const struct rte_flow_action *action, 978 struct rte_flow_error *error) 979 { 980 const struct mlx5_rte_flow_action_set_tag *conf = action->conf; 981 struct mlx5_modification_cmd *actions = resource->actions; 982 uint32_t i = resource->actions_num; 983 984 if (i >= MLX5_MAX_MODIFY_NUM) 985 return rte_flow_error_set(error, EINVAL, 986 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 987 "too many items to modify"); 988 MLX5_ASSERT(conf->id != REG_NON); 989 MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); 990 actions[i] = (struct mlx5_modification_cmd) { 991 .action_type = MLX5_MODIFICATION_TYPE_SET, 992 .field = reg_to_field[conf->id], 993 }; 994 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 995 actions[i].data1 = rte_cpu_to_be_32(conf->data); 996 ++i; 997 resource->actions_num = i; 998 return 0; 999 } 1000 1001 /** 1002 * Convert SET_TAG action to DV specification. 1003 * 1004 * @param[in] dev 1005 * Pointer to the rte_eth_dev structure. 1006 * @param[in,out] resource 1007 * Pointer to the modify-header resource. 1008 * @param[in] conf 1009 * Pointer to action specification. 1010 * @param[out] error 1011 * Pointer to the error structure. 1012 * 1013 * @return 1014 * 0 on success, a negative errno value otherwise and rte_errno is set. 1015 */ 1016 static int 1017 flow_dv_convert_action_set_tag 1018 (struct rte_eth_dev *dev, 1019 struct mlx5_flow_dv_modify_hdr_resource *resource, 1020 const struct rte_flow_action_set_tag *conf, 1021 struct rte_flow_error *error) 1022 { 1023 rte_be32_t data = rte_cpu_to_be_32(conf->data); 1024 rte_be32_t mask = rte_cpu_to_be_32(conf->mask); 1025 struct rte_flow_item item = { 1026 .spec = &data, 1027 .mask = &mask, 1028 }; 1029 struct field_modify_info reg_c_x[] = { 1030 [1] = {0, 0, 0}, 1031 }; 1032 enum mlx5_modification_field reg_type; 1033 int ret; 1034 1035 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 1036 if (ret < 0) 1037 return ret; 1038 MLX5_ASSERT(ret != REG_NON); 1039 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field)); 1040 reg_type = reg_to_field[ret]; 1041 MLX5_ASSERT(reg_type > 0); 1042 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type}; 1043 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1044 MLX5_MODIFICATION_TYPE_SET, error); 1045 } 1046 1047 /** 1048 * Convert internal COPY_REG action to DV specification. 1049 * 1050 * @param[in] dev 1051 * Pointer to the rte_eth_dev structure. 1052 * @param[in,out] res 1053 * Pointer to the modify-header resource. 1054 * @param[in] action 1055 * Pointer to action specification. 1056 * @param[out] error 1057 * Pointer to the error structure. 1058 * 1059 * @return 1060 * 0 on success, a negative errno value otherwise and rte_errno is set. 1061 */ 1062 static int 1063 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, 1064 struct mlx5_flow_dv_modify_hdr_resource *res, 1065 const struct rte_flow_action *action, 1066 struct rte_flow_error *error) 1067 { 1068 const struct mlx5_flow_action_copy_mreg *conf = action->conf; 1069 rte_be32_t mask = RTE_BE32(UINT32_MAX); 1070 struct rte_flow_item item = { 1071 .spec = NULL, 1072 .mask = &mask, 1073 }; 1074 struct field_modify_info reg_src[] = { 1075 {4, 0, reg_to_field[conf->src]}, 1076 {0, 0, 0}, 1077 }; 1078 struct field_modify_info reg_dst = { 1079 .offset = 0, 1080 .id = reg_to_field[conf->dst], 1081 }; 1082 /* Adjust reg_c[0] usage according to reported mask. */ 1083 if (conf->dst == REG_C_0 || conf->src == REG_C_0) { 1084 struct mlx5_priv *priv = dev->data->dev_private; 1085 uint32_t reg_c0 = priv->sh->dv_regc0_mask; 1086 1087 MLX5_ASSERT(reg_c0); 1088 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY); 1089 if (conf->dst == REG_C_0) { 1090 /* Copy to reg_c[0], within mask only. */ 1091 reg_dst.offset = rte_bsf32(reg_c0); 1092 /* 1093 * Mask is ignoring the enianness, because 1094 * there is no conversion in datapath. 1095 */ 1096 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1097 /* Copy from destination lower bits to reg_c[0]. */ 1098 mask = reg_c0 >> reg_dst.offset; 1099 #else 1100 /* Copy from destination upper bits to reg_c[0]. */ 1101 mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT - 1102 rte_fls_u32(reg_c0)); 1103 #endif 1104 } else { 1105 mask = rte_cpu_to_be_32(reg_c0); 1106 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1107 /* Copy from reg_c[0] to destination lower bits. */ 1108 reg_dst.offset = 0; 1109 #else 1110 /* Copy from reg_c[0] to destination upper bits. */ 1111 reg_dst.offset = sizeof(reg_c0) * CHAR_BIT - 1112 (rte_fls_u32(reg_c0) - 1113 rte_bsf32(reg_c0)); 1114 #endif 1115 } 1116 } 1117 return flow_dv_convert_modify_action(&item, 1118 reg_src, ®_dst, res, 1119 MLX5_MODIFICATION_TYPE_COPY, 1120 error); 1121 } 1122 1123 /** 1124 * Convert MARK action to DV specification. This routine is used 1125 * in extensive metadata only and requires metadata register to be 1126 * handled. In legacy mode hardware tag resource is engaged. 1127 * 1128 * @param[in] dev 1129 * Pointer to the rte_eth_dev structure. 1130 * @param[in] conf 1131 * Pointer to MARK action specification. 1132 * @param[in,out] resource 1133 * Pointer to the modify-header resource. 1134 * @param[out] error 1135 * Pointer to the error structure. 1136 * 1137 * @return 1138 * 0 on success, a negative errno value otherwise and rte_errno is set. 1139 */ 1140 static int 1141 flow_dv_convert_action_mark(struct rte_eth_dev *dev, 1142 const struct rte_flow_action_mark *conf, 1143 struct mlx5_flow_dv_modify_hdr_resource *resource, 1144 struct rte_flow_error *error) 1145 { 1146 struct mlx5_priv *priv = dev->data->dev_private; 1147 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK & 1148 priv->sh->dv_mark_mask); 1149 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask; 1150 struct rte_flow_item item = { 1151 .spec = &data, 1152 .mask = &mask, 1153 }; 1154 struct field_modify_info reg_c_x[] = { 1155 [1] = {0, 0, 0}, 1156 }; 1157 int reg; 1158 1159 if (!mask) 1160 return rte_flow_error_set(error, EINVAL, 1161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1162 NULL, "zero mark action mask"); 1163 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1164 if (reg < 0) 1165 return reg; 1166 MLX5_ASSERT(reg > 0); 1167 if (reg == REG_C_0) { 1168 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1169 uint32_t shl_c0 = rte_bsf32(msk_c0); 1170 1171 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); 1172 mask = rte_cpu_to_be_32(mask) & msk_c0; 1173 mask = rte_cpu_to_be_32(mask << shl_c0); 1174 } 1175 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1176 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1177 MLX5_MODIFICATION_TYPE_SET, error); 1178 } 1179 1180 /** 1181 * Get metadata register index for specified steering domain. 1182 * 1183 * @param[in] dev 1184 * Pointer to the rte_eth_dev structure. 1185 * @param[in] attr 1186 * Attributes of flow to determine steering domain. 1187 * @param[out] error 1188 * Pointer to the error structure. 1189 * 1190 * @return 1191 * positive index on success, a negative errno value otherwise 1192 * and rte_errno is set. 1193 */ 1194 static enum modify_reg 1195 flow_dv_get_metadata_reg(struct rte_eth_dev *dev, 1196 const struct rte_flow_attr *attr, 1197 struct rte_flow_error *error) 1198 { 1199 int reg = 1200 mlx5_flow_get_reg_id(dev, attr->transfer ? 1201 MLX5_METADATA_FDB : 1202 attr->egress ? 1203 MLX5_METADATA_TX : 1204 MLX5_METADATA_RX, 0, error); 1205 if (reg < 0) 1206 return rte_flow_error_set(error, 1207 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 1208 NULL, "unavailable " 1209 "metadata register"); 1210 return reg; 1211 } 1212 1213 /** 1214 * Convert SET_META action to DV specification. 1215 * 1216 * @param[in] dev 1217 * Pointer to the rte_eth_dev structure. 1218 * @param[in,out] resource 1219 * Pointer to the modify-header resource. 1220 * @param[in] attr 1221 * Attributes of flow that includes this item. 1222 * @param[in] conf 1223 * Pointer to action specification. 1224 * @param[out] error 1225 * Pointer to the error structure. 1226 * 1227 * @return 1228 * 0 on success, a negative errno value otherwise and rte_errno is set. 1229 */ 1230 static int 1231 flow_dv_convert_action_set_meta 1232 (struct rte_eth_dev *dev, 1233 struct mlx5_flow_dv_modify_hdr_resource *resource, 1234 const struct rte_flow_attr *attr, 1235 const struct rte_flow_action_set_meta *conf, 1236 struct rte_flow_error *error) 1237 { 1238 uint32_t data = conf->data; 1239 uint32_t mask = conf->mask; 1240 struct rte_flow_item item = { 1241 .spec = &data, 1242 .mask = &mask, 1243 }; 1244 struct field_modify_info reg_c_x[] = { 1245 [1] = {0, 0, 0}, 1246 }; 1247 int reg = flow_dv_get_metadata_reg(dev, attr, error); 1248 1249 if (reg < 0) 1250 return reg; 1251 /* 1252 * In datapath code there is no endianness 1253 * coversions for perfromance reasons, all 1254 * pattern conversions are done in rte_flow. 1255 */ 1256 if (reg == REG_C_0) { 1257 struct mlx5_priv *priv = dev->data->dev_private; 1258 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1259 uint32_t shl_c0; 1260 1261 MLX5_ASSERT(msk_c0); 1262 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1263 shl_c0 = rte_bsf32(msk_c0); 1264 #else 1265 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0); 1266 #endif 1267 mask <<= shl_c0; 1268 data <<= shl_c0; 1269 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask))); 1270 } 1271 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1272 /* The routine expects parameters in memory as big-endian ones. */ 1273 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1274 MLX5_MODIFICATION_TYPE_SET, error); 1275 } 1276 1277 /** 1278 * Convert modify-header set IPv4 DSCP action to DV specification. 1279 * 1280 * @param[in,out] resource 1281 * Pointer to the modify-header resource. 1282 * @param[in] action 1283 * Pointer to action specification. 1284 * @param[out] error 1285 * Pointer to the error structure. 1286 * 1287 * @return 1288 * 0 on success, a negative errno value otherwise and rte_errno is set. 1289 */ 1290 static int 1291 flow_dv_convert_action_modify_ipv4_dscp 1292 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1293 const struct rte_flow_action *action, 1294 struct rte_flow_error *error) 1295 { 1296 const struct rte_flow_action_set_dscp *conf = 1297 (const struct rte_flow_action_set_dscp *)(action->conf); 1298 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 1299 struct rte_flow_item_ipv4 ipv4; 1300 struct rte_flow_item_ipv4 ipv4_mask; 1301 1302 memset(&ipv4, 0, sizeof(ipv4)); 1303 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 1304 ipv4.hdr.type_of_service = conf->dscp; 1305 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; 1306 item.spec = &ipv4; 1307 item.mask = &ipv4_mask; 1308 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 1309 MLX5_MODIFICATION_TYPE_SET, error); 1310 } 1311 1312 /** 1313 * Convert modify-header set IPv6 DSCP action to DV specification. 1314 * 1315 * @param[in,out] resource 1316 * Pointer to the modify-header resource. 1317 * @param[in] action 1318 * Pointer to action specification. 1319 * @param[out] error 1320 * Pointer to the error structure. 1321 * 1322 * @return 1323 * 0 on success, a negative errno value otherwise and rte_errno is set. 1324 */ 1325 static int 1326 flow_dv_convert_action_modify_ipv6_dscp 1327 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1328 const struct rte_flow_action *action, 1329 struct rte_flow_error *error) 1330 { 1331 const struct rte_flow_action_set_dscp *conf = 1332 (const struct rte_flow_action_set_dscp *)(action->conf); 1333 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 1334 struct rte_flow_item_ipv6 ipv6; 1335 struct rte_flow_item_ipv6 ipv6_mask; 1336 1337 memset(&ipv6, 0, sizeof(ipv6)); 1338 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 1339 /* 1340 * Even though the DSCP bits offset of IPv6 is not byte aligned, 1341 * rdma-core only accept the DSCP bits byte aligned start from 1342 * bit 0 to 5 as to be compatible with IPv4. No need to shift the 1343 * bits in IPv6 case as rdma-core requires byte aligned value. 1344 */ 1345 ipv6.hdr.vtc_flow = conf->dscp; 1346 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; 1347 item.spec = &ipv6; 1348 item.mask = &ipv6_mask; 1349 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 1350 MLX5_MODIFICATION_TYPE_SET, error); 1351 } 1352 1353 /** 1354 * Validate MARK item. 1355 * 1356 * @param[in] dev 1357 * Pointer to the rte_eth_dev structure. 1358 * @param[in] item 1359 * Item specification. 1360 * @param[in] attr 1361 * Attributes of flow that includes this item. 1362 * @param[out] error 1363 * Pointer to error structure. 1364 * 1365 * @return 1366 * 0 on success, a negative errno value otherwise and rte_errno is set. 1367 */ 1368 static int 1369 flow_dv_validate_item_mark(struct rte_eth_dev *dev, 1370 const struct rte_flow_item *item, 1371 const struct rte_flow_attr *attr __rte_unused, 1372 struct rte_flow_error *error) 1373 { 1374 struct mlx5_priv *priv = dev->data->dev_private; 1375 struct mlx5_dev_config *config = &priv->config; 1376 const struct rte_flow_item_mark *spec = item->spec; 1377 const struct rte_flow_item_mark *mask = item->mask; 1378 const struct rte_flow_item_mark nic_mask = { 1379 .id = priv->sh->dv_mark_mask, 1380 }; 1381 int ret; 1382 1383 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 1384 return rte_flow_error_set(error, ENOTSUP, 1385 RTE_FLOW_ERROR_TYPE_ITEM, item, 1386 "extended metadata feature" 1387 " isn't enabled"); 1388 if (!mlx5_flow_ext_mreg_supported(dev)) 1389 return rte_flow_error_set(error, ENOTSUP, 1390 RTE_FLOW_ERROR_TYPE_ITEM, item, 1391 "extended metadata register" 1392 " isn't supported"); 1393 if (!nic_mask.id) 1394 return rte_flow_error_set(error, ENOTSUP, 1395 RTE_FLOW_ERROR_TYPE_ITEM, item, 1396 "extended metadata register" 1397 " isn't available"); 1398 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1399 if (ret < 0) 1400 return ret; 1401 if (!spec) 1402 return rte_flow_error_set(error, EINVAL, 1403 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1404 item->spec, 1405 "data cannot be empty"); 1406 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id)) 1407 return rte_flow_error_set(error, EINVAL, 1408 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1409 &spec->id, 1410 "mark id exceeds the limit"); 1411 if (!mask) 1412 mask = &nic_mask; 1413 if (!mask->id) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1416 "mask cannot be zero"); 1417 1418 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1419 (const uint8_t *)&nic_mask, 1420 sizeof(struct rte_flow_item_mark), 1421 error); 1422 if (ret < 0) 1423 return ret; 1424 return 0; 1425 } 1426 1427 /** 1428 * Validate META item. 1429 * 1430 * @param[in] dev 1431 * Pointer to the rte_eth_dev structure. 1432 * @param[in] item 1433 * Item specification. 1434 * @param[in] attr 1435 * Attributes of flow that includes this item. 1436 * @param[out] error 1437 * Pointer to error structure. 1438 * 1439 * @return 1440 * 0 on success, a negative errno value otherwise and rte_errno is set. 1441 */ 1442 static int 1443 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, 1444 const struct rte_flow_item *item, 1445 const struct rte_flow_attr *attr, 1446 struct rte_flow_error *error) 1447 { 1448 struct mlx5_priv *priv = dev->data->dev_private; 1449 struct mlx5_dev_config *config = &priv->config; 1450 const struct rte_flow_item_meta *spec = item->spec; 1451 const struct rte_flow_item_meta *mask = item->mask; 1452 struct rte_flow_item_meta nic_mask = { 1453 .data = UINT32_MAX 1454 }; 1455 int reg; 1456 int ret; 1457 1458 if (!spec) 1459 return rte_flow_error_set(error, EINVAL, 1460 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1461 item->spec, 1462 "data cannot be empty"); 1463 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1464 if (!mlx5_flow_ext_mreg_supported(dev)) 1465 return rte_flow_error_set(error, ENOTSUP, 1466 RTE_FLOW_ERROR_TYPE_ITEM, item, 1467 "extended metadata register" 1468 " isn't supported"); 1469 reg = flow_dv_get_metadata_reg(dev, attr, error); 1470 if (reg < 0) 1471 return reg; 1472 if (reg == REG_B) 1473 return rte_flow_error_set(error, ENOTSUP, 1474 RTE_FLOW_ERROR_TYPE_ITEM, item, 1475 "match on reg_b " 1476 "isn't supported"); 1477 if (reg != REG_A) 1478 nic_mask.data = priv->sh->dv_meta_mask; 1479 } else if (attr->transfer) { 1480 return rte_flow_error_set(error, ENOTSUP, 1481 RTE_FLOW_ERROR_TYPE_ITEM, item, 1482 "extended metadata feature " 1483 "should be enabled when " 1484 "meta item is requested " 1485 "with e-switch mode "); 1486 } 1487 if (!mask) 1488 mask = &rte_flow_item_meta_mask; 1489 if (!mask->data) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1492 "mask cannot be zero"); 1493 1494 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1495 (const uint8_t *)&nic_mask, 1496 sizeof(struct rte_flow_item_meta), 1497 error); 1498 return ret; 1499 } 1500 1501 /** 1502 * Validate TAG item. 1503 * 1504 * @param[in] dev 1505 * Pointer to the rte_eth_dev structure. 1506 * @param[in] item 1507 * Item specification. 1508 * @param[in] attr 1509 * Attributes of flow that includes this item. 1510 * @param[out] error 1511 * Pointer to error structure. 1512 * 1513 * @return 1514 * 0 on success, a negative errno value otherwise and rte_errno is set. 1515 */ 1516 static int 1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev, 1518 const struct rte_flow_item *item, 1519 const struct rte_flow_attr *attr __rte_unused, 1520 struct rte_flow_error *error) 1521 { 1522 const struct rte_flow_item_tag *spec = item->spec; 1523 const struct rte_flow_item_tag *mask = item->mask; 1524 const struct rte_flow_item_tag nic_mask = { 1525 .data = RTE_BE32(UINT32_MAX), 1526 .index = 0xff, 1527 }; 1528 int ret; 1529 1530 if (!mlx5_flow_ext_mreg_supported(dev)) 1531 return rte_flow_error_set(error, ENOTSUP, 1532 RTE_FLOW_ERROR_TYPE_ITEM, item, 1533 "extensive metadata register" 1534 " isn't supported"); 1535 if (!spec) 1536 return rte_flow_error_set(error, EINVAL, 1537 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1538 item->spec, 1539 "data cannot be empty"); 1540 if (!mask) 1541 mask = &rte_flow_item_tag_mask; 1542 if (!mask->data) 1543 return rte_flow_error_set(error, EINVAL, 1544 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1545 "mask cannot be zero"); 1546 1547 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1548 (const uint8_t *)&nic_mask, 1549 sizeof(struct rte_flow_item_tag), 1550 error); 1551 if (ret < 0) 1552 return ret; 1553 if (mask->index != 0xff) 1554 return rte_flow_error_set(error, EINVAL, 1555 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1556 "partial mask for tag index" 1557 " is not supported"); 1558 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error); 1559 if (ret < 0) 1560 return ret; 1561 MLX5_ASSERT(ret != REG_NON); 1562 return 0; 1563 } 1564 1565 /** 1566 * Validate vport item. 1567 * 1568 * @param[in] dev 1569 * Pointer to the rte_eth_dev structure. 1570 * @param[in] item 1571 * Item specification. 1572 * @param[in] attr 1573 * Attributes of flow that includes this item. 1574 * @param[in] item_flags 1575 * Bit-fields that holds the items detected until now. 1576 * @param[out] error 1577 * Pointer to error structure. 1578 * 1579 * @return 1580 * 0 on success, a negative errno value otherwise and rte_errno is set. 1581 */ 1582 static int 1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev, 1584 const struct rte_flow_item *item, 1585 const struct rte_flow_attr *attr, 1586 uint64_t item_flags, 1587 struct rte_flow_error *error) 1588 { 1589 const struct rte_flow_item_port_id *spec = item->spec; 1590 const struct rte_flow_item_port_id *mask = item->mask; 1591 const struct rte_flow_item_port_id switch_mask = { 1592 .id = 0xffffffff, 1593 }; 1594 struct mlx5_priv *esw_priv; 1595 struct mlx5_priv *dev_priv; 1596 int ret; 1597 1598 if (!attr->transfer) 1599 return rte_flow_error_set(error, EINVAL, 1600 RTE_FLOW_ERROR_TYPE_ITEM, 1601 NULL, 1602 "match on port id is valid only" 1603 " when transfer flag is enabled"); 1604 if (item_flags & MLX5_FLOW_ITEM_PORT_ID) 1605 return rte_flow_error_set(error, ENOTSUP, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "multiple source ports are not" 1608 " supported"); 1609 if (!mask) 1610 mask = &switch_mask; 1611 if (mask->id != 0xffffffff) 1612 return rte_flow_error_set(error, ENOTSUP, 1613 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 1614 mask, 1615 "no support for partial mask on" 1616 " \"id\" field"); 1617 ret = mlx5_flow_item_acceptable 1618 (item, (const uint8_t *)mask, 1619 (const uint8_t *)&rte_flow_item_port_id_mask, 1620 sizeof(struct rte_flow_item_port_id), 1621 error); 1622 if (ret) 1623 return ret; 1624 if (!spec) 1625 return 0; 1626 esw_priv = mlx5_port_to_eswitch_info(spec->id, false); 1627 if (!esw_priv) 1628 return rte_flow_error_set(error, rte_errno, 1629 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1630 "failed to obtain E-Switch info for" 1631 " port"); 1632 dev_priv = mlx5_dev_to_eswitch_info(dev); 1633 if (!dev_priv) 1634 return rte_flow_error_set(error, rte_errno, 1635 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1636 NULL, 1637 "failed to obtain E-Switch info"); 1638 if (esw_priv->domain_id != dev_priv->domain_id) 1639 return rte_flow_error_set(error, EINVAL, 1640 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1641 "cannot match on a port from a" 1642 " different E-Switch"); 1643 return 0; 1644 } 1645 1646 /** 1647 * Validate VLAN item. 1648 * 1649 * @param[in] item 1650 * Item specification. 1651 * @param[in] item_flags 1652 * Bit-fields that holds the items detected until now. 1653 * @param[in] dev 1654 * Ethernet device flow is being created on. 1655 * @param[out] error 1656 * Pointer to error structure. 1657 * 1658 * @return 1659 * 0 on success, a negative errno value otherwise and rte_errno is set. 1660 */ 1661 static int 1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item, 1663 uint64_t item_flags, 1664 struct rte_eth_dev *dev, 1665 struct rte_flow_error *error) 1666 { 1667 const struct rte_flow_item_vlan *mask = item->mask; 1668 const struct rte_flow_item_vlan nic_mask = { 1669 .tci = RTE_BE16(UINT16_MAX), 1670 .inner_type = RTE_BE16(UINT16_MAX), 1671 }; 1672 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1673 int ret; 1674 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1675 MLX5_FLOW_LAYER_INNER_L4) : 1676 (MLX5_FLOW_LAYER_OUTER_L3 | 1677 MLX5_FLOW_LAYER_OUTER_L4); 1678 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1679 MLX5_FLOW_LAYER_OUTER_VLAN; 1680 1681 if (item_flags & vlanm) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ITEM, item, 1684 "multiple VLAN layers not supported"); 1685 else if ((item_flags & l34m) != 0) 1686 return rte_flow_error_set(error, EINVAL, 1687 RTE_FLOW_ERROR_TYPE_ITEM, item, 1688 "VLAN cannot follow L3/L4 layer"); 1689 if (!mask) 1690 mask = &rte_flow_item_vlan_mask; 1691 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1692 (const uint8_t *)&nic_mask, 1693 sizeof(struct rte_flow_item_vlan), 1694 error); 1695 if (ret) 1696 return ret; 1697 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1698 struct mlx5_priv *priv = dev->data->dev_private; 1699 1700 if (priv->vmwa_context) { 1701 /* 1702 * Non-NULL context means we have a virtual machine 1703 * and SR-IOV enabled, we have to create VLAN interface 1704 * to make hypervisor to setup E-Switch vport 1705 * context correctly. We avoid creating the multiple 1706 * VLAN interfaces, so we cannot support VLAN tag mask. 1707 */ 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, 1710 item, 1711 "VLAN tag mask is not" 1712 " supported in virtual" 1713 " environment"); 1714 } 1715 } 1716 return 0; 1717 } 1718 1719 /* 1720 * GTP flags are contained in 1 byte of the format: 1721 * ------------------------------------------- 1722 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 | 1723 * |-----------------------------------------| 1724 * | value | Version | PT | Res | E | S | PN | 1725 * ------------------------------------------- 1726 * 1727 * Matching is supported only for GTP flags E, S, PN. 1728 */ 1729 #define MLX5_GTP_FLAGS_MASK 0x07 1730 1731 /** 1732 * Validate GTP item. 1733 * 1734 * @param[in] dev 1735 * Pointer to the rte_eth_dev structure. 1736 * @param[in] item 1737 * Item specification. 1738 * @param[in] item_flags 1739 * Bit-fields that holds the items detected until now. 1740 * @param[out] error 1741 * Pointer to error structure. 1742 * 1743 * @return 1744 * 0 on success, a negative errno value otherwise and rte_errno is set. 1745 */ 1746 static int 1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev, 1748 const struct rte_flow_item *item, 1749 uint64_t item_flags, 1750 struct rte_flow_error *error) 1751 { 1752 struct mlx5_priv *priv = dev->data->dev_private; 1753 const struct rte_flow_item_gtp *spec = item->spec; 1754 const struct rte_flow_item_gtp *mask = item->mask; 1755 const struct rte_flow_item_gtp nic_mask = { 1756 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK, 1757 .msg_type = 0xff, 1758 .teid = RTE_BE32(0xffffffff), 1759 }; 1760 1761 if (!priv->config.hca_attr.tunnel_stateless_gtp) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ITEM, item, 1764 "GTP support is not enabled"); 1765 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1766 return rte_flow_error_set(error, ENOTSUP, 1767 RTE_FLOW_ERROR_TYPE_ITEM, item, 1768 "multiple tunnel layers not" 1769 " supported"); 1770 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ITEM, item, 1773 "no outer UDP layer found"); 1774 if (!mask) 1775 mask = &rte_flow_item_gtp_mask; 1776 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK) 1777 return rte_flow_error_set(error, ENOTSUP, 1778 RTE_FLOW_ERROR_TYPE_ITEM, item, 1779 "Match is supported for GTP" 1780 " flags only"); 1781 return mlx5_flow_item_acceptable 1782 (item, (const uint8_t *)mask, 1783 (const uint8_t *)&nic_mask, 1784 sizeof(struct rte_flow_item_gtp), 1785 error); 1786 } 1787 1788 /** 1789 * Validate the pop VLAN action. 1790 * 1791 * @param[in] dev 1792 * Pointer to the rte_eth_dev structure. 1793 * @param[in] action_flags 1794 * Holds the actions detected until now. 1795 * @param[in] action 1796 * Pointer to the pop vlan action. 1797 * @param[in] item_flags 1798 * The items found in this flow rule. 1799 * @param[in] attr 1800 * Pointer to flow attributes. 1801 * @param[out] error 1802 * Pointer to error structure. 1803 * 1804 * @return 1805 * 0 on success, a negative errno value otherwise and rte_errno is set. 1806 */ 1807 static int 1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, 1809 uint64_t action_flags, 1810 const struct rte_flow_action *action, 1811 uint64_t item_flags, 1812 const struct rte_flow_attr *attr, 1813 struct rte_flow_error *error) 1814 { 1815 const struct mlx5_priv *priv = dev->data->dev_private; 1816 1817 (void)action; 1818 (void)attr; 1819 if (!priv->sh->pop_vlan_action) 1820 return rte_flow_error_set(error, ENOTSUP, 1821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1822 NULL, 1823 "pop vlan action is not supported"); 1824 if (attr->egress) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 1827 NULL, 1828 "pop vlan action not supported for " 1829 "egress"); 1830 if (action_flags & MLX5_FLOW_VLAN_ACTIONS) 1831 return rte_flow_error_set(error, ENOTSUP, 1832 RTE_FLOW_ERROR_TYPE_ACTION, action, 1833 "no support for multiple VLAN " 1834 "actions"); 1835 /* Pop VLAN with preceding Decap requires inner header with VLAN. */ 1836 if ((action_flags & MLX5_FLOW_ACTION_DECAP) && 1837 !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN)) 1838 return rte_flow_error_set(error, ENOTSUP, 1839 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1840 NULL, 1841 "cannot pop vlan after decap without " 1842 "match on inner vlan in the flow"); 1843 /* Pop VLAN without preceding Decap requires outer header with VLAN. */ 1844 if (!(action_flags & MLX5_FLOW_ACTION_DECAP) && 1845 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 1846 return rte_flow_error_set(error, ENOTSUP, 1847 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1848 NULL, 1849 "cannot pop vlan without a " 1850 "match on (outer) vlan in the flow"); 1851 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1852 return rte_flow_error_set(error, EINVAL, 1853 RTE_FLOW_ERROR_TYPE_ACTION, action, 1854 "wrong action order, port_id should " 1855 "be after pop VLAN action"); 1856 if (!attr->transfer && priv->representor) 1857 return rte_flow_error_set(error, ENOTSUP, 1858 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1859 "pop vlan action for VF representor " 1860 "not supported on NIC table"); 1861 return 0; 1862 } 1863 1864 /** 1865 * Get VLAN default info from vlan match info. 1866 * 1867 * @param[in] items 1868 * the list of item specifications. 1869 * @param[out] vlan 1870 * pointer VLAN info to fill to. 1871 * 1872 * @return 1873 * 0 on success, a negative errno value otherwise and rte_errno is set. 1874 */ 1875 static void 1876 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, 1877 struct rte_vlan_hdr *vlan) 1878 { 1879 const struct rte_flow_item_vlan nic_mask = { 1880 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK | 1881 MLX5DV_FLOW_VLAN_VID_MASK), 1882 .inner_type = RTE_BE16(0xffff), 1883 }; 1884 1885 if (items == NULL) 1886 return; 1887 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1888 int type = items->type; 1889 1890 if (type == RTE_FLOW_ITEM_TYPE_VLAN || 1891 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 1892 break; 1893 } 1894 if (items->type != RTE_FLOW_ITEM_TYPE_END) { 1895 const struct rte_flow_item_vlan *vlan_m = items->mask; 1896 const struct rte_flow_item_vlan *vlan_v = items->spec; 1897 1898 /* If VLAN item in pattern doesn't contain data, return here. */ 1899 if (!vlan_v) 1900 return; 1901 if (!vlan_m) 1902 vlan_m = &nic_mask; 1903 /* Only full match values are accepted */ 1904 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) == 1905 MLX5DV_FLOW_VLAN_PCP_MASK_BE) { 1906 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 1907 vlan->vlan_tci |= 1908 rte_be_to_cpu_16(vlan_v->tci & 1909 MLX5DV_FLOW_VLAN_PCP_MASK_BE); 1910 } 1911 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) == 1912 MLX5DV_FLOW_VLAN_VID_MASK_BE) { 1913 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 1914 vlan->vlan_tci |= 1915 rte_be_to_cpu_16(vlan_v->tci & 1916 MLX5DV_FLOW_VLAN_VID_MASK_BE); 1917 } 1918 if (vlan_m->inner_type == nic_mask.inner_type) 1919 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type & 1920 vlan_m->inner_type); 1921 } 1922 } 1923 1924 /** 1925 * Validate the push VLAN action. 1926 * 1927 * @param[in] dev 1928 * Pointer to the rte_eth_dev structure. 1929 * @param[in] action_flags 1930 * Holds the actions detected until now. 1931 * @param[in] item_flags 1932 * The items found in this flow rule. 1933 * @param[in] action 1934 * Pointer to the action structure. 1935 * @param[in] attr 1936 * Pointer to flow attributes 1937 * @param[out] error 1938 * Pointer to error structure. 1939 * 1940 * @return 1941 * 0 on success, a negative errno value otherwise and rte_errno is set. 1942 */ 1943 static int 1944 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, 1945 uint64_t action_flags, 1946 const struct rte_flow_item_vlan *vlan_m, 1947 const struct rte_flow_action *action, 1948 const struct rte_flow_attr *attr, 1949 struct rte_flow_error *error) 1950 { 1951 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; 1952 const struct mlx5_priv *priv = dev->data->dev_private; 1953 1954 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && 1955 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ACTION, action, 1958 "invalid vlan ethertype"); 1959 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1960 return rte_flow_error_set(error, EINVAL, 1961 RTE_FLOW_ERROR_TYPE_ACTION, action, 1962 "wrong action order, port_id should " 1963 "be after push VLAN"); 1964 if (!attr->transfer && priv->representor) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1967 "push vlan action for VF representor " 1968 "not supported on NIC table"); 1969 if (vlan_m && 1970 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) && 1971 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) != 1972 MLX5DV_FLOW_VLAN_PCP_MASK_BE && 1973 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) && 1974 !(mlx5_flow_find_action 1975 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP))) 1976 return rte_flow_error_set(error, EINVAL, 1977 RTE_FLOW_ERROR_TYPE_ACTION, action, 1978 "not full match mask on VLAN PCP and " 1979 "there is no of_set_vlan_pcp action, " 1980 "push VLAN action cannot figure out " 1981 "PCP value"); 1982 if (vlan_m && 1983 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) && 1984 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) != 1985 MLX5DV_FLOW_VLAN_VID_MASK_BE && 1986 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) && 1987 !(mlx5_flow_find_action 1988 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))) 1989 return rte_flow_error_set(error, EINVAL, 1990 RTE_FLOW_ERROR_TYPE_ACTION, action, 1991 "not full match mask on VLAN VID and " 1992 "there is no of_set_vlan_vid action, " 1993 "push VLAN action cannot figure out " 1994 "VID value"); 1995 (void)attr; 1996 return 0; 1997 } 1998 1999 /** 2000 * Validate the set VLAN PCP. 2001 * 2002 * @param[in] action_flags 2003 * Holds the actions detected until now. 2004 * @param[in] actions 2005 * Pointer to the list of actions remaining in the flow rule. 2006 * @param[out] error 2007 * Pointer to error structure. 2008 * 2009 * @return 2010 * 0 on success, a negative errno value otherwise and rte_errno is set. 2011 */ 2012 static int 2013 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, 2014 const struct rte_flow_action actions[], 2015 struct rte_flow_error *error) 2016 { 2017 const struct rte_flow_action *action = actions; 2018 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; 2019 2020 if (conf->vlan_pcp > 7) 2021 return rte_flow_error_set(error, EINVAL, 2022 RTE_FLOW_ERROR_TYPE_ACTION, action, 2023 "VLAN PCP value is too big"); 2024 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) 2025 return rte_flow_error_set(error, ENOTSUP, 2026 RTE_FLOW_ERROR_TYPE_ACTION, action, 2027 "set VLAN PCP action must follow " 2028 "the push VLAN action"); 2029 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) 2030 return rte_flow_error_set(error, ENOTSUP, 2031 RTE_FLOW_ERROR_TYPE_ACTION, action, 2032 "Multiple VLAN PCP modification are " 2033 "not supported"); 2034 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ACTION, action, 2037 "wrong action order, port_id should " 2038 "be after set VLAN PCP"); 2039 return 0; 2040 } 2041 2042 /** 2043 * Validate the set VLAN VID. 2044 * 2045 * @param[in] item_flags 2046 * Holds the items detected in this rule. 2047 * @param[in] action_flags 2048 * Holds the actions detected until now. 2049 * @param[in] actions 2050 * Pointer to the list of actions remaining in the flow rule. 2051 * @param[out] error 2052 * Pointer to error structure. 2053 * 2054 * @return 2055 * 0 on success, a negative errno value otherwise and rte_errno is set. 2056 */ 2057 static int 2058 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, 2059 uint64_t action_flags, 2060 const struct rte_flow_action actions[], 2061 struct rte_flow_error *error) 2062 { 2063 const struct rte_flow_action *action = actions; 2064 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; 2065 2066 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE) 2067 return rte_flow_error_set(error, EINVAL, 2068 RTE_FLOW_ERROR_TYPE_ACTION, action, 2069 "VLAN VID value is too big"); 2070 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && 2071 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 2072 return rte_flow_error_set(error, ENOTSUP, 2073 RTE_FLOW_ERROR_TYPE_ACTION, action, 2074 "set VLAN VID action must follow push" 2075 " VLAN action or match on VLAN item"); 2076 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) 2077 return rte_flow_error_set(error, ENOTSUP, 2078 RTE_FLOW_ERROR_TYPE_ACTION, action, 2079 "Multiple VLAN VID modifications are " 2080 "not supported"); 2081 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2082 return rte_flow_error_set(error, EINVAL, 2083 RTE_FLOW_ERROR_TYPE_ACTION, action, 2084 "wrong action order, port_id should " 2085 "be after set VLAN VID"); 2086 return 0; 2087 } 2088 2089 /* 2090 * Validate the FLAG action. 2091 * 2092 * @param[in] dev 2093 * Pointer to the rte_eth_dev structure. 2094 * @param[in] action_flags 2095 * Holds the actions detected until now. 2096 * @param[in] attr 2097 * Pointer to flow attributes 2098 * @param[out] error 2099 * Pointer to error structure. 2100 * 2101 * @return 2102 * 0 on success, a negative errno value otherwise and rte_errno is set. 2103 */ 2104 static int 2105 flow_dv_validate_action_flag(struct rte_eth_dev *dev, 2106 uint64_t action_flags, 2107 const struct rte_flow_attr *attr, 2108 struct rte_flow_error *error) 2109 { 2110 struct mlx5_priv *priv = dev->data->dev_private; 2111 struct mlx5_dev_config *config = &priv->config; 2112 int ret; 2113 2114 /* Fall back if no extended metadata register support. */ 2115 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2116 return mlx5_flow_validate_action_flag(action_flags, attr, 2117 error); 2118 /* Extensive metadata mode requires registers. */ 2119 if (!mlx5_flow_ext_mreg_supported(dev)) 2120 return rte_flow_error_set(error, ENOTSUP, 2121 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2122 "no metadata registers " 2123 "to support flag action"); 2124 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT)) 2125 return rte_flow_error_set(error, ENOTSUP, 2126 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2127 "extended metadata register" 2128 " isn't available"); 2129 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2130 if (ret < 0) 2131 return ret; 2132 MLX5_ASSERT(ret > 0); 2133 if (action_flags & MLX5_FLOW_ACTION_MARK) 2134 return rte_flow_error_set(error, EINVAL, 2135 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2136 "can't mark and flag in same flow"); 2137 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2138 return rte_flow_error_set(error, EINVAL, 2139 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2140 "can't have 2 flag" 2141 " actions in same flow"); 2142 return 0; 2143 } 2144 2145 /** 2146 * Validate MARK action. 2147 * 2148 * @param[in] dev 2149 * Pointer to the rte_eth_dev structure. 2150 * @param[in] action 2151 * Pointer to action. 2152 * @param[in] action_flags 2153 * Holds the actions detected until now. 2154 * @param[in] attr 2155 * Pointer to flow attributes 2156 * @param[out] error 2157 * Pointer to error structure. 2158 * 2159 * @return 2160 * 0 on success, a negative errno value otherwise and rte_errno is set. 2161 */ 2162 static int 2163 flow_dv_validate_action_mark(struct rte_eth_dev *dev, 2164 const struct rte_flow_action *action, 2165 uint64_t action_flags, 2166 const struct rte_flow_attr *attr, 2167 struct rte_flow_error *error) 2168 { 2169 struct mlx5_priv *priv = dev->data->dev_private; 2170 struct mlx5_dev_config *config = &priv->config; 2171 const struct rte_flow_action_mark *mark = action->conf; 2172 int ret; 2173 2174 /* Fall back if no extended metadata register support. */ 2175 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2176 return mlx5_flow_validate_action_mark(action, action_flags, 2177 attr, error); 2178 /* Extensive metadata mode requires registers. */ 2179 if (!mlx5_flow_ext_mreg_supported(dev)) 2180 return rte_flow_error_set(error, ENOTSUP, 2181 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2182 "no metadata registers " 2183 "to support mark action"); 2184 if (!priv->sh->dv_mark_mask) 2185 return rte_flow_error_set(error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2187 "extended metadata register" 2188 " isn't available"); 2189 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2190 if (ret < 0) 2191 return ret; 2192 MLX5_ASSERT(ret > 0); 2193 if (!mark) 2194 return rte_flow_error_set(error, EINVAL, 2195 RTE_FLOW_ERROR_TYPE_ACTION, action, 2196 "configuration cannot be null"); 2197 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask)) 2198 return rte_flow_error_set(error, EINVAL, 2199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2200 &mark->id, 2201 "mark id exceeds the limit"); 2202 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2203 return rte_flow_error_set(error, EINVAL, 2204 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2205 "can't flag and mark in same flow"); 2206 if (action_flags & MLX5_FLOW_ACTION_MARK) 2207 return rte_flow_error_set(error, EINVAL, 2208 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2209 "can't have 2 mark actions in same" 2210 " flow"); 2211 return 0; 2212 } 2213 2214 /** 2215 * Validate SET_META action. 2216 * 2217 * @param[in] dev 2218 * Pointer to the rte_eth_dev structure. 2219 * @param[in] action 2220 * Pointer to the action structure. 2221 * @param[in] action_flags 2222 * Holds the actions detected until now. 2223 * @param[in] attr 2224 * Pointer to flow attributes 2225 * @param[out] error 2226 * Pointer to error structure. 2227 * 2228 * @return 2229 * 0 on success, a negative errno value otherwise and rte_errno is set. 2230 */ 2231 static int 2232 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, 2233 const struct rte_flow_action *action, 2234 uint64_t action_flags __rte_unused, 2235 const struct rte_flow_attr *attr, 2236 struct rte_flow_error *error) 2237 { 2238 const struct rte_flow_action_set_meta *conf; 2239 uint32_t nic_mask = UINT32_MAX; 2240 int reg; 2241 2242 if (!mlx5_flow_ext_mreg_supported(dev)) 2243 return rte_flow_error_set(error, ENOTSUP, 2244 RTE_FLOW_ERROR_TYPE_ACTION, action, 2245 "extended metadata register" 2246 " isn't supported"); 2247 reg = flow_dv_get_metadata_reg(dev, attr, error); 2248 if (reg < 0) 2249 return reg; 2250 if (reg != REG_A && reg != REG_B) { 2251 struct mlx5_priv *priv = dev->data->dev_private; 2252 2253 nic_mask = priv->sh->dv_meta_mask; 2254 } 2255 if (!(action->conf)) 2256 return rte_flow_error_set(error, EINVAL, 2257 RTE_FLOW_ERROR_TYPE_ACTION, action, 2258 "configuration cannot be null"); 2259 conf = (const struct rte_flow_action_set_meta *)action->conf; 2260 if (!conf->mask) 2261 return rte_flow_error_set(error, EINVAL, 2262 RTE_FLOW_ERROR_TYPE_ACTION, action, 2263 "zero mask doesn't have any effect"); 2264 if (conf->mask & ~nic_mask) 2265 return rte_flow_error_set(error, EINVAL, 2266 RTE_FLOW_ERROR_TYPE_ACTION, action, 2267 "meta data must be within reg C0"); 2268 return 0; 2269 } 2270 2271 /** 2272 * Validate SET_TAG action. 2273 * 2274 * @param[in] dev 2275 * Pointer to the rte_eth_dev structure. 2276 * @param[in] action 2277 * Pointer to the action structure. 2278 * @param[in] action_flags 2279 * Holds the actions detected until now. 2280 * @param[in] attr 2281 * Pointer to flow attributes 2282 * @param[out] error 2283 * Pointer to error structure. 2284 * 2285 * @return 2286 * 0 on success, a negative errno value otherwise and rte_errno is set. 2287 */ 2288 static int 2289 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, 2290 const struct rte_flow_action *action, 2291 uint64_t action_flags, 2292 const struct rte_flow_attr *attr, 2293 struct rte_flow_error *error) 2294 { 2295 const struct rte_flow_action_set_tag *conf; 2296 const uint64_t terminal_action_flags = 2297 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | 2298 MLX5_FLOW_ACTION_RSS; 2299 int ret; 2300 2301 if (!mlx5_flow_ext_mreg_supported(dev)) 2302 return rte_flow_error_set(error, ENOTSUP, 2303 RTE_FLOW_ERROR_TYPE_ACTION, action, 2304 "extensive metadata register" 2305 " isn't supported"); 2306 if (!(action->conf)) 2307 return rte_flow_error_set(error, EINVAL, 2308 RTE_FLOW_ERROR_TYPE_ACTION, action, 2309 "configuration cannot be null"); 2310 conf = (const struct rte_flow_action_set_tag *)action->conf; 2311 if (!conf->mask) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ACTION, action, 2314 "zero mask doesn't have any effect"); 2315 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 2316 if (ret < 0) 2317 return ret; 2318 if (!attr->transfer && attr->ingress && 2319 (action_flags & terminal_action_flags)) 2320 return rte_flow_error_set(error, EINVAL, 2321 RTE_FLOW_ERROR_TYPE_ACTION, action, 2322 "set_tag has no effect" 2323 " with terminal actions"); 2324 return 0; 2325 } 2326 2327 /** 2328 * Validate count action. 2329 * 2330 * @param[in] dev 2331 * Pointer to rte_eth_dev structure. 2332 * @param[out] error 2333 * Pointer to error structure. 2334 * 2335 * @return 2336 * 0 on success, a negative errno value otherwise and rte_errno is set. 2337 */ 2338 static int 2339 flow_dv_validate_action_count(struct rte_eth_dev *dev, 2340 struct rte_flow_error *error) 2341 { 2342 struct mlx5_priv *priv = dev->data->dev_private; 2343 2344 if (!priv->config.devx) 2345 goto notsup_err; 2346 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS 2347 return 0; 2348 #endif 2349 notsup_err: 2350 return rte_flow_error_set 2351 (error, ENOTSUP, 2352 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2353 NULL, 2354 "count action not supported"); 2355 } 2356 2357 /** 2358 * Validate the L2 encap action. 2359 * 2360 * @param[in] dev 2361 * Pointer to the rte_eth_dev structure. 2362 * @param[in] action_flags 2363 * Holds the actions detected until now. 2364 * @param[in] action 2365 * Pointer to the action structure. 2366 * @param[in] attr 2367 * Pointer to flow attributes. 2368 * @param[out] error 2369 * Pointer to error structure. 2370 * 2371 * @return 2372 * 0 on success, a negative errno value otherwise and rte_errno is set. 2373 */ 2374 static int 2375 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, 2376 uint64_t action_flags, 2377 const struct rte_flow_action *action, 2378 const struct rte_flow_attr *attr, 2379 struct rte_flow_error *error) 2380 { 2381 const struct mlx5_priv *priv = dev->data->dev_private; 2382 2383 if (!(action->conf)) 2384 return rte_flow_error_set(error, EINVAL, 2385 RTE_FLOW_ERROR_TYPE_ACTION, action, 2386 "configuration cannot be null"); 2387 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 2388 return rte_flow_error_set(error, EINVAL, 2389 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2390 "can only have a single encap action " 2391 "in a flow"); 2392 if (!attr->transfer && priv->representor) 2393 return rte_flow_error_set(error, ENOTSUP, 2394 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2395 "encap action for VF representor " 2396 "not supported on NIC table"); 2397 return 0; 2398 } 2399 2400 /** 2401 * Validate a decap action. 2402 * 2403 * @param[in] dev 2404 * Pointer to the rte_eth_dev structure. 2405 * @param[in] action_flags 2406 * Holds the actions detected until now. 2407 * @param[in] attr 2408 * Pointer to flow attributes 2409 * @param[out] error 2410 * Pointer to error structure. 2411 * 2412 * @return 2413 * 0 on success, a negative errno value otherwise and rte_errno is set. 2414 */ 2415 static int 2416 flow_dv_validate_action_decap(struct rte_eth_dev *dev, 2417 uint64_t action_flags, 2418 const struct rte_flow_attr *attr, 2419 struct rte_flow_error *error) 2420 { 2421 const struct mlx5_priv *priv = dev->data->dev_private; 2422 2423 if (priv->config.hca_attr.scatter_fcs_w_decap_disable && 2424 !priv->config.decap_en) 2425 return rte_flow_error_set(error, ENOTSUP, 2426 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2427 "decap is not enabled"); 2428 if (action_flags & MLX5_FLOW_XCAP_ACTIONS) 2429 return rte_flow_error_set(error, ENOTSUP, 2430 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2431 action_flags & 2432 MLX5_FLOW_ACTION_DECAP ? "can only " 2433 "have a single decap action" : "decap " 2434 "after encap is not supported"); 2435 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 2436 return rte_flow_error_set(error, EINVAL, 2437 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2438 "can't have decap action after" 2439 " modify action"); 2440 if (attr->egress) 2441 return rte_flow_error_set(error, ENOTSUP, 2442 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 2443 NULL, 2444 "decap action not supported for " 2445 "egress"); 2446 if (!attr->transfer && priv->representor) 2447 return rte_flow_error_set(error, ENOTSUP, 2448 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2449 "decap action for VF representor " 2450 "not supported on NIC table"); 2451 return 0; 2452 } 2453 2454 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; 2455 2456 /** 2457 * Validate the raw encap and decap actions. 2458 * 2459 * @param[in] dev 2460 * Pointer to the rte_eth_dev structure. 2461 * @param[in] decap 2462 * Pointer to the decap action. 2463 * @param[in] encap 2464 * Pointer to the encap action. 2465 * @param[in] attr 2466 * Pointer to flow attributes 2467 * @param[in/out] action_flags 2468 * Holds the actions detected until now. 2469 * @param[out] actions_n 2470 * pointer to the number of actions counter. 2471 * @param[out] error 2472 * Pointer to error structure. 2473 * 2474 * @return 2475 * 0 on success, a negative errno value otherwise and rte_errno is set. 2476 */ 2477 static int 2478 flow_dv_validate_action_raw_encap_decap 2479 (struct rte_eth_dev *dev, 2480 const struct rte_flow_action_raw_decap *decap, 2481 const struct rte_flow_action_raw_encap *encap, 2482 const struct rte_flow_attr *attr, uint64_t *action_flags, 2483 int *actions_n, struct rte_flow_error *error) 2484 { 2485 const struct mlx5_priv *priv = dev->data->dev_private; 2486 int ret; 2487 2488 if (encap && (!encap->size || !encap->data)) 2489 return rte_flow_error_set(error, EINVAL, 2490 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2491 "raw encap data cannot be empty"); 2492 if (decap && encap) { 2493 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE && 2494 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2495 /* L3 encap. */ 2496 decap = NULL; 2497 else if (encap->size <= 2498 MLX5_ENCAPSULATION_DECISION_SIZE && 2499 decap->size > 2500 MLX5_ENCAPSULATION_DECISION_SIZE) 2501 /* L3 decap. */ 2502 encap = NULL; 2503 else if (encap->size > 2504 MLX5_ENCAPSULATION_DECISION_SIZE && 2505 decap->size > 2506 MLX5_ENCAPSULATION_DECISION_SIZE) 2507 /* 2 L2 actions: encap and decap. */ 2508 ; 2509 else 2510 return rte_flow_error_set(error, 2511 ENOTSUP, 2512 RTE_FLOW_ERROR_TYPE_ACTION, 2513 NULL, "unsupported too small " 2514 "raw decap and too small raw " 2515 "encap combination"); 2516 } 2517 if (decap) { 2518 ret = flow_dv_validate_action_decap(dev, *action_flags, attr, 2519 error); 2520 if (ret < 0) 2521 return ret; 2522 *action_flags |= MLX5_FLOW_ACTION_DECAP; 2523 ++(*actions_n); 2524 } 2525 if (encap) { 2526 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE) 2527 return rte_flow_error_set(error, ENOTSUP, 2528 RTE_FLOW_ERROR_TYPE_ACTION, 2529 NULL, 2530 "small raw encap size"); 2531 if (*action_flags & MLX5_FLOW_ACTION_ENCAP) 2532 return rte_flow_error_set(error, EINVAL, 2533 RTE_FLOW_ERROR_TYPE_ACTION, 2534 NULL, 2535 "more than one encap action"); 2536 if (!attr->transfer && priv->representor) 2537 return rte_flow_error_set 2538 (error, ENOTSUP, 2539 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2540 "encap action for VF representor " 2541 "not supported on NIC table"); 2542 *action_flags |= MLX5_FLOW_ACTION_ENCAP; 2543 ++(*actions_n); 2544 } 2545 return 0; 2546 } 2547 2548 /** 2549 * Find existing encap/decap resource or create and register a new one. 2550 * 2551 * @param[in, out] dev 2552 * Pointer to rte_eth_dev structure. 2553 * @param[in, out] resource 2554 * Pointer to encap/decap resource. 2555 * @parm[in, out] dev_flow 2556 * Pointer to the dev_flow. 2557 * @param[out] error 2558 * pointer to error structure. 2559 * 2560 * @return 2561 * 0 on success otherwise -errno and errno is set. 2562 */ 2563 static int 2564 flow_dv_encap_decap_resource_register 2565 (struct rte_eth_dev *dev, 2566 struct mlx5_flow_dv_encap_decap_resource *resource, 2567 struct mlx5_flow *dev_flow, 2568 struct rte_flow_error *error) 2569 { 2570 struct mlx5_priv *priv = dev->data->dev_private; 2571 struct mlx5_dev_ctx_shared *sh = priv->sh; 2572 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 2573 struct mlx5dv_dr_domain *domain; 2574 uint32_t idx = 0; 2575 int ret; 2576 2577 resource->flags = dev_flow->dv.group ? 0 : 1; 2578 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2579 domain = sh->fdb_domain; 2580 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2581 domain = sh->rx_domain; 2582 else 2583 domain = sh->tx_domain; 2584 /* Lookup a matching resource from cache. */ 2585 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx, 2586 cache_resource, next) { 2587 if (resource->reformat_type == cache_resource->reformat_type && 2588 resource->ft_type == cache_resource->ft_type && 2589 resource->flags == cache_resource->flags && 2590 resource->size == cache_resource->size && 2591 !memcmp((const void *)resource->buf, 2592 (const void *)cache_resource->buf, 2593 resource->size)) { 2594 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", 2595 (void *)cache_resource, 2596 rte_atomic32_read(&cache_resource->refcnt)); 2597 rte_atomic32_inc(&cache_resource->refcnt); 2598 dev_flow->handle->dvh.rix_encap_decap = idx; 2599 dev_flow->dv.encap_decap = cache_resource; 2600 return 0; 2601 } 2602 } 2603 /* Register new encap/decap resource. */ 2604 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 2605 &dev_flow->handle->dvh.rix_encap_decap); 2606 if (!cache_resource) 2607 return rte_flow_error_set(error, ENOMEM, 2608 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2609 "cannot allocate resource memory"); 2610 *cache_resource = *resource; 2611 ret = mlx5_flow_os_create_flow_action_packet_reformat 2612 (sh->ctx, domain, cache_resource, 2613 &cache_resource->action); 2614 if (ret) { 2615 mlx5_free(cache_resource); 2616 return rte_flow_error_set(error, ENOMEM, 2617 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2618 NULL, "cannot create action"); 2619 } 2620 rte_atomic32_init(&cache_resource->refcnt); 2621 rte_atomic32_inc(&cache_resource->refcnt); 2622 ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps, 2623 dev_flow->handle->dvh.rix_encap_decap, cache_resource, 2624 next); 2625 dev_flow->dv.encap_decap = cache_resource; 2626 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", 2627 (void *)cache_resource, 2628 rte_atomic32_read(&cache_resource->refcnt)); 2629 return 0; 2630 } 2631 2632 /** 2633 * Find existing table jump resource or create and register a new one. 2634 * 2635 * @param[in, out] dev 2636 * Pointer to rte_eth_dev structure. 2637 * @param[in, out] tbl 2638 * Pointer to flow table resource. 2639 * @parm[in, out] dev_flow 2640 * Pointer to the dev_flow. 2641 * @param[out] error 2642 * pointer to error structure. 2643 * 2644 * @return 2645 * 0 on success otherwise -errno and errno is set. 2646 */ 2647 static int 2648 flow_dv_jump_tbl_resource_register 2649 (struct rte_eth_dev *dev __rte_unused, 2650 struct mlx5_flow_tbl_resource *tbl, 2651 struct mlx5_flow *dev_flow, 2652 struct rte_flow_error *error) 2653 { 2654 struct mlx5_flow_tbl_data_entry *tbl_data = 2655 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 2656 int cnt, ret; 2657 2658 MLX5_ASSERT(tbl); 2659 cnt = rte_atomic32_read(&tbl_data->jump.refcnt); 2660 if (!cnt) { 2661 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl 2662 (tbl->obj, &tbl_data->jump.action); 2663 if (ret) 2664 return rte_flow_error_set(error, ENOMEM, 2665 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2666 NULL, "cannot create jump action"); 2667 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", 2668 (void *)&tbl_data->jump, cnt); 2669 } else { 2670 /* old jump should not make the table ref++. */ 2671 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 2672 MLX5_ASSERT(tbl_data->jump.action); 2673 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", 2674 (void *)&tbl_data->jump, cnt); 2675 } 2676 rte_atomic32_inc(&tbl_data->jump.refcnt); 2677 dev_flow->handle->rix_jump = tbl_data->idx; 2678 dev_flow->dv.jump = &tbl_data->jump; 2679 return 0; 2680 } 2681 2682 /** 2683 * Find existing default miss resource or create and register a new one. 2684 * 2685 * @param[in, out] dev 2686 * Pointer to rte_eth_dev structure. 2687 * @param[out] error 2688 * pointer to error structure. 2689 * 2690 * @return 2691 * 0 on success otherwise -errno and errno is set. 2692 */ 2693 static int 2694 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev, 2695 struct rte_flow_error *error) 2696 { 2697 struct mlx5_priv *priv = dev->data->dev_private; 2698 struct mlx5_dev_ctx_shared *sh = priv->sh; 2699 struct mlx5_flow_default_miss_resource *cache_resource = 2700 &sh->default_miss; 2701 int cnt = rte_atomic32_read(&cache_resource->refcnt); 2702 2703 if (!cnt) { 2704 MLX5_ASSERT(cache_resource->action); 2705 cache_resource->action = 2706 mlx5_glue->dr_create_flow_action_default_miss(); 2707 if (!cache_resource->action) 2708 return rte_flow_error_set(error, ENOMEM, 2709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2710 "cannot create default miss action"); 2711 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++", 2712 (void *)cache_resource->action, cnt); 2713 } 2714 rte_atomic32_inc(&cache_resource->refcnt); 2715 return 0; 2716 } 2717 2718 /** 2719 * Find existing table port ID resource or create and register a new one. 2720 * 2721 * @param[in, out] dev 2722 * Pointer to rte_eth_dev structure. 2723 * @param[in, out] resource 2724 * Pointer to port ID action resource. 2725 * @parm[in, out] dev_flow 2726 * Pointer to the dev_flow. 2727 * @param[out] error 2728 * pointer to error structure. 2729 * 2730 * @return 2731 * 0 on success otherwise -errno and errno is set. 2732 */ 2733 static int 2734 flow_dv_port_id_action_resource_register 2735 (struct rte_eth_dev *dev, 2736 struct mlx5_flow_dv_port_id_action_resource *resource, 2737 struct mlx5_flow *dev_flow, 2738 struct rte_flow_error *error) 2739 { 2740 struct mlx5_priv *priv = dev->data->dev_private; 2741 struct mlx5_dev_ctx_shared *sh = priv->sh; 2742 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 2743 uint32_t idx = 0; 2744 int ret; 2745 2746 /* Lookup a matching resource from cache. */ 2747 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, 2748 idx, cache_resource, next) { 2749 if (resource->port_id == cache_resource->port_id) { 2750 DRV_LOG(DEBUG, "port id action resource resource %p: " 2751 "refcnt %d++", 2752 (void *)cache_resource, 2753 rte_atomic32_read(&cache_resource->refcnt)); 2754 rte_atomic32_inc(&cache_resource->refcnt); 2755 dev_flow->handle->rix_port_id_action = idx; 2756 dev_flow->dv.port_id_action = cache_resource; 2757 return 0; 2758 } 2759 } 2760 /* Register new port id action resource. */ 2761 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], 2762 &dev_flow->handle->rix_port_id_action); 2763 if (!cache_resource) 2764 return rte_flow_error_set(error, ENOMEM, 2765 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2766 "cannot allocate resource memory"); 2767 *cache_resource = *resource; 2768 ret = mlx5_flow_os_create_flow_action_dest_port 2769 (priv->sh->fdb_domain, resource->port_id, 2770 &cache_resource->action); 2771 if (ret) { 2772 mlx5_free(cache_resource); 2773 return rte_flow_error_set(error, ENOMEM, 2774 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2775 NULL, "cannot create action"); 2776 } 2777 rte_atomic32_init(&cache_resource->refcnt); 2778 rte_atomic32_inc(&cache_resource->refcnt); 2779 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, 2780 dev_flow->handle->rix_port_id_action, cache_resource, 2781 next); 2782 dev_flow->dv.port_id_action = cache_resource; 2783 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", 2784 (void *)cache_resource, 2785 rte_atomic32_read(&cache_resource->refcnt)); 2786 return 0; 2787 } 2788 2789 /** 2790 * Find existing push vlan resource or create and register a new one. 2791 * 2792 * @param [in, out] dev 2793 * Pointer to rte_eth_dev structure. 2794 * @param[in, out] resource 2795 * Pointer to port ID action resource. 2796 * @parm[in, out] dev_flow 2797 * Pointer to the dev_flow. 2798 * @param[out] error 2799 * pointer to error structure. 2800 * 2801 * @return 2802 * 0 on success otherwise -errno and errno is set. 2803 */ 2804 static int 2805 flow_dv_push_vlan_action_resource_register 2806 (struct rte_eth_dev *dev, 2807 struct mlx5_flow_dv_push_vlan_action_resource *resource, 2808 struct mlx5_flow *dev_flow, 2809 struct rte_flow_error *error) 2810 { 2811 struct mlx5_priv *priv = dev->data->dev_private; 2812 struct mlx5_dev_ctx_shared *sh = priv->sh; 2813 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 2814 struct mlx5dv_dr_domain *domain; 2815 uint32_t idx = 0; 2816 int ret; 2817 2818 /* Lookup a matching resource from cache. */ 2819 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2820 sh->push_vlan_action_list, idx, cache_resource, next) { 2821 if (resource->vlan_tag == cache_resource->vlan_tag && 2822 resource->ft_type == cache_resource->ft_type) { 2823 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: " 2824 "refcnt %d++", 2825 (void *)cache_resource, 2826 rte_atomic32_read(&cache_resource->refcnt)); 2827 rte_atomic32_inc(&cache_resource->refcnt); 2828 dev_flow->handle->dvh.rix_push_vlan = idx; 2829 dev_flow->dv.push_vlan_res = cache_resource; 2830 return 0; 2831 } 2832 } 2833 /* Register new push_vlan action resource. */ 2834 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2835 &dev_flow->handle->dvh.rix_push_vlan); 2836 if (!cache_resource) 2837 return rte_flow_error_set(error, ENOMEM, 2838 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2839 "cannot allocate resource memory"); 2840 *cache_resource = *resource; 2841 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2842 domain = sh->fdb_domain; 2843 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2844 domain = sh->rx_domain; 2845 else 2846 domain = sh->tx_domain; 2847 ret = mlx5_flow_os_create_flow_action_push_vlan 2848 (domain, resource->vlan_tag, 2849 &cache_resource->action); 2850 if (ret) { 2851 mlx5_free(cache_resource); 2852 return rte_flow_error_set(error, ENOMEM, 2853 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2854 NULL, "cannot create action"); 2855 } 2856 rte_atomic32_init(&cache_resource->refcnt); 2857 rte_atomic32_inc(&cache_resource->refcnt); 2858 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2859 &sh->push_vlan_action_list, 2860 dev_flow->handle->dvh.rix_push_vlan, 2861 cache_resource, next); 2862 dev_flow->dv.push_vlan_res = cache_resource; 2863 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", 2864 (void *)cache_resource, 2865 rte_atomic32_read(&cache_resource->refcnt)); 2866 return 0; 2867 } 2868 /** 2869 * Get the size of specific rte_flow_item_type hdr size 2870 * 2871 * @param[in] item_type 2872 * Tested rte_flow_item_type. 2873 * 2874 * @return 2875 * sizeof struct item_type, 0 if void or irrelevant. 2876 */ 2877 static size_t 2878 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type) 2879 { 2880 size_t retval; 2881 2882 switch (item_type) { 2883 case RTE_FLOW_ITEM_TYPE_ETH: 2884 retval = sizeof(struct rte_ether_hdr); 2885 break; 2886 case RTE_FLOW_ITEM_TYPE_VLAN: 2887 retval = sizeof(struct rte_vlan_hdr); 2888 break; 2889 case RTE_FLOW_ITEM_TYPE_IPV4: 2890 retval = sizeof(struct rte_ipv4_hdr); 2891 break; 2892 case RTE_FLOW_ITEM_TYPE_IPV6: 2893 retval = sizeof(struct rte_ipv6_hdr); 2894 break; 2895 case RTE_FLOW_ITEM_TYPE_UDP: 2896 retval = sizeof(struct rte_udp_hdr); 2897 break; 2898 case RTE_FLOW_ITEM_TYPE_TCP: 2899 retval = sizeof(struct rte_tcp_hdr); 2900 break; 2901 case RTE_FLOW_ITEM_TYPE_VXLAN: 2902 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 2903 retval = sizeof(struct rte_vxlan_hdr); 2904 break; 2905 case RTE_FLOW_ITEM_TYPE_GRE: 2906 case RTE_FLOW_ITEM_TYPE_NVGRE: 2907 retval = sizeof(struct rte_gre_hdr); 2908 break; 2909 case RTE_FLOW_ITEM_TYPE_MPLS: 2910 retval = sizeof(struct rte_mpls_hdr); 2911 break; 2912 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 2913 default: 2914 retval = 0; 2915 break; 2916 } 2917 return retval; 2918 } 2919 2920 #define MLX5_ENCAP_IPV4_VERSION 0x40 2921 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 2922 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 2923 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 2924 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 2925 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 2926 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 2927 2928 /** 2929 * Convert the encap action data from list of rte_flow_item to raw buffer 2930 * 2931 * @param[in] items 2932 * Pointer to rte_flow_item objects list. 2933 * @param[out] buf 2934 * Pointer to the output buffer. 2935 * @param[out] size 2936 * Pointer to the output buffer size. 2937 * @param[out] error 2938 * Pointer to the error structure. 2939 * 2940 * @return 2941 * 0 on success, a negative errno value otherwise and rte_errno is set. 2942 */ 2943 static int 2944 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 2945 size_t *size, struct rte_flow_error *error) 2946 { 2947 struct rte_ether_hdr *eth = NULL; 2948 struct rte_vlan_hdr *vlan = NULL; 2949 struct rte_ipv4_hdr *ipv4 = NULL; 2950 struct rte_ipv6_hdr *ipv6 = NULL; 2951 struct rte_udp_hdr *udp = NULL; 2952 struct rte_vxlan_hdr *vxlan = NULL; 2953 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; 2954 struct rte_gre_hdr *gre = NULL; 2955 size_t len; 2956 size_t temp_size = 0; 2957 2958 if (!items) 2959 return rte_flow_error_set(error, EINVAL, 2960 RTE_FLOW_ERROR_TYPE_ACTION, 2961 NULL, "invalid empty data"); 2962 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2963 len = flow_dv_get_item_hdr_len(items->type); 2964 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 2965 return rte_flow_error_set(error, EINVAL, 2966 RTE_FLOW_ERROR_TYPE_ACTION, 2967 (void *)items->type, 2968 "items total size is too big" 2969 " for encap action"); 2970 rte_memcpy((void *)&buf[temp_size], items->spec, len); 2971 switch (items->type) { 2972 case RTE_FLOW_ITEM_TYPE_ETH: 2973 eth = (struct rte_ether_hdr *)&buf[temp_size]; 2974 break; 2975 case RTE_FLOW_ITEM_TYPE_VLAN: 2976 vlan = (struct rte_vlan_hdr *)&buf[temp_size]; 2977 if (!eth) 2978 return rte_flow_error_set(error, EINVAL, 2979 RTE_FLOW_ERROR_TYPE_ACTION, 2980 (void *)items->type, 2981 "eth header not found"); 2982 if (!eth->ether_type) 2983 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); 2984 break; 2985 case RTE_FLOW_ITEM_TYPE_IPV4: 2986 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; 2987 if (!vlan && !eth) 2988 return rte_flow_error_set(error, EINVAL, 2989 RTE_FLOW_ERROR_TYPE_ACTION, 2990 (void *)items->type, 2991 "neither eth nor vlan" 2992 " header found"); 2993 if (vlan && !vlan->eth_proto) 2994 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); 2995 else if (eth && !eth->ether_type) 2996 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); 2997 if (!ipv4->version_ihl) 2998 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 2999 MLX5_ENCAP_IPV4_IHL_MIN; 3000 if (!ipv4->time_to_live) 3001 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 3002 break; 3003 case RTE_FLOW_ITEM_TYPE_IPV6: 3004 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; 3005 if (!vlan && !eth) 3006 return rte_flow_error_set(error, EINVAL, 3007 RTE_FLOW_ERROR_TYPE_ACTION, 3008 (void *)items->type, 3009 "neither eth nor vlan" 3010 " header found"); 3011 if (vlan && !vlan->eth_proto) 3012 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3013 else if (eth && !eth->ether_type) 3014 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3015 if (!ipv6->vtc_flow) 3016 ipv6->vtc_flow = 3017 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 3018 if (!ipv6->hop_limits) 3019 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 3020 break; 3021 case RTE_FLOW_ITEM_TYPE_UDP: 3022 udp = (struct rte_udp_hdr *)&buf[temp_size]; 3023 if (!ipv4 && !ipv6) 3024 return rte_flow_error_set(error, EINVAL, 3025 RTE_FLOW_ERROR_TYPE_ACTION, 3026 (void *)items->type, 3027 "ip header not found"); 3028 if (ipv4 && !ipv4->next_proto_id) 3029 ipv4->next_proto_id = IPPROTO_UDP; 3030 else if (ipv6 && !ipv6->proto) 3031 ipv6->proto = IPPROTO_UDP; 3032 break; 3033 case RTE_FLOW_ITEM_TYPE_VXLAN: 3034 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; 3035 if (!udp) 3036 return rte_flow_error_set(error, EINVAL, 3037 RTE_FLOW_ERROR_TYPE_ACTION, 3038 (void *)items->type, 3039 "udp header not found"); 3040 if (!udp->dst_port) 3041 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 3042 if (!vxlan->vx_flags) 3043 vxlan->vx_flags = 3044 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 3045 break; 3046 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 3047 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; 3048 if (!udp) 3049 return rte_flow_error_set(error, EINVAL, 3050 RTE_FLOW_ERROR_TYPE_ACTION, 3051 (void *)items->type, 3052 "udp header not found"); 3053 if (!vxlan_gpe->proto) 3054 return rte_flow_error_set(error, EINVAL, 3055 RTE_FLOW_ERROR_TYPE_ACTION, 3056 (void *)items->type, 3057 "next protocol not found"); 3058 if (!udp->dst_port) 3059 udp->dst_port = 3060 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 3061 if (!vxlan_gpe->vx_flags) 3062 vxlan_gpe->vx_flags = 3063 MLX5_ENCAP_VXLAN_GPE_FLAGS; 3064 break; 3065 case RTE_FLOW_ITEM_TYPE_GRE: 3066 case RTE_FLOW_ITEM_TYPE_NVGRE: 3067 gre = (struct rte_gre_hdr *)&buf[temp_size]; 3068 if (!gre->proto) 3069 return rte_flow_error_set(error, EINVAL, 3070 RTE_FLOW_ERROR_TYPE_ACTION, 3071 (void *)items->type, 3072 "next protocol not found"); 3073 if (!ipv4 && !ipv6) 3074 return rte_flow_error_set(error, EINVAL, 3075 RTE_FLOW_ERROR_TYPE_ACTION, 3076 (void *)items->type, 3077 "ip header not found"); 3078 if (ipv4 && !ipv4->next_proto_id) 3079 ipv4->next_proto_id = IPPROTO_GRE; 3080 else if (ipv6 && !ipv6->proto) 3081 ipv6->proto = IPPROTO_GRE; 3082 break; 3083 case RTE_FLOW_ITEM_TYPE_VOID: 3084 break; 3085 default: 3086 return rte_flow_error_set(error, EINVAL, 3087 RTE_FLOW_ERROR_TYPE_ACTION, 3088 (void *)items->type, 3089 "unsupported item type"); 3090 break; 3091 } 3092 temp_size += len; 3093 } 3094 *size = temp_size; 3095 return 0; 3096 } 3097 3098 static int 3099 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) 3100 { 3101 struct rte_ether_hdr *eth = NULL; 3102 struct rte_vlan_hdr *vlan = NULL; 3103 struct rte_ipv6_hdr *ipv6 = NULL; 3104 struct rte_udp_hdr *udp = NULL; 3105 char *next_hdr; 3106 uint16_t proto; 3107 3108 eth = (struct rte_ether_hdr *)data; 3109 next_hdr = (char *)(eth + 1); 3110 proto = RTE_BE16(eth->ether_type); 3111 3112 /* VLAN skipping */ 3113 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { 3114 vlan = (struct rte_vlan_hdr *)next_hdr; 3115 proto = RTE_BE16(vlan->eth_proto); 3116 next_hdr += sizeof(struct rte_vlan_hdr); 3117 } 3118 3119 /* HW calculates IPv4 csum. no need to proceed */ 3120 if (proto == RTE_ETHER_TYPE_IPV4) 3121 return 0; 3122 3123 /* non IPv4/IPv6 header. not supported */ 3124 if (proto != RTE_ETHER_TYPE_IPV6) { 3125 return rte_flow_error_set(error, ENOTSUP, 3126 RTE_FLOW_ERROR_TYPE_ACTION, 3127 NULL, "Cannot offload non IPv4/IPv6"); 3128 } 3129 3130 ipv6 = (struct rte_ipv6_hdr *)next_hdr; 3131 3132 /* ignore non UDP */ 3133 if (ipv6->proto != IPPROTO_UDP) 3134 return 0; 3135 3136 udp = (struct rte_udp_hdr *)(ipv6 + 1); 3137 udp->dgram_cksum = 0; 3138 3139 return 0; 3140 } 3141 3142 /** 3143 * Convert L2 encap action to DV specification. 3144 * 3145 * @param[in] dev 3146 * Pointer to rte_eth_dev structure. 3147 * @param[in] action 3148 * Pointer to action structure. 3149 * @param[in, out] dev_flow 3150 * Pointer to the mlx5_flow. 3151 * @param[in] transfer 3152 * Mark if the flow is E-Switch flow. 3153 * @param[out] error 3154 * Pointer to the error structure. 3155 * 3156 * @return 3157 * 0 on success, a negative errno value otherwise and rte_errno is set. 3158 */ 3159 static int 3160 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 3161 const struct rte_flow_action *action, 3162 struct mlx5_flow *dev_flow, 3163 uint8_t transfer, 3164 struct rte_flow_error *error) 3165 { 3166 const struct rte_flow_item *encap_data; 3167 const struct rte_flow_action_raw_encap *raw_encap_data; 3168 struct mlx5_flow_dv_encap_decap_resource res = { 3169 .reformat_type = 3170 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 3171 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3172 MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 3173 }; 3174 3175 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 3176 raw_encap_data = 3177 (const struct rte_flow_action_raw_encap *)action->conf; 3178 res.size = raw_encap_data->size; 3179 memcpy(res.buf, raw_encap_data->data, res.size); 3180 } else { 3181 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 3182 encap_data = 3183 ((const struct rte_flow_action_vxlan_encap *) 3184 action->conf)->definition; 3185 else 3186 encap_data = 3187 ((const struct rte_flow_action_nvgre_encap *) 3188 action->conf)->definition; 3189 if (flow_dv_convert_encap_data(encap_data, res.buf, 3190 &res.size, error)) 3191 return -rte_errno; 3192 } 3193 if (flow_dv_zero_encap_udp_csum(res.buf, error)) 3194 return -rte_errno; 3195 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3196 return rte_flow_error_set(error, EINVAL, 3197 RTE_FLOW_ERROR_TYPE_ACTION, 3198 NULL, "can't create L2 encap action"); 3199 return 0; 3200 } 3201 3202 /** 3203 * Convert L2 decap action to DV specification. 3204 * 3205 * @param[in] dev 3206 * Pointer to rte_eth_dev structure. 3207 * @param[in, out] dev_flow 3208 * Pointer to the mlx5_flow. 3209 * @param[in] transfer 3210 * Mark if the flow is E-Switch flow. 3211 * @param[out] error 3212 * Pointer to the error structure. 3213 * 3214 * @return 3215 * 0 on success, a negative errno value otherwise and rte_errno is set. 3216 */ 3217 static int 3218 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 3219 struct mlx5_flow *dev_flow, 3220 uint8_t transfer, 3221 struct rte_flow_error *error) 3222 { 3223 struct mlx5_flow_dv_encap_decap_resource res = { 3224 .size = 0, 3225 .reformat_type = 3226 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 3227 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3228 MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 3229 }; 3230 3231 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3232 return rte_flow_error_set(error, EINVAL, 3233 RTE_FLOW_ERROR_TYPE_ACTION, 3234 NULL, "can't create L2 decap action"); 3235 return 0; 3236 } 3237 3238 /** 3239 * Convert raw decap/encap (L3 tunnel) action to DV specification. 3240 * 3241 * @param[in] dev 3242 * Pointer to rte_eth_dev structure. 3243 * @param[in] action 3244 * Pointer to action structure. 3245 * @param[in, out] dev_flow 3246 * Pointer to the mlx5_flow. 3247 * @param[in] attr 3248 * Pointer to the flow attributes. 3249 * @param[out] error 3250 * Pointer to the error structure. 3251 * 3252 * @return 3253 * 0 on success, a negative errno value otherwise and rte_errno is set. 3254 */ 3255 static int 3256 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 3257 const struct rte_flow_action *action, 3258 struct mlx5_flow *dev_flow, 3259 const struct rte_flow_attr *attr, 3260 struct rte_flow_error *error) 3261 { 3262 const struct rte_flow_action_raw_encap *encap_data; 3263 struct mlx5_flow_dv_encap_decap_resource res; 3264 3265 memset(&res, 0, sizeof(res)); 3266 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 3267 res.size = encap_data->size; 3268 memcpy(res.buf, encap_data->data, res.size); 3269 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? 3270 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : 3271 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3272 if (attr->transfer) 3273 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3274 else 3275 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3276 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3277 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3278 return rte_flow_error_set(error, EINVAL, 3279 RTE_FLOW_ERROR_TYPE_ACTION, 3280 NULL, "can't create encap action"); 3281 return 0; 3282 } 3283 3284 /** 3285 * Create action push VLAN. 3286 * 3287 * @param[in] dev 3288 * Pointer to rte_eth_dev structure. 3289 * @param[in] attr 3290 * Pointer to the flow attributes. 3291 * @param[in] vlan 3292 * Pointer to the vlan to push to the Ethernet header. 3293 * @param[in, out] dev_flow 3294 * Pointer to the mlx5_flow. 3295 * @param[out] error 3296 * Pointer to the error structure. 3297 * 3298 * @return 3299 * 0 on success, a negative errno value otherwise and rte_errno is set. 3300 */ 3301 static int 3302 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, 3303 const struct rte_flow_attr *attr, 3304 const struct rte_vlan_hdr *vlan, 3305 struct mlx5_flow *dev_flow, 3306 struct rte_flow_error *error) 3307 { 3308 struct mlx5_flow_dv_push_vlan_action_resource res; 3309 3310 memset(&res, 0, sizeof(res)); 3311 res.vlan_tag = 3312 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 | 3313 vlan->vlan_tci); 3314 if (attr->transfer) 3315 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3316 else 3317 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3318 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3319 return flow_dv_push_vlan_action_resource_register 3320 (dev, &res, dev_flow, error); 3321 } 3322 3323 /** 3324 * Validate the modify-header actions. 3325 * 3326 * @param[in] action_flags 3327 * Holds the actions detected until now. 3328 * @param[in] action 3329 * Pointer to the modify action. 3330 * @param[out] error 3331 * Pointer to error structure. 3332 * 3333 * @return 3334 * 0 on success, a negative errno value otherwise and rte_errno is set. 3335 */ 3336 static int 3337 flow_dv_validate_action_modify_hdr(const uint64_t action_flags, 3338 const struct rte_flow_action *action, 3339 struct rte_flow_error *error) 3340 { 3341 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) 3342 return rte_flow_error_set(error, EINVAL, 3343 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3344 NULL, "action configuration not set"); 3345 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 3346 return rte_flow_error_set(error, EINVAL, 3347 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3348 "can't have encap action before" 3349 " modify action"); 3350 return 0; 3351 } 3352 3353 /** 3354 * Validate the modify-header MAC address actions. 3355 * 3356 * @param[in] action_flags 3357 * Holds the actions detected until now. 3358 * @param[in] action 3359 * Pointer to the modify action. 3360 * @param[in] item_flags 3361 * Holds the items detected. 3362 * @param[out] error 3363 * Pointer to error structure. 3364 * 3365 * @return 3366 * 0 on success, a negative errno value otherwise and rte_errno is set. 3367 */ 3368 static int 3369 flow_dv_validate_action_modify_mac(const uint64_t action_flags, 3370 const struct rte_flow_action *action, 3371 const uint64_t item_flags, 3372 struct rte_flow_error *error) 3373 { 3374 int ret = 0; 3375 3376 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3377 if (!ret) { 3378 if (!(item_flags & MLX5_FLOW_LAYER_L2)) 3379 return rte_flow_error_set(error, EINVAL, 3380 RTE_FLOW_ERROR_TYPE_ACTION, 3381 NULL, 3382 "no L2 item in pattern"); 3383 } 3384 return ret; 3385 } 3386 3387 /** 3388 * Validate the modify-header IPv4 address actions. 3389 * 3390 * @param[in] action_flags 3391 * Holds the actions detected until now. 3392 * @param[in] action 3393 * Pointer to the modify action. 3394 * @param[in] item_flags 3395 * Holds the items detected. 3396 * @param[out] error 3397 * Pointer to error structure. 3398 * 3399 * @return 3400 * 0 on success, a negative errno value otherwise and rte_errno is set. 3401 */ 3402 static int 3403 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, 3404 const struct rte_flow_action *action, 3405 const uint64_t item_flags, 3406 struct rte_flow_error *error) 3407 { 3408 int ret = 0; 3409 uint64_t layer; 3410 3411 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3412 if (!ret) { 3413 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3414 MLX5_FLOW_LAYER_INNER_L3_IPV4 : 3415 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3416 if (!(item_flags & layer)) 3417 return rte_flow_error_set(error, EINVAL, 3418 RTE_FLOW_ERROR_TYPE_ACTION, 3419 NULL, 3420 "no ipv4 item in pattern"); 3421 } 3422 return ret; 3423 } 3424 3425 /** 3426 * Validate the modify-header IPv6 address actions. 3427 * 3428 * @param[in] action_flags 3429 * Holds the actions detected until now. 3430 * @param[in] action 3431 * Pointer to the modify action. 3432 * @param[in] item_flags 3433 * Holds the items detected. 3434 * @param[out] error 3435 * Pointer to error structure. 3436 * 3437 * @return 3438 * 0 on success, a negative errno value otherwise and rte_errno is set. 3439 */ 3440 static int 3441 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, 3442 const struct rte_flow_action *action, 3443 const uint64_t item_flags, 3444 struct rte_flow_error *error) 3445 { 3446 int ret = 0; 3447 uint64_t layer; 3448 3449 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3450 if (!ret) { 3451 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3452 MLX5_FLOW_LAYER_INNER_L3_IPV6 : 3453 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3454 if (!(item_flags & layer)) 3455 return rte_flow_error_set(error, EINVAL, 3456 RTE_FLOW_ERROR_TYPE_ACTION, 3457 NULL, 3458 "no ipv6 item in pattern"); 3459 } 3460 return ret; 3461 } 3462 3463 /** 3464 * Validate the modify-header TP actions. 3465 * 3466 * @param[in] action_flags 3467 * Holds the actions detected until now. 3468 * @param[in] action 3469 * Pointer to the modify action. 3470 * @param[in] item_flags 3471 * Holds the items detected. 3472 * @param[out] error 3473 * Pointer to error structure. 3474 * 3475 * @return 3476 * 0 on success, a negative errno value otherwise and rte_errno is set. 3477 */ 3478 static int 3479 flow_dv_validate_action_modify_tp(const uint64_t action_flags, 3480 const struct rte_flow_action *action, 3481 const uint64_t item_flags, 3482 struct rte_flow_error *error) 3483 { 3484 int ret = 0; 3485 uint64_t layer; 3486 3487 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3488 if (!ret) { 3489 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3490 MLX5_FLOW_LAYER_INNER_L4 : 3491 MLX5_FLOW_LAYER_OUTER_L4; 3492 if (!(item_flags & layer)) 3493 return rte_flow_error_set(error, EINVAL, 3494 RTE_FLOW_ERROR_TYPE_ACTION, 3495 NULL, "no transport layer " 3496 "in pattern"); 3497 } 3498 return ret; 3499 } 3500 3501 /** 3502 * Validate the modify-header actions of increment/decrement 3503 * TCP Sequence-number. 3504 * 3505 * @param[in] action_flags 3506 * Holds the actions detected until now. 3507 * @param[in] action 3508 * Pointer to the modify action. 3509 * @param[in] item_flags 3510 * Holds the items detected. 3511 * @param[out] error 3512 * Pointer to error structure. 3513 * 3514 * @return 3515 * 0 on success, a negative errno value otherwise and rte_errno is set. 3516 */ 3517 static int 3518 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, 3519 const struct rte_flow_action *action, 3520 const uint64_t item_flags, 3521 struct rte_flow_error *error) 3522 { 3523 int ret = 0; 3524 uint64_t layer; 3525 3526 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3527 if (!ret) { 3528 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3529 MLX5_FLOW_LAYER_INNER_L4_TCP : 3530 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3531 if (!(item_flags & layer)) 3532 return rte_flow_error_set(error, EINVAL, 3533 RTE_FLOW_ERROR_TYPE_ACTION, 3534 NULL, "no TCP item in" 3535 " pattern"); 3536 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && 3537 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || 3538 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && 3539 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) 3540 return rte_flow_error_set(error, EINVAL, 3541 RTE_FLOW_ERROR_TYPE_ACTION, 3542 NULL, 3543 "cannot decrease and increase" 3544 " TCP sequence number" 3545 " at the same time"); 3546 } 3547 return ret; 3548 } 3549 3550 /** 3551 * Validate the modify-header actions of increment/decrement 3552 * TCP Acknowledgment number. 3553 * 3554 * @param[in] action_flags 3555 * Holds the actions detected until now. 3556 * @param[in] action 3557 * Pointer to the modify action. 3558 * @param[in] item_flags 3559 * Holds the items detected. 3560 * @param[out] error 3561 * Pointer to error structure. 3562 * 3563 * @return 3564 * 0 on success, a negative errno value otherwise and rte_errno is set. 3565 */ 3566 static int 3567 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, 3568 const struct rte_flow_action *action, 3569 const uint64_t item_flags, 3570 struct rte_flow_error *error) 3571 { 3572 int ret = 0; 3573 uint64_t layer; 3574 3575 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3576 if (!ret) { 3577 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3578 MLX5_FLOW_LAYER_INNER_L4_TCP : 3579 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3580 if (!(item_flags & layer)) 3581 return rte_flow_error_set(error, EINVAL, 3582 RTE_FLOW_ERROR_TYPE_ACTION, 3583 NULL, "no TCP item in" 3584 " pattern"); 3585 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && 3586 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || 3587 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && 3588 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) 3589 return rte_flow_error_set(error, EINVAL, 3590 RTE_FLOW_ERROR_TYPE_ACTION, 3591 NULL, 3592 "cannot decrease and increase" 3593 " TCP acknowledgment number" 3594 " at the same time"); 3595 } 3596 return ret; 3597 } 3598 3599 /** 3600 * Validate the modify-header TTL actions. 3601 * 3602 * @param[in] action_flags 3603 * Holds the actions detected until now. 3604 * @param[in] action 3605 * Pointer to the modify action. 3606 * @param[in] item_flags 3607 * Holds the items detected. 3608 * @param[out] error 3609 * Pointer to error structure. 3610 * 3611 * @return 3612 * 0 on success, a negative errno value otherwise and rte_errno is set. 3613 */ 3614 static int 3615 flow_dv_validate_action_modify_ttl(const uint64_t action_flags, 3616 const struct rte_flow_action *action, 3617 const uint64_t item_flags, 3618 struct rte_flow_error *error) 3619 { 3620 int ret = 0; 3621 uint64_t layer; 3622 3623 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3624 if (!ret) { 3625 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3626 MLX5_FLOW_LAYER_INNER_L3 : 3627 MLX5_FLOW_LAYER_OUTER_L3; 3628 if (!(item_flags & layer)) 3629 return rte_flow_error_set(error, EINVAL, 3630 RTE_FLOW_ERROR_TYPE_ACTION, 3631 NULL, 3632 "no IP protocol in pattern"); 3633 } 3634 return ret; 3635 } 3636 3637 /** 3638 * Validate jump action. 3639 * 3640 * @param[in] action 3641 * Pointer to the jump action. 3642 * @param[in] action_flags 3643 * Holds the actions detected until now. 3644 * @param[in] attributes 3645 * Pointer to flow attributes 3646 * @param[in] external 3647 * Action belongs to flow rule created by request external to PMD. 3648 * @param[out] error 3649 * Pointer to error structure. 3650 * 3651 * @return 3652 * 0 on success, a negative errno value otherwise and rte_errno is set. 3653 */ 3654 static int 3655 flow_dv_validate_action_jump(const struct rte_flow_action *action, 3656 uint64_t action_flags, 3657 const struct rte_flow_attr *attributes, 3658 bool external, struct rte_flow_error *error) 3659 { 3660 uint32_t target_group, table; 3661 int ret = 0; 3662 3663 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3664 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3665 return rte_flow_error_set(error, EINVAL, 3666 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3667 "can't have 2 fate actions in" 3668 " same flow"); 3669 if (action_flags & MLX5_FLOW_ACTION_METER) 3670 return rte_flow_error_set(error, ENOTSUP, 3671 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3672 "jump with meter not support"); 3673 if (!action->conf) 3674 return rte_flow_error_set(error, EINVAL, 3675 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3676 NULL, "action configuration not set"); 3677 target_group = 3678 ((const struct rte_flow_action_jump *)action->conf)->group; 3679 ret = mlx5_flow_group_to_table(attributes, external, target_group, 3680 true, &table, error); 3681 if (ret) 3682 return ret; 3683 if (attributes->group == target_group) 3684 return rte_flow_error_set(error, EINVAL, 3685 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3686 "target group must be other than" 3687 " the current flow group"); 3688 return 0; 3689 } 3690 3691 /* 3692 * Validate the port_id action. 3693 * 3694 * @param[in] dev 3695 * Pointer to rte_eth_dev structure. 3696 * @param[in] action_flags 3697 * Bit-fields that holds the actions detected until now. 3698 * @param[in] action 3699 * Port_id RTE action structure. 3700 * @param[in] attr 3701 * Attributes of flow that includes this action. 3702 * @param[out] error 3703 * Pointer to error structure. 3704 * 3705 * @return 3706 * 0 on success, a negative errno value otherwise and rte_errno is set. 3707 */ 3708 static int 3709 flow_dv_validate_action_port_id(struct rte_eth_dev *dev, 3710 uint64_t action_flags, 3711 const struct rte_flow_action *action, 3712 const struct rte_flow_attr *attr, 3713 struct rte_flow_error *error) 3714 { 3715 const struct rte_flow_action_port_id *port_id; 3716 struct mlx5_priv *act_priv; 3717 struct mlx5_priv *dev_priv; 3718 uint16_t port; 3719 3720 if (!attr->transfer) 3721 return rte_flow_error_set(error, ENOTSUP, 3722 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3723 NULL, 3724 "port id action is valid in transfer" 3725 " mode only"); 3726 if (!action || !action->conf) 3727 return rte_flow_error_set(error, ENOTSUP, 3728 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3729 NULL, 3730 "port id action parameters must be" 3731 " specified"); 3732 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3733 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3734 return rte_flow_error_set(error, EINVAL, 3735 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3736 "can have only one fate actions in" 3737 " a flow"); 3738 dev_priv = mlx5_dev_to_eswitch_info(dev); 3739 if (!dev_priv) 3740 return rte_flow_error_set(error, rte_errno, 3741 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3742 NULL, 3743 "failed to obtain E-Switch info"); 3744 port_id = action->conf; 3745 port = port_id->original ? dev->data->port_id : port_id->id; 3746 act_priv = mlx5_port_to_eswitch_info(port, false); 3747 if (!act_priv) 3748 return rte_flow_error_set 3749 (error, rte_errno, 3750 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id, 3751 "failed to obtain E-Switch port id for port"); 3752 if (act_priv->domain_id != dev_priv->domain_id) 3753 return rte_flow_error_set 3754 (error, EINVAL, 3755 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3756 "port does not belong to" 3757 " E-Switch being configured"); 3758 return 0; 3759 } 3760 3761 /** 3762 * Get the maximum number of modify header actions. 3763 * 3764 * @param dev 3765 * Pointer to rte_eth_dev structure. 3766 * @param flags 3767 * Flags bits to check if root level. 3768 * 3769 * @return 3770 * Max number of modify header actions device can support. 3771 */ 3772 static inline unsigned int 3773 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, 3774 uint64_t flags) 3775 { 3776 /* 3777 * There's no way to directly query the max capacity from FW. 3778 * The maximal value on root table should be assumed to be supported. 3779 */ 3780 if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) 3781 return MLX5_MAX_MODIFY_NUM; 3782 else 3783 return MLX5_ROOT_TBL_MODIFY_NUM; 3784 } 3785 3786 /** 3787 * Validate the meter action. 3788 * 3789 * @param[in] dev 3790 * Pointer to rte_eth_dev structure. 3791 * @param[in] action_flags 3792 * Bit-fields that holds the actions detected until now. 3793 * @param[in] action 3794 * Pointer to the meter action. 3795 * @param[in] attr 3796 * Attributes of flow that includes this action. 3797 * @param[out] error 3798 * Pointer to error structure. 3799 * 3800 * @return 3801 * 0 on success, a negative errno value otherwise and rte_ernno is set. 3802 */ 3803 static int 3804 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, 3805 uint64_t action_flags, 3806 const struct rte_flow_action *action, 3807 const struct rte_flow_attr *attr, 3808 struct rte_flow_error *error) 3809 { 3810 struct mlx5_priv *priv = dev->data->dev_private; 3811 const struct rte_flow_action_meter *am = action->conf; 3812 struct mlx5_flow_meter *fm; 3813 3814 if (!am) 3815 return rte_flow_error_set(error, EINVAL, 3816 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3817 "meter action conf is NULL"); 3818 3819 if (action_flags & MLX5_FLOW_ACTION_METER) 3820 return rte_flow_error_set(error, ENOTSUP, 3821 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3822 "meter chaining not support"); 3823 if (action_flags & MLX5_FLOW_ACTION_JUMP) 3824 return rte_flow_error_set(error, ENOTSUP, 3825 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3826 "meter with jump not support"); 3827 if (!priv->mtr_en) 3828 return rte_flow_error_set(error, ENOTSUP, 3829 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3830 NULL, 3831 "meter action not supported"); 3832 fm = mlx5_flow_meter_find(priv, am->mtr_id); 3833 if (!fm) 3834 return rte_flow_error_set(error, EINVAL, 3835 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3836 "Meter not found"); 3837 if (fm->ref_cnt && (!(fm->transfer == attr->transfer || 3838 (!fm->ingress && !attr->ingress && attr->egress) || 3839 (!fm->egress && !attr->egress && attr->ingress)))) 3840 return rte_flow_error_set(error, EINVAL, 3841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3842 "Flow attributes are either invalid " 3843 "or have a conflict with current " 3844 "meter attributes"); 3845 return 0; 3846 } 3847 3848 /** 3849 * Validate the age action. 3850 * 3851 * @param[in] action_flags 3852 * Holds the actions detected until now. 3853 * @param[in] action 3854 * Pointer to the age action. 3855 * @param[in] dev 3856 * Pointer to the Ethernet device structure. 3857 * @param[out] error 3858 * Pointer to error structure. 3859 * 3860 * @return 3861 * 0 on success, a negative errno value otherwise and rte_errno is set. 3862 */ 3863 static int 3864 flow_dv_validate_action_age(uint64_t action_flags, 3865 const struct rte_flow_action *action, 3866 struct rte_eth_dev *dev, 3867 struct rte_flow_error *error) 3868 { 3869 struct mlx5_priv *priv = dev->data->dev_private; 3870 const struct rte_flow_action_age *age = action->conf; 3871 3872 if (!priv->config.devx || priv->counter_fallback) 3873 return rte_flow_error_set(error, ENOTSUP, 3874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3875 NULL, 3876 "age action not supported"); 3877 if (!(action->conf)) 3878 return rte_flow_error_set(error, EINVAL, 3879 RTE_FLOW_ERROR_TYPE_ACTION, action, 3880 "configuration cannot be null"); 3881 if (age->timeout >= UINT16_MAX / 2 / 10) 3882 return rte_flow_error_set(error, ENOTSUP, 3883 RTE_FLOW_ERROR_TYPE_ACTION, action, 3884 "Max age time: 3275 seconds"); 3885 if (action_flags & MLX5_FLOW_ACTION_AGE) 3886 return rte_flow_error_set(error, EINVAL, 3887 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3888 "Duplicate age ctions set"); 3889 return 0; 3890 } 3891 3892 /** 3893 * Validate the modify-header IPv4 DSCP actions. 3894 * 3895 * @param[in] action_flags 3896 * Holds the actions detected until now. 3897 * @param[in] action 3898 * Pointer to the modify action. 3899 * @param[in] item_flags 3900 * Holds the items detected. 3901 * @param[out] error 3902 * Pointer to error structure. 3903 * 3904 * @return 3905 * 0 on success, a negative errno value otherwise and rte_errno is set. 3906 */ 3907 static int 3908 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, 3909 const struct rte_flow_action *action, 3910 const uint64_t item_flags, 3911 struct rte_flow_error *error) 3912 { 3913 int ret = 0; 3914 3915 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3916 if (!ret) { 3917 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) 3918 return rte_flow_error_set(error, EINVAL, 3919 RTE_FLOW_ERROR_TYPE_ACTION, 3920 NULL, 3921 "no ipv4 item in pattern"); 3922 } 3923 return ret; 3924 } 3925 3926 /** 3927 * Validate the modify-header IPv6 DSCP actions. 3928 * 3929 * @param[in] action_flags 3930 * Holds the actions detected until now. 3931 * @param[in] action 3932 * Pointer to the modify action. 3933 * @param[in] item_flags 3934 * Holds the items detected. 3935 * @param[out] error 3936 * Pointer to error structure. 3937 * 3938 * @return 3939 * 0 on success, a negative errno value otherwise and rte_errno is set. 3940 */ 3941 static int 3942 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, 3943 const struct rte_flow_action *action, 3944 const uint64_t item_flags, 3945 struct rte_flow_error *error) 3946 { 3947 int ret = 0; 3948 3949 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3950 if (!ret) { 3951 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) 3952 return rte_flow_error_set(error, EINVAL, 3953 RTE_FLOW_ERROR_TYPE_ACTION, 3954 NULL, 3955 "no ipv6 item in pattern"); 3956 } 3957 return ret; 3958 } 3959 3960 /** 3961 * Match modify-header resource. 3962 * 3963 * @param entry 3964 * Pointer to exist resource entry object. 3965 * @param ctx 3966 * Pointer to new modify-header resource. 3967 * 3968 * @return 3969 * 0 on matching, -1 otherwise. 3970 */ 3971 static int 3972 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx) 3973 { 3974 struct mlx5_flow_dv_modify_hdr_resource *resource; 3975 struct mlx5_flow_dv_modify_hdr_resource *cache_resource; 3976 uint32_t actions_len; 3977 3978 resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx; 3979 cache_resource = container_of(entry, 3980 struct mlx5_flow_dv_modify_hdr_resource, 3981 entry); 3982 actions_len = resource->actions_num * sizeof(resource->actions[0]); 3983 if (resource->entry.key == cache_resource->entry.key && 3984 resource->ft_type == cache_resource->ft_type && 3985 resource->actions_num == cache_resource->actions_num && 3986 resource->flags == cache_resource->flags && 3987 !memcmp((const void *)resource->actions, 3988 (const void *)cache_resource->actions, 3989 actions_len)) 3990 return 0; 3991 return -1; 3992 } 3993 3994 /** 3995 * Find existing modify-header resource or create and register a new one. 3996 * 3997 * @param dev[in, out] 3998 * Pointer to rte_eth_dev structure. 3999 * @param[in, out] resource 4000 * Pointer to modify-header resource. 4001 * @parm[in, out] dev_flow 4002 * Pointer to the dev_flow. 4003 * @param[out] error 4004 * pointer to error structure. 4005 * 4006 * @return 4007 * 0 on success otherwise -errno and errno is set. 4008 */ 4009 static int 4010 flow_dv_modify_hdr_resource_register 4011 (struct rte_eth_dev *dev, 4012 struct mlx5_flow_dv_modify_hdr_resource *resource, 4013 struct mlx5_flow *dev_flow, 4014 struct rte_flow_error *error) 4015 { 4016 struct mlx5_priv *priv = dev->data->dev_private; 4017 struct mlx5_dev_ctx_shared *sh = priv->sh; 4018 struct mlx5_flow_dv_modify_hdr_resource *cache_resource; 4019 struct mlx5dv_dr_domain *ns; 4020 uint32_t actions_len; 4021 struct mlx5_hlist_entry *entry; 4022 union mlx5_flow_modify_hdr_key hdr_mod_key = { 4023 { 4024 .ft_type = resource->ft_type, 4025 .actions_num = resource->actions_num, 4026 .group = dev_flow->dv.group, 4027 .cksum = 0, 4028 } 4029 }; 4030 int ret; 4031 4032 resource->flags = dev_flow->dv.group ? 0 : 4033 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 4034 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, 4035 resource->flags)) 4036 return rte_flow_error_set(error, EOVERFLOW, 4037 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 4038 "too many modify header items"); 4039 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 4040 ns = sh->fdb_domain; 4041 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 4042 ns = sh->tx_domain; 4043 else 4044 ns = sh->rx_domain; 4045 /* Lookup a matching resource from cache. */ 4046 actions_len = resource->actions_num * sizeof(resource->actions[0]); 4047 hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0); 4048 resource->entry.key = hdr_mod_key.v64; 4049 entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key, 4050 flow_dv_modify_hdr_resource_match, 4051 (void *)resource); 4052 if (entry) { 4053 cache_resource = container_of(entry, 4054 struct mlx5_flow_dv_modify_hdr_resource, 4055 entry); 4056 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", 4057 (void *)cache_resource, 4058 rte_atomic32_read(&cache_resource->refcnt)); 4059 rte_atomic32_inc(&cache_resource->refcnt); 4060 dev_flow->handle->dvh.modify_hdr = cache_resource; 4061 return 0; 4062 4063 } 4064 /* Register new modify-header resource. */ 4065 cache_resource = mlx5_malloc(MLX5_MEM_ZERO, 4066 sizeof(*cache_resource) + actions_len, 0, 4067 SOCKET_ID_ANY); 4068 if (!cache_resource) 4069 return rte_flow_error_set(error, ENOMEM, 4070 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4071 "cannot allocate resource memory"); 4072 *cache_resource = *resource; 4073 rte_memcpy(cache_resource->actions, resource->actions, actions_len); 4074 ret = mlx5_flow_os_create_flow_action_modify_header 4075 (sh->ctx, ns, cache_resource, 4076 actions_len, &cache_resource->action); 4077 if (ret) { 4078 mlx5_free(cache_resource); 4079 return rte_flow_error_set(error, ENOMEM, 4080 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4081 NULL, "cannot create action"); 4082 } 4083 rte_atomic32_init(&cache_resource->refcnt); 4084 rte_atomic32_inc(&cache_resource->refcnt); 4085 if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry, 4086 flow_dv_modify_hdr_resource_match, 4087 (void *)cache_resource)) { 4088 claim_zero(mlx5_flow_os_destroy_flow_action 4089 (cache_resource->action)); 4090 mlx5_free(cache_resource); 4091 return rte_flow_error_set(error, EEXIST, 4092 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4093 NULL, "action exist"); 4094 } 4095 dev_flow->handle->dvh.modify_hdr = cache_resource; 4096 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", 4097 (void *)cache_resource, 4098 rte_atomic32_read(&cache_resource->refcnt)); 4099 return 0; 4100 } 4101 4102 /** 4103 * Get DV flow counter by index. 4104 * 4105 * @param[in] dev 4106 * Pointer to the Ethernet device structure. 4107 * @param[in] idx 4108 * mlx5 flow counter index in the container. 4109 * @param[out] ppool 4110 * mlx5 flow counter pool in the container, 4111 * 4112 * @return 4113 * Pointer to the counter, NULL otherwise. 4114 */ 4115 static struct mlx5_flow_counter * 4116 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev, 4117 uint32_t idx, 4118 struct mlx5_flow_counter_pool **ppool) 4119 { 4120 struct mlx5_priv *priv = dev->data->dev_private; 4121 struct mlx5_pools_container *cont; 4122 struct mlx5_flow_counter_pool *pool; 4123 uint32_t batch = 0, age = 0; 4124 4125 idx--; 4126 age = MLX_CNT_IS_AGE(idx); 4127 idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx; 4128 if (idx >= MLX5_CNT_BATCH_OFFSET) { 4129 idx -= MLX5_CNT_BATCH_OFFSET; 4130 batch = 1; 4131 } 4132 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4133 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n); 4134 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; 4135 MLX5_ASSERT(pool); 4136 if (ppool) 4137 *ppool = pool; 4138 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 4139 } 4140 4141 /** 4142 * Check the devx counter belongs to the pool. 4143 * 4144 * @param[in] pool 4145 * Pointer to the counter pool. 4146 * @param[in] id 4147 * The counter devx ID. 4148 * 4149 * @return 4150 * True if counter belongs to the pool, false otherwise. 4151 */ 4152 static bool 4153 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id) 4154 { 4155 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * 4156 MLX5_COUNTERS_PER_POOL; 4157 4158 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) 4159 return true; 4160 return false; 4161 } 4162 4163 /** 4164 * Get a pool by devx counter ID. 4165 * 4166 * @param[in] cont 4167 * Pointer to the counter container. 4168 * @param[in] id 4169 * The counter devx ID. 4170 * 4171 * @return 4172 * The counter pool pointer if exists, NULL otherwise, 4173 */ 4174 static struct mlx5_flow_counter_pool * 4175 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) 4176 { 4177 uint32_t i; 4178 4179 /* Check last used pool. */ 4180 if (cont->last_pool_idx != POOL_IDX_INVALID && 4181 flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id)) 4182 return cont->pools[cont->last_pool_idx]; 4183 /* ID out of range means no suitable pool in the container. */ 4184 if (id > cont->max_id || id < cont->min_id) 4185 return NULL; 4186 /* 4187 * Find the pool from the end of the container, since mostly counter 4188 * ID is sequence increasing, and the last pool should be the needed 4189 * one. 4190 */ 4191 i = rte_atomic16_read(&cont->n_valid); 4192 while (i--) { 4193 struct mlx5_flow_counter_pool *pool = cont->pools[i]; 4194 4195 if (flow_dv_is_counter_in_pool(pool, id)) 4196 return pool; 4197 } 4198 return NULL; 4199 } 4200 4201 /** 4202 * Allocate a new memory for the counter values wrapped by all the needed 4203 * management. 4204 * 4205 * @param[in] dev 4206 * Pointer to the Ethernet device structure. 4207 * @param[in] raws_n 4208 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters. 4209 * 4210 * @return 4211 * The new memory management pointer on success, otherwise NULL and rte_errno 4212 * is set. 4213 */ 4214 static struct mlx5_counter_stats_mem_mng * 4215 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) 4216 { 4217 struct mlx5_priv *priv = dev->data->dev_private; 4218 struct mlx5_dev_ctx_shared *sh = priv->sh; 4219 struct mlx5_devx_mkey_attr mkey_attr; 4220 struct mlx5_counter_stats_mem_mng *mem_mng; 4221 volatile struct flow_counter_stats *raw_data; 4222 int size = (sizeof(struct flow_counter_stats) * 4223 MLX5_COUNTERS_PER_POOL + 4224 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 4225 sizeof(struct mlx5_counter_stats_mem_mng); 4226 size_t pgsize = rte_mem_page_size(); 4227 if (pgsize == (size_t)-1) { 4228 DRV_LOG(ERR, "Failed to get mem page size"); 4229 rte_errno = ENOMEM; 4230 return NULL; 4231 } 4232 uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, 4233 SOCKET_ID_ANY); 4234 int i; 4235 4236 if (!mem) { 4237 rte_errno = ENOMEM; 4238 return NULL; 4239 } 4240 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 4241 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 4242 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 4243 IBV_ACCESS_LOCAL_WRITE); 4244 if (!mem_mng->umem) { 4245 rte_errno = errno; 4246 mlx5_free(mem); 4247 return NULL; 4248 } 4249 mkey_attr.addr = (uintptr_t)mem; 4250 mkey_attr.size = size; 4251 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 4252 mkey_attr.pd = sh->pdn; 4253 mkey_attr.log_entity_size = 0; 4254 mkey_attr.pg_access = 0; 4255 mkey_attr.klm_array = NULL; 4256 mkey_attr.klm_num = 0; 4257 if (priv->config.hca_attr.relaxed_ordering_write && 4258 priv->config.hca_attr.relaxed_ordering_read && 4259 !haswell_broadwell_cpu) 4260 mkey_attr.relaxed_ordering = 1; 4261 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 4262 if (!mem_mng->dm) { 4263 mlx5_glue->devx_umem_dereg(mem_mng->umem); 4264 rte_errno = errno; 4265 mlx5_free(mem); 4266 return NULL; 4267 } 4268 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 4269 raw_data = (volatile struct flow_counter_stats *)mem; 4270 for (i = 0; i < raws_n; ++i) { 4271 mem_mng->raws[i].mem_mng = mem_mng; 4272 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 4273 } 4274 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 4275 return mem_mng; 4276 } 4277 4278 /** 4279 * Resize a counter container. 4280 * 4281 * @param[in] dev 4282 * Pointer to the Ethernet device structure. 4283 * @param[in] batch 4284 * Whether the pool is for counter that was allocated by batch command. 4285 * @param[in] age 4286 * Whether the pool is for Aging counter. 4287 * 4288 * @return 4289 * 0 on success, otherwise negative errno value and rte_errno is set. 4290 */ 4291 static int 4292 flow_dv_container_resize(struct rte_eth_dev *dev, 4293 uint32_t batch, uint32_t age) 4294 { 4295 struct mlx5_priv *priv = dev->data->dev_private; 4296 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4297 age); 4298 struct mlx5_counter_stats_mem_mng *mem_mng = NULL; 4299 void *old_pools = cont->pools; 4300 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; 4301 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; 4302 void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); 4303 4304 if (!pools) { 4305 rte_errno = ENOMEM; 4306 return -ENOMEM; 4307 } 4308 if (old_pools) 4309 memcpy(pools, old_pools, cont->n * 4310 sizeof(struct mlx5_flow_counter_pool *)); 4311 /* 4312 * Fallback mode query the counter directly, no background query 4313 * resources are needed. 4314 */ 4315 if (!priv->counter_fallback) { 4316 int i; 4317 4318 mem_mng = flow_dv_create_counter_stat_mem_mng(dev, 4319 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); 4320 if (!mem_mng) { 4321 mlx5_free(pools); 4322 return -ENOMEM; 4323 } 4324 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 4325 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, 4326 mem_mng->raws + 4327 MLX5_CNT_CONTAINER_RESIZE + 4328 i, next); 4329 } 4330 rte_spinlock_lock(&cont->resize_sl); 4331 cont->n = resize; 4332 cont->mem_mng = mem_mng; 4333 cont->pools = pools; 4334 rte_spinlock_unlock(&cont->resize_sl); 4335 if (old_pools) 4336 mlx5_free(old_pools); 4337 return 0; 4338 } 4339 4340 /** 4341 * Query a devx flow counter. 4342 * 4343 * @param[in] dev 4344 * Pointer to the Ethernet device structure. 4345 * @param[in] cnt 4346 * Index to the flow counter. 4347 * @param[out] pkts 4348 * The statistics value of packets. 4349 * @param[out] bytes 4350 * The statistics value of bytes. 4351 * 4352 * @return 4353 * 0 on success, otherwise a negative errno value and rte_errno is set. 4354 */ 4355 static inline int 4356 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts, 4357 uint64_t *bytes) 4358 { 4359 struct mlx5_priv *priv = dev->data->dev_private; 4360 struct mlx5_flow_counter_pool *pool = NULL; 4361 struct mlx5_flow_counter *cnt; 4362 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4363 int offset; 4364 4365 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4366 MLX5_ASSERT(pool); 4367 if (counter < MLX5_CNT_BATCH_OFFSET) { 4368 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4369 if (priv->counter_fallback) 4370 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0, 4371 0, pkts, bytes, 0, NULL, NULL, 0); 4372 } 4373 4374 rte_spinlock_lock(&pool->sl); 4375 /* 4376 * The single counters allocation may allocate smaller ID than the 4377 * current allocated in parallel to the host reading. 4378 * In this case the new counter values must be reported as 0. 4379 */ 4380 if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) { 4381 *pkts = 0; 4382 *bytes = 0; 4383 } else { 4384 offset = MLX5_CNT_ARRAY_IDX(pool, cnt); 4385 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); 4386 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); 4387 } 4388 rte_spinlock_unlock(&pool->sl); 4389 return 0; 4390 } 4391 4392 /** 4393 * Create and initialize a new counter pool. 4394 * 4395 * @param[in] dev 4396 * Pointer to the Ethernet device structure. 4397 * @param[out] dcs 4398 * The devX counter handle. 4399 * @param[in] batch 4400 * Whether the pool is for counter that was allocated by batch command. 4401 * @param[in] age 4402 * Whether the pool is for counter that was allocated for aging. 4403 * @param[in/out] cont_cur 4404 * Pointer to the container pointer, it will be update in pool resize. 4405 * 4406 * @return 4407 * The pool container pointer on success, NULL otherwise and rte_errno is set. 4408 */ 4409 static struct mlx5_flow_counter_pool * 4410 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, 4411 uint32_t batch, uint32_t age) 4412 { 4413 struct mlx5_priv *priv = dev->data->dev_private; 4414 struct mlx5_flow_counter_pool *pool; 4415 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4416 age); 4417 int16_t n_valid = rte_atomic16_read(&cont->n_valid); 4418 uint32_t size = sizeof(*pool); 4419 4420 if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age)) 4421 return NULL; 4422 size += MLX5_COUNTERS_PER_POOL * CNT_SIZE; 4423 size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE); 4424 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE); 4425 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); 4426 if (!pool) { 4427 rte_errno = ENOMEM; 4428 return NULL; 4429 } 4430 pool->min_dcs = dcs; 4431 if (!priv->counter_fallback) 4432 pool->raw = cont->mem_mng->raws + n_valid % 4433 MLX5_CNT_CONTAINER_RESIZE; 4434 pool->raw_hw = NULL; 4435 pool->type = 0; 4436 pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT); 4437 pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE); 4438 pool->query_gen = 0; 4439 rte_spinlock_init(&pool->sl); 4440 TAILQ_INIT(&pool->counters[0]); 4441 TAILQ_INIT(&pool->counters[1]); 4442 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); 4443 pool->index = n_valid; 4444 cont->pools[n_valid] = pool; 4445 if (!batch) { 4446 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL); 4447 4448 if (base < cont->min_id) 4449 cont->min_id = base; 4450 if (base > cont->max_id) 4451 cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1; 4452 cont->last_pool_idx = pool->index; 4453 } 4454 /* Pool initialization must be updated before host thread access. */ 4455 rte_io_wmb(); 4456 rte_atomic16_add(&cont->n_valid, 1); 4457 return pool; 4458 } 4459 4460 /** 4461 * Restore skipped counters in the pool. 4462 * 4463 * As counter pool query requires the first counter dcs 4464 * ID start with 4 alinged, if the pool counters with 4465 * min_dcs ID are not aligned with 4, the counters will 4466 * be skipped. 4467 * Once other min_dcs ID less than these skipped counter 4468 * dcs ID appears, the skipped counters will be safe to 4469 * use. 4470 * Should be called when min_dcs is updated. 4471 * 4472 * @param[in] pool 4473 * Current counter pool. 4474 * @param[in] last_min_dcs 4475 * Last min_dcs. 4476 */ 4477 static void 4478 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool, 4479 struct mlx5_devx_obj *last_min_dcs) 4480 { 4481 struct mlx5_flow_counter_ext *cnt_ext; 4482 uint32_t offset, new_offset; 4483 uint32_t skip_cnt = 0; 4484 uint32_t i; 4485 4486 if (!pool->skip_cnt) 4487 return; 4488 /* 4489 * If last min_dcs is not valid. The skipped counter may even after 4490 * last min_dcs, set the offset to the whole pool. 4491 */ 4492 if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) 4493 offset = MLX5_COUNTERS_PER_POOL; 4494 else 4495 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL; 4496 new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL; 4497 /* 4498 * Check the counters from 1 to the last_min_dcs range. Counters 4499 * before new min_dcs indicates pool still has skipped counters. 4500 * Counters be skipped after new min_dcs will be ready to use. 4501 * Offset 0 counter must be empty or min_dcs, start from 1. 4502 */ 4503 for (i = 1; i < offset; i++) { 4504 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); 4505 if (cnt_ext->skipped) { 4506 if (i > new_offset) { 4507 cnt_ext->skipped = 0; 4508 TAILQ_INSERT_TAIL 4509 (&pool->counters[pool->query_gen], 4510 MLX5_POOL_GET_CNT(pool, i), next); 4511 } else { 4512 skip_cnt++; 4513 } 4514 } 4515 } 4516 if (!skip_cnt) 4517 pool->skip_cnt = 0; 4518 } 4519 4520 /** 4521 * Prepare a new counter and/or a new counter pool. 4522 * 4523 * @param[in] dev 4524 * Pointer to the Ethernet device structure. 4525 * @param[out] cnt_free 4526 * Where to put the pointer of a new counter. 4527 * @param[in] batch 4528 * Whether the pool is for counter that was allocated by batch command. 4529 * @param[in] age 4530 * Whether the pool is for counter that was allocated for aging. 4531 * 4532 * @return 4533 * The counter pool pointer and @p cnt_free is set on success, 4534 * NULL otherwise and rte_errno is set. 4535 */ 4536 static struct mlx5_flow_counter_pool * 4537 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, 4538 struct mlx5_flow_counter **cnt_free, 4539 uint32_t batch, uint32_t age) 4540 { 4541 struct mlx5_priv *priv = dev->data->dev_private; 4542 struct mlx5_pools_container *cont; 4543 struct mlx5_flow_counter_pool *pool; 4544 struct mlx5_counters tmp_tq; 4545 struct mlx5_devx_obj *last_min_dcs; 4546 struct mlx5_devx_obj *dcs = NULL; 4547 struct mlx5_flow_counter *cnt; 4548 uint32_t add2other; 4549 uint32_t i; 4550 4551 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4552 if (!batch) { 4553 retry: 4554 add2other = 0; 4555 /* bulk_bitmap must be 0 for single counter allocation. */ 4556 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); 4557 if (!dcs) 4558 return NULL; 4559 pool = flow_dv_find_pool_by_id(cont, dcs->id); 4560 /* Check if counter belongs to exist pool ID range. */ 4561 if (!pool) { 4562 pool = flow_dv_find_pool_by_id 4563 (MLX5_CNT_CONTAINER 4564 (priv->sh, batch, (age ^ 0x1)), dcs->id); 4565 /* 4566 * Pool eixsts, counter will be added to the other 4567 * container, need to reallocate it later. 4568 */ 4569 if (pool) { 4570 add2other = 1; 4571 } else { 4572 pool = flow_dv_pool_create(dev, dcs, batch, 4573 age); 4574 if (!pool) { 4575 mlx5_devx_cmd_destroy(dcs); 4576 return NULL; 4577 } 4578 } 4579 } 4580 if ((dcs->id < pool->min_dcs->id || 4581 pool->min_dcs->id & 4582 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) && 4583 !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) { 4584 /* 4585 * Update the pool min_dcs only if current dcs is 4586 * valid and exist min_dcs is not valid or greater 4587 * than new dcs. 4588 */ 4589 last_min_dcs = pool->min_dcs; 4590 rte_atomic64_set(&pool->a64_dcs, 4591 (int64_t)(uintptr_t)dcs); 4592 /* 4593 * Restore any skipped counters if the new min_dcs 4594 * ID is smaller or min_dcs is not valid. 4595 */ 4596 if (dcs->id < last_min_dcs->id || 4597 last_min_dcs->id & 4598 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) 4599 flow_dv_counter_restore(pool, last_min_dcs); 4600 } 4601 i = dcs->id % MLX5_COUNTERS_PER_POOL; 4602 cnt = MLX5_POOL_GET_CNT(pool, i); 4603 cnt->pool = pool; 4604 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs; 4605 /* 4606 * If min_dcs is not valid, it means the new allocated dcs 4607 * also fail to become the valid min_dcs, just skip it. 4608 * Or if min_dcs is valid, and new dcs ID is smaller than 4609 * min_dcs, but not become the min_dcs, also skip it. 4610 */ 4611 if (pool->min_dcs->id & 4612 (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) || 4613 dcs->id < pool->min_dcs->id) { 4614 MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1; 4615 pool->skip_cnt = 1; 4616 goto retry; 4617 } 4618 if (add2other) { 4619 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], 4620 cnt, next); 4621 goto retry; 4622 } 4623 *cnt_free = cnt; 4624 return pool; 4625 } 4626 /* bulk_bitmap is in 128 counters units. */ 4627 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) 4628 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); 4629 if (!dcs) { 4630 rte_errno = ENODATA; 4631 return NULL; 4632 } 4633 pool = flow_dv_pool_create(dev, dcs, batch, age); 4634 if (!pool) { 4635 mlx5_devx_cmd_destroy(dcs); 4636 return NULL; 4637 } 4638 TAILQ_INIT(&tmp_tq); 4639 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) { 4640 cnt = MLX5_POOL_GET_CNT(pool, i); 4641 cnt->pool = pool; 4642 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next); 4643 } 4644 rte_spinlock_lock(&cont->csl); 4645 TAILQ_CONCAT(&cont->counters, &tmp_tq, next); 4646 rte_spinlock_unlock(&cont->csl); 4647 *cnt_free = MLX5_POOL_GET_CNT(pool, 0); 4648 (*cnt_free)->pool = pool; 4649 return pool; 4650 } 4651 4652 /** 4653 * Search for existed shared counter. 4654 * 4655 * @param[in] dev 4656 * Pointer to the Ethernet device structure. 4657 * @param[in] id 4658 * The shared counter ID to search. 4659 * @param[out] ppool 4660 * mlx5 flow counter pool in the container, 4661 * 4662 * @return 4663 * NULL if not existed, otherwise pointer to the shared extend counter. 4664 */ 4665 static struct mlx5_flow_counter_ext * 4666 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id, 4667 struct mlx5_flow_counter_pool **ppool) 4668 { 4669 struct mlx5_priv *priv = dev->data->dev_private; 4670 union mlx5_l3t_data data; 4671 uint32_t cnt_idx; 4672 4673 if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword) 4674 return NULL; 4675 cnt_idx = data.dword; 4676 /* 4677 * Shared counters don't have age info. The counter extend is after 4678 * the counter datat structure. 4679 */ 4680 return (struct mlx5_flow_counter_ext *) 4681 ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1); 4682 } 4683 4684 /** 4685 * Allocate a flow counter. 4686 * 4687 * @param[in] dev 4688 * Pointer to the Ethernet device structure. 4689 * @param[in] shared 4690 * Indicate if this counter is shared with other flows. 4691 * @param[in] id 4692 * Counter identifier. 4693 * @param[in] group 4694 * Counter flow group. 4695 * @param[in] age 4696 * Whether the counter was allocated for aging. 4697 * 4698 * @return 4699 * Index to flow counter on success, 0 otherwise and rte_errno is set. 4700 */ 4701 static uint32_t 4702 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, 4703 uint16_t group, uint32_t age) 4704 { 4705 struct mlx5_priv *priv = dev->data->dev_private; 4706 struct mlx5_flow_counter_pool *pool = NULL; 4707 struct mlx5_flow_counter *cnt_free = NULL; 4708 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4709 /* 4710 * Currently group 0 flow counter cannot be assigned to a flow if it is 4711 * not the first one in the batch counter allocation, so it is better 4712 * to allocate counters one by one for these flows in a separate 4713 * container. 4714 * A counter can be shared between different groups so need to take 4715 * shared counters from the single container. 4716 */ 4717 uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0; 4718 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4719 age); 4720 uint32_t cnt_idx; 4721 4722 if (!priv->config.devx) { 4723 rte_errno = ENOTSUP; 4724 return 0; 4725 } 4726 if (shared) { 4727 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool); 4728 if (cnt_ext) { 4729 if (cnt_ext->ref_cnt + 1 == 0) { 4730 rte_errno = E2BIG; 4731 return 0; 4732 } 4733 cnt_ext->ref_cnt++; 4734 cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL + 4735 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL) 4736 + 1; 4737 return cnt_idx; 4738 } 4739 } 4740 /* Get free counters from container. */ 4741 rte_spinlock_lock(&cont->csl); 4742 cnt_free = TAILQ_FIRST(&cont->counters); 4743 if (cnt_free) 4744 TAILQ_REMOVE(&cont->counters, cnt_free, next); 4745 rte_spinlock_unlock(&cont->csl); 4746 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, 4747 batch, age)) 4748 goto err; 4749 pool = cnt_free->pool; 4750 if (!batch) 4751 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free); 4752 /* Create a DV counter action only in the first time usage. */ 4753 if (!cnt_free->action) { 4754 uint16_t offset; 4755 struct mlx5_devx_obj *dcs; 4756 int ret; 4757 4758 if (batch) { 4759 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free); 4760 dcs = pool->min_dcs; 4761 } else { 4762 offset = 0; 4763 dcs = cnt_ext->dcs; 4764 } 4765 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset, 4766 &cnt_free->action); 4767 if (ret) { 4768 rte_errno = errno; 4769 goto err; 4770 } 4771 } 4772 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index, 4773 MLX5_CNT_ARRAY_IDX(pool, cnt_free)); 4774 cnt_idx += batch * MLX5_CNT_BATCH_OFFSET; 4775 cnt_idx += age * MLX5_CNT_AGE_OFFSET; 4776 /* Update the counter reset values. */ 4777 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits, 4778 &cnt_free->bytes)) 4779 goto err; 4780 if (cnt_ext) { 4781 cnt_ext->shared = shared; 4782 cnt_ext->ref_cnt = 1; 4783 cnt_ext->id = id; 4784 if (shared) { 4785 union mlx5_l3t_data data; 4786 4787 data.dword = cnt_idx; 4788 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data)) 4789 return 0; 4790 } 4791 } 4792 if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on) 4793 /* Start the asynchronous batch query by the host thread. */ 4794 mlx5_set_query_alarm(priv->sh); 4795 return cnt_idx; 4796 err: 4797 if (cnt_free) { 4798 cnt_free->pool = pool; 4799 rte_spinlock_lock(&cont->csl); 4800 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next); 4801 rte_spinlock_unlock(&cont->csl); 4802 } 4803 return 0; 4804 } 4805 4806 /** 4807 * Get age param from counter index. 4808 * 4809 * @param[in] dev 4810 * Pointer to the Ethernet device structure. 4811 * @param[in] counter 4812 * Index to the counter handler. 4813 * 4814 * @return 4815 * The aging parameter specified for the counter index. 4816 */ 4817 static struct mlx5_age_param* 4818 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev, 4819 uint32_t counter) 4820 { 4821 struct mlx5_flow_counter *cnt; 4822 struct mlx5_flow_counter_pool *pool = NULL; 4823 4824 flow_dv_counter_get_by_idx(dev, counter, &pool); 4825 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL; 4826 cnt = MLX5_POOL_GET_CNT(pool, counter); 4827 return MLX5_CNT_TO_AGE(cnt); 4828 } 4829 4830 /** 4831 * Remove a flow counter from aged counter list. 4832 * 4833 * @param[in] dev 4834 * Pointer to the Ethernet device structure. 4835 * @param[in] counter 4836 * Index to the counter handler. 4837 * @param[in] cnt 4838 * Pointer to the counter handler. 4839 */ 4840 static void 4841 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, 4842 uint32_t counter, struct mlx5_flow_counter *cnt) 4843 { 4844 struct mlx5_age_info *age_info; 4845 struct mlx5_age_param *age_param; 4846 struct mlx5_priv *priv = dev->data->dev_private; 4847 4848 age_info = GET_PORT_AGE_INFO(priv); 4849 age_param = flow_dv_counter_idx_get_age(dev, counter); 4850 if (rte_atomic16_cmpset((volatile uint16_t *) 4851 &age_param->state, 4852 AGE_CANDIDATE, AGE_FREE) 4853 != AGE_CANDIDATE) { 4854 /** 4855 * We need the lock even it is age timeout, 4856 * since counter may still in process. 4857 */ 4858 rte_spinlock_lock(&age_info->aged_sl); 4859 TAILQ_REMOVE(&age_info->aged_counters, cnt, next); 4860 rte_spinlock_unlock(&age_info->aged_sl); 4861 } 4862 rte_atomic16_set(&age_param->state, AGE_FREE); 4863 } 4864 /** 4865 * Release a flow counter. 4866 * 4867 * @param[in] dev 4868 * Pointer to the Ethernet device structure. 4869 * @param[in] counter 4870 * Index to the counter handler. 4871 */ 4872 static void 4873 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) 4874 { 4875 struct mlx5_priv *priv = dev->data->dev_private; 4876 struct mlx5_flow_counter_pool *pool = NULL; 4877 struct mlx5_flow_counter *cnt; 4878 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4879 4880 if (!counter) 4881 return; 4882 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4883 MLX5_ASSERT(pool); 4884 if (counter < MLX5_CNT_BATCH_OFFSET) { 4885 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4886 if (cnt_ext) { 4887 if (--cnt_ext->ref_cnt) 4888 return; 4889 if (cnt_ext->shared) 4890 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, 4891 cnt_ext->id); 4892 } 4893 } 4894 if (IS_AGE_POOL(pool)) 4895 flow_dv_counter_remove_from_age(dev, counter, cnt); 4896 cnt->pool = pool; 4897 /* 4898 * Put the counter back to list to be updated in none fallback mode. 4899 * Currently, we are using two list alternately, while one is in query, 4900 * add the freed counter to the other list based on the pool query_gen 4901 * value. After query finishes, add counter the list to the global 4902 * container counter list. The list changes while query starts. In 4903 * this case, lock will not be needed as query callback and release 4904 * function both operate with the different list. 4905 * 4906 */ 4907 if (!priv->counter_fallback) 4908 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next); 4909 else 4910 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER 4911 (priv->sh, 0, 0))->counters), 4912 cnt, next); 4913 } 4914 4915 /** 4916 * Verify the @p attributes will be correctly understood by the NIC and store 4917 * them in the @p flow if everything is correct. 4918 * 4919 * @param[in] dev 4920 * Pointer to dev struct. 4921 * @param[in] attributes 4922 * Pointer to flow attributes 4923 * @param[in] external 4924 * This flow rule is created by request external to PMD. 4925 * @param[out] error 4926 * Pointer to error structure. 4927 * 4928 * @return 4929 * - 0 on success and non root table. 4930 * - 1 on success and root table. 4931 * - a negative errno value otherwise and rte_errno is set. 4932 */ 4933 static int 4934 flow_dv_validate_attributes(struct rte_eth_dev *dev, 4935 const struct rte_flow_attr *attributes, 4936 bool external __rte_unused, 4937 struct rte_flow_error *error) 4938 { 4939 struct mlx5_priv *priv = dev->data->dev_private; 4940 uint32_t priority_max = priv->config.flow_prio - 1; 4941 int ret = 0; 4942 4943 #ifndef HAVE_MLX5DV_DR 4944 if (attributes->group) 4945 return rte_flow_error_set(error, ENOTSUP, 4946 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 4947 NULL, 4948 "groups are not supported"); 4949 #else 4950 uint32_t table = 0; 4951 4952 ret = mlx5_flow_group_to_table(attributes, external, 4953 attributes->group, !!priv->fdb_def_rule, 4954 &table, error); 4955 if (ret) 4956 return ret; 4957 if (!table) 4958 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 4959 #endif 4960 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 4961 attributes->priority >= priority_max) 4962 return rte_flow_error_set(error, ENOTSUP, 4963 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 4964 NULL, 4965 "priority out of range"); 4966 if (attributes->transfer) { 4967 if (!priv->config.dv_esw_en) 4968 return rte_flow_error_set 4969 (error, ENOTSUP, 4970 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4971 "E-Switch dr is not supported"); 4972 if (!(priv->representor || priv->master)) 4973 return rte_flow_error_set 4974 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4975 NULL, "E-Switch configuration can only be" 4976 " done by a master or a representor device"); 4977 if (attributes->egress) 4978 return rte_flow_error_set 4979 (error, ENOTSUP, 4980 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, 4981 "egress is not supported"); 4982 } 4983 if (!(attributes->egress ^ attributes->ingress)) 4984 return rte_flow_error_set(error, ENOTSUP, 4985 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 4986 "must specify exactly one of " 4987 "ingress or egress"); 4988 return ret; 4989 } 4990 4991 /** 4992 * Internal validation function. For validating both actions and items. 4993 * 4994 * @param[in] dev 4995 * Pointer to the rte_eth_dev structure. 4996 * @param[in] attr 4997 * Pointer to the flow attributes. 4998 * @param[in] items 4999 * Pointer to the list of items. 5000 * @param[in] actions 5001 * Pointer to the list of actions. 5002 * @param[in] external 5003 * This flow rule is created by request external to PMD. 5004 * @param[in] hairpin 5005 * Number of hairpin TX actions, 0 means classic flow. 5006 * @param[out] error 5007 * Pointer to the error structure. 5008 * 5009 * @return 5010 * 0 on success, a negative errno value otherwise and rte_errno is set. 5011 */ 5012 static int 5013 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 5014 const struct rte_flow_item items[], 5015 const struct rte_flow_action actions[], 5016 bool external, int hairpin, struct rte_flow_error *error) 5017 { 5018 int ret; 5019 uint64_t action_flags = 0; 5020 uint64_t item_flags = 0; 5021 uint64_t last_item = 0; 5022 uint8_t next_protocol = 0xff; 5023 uint16_t ether_type = 0; 5024 int actions_n = 0; 5025 uint8_t item_ipv6_proto = 0; 5026 const struct rte_flow_item *gre_item = NULL; 5027 const struct rte_flow_action_raw_decap *decap; 5028 const struct rte_flow_action_raw_encap *encap; 5029 const struct rte_flow_action_rss *rss; 5030 const struct rte_flow_item_tcp nic_tcp_mask = { 5031 .hdr = { 5032 .tcp_flags = 0xFF, 5033 .src_port = RTE_BE16(UINT16_MAX), 5034 .dst_port = RTE_BE16(UINT16_MAX), 5035 } 5036 }; 5037 const struct rte_flow_item_ipv4 nic_ipv4_mask = { 5038 .hdr = { 5039 .src_addr = RTE_BE32(0xffffffff), 5040 .dst_addr = RTE_BE32(0xffffffff), 5041 .type_of_service = 0xff, 5042 .next_proto_id = 0xff, 5043 .time_to_live = 0xff, 5044 }, 5045 }; 5046 const struct rte_flow_item_ipv6 nic_ipv6_mask = { 5047 .hdr = { 5048 .src_addr = 5049 "\xff\xff\xff\xff\xff\xff\xff\xff" 5050 "\xff\xff\xff\xff\xff\xff\xff\xff", 5051 .dst_addr = 5052 "\xff\xff\xff\xff\xff\xff\xff\xff" 5053 "\xff\xff\xff\xff\xff\xff\xff\xff", 5054 .vtc_flow = RTE_BE32(0xffffffff), 5055 .proto = 0xff, 5056 .hop_limits = 0xff, 5057 }, 5058 }; 5059 const struct rte_flow_item_ecpri nic_ecpri_mask = { 5060 .hdr = { 5061 .common = { 5062 .u32 = 5063 RTE_BE32(((const struct rte_ecpri_common_hdr) { 5064 .type = 0xFF, 5065 }).u32), 5066 }, 5067 .dummy[0] = 0xffffffff, 5068 }, 5069 }; 5070 struct mlx5_priv *priv = dev->data->dev_private; 5071 struct mlx5_dev_config *dev_conf = &priv->config; 5072 uint16_t queue_index = 0xFFFF; 5073 const struct rte_flow_item_vlan *vlan_m = NULL; 5074 int16_t rw_act_num = 0; 5075 uint64_t is_root; 5076 5077 if (items == NULL) 5078 return -1; 5079 ret = flow_dv_validate_attributes(dev, attr, external, error); 5080 if (ret < 0) 5081 return ret; 5082 is_root = (uint64_t)ret; 5083 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5084 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 5085 int type = items->type; 5086 5087 if (!mlx5_flow_os_item_supported(type)) 5088 return rte_flow_error_set(error, ENOTSUP, 5089 RTE_FLOW_ERROR_TYPE_ITEM, 5090 NULL, "item not supported"); 5091 switch (type) { 5092 case RTE_FLOW_ITEM_TYPE_VOID: 5093 break; 5094 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5095 ret = flow_dv_validate_item_port_id 5096 (dev, items, attr, item_flags, error); 5097 if (ret < 0) 5098 return ret; 5099 last_item = MLX5_FLOW_ITEM_PORT_ID; 5100 break; 5101 case RTE_FLOW_ITEM_TYPE_ETH: 5102 ret = mlx5_flow_validate_item_eth(items, item_flags, 5103 error); 5104 if (ret < 0) 5105 return ret; 5106 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 5107 MLX5_FLOW_LAYER_OUTER_L2; 5108 if (items->mask != NULL && items->spec != NULL) { 5109 ether_type = 5110 ((const struct rte_flow_item_eth *) 5111 items->spec)->type; 5112 ether_type &= 5113 ((const struct rte_flow_item_eth *) 5114 items->mask)->type; 5115 ether_type = rte_be_to_cpu_16(ether_type); 5116 } else { 5117 ether_type = 0; 5118 } 5119 break; 5120 case RTE_FLOW_ITEM_TYPE_VLAN: 5121 ret = flow_dv_validate_item_vlan(items, item_flags, 5122 dev, error); 5123 if (ret < 0) 5124 return ret; 5125 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 5126 MLX5_FLOW_LAYER_OUTER_VLAN; 5127 if (items->mask != NULL && items->spec != NULL) { 5128 ether_type = 5129 ((const struct rte_flow_item_vlan *) 5130 items->spec)->inner_type; 5131 ether_type &= 5132 ((const struct rte_flow_item_vlan *) 5133 items->mask)->inner_type; 5134 ether_type = rte_be_to_cpu_16(ether_type); 5135 } else { 5136 ether_type = 0; 5137 } 5138 /* Store outer VLAN mask for of_push_vlan action. */ 5139 if (!tunnel) 5140 vlan_m = items->mask; 5141 break; 5142 case RTE_FLOW_ITEM_TYPE_IPV4: 5143 mlx5_flow_tunnel_ip_check(items, next_protocol, 5144 &item_flags, &tunnel); 5145 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 5146 last_item, 5147 ether_type, 5148 &nic_ipv4_mask, 5149 error); 5150 if (ret < 0) 5151 return ret; 5152 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 5153 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 5154 if (items->mask != NULL && 5155 ((const struct rte_flow_item_ipv4 *) 5156 items->mask)->hdr.next_proto_id) { 5157 next_protocol = 5158 ((const struct rte_flow_item_ipv4 *) 5159 (items->spec))->hdr.next_proto_id; 5160 next_protocol &= 5161 ((const struct rte_flow_item_ipv4 *) 5162 (items->mask))->hdr.next_proto_id; 5163 } else { 5164 /* Reset for inner layer. */ 5165 next_protocol = 0xff; 5166 } 5167 break; 5168 case RTE_FLOW_ITEM_TYPE_IPV6: 5169 mlx5_flow_tunnel_ip_check(items, next_protocol, 5170 &item_flags, &tunnel); 5171 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 5172 last_item, 5173 ether_type, 5174 &nic_ipv6_mask, 5175 error); 5176 if (ret < 0) 5177 return ret; 5178 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 5179 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 5180 if (items->mask != NULL && 5181 ((const struct rte_flow_item_ipv6 *) 5182 items->mask)->hdr.proto) { 5183 item_ipv6_proto = 5184 ((const struct rte_flow_item_ipv6 *) 5185 items->spec)->hdr.proto; 5186 next_protocol = 5187 ((const struct rte_flow_item_ipv6 *) 5188 items->spec)->hdr.proto; 5189 next_protocol &= 5190 ((const struct rte_flow_item_ipv6 *) 5191 items->mask)->hdr.proto; 5192 } else { 5193 /* Reset for inner layer. */ 5194 next_protocol = 0xff; 5195 } 5196 break; 5197 case RTE_FLOW_ITEM_TYPE_TCP: 5198 ret = mlx5_flow_validate_item_tcp 5199 (items, item_flags, 5200 next_protocol, 5201 &nic_tcp_mask, 5202 error); 5203 if (ret < 0) 5204 return ret; 5205 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 5206 MLX5_FLOW_LAYER_OUTER_L4_TCP; 5207 break; 5208 case RTE_FLOW_ITEM_TYPE_UDP: 5209 ret = mlx5_flow_validate_item_udp(items, item_flags, 5210 next_protocol, 5211 error); 5212 if (ret < 0) 5213 return ret; 5214 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 5215 MLX5_FLOW_LAYER_OUTER_L4_UDP; 5216 break; 5217 case RTE_FLOW_ITEM_TYPE_GRE: 5218 ret = mlx5_flow_validate_item_gre(items, item_flags, 5219 next_protocol, error); 5220 if (ret < 0) 5221 return ret; 5222 gre_item = items; 5223 last_item = MLX5_FLOW_LAYER_GRE; 5224 break; 5225 case RTE_FLOW_ITEM_TYPE_NVGRE: 5226 ret = mlx5_flow_validate_item_nvgre(items, item_flags, 5227 next_protocol, 5228 error); 5229 if (ret < 0) 5230 return ret; 5231 last_item = MLX5_FLOW_LAYER_NVGRE; 5232 break; 5233 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 5234 ret = mlx5_flow_validate_item_gre_key 5235 (items, item_flags, gre_item, error); 5236 if (ret < 0) 5237 return ret; 5238 last_item = MLX5_FLOW_LAYER_GRE_KEY; 5239 break; 5240 case RTE_FLOW_ITEM_TYPE_VXLAN: 5241 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 5242 error); 5243 if (ret < 0) 5244 return ret; 5245 last_item = MLX5_FLOW_LAYER_VXLAN; 5246 break; 5247 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 5248 ret = mlx5_flow_validate_item_vxlan_gpe(items, 5249 item_flags, dev, 5250 error); 5251 if (ret < 0) 5252 return ret; 5253 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 5254 break; 5255 case RTE_FLOW_ITEM_TYPE_GENEVE: 5256 ret = mlx5_flow_validate_item_geneve(items, 5257 item_flags, dev, 5258 error); 5259 if (ret < 0) 5260 return ret; 5261 last_item = MLX5_FLOW_LAYER_GENEVE; 5262 break; 5263 case RTE_FLOW_ITEM_TYPE_MPLS: 5264 ret = mlx5_flow_validate_item_mpls(dev, items, 5265 item_flags, 5266 last_item, error); 5267 if (ret < 0) 5268 return ret; 5269 last_item = MLX5_FLOW_LAYER_MPLS; 5270 break; 5271 5272 case RTE_FLOW_ITEM_TYPE_MARK: 5273 ret = flow_dv_validate_item_mark(dev, items, attr, 5274 error); 5275 if (ret < 0) 5276 return ret; 5277 last_item = MLX5_FLOW_ITEM_MARK; 5278 break; 5279 case RTE_FLOW_ITEM_TYPE_META: 5280 ret = flow_dv_validate_item_meta(dev, items, attr, 5281 error); 5282 if (ret < 0) 5283 return ret; 5284 last_item = MLX5_FLOW_ITEM_METADATA; 5285 break; 5286 case RTE_FLOW_ITEM_TYPE_ICMP: 5287 ret = mlx5_flow_validate_item_icmp(items, item_flags, 5288 next_protocol, 5289 error); 5290 if (ret < 0) 5291 return ret; 5292 last_item = MLX5_FLOW_LAYER_ICMP; 5293 break; 5294 case RTE_FLOW_ITEM_TYPE_ICMP6: 5295 ret = mlx5_flow_validate_item_icmp6(items, item_flags, 5296 next_protocol, 5297 error); 5298 if (ret < 0) 5299 return ret; 5300 item_ipv6_proto = IPPROTO_ICMPV6; 5301 last_item = MLX5_FLOW_LAYER_ICMP6; 5302 break; 5303 case RTE_FLOW_ITEM_TYPE_TAG: 5304 ret = flow_dv_validate_item_tag(dev, items, 5305 attr, error); 5306 if (ret < 0) 5307 return ret; 5308 last_item = MLX5_FLOW_ITEM_TAG; 5309 break; 5310 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 5311 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 5312 break; 5313 case RTE_FLOW_ITEM_TYPE_GTP: 5314 ret = flow_dv_validate_item_gtp(dev, items, item_flags, 5315 error); 5316 if (ret < 0) 5317 return ret; 5318 last_item = MLX5_FLOW_LAYER_GTP; 5319 break; 5320 case RTE_FLOW_ITEM_TYPE_ECPRI: 5321 /* Capacity will be checked in the translate stage. */ 5322 ret = mlx5_flow_validate_item_ecpri(items, item_flags, 5323 last_item, 5324 ether_type, 5325 &nic_ecpri_mask, 5326 error); 5327 if (ret < 0) 5328 return ret; 5329 last_item = MLX5_FLOW_LAYER_ECPRI; 5330 break; 5331 default: 5332 return rte_flow_error_set(error, ENOTSUP, 5333 RTE_FLOW_ERROR_TYPE_ITEM, 5334 NULL, "item not supported"); 5335 } 5336 item_flags |= last_item; 5337 } 5338 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5339 int type = actions->type; 5340 5341 if (!mlx5_flow_os_action_supported(type)) 5342 return rte_flow_error_set(error, ENOTSUP, 5343 RTE_FLOW_ERROR_TYPE_ACTION, 5344 actions, 5345 "action not supported"); 5346 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 5347 return rte_flow_error_set(error, ENOTSUP, 5348 RTE_FLOW_ERROR_TYPE_ACTION, 5349 actions, "too many actions"); 5350 switch (type) { 5351 case RTE_FLOW_ACTION_TYPE_VOID: 5352 break; 5353 case RTE_FLOW_ACTION_TYPE_PORT_ID: 5354 ret = flow_dv_validate_action_port_id(dev, 5355 action_flags, 5356 actions, 5357 attr, 5358 error); 5359 if (ret) 5360 return ret; 5361 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 5362 ++actions_n; 5363 break; 5364 case RTE_FLOW_ACTION_TYPE_FLAG: 5365 ret = flow_dv_validate_action_flag(dev, action_flags, 5366 attr, error); 5367 if (ret < 0) 5368 return ret; 5369 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5370 /* Count all modify-header actions as one. */ 5371 if (!(action_flags & 5372 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5373 ++actions_n; 5374 action_flags |= MLX5_FLOW_ACTION_FLAG | 5375 MLX5_FLOW_ACTION_MARK_EXT; 5376 } else { 5377 action_flags |= MLX5_FLOW_ACTION_FLAG; 5378 ++actions_n; 5379 } 5380 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5381 break; 5382 case RTE_FLOW_ACTION_TYPE_MARK: 5383 ret = flow_dv_validate_action_mark(dev, actions, 5384 action_flags, 5385 attr, error); 5386 if (ret < 0) 5387 return ret; 5388 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5389 /* Count all modify-header actions as one. */ 5390 if (!(action_flags & 5391 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5392 ++actions_n; 5393 action_flags |= MLX5_FLOW_ACTION_MARK | 5394 MLX5_FLOW_ACTION_MARK_EXT; 5395 } else { 5396 action_flags |= MLX5_FLOW_ACTION_MARK; 5397 ++actions_n; 5398 } 5399 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5400 break; 5401 case RTE_FLOW_ACTION_TYPE_SET_META: 5402 ret = flow_dv_validate_action_set_meta(dev, actions, 5403 action_flags, 5404 attr, error); 5405 if (ret < 0) 5406 return ret; 5407 /* Count all modify-header actions as one action. */ 5408 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5409 ++actions_n; 5410 action_flags |= MLX5_FLOW_ACTION_SET_META; 5411 rw_act_num += MLX5_ACT_NUM_SET_META; 5412 break; 5413 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5414 ret = flow_dv_validate_action_set_tag(dev, actions, 5415 action_flags, 5416 attr, error); 5417 if (ret < 0) 5418 return ret; 5419 /* Count all modify-header actions as one action. */ 5420 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5421 ++actions_n; 5422 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 5423 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5424 break; 5425 case RTE_FLOW_ACTION_TYPE_DROP: 5426 ret = mlx5_flow_validate_action_drop(action_flags, 5427 attr, error); 5428 if (ret < 0) 5429 return ret; 5430 action_flags |= MLX5_FLOW_ACTION_DROP; 5431 ++actions_n; 5432 break; 5433 case RTE_FLOW_ACTION_TYPE_QUEUE: 5434 ret = mlx5_flow_validate_action_queue(actions, 5435 action_flags, dev, 5436 attr, error); 5437 if (ret < 0) 5438 return ret; 5439 queue_index = ((const struct rte_flow_action_queue *) 5440 (actions->conf))->index; 5441 action_flags |= MLX5_FLOW_ACTION_QUEUE; 5442 ++actions_n; 5443 break; 5444 case RTE_FLOW_ACTION_TYPE_RSS: 5445 rss = actions->conf; 5446 ret = mlx5_flow_validate_action_rss(actions, 5447 action_flags, dev, 5448 attr, item_flags, 5449 error); 5450 if (ret < 0) 5451 return ret; 5452 if (rss != NULL && rss->queue_num) 5453 queue_index = rss->queue[0]; 5454 action_flags |= MLX5_FLOW_ACTION_RSS; 5455 ++actions_n; 5456 break; 5457 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 5458 ret = 5459 mlx5_flow_validate_action_default_miss(action_flags, 5460 attr, error); 5461 if (ret < 0) 5462 return ret; 5463 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 5464 ++actions_n; 5465 break; 5466 case RTE_FLOW_ACTION_TYPE_COUNT: 5467 ret = flow_dv_validate_action_count(dev, error); 5468 if (ret < 0) 5469 return ret; 5470 action_flags |= MLX5_FLOW_ACTION_COUNT; 5471 ++actions_n; 5472 break; 5473 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5474 if (flow_dv_validate_action_pop_vlan(dev, 5475 action_flags, 5476 actions, 5477 item_flags, attr, 5478 error)) 5479 return -rte_errno; 5480 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 5481 ++actions_n; 5482 break; 5483 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5484 ret = flow_dv_validate_action_push_vlan(dev, 5485 action_flags, 5486 vlan_m, 5487 actions, attr, 5488 error); 5489 if (ret < 0) 5490 return ret; 5491 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 5492 ++actions_n; 5493 break; 5494 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5495 ret = flow_dv_validate_action_set_vlan_pcp 5496 (action_flags, actions, error); 5497 if (ret < 0) 5498 return ret; 5499 /* Count PCP with push_vlan command. */ 5500 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; 5501 break; 5502 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5503 ret = flow_dv_validate_action_set_vlan_vid 5504 (item_flags, action_flags, 5505 actions, error); 5506 if (ret < 0) 5507 return ret; 5508 /* Count VID with push_vlan command. */ 5509 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 5510 rw_act_num += MLX5_ACT_NUM_MDF_VID; 5511 break; 5512 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5513 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5514 ret = flow_dv_validate_action_l2_encap(dev, 5515 action_flags, 5516 actions, attr, 5517 error); 5518 if (ret < 0) 5519 return ret; 5520 action_flags |= MLX5_FLOW_ACTION_ENCAP; 5521 ++actions_n; 5522 break; 5523 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5524 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5525 ret = flow_dv_validate_action_decap(dev, action_flags, 5526 attr, error); 5527 if (ret < 0) 5528 return ret; 5529 action_flags |= MLX5_FLOW_ACTION_DECAP; 5530 ++actions_n; 5531 break; 5532 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5533 ret = flow_dv_validate_action_raw_encap_decap 5534 (dev, NULL, actions->conf, attr, &action_flags, 5535 &actions_n, error); 5536 if (ret < 0) 5537 return ret; 5538 break; 5539 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5540 decap = actions->conf; 5541 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5542 ; 5543 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5544 encap = NULL; 5545 actions--; 5546 } else { 5547 encap = actions->conf; 5548 } 5549 ret = flow_dv_validate_action_raw_encap_decap 5550 (dev, 5551 decap ? decap : &empty_decap, encap, 5552 attr, &action_flags, &actions_n, 5553 error); 5554 if (ret < 0) 5555 return ret; 5556 break; 5557 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5558 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5559 ret = flow_dv_validate_action_modify_mac(action_flags, 5560 actions, 5561 item_flags, 5562 error); 5563 if (ret < 0) 5564 return ret; 5565 /* Count all modify-header actions as one action. */ 5566 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5567 ++actions_n; 5568 action_flags |= actions->type == 5569 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 5570 MLX5_FLOW_ACTION_SET_MAC_SRC : 5571 MLX5_FLOW_ACTION_SET_MAC_DST; 5572 /* 5573 * Even if the source and destination MAC addresses have 5574 * overlap in the header with 4B alignment, the convert 5575 * function will handle them separately and 4 SW actions 5576 * will be created. And 2 actions will be added each 5577 * time no matter how many bytes of address will be set. 5578 */ 5579 rw_act_num += MLX5_ACT_NUM_MDF_MAC; 5580 break; 5581 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5582 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5583 ret = flow_dv_validate_action_modify_ipv4(action_flags, 5584 actions, 5585 item_flags, 5586 error); 5587 if (ret < 0) 5588 return ret; 5589 /* Count all modify-header actions as one action. */ 5590 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5591 ++actions_n; 5592 action_flags |= actions->type == 5593 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 5594 MLX5_FLOW_ACTION_SET_IPV4_SRC : 5595 MLX5_FLOW_ACTION_SET_IPV4_DST; 5596 rw_act_num += MLX5_ACT_NUM_MDF_IPV4; 5597 break; 5598 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5599 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5600 ret = flow_dv_validate_action_modify_ipv6(action_flags, 5601 actions, 5602 item_flags, 5603 error); 5604 if (ret < 0) 5605 return ret; 5606 if (item_ipv6_proto == IPPROTO_ICMPV6) 5607 return rte_flow_error_set(error, ENOTSUP, 5608 RTE_FLOW_ERROR_TYPE_ACTION, 5609 actions, 5610 "Can't change header " 5611 "with ICMPv6 proto"); 5612 /* Count all modify-header actions as one action. */ 5613 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5614 ++actions_n; 5615 action_flags |= actions->type == 5616 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 5617 MLX5_FLOW_ACTION_SET_IPV6_SRC : 5618 MLX5_FLOW_ACTION_SET_IPV6_DST; 5619 rw_act_num += MLX5_ACT_NUM_MDF_IPV6; 5620 break; 5621 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5622 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5623 ret = flow_dv_validate_action_modify_tp(action_flags, 5624 actions, 5625 item_flags, 5626 error); 5627 if (ret < 0) 5628 return ret; 5629 /* Count all modify-header actions as one action. */ 5630 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5631 ++actions_n; 5632 action_flags |= actions->type == 5633 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 5634 MLX5_FLOW_ACTION_SET_TP_SRC : 5635 MLX5_FLOW_ACTION_SET_TP_DST; 5636 rw_act_num += MLX5_ACT_NUM_MDF_PORT; 5637 break; 5638 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5639 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5640 ret = flow_dv_validate_action_modify_ttl(action_flags, 5641 actions, 5642 item_flags, 5643 error); 5644 if (ret < 0) 5645 return ret; 5646 /* Count all modify-header actions as one action. */ 5647 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5648 ++actions_n; 5649 action_flags |= actions->type == 5650 RTE_FLOW_ACTION_TYPE_SET_TTL ? 5651 MLX5_FLOW_ACTION_SET_TTL : 5652 MLX5_FLOW_ACTION_DEC_TTL; 5653 rw_act_num += MLX5_ACT_NUM_MDF_TTL; 5654 break; 5655 case RTE_FLOW_ACTION_TYPE_JUMP: 5656 ret = flow_dv_validate_action_jump(actions, 5657 action_flags, 5658 attr, external, 5659 error); 5660 if (ret) 5661 return ret; 5662 ++actions_n; 5663 action_flags |= MLX5_FLOW_ACTION_JUMP; 5664 break; 5665 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5666 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5667 ret = flow_dv_validate_action_modify_tcp_seq 5668 (action_flags, 5669 actions, 5670 item_flags, 5671 error); 5672 if (ret < 0) 5673 return ret; 5674 /* Count all modify-header actions as one action. */ 5675 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5676 ++actions_n; 5677 action_flags |= actions->type == 5678 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 5679 MLX5_FLOW_ACTION_INC_TCP_SEQ : 5680 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 5681 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ; 5682 break; 5683 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5684 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5685 ret = flow_dv_validate_action_modify_tcp_ack 5686 (action_flags, 5687 actions, 5688 item_flags, 5689 error); 5690 if (ret < 0) 5691 return ret; 5692 /* Count all modify-header actions as one action. */ 5693 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5694 ++actions_n; 5695 action_flags |= actions->type == 5696 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 5697 MLX5_FLOW_ACTION_INC_TCP_ACK : 5698 MLX5_FLOW_ACTION_DEC_TCP_ACK; 5699 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK; 5700 break; 5701 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 5702 break; 5703 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 5704 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 5705 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5706 break; 5707 case RTE_FLOW_ACTION_TYPE_METER: 5708 ret = mlx5_flow_validate_action_meter(dev, 5709 action_flags, 5710 actions, attr, 5711 error); 5712 if (ret < 0) 5713 return ret; 5714 action_flags |= MLX5_FLOW_ACTION_METER; 5715 ++actions_n; 5716 /* Meter action will add one more TAG action. */ 5717 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5718 break; 5719 case RTE_FLOW_ACTION_TYPE_AGE: 5720 ret = flow_dv_validate_action_age(action_flags, 5721 actions, dev, 5722 error); 5723 if (ret < 0) 5724 return ret; 5725 action_flags |= MLX5_FLOW_ACTION_AGE; 5726 ++actions_n; 5727 break; 5728 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5729 ret = flow_dv_validate_action_modify_ipv4_dscp 5730 (action_flags, 5731 actions, 5732 item_flags, 5733 error); 5734 if (ret < 0) 5735 return ret; 5736 /* Count all modify-header actions as one action. */ 5737 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5738 ++actions_n; 5739 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 5740 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5741 break; 5742 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5743 ret = flow_dv_validate_action_modify_ipv6_dscp 5744 (action_flags, 5745 actions, 5746 item_flags, 5747 error); 5748 if (ret < 0) 5749 return ret; 5750 /* Count all modify-header actions as one action. */ 5751 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5752 ++actions_n; 5753 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 5754 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5755 break; 5756 default: 5757 return rte_flow_error_set(error, ENOTSUP, 5758 RTE_FLOW_ERROR_TYPE_ACTION, 5759 actions, 5760 "action not supported"); 5761 } 5762 } 5763 /* 5764 * Validate the drop action mutual exclusion with other actions. 5765 * Drop action is mutually-exclusive with any other action, except for 5766 * Count action. 5767 */ 5768 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 5769 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 5770 return rte_flow_error_set(error, EINVAL, 5771 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5772 "Drop action is mutually-exclusive " 5773 "with any other action, except for " 5774 "Count action"); 5775 /* Eswitch has few restrictions on using items and actions */ 5776 if (attr->transfer) { 5777 if (!mlx5_flow_ext_mreg_supported(dev) && 5778 action_flags & MLX5_FLOW_ACTION_FLAG) 5779 return rte_flow_error_set(error, ENOTSUP, 5780 RTE_FLOW_ERROR_TYPE_ACTION, 5781 NULL, 5782 "unsupported action FLAG"); 5783 if (!mlx5_flow_ext_mreg_supported(dev) && 5784 action_flags & MLX5_FLOW_ACTION_MARK) 5785 return rte_flow_error_set(error, ENOTSUP, 5786 RTE_FLOW_ERROR_TYPE_ACTION, 5787 NULL, 5788 "unsupported action MARK"); 5789 if (action_flags & MLX5_FLOW_ACTION_QUEUE) 5790 return rte_flow_error_set(error, ENOTSUP, 5791 RTE_FLOW_ERROR_TYPE_ACTION, 5792 NULL, 5793 "unsupported action QUEUE"); 5794 if (action_flags & MLX5_FLOW_ACTION_RSS) 5795 return rte_flow_error_set(error, ENOTSUP, 5796 RTE_FLOW_ERROR_TYPE_ACTION, 5797 NULL, 5798 "unsupported action RSS"); 5799 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 5800 return rte_flow_error_set(error, EINVAL, 5801 RTE_FLOW_ERROR_TYPE_ACTION, 5802 actions, 5803 "no fate action is found"); 5804 } else { 5805 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 5806 return rte_flow_error_set(error, EINVAL, 5807 RTE_FLOW_ERROR_TYPE_ACTION, 5808 actions, 5809 "no fate action is found"); 5810 } 5811 /* Continue validation for Xcap and VLAN actions.*/ 5812 if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS | 5813 MLX5_FLOW_VLAN_ACTIONS)) && 5814 (queue_index == 0xFFFF || 5815 mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) { 5816 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) == 5817 MLX5_FLOW_XCAP_ACTIONS) 5818 return rte_flow_error_set(error, ENOTSUP, 5819 RTE_FLOW_ERROR_TYPE_ACTION, 5820 NULL, "encap and decap " 5821 "combination aren't supported"); 5822 if (!attr->transfer && attr->ingress) { 5823 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 5824 return rte_flow_error_set 5825 (error, ENOTSUP, 5826 RTE_FLOW_ERROR_TYPE_ACTION, 5827 NULL, "encap is not supported" 5828 " for ingress traffic"); 5829 else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 5830 return rte_flow_error_set 5831 (error, ENOTSUP, 5832 RTE_FLOW_ERROR_TYPE_ACTION, 5833 NULL, "push VLAN action not " 5834 "supported for ingress"); 5835 else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) == 5836 MLX5_FLOW_VLAN_ACTIONS) 5837 return rte_flow_error_set 5838 (error, ENOTSUP, 5839 RTE_FLOW_ERROR_TYPE_ACTION, 5840 NULL, "no support for " 5841 "multiple VLAN actions"); 5842 } 5843 } 5844 /* Hairpin flow will add one more TAG action. */ 5845 if (hairpin > 0) 5846 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5847 /* extra metadata enabled: one more TAG action will be add. */ 5848 if (dev_conf->dv_flow_en && 5849 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 5850 mlx5_flow_ext_mreg_supported(dev)) 5851 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5852 if ((uint32_t)rw_act_num > 5853 flow_dv_modify_hdr_action_max(dev, is_root)) { 5854 return rte_flow_error_set(error, ENOTSUP, 5855 RTE_FLOW_ERROR_TYPE_ACTION, 5856 NULL, "too many header modify" 5857 " actions to support"); 5858 } 5859 return 0; 5860 } 5861 5862 /** 5863 * Internal preparation function. Allocates the DV flow size, 5864 * this size is constant. 5865 * 5866 * @param[in] dev 5867 * Pointer to the rte_eth_dev structure. 5868 * @param[in] attr 5869 * Pointer to the flow attributes. 5870 * @param[in] items 5871 * Pointer to the list of items. 5872 * @param[in] actions 5873 * Pointer to the list of actions. 5874 * @param[out] error 5875 * Pointer to the error structure. 5876 * 5877 * @return 5878 * Pointer to mlx5_flow object on success, 5879 * otherwise NULL and rte_errno is set. 5880 */ 5881 static struct mlx5_flow * 5882 flow_dv_prepare(struct rte_eth_dev *dev, 5883 const struct rte_flow_attr *attr __rte_unused, 5884 const struct rte_flow_item items[] __rte_unused, 5885 const struct rte_flow_action actions[] __rte_unused, 5886 struct rte_flow_error *error) 5887 { 5888 uint32_t handle_idx = 0; 5889 struct mlx5_flow *dev_flow; 5890 struct mlx5_flow_handle *dev_handle; 5891 struct mlx5_priv *priv = dev->data->dev_private; 5892 5893 /* In case of corrupting the memory. */ 5894 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 5895 rte_flow_error_set(error, ENOSPC, 5896 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5897 "not free temporary device flow"); 5898 return NULL; 5899 } 5900 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 5901 &handle_idx); 5902 if (!dev_handle) { 5903 rte_flow_error_set(error, ENOMEM, 5904 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5905 "not enough memory to create flow handle"); 5906 return NULL; 5907 } 5908 /* No multi-thread supporting. */ 5909 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; 5910 dev_flow->handle = dev_handle; 5911 dev_flow->handle_idx = handle_idx; 5912 /* 5913 * In some old rdma-core releases, before continuing, a check of the 5914 * length of matching parameter will be done at first. It needs to use 5915 * the length without misc4 param. If the flow has misc4 support, then 5916 * the length needs to be adjusted accordingly. Each param member is 5917 * aligned with a 64B boundary naturally. 5918 */ 5919 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) - 5920 MLX5_ST_SZ_BYTES(fte_match_set_misc4); 5921 /* 5922 * The matching value needs to be cleared to 0 before using. In the 5923 * past, it will be automatically cleared when using rte_*alloc 5924 * API. The time consumption will be almost the same as before. 5925 */ 5926 memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); 5927 dev_flow->ingress = attr->ingress; 5928 dev_flow->dv.transfer = attr->transfer; 5929 return dev_flow; 5930 } 5931 5932 #ifdef RTE_LIBRTE_MLX5_DEBUG 5933 /** 5934 * Sanity check for match mask and value. Similar to check_valid_spec() in 5935 * kernel driver. If unmasked bit is present in value, it returns failure. 5936 * 5937 * @param match_mask 5938 * pointer to match mask buffer. 5939 * @param match_value 5940 * pointer to match value buffer. 5941 * 5942 * @return 5943 * 0 if valid, -EINVAL otherwise. 5944 */ 5945 static int 5946 flow_dv_check_valid_spec(void *match_mask, void *match_value) 5947 { 5948 uint8_t *m = match_mask; 5949 uint8_t *v = match_value; 5950 unsigned int i; 5951 5952 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { 5953 if (v[i] & ~m[i]) { 5954 DRV_LOG(ERR, 5955 "match_value differs from match_criteria" 5956 " %p[%u] != %p[%u]", 5957 match_value, i, match_mask, i); 5958 return -EINVAL; 5959 } 5960 } 5961 return 0; 5962 } 5963 #endif 5964 5965 /** 5966 * Add match of ip_version. 5967 * 5968 * @param[in] group 5969 * Flow group. 5970 * @param[in] headers_v 5971 * Values header pointer. 5972 * @param[in] headers_m 5973 * Masks header pointer. 5974 * @param[in] ip_version 5975 * The IP version to set. 5976 */ 5977 static inline void 5978 flow_dv_set_match_ip_version(uint32_t group, 5979 void *headers_v, 5980 void *headers_m, 5981 uint8_t ip_version) 5982 { 5983 if (group == 0) 5984 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 5985 else 5986 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 5987 ip_version); 5988 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version); 5989 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0); 5990 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0); 5991 } 5992 5993 /** 5994 * Add Ethernet item to matcher and to the value. 5995 * 5996 * @param[in, out] matcher 5997 * Flow matcher. 5998 * @param[in, out] key 5999 * Flow matcher value. 6000 * @param[in] item 6001 * Flow pattern to translate. 6002 * @param[in] inner 6003 * Item is inner pattern. 6004 */ 6005 static void 6006 flow_dv_translate_item_eth(void *matcher, void *key, 6007 const struct rte_flow_item *item, int inner, 6008 uint32_t group) 6009 { 6010 const struct rte_flow_item_eth *eth_m = item->mask; 6011 const struct rte_flow_item_eth *eth_v = item->spec; 6012 const struct rte_flow_item_eth nic_mask = { 6013 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 6014 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 6015 .type = RTE_BE16(0xffff), 6016 }; 6017 void *headers_m; 6018 void *headers_v; 6019 char *l24_v; 6020 unsigned int i; 6021 6022 if (!eth_v) 6023 return; 6024 if (!eth_m) 6025 eth_m = &nic_mask; 6026 if (inner) { 6027 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6028 inner_headers); 6029 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6030 } else { 6031 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6032 outer_headers); 6033 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6034 } 6035 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), 6036 ð_m->dst, sizeof(eth_m->dst)); 6037 /* The value must be in the range of the mask. */ 6038 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); 6039 for (i = 0; i < sizeof(eth_m->dst); ++i) 6040 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 6041 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), 6042 ð_m->src, sizeof(eth_m->src)); 6043 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); 6044 /* The value must be in the range of the mask. */ 6045 for (i = 0; i < sizeof(eth_m->dst); ++i) 6046 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 6047 if (eth_v->type) { 6048 /* When ethertype is present set mask for tagged VLAN. */ 6049 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6050 /* Set value for tagged VLAN if ethertype is 802.1Q. */ 6051 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 6052 eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 6053 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 6054 1); 6055 /* Return here to avoid setting match on ethertype. */ 6056 return; 6057 } 6058 } 6059 /* 6060 * HW supports match on one Ethertype, the Ethertype following the last 6061 * VLAN tag of the packet (see PRM). 6062 * Set match on ethertype only if ETH header is not followed by VLAN. 6063 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 6064 * ethertype, and use ip_version field instead. 6065 * eCPRI over Ether layer will use type value 0xAEFE. 6066 */ 6067 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 6068 eth_m->type == 0xFFFF) { 6069 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6070 } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 6071 eth_m->type == 0xFFFF) { 6072 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6073 } else { 6074 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 6075 rte_be_to_cpu_16(eth_m->type)); 6076 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6077 ethertype); 6078 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 6079 } 6080 } 6081 6082 /** 6083 * Add VLAN item to matcher and to the value. 6084 * 6085 * @param[in, out] dev_flow 6086 * Flow descriptor. 6087 * @param[in, out] matcher 6088 * Flow matcher. 6089 * @param[in, out] key 6090 * Flow matcher value. 6091 * @param[in] item 6092 * Flow pattern to translate. 6093 * @param[in] inner 6094 * Item is inner pattern. 6095 */ 6096 static void 6097 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, 6098 void *matcher, void *key, 6099 const struct rte_flow_item *item, 6100 int inner, uint32_t group) 6101 { 6102 const struct rte_flow_item_vlan *vlan_m = item->mask; 6103 const struct rte_flow_item_vlan *vlan_v = item->spec; 6104 void *headers_m; 6105 void *headers_v; 6106 uint16_t tci_m; 6107 uint16_t tci_v; 6108 6109 if (inner) { 6110 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6111 inner_headers); 6112 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6113 } else { 6114 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6115 outer_headers); 6116 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6117 /* 6118 * This is workaround, masks are not supported, 6119 * and pre-validated. 6120 */ 6121 if (vlan_v) 6122 dev_flow->handle->vf_vlan.tag = 6123 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; 6124 } 6125 /* 6126 * When VLAN item exists in flow, mark packet as tagged, 6127 * even if TCI is not specified. 6128 */ 6129 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6130 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 6131 if (!vlan_v) 6132 return; 6133 if (!vlan_m) 6134 vlan_m = &rte_flow_item_vlan_mask; 6135 tci_m = rte_be_to_cpu_16(vlan_m->tci); 6136 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 6137 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); 6138 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); 6139 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); 6140 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); 6141 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); 6142 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); 6143 /* 6144 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 6145 * ethertype, and use ip_version field instead. 6146 */ 6147 if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 6148 vlan_m->inner_type == 0xFFFF) { 6149 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6150 } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 6151 vlan_m->inner_type == 0xFFFF) { 6152 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6153 } else { 6154 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 6155 rte_be_to_cpu_16(vlan_m->inner_type)); 6156 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 6157 rte_be_to_cpu_16(vlan_m->inner_type & 6158 vlan_v->inner_type)); 6159 } 6160 } 6161 6162 /** 6163 * Add IPV4 item to matcher and to the value. 6164 * 6165 * @param[in, out] matcher 6166 * Flow matcher. 6167 * @param[in, out] key 6168 * Flow matcher value. 6169 * @param[in] item 6170 * Flow pattern to translate. 6171 * @param[in] item_flags 6172 * Bit-fields that holds the items detected until now. 6173 * @param[in] inner 6174 * Item is inner pattern. 6175 * @param[in] group 6176 * The group to insert the rule. 6177 */ 6178 static void 6179 flow_dv_translate_item_ipv4(void *matcher, void *key, 6180 const struct rte_flow_item *item, 6181 const uint64_t item_flags, 6182 int inner, uint32_t group) 6183 { 6184 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 6185 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 6186 const struct rte_flow_item_ipv4 nic_mask = { 6187 .hdr = { 6188 .src_addr = RTE_BE32(0xffffffff), 6189 .dst_addr = RTE_BE32(0xffffffff), 6190 .type_of_service = 0xff, 6191 .next_proto_id = 0xff, 6192 .time_to_live = 0xff, 6193 }, 6194 }; 6195 void *headers_m; 6196 void *headers_v; 6197 char *l24_m; 6198 char *l24_v; 6199 uint8_t tos; 6200 6201 if (inner) { 6202 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6203 inner_headers); 6204 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6205 } else { 6206 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6207 outer_headers); 6208 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6209 } 6210 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6211 /* 6212 * On outer header (which must contains L2), or inner header with L2, 6213 * set cvlan_tag mask bit to mark this packet as untagged. 6214 * This should be done even if item->spec is empty. 6215 */ 6216 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6217 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6218 if (!ipv4_v) 6219 return; 6220 if (!ipv4_m) 6221 ipv4_m = &nic_mask; 6222 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6223 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6224 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6225 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6226 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 6227 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 6228 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6229 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6230 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6231 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6232 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 6233 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 6234 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 6235 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 6236 ipv4_m->hdr.type_of_service); 6237 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 6238 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 6239 ipv4_m->hdr.type_of_service >> 2); 6240 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 6241 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6242 ipv4_m->hdr.next_proto_id); 6243 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6244 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 6245 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6246 ipv4_m->hdr.time_to_live); 6247 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6248 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live); 6249 } 6250 6251 /** 6252 * Add IPV6 item to matcher and to the value. 6253 * 6254 * @param[in, out] matcher 6255 * Flow matcher. 6256 * @param[in, out] key 6257 * Flow matcher value. 6258 * @param[in] item 6259 * Flow pattern to translate. 6260 * @param[in] item_flags 6261 * Bit-fields that holds the items detected until now. 6262 * @param[in] inner 6263 * Item is inner pattern. 6264 * @param[in] group 6265 * The group to insert the rule. 6266 */ 6267 static void 6268 flow_dv_translate_item_ipv6(void *matcher, void *key, 6269 const struct rte_flow_item *item, 6270 const uint64_t item_flags, 6271 int inner, uint32_t group) 6272 { 6273 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 6274 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 6275 const struct rte_flow_item_ipv6 nic_mask = { 6276 .hdr = { 6277 .src_addr = 6278 "\xff\xff\xff\xff\xff\xff\xff\xff" 6279 "\xff\xff\xff\xff\xff\xff\xff\xff", 6280 .dst_addr = 6281 "\xff\xff\xff\xff\xff\xff\xff\xff" 6282 "\xff\xff\xff\xff\xff\xff\xff\xff", 6283 .vtc_flow = RTE_BE32(0xffffffff), 6284 .proto = 0xff, 6285 .hop_limits = 0xff, 6286 }, 6287 }; 6288 void *headers_m; 6289 void *headers_v; 6290 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6291 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6292 char *l24_m; 6293 char *l24_v; 6294 uint32_t vtc_m; 6295 uint32_t vtc_v; 6296 int i; 6297 int size; 6298 6299 if (inner) { 6300 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6301 inner_headers); 6302 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6303 } else { 6304 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6305 outer_headers); 6306 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6307 } 6308 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6309 /* 6310 * On outer header (which must contains L2), or inner header with L2, 6311 * set cvlan_tag mask bit to mark this packet as untagged. 6312 * This should be done even if item->spec is empty. 6313 */ 6314 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6315 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6316 if (!ipv6_v) 6317 return; 6318 if (!ipv6_m) 6319 ipv6_m = &nic_mask; 6320 size = sizeof(ipv6_m->hdr.dst_addr); 6321 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6322 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6323 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6324 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6325 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 6326 for (i = 0; i < size; ++i) 6327 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 6328 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6329 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6330 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6331 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6332 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 6333 for (i = 0; i < size; ++i) 6334 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 6335 /* TOS. */ 6336 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 6337 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 6338 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 6339 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 6340 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 6341 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 6342 /* Label. */ 6343 if (inner) { 6344 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 6345 vtc_m); 6346 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 6347 vtc_v); 6348 } else { 6349 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 6350 vtc_m); 6351 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 6352 vtc_v); 6353 } 6354 /* Protocol. */ 6355 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6356 ipv6_m->hdr.proto); 6357 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6358 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 6359 /* Hop limit. */ 6360 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6361 ipv6_m->hdr.hop_limits); 6362 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6363 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits); 6364 } 6365 6366 /** 6367 * Add TCP item to matcher and to the value. 6368 * 6369 * @param[in, out] matcher 6370 * Flow matcher. 6371 * @param[in, out] key 6372 * Flow matcher value. 6373 * @param[in] item 6374 * Flow pattern to translate. 6375 * @param[in] inner 6376 * Item is inner pattern. 6377 */ 6378 static void 6379 flow_dv_translate_item_tcp(void *matcher, void *key, 6380 const struct rte_flow_item *item, 6381 int inner) 6382 { 6383 const struct rte_flow_item_tcp *tcp_m = item->mask; 6384 const struct rte_flow_item_tcp *tcp_v = item->spec; 6385 void *headers_m; 6386 void *headers_v; 6387 6388 if (inner) { 6389 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6390 inner_headers); 6391 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6392 } else { 6393 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6394 outer_headers); 6395 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6396 } 6397 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6398 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 6399 if (!tcp_v) 6400 return; 6401 if (!tcp_m) 6402 tcp_m = &rte_flow_item_tcp_mask; 6403 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 6404 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 6405 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 6406 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 6407 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 6408 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 6409 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 6410 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 6411 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, 6412 tcp_m->hdr.tcp_flags); 6413 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 6414 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); 6415 } 6416 6417 /** 6418 * Add UDP item to matcher and to the value. 6419 * 6420 * @param[in, out] matcher 6421 * Flow matcher. 6422 * @param[in, out] key 6423 * Flow matcher value. 6424 * @param[in] item 6425 * Flow pattern to translate. 6426 * @param[in] inner 6427 * Item is inner pattern. 6428 */ 6429 static void 6430 flow_dv_translate_item_udp(void *matcher, void *key, 6431 const struct rte_flow_item *item, 6432 int inner) 6433 { 6434 const struct rte_flow_item_udp *udp_m = item->mask; 6435 const struct rte_flow_item_udp *udp_v = item->spec; 6436 void *headers_m; 6437 void *headers_v; 6438 6439 if (inner) { 6440 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6441 inner_headers); 6442 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6443 } else { 6444 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6445 outer_headers); 6446 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6447 } 6448 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6449 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 6450 if (!udp_v) 6451 return; 6452 if (!udp_m) 6453 udp_m = &rte_flow_item_udp_mask; 6454 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 6455 rte_be_to_cpu_16(udp_m->hdr.src_port)); 6456 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 6457 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 6458 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 6459 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 6460 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6461 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 6462 } 6463 6464 /** 6465 * Add GRE optional Key item to matcher and to the value. 6466 * 6467 * @param[in, out] matcher 6468 * Flow matcher. 6469 * @param[in, out] key 6470 * Flow matcher value. 6471 * @param[in] item 6472 * Flow pattern to translate. 6473 * @param[in] inner 6474 * Item is inner pattern. 6475 */ 6476 static void 6477 flow_dv_translate_item_gre_key(void *matcher, void *key, 6478 const struct rte_flow_item *item) 6479 { 6480 const rte_be32_t *key_m = item->mask; 6481 const rte_be32_t *key_v = item->spec; 6482 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6483 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6484 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 6485 6486 /* GRE K bit must be on and should already be validated */ 6487 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); 6488 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); 6489 if (!key_v) 6490 return; 6491 if (!key_m) 6492 key_m = &gre_key_default_mask; 6493 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, 6494 rte_be_to_cpu_32(*key_m) >> 8); 6495 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, 6496 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); 6497 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, 6498 rte_be_to_cpu_32(*key_m) & 0xFF); 6499 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, 6500 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); 6501 } 6502 6503 /** 6504 * Add GRE item to matcher and to the value. 6505 * 6506 * @param[in, out] matcher 6507 * Flow matcher. 6508 * @param[in, out] key 6509 * Flow matcher value. 6510 * @param[in] item 6511 * Flow pattern to translate. 6512 * @param[in] inner 6513 * Item is inner pattern. 6514 */ 6515 static void 6516 flow_dv_translate_item_gre(void *matcher, void *key, 6517 const struct rte_flow_item *item, 6518 int inner) 6519 { 6520 const struct rte_flow_item_gre *gre_m = item->mask; 6521 const struct rte_flow_item_gre *gre_v = item->spec; 6522 void *headers_m; 6523 void *headers_v; 6524 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6525 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6526 struct { 6527 union { 6528 __extension__ 6529 struct { 6530 uint16_t version:3; 6531 uint16_t rsvd0:9; 6532 uint16_t s_present:1; 6533 uint16_t k_present:1; 6534 uint16_t rsvd_bit1:1; 6535 uint16_t c_present:1; 6536 }; 6537 uint16_t value; 6538 }; 6539 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; 6540 6541 if (inner) { 6542 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6543 inner_headers); 6544 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6545 } else { 6546 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6547 outer_headers); 6548 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6549 } 6550 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6551 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 6552 if (!gre_v) 6553 return; 6554 if (!gre_m) 6555 gre_m = &rte_flow_item_gre_mask; 6556 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 6557 rte_be_to_cpu_16(gre_m->protocol)); 6558 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6559 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); 6560 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); 6561 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); 6562 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, 6563 gre_crks_rsvd0_ver_m.c_present); 6564 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, 6565 gre_crks_rsvd0_ver_v.c_present & 6566 gre_crks_rsvd0_ver_m.c_present); 6567 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 6568 gre_crks_rsvd0_ver_m.k_present); 6569 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 6570 gre_crks_rsvd0_ver_v.k_present & 6571 gre_crks_rsvd0_ver_m.k_present); 6572 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, 6573 gre_crks_rsvd0_ver_m.s_present); 6574 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, 6575 gre_crks_rsvd0_ver_v.s_present & 6576 gre_crks_rsvd0_ver_m.s_present); 6577 } 6578 6579 /** 6580 * Add NVGRE item to matcher and to the value. 6581 * 6582 * @param[in, out] matcher 6583 * Flow matcher. 6584 * @param[in, out] key 6585 * Flow matcher value. 6586 * @param[in] item 6587 * Flow pattern to translate. 6588 * @param[in] inner 6589 * Item is inner pattern. 6590 */ 6591 static void 6592 flow_dv_translate_item_nvgre(void *matcher, void *key, 6593 const struct rte_flow_item *item, 6594 int inner) 6595 { 6596 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 6597 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 6598 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6599 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6600 const char *tni_flow_id_m; 6601 const char *tni_flow_id_v; 6602 char *gre_key_m; 6603 char *gre_key_v; 6604 int size; 6605 int i; 6606 6607 /* For NVGRE, GRE header fields must be set with defined values. */ 6608 const struct rte_flow_item_gre gre_spec = { 6609 .c_rsvd0_ver = RTE_BE16(0x2000), 6610 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB) 6611 }; 6612 const struct rte_flow_item_gre gre_mask = { 6613 .c_rsvd0_ver = RTE_BE16(0xB000), 6614 .protocol = RTE_BE16(UINT16_MAX), 6615 }; 6616 const struct rte_flow_item gre_item = { 6617 .spec = &gre_spec, 6618 .mask = &gre_mask, 6619 .last = NULL, 6620 }; 6621 flow_dv_translate_item_gre(matcher, key, &gre_item, inner); 6622 if (!nvgre_v) 6623 return; 6624 if (!nvgre_m) 6625 nvgre_m = &rte_flow_item_nvgre_mask; 6626 tni_flow_id_m = (const char *)nvgre_m->tni; 6627 tni_flow_id_v = (const char *)nvgre_v->tni; 6628 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 6629 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 6630 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 6631 memcpy(gre_key_m, tni_flow_id_m, size); 6632 for (i = 0; i < size; ++i) 6633 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 6634 } 6635 6636 /** 6637 * Add VXLAN item to matcher and to the value. 6638 * 6639 * @param[in, out] matcher 6640 * Flow matcher. 6641 * @param[in, out] key 6642 * Flow matcher value. 6643 * @param[in] item 6644 * Flow pattern to translate. 6645 * @param[in] inner 6646 * Item is inner pattern. 6647 */ 6648 static void 6649 flow_dv_translate_item_vxlan(void *matcher, void *key, 6650 const struct rte_flow_item *item, 6651 int inner) 6652 { 6653 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 6654 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 6655 void *headers_m; 6656 void *headers_v; 6657 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6658 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6659 char *vni_m; 6660 char *vni_v; 6661 uint16_t dport; 6662 int size; 6663 int i; 6664 6665 if (inner) { 6666 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6667 inner_headers); 6668 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6669 } else { 6670 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6671 outer_headers); 6672 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6673 } 6674 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6675 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6676 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6677 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6678 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6679 } 6680 if (!vxlan_v) 6681 return; 6682 if (!vxlan_m) 6683 vxlan_m = &rte_flow_item_vxlan_mask; 6684 size = sizeof(vxlan_m->vni); 6685 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 6686 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 6687 memcpy(vni_m, vxlan_m->vni, size); 6688 for (i = 0; i < size; ++i) 6689 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6690 } 6691 6692 /** 6693 * Add VXLAN-GPE item to matcher and to the value. 6694 * 6695 * @param[in, out] matcher 6696 * Flow matcher. 6697 * @param[in, out] key 6698 * Flow matcher value. 6699 * @param[in] item 6700 * Flow pattern to translate. 6701 * @param[in] inner 6702 * Item is inner pattern. 6703 */ 6704 6705 static void 6706 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key, 6707 const struct rte_flow_item *item, int inner) 6708 { 6709 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask; 6710 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec; 6711 void *headers_m; 6712 void *headers_v; 6713 void *misc_m = 6714 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3); 6715 void *misc_v = 6716 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 6717 char *vni_m; 6718 char *vni_v; 6719 uint16_t dport; 6720 int size; 6721 int i; 6722 uint8_t flags_m = 0xff; 6723 uint8_t flags_v = 0xc; 6724 6725 if (inner) { 6726 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6727 inner_headers); 6728 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6729 } else { 6730 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6731 outer_headers); 6732 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6733 } 6734 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6735 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6736 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6737 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6738 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6739 } 6740 if (!vxlan_v) 6741 return; 6742 if (!vxlan_m) 6743 vxlan_m = &rte_flow_item_vxlan_gpe_mask; 6744 size = sizeof(vxlan_m->vni); 6745 vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni); 6746 vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni); 6747 memcpy(vni_m, vxlan_m->vni, size); 6748 for (i = 0; i < size; ++i) 6749 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6750 if (vxlan_m->flags) { 6751 flags_m = vxlan_m->flags; 6752 flags_v = vxlan_v->flags; 6753 } 6754 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m); 6755 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v); 6756 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol, 6757 vxlan_m->protocol); 6758 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol, 6759 vxlan_v->protocol); 6760 } 6761 6762 /** 6763 * Add Geneve item to matcher and to the value. 6764 * 6765 * @param[in, out] matcher 6766 * Flow matcher. 6767 * @param[in, out] key 6768 * Flow matcher value. 6769 * @param[in] item 6770 * Flow pattern to translate. 6771 * @param[in] inner 6772 * Item is inner pattern. 6773 */ 6774 6775 static void 6776 flow_dv_translate_item_geneve(void *matcher, void *key, 6777 const struct rte_flow_item *item, int inner) 6778 { 6779 const struct rte_flow_item_geneve *geneve_m = item->mask; 6780 const struct rte_flow_item_geneve *geneve_v = item->spec; 6781 void *headers_m; 6782 void *headers_v; 6783 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6784 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6785 uint16_t dport; 6786 uint16_t gbhdr_m; 6787 uint16_t gbhdr_v; 6788 char *vni_m; 6789 char *vni_v; 6790 size_t size, i; 6791 6792 if (inner) { 6793 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6794 inner_headers); 6795 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6796 } else { 6797 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6798 outer_headers); 6799 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6800 } 6801 dport = MLX5_UDP_PORT_GENEVE; 6802 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6803 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6804 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6805 } 6806 if (!geneve_v) 6807 return; 6808 if (!geneve_m) 6809 geneve_m = &rte_flow_item_geneve_mask; 6810 size = sizeof(geneve_m->vni); 6811 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni); 6812 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni); 6813 memcpy(vni_m, geneve_m->vni, size); 6814 for (i = 0; i < size; ++i) 6815 vni_v[i] = vni_m[i] & geneve_v->vni[i]; 6816 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, 6817 rte_be_to_cpu_16(geneve_m->protocol)); 6818 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type, 6819 rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol)); 6820 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0); 6821 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0); 6822 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam, 6823 MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6824 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam, 6825 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6826 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, 6827 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6828 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, 6829 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) & 6830 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6831 } 6832 6833 /** 6834 * Add MPLS item to matcher and to the value. 6835 * 6836 * @param[in, out] matcher 6837 * Flow matcher. 6838 * @param[in, out] key 6839 * Flow matcher value. 6840 * @param[in] item 6841 * Flow pattern to translate. 6842 * @param[in] prev_layer 6843 * The protocol layer indicated in previous item. 6844 * @param[in] inner 6845 * Item is inner pattern. 6846 */ 6847 static void 6848 flow_dv_translate_item_mpls(void *matcher, void *key, 6849 const struct rte_flow_item *item, 6850 uint64_t prev_layer, 6851 int inner) 6852 { 6853 const uint32_t *in_mpls_m = item->mask; 6854 const uint32_t *in_mpls_v = item->spec; 6855 uint32_t *out_mpls_m = 0; 6856 uint32_t *out_mpls_v = 0; 6857 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6858 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6859 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 6860 misc_parameters_2); 6861 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6862 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 6863 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6864 6865 switch (prev_layer) { 6866 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6867 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff); 6868 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6869 MLX5_UDP_PORT_MPLS); 6870 break; 6871 case MLX5_FLOW_LAYER_GRE: 6872 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); 6873 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6874 RTE_ETHER_TYPE_MPLS); 6875 break; 6876 default: 6877 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6878 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6879 IPPROTO_MPLS); 6880 break; 6881 } 6882 if (!in_mpls_v) 6883 return; 6884 if (!in_mpls_m) 6885 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 6886 switch (prev_layer) { 6887 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6888 out_mpls_m = 6889 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6890 outer_first_mpls_over_udp); 6891 out_mpls_v = 6892 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6893 outer_first_mpls_over_udp); 6894 break; 6895 case MLX5_FLOW_LAYER_GRE: 6896 out_mpls_m = 6897 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6898 outer_first_mpls_over_gre); 6899 out_mpls_v = 6900 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6901 outer_first_mpls_over_gre); 6902 break; 6903 default: 6904 /* Inner MPLS not over GRE is not supported. */ 6905 if (!inner) { 6906 out_mpls_m = 6907 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6908 misc2_m, 6909 outer_first_mpls); 6910 out_mpls_v = 6911 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6912 misc2_v, 6913 outer_first_mpls); 6914 } 6915 break; 6916 } 6917 if (out_mpls_m && out_mpls_v) { 6918 *out_mpls_m = *in_mpls_m; 6919 *out_mpls_v = *in_mpls_v & *in_mpls_m; 6920 } 6921 } 6922 6923 /** 6924 * Add metadata register item to matcher 6925 * 6926 * @param[in, out] matcher 6927 * Flow matcher. 6928 * @param[in, out] key 6929 * Flow matcher value. 6930 * @param[in] reg_type 6931 * Type of device metadata register 6932 * @param[in] value 6933 * Register value 6934 * @param[in] mask 6935 * Register mask 6936 */ 6937 static void 6938 flow_dv_match_meta_reg(void *matcher, void *key, 6939 enum modify_reg reg_type, 6940 uint32_t data, uint32_t mask) 6941 { 6942 void *misc2_m = 6943 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 6944 void *misc2_v = 6945 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6946 uint32_t temp; 6947 6948 data &= mask; 6949 switch (reg_type) { 6950 case REG_A: 6951 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask); 6952 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data); 6953 break; 6954 case REG_B: 6955 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask); 6956 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); 6957 break; 6958 case REG_C_0: 6959 /* 6960 * The metadata register C0 field might be divided into 6961 * source vport index and META item value, we should set 6962 * this field according to specified mask, not as whole one. 6963 */ 6964 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); 6965 temp |= mask; 6966 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); 6967 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); 6968 temp &= ~mask; 6969 temp |= data; 6970 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); 6971 break; 6972 case REG_C_1: 6973 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); 6974 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data); 6975 break; 6976 case REG_C_2: 6977 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask); 6978 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data); 6979 break; 6980 case REG_C_3: 6981 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask); 6982 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data); 6983 break; 6984 case REG_C_4: 6985 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask); 6986 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data); 6987 break; 6988 case REG_C_5: 6989 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask); 6990 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data); 6991 break; 6992 case REG_C_6: 6993 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask); 6994 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data); 6995 break; 6996 case REG_C_7: 6997 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask); 6998 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data); 6999 break; 7000 default: 7001 MLX5_ASSERT(false); 7002 break; 7003 } 7004 } 7005 7006 /** 7007 * Add MARK item to matcher 7008 * 7009 * @param[in] dev 7010 * The device to configure through. 7011 * @param[in, out] matcher 7012 * Flow matcher. 7013 * @param[in, out] key 7014 * Flow matcher value. 7015 * @param[in] item 7016 * Flow pattern to translate. 7017 */ 7018 static void 7019 flow_dv_translate_item_mark(struct rte_eth_dev *dev, 7020 void *matcher, void *key, 7021 const struct rte_flow_item *item) 7022 { 7023 struct mlx5_priv *priv = dev->data->dev_private; 7024 const struct rte_flow_item_mark *mark; 7025 uint32_t value; 7026 uint32_t mask; 7027 7028 mark = item->mask ? (const void *)item->mask : 7029 &rte_flow_item_mark_mask; 7030 mask = mark->id & priv->sh->dv_mark_mask; 7031 mark = (const void *)item->spec; 7032 MLX5_ASSERT(mark); 7033 value = mark->id & priv->sh->dv_mark_mask & mask; 7034 if (mask) { 7035 enum modify_reg reg; 7036 7037 /* Get the metadata register index for the mark. */ 7038 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); 7039 MLX5_ASSERT(reg > 0); 7040 if (reg == REG_C_0) { 7041 struct mlx5_priv *priv = dev->data->dev_private; 7042 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 7043 uint32_t shl_c0 = rte_bsf32(msk_c0); 7044 7045 mask &= msk_c0; 7046 mask <<= shl_c0; 7047 value <<= shl_c0; 7048 } 7049 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 7050 } 7051 } 7052 7053 /** 7054 * Add META item to matcher 7055 * 7056 * @param[in] dev 7057 * The devich to configure through. 7058 * @param[in, out] matcher 7059 * Flow matcher. 7060 * @param[in, out] key 7061 * Flow matcher value. 7062 * @param[in] attr 7063 * Attributes of flow that includes this item. 7064 * @param[in] item 7065 * Flow pattern to translate. 7066 */ 7067 static void 7068 flow_dv_translate_item_meta(struct rte_eth_dev *dev, 7069 void *matcher, void *key, 7070 const struct rte_flow_attr *attr, 7071 const struct rte_flow_item *item) 7072 { 7073 const struct rte_flow_item_meta *meta_m; 7074 const struct rte_flow_item_meta *meta_v; 7075 7076 meta_m = (const void *)item->mask; 7077 if (!meta_m) 7078 meta_m = &rte_flow_item_meta_mask; 7079 meta_v = (const void *)item->spec; 7080 if (meta_v) { 7081 int reg; 7082 uint32_t value = meta_v->data; 7083 uint32_t mask = meta_m->data; 7084 7085 reg = flow_dv_get_metadata_reg(dev, attr, NULL); 7086 if (reg < 0) 7087 return; 7088 /* 7089 * In datapath code there is no endianness 7090 * coversions for perfromance reasons, all 7091 * pattern conversions are done in rte_flow. 7092 */ 7093 value = rte_cpu_to_be_32(value); 7094 mask = rte_cpu_to_be_32(mask); 7095 if (reg == REG_C_0) { 7096 struct mlx5_priv *priv = dev->data->dev_private; 7097 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 7098 uint32_t shl_c0 = rte_bsf32(msk_c0); 7099 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 7100 uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); 7101 7102 value >>= shr_c0; 7103 mask >>= shr_c0; 7104 #endif 7105 value <<= shl_c0; 7106 mask <<= shl_c0; 7107 MLX5_ASSERT(msk_c0); 7108 MLX5_ASSERT(!(~msk_c0 & mask)); 7109 } 7110 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 7111 } 7112 } 7113 7114 /** 7115 * Add vport metadata Reg C0 item to matcher 7116 * 7117 * @param[in, out] matcher 7118 * Flow matcher. 7119 * @param[in, out] key 7120 * Flow matcher value. 7121 * @param[in] reg 7122 * Flow pattern to translate. 7123 */ 7124 static void 7125 flow_dv_translate_item_meta_vport(void *matcher, void *key, 7126 uint32_t value, uint32_t mask) 7127 { 7128 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask); 7129 } 7130 7131 /** 7132 * Add tag item to matcher 7133 * 7134 * @param[in] dev 7135 * The devich to configure through. 7136 * @param[in, out] matcher 7137 * Flow matcher. 7138 * @param[in, out] key 7139 * Flow matcher value. 7140 * @param[in] item 7141 * Flow pattern to translate. 7142 */ 7143 static void 7144 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, 7145 void *matcher, void *key, 7146 const struct rte_flow_item *item) 7147 { 7148 const struct mlx5_rte_flow_item_tag *tag_v = item->spec; 7149 const struct mlx5_rte_flow_item_tag *tag_m = item->mask; 7150 uint32_t mask, value; 7151 7152 MLX5_ASSERT(tag_v); 7153 value = tag_v->data; 7154 mask = tag_m ? tag_m->data : UINT32_MAX; 7155 if (tag_v->id == REG_C_0) { 7156 struct mlx5_priv *priv = dev->data->dev_private; 7157 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 7158 uint32_t shl_c0 = rte_bsf32(msk_c0); 7159 7160 mask &= msk_c0; 7161 mask <<= shl_c0; 7162 value <<= shl_c0; 7163 } 7164 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); 7165 } 7166 7167 /** 7168 * Add TAG item to matcher 7169 * 7170 * @param[in] dev 7171 * The devich to configure through. 7172 * @param[in, out] matcher 7173 * Flow matcher. 7174 * @param[in, out] key 7175 * Flow matcher value. 7176 * @param[in] item 7177 * Flow pattern to translate. 7178 */ 7179 static void 7180 flow_dv_translate_item_tag(struct rte_eth_dev *dev, 7181 void *matcher, void *key, 7182 const struct rte_flow_item *item) 7183 { 7184 const struct rte_flow_item_tag *tag_v = item->spec; 7185 const struct rte_flow_item_tag *tag_m = item->mask; 7186 enum modify_reg reg; 7187 7188 MLX5_ASSERT(tag_v); 7189 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask; 7190 /* Get the metadata register index for the tag. */ 7191 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL); 7192 MLX5_ASSERT(reg > 0); 7193 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data); 7194 } 7195 7196 /** 7197 * Add source vport match to the specified matcher. 7198 * 7199 * @param[in, out] matcher 7200 * Flow matcher. 7201 * @param[in, out] key 7202 * Flow matcher value. 7203 * @param[in] port 7204 * Source vport value to match 7205 * @param[in] mask 7206 * Mask 7207 */ 7208 static void 7209 flow_dv_translate_item_source_vport(void *matcher, void *key, 7210 int16_t port, uint16_t mask) 7211 { 7212 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 7213 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 7214 7215 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); 7216 MLX5_SET(fte_match_set_misc, misc_v, source_port, port); 7217 } 7218 7219 /** 7220 * Translate port-id item to eswitch match on port-id. 7221 * 7222 * @param[in] dev 7223 * The devich to configure through. 7224 * @param[in, out] matcher 7225 * Flow matcher. 7226 * @param[in, out] key 7227 * Flow matcher value. 7228 * @param[in] item 7229 * Flow pattern to translate. 7230 * 7231 * @return 7232 * 0 on success, a negative errno value otherwise. 7233 */ 7234 static int 7235 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, 7236 void *key, const struct rte_flow_item *item) 7237 { 7238 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; 7239 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; 7240 struct mlx5_priv *priv; 7241 uint16_t mask, id; 7242 7243 mask = pid_m ? pid_m->id : 0xffff; 7244 id = pid_v ? pid_v->id : dev->data->port_id; 7245 priv = mlx5_port_to_eswitch_info(id, item == NULL); 7246 if (!priv) 7247 return -rte_errno; 7248 /* Translate to vport field or to metadata, depending on mode. */ 7249 if (priv->vport_meta_mask) 7250 flow_dv_translate_item_meta_vport(matcher, key, 7251 priv->vport_meta_tag, 7252 priv->vport_meta_mask); 7253 else 7254 flow_dv_translate_item_source_vport(matcher, key, 7255 priv->vport_id, mask); 7256 return 0; 7257 } 7258 7259 /** 7260 * Add ICMP6 item to matcher and to the value. 7261 * 7262 * @param[in, out] matcher 7263 * Flow matcher. 7264 * @param[in, out] key 7265 * Flow matcher value. 7266 * @param[in] item 7267 * Flow pattern to translate. 7268 * @param[in] inner 7269 * Item is inner pattern. 7270 */ 7271 static void 7272 flow_dv_translate_item_icmp6(void *matcher, void *key, 7273 const struct rte_flow_item *item, 7274 int inner) 7275 { 7276 const struct rte_flow_item_icmp6 *icmp6_m = item->mask; 7277 const struct rte_flow_item_icmp6 *icmp6_v = item->spec; 7278 void *headers_m; 7279 void *headers_v; 7280 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7281 misc_parameters_3); 7282 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7283 if (inner) { 7284 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7285 inner_headers); 7286 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7287 } else { 7288 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7289 outer_headers); 7290 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7291 } 7292 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7293 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); 7294 if (!icmp6_v) 7295 return; 7296 if (!icmp6_m) 7297 icmp6_m = &rte_flow_item_icmp6_mask; 7298 /* 7299 * Force flow only to match the non-fragmented IPv6 ICMPv6 packets. 7300 * If only the protocol is specified, no need to match the frag. 7301 */ 7302 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7303 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7304 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); 7305 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, 7306 icmp6_v->type & icmp6_m->type); 7307 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); 7308 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, 7309 icmp6_v->code & icmp6_m->code); 7310 } 7311 7312 /** 7313 * Add ICMP item to matcher and to the value. 7314 * 7315 * @param[in, out] matcher 7316 * Flow matcher. 7317 * @param[in, out] key 7318 * Flow matcher value. 7319 * @param[in] item 7320 * Flow pattern to translate. 7321 * @param[in] inner 7322 * Item is inner pattern. 7323 */ 7324 static void 7325 flow_dv_translate_item_icmp(void *matcher, void *key, 7326 const struct rte_flow_item *item, 7327 int inner) 7328 { 7329 const struct rte_flow_item_icmp *icmp_m = item->mask; 7330 const struct rte_flow_item_icmp *icmp_v = item->spec; 7331 void *headers_m; 7332 void *headers_v; 7333 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7334 misc_parameters_3); 7335 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7336 if (inner) { 7337 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7338 inner_headers); 7339 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7340 } else { 7341 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7342 outer_headers); 7343 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7344 } 7345 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7346 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); 7347 if (!icmp_v) 7348 return; 7349 if (!icmp_m) 7350 icmp_m = &rte_flow_item_icmp_mask; 7351 /* 7352 * Force flow only to match the non-fragmented IPv4 ICMP packets. 7353 * If only the protocol is specified, no need to match the frag. 7354 */ 7355 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7356 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7357 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, 7358 icmp_m->hdr.icmp_type); 7359 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, 7360 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); 7361 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, 7362 icmp_m->hdr.icmp_code); 7363 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, 7364 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); 7365 } 7366 7367 /** 7368 * Add GTP item to matcher and to the value. 7369 * 7370 * @param[in, out] matcher 7371 * Flow matcher. 7372 * @param[in, out] key 7373 * Flow matcher value. 7374 * @param[in] item 7375 * Flow pattern to translate. 7376 * @param[in] inner 7377 * Item is inner pattern. 7378 */ 7379 static void 7380 flow_dv_translate_item_gtp(void *matcher, void *key, 7381 const struct rte_flow_item *item, int inner) 7382 { 7383 const struct rte_flow_item_gtp *gtp_m = item->mask; 7384 const struct rte_flow_item_gtp *gtp_v = item->spec; 7385 void *headers_m; 7386 void *headers_v; 7387 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7388 misc_parameters_3); 7389 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7390 uint16_t dport = RTE_GTPU_UDP_PORT; 7391 7392 if (inner) { 7393 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7394 inner_headers); 7395 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7396 } else { 7397 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7398 outer_headers); 7399 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7400 } 7401 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 7402 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 7403 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 7404 } 7405 if (!gtp_v) 7406 return; 7407 if (!gtp_m) 7408 gtp_m = &rte_flow_item_gtp_mask; 7409 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, 7410 gtp_m->v_pt_rsv_flags); 7411 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, 7412 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags); 7413 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); 7414 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, 7415 gtp_v->msg_type & gtp_m->msg_type); 7416 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, 7417 rte_be_to_cpu_32(gtp_m->teid)); 7418 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, 7419 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); 7420 } 7421 7422 /** 7423 * Add eCPRI item to matcher and to the value. 7424 * 7425 * @param[in] dev 7426 * The devich to configure through. 7427 * @param[in, out] matcher 7428 * Flow matcher. 7429 * @param[in, out] key 7430 * Flow matcher value. 7431 * @param[in] item 7432 * Flow pattern to translate. 7433 * @param[in] samples 7434 * Sample IDs to be used in the matching. 7435 */ 7436 static void 7437 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher, 7438 void *key, const struct rte_flow_item *item) 7439 { 7440 struct mlx5_priv *priv = dev->data->dev_private; 7441 const struct rte_flow_item_ecpri *ecpri_m = item->mask; 7442 const struct rte_flow_item_ecpri *ecpri_v = item->spec; 7443 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, 7444 misc_parameters_4); 7445 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); 7446 uint32_t *samples; 7447 void *dw_m; 7448 void *dw_v; 7449 7450 if (!ecpri_v) 7451 return; 7452 if (!ecpri_m) 7453 ecpri_m = &rte_flow_item_ecpri_mask; 7454 /* 7455 * Maximal four DW samples are supported in a single matching now. 7456 * Two are used now for a eCPRI matching: 7457 * 1. Type: one byte, mask should be 0x00ff0000 in network order 7458 * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000 7459 * if any. 7460 */ 7461 if (!ecpri_m->hdr.common.u32) 7462 return; 7463 samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids; 7464 /* Need to take the whole DW as the mask to fill the entry. */ 7465 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 7466 prog_sample_field_value_0); 7467 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 7468 prog_sample_field_value_0); 7469 /* Already big endian (network order) in the header. */ 7470 *(uint32_t *)dw_m = ecpri_m->hdr.common.u32; 7471 *(uint32_t *)dw_v = ecpri_v->hdr.common.u32; 7472 /* Sample#0, used for matching type, offset 0. */ 7473 MLX5_SET(fte_match_set_misc4, misc4_m, 7474 prog_sample_field_id_0, samples[0]); 7475 /* It makes no sense to set the sample ID in the mask field. */ 7476 MLX5_SET(fte_match_set_misc4, misc4_v, 7477 prog_sample_field_id_0, samples[0]); 7478 /* 7479 * Checking if message body part needs to be matched. 7480 * Some wildcard rules only matching type field should be supported. 7481 */ 7482 if (ecpri_m->hdr.dummy[0]) { 7483 switch (ecpri_v->hdr.common.type) { 7484 case RTE_ECPRI_MSG_TYPE_IQ_DATA: 7485 case RTE_ECPRI_MSG_TYPE_RTC_CTRL: 7486 case RTE_ECPRI_MSG_TYPE_DLY_MSR: 7487 dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m, 7488 prog_sample_field_value_1); 7489 dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v, 7490 prog_sample_field_value_1); 7491 *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0]; 7492 *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0]; 7493 /* Sample#1, to match message body, offset 4. */ 7494 MLX5_SET(fte_match_set_misc4, misc4_m, 7495 prog_sample_field_id_1, samples[1]); 7496 MLX5_SET(fte_match_set_misc4, misc4_v, 7497 prog_sample_field_id_1, samples[1]); 7498 break; 7499 default: 7500 /* Others, do not match any sample ID. */ 7501 break; 7502 } 7503 } 7504 } 7505 7506 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 7507 7508 #define HEADER_IS_ZERO(match_criteria, headers) \ 7509 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 7510 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 7511 7512 /** 7513 * Calculate flow matcher enable bitmap. 7514 * 7515 * @param match_criteria 7516 * Pointer to flow matcher criteria. 7517 * 7518 * @return 7519 * Bitmap of enabled fields. 7520 */ 7521 static uint8_t 7522 flow_dv_matcher_enable(uint32_t *match_criteria) 7523 { 7524 uint8_t match_criteria_enable; 7525 7526 match_criteria_enable = 7527 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 7528 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 7529 match_criteria_enable |= 7530 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 7531 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 7532 match_criteria_enable |= 7533 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 7534 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 7535 match_criteria_enable |= 7536 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 7537 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 7538 match_criteria_enable |= 7539 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << 7540 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; 7541 match_criteria_enable |= 7542 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) << 7543 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT; 7544 return match_criteria_enable; 7545 } 7546 7547 7548 /** 7549 * Get a flow table. 7550 * 7551 * @param[in, out] dev 7552 * Pointer to rte_eth_dev structure. 7553 * @param[in] table_id 7554 * Table id to use. 7555 * @param[in] egress 7556 * Direction of the table. 7557 * @param[in] transfer 7558 * E-Switch or NIC flow. 7559 * @param[out] error 7560 * pointer to error structure. 7561 * 7562 * @return 7563 * Returns tables resource based on the index, NULL in case of failed. 7564 */ 7565 static struct mlx5_flow_tbl_resource * 7566 flow_dv_tbl_resource_get(struct rte_eth_dev *dev, 7567 uint32_t table_id, uint8_t egress, 7568 uint8_t transfer, 7569 struct rte_flow_error *error) 7570 { 7571 struct mlx5_priv *priv = dev->data->dev_private; 7572 struct mlx5_dev_ctx_shared *sh = priv->sh; 7573 struct mlx5_flow_tbl_resource *tbl; 7574 union mlx5_flow_tbl_key table_key = { 7575 { 7576 .table_id = table_id, 7577 .reserved = 0, 7578 .domain = !!transfer, 7579 .direction = !!egress, 7580 } 7581 }; 7582 struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, 7583 table_key.v64); 7584 struct mlx5_flow_tbl_data_entry *tbl_data; 7585 uint32_t idx = 0; 7586 int ret; 7587 void *domain; 7588 7589 if (pos) { 7590 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, 7591 entry); 7592 tbl = &tbl_data->tbl; 7593 rte_atomic32_inc(&tbl->refcnt); 7594 return tbl; 7595 } 7596 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); 7597 if (!tbl_data) { 7598 rte_flow_error_set(error, ENOMEM, 7599 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7600 NULL, 7601 "cannot allocate flow table data entry"); 7602 return NULL; 7603 } 7604 tbl_data->idx = idx; 7605 tbl = &tbl_data->tbl; 7606 pos = &tbl_data->entry; 7607 if (transfer) 7608 domain = sh->fdb_domain; 7609 else if (egress) 7610 domain = sh->tx_domain; 7611 else 7612 domain = sh->rx_domain; 7613 ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj); 7614 if (ret) { 7615 rte_flow_error_set(error, ENOMEM, 7616 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7617 NULL, "cannot create flow table object"); 7618 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7619 return NULL; 7620 } 7621 /* 7622 * No multi-threads now, but still better to initialize the reference 7623 * count before insert it into the hash list. 7624 */ 7625 rte_atomic32_init(&tbl->refcnt); 7626 /* Jump action reference count is initialized here. */ 7627 rte_atomic32_init(&tbl_data->jump.refcnt); 7628 pos->key = table_key.v64; 7629 ret = mlx5_hlist_insert(sh->flow_tbls, pos); 7630 if (ret < 0) { 7631 rte_flow_error_set(error, -ret, 7632 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7633 "cannot insert flow table data entry"); 7634 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 7635 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7636 } 7637 rte_atomic32_inc(&tbl->refcnt); 7638 return tbl; 7639 } 7640 7641 /** 7642 * Release a flow table. 7643 * 7644 * @param[in] dev 7645 * Pointer to rte_eth_dev structure. 7646 * @param[in] tbl 7647 * Table resource to be released. 7648 * 7649 * @return 7650 * Returns 0 if table was released, else return 1; 7651 */ 7652 static int 7653 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 7654 struct mlx5_flow_tbl_resource *tbl) 7655 { 7656 struct mlx5_priv *priv = dev->data->dev_private; 7657 struct mlx5_dev_ctx_shared *sh = priv->sh; 7658 struct mlx5_flow_tbl_data_entry *tbl_data = 7659 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7660 7661 if (!tbl) 7662 return 0; 7663 if (rte_atomic32_dec_and_test(&tbl->refcnt)) { 7664 struct mlx5_hlist_entry *pos = &tbl_data->entry; 7665 7666 mlx5_flow_os_destroy_flow_tbl(tbl->obj); 7667 tbl->obj = NULL; 7668 /* remove the entry from the hash list and free memory. */ 7669 mlx5_hlist_remove(sh->flow_tbls, pos); 7670 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], 7671 tbl_data->idx); 7672 return 0; 7673 } 7674 return 1; 7675 } 7676 7677 /** 7678 * Register the flow matcher. 7679 * 7680 * @param[in, out] dev 7681 * Pointer to rte_eth_dev structure. 7682 * @param[in, out] matcher 7683 * Pointer to flow matcher. 7684 * @param[in, out] key 7685 * Pointer to flow table key. 7686 * @parm[in, out] dev_flow 7687 * Pointer to the dev_flow. 7688 * @param[out] error 7689 * pointer to error structure. 7690 * 7691 * @return 7692 * 0 on success otherwise -errno and errno is set. 7693 */ 7694 static int 7695 flow_dv_matcher_register(struct rte_eth_dev *dev, 7696 struct mlx5_flow_dv_matcher *matcher, 7697 union mlx5_flow_tbl_key *key, 7698 struct mlx5_flow *dev_flow, 7699 struct rte_flow_error *error) 7700 { 7701 struct mlx5_priv *priv = dev->data->dev_private; 7702 struct mlx5_dev_ctx_shared *sh = priv->sh; 7703 struct mlx5_flow_dv_matcher *cache_matcher; 7704 struct mlx5dv_flow_matcher_attr dv_attr = { 7705 .type = IBV_FLOW_ATTR_NORMAL, 7706 .match_mask = (void *)&matcher->mask, 7707 }; 7708 struct mlx5_flow_tbl_resource *tbl; 7709 struct mlx5_flow_tbl_data_entry *tbl_data; 7710 int ret; 7711 7712 tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, 7713 key->domain, error); 7714 if (!tbl) 7715 return -rte_errno; /* No need to refill the error info */ 7716 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7717 /* Lookup from cache. */ 7718 LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { 7719 if (matcher->crc == cache_matcher->crc && 7720 matcher->priority == cache_matcher->priority && 7721 !memcmp((const void *)matcher->mask.buf, 7722 (const void *)cache_matcher->mask.buf, 7723 cache_matcher->mask.size)) { 7724 DRV_LOG(DEBUG, 7725 "%s group %u priority %hd use %s " 7726 "matcher %p: refcnt %d++", 7727 key->domain ? "FDB" : "NIC", key->table_id, 7728 cache_matcher->priority, 7729 key->direction ? "tx" : "rx", 7730 (void *)cache_matcher, 7731 rte_atomic32_read(&cache_matcher->refcnt)); 7732 rte_atomic32_inc(&cache_matcher->refcnt); 7733 dev_flow->handle->dvh.matcher = cache_matcher; 7734 /* old matcher should not make the table ref++. */ 7735 flow_dv_tbl_resource_release(dev, tbl); 7736 return 0; 7737 } 7738 } 7739 /* Register new matcher. */ 7740 cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0, 7741 SOCKET_ID_ANY); 7742 if (!cache_matcher) { 7743 flow_dv_tbl_resource_release(dev, tbl); 7744 return rte_flow_error_set(error, ENOMEM, 7745 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7746 "cannot allocate matcher memory"); 7747 } 7748 *cache_matcher = *matcher; 7749 dv_attr.match_criteria_enable = 7750 flow_dv_matcher_enable(cache_matcher->mask.buf); 7751 dv_attr.priority = matcher->priority; 7752 if (key->direction) 7753 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 7754 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj, 7755 &cache_matcher->matcher_object); 7756 if (ret) { 7757 mlx5_free(cache_matcher); 7758 #ifdef HAVE_MLX5DV_DR 7759 flow_dv_tbl_resource_release(dev, tbl); 7760 #endif 7761 return rte_flow_error_set(error, ENOMEM, 7762 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7763 NULL, "cannot create matcher"); 7764 } 7765 /* Save the table information */ 7766 cache_matcher->tbl = tbl; 7767 rte_atomic32_init(&cache_matcher->refcnt); 7768 /* only matcher ref++, table ref++ already done above in get API. */ 7769 rte_atomic32_inc(&cache_matcher->refcnt); 7770 LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); 7771 dev_flow->handle->dvh.matcher = cache_matcher; 7772 DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", 7773 key->domain ? "FDB" : "NIC", key->table_id, 7774 cache_matcher->priority, 7775 key->direction ? "tx" : "rx", (void *)cache_matcher, 7776 rte_atomic32_read(&cache_matcher->refcnt)); 7777 return 0; 7778 } 7779 7780 /** 7781 * Find existing tag resource or create and register a new one. 7782 * 7783 * @param dev[in, out] 7784 * Pointer to rte_eth_dev structure. 7785 * @param[in, out] tag_be24 7786 * Tag value in big endian then R-shift 8. 7787 * @parm[in, out] dev_flow 7788 * Pointer to the dev_flow. 7789 * @param[out] error 7790 * pointer to error structure. 7791 * 7792 * @return 7793 * 0 on success otherwise -errno and errno is set. 7794 */ 7795 static int 7796 flow_dv_tag_resource_register 7797 (struct rte_eth_dev *dev, 7798 uint32_t tag_be24, 7799 struct mlx5_flow *dev_flow, 7800 struct rte_flow_error *error) 7801 { 7802 struct mlx5_priv *priv = dev->data->dev_private; 7803 struct mlx5_dev_ctx_shared *sh = priv->sh; 7804 struct mlx5_flow_dv_tag_resource *cache_resource; 7805 struct mlx5_hlist_entry *entry; 7806 int ret; 7807 7808 /* Lookup a matching resource from cache. */ 7809 entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); 7810 if (entry) { 7811 cache_resource = container_of 7812 (entry, struct mlx5_flow_dv_tag_resource, entry); 7813 rte_atomic32_inc(&cache_resource->refcnt); 7814 dev_flow->handle->dvh.rix_tag = cache_resource->idx; 7815 dev_flow->dv.tag_resource = cache_resource; 7816 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", 7817 (void *)cache_resource, 7818 rte_atomic32_read(&cache_resource->refcnt)); 7819 return 0; 7820 } 7821 /* Register new resource. */ 7822 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], 7823 &dev_flow->handle->dvh.rix_tag); 7824 if (!cache_resource) 7825 return rte_flow_error_set(error, ENOMEM, 7826 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7827 "cannot allocate resource memory"); 7828 cache_resource->entry.key = (uint64_t)tag_be24; 7829 ret = mlx5_flow_os_create_flow_action_tag(tag_be24, 7830 &cache_resource->action); 7831 if (ret) { 7832 mlx5_free(cache_resource); 7833 return rte_flow_error_set(error, ENOMEM, 7834 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7835 NULL, "cannot create action"); 7836 } 7837 rte_atomic32_init(&cache_resource->refcnt); 7838 rte_atomic32_inc(&cache_resource->refcnt); 7839 if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { 7840 mlx5_flow_os_destroy_flow_action(cache_resource->action); 7841 mlx5_free(cache_resource); 7842 return rte_flow_error_set(error, EEXIST, 7843 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7844 NULL, "cannot insert tag"); 7845 } 7846 dev_flow->dv.tag_resource = cache_resource; 7847 DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", 7848 (void *)cache_resource, 7849 rte_atomic32_read(&cache_resource->refcnt)); 7850 return 0; 7851 } 7852 7853 /** 7854 * Release the tag. 7855 * 7856 * @param dev 7857 * Pointer to Ethernet device. 7858 * @param tag_idx 7859 * Tag index. 7860 * 7861 * @return 7862 * 1 while a reference on it exists, 0 when freed. 7863 */ 7864 static int 7865 flow_dv_tag_release(struct rte_eth_dev *dev, 7866 uint32_t tag_idx) 7867 { 7868 struct mlx5_priv *priv = dev->data->dev_private; 7869 struct mlx5_dev_ctx_shared *sh = priv->sh; 7870 struct mlx5_flow_dv_tag_resource *tag; 7871 7872 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7873 if (!tag) 7874 return 0; 7875 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", 7876 dev->data->port_id, (void *)tag, 7877 rte_atomic32_read(&tag->refcnt)); 7878 if (rte_atomic32_dec_and_test(&tag->refcnt)) { 7879 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action)); 7880 mlx5_hlist_remove(sh->tag_table, &tag->entry); 7881 DRV_LOG(DEBUG, "port %u tag %p: removed", 7882 dev->data->port_id, (void *)tag); 7883 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7884 return 0; 7885 } 7886 return 1; 7887 } 7888 7889 /** 7890 * Translate port ID action to vport. 7891 * 7892 * @param[in] dev 7893 * Pointer to rte_eth_dev structure. 7894 * @param[in] action 7895 * Pointer to the port ID action. 7896 * @param[out] dst_port_id 7897 * The target port ID. 7898 * @param[out] error 7899 * Pointer to the error structure. 7900 * 7901 * @return 7902 * 0 on success, a negative errno value otherwise and rte_errno is set. 7903 */ 7904 static int 7905 flow_dv_translate_action_port_id(struct rte_eth_dev *dev, 7906 const struct rte_flow_action *action, 7907 uint32_t *dst_port_id, 7908 struct rte_flow_error *error) 7909 { 7910 uint32_t port; 7911 struct mlx5_priv *priv; 7912 const struct rte_flow_action_port_id *conf = 7913 (const struct rte_flow_action_port_id *)action->conf; 7914 7915 port = conf->original ? dev->data->port_id : conf->id; 7916 priv = mlx5_port_to_eswitch_info(port, false); 7917 if (!priv) 7918 return rte_flow_error_set(error, -rte_errno, 7919 RTE_FLOW_ERROR_TYPE_ACTION, 7920 NULL, 7921 "No eswitch info was found for port"); 7922 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 7923 /* 7924 * This parameter is transferred to 7925 * mlx5dv_dr_action_create_dest_ib_port(). 7926 */ 7927 *dst_port_id = priv->dev_port; 7928 #else 7929 /* 7930 * Legacy mode, no LAG configurations is supported. 7931 * This parameter is transferred to 7932 * mlx5dv_dr_action_create_dest_vport(). 7933 */ 7934 *dst_port_id = priv->vport_id; 7935 #endif 7936 return 0; 7937 } 7938 7939 /** 7940 * Create a counter with aging configuration. 7941 * 7942 * @param[in] dev 7943 * Pointer to rte_eth_dev structure. 7944 * @param[out] count 7945 * Pointer to the counter action configuration. 7946 * @param[in] age 7947 * Pointer to the aging action configuration. 7948 * 7949 * @return 7950 * Index to flow counter on success, 0 otherwise. 7951 */ 7952 static uint32_t 7953 flow_dv_translate_create_counter(struct rte_eth_dev *dev, 7954 struct mlx5_flow *dev_flow, 7955 const struct rte_flow_action_count *count, 7956 const struct rte_flow_action_age *age) 7957 { 7958 uint32_t counter; 7959 struct mlx5_age_param *age_param; 7960 7961 counter = flow_dv_counter_alloc(dev, 7962 count ? count->shared : 0, 7963 count ? count->id : 0, 7964 dev_flow->dv.group, !!age); 7965 if (!counter || age == NULL) 7966 return counter; 7967 age_param = flow_dv_counter_idx_get_age(dev, counter); 7968 /* 7969 * The counter age accuracy may have a bit delay. Have 3/4 7970 * second bias on the timeount in order to let it age in time. 7971 */ 7972 age_param->context = age->context ? age->context : 7973 (void *)(uintptr_t)(dev_flow->flow_idx); 7974 /* 7975 * The counter age accuracy may have a bit delay. Have 3/4 7976 * second bias on the timeount in order to let it age in time. 7977 */ 7978 age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY; 7979 /* Set expire time in unit of 0.1 sec. */ 7980 age_param->port_id = dev->data->port_id; 7981 age_param->expire = age_param->timeout + 7982 rte_rdtsc() / (rte_get_tsc_hz() / 10); 7983 rte_atomic16_set(&age_param->state, AGE_CANDIDATE); 7984 return counter; 7985 } 7986 /** 7987 * Add Tx queue matcher 7988 * 7989 * @param[in] dev 7990 * Pointer to the dev struct. 7991 * @param[in, out] matcher 7992 * Flow matcher. 7993 * @param[in, out] key 7994 * Flow matcher value. 7995 * @param[in] item 7996 * Flow pattern to translate. 7997 * @param[in] inner 7998 * Item is inner pattern. 7999 */ 8000 static void 8001 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, 8002 void *matcher, void *key, 8003 const struct rte_flow_item *item) 8004 { 8005 const struct mlx5_rte_flow_item_tx_queue *queue_m; 8006 const struct mlx5_rte_flow_item_tx_queue *queue_v; 8007 void *misc_m = 8008 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 8009 void *misc_v = 8010 MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 8011 struct mlx5_txq_ctrl *txq; 8012 uint32_t queue; 8013 8014 8015 queue_m = (const void *)item->mask; 8016 if (!queue_m) 8017 return; 8018 queue_v = (const void *)item->spec; 8019 if (!queue_v) 8020 return; 8021 txq = mlx5_txq_get(dev, queue_v->queue); 8022 if (!txq) 8023 return; 8024 queue = txq->obj->sq->id; 8025 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue); 8026 MLX5_SET(fte_match_set_misc, misc_v, source_sqn, 8027 queue & queue_m->queue); 8028 mlx5_txq_release(dev, queue_v->queue); 8029 } 8030 8031 /** 8032 * Set the hash fields according to the @p flow information. 8033 * 8034 * @param[in] dev_flow 8035 * Pointer to the mlx5_flow. 8036 * @param[in] rss_desc 8037 * Pointer to the mlx5_flow_rss_desc. 8038 */ 8039 static void 8040 flow_dv_hashfields_set(struct mlx5_flow *dev_flow, 8041 struct mlx5_flow_rss_desc *rss_desc) 8042 { 8043 uint64_t items = dev_flow->handle->layers; 8044 int rss_inner = 0; 8045 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types); 8046 8047 dev_flow->hash_fields = 0; 8048 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 8049 if (rss_desc->level >= 2) { 8050 dev_flow->hash_fields |= IBV_RX_HASH_INNER; 8051 rss_inner = 1; 8052 } 8053 #endif 8054 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || 8055 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { 8056 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 8057 if (rss_types & ETH_RSS_L3_SRC_ONLY) 8058 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4; 8059 else if (rss_types & ETH_RSS_L3_DST_ONLY) 8060 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4; 8061 else 8062 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH; 8063 } 8064 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || 8065 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) { 8066 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 8067 if (rss_types & ETH_RSS_L3_SRC_ONLY) 8068 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6; 8069 else if (rss_types & ETH_RSS_L3_DST_ONLY) 8070 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6; 8071 else 8072 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; 8073 } 8074 } 8075 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || 8076 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { 8077 if (rss_types & ETH_RSS_UDP) { 8078 if (rss_types & ETH_RSS_L4_SRC_ONLY) 8079 dev_flow->hash_fields |= 8080 IBV_RX_HASH_SRC_PORT_UDP; 8081 else if (rss_types & ETH_RSS_L4_DST_ONLY) 8082 dev_flow->hash_fields |= 8083 IBV_RX_HASH_DST_PORT_UDP; 8084 else 8085 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH; 8086 } 8087 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || 8088 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) { 8089 if (rss_types & ETH_RSS_TCP) { 8090 if (rss_types & ETH_RSS_L4_SRC_ONLY) 8091 dev_flow->hash_fields |= 8092 IBV_RX_HASH_SRC_PORT_TCP; 8093 else if (rss_types & ETH_RSS_L4_DST_ONLY) 8094 dev_flow->hash_fields |= 8095 IBV_RX_HASH_DST_PORT_TCP; 8096 else 8097 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; 8098 } 8099 } 8100 } 8101 8102 /** 8103 * Fill the flow with DV spec, lock free 8104 * (mutex should be acquired by caller). 8105 * 8106 * @param[in] dev 8107 * Pointer to rte_eth_dev structure. 8108 * @param[in, out] dev_flow 8109 * Pointer to the sub flow. 8110 * @param[in] attr 8111 * Pointer to the flow attributes. 8112 * @param[in] items 8113 * Pointer to the list of items. 8114 * @param[in] actions 8115 * Pointer to the list of actions. 8116 * @param[out] error 8117 * Pointer to the error structure. 8118 * 8119 * @return 8120 * 0 on success, a negative errno value otherwise and rte_errno is set. 8121 */ 8122 static int 8123 __flow_dv_translate(struct rte_eth_dev *dev, 8124 struct mlx5_flow *dev_flow, 8125 const struct rte_flow_attr *attr, 8126 const struct rte_flow_item items[], 8127 const struct rte_flow_action actions[], 8128 struct rte_flow_error *error) 8129 { 8130 struct mlx5_priv *priv = dev->data->dev_private; 8131 struct mlx5_dev_config *dev_conf = &priv->config; 8132 struct rte_flow *flow = dev_flow->flow; 8133 struct mlx5_flow_handle *handle = dev_flow->handle; 8134 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 8135 priv->rss_desc) 8136 [!!priv->flow_nested_idx]; 8137 uint64_t item_flags = 0; 8138 uint64_t last_item = 0; 8139 uint64_t action_flags = 0; 8140 uint64_t priority = attr->priority; 8141 struct mlx5_flow_dv_matcher matcher = { 8142 .mask = { 8143 .size = sizeof(matcher.mask.buf) - 8144 MLX5_ST_SZ_BYTES(fte_match_set_misc4), 8145 }, 8146 }; 8147 int actions_n = 0; 8148 bool actions_end = false; 8149 union { 8150 struct mlx5_flow_dv_modify_hdr_resource res; 8151 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 8152 sizeof(struct mlx5_modification_cmd) * 8153 (MLX5_MAX_MODIFY_NUM + 1)]; 8154 } mhdr_dummy; 8155 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; 8156 const struct rte_flow_action_count *count = NULL; 8157 const struct rte_flow_action_age *age = NULL; 8158 union flow_dv_attr flow_attr = { .attr = 0 }; 8159 uint32_t tag_be; 8160 union mlx5_flow_tbl_key tbl_key; 8161 uint32_t modify_action_position = UINT32_MAX; 8162 void *match_mask = matcher.mask.buf; 8163 void *match_value = dev_flow->dv.value.buf; 8164 uint8_t next_protocol = 0xff; 8165 struct rte_vlan_hdr vlan = { 0 }; 8166 uint32_t table; 8167 int ret = 0; 8168 8169 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 8170 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 8171 ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, 8172 !!priv->fdb_def_rule, &table, error); 8173 if (ret) 8174 return ret; 8175 dev_flow->dv.group = table; 8176 if (attr->transfer) 8177 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 8178 if (priority == MLX5_FLOW_PRIO_RSVD) 8179 priority = dev_conf->flow_prio - 1; 8180 /* number of actions must be set to 0 in case of dirty stack. */ 8181 mhdr_res->actions_num = 0; 8182 for (; !actions_end ; actions++) { 8183 const struct rte_flow_action_queue *queue; 8184 const struct rte_flow_action_rss *rss; 8185 const struct rte_flow_action *action = actions; 8186 const uint8_t *rss_key; 8187 const struct rte_flow_action_jump *jump_data; 8188 const struct rte_flow_action_meter *mtr; 8189 struct mlx5_flow_tbl_resource *tbl; 8190 uint32_t port_id = 0; 8191 struct mlx5_flow_dv_port_id_action_resource port_id_resource; 8192 int action_type = actions->type; 8193 const struct rte_flow_action *found_action = NULL; 8194 struct mlx5_flow_meter *fm = NULL; 8195 8196 if (!mlx5_flow_os_action_supported(action_type)) 8197 return rte_flow_error_set(error, ENOTSUP, 8198 RTE_FLOW_ERROR_TYPE_ACTION, 8199 actions, 8200 "action not supported"); 8201 switch (action_type) { 8202 case RTE_FLOW_ACTION_TYPE_VOID: 8203 break; 8204 case RTE_FLOW_ACTION_TYPE_PORT_ID: 8205 if (flow_dv_translate_action_port_id(dev, action, 8206 &port_id, error)) 8207 return -rte_errno; 8208 port_id_resource.port_id = port_id; 8209 MLX5_ASSERT(!handle->rix_port_id_action); 8210 if (flow_dv_port_id_action_resource_register 8211 (dev, &port_id_resource, dev_flow, error)) 8212 return -rte_errno; 8213 dev_flow->dv.actions[actions_n++] = 8214 dev_flow->dv.port_id_action->action; 8215 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 8216 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID; 8217 break; 8218 case RTE_FLOW_ACTION_TYPE_FLAG: 8219 action_flags |= MLX5_FLOW_ACTION_FLAG; 8220 dev_flow->handle->mark = 1; 8221 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 8222 struct rte_flow_action_mark mark = { 8223 .id = MLX5_FLOW_MARK_DEFAULT, 8224 }; 8225 8226 if (flow_dv_convert_action_mark(dev, &mark, 8227 mhdr_res, 8228 error)) 8229 return -rte_errno; 8230 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 8231 break; 8232 } 8233 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 8234 /* 8235 * Only one FLAG or MARK is supported per device flow 8236 * right now. So the pointer to the tag resource must be 8237 * zero before the register process. 8238 */ 8239 MLX5_ASSERT(!handle->dvh.rix_tag); 8240 if (flow_dv_tag_resource_register(dev, tag_be, 8241 dev_flow, error)) 8242 return -rte_errno; 8243 MLX5_ASSERT(dev_flow->dv.tag_resource); 8244 dev_flow->dv.actions[actions_n++] = 8245 dev_flow->dv.tag_resource->action; 8246 break; 8247 case RTE_FLOW_ACTION_TYPE_MARK: 8248 action_flags |= MLX5_FLOW_ACTION_MARK; 8249 dev_flow->handle->mark = 1; 8250 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 8251 const struct rte_flow_action_mark *mark = 8252 (const struct rte_flow_action_mark *) 8253 actions->conf; 8254 8255 if (flow_dv_convert_action_mark(dev, mark, 8256 mhdr_res, 8257 error)) 8258 return -rte_errno; 8259 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 8260 break; 8261 } 8262 /* Fall-through */ 8263 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 8264 /* Legacy (non-extensive) MARK action. */ 8265 tag_be = mlx5_flow_mark_set 8266 (((const struct rte_flow_action_mark *) 8267 (actions->conf))->id); 8268 MLX5_ASSERT(!handle->dvh.rix_tag); 8269 if (flow_dv_tag_resource_register(dev, tag_be, 8270 dev_flow, error)) 8271 return -rte_errno; 8272 MLX5_ASSERT(dev_flow->dv.tag_resource); 8273 dev_flow->dv.actions[actions_n++] = 8274 dev_flow->dv.tag_resource->action; 8275 break; 8276 case RTE_FLOW_ACTION_TYPE_SET_META: 8277 if (flow_dv_convert_action_set_meta 8278 (dev, mhdr_res, attr, 8279 (const struct rte_flow_action_set_meta *) 8280 actions->conf, error)) 8281 return -rte_errno; 8282 action_flags |= MLX5_FLOW_ACTION_SET_META; 8283 break; 8284 case RTE_FLOW_ACTION_TYPE_SET_TAG: 8285 if (flow_dv_convert_action_set_tag 8286 (dev, mhdr_res, 8287 (const struct rte_flow_action_set_tag *) 8288 actions->conf, error)) 8289 return -rte_errno; 8290 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8291 break; 8292 case RTE_FLOW_ACTION_TYPE_DROP: 8293 action_flags |= MLX5_FLOW_ACTION_DROP; 8294 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 8295 break; 8296 case RTE_FLOW_ACTION_TYPE_QUEUE: 8297 queue = actions->conf; 8298 rss_desc->queue_num = 1; 8299 rss_desc->queue[0] = queue->index; 8300 action_flags |= MLX5_FLOW_ACTION_QUEUE; 8301 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8302 break; 8303 case RTE_FLOW_ACTION_TYPE_RSS: 8304 rss = actions->conf; 8305 memcpy(rss_desc->queue, rss->queue, 8306 rss->queue_num * sizeof(uint16_t)); 8307 rss_desc->queue_num = rss->queue_num; 8308 /* NULL RSS key indicates default RSS key. */ 8309 rss_key = !rss->key ? rss_hash_default_key : rss->key; 8310 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 8311 /* 8312 * rss->level and rss.types should be set in advance 8313 * when expanding items for RSS. 8314 */ 8315 action_flags |= MLX5_FLOW_ACTION_RSS; 8316 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8317 break; 8318 case RTE_FLOW_ACTION_TYPE_AGE: 8319 case RTE_FLOW_ACTION_TYPE_COUNT: 8320 if (!dev_conf->devx) { 8321 return rte_flow_error_set 8322 (error, ENOTSUP, 8323 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8324 NULL, 8325 "count action not supported"); 8326 } 8327 /* Save information first, will apply later. */ 8328 if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT) 8329 count = action->conf; 8330 else 8331 age = action->conf; 8332 action_flags |= MLX5_FLOW_ACTION_COUNT; 8333 break; 8334 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 8335 dev_flow->dv.actions[actions_n++] = 8336 priv->sh->pop_vlan_action; 8337 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 8338 break; 8339 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 8340 if (!(action_flags & 8341 MLX5_FLOW_ACTION_OF_SET_VLAN_VID)) 8342 flow_dev_get_vlan_info_from_items(items, &vlan); 8343 vlan.eth_proto = rte_be_to_cpu_16 8344 ((((const struct rte_flow_action_of_push_vlan *) 8345 actions->conf)->ethertype)); 8346 found_action = mlx5_flow_find_action 8347 (actions + 1, 8348 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID); 8349 if (found_action) 8350 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8351 found_action = mlx5_flow_find_action 8352 (actions + 1, 8353 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP); 8354 if (found_action) 8355 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8356 if (flow_dv_create_action_push_vlan 8357 (dev, attr, &vlan, dev_flow, error)) 8358 return -rte_errno; 8359 dev_flow->dv.actions[actions_n++] = 8360 dev_flow->dv.push_vlan_res->action; 8361 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 8362 break; 8363 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 8364 /* of_vlan_push action handled this action */ 8365 MLX5_ASSERT(action_flags & 8366 MLX5_FLOW_ACTION_OF_PUSH_VLAN); 8367 break; 8368 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 8369 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 8370 break; 8371 flow_dev_get_vlan_info_from_items(items, &vlan); 8372 mlx5_update_vlan_vid_pcp(actions, &vlan); 8373 /* If no VLAN push - this is a modify header action */ 8374 if (flow_dv_convert_action_modify_vlan_vid 8375 (mhdr_res, actions, error)) 8376 return -rte_errno; 8377 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 8378 break; 8379 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 8380 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 8381 if (flow_dv_create_action_l2_encap(dev, actions, 8382 dev_flow, 8383 attr->transfer, 8384 error)) 8385 return -rte_errno; 8386 dev_flow->dv.actions[actions_n++] = 8387 dev_flow->dv.encap_decap->action; 8388 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8389 break; 8390 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 8391 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 8392 if (flow_dv_create_action_l2_decap(dev, dev_flow, 8393 attr->transfer, 8394 error)) 8395 return -rte_errno; 8396 dev_flow->dv.actions[actions_n++] = 8397 dev_flow->dv.encap_decap->action; 8398 action_flags |= MLX5_FLOW_ACTION_DECAP; 8399 break; 8400 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 8401 /* Handle encap with preceding decap. */ 8402 if (action_flags & MLX5_FLOW_ACTION_DECAP) { 8403 if (flow_dv_create_action_raw_encap 8404 (dev, actions, dev_flow, attr, error)) 8405 return -rte_errno; 8406 dev_flow->dv.actions[actions_n++] = 8407 dev_flow->dv.encap_decap->action; 8408 } else { 8409 /* Handle encap without preceding decap. */ 8410 if (flow_dv_create_action_l2_encap 8411 (dev, actions, dev_flow, attr->transfer, 8412 error)) 8413 return -rte_errno; 8414 dev_flow->dv.actions[actions_n++] = 8415 dev_flow->dv.encap_decap->action; 8416 } 8417 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8418 break; 8419 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 8420 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID) 8421 ; 8422 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 8423 if (flow_dv_create_action_l2_decap 8424 (dev, dev_flow, attr->transfer, error)) 8425 return -rte_errno; 8426 dev_flow->dv.actions[actions_n++] = 8427 dev_flow->dv.encap_decap->action; 8428 } 8429 /* If decap is followed by encap, handle it at encap. */ 8430 action_flags |= MLX5_FLOW_ACTION_DECAP; 8431 break; 8432 case RTE_FLOW_ACTION_TYPE_JUMP: 8433 jump_data = action->conf; 8434 ret = mlx5_flow_group_to_table(attr, dev_flow->external, 8435 jump_data->group, 8436 !!priv->fdb_def_rule, 8437 &table, error); 8438 if (ret) 8439 return ret; 8440 tbl = flow_dv_tbl_resource_get(dev, table, 8441 attr->egress, 8442 attr->transfer, error); 8443 if (!tbl) 8444 return rte_flow_error_set 8445 (error, errno, 8446 RTE_FLOW_ERROR_TYPE_ACTION, 8447 NULL, 8448 "cannot create jump action."); 8449 if (flow_dv_jump_tbl_resource_register 8450 (dev, tbl, dev_flow, error)) { 8451 flow_dv_tbl_resource_release(dev, tbl); 8452 return rte_flow_error_set 8453 (error, errno, 8454 RTE_FLOW_ERROR_TYPE_ACTION, 8455 NULL, 8456 "cannot create jump action."); 8457 } 8458 dev_flow->dv.actions[actions_n++] = 8459 dev_flow->dv.jump->action; 8460 action_flags |= MLX5_FLOW_ACTION_JUMP; 8461 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; 8462 break; 8463 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 8464 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 8465 if (flow_dv_convert_action_modify_mac 8466 (mhdr_res, actions, error)) 8467 return -rte_errno; 8468 action_flags |= actions->type == 8469 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 8470 MLX5_FLOW_ACTION_SET_MAC_SRC : 8471 MLX5_FLOW_ACTION_SET_MAC_DST; 8472 break; 8473 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 8474 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 8475 if (flow_dv_convert_action_modify_ipv4 8476 (mhdr_res, actions, error)) 8477 return -rte_errno; 8478 action_flags |= actions->type == 8479 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 8480 MLX5_FLOW_ACTION_SET_IPV4_SRC : 8481 MLX5_FLOW_ACTION_SET_IPV4_DST; 8482 break; 8483 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 8484 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 8485 if (flow_dv_convert_action_modify_ipv6 8486 (mhdr_res, actions, error)) 8487 return -rte_errno; 8488 action_flags |= actions->type == 8489 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 8490 MLX5_FLOW_ACTION_SET_IPV6_SRC : 8491 MLX5_FLOW_ACTION_SET_IPV6_DST; 8492 break; 8493 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 8494 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 8495 if (flow_dv_convert_action_modify_tp 8496 (mhdr_res, actions, items, 8497 &flow_attr, dev_flow, !!(action_flags & 8498 MLX5_FLOW_ACTION_DECAP), error)) 8499 return -rte_errno; 8500 action_flags |= actions->type == 8501 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 8502 MLX5_FLOW_ACTION_SET_TP_SRC : 8503 MLX5_FLOW_ACTION_SET_TP_DST; 8504 break; 8505 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 8506 if (flow_dv_convert_action_modify_dec_ttl 8507 (mhdr_res, items, &flow_attr, dev_flow, 8508 !!(action_flags & 8509 MLX5_FLOW_ACTION_DECAP), error)) 8510 return -rte_errno; 8511 action_flags |= MLX5_FLOW_ACTION_DEC_TTL; 8512 break; 8513 case RTE_FLOW_ACTION_TYPE_SET_TTL: 8514 if (flow_dv_convert_action_modify_ttl 8515 (mhdr_res, actions, items, &flow_attr, 8516 dev_flow, !!(action_flags & 8517 MLX5_FLOW_ACTION_DECAP), error)) 8518 return -rte_errno; 8519 action_flags |= MLX5_FLOW_ACTION_SET_TTL; 8520 break; 8521 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 8522 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 8523 if (flow_dv_convert_action_modify_tcp_seq 8524 (mhdr_res, actions, error)) 8525 return -rte_errno; 8526 action_flags |= actions->type == 8527 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 8528 MLX5_FLOW_ACTION_INC_TCP_SEQ : 8529 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 8530 break; 8531 8532 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 8533 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 8534 if (flow_dv_convert_action_modify_tcp_ack 8535 (mhdr_res, actions, error)) 8536 return -rte_errno; 8537 action_flags |= actions->type == 8538 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 8539 MLX5_FLOW_ACTION_INC_TCP_ACK : 8540 MLX5_FLOW_ACTION_DEC_TCP_ACK; 8541 break; 8542 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 8543 if (flow_dv_convert_action_set_reg 8544 (mhdr_res, actions, error)) 8545 return -rte_errno; 8546 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8547 break; 8548 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 8549 if (flow_dv_convert_action_copy_mreg 8550 (dev, mhdr_res, actions, error)) 8551 return -rte_errno; 8552 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8553 break; 8554 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 8555 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 8556 dev_flow->handle->fate_action = 8557 MLX5_FLOW_FATE_DEFAULT_MISS; 8558 break; 8559 case RTE_FLOW_ACTION_TYPE_METER: 8560 mtr = actions->conf; 8561 if (!flow->meter) { 8562 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, 8563 attr, error); 8564 if (!fm) 8565 return rte_flow_error_set(error, 8566 rte_errno, 8567 RTE_FLOW_ERROR_TYPE_ACTION, 8568 NULL, 8569 "meter not found " 8570 "or invalid parameters"); 8571 flow->meter = fm->idx; 8572 } 8573 /* Set the meter action. */ 8574 if (!fm) { 8575 fm = mlx5_ipool_get(priv->sh->ipool 8576 [MLX5_IPOOL_MTR], flow->meter); 8577 if (!fm) 8578 return rte_flow_error_set(error, 8579 rte_errno, 8580 RTE_FLOW_ERROR_TYPE_ACTION, 8581 NULL, 8582 "meter not found " 8583 "or invalid parameters"); 8584 } 8585 dev_flow->dv.actions[actions_n++] = 8586 fm->mfts->meter_action; 8587 action_flags |= MLX5_FLOW_ACTION_METER; 8588 break; 8589 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 8590 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, 8591 actions, error)) 8592 return -rte_errno; 8593 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 8594 break; 8595 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 8596 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, 8597 actions, error)) 8598 return -rte_errno; 8599 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 8600 break; 8601 case RTE_FLOW_ACTION_TYPE_END: 8602 actions_end = true; 8603 if (mhdr_res->actions_num) { 8604 /* create modify action if needed. */ 8605 if (flow_dv_modify_hdr_resource_register 8606 (dev, mhdr_res, dev_flow, error)) 8607 return -rte_errno; 8608 dev_flow->dv.actions[modify_action_position] = 8609 handle->dvh.modify_hdr->action; 8610 } 8611 if (action_flags & MLX5_FLOW_ACTION_COUNT) { 8612 flow->counter = 8613 flow_dv_translate_create_counter(dev, 8614 dev_flow, count, age); 8615 8616 if (!flow->counter) 8617 return rte_flow_error_set 8618 (error, rte_errno, 8619 RTE_FLOW_ERROR_TYPE_ACTION, 8620 NULL, 8621 "cannot create counter" 8622 " object."); 8623 dev_flow->dv.actions[actions_n++] = 8624 (flow_dv_counter_get_by_idx(dev, 8625 flow->counter, NULL))->action; 8626 } 8627 break; 8628 default: 8629 break; 8630 } 8631 if (mhdr_res->actions_num && 8632 modify_action_position == UINT32_MAX) 8633 modify_action_position = actions_n++; 8634 } 8635 dev_flow->dv.actions_n = actions_n; 8636 dev_flow->act_flags = action_flags; 8637 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 8638 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 8639 int item_type = items->type; 8640 8641 if (!mlx5_flow_os_item_supported(item_type)) 8642 return rte_flow_error_set(error, ENOTSUP, 8643 RTE_FLOW_ERROR_TYPE_ITEM, 8644 NULL, "item not supported"); 8645 switch (item_type) { 8646 case RTE_FLOW_ITEM_TYPE_PORT_ID: 8647 flow_dv_translate_item_port_id(dev, match_mask, 8648 match_value, items); 8649 last_item = MLX5_FLOW_ITEM_PORT_ID; 8650 break; 8651 case RTE_FLOW_ITEM_TYPE_ETH: 8652 flow_dv_translate_item_eth(match_mask, match_value, 8653 items, tunnel, 8654 dev_flow->dv.group); 8655 matcher.priority = action_flags & 8656 MLX5_FLOW_ACTION_DEFAULT_MISS && 8657 !dev_flow->external ? 8658 MLX5_PRIORITY_MAP_L3 : 8659 MLX5_PRIORITY_MAP_L2; 8660 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 8661 MLX5_FLOW_LAYER_OUTER_L2; 8662 break; 8663 case RTE_FLOW_ITEM_TYPE_VLAN: 8664 flow_dv_translate_item_vlan(dev_flow, 8665 match_mask, match_value, 8666 items, tunnel, 8667 dev_flow->dv.group); 8668 matcher.priority = MLX5_PRIORITY_MAP_L2; 8669 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 8670 MLX5_FLOW_LAYER_INNER_VLAN) : 8671 (MLX5_FLOW_LAYER_OUTER_L2 | 8672 MLX5_FLOW_LAYER_OUTER_VLAN); 8673 break; 8674 case RTE_FLOW_ITEM_TYPE_IPV4: 8675 mlx5_flow_tunnel_ip_check(items, next_protocol, 8676 &item_flags, &tunnel); 8677 flow_dv_translate_item_ipv4(match_mask, match_value, 8678 items, item_flags, tunnel, 8679 dev_flow->dv.group); 8680 matcher.priority = MLX5_PRIORITY_MAP_L3; 8681 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 8682 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 8683 if (items->mask != NULL && 8684 ((const struct rte_flow_item_ipv4 *) 8685 items->mask)->hdr.next_proto_id) { 8686 next_protocol = 8687 ((const struct rte_flow_item_ipv4 *) 8688 (items->spec))->hdr.next_proto_id; 8689 next_protocol &= 8690 ((const struct rte_flow_item_ipv4 *) 8691 (items->mask))->hdr.next_proto_id; 8692 } else { 8693 /* Reset for inner layer. */ 8694 next_protocol = 0xff; 8695 } 8696 break; 8697 case RTE_FLOW_ITEM_TYPE_IPV6: 8698 mlx5_flow_tunnel_ip_check(items, next_protocol, 8699 &item_flags, &tunnel); 8700 flow_dv_translate_item_ipv6(match_mask, match_value, 8701 items, item_flags, tunnel, 8702 dev_flow->dv.group); 8703 matcher.priority = MLX5_PRIORITY_MAP_L3; 8704 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 8705 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 8706 if (items->mask != NULL && 8707 ((const struct rte_flow_item_ipv6 *) 8708 items->mask)->hdr.proto) { 8709 next_protocol = 8710 ((const struct rte_flow_item_ipv6 *) 8711 items->spec)->hdr.proto; 8712 next_protocol &= 8713 ((const struct rte_flow_item_ipv6 *) 8714 items->mask)->hdr.proto; 8715 } else { 8716 /* Reset for inner layer. */ 8717 next_protocol = 0xff; 8718 } 8719 break; 8720 case RTE_FLOW_ITEM_TYPE_TCP: 8721 flow_dv_translate_item_tcp(match_mask, match_value, 8722 items, tunnel); 8723 matcher.priority = MLX5_PRIORITY_MAP_L4; 8724 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 8725 MLX5_FLOW_LAYER_OUTER_L4_TCP; 8726 break; 8727 case RTE_FLOW_ITEM_TYPE_UDP: 8728 flow_dv_translate_item_udp(match_mask, match_value, 8729 items, tunnel); 8730 matcher.priority = MLX5_PRIORITY_MAP_L4; 8731 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 8732 MLX5_FLOW_LAYER_OUTER_L4_UDP; 8733 break; 8734 case RTE_FLOW_ITEM_TYPE_GRE: 8735 flow_dv_translate_item_gre(match_mask, match_value, 8736 items, tunnel); 8737 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8738 last_item = MLX5_FLOW_LAYER_GRE; 8739 break; 8740 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 8741 flow_dv_translate_item_gre_key(match_mask, 8742 match_value, items); 8743 last_item = MLX5_FLOW_LAYER_GRE_KEY; 8744 break; 8745 case RTE_FLOW_ITEM_TYPE_NVGRE: 8746 flow_dv_translate_item_nvgre(match_mask, match_value, 8747 items, tunnel); 8748 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8749 last_item = MLX5_FLOW_LAYER_GRE; 8750 break; 8751 case RTE_FLOW_ITEM_TYPE_VXLAN: 8752 flow_dv_translate_item_vxlan(match_mask, match_value, 8753 items, tunnel); 8754 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8755 last_item = MLX5_FLOW_LAYER_VXLAN; 8756 break; 8757 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 8758 flow_dv_translate_item_vxlan_gpe(match_mask, 8759 match_value, items, 8760 tunnel); 8761 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8762 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 8763 break; 8764 case RTE_FLOW_ITEM_TYPE_GENEVE: 8765 flow_dv_translate_item_geneve(match_mask, match_value, 8766 items, tunnel); 8767 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8768 last_item = MLX5_FLOW_LAYER_GENEVE; 8769 break; 8770 case RTE_FLOW_ITEM_TYPE_MPLS: 8771 flow_dv_translate_item_mpls(match_mask, match_value, 8772 items, last_item, tunnel); 8773 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8774 last_item = MLX5_FLOW_LAYER_MPLS; 8775 break; 8776 case RTE_FLOW_ITEM_TYPE_MARK: 8777 flow_dv_translate_item_mark(dev, match_mask, 8778 match_value, items); 8779 last_item = MLX5_FLOW_ITEM_MARK; 8780 break; 8781 case RTE_FLOW_ITEM_TYPE_META: 8782 flow_dv_translate_item_meta(dev, match_mask, 8783 match_value, attr, items); 8784 last_item = MLX5_FLOW_ITEM_METADATA; 8785 break; 8786 case RTE_FLOW_ITEM_TYPE_ICMP: 8787 flow_dv_translate_item_icmp(match_mask, match_value, 8788 items, tunnel); 8789 last_item = MLX5_FLOW_LAYER_ICMP; 8790 break; 8791 case RTE_FLOW_ITEM_TYPE_ICMP6: 8792 flow_dv_translate_item_icmp6(match_mask, match_value, 8793 items, tunnel); 8794 last_item = MLX5_FLOW_LAYER_ICMP6; 8795 break; 8796 case RTE_FLOW_ITEM_TYPE_TAG: 8797 flow_dv_translate_item_tag(dev, match_mask, 8798 match_value, items); 8799 last_item = MLX5_FLOW_ITEM_TAG; 8800 break; 8801 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 8802 flow_dv_translate_mlx5_item_tag(dev, match_mask, 8803 match_value, items); 8804 last_item = MLX5_FLOW_ITEM_TAG; 8805 break; 8806 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 8807 flow_dv_translate_item_tx_queue(dev, match_mask, 8808 match_value, 8809 items); 8810 last_item = MLX5_FLOW_ITEM_TX_QUEUE; 8811 break; 8812 case RTE_FLOW_ITEM_TYPE_GTP: 8813 flow_dv_translate_item_gtp(match_mask, match_value, 8814 items, tunnel); 8815 matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc); 8816 last_item = MLX5_FLOW_LAYER_GTP; 8817 break; 8818 case RTE_FLOW_ITEM_TYPE_ECPRI: 8819 if (!mlx5_flex_parser_ecpri_exist(dev)) { 8820 /* Create it only the first time to be used. */ 8821 ret = mlx5_flex_parser_ecpri_alloc(dev); 8822 if (ret) 8823 return rte_flow_error_set 8824 (error, -ret, 8825 RTE_FLOW_ERROR_TYPE_ITEM, 8826 NULL, 8827 "cannot create eCPRI parser"); 8828 } 8829 /* Adjust the length matcher and device flow value. */ 8830 matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param); 8831 dev_flow->dv.value.size = 8832 MLX5_ST_SZ_BYTES(fte_match_param); 8833 flow_dv_translate_item_ecpri(dev, match_mask, 8834 match_value, items); 8835 /* No other protocol should follow eCPRI layer. */ 8836 last_item = MLX5_FLOW_LAYER_ECPRI; 8837 break; 8838 default: 8839 break; 8840 } 8841 item_flags |= last_item; 8842 } 8843 /* 8844 * When E-Switch mode is enabled, we have two cases where we need to 8845 * set the source port manually. 8846 * The first one, is in case of Nic steering rule, and the second is 8847 * E-Switch rule where no port_id item was found. In both cases 8848 * the source port is set according the current port in use. 8849 */ 8850 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && 8851 (priv->representor || priv->master)) { 8852 if (flow_dv_translate_item_port_id(dev, match_mask, 8853 match_value, NULL)) 8854 return -rte_errno; 8855 } 8856 #ifdef RTE_LIBRTE_MLX5_DEBUG 8857 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf, 8858 dev_flow->dv.value.buf)); 8859 #endif 8860 /* 8861 * Layers may be already initialized from prefix flow if this dev_flow 8862 * is the suffix flow. 8863 */ 8864 handle->layers |= item_flags; 8865 if (action_flags & MLX5_FLOW_ACTION_RSS) 8866 flow_dv_hashfields_set(dev_flow, rss_desc); 8867 /* Register matcher. */ 8868 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 8869 matcher.mask.size); 8870 matcher.priority = mlx5_flow_adjust_priority(dev, priority, 8871 matcher.priority); 8872 /* reserved field no needs to be set to 0 here. */ 8873 tbl_key.domain = attr->transfer; 8874 tbl_key.direction = attr->egress; 8875 tbl_key.table_id = dev_flow->dv.group; 8876 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) 8877 return -rte_errno; 8878 return 0; 8879 } 8880 8881 /** 8882 * Apply the flow to the NIC, lock free, 8883 * (mutex should be acquired by caller). 8884 * 8885 * @param[in] dev 8886 * Pointer to the Ethernet device structure. 8887 * @param[in, out] flow 8888 * Pointer to flow structure. 8889 * @param[out] error 8890 * Pointer to error structure. 8891 * 8892 * @return 8893 * 0 on success, a negative errno value otherwise and rte_errno is set. 8894 */ 8895 static int 8896 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 8897 struct rte_flow_error *error) 8898 { 8899 struct mlx5_flow_dv_workspace *dv; 8900 struct mlx5_flow_handle *dh; 8901 struct mlx5_flow_handle_dv *dv_h; 8902 struct mlx5_flow *dev_flow; 8903 struct mlx5_priv *priv = dev->data->dev_private; 8904 uint32_t handle_idx; 8905 int n; 8906 int err; 8907 int idx; 8908 8909 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) { 8910 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; 8911 dv = &dev_flow->dv; 8912 dh = dev_flow->handle; 8913 dv_h = &dh->dvh; 8914 n = dv->actions_n; 8915 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 8916 if (dv->transfer) { 8917 dv->actions[n++] = priv->sh->esw_drop_action; 8918 } else { 8919 struct mlx5_hrxq *drop_hrxq; 8920 drop_hrxq = mlx5_drop_action_create(dev); 8921 if (!drop_hrxq) { 8922 rte_flow_error_set 8923 (error, errno, 8924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8925 NULL, 8926 "cannot get drop hash queue"); 8927 goto error; 8928 } 8929 /* 8930 * Drop queues will be released by the specify 8931 * mlx5_drop_action_destroy() function. Assign 8932 * the special index to hrxq to mark the queue 8933 * has been allocated. 8934 */ 8935 dh->rix_hrxq = UINT32_MAX; 8936 dv->actions[n++] = drop_hrxq->action; 8937 } 8938 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 8939 struct mlx5_hrxq *hrxq; 8940 uint32_t hrxq_idx; 8941 struct mlx5_flow_rss_desc *rss_desc = 8942 &((struct mlx5_flow_rss_desc *)priv->rss_desc) 8943 [!!priv->flow_nested_idx]; 8944 8945 MLX5_ASSERT(rss_desc->queue_num); 8946 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, 8947 MLX5_RSS_HASH_KEY_LEN, 8948 dev_flow->hash_fields, 8949 rss_desc->queue, 8950 rss_desc->queue_num); 8951 if (!hrxq_idx) { 8952 hrxq_idx = mlx5_hrxq_new 8953 (dev, rss_desc->key, 8954 MLX5_RSS_HASH_KEY_LEN, 8955 dev_flow->hash_fields, 8956 rss_desc->queue, 8957 rss_desc->queue_num, 8958 !!(dh->layers & 8959 MLX5_FLOW_LAYER_TUNNEL)); 8960 } 8961 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 8962 hrxq_idx); 8963 if (!hrxq) { 8964 rte_flow_error_set 8965 (error, rte_errno, 8966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8967 "cannot get hash queue"); 8968 goto error; 8969 } 8970 dh->rix_hrxq = hrxq_idx; 8971 dv->actions[n++] = hrxq->action; 8972 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { 8973 if (flow_dv_default_miss_resource_register 8974 (dev, error)) { 8975 rte_flow_error_set 8976 (error, rte_errno, 8977 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8978 "cannot create default miss resource"); 8979 goto error_default_miss; 8980 } 8981 dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS; 8982 dv->actions[n++] = priv->sh->default_miss.action; 8983 } 8984 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object, 8985 (void *)&dv->value, n, 8986 dv->actions, &dh->drv_flow); 8987 if (err) { 8988 rte_flow_error_set(error, errno, 8989 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8990 NULL, 8991 "hardware refuses to create flow"); 8992 goto error; 8993 } 8994 if (priv->vmwa_context && 8995 dh->vf_vlan.tag && !dh->vf_vlan.created) { 8996 /* 8997 * The rule contains the VLAN pattern. 8998 * For VF we are going to create VLAN 8999 * interface to make hypervisor set correct 9000 * e-Switch vport context. 9001 */ 9002 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan); 9003 } 9004 } 9005 return 0; 9006 error: 9007 if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 9008 flow_dv_default_miss_resource_release(dev); 9009 error_default_miss: 9010 err = rte_errno; /* Save rte_errno before cleanup. */ 9011 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 9012 handle_idx, dh, next) { 9013 /* hrxq is union, don't clear it if the flag is not set. */ 9014 if (dh->rix_hrxq) { 9015 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 9016 mlx5_drop_action_destroy(dev); 9017 dh->rix_hrxq = 0; 9018 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 9019 mlx5_hrxq_release(dev, dh->rix_hrxq); 9020 dh->rix_hrxq = 0; 9021 } 9022 } 9023 if (dh->vf_vlan.tag && dh->vf_vlan.created) 9024 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 9025 } 9026 rte_errno = err; /* Restore rte_errno. */ 9027 return -rte_errno; 9028 } 9029 9030 /** 9031 * Release the flow matcher. 9032 * 9033 * @param dev 9034 * Pointer to Ethernet device. 9035 * @param handle 9036 * Pointer to mlx5_flow_handle. 9037 * 9038 * @return 9039 * 1 while a reference on it exists, 0 when freed. 9040 */ 9041 static int 9042 flow_dv_matcher_release(struct rte_eth_dev *dev, 9043 struct mlx5_flow_handle *handle) 9044 { 9045 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; 9046 9047 MLX5_ASSERT(matcher->matcher_object); 9048 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", 9049 dev->data->port_id, (void *)matcher, 9050 rte_atomic32_read(&matcher->refcnt)); 9051 if (rte_atomic32_dec_and_test(&matcher->refcnt)) { 9052 claim_zero(mlx5_flow_os_destroy_flow_matcher 9053 (matcher->matcher_object)); 9054 LIST_REMOVE(matcher, next); 9055 /* table ref-- in release interface. */ 9056 flow_dv_tbl_resource_release(dev, matcher->tbl); 9057 mlx5_free(matcher); 9058 DRV_LOG(DEBUG, "port %u matcher %p: removed", 9059 dev->data->port_id, (void *)matcher); 9060 return 0; 9061 } 9062 return 1; 9063 } 9064 9065 /** 9066 * Release an encap/decap resource. 9067 * 9068 * @param dev 9069 * Pointer to Ethernet device. 9070 * @param handle 9071 * Pointer to mlx5_flow_handle. 9072 * 9073 * @return 9074 * 1 while a reference on it exists, 0 when freed. 9075 */ 9076 static int 9077 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, 9078 struct mlx5_flow_handle *handle) 9079 { 9080 struct mlx5_priv *priv = dev->data->dev_private; 9081 uint32_t idx = handle->dvh.rix_encap_decap; 9082 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 9083 9084 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 9085 idx); 9086 if (!cache_resource) 9087 return 0; 9088 MLX5_ASSERT(cache_resource->action); 9089 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", 9090 (void *)cache_resource, 9091 rte_atomic32_read(&cache_resource->refcnt)); 9092 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9093 claim_zero(mlx5_flow_os_destroy_flow_action 9094 (cache_resource->action)); 9095 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 9096 &priv->sh->encaps_decaps, idx, 9097 cache_resource, next); 9098 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); 9099 DRV_LOG(DEBUG, "encap/decap resource %p: removed", 9100 (void *)cache_resource); 9101 return 0; 9102 } 9103 return 1; 9104 } 9105 9106 /** 9107 * Release an jump to table action resource. 9108 * 9109 * @param dev 9110 * Pointer to Ethernet device. 9111 * @param handle 9112 * Pointer to mlx5_flow_handle. 9113 * 9114 * @return 9115 * 1 while a reference on it exists, 0 when freed. 9116 */ 9117 static int 9118 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, 9119 struct mlx5_flow_handle *handle) 9120 { 9121 struct mlx5_priv *priv = dev->data->dev_private; 9122 struct mlx5_flow_dv_jump_tbl_resource *cache_resource; 9123 struct mlx5_flow_tbl_data_entry *tbl_data; 9124 9125 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], 9126 handle->rix_jump); 9127 if (!tbl_data) 9128 return 0; 9129 cache_resource = &tbl_data->jump; 9130 MLX5_ASSERT(cache_resource->action); 9131 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", 9132 (void *)cache_resource, 9133 rte_atomic32_read(&cache_resource->refcnt)); 9134 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9135 claim_zero(mlx5_flow_os_destroy_flow_action 9136 (cache_resource->action)); 9137 /* jump action memory free is inside the table release. */ 9138 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 9139 DRV_LOG(DEBUG, "jump table resource %p: removed", 9140 (void *)cache_resource); 9141 return 0; 9142 } 9143 return 1; 9144 } 9145 9146 /** 9147 * Release a default miss resource. 9148 * 9149 * @param dev 9150 * Pointer to Ethernet device. 9151 * @return 9152 * 1 while a reference on it exists, 0 when freed. 9153 */ 9154 static int 9155 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev) 9156 { 9157 struct mlx5_priv *priv = dev->data->dev_private; 9158 struct mlx5_dev_ctx_shared *sh = priv->sh; 9159 struct mlx5_flow_default_miss_resource *cache_resource = 9160 &sh->default_miss; 9161 9162 MLX5_ASSERT(cache_resource->action); 9163 DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--", 9164 (void *)cache_resource->action, 9165 rte_atomic32_read(&cache_resource->refcnt)); 9166 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9167 claim_zero(mlx5_glue->destroy_flow_action 9168 (cache_resource->action)); 9169 DRV_LOG(DEBUG, "default miss resource %p: removed", 9170 (void *)cache_resource->action); 9171 return 0; 9172 } 9173 return 1; 9174 } 9175 9176 /** 9177 * Release a modify-header resource. 9178 * 9179 * @param dev 9180 * Pointer to Ethernet device. 9181 * @param handle 9182 * Pointer to mlx5_flow_handle. 9183 * 9184 * @return 9185 * 1 while a reference on it exists, 0 when freed. 9186 */ 9187 static int 9188 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev, 9189 struct mlx5_flow_handle *handle) 9190 { 9191 struct mlx5_priv *priv = dev->data->dev_private; 9192 struct mlx5_flow_dv_modify_hdr_resource *cache_resource = 9193 handle->dvh.modify_hdr; 9194 9195 MLX5_ASSERT(cache_resource->action); 9196 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", 9197 (void *)cache_resource, 9198 rte_atomic32_read(&cache_resource->refcnt)); 9199 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9200 claim_zero(mlx5_flow_os_destroy_flow_action 9201 (cache_resource->action)); 9202 mlx5_hlist_remove(priv->sh->modify_cmds, 9203 &cache_resource->entry); 9204 mlx5_free(cache_resource); 9205 DRV_LOG(DEBUG, "modify-header resource %p: removed", 9206 (void *)cache_resource); 9207 return 0; 9208 } 9209 return 1; 9210 } 9211 9212 /** 9213 * Release port ID action resource. 9214 * 9215 * @param dev 9216 * Pointer to Ethernet device. 9217 * @param handle 9218 * Pointer to mlx5_flow_handle. 9219 * 9220 * @return 9221 * 1 while a reference on it exists, 0 when freed. 9222 */ 9223 static int 9224 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, 9225 struct mlx5_flow_handle *handle) 9226 { 9227 struct mlx5_priv *priv = dev->data->dev_private; 9228 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 9229 uint32_t idx = handle->rix_port_id_action; 9230 9231 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 9232 idx); 9233 if (!cache_resource) 9234 return 0; 9235 MLX5_ASSERT(cache_resource->action); 9236 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", 9237 (void *)cache_resource, 9238 rte_atomic32_read(&cache_resource->refcnt)); 9239 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9240 claim_zero(mlx5_flow_os_destroy_flow_action 9241 (cache_resource->action)); 9242 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 9243 &priv->sh->port_id_action_list, idx, 9244 cache_resource, next); 9245 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); 9246 DRV_LOG(DEBUG, "port id action resource %p: removed", 9247 (void *)cache_resource); 9248 return 0; 9249 } 9250 return 1; 9251 } 9252 9253 /** 9254 * Release push vlan action resource. 9255 * 9256 * @param dev 9257 * Pointer to Ethernet device. 9258 * @param handle 9259 * Pointer to mlx5_flow_handle. 9260 * 9261 * @return 9262 * 1 while a reference on it exists, 0 when freed. 9263 */ 9264 static int 9265 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, 9266 struct mlx5_flow_handle *handle) 9267 { 9268 struct mlx5_priv *priv = dev->data->dev_private; 9269 uint32_t idx = handle->dvh.rix_push_vlan; 9270 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 9271 9272 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 9273 idx); 9274 if (!cache_resource) 9275 return 0; 9276 MLX5_ASSERT(cache_resource->action); 9277 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", 9278 (void *)cache_resource, 9279 rte_atomic32_read(&cache_resource->refcnt)); 9280 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 9281 claim_zero(mlx5_flow_os_destroy_flow_action 9282 (cache_resource->action)); 9283 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 9284 &priv->sh->push_vlan_action_list, idx, 9285 cache_resource, next); 9286 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); 9287 DRV_LOG(DEBUG, "push vlan action resource %p: removed", 9288 (void *)cache_resource); 9289 return 0; 9290 } 9291 return 1; 9292 } 9293 9294 /** 9295 * Release the fate resource. 9296 * 9297 * @param dev 9298 * Pointer to Ethernet device. 9299 * @param handle 9300 * Pointer to mlx5_flow_handle. 9301 */ 9302 static void 9303 flow_dv_fate_resource_release(struct rte_eth_dev *dev, 9304 struct mlx5_flow_handle *handle) 9305 { 9306 if (!handle->rix_fate) 9307 return; 9308 switch (handle->fate_action) { 9309 case MLX5_FLOW_FATE_DROP: 9310 mlx5_drop_action_destroy(dev); 9311 break; 9312 case MLX5_FLOW_FATE_QUEUE: 9313 mlx5_hrxq_release(dev, handle->rix_hrxq); 9314 break; 9315 case MLX5_FLOW_FATE_JUMP: 9316 flow_dv_jump_tbl_resource_release(dev, handle); 9317 break; 9318 case MLX5_FLOW_FATE_PORT_ID: 9319 flow_dv_port_id_action_resource_release(dev, handle); 9320 break; 9321 case MLX5_FLOW_FATE_DEFAULT_MISS: 9322 flow_dv_default_miss_resource_release(dev); 9323 break; 9324 default: 9325 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); 9326 break; 9327 } 9328 handle->rix_fate = 0; 9329 } 9330 9331 /** 9332 * Remove the flow from the NIC but keeps it in memory. 9333 * Lock free, (mutex should be acquired by caller). 9334 * 9335 * @param[in] dev 9336 * Pointer to Ethernet device. 9337 * @param[in, out] flow 9338 * Pointer to flow structure. 9339 */ 9340 static void 9341 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 9342 { 9343 struct mlx5_flow_handle *dh; 9344 uint32_t handle_idx; 9345 struct mlx5_priv *priv = dev->data->dev_private; 9346 9347 if (!flow) 9348 return; 9349 handle_idx = flow->dev_handles; 9350 while (handle_idx) { 9351 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9352 handle_idx); 9353 if (!dh) 9354 return; 9355 if (dh->drv_flow) { 9356 claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow)); 9357 dh->drv_flow = NULL; 9358 } 9359 if (dh->fate_action == MLX5_FLOW_FATE_DROP || 9360 dh->fate_action == MLX5_FLOW_FATE_QUEUE || 9361 dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 9362 flow_dv_fate_resource_release(dev, dh); 9363 if (dh->vf_vlan.tag && dh->vf_vlan.created) 9364 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 9365 handle_idx = dh->next.next; 9366 } 9367 } 9368 9369 /** 9370 * Remove the flow from the NIC and the memory. 9371 * Lock free, (mutex should be acquired by caller). 9372 * 9373 * @param[in] dev 9374 * Pointer to the Ethernet device structure. 9375 * @param[in, out] flow 9376 * Pointer to flow structure. 9377 */ 9378 static void 9379 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 9380 { 9381 struct mlx5_flow_handle *dev_handle; 9382 struct mlx5_priv *priv = dev->data->dev_private; 9383 9384 if (!flow) 9385 return; 9386 __flow_dv_remove(dev, flow); 9387 if (flow->counter) { 9388 flow_dv_counter_release(dev, flow->counter); 9389 flow->counter = 0; 9390 } 9391 if (flow->meter) { 9392 struct mlx5_flow_meter *fm; 9393 9394 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], 9395 flow->meter); 9396 if (fm) 9397 mlx5_flow_meter_detach(fm); 9398 flow->meter = 0; 9399 } 9400 while (flow->dev_handles) { 9401 uint32_t tmp_idx = flow->dev_handles; 9402 9403 dev_handle = mlx5_ipool_get(priv->sh->ipool 9404 [MLX5_IPOOL_MLX5_FLOW], tmp_idx); 9405 if (!dev_handle) 9406 return; 9407 flow->dev_handles = dev_handle->next.next; 9408 if (dev_handle->dvh.matcher) 9409 flow_dv_matcher_release(dev, dev_handle); 9410 if (dev_handle->dvh.rix_encap_decap) 9411 flow_dv_encap_decap_resource_release(dev, dev_handle); 9412 if (dev_handle->dvh.modify_hdr) 9413 flow_dv_modify_hdr_resource_release(dev, dev_handle); 9414 if (dev_handle->dvh.rix_push_vlan) 9415 flow_dv_push_vlan_action_resource_release(dev, 9416 dev_handle); 9417 if (dev_handle->dvh.rix_tag) 9418 flow_dv_tag_release(dev, 9419 dev_handle->dvh.rix_tag); 9420 flow_dv_fate_resource_release(dev, dev_handle); 9421 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9422 tmp_idx); 9423 } 9424 } 9425 9426 /** 9427 * Query a dv flow rule for its statistics via devx. 9428 * 9429 * @param[in] dev 9430 * Pointer to Ethernet device. 9431 * @param[in] flow 9432 * Pointer to the sub flow. 9433 * @param[out] data 9434 * data retrieved by the query. 9435 * @param[out] error 9436 * Perform verbose error reporting if not NULL. 9437 * 9438 * @return 9439 * 0 on success, a negative errno value otherwise and rte_errno is set. 9440 */ 9441 static int 9442 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, 9443 void *data, struct rte_flow_error *error) 9444 { 9445 struct mlx5_priv *priv = dev->data->dev_private; 9446 struct rte_flow_query_count *qc = data; 9447 9448 if (!priv->config.devx) 9449 return rte_flow_error_set(error, ENOTSUP, 9450 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9451 NULL, 9452 "counters are not supported"); 9453 if (flow->counter) { 9454 uint64_t pkts, bytes; 9455 struct mlx5_flow_counter *cnt; 9456 9457 cnt = flow_dv_counter_get_by_idx(dev, flow->counter, 9458 NULL); 9459 int err = _flow_dv_query_count(dev, flow->counter, &pkts, 9460 &bytes); 9461 9462 if (err) 9463 return rte_flow_error_set(error, -err, 9464 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9465 NULL, "cannot read counters"); 9466 qc->hits_set = 1; 9467 qc->bytes_set = 1; 9468 qc->hits = pkts - cnt->hits; 9469 qc->bytes = bytes - cnt->bytes; 9470 if (qc->reset) { 9471 cnt->hits = pkts; 9472 cnt->bytes = bytes; 9473 } 9474 return 0; 9475 } 9476 return rte_flow_error_set(error, EINVAL, 9477 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9478 NULL, 9479 "counters are not available"); 9480 } 9481 9482 /** 9483 * Query a flow. 9484 * 9485 * @see rte_flow_query() 9486 * @see rte_flow_ops 9487 */ 9488 static int 9489 flow_dv_query(struct rte_eth_dev *dev, 9490 struct rte_flow *flow __rte_unused, 9491 const struct rte_flow_action *actions __rte_unused, 9492 void *data __rte_unused, 9493 struct rte_flow_error *error __rte_unused) 9494 { 9495 int ret = -EINVAL; 9496 9497 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 9498 switch (actions->type) { 9499 case RTE_FLOW_ACTION_TYPE_VOID: 9500 break; 9501 case RTE_FLOW_ACTION_TYPE_COUNT: 9502 ret = flow_dv_query_count(dev, flow, data, error); 9503 break; 9504 default: 9505 return rte_flow_error_set(error, ENOTSUP, 9506 RTE_FLOW_ERROR_TYPE_ACTION, 9507 actions, 9508 "action not supported"); 9509 } 9510 } 9511 return ret; 9512 } 9513 9514 /** 9515 * Destroy the meter table set. 9516 * Lock free, (mutex should be acquired by caller). 9517 * 9518 * @param[in] dev 9519 * Pointer to Ethernet device. 9520 * @param[in] tbl 9521 * Pointer to the meter table set. 9522 * 9523 * @return 9524 * Always 0. 9525 */ 9526 static int 9527 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, 9528 struct mlx5_meter_domains_infos *tbl) 9529 { 9530 struct mlx5_priv *priv = dev->data->dev_private; 9531 struct mlx5_meter_domains_infos *mtd = 9532 (struct mlx5_meter_domains_infos *)tbl; 9533 9534 if (!mtd || !priv->config.dv_flow_en) 9535 return 0; 9536 if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) 9537 claim_zero(mlx5_flow_os_destroy_flow 9538 (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); 9539 if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) 9540 claim_zero(mlx5_flow_os_destroy_flow 9541 (mtd->egress.policer_rules[RTE_MTR_DROPPED])); 9542 if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) 9543 claim_zero(mlx5_flow_os_destroy_flow 9544 (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); 9545 if (mtd->egress.color_matcher) 9546 claim_zero(mlx5_flow_os_destroy_flow_matcher 9547 (mtd->egress.color_matcher)); 9548 if (mtd->egress.any_matcher) 9549 claim_zero(mlx5_flow_os_destroy_flow_matcher 9550 (mtd->egress.any_matcher)); 9551 if (mtd->egress.tbl) 9552 flow_dv_tbl_resource_release(dev, mtd->egress.tbl); 9553 if (mtd->egress.sfx_tbl) 9554 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); 9555 if (mtd->ingress.color_matcher) 9556 claim_zero(mlx5_flow_os_destroy_flow_matcher 9557 (mtd->ingress.color_matcher)); 9558 if (mtd->ingress.any_matcher) 9559 claim_zero(mlx5_flow_os_destroy_flow_matcher 9560 (mtd->ingress.any_matcher)); 9561 if (mtd->ingress.tbl) 9562 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); 9563 if (mtd->ingress.sfx_tbl) 9564 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); 9565 if (mtd->transfer.color_matcher) 9566 claim_zero(mlx5_flow_os_destroy_flow_matcher 9567 (mtd->transfer.color_matcher)); 9568 if (mtd->transfer.any_matcher) 9569 claim_zero(mlx5_flow_os_destroy_flow_matcher 9570 (mtd->transfer.any_matcher)); 9571 if (mtd->transfer.tbl) 9572 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); 9573 if (mtd->transfer.sfx_tbl) 9574 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); 9575 if (mtd->drop_actn) 9576 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); 9577 mlx5_free(mtd); 9578 return 0; 9579 } 9580 9581 /* Number of meter flow actions, count and jump or count and drop. */ 9582 #define METER_ACTIONS 2 9583 9584 /** 9585 * Create specify domain meter table and suffix table. 9586 * 9587 * @param[in] dev 9588 * Pointer to Ethernet device. 9589 * @param[in,out] mtb 9590 * Pointer to DV meter table set. 9591 * @param[in] egress 9592 * Table attribute. 9593 * @param[in] transfer 9594 * Table attribute. 9595 * @param[in] color_reg_c_idx 9596 * Reg C index for color match. 9597 * 9598 * @return 9599 * 0 on success, -1 otherwise and rte_errno is set. 9600 */ 9601 static int 9602 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, 9603 struct mlx5_meter_domains_infos *mtb, 9604 uint8_t egress, uint8_t transfer, 9605 uint32_t color_reg_c_idx) 9606 { 9607 struct mlx5_priv *priv = dev->data->dev_private; 9608 struct mlx5_dev_ctx_shared *sh = priv->sh; 9609 struct mlx5_flow_dv_match_params mask = { 9610 .size = sizeof(mask.buf), 9611 }; 9612 struct mlx5_flow_dv_match_params value = { 9613 .size = sizeof(value.buf), 9614 }; 9615 struct mlx5dv_flow_matcher_attr dv_attr = { 9616 .type = IBV_FLOW_ATTR_NORMAL, 9617 .priority = 0, 9618 .match_criteria_enable = 0, 9619 .match_mask = (void *)&mask, 9620 }; 9621 void *actions[METER_ACTIONS]; 9622 struct mlx5_meter_domain_info *dtb; 9623 struct rte_flow_error error; 9624 int i = 0; 9625 int ret; 9626 9627 if (transfer) 9628 dtb = &mtb->transfer; 9629 else if (egress) 9630 dtb = &mtb->egress; 9631 else 9632 dtb = &mtb->ingress; 9633 /* Create the meter table with METER level. */ 9634 dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, 9635 egress, transfer, &error); 9636 if (!dtb->tbl) { 9637 DRV_LOG(ERR, "Failed to create meter policer table."); 9638 return -1; 9639 } 9640 /* Create the meter suffix table with SUFFIX level. */ 9641 dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, 9642 MLX5_FLOW_TABLE_LEVEL_SUFFIX, 9643 egress, transfer, &error); 9644 if (!dtb->sfx_tbl) { 9645 DRV_LOG(ERR, "Failed to create meter suffix table."); 9646 return -1; 9647 } 9648 /* Create matchers, Any and Color. */ 9649 dv_attr.priority = 3; 9650 dv_attr.match_criteria_enable = 0; 9651 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, 9652 &dtb->any_matcher); 9653 if (ret) { 9654 DRV_LOG(ERR, "Failed to create meter" 9655 " policer default matcher."); 9656 goto error_exit; 9657 } 9658 dv_attr.priority = 0; 9659 dv_attr.match_criteria_enable = 9660 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 9661 flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, 9662 rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); 9663 ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, 9664 &dtb->color_matcher); 9665 if (ret) { 9666 DRV_LOG(ERR, "Failed to create meter policer color matcher."); 9667 goto error_exit; 9668 } 9669 if (mtb->count_actns[RTE_MTR_DROPPED]) 9670 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; 9671 actions[i++] = mtb->drop_actn; 9672 /* Default rule: lowest priority, match any, actions: drop. */ 9673 ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, 9674 actions, 9675 &dtb->policer_rules[RTE_MTR_DROPPED]); 9676 if (ret) { 9677 DRV_LOG(ERR, "Failed to create meter policer drop rule."); 9678 goto error_exit; 9679 } 9680 return 0; 9681 error_exit: 9682 return -1; 9683 } 9684 9685 /** 9686 * Create the needed meter and suffix tables. 9687 * Lock free, (mutex should be acquired by caller). 9688 * 9689 * @param[in] dev 9690 * Pointer to Ethernet device. 9691 * @param[in] fm 9692 * Pointer to the flow meter. 9693 * 9694 * @return 9695 * Pointer to table set on success, NULL otherwise and rte_errno is set. 9696 */ 9697 static struct mlx5_meter_domains_infos * 9698 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, 9699 const struct mlx5_flow_meter *fm) 9700 { 9701 struct mlx5_priv *priv = dev->data->dev_private; 9702 struct mlx5_meter_domains_infos *mtb; 9703 int ret; 9704 int i; 9705 9706 if (!priv->mtr_en) { 9707 rte_errno = ENOTSUP; 9708 return NULL; 9709 } 9710 mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY); 9711 if (!mtb) { 9712 DRV_LOG(ERR, "Failed to allocate memory for meter."); 9713 return NULL; 9714 } 9715 /* Create meter count actions */ 9716 for (i = 0; i <= RTE_MTR_DROPPED; i++) { 9717 struct mlx5_flow_counter *cnt; 9718 if (!fm->policer_stats.cnt[i]) 9719 continue; 9720 cnt = flow_dv_counter_get_by_idx(dev, 9721 fm->policer_stats.cnt[i], NULL); 9722 mtb->count_actns[i] = cnt->action; 9723 } 9724 /* Create drop action. */ 9725 ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); 9726 if (ret) { 9727 DRV_LOG(ERR, "Failed to create drop action."); 9728 goto error_exit; 9729 } 9730 /* Egress meter table. */ 9731 ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); 9732 if (ret) { 9733 DRV_LOG(ERR, "Failed to prepare egress meter table."); 9734 goto error_exit; 9735 } 9736 /* Ingress meter table. */ 9737 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); 9738 if (ret) { 9739 DRV_LOG(ERR, "Failed to prepare ingress meter table."); 9740 goto error_exit; 9741 } 9742 /* FDB meter table. */ 9743 if (priv->config.dv_esw_en) { 9744 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, 9745 priv->mtr_color_reg); 9746 if (ret) { 9747 DRV_LOG(ERR, "Failed to prepare fdb meter table."); 9748 goto error_exit; 9749 } 9750 } 9751 return mtb; 9752 error_exit: 9753 flow_dv_destroy_mtr_tbl(dev, mtb); 9754 return NULL; 9755 } 9756 9757 /** 9758 * Destroy domain policer rule. 9759 * 9760 * @param[in] dt 9761 * Pointer to domain table. 9762 */ 9763 static void 9764 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) 9765 { 9766 int i; 9767 9768 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9769 if (dt->policer_rules[i]) { 9770 claim_zero(mlx5_flow_os_destroy_flow 9771 (dt->policer_rules[i])); 9772 dt->policer_rules[i] = NULL; 9773 } 9774 } 9775 if (dt->jump_actn) { 9776 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); 9777 dt->jump_actn = NULL; 9778 } 9779 } 9780 9781 /** 9782 * Destroy policer rules. 9783 * 9784 * @param[in] dev 9785 * Pointer to Ethernet device. 9786 * @param[in] fm 9787 * Pointer to flow meter structure. 9788 * @param[in] attr 9789 * Pointer to flow attributes. 9790 * 9791 * @return 9792 * Always 0. 9793 */ 9794 static int 9795 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, 9796 const struct mlx5_flow_meter *fm, 9797 const struct rte_flow_attr *attr) 9798 { 9799 struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; 9800 9801 if (!mtb) 9802 return 0; 9803 if (attr->egress) 9804 flow_dv_destroy_domain_policer_rule(&mtb->egress); 9805 if (attr->ingress) 9806 flow_dv_destroy_domain_policer_rule(&mtb->ingress); 9807 if (attr->transfer) 9808 flow_dv_destroy_domain_policer_rule(&mtb->transfer); 9809 return 0; 9810 } 9811 9812 /** 9813 * Create specify domain meter policer rule. 9814 * 9815 * @param[in] fm 9816 * Pointer to flow meter structure. 9817 * @param[in] mtb 9818 * Pointer to DV meter table set. 9819 * @param[in] mtr_reg_c 9820 * Color match REG_C. 9821 * 9822 * @return 9823 * 0 on success, -1 otherwise. 9824 */ 9825 static int 9826 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, 9827 struct mlx5_meter_domain_info *dtb, 9828 uint8_t mtr_reg_c) 9829 { 9830 struct mlx5_flow_dv_match_params matcher = { 9831 .size = sizeof(matcher.buf), 9832 }; 9833 struct mlx5_flow_dv_match_params value = { 9834 .size = sizeof(value.buf), 9835 }; 9836 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9837 void *actions[METER_ACTIONS]; 9838 int i; 9839 int ret = 0; 9840 9841 /* Create jump action. */ 9842 if (!dtb->jump_actn) 9843 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl 9844 (dtb->sfx_tbl->obj, &dtb->jump_actn); 9845 if (ret) { 9846 DRV_LOG(ERR, "Failed to create policer jump action."); 9847 goto error; 9848 } 9849 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9850 int j = 0; 9851 9852 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, 9853 rte_col_2_mlx5_col(i), UINT8_MAX); 9854 if (mtb->count_actns[i]) 9855 actions[j++] = mtb->count_actns[i]; 9856 if (fm->action[i] == MTR_POLICER_ACTION_DROP) 9857 actions[j++] = mtb->drop_actn; 9858 else 9859 actions[j++] = dtb->jump_actn; 9860 ret = mlx5_flow_os_create_flow(dtb->color_matcher, 9861 (void *)&value, j, actions, 9862 &dtb->policer_rules[i]); 9863 if (ret) { 9864 DRV_LOG(ERR, "Failed to create policer rule."); 9865 goto error; 9866 } 9867 } 9868 return 0; 9869 error: 9870 rte_errno = errno; 9871 return -1; 9872 } 9873 9874 /** 9875 * Create policer rules. 9876 * 9877 * @param[in] dev 9878 * Pointer to Ethernet device. 9879 * @param[in] fm 9880 * Pointer to flow meter structure. 9881 * @param[in] attr 9882 * Pointer to flow attributes. 9883 * 9884 * @return 9885 * 0 on success, -1 otherwise. 9886 */ 9887 static int 9888 flow_dv_create_policer_rules(struct rte_eth_dev *dev, 9889 struct mlx5_flow_meter *fm, 9890 const struct rte_flow_attr *attr) 9891 { 9892 struct mlx5_priv *priv = dev->data->dev_private; 9893 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9894 int ret; 9895 9896 if (attr->egress) { 9897 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, 9898 priv->mtr_color_reg); 9899 if (ret) { 9900 DRV_LOG(ERR, "Failed to create egress policer."); 9901 goto error; 9902 } 9903 } 9904 if (attr->ingress) { 9905 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, 9906 priv->mtr_color_reg); 9907 if (ret) { 9908 DRV_LOG(ERR, "Failed to create ingress policer."); 9909 goto error; 9910 } 9911 } 9912 if (attr->transfer) { 9913 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, 9914 priv->mtr_color_reg); 9915 if (ret) { 9916 DRV_LOG(ERR, "Failed to create transfer policer."); 9917 goto error; 9918 } 9919 } 9920 return 0; 9921 error: 9922 flow_dv_destroy_policer_rules(dev, fm, attr); 9923 return -1; 9924 } 9925 9926 /** 9927 * Query a devx counter. 9928 * 9929 * @param[in] dev 9930 * Pointer to the Ethernet device structure. 9931 * @param[in] cnt 9932 * Index to the flow counter. 9933 * @param[in] clear 9934 * Set to clear the counter statistics. 9935 * @param[out] pkts 9936 * The statistics value of packets. 9937 * @param[out] bytes 9938 * The statistics value of bytes. 9939 * 9940 * @return 9941 * 0 on success, otherwise return -1. 9942 */ 9943 static int 9944 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, 9945 uint64_t *pkts, uint64_t *bytes) 9946 { 9947 struct mlx5_priv *priv = dev->data->dev_private; 9948 struct mlx5_flow_counter *cnt; 9949 uint64_t inn_pkts, inn_bytes; 9950 int ret; 9951 9952 if (!priv->config.devx) 9953 return -1; 9954 9955 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes); 9956 if (ret) 9957 return -1; 9958 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL); 9959 *pkts = inn_pkts - cnt->hits; 9960 *bytes = inn_bytes - cnt->bytes; 9961 if (clear) { 9962 cnt->hits = inn_pkts; 9963 cnt->bytes = inn_bytes; 9964 } 9965 return 0; 9966 } 9967 9968 /** 9969 * Get aged-out flows. 9970 * 9971 * @param[in] dev 9972 * Pointer to the Ethernet device structure. 9973 * @param[in] context 9974 * The address of an array of pointers to the aged-out flows contexts. 9975 * @param[in] nb_contexts 9976 * The length of context array pointers. 9977 * @param[out] error 9978 * Perform verbose error reporting if not NULL. Initialized in case of 9979 * error only. 9980 * 9981 * @return 9982 * how many contexts get in success, otherwise negative errno value. 9983 * if nb_contexts is 0, return the amount of all aged contexts. 9984 * if nb_contexts is not 0 , return the amount of aged flows reported 9985 * in the context array. 9986 * @note: only stub for now 9987 */ 9988 static int 9989 flow_get_aged_flows(struct rte_eth_dev *dev, 9990 void **context, 9991 uint32_t nb_contexts, 9992 struct rte_flow_error *error) 9993 { 9994 struct mlx5_priv *priv = dev->data->dev_private; 9995 struct mlx5_age_info *age_info; 9996 struct mlx5_age_param *age_param; 9997 struct mlx5_flow_counter *counter; 9998 int nb_flows = 0; 9999 10000 if (nb_contexts && !context) 10001 return rte_flow_error_set(error, EINVAL, 10002 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10003 NULL, 10004 "Should assign at least one flow or" 10005 " context to get if nb_contexts != 0"); 10006 age_info = GET_PORT_AGE_INFO(priv); 10007 rte_spinlock_lock(&age_info->aged_sl); 10008 TAILQ_FOREACH(counter, &age_info->aged_counters, next) { 10009 nb_flows++; 10010 if (nb_contexts) { 10011 age_param = MLX5_CNT_TO_AGE(counter); 10012 context[nb_flows - 1] = age_param->context; 10013 if (!(--nb_contexts)) 10014 break; 10015 } 10016 } 10017 rte_spinlock_unlock(&age_info->aged_sl); 10018 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 10019 return nb_flows; 10020 } 10021 10022 /* 10023 * Mutex-protected thunk to lock-free __flow_dv_translate(). 10024 */ 10025 static int 10026 flow_dv_translate(struct rte_eth_dev *dev, 10027 struct mlx5_flow *dev_flow, 10028 const struct rte_flow_attr *attr, 10029 const struct rte_flow_item items[], 10030 const struct rte_flow_action actions[], 10031 struct rte_flow_error *error) 10032 { 10033 int ret; 10034 10035 flow_dv_shared_lock(dev); 10036 ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); 10037 flow_dv_shared_unlock(dev); 10038 return ret; 10039 } 10040 10041 /* 10042 * Mutex-protected thunk to lock-free __flow_dv_apply(). 10043 */ 10044 static int 10045 flow_dv_apply(struct rte_eth_dev *dev, 10046 struct rte_flow *flow, 10047 struct rte_flow_error *error) 10048 { 10049 int ret; 10050 10051 flow_dv_shared_lock(dev); 10052 ret = __flow_dv_apply(dev, flow, error); 10053 flow_dv_shared_unlock(dev); 10054 return ret; 10055 } 10056 10057 /* 10058 * Mutex-protected thunk to lock-free __flow_dv_remove(). 10059 */ 10060 static void 10061 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 10062 { 10063 flow_dv_shared_lock(dev); 10064 __flow_dv_remove(dev, flow); 10065 flow_dv_shared_unlock(dev); 10066 } 10067 10068 /* 10069 * Mutex-protected thunk to lock-free __flow_dv_destroy(). 10070 */ 10071 static void 10072 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 10073 { 10074 flow_dv_shared_lock(dev); 10075 __flow_dv_destroy(dev, flow); 10076 flow_dv_shared_unlock(dev); 10077 } 10078 10079 /* 10080 * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). 10081 */ 10082 static uint32_t 10083 flow_dv_counter_allocate(struct rte_eth_dev *dev) 10084 { 10085 uint32_t cnt; 10086 10087 flow_dv_shared_lock(dev); 10088 cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0); 10089 flow_dv_shared_unlock(dev); 10090 return cnt; 10091 } 10092 10093 /* 10094 * Mutex-protected thunk to lock-free flow_dv_counter_release(). 10095 */ 10096 static void 10097 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 10098 { 10099 flow_dv_shared_lock(dev); 10100 flow_dv_counter_release(dev, cnt); 10101 flow_dv_shared_unlock(dev); 10102 } 10103 10104 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 10105 .validate = flow_dv_validate, 10106 .prepare = flow_dv_prepare, 10107 .translate = flow_dv_translate, 10108 .apply = flow_dv_apply, 10109 .remove = flow_dv_remove, 10110 .destroy = flow_dv_destroy, 10111 .query = flow_dv_query, 10112 .create_mtr_tbls = flow_dv_create_mtr_tbl, 10113 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, 10114 .create_policer_rules = flow_dv_create_policer_rules, 10115 .destroy_policer_rules = flow_dv_destroy_policer_rules, 10116 .counter_alloc = flow_dv_counter_allocate, 10117 .counter_free = flow_dv_counter_free, 10118 .counter_query = flow_dv_counter_query, 10119 .get_aged_flows = flow_get_aged_flows, 10120 }; 10121 10122 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10123 10124