1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <unistd.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_common.h> 22 #include <rte_ether.h> 23 #include <rte_ethdev_driver.h> 24 #include <rte_flow.h> 25 #include <rte_flow_driver.h> 26 #include <rte_malloc.h> 27 #include <rte_cycles.h> 28 #include <rte_ip.h> 29 #include <rte_gre.h> 30 #include <rte_vxlan.h> 31 #include <rte_gtp.h> 32 33 #include <mlx5_glue.h> 34 #include <mlx5_devx_cmds.h> 35 #include <mlx5_prm.h> 36 37 #include "mlx5_defs.h" 38 #include "mlx5.h" 39 #include "mlx5_common_os.h" 40 #include "mlx5_flow.h" 41 #include "mlx5_rxtx.h" 42 43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 44 45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS 46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 47 #endif 48 49 #ifndef HAVE_MLX5DV_DR_ESWITCH 50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB 51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0 52 #endif 53 #endif 54 55 #ifndef HAVE_MLX5DV_DR 56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 57 #endif 58 59 /* VLAN header definitions */ 60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13 61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT) 62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff 63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK) 64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK) 65 66 union flow_dv_attr { 67 struct { 68 uint32_t valid:1; 69 uint32_t ipv4:1; 70 uint32_t ipv6:1; 71 uint32_t tcp:1; 72 uint32_t udp:1; 73 uint32_t reserved:27; 74 }; 75 uint32_t attr; 76 }; 77 78 static int 79 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 80 struct mlx5_flow_tbl_resource *tbl); 81 82 static int 83 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev); 84 85 /** 86 * Initialize flow attributes structure according to flow items' types. 87 * 88 * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel 89 * mode. For tunnel mode, the items to be modified are the outermost ones. 90 * 91 * @param[in] item 92 * Pointer to item specification. 93 * @param[out] attr 94 * Pointer to flow attributes structure. 95 * @param[in] dev_flow 96 * Pointer to the sub flow. 97 * @param[in] tunnel_decap 98 * Whether action is after tunnel decapsulation. 99 */ 100 static void 101 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr, 102 struct mlx5_flow *dev_flow, bool tunnel_decap) 103 { 104 uint64_t layers = dev_flow->handle->layers; 105 106 /* 107 * If layers is already initialized, it means this dev_flow is the 108 * suffix flow, the layers flags is set by the prefix flow. Need to 109 * use the layer flags from prefix flow as the suffix flow may not 110 * have the user defined items as the flow is split. 111 */ 112 if (layers) { 113 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4) 114 attr->ipv4 = 1; 115 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6) 116 attr->ipv6 = 1; 117 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP) 118 attr->tcp = 1; 119 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP) 120 attr->udp = 1; 121 attr->valid = 1; 122 return; 123 } 124 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 125 uint8_t next_protocol = 0xff; 126 switch (item->type) { 127 case RTE_FLOW_ITEM_TYPE_GRE: 128 case RTE_FLOW_ITEM_TYPE_NVGRE: 129 case RTE_FLOW_ITEM_TYPE_VXLAN: 130 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 131 case RTE_FLOW_ITEM_TYPE_GENEVE: 132 case RTE_FLOW_ITEM_TYPE_MPLS: 133 if (tunnel_decap) 134 attr->attr = 0; 135 break; 136 case RTE_FLOW_ITEM_TYPE_IPV4: 137 if (!attr->ipv6) 138 attr->ipv4 = 1; 139 if (item->mask != NULL && 140 ((const struct rte_flow_item_ipv4 *) 141 item->mask)->hdr.next_proto_id) 142 next_protocol = 143 ((const struct rte_flow_item_ipv4 *) 144 (item->spec))->hdr.next_proto_id & 145 ((const struct rte_flow_item_ipv4 *) 146 (item->mask))->hdr.next_proto_id; 147 if ((next_protocol == IPPROTO_IPIP || 148 next_protocol == IPPROTO_IPV6) && tunnel_decap) 149 attr->attr = 0; 150 break; 151 case RTE_FLOW_ITEM_TYPE_IPV6: 152 if (!attr->ipv4) 153 attr->ipv6 = 1; 154 if (item->mask != NULL && 155 ((const struct rte_flow_item_ipv6 *) 156 item->mask)->hdr.proto) 157 next_protocol = 158 ((const struct rte_flow_item_ipv6 *) 159 (item->spec))->hdr.proto & 160 ((const struct rte_flow_item_ipv6 *) 161 (item->mask))->hdr.proto; 162 if ((next_protocol == IPPROTO_IPIP || 163 next_protocol == IPPROTO_IPV6) && tunnel_decap) 164 attr->attr = 0; 165 break; 166 case RTE_FLOW_ITEM_TYPE_UDP: 167 if (!attr->tcp) 168 attr->udp = 1; 169 break; 170 case RTE_FLOW_ITEM_TYPE_TCP: 171 if (!attr->udp) 172 attr->tcp = 1; 173 break; 174 default: 175 break; 176 } 177 } 178 attr->valid = 1; 179 } 180 181 /** 182 * Convert rte_mtr_color to mlx5 color. 183 * 184 * @param[in] rcol 185 * rte_mtr_color. 186 * 187 * @return 188 * mlx5 color. 189 */ 190 static int 191 rte_col_2_mlx5_col(enum rte_color rcol) 192 { 193 switch (rcol) { 194 case RTE_COLOR_GREEN: 195 return MLX5_FLOW_COLOR_GREEN; 196 case RTE_COLOR_YELLOW: 197 return MLX5_FLOW_COLOR_YELLOW; 198 case RTE_COLOR_RED: 199 return MLX5_FLOW_COLOR_RED; 200 default: 201 break; 202 } 203 return MLX5_FLOW_COLOR_UNDEFINED; 204 } 205 206 struct field_modify_info { 207 uint32_t size; /* Size of field in protocol header, in bytes. */ 208 uint32_t offset; /* Offset of field in protocol header, in bytes. */ 209 enum mlx5_modification_field id; 210 }; 211 212 struct field_modify_info modify_eth[] = { 213 {4, 0, MLX5_MODI_OUT_DMAC_47_16}, 214 {2, 4, MLX5_MODI_OUT_DMAC_15_0}, 215 {4, 6, MLX5_MODI_OUT_SMAC_47_16}, 216 {2, 10, MLX5_MODI_OUT_SMAC_15_0}, 217 {0, 0, 0}, 218 }; 219 220 struct field_modify_info modify_vlan_out_first_vid[] = { 221 /* Size in bits !!! */ 222 {12, 0, MLX5_MODI_OUT_FIRST_VID}, 223 {0, 0, 0}, 224 }; 225 226 struct field_modify_info modify_ipv4[] = { 227 {1, 1, MLX5_MODI_OUT_IP_DSCP}, 228 {1, 8, MLX5_MODI_OUT_IPV4_TTL}, 229 {4, 12, MLX5_MODI_OUT_SIPV4}, 230 {4, 16, MLX5_MODI_OUT_DIPV4}, 231 {0, 0, 0}, 232 }; 233 234 struct field_modify_info modify_ipv6[] = { 235 {1, 0, MLX5_MODI_OUT_IP_DSCP}, 236 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, 237 {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, 238 {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, 239 {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, 240 {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, 241 {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, 242 {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, 243 {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, 244 {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, 245 {0, 0, 0}, 246 }; 247 248 struct field_modify_info modify_udp[] = { 249 {2, 0, MLX5_MODI_OUT_UDP_SPORT}, 250 {2, 2, MLX5_MODI_OUT_UDP_DPORT}, 251 {0, 0, 0}, 252 }; 253 254 struct field_modify_info modify_tcp[] = { 255 {2, 0, MLX5_MODI_OUT_TCP_SPORT}, 256 {2, 2, MLX5_MODI_OUT_TCP_DPORT}, 257 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, 258 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, 259 {0, 0, 0}, 260 }; 261 262 static void 263 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused, 264 uint8_t next_protocol, uint64_t *item_flags, 265 int *tunnel) 266 { 267 MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 || 268 item->type == RTE_FLOW_ITEM_TYPE_IPV6); 269 if (next_protocol == IPPROTO_IPIP) { 270 *item_flags |= MLX5_FLOW_LAYER_IPIP; 271 *tunnel = 1; 272 } 273 if (next_protocol == IPPROTO_IPV6) { 274 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; 275 *tunnel = 1; 276 } 277 } 278 279 /** 280 * Acquire the synchronizing object to protect multithreaded access 281 * to shared dv context. Lock occurs only if context is actually 282 * shared, i.e. we have multiport IB device and representors are 283 * created. 284 * 285 * @param[in] dev 286 * Pointer to the rte_eth_dev structure. 287 */ 288 static void 289 flow_dv_shared_lock(struct rte_eth_dev *dev) 290 { 291 struct mlx5_priv *priv = dev->data->dev_private; 292 struct mlx5_dev_ctx_shared *sh = priv->sh; 293 294 if (sh->dv_refcnt > 1) { 295 int ret; 296 297 ret = pthread_mutex_lock(&sh->dv_mutex); 298 MLX5_ASSERT(!ret); 299 (void)ret; 300 } 301 } 302 303 static void 304 flow_dv_shared_unlock(struct rte_eth_dev *dev) 305 { 306 struct mlx5_priv *priv = dev->data->dev_private; 307 struct mlx5_dev_ctx_shared *sh = priv->sh; 308 309 if (sh->dv_refcnt > 1) { 310 int ret; 311 312 ret = pthread_mutex_unlock(&sh->dv_mutex); 313 MLX5_ASSERT(!ret); 314 (void)ret; 315 } 316 } 317 318 /* Update VLAN's VID/PCP based on input rte_flow_action. 319 * 320 * @param[in] action 321 * Pointer to struct rte_flow_action. 322 * @param[out] vlan 323 * Pointer to struct rte_vlan_hdr. 324 */ 325 static void 326 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action, 327 struct rte_vlan_hdr *vlan) 328 { 329 uint16_t vlan_tci; 330 if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) { 331 vlan_tci = 332 ((const struct rte_flow_action_of_set_vlan_pcp *) 333 action->conf)->vlan_pcp; 334 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT; 335 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 336 vlan->vlan_tci |= vlan_tci; 337 } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) { 338 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 339 vlan->vlan_tci |= rte_be_to_cpu_16 340 (((const struct rte_flow_action_of_set_vlan_vid *) 341 action->conf)->vlan_vid); 342 } 343 } 344 345 /** 346 * Fetch 1, 2, 3 or 4 byte field from the byte array 347 * and return as unsigned integer in host-endian format. 348 * 349 * @param[in] data 350 * Pointer to data array. 351 * @param[in] size 352 * Size of field to extract. 353 * 354 * @return 355 * converted field in host endian format. 356 */ 357 static inline uint32_t 358 flow_dv_fetch_field(const uint8_t *data, uint32_t size) 359 { 360 uint32_t ret; 361 362 switch (size) { 363 case 1: 364 ret = *data; 365 break; 366 case 2: 367 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 368 break; 369 case 3: 370 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data); 371 ret = (ret << 8) | *(data + sizeof(uint16_t)); 372 break; 373 case 4: 374 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data); 375 break; 376 default: 377 MLX5_ASSERT(false); 378 ret = 0; 379 break; 380 } 381 return ret; 382 } 383 384 /** 385 * Convert modify-header action to DV specification. 386 * 387 * Data length of each action is determined by provided field description 388 * and the item mask. Data bit offset and width of each action is determined 389 * by provided item mask. 390 * 391 * @param[in] item 392 * Pointer to item specification. 393 * @param[in] field 394 * Pointer to field modification information. 395 * For MLX5_MODIFICATION_TYPE_SET specifies destination field. 396 * For MLX5_MODIFICATION_TYPE_ADD specifies destination field. 397 * For MLX5_MODIFICATION_TYPE_COPY specifies source field. 398 * @param[in] dcopy 399 * Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type. 400 * Negative offset value sets the same offset as source offset. 401 * size field is ignored, value is taken from source field. 402 * @param[in,out] resource 403 * Pointer to the modify-header resource. 404 * @param[in] type 405 * Type of modification. 406 * @param[out] error 407 * Pointer to the error structure. 408 * 409 * @return 410 * 0 on success, a negative errno value otherwise and rte_errno is set. 411 */ 412 static int 413 flow_dv_convert_modify_action(struct rte_flow_item *item, 414 struct field_modify_info *field, 415 struct field_modify_info *dcopy, 416 struct mlx5_flow_dv_modify_hdr_resource *resource, 417 uint32_t type, struct rte_flow_error *error) 418 { 419 uint32_t i = resource->actions_num; 420 struct mlx5_modification_cmd *actions = resource->actions; 421 422 /* 423 * The item and mask are provided in big-endian format. 424 * The fields should be presented as in big-endian format either. 425 * Mask must be always present, it defines the actual field width. 426 */ 427 MLX5_ASSERT(item->mask); 428 MLX5_ASSERT(field->size); 429 do { 430 unsigned int size_b; 431 unsigned int off_b; 432 uint32_t mask; 433 uint32_t data; 434 435 if (i >= MLX5_MAX_MODIFY_NUM) 436 return rte_flow_error_set(error, EINVAL, 437 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 438 "too many items to modify"); 439 /* Fetch variable byte size mask from the array. */ 440 mask = flow_dv_fetch_field((const uint8_t *)item->mask + 441 field->offset, field->size); 442 if (!mask) { 443 ++field; 444 continue; 445 } 446 /* Deduce actual data width in bits from mask value. */ 447 off_b = rte_bsf32(mask); 448 size_b = sizeof(uint32_t) * CHAR_BIT - 449 off_b - __builtin_clz(mask); 450 MLX5_ASSERT(size_b); 451 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b; 452 actions[i] = (struct mlx5_modification_cmd) { 453 .action_type = type, 454 .field = field->id, 455 .offset = off_b, 456 .length = size_b, 457 }; 458 /* Convert entire record to expected big-endian format. */ 459 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 460 if (type == MLX5_MODIFICATION_TYPE_COPY) { 461 MLX5_ASSERT(dcopy); 462 actions[i].dst_field = dcopy->id; 463 actions[i].dst_offset = 464 (int)dcopy->offset < 0 ? off_b : dcopy->offset; 465 /* Convert entire record to big-endian format. */ 466 actions[i].data1 = rte_cpu_to_be_32(actions[i].data1); 467 } else { 468 MLX5_ASSERT(item->spec); 469 data = flow_dv_fetch_field((const uint8_t *)item->spec + 470 field->offset, field->size); 471 /* Shift out the trailing masked bits from data. */ 472 data = (data & mask) >> off_b; 473 actions[i].data1 = rte_cpu_to_be_32(data); 474 } 475 ++i; 476 ++field; 477 } while (field->size); 478 if (resource->actions_num == i) 479 return rte_flow_error_set(error, EINVAL, 480 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 481 "invalid modification flow item"); 482 resource->actions_num = i; 483 return 0; 484 } 485 486 /** 487 * Convert modify-header set IPv4 address action to DV specification. 488 * 489 * @param[in,out] resource 490 * Pointer to the modify-header resource. 491 * @param[in] action 492 * Pointer to action specification. 493 * @param[out] error 494 * Pointer to the error structure. 495 * 496 * @return 497 * 0 on success, a negative errno value otherwise and rte_errno is set. 498 */ 499 static int 500 flow_dv_convert_action_modify_ipv4 501 (struct mlx5_flow_dv_modify_hdr_resource *resource, 502 const struct rte_flow_action *action, 503 struct rte_flow_error *error) 504 { 505 const struct rte_flow_action_set_ipv4 *conf = 506 (const struct rte_flow_action_set_ipv4 *)(action->conf); 507 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 508 struct rte_flow_item_ipv4 ipv4; 509 struct rte_flow_item_ipv4 ipv4_mask; 510 511 memset(&ipv4, 0, sizeof(ipv4)); 512 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 513 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { 514 ipv4.hdr.src_addr = conf->ipv4_addr; 515 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; 516 } else { 517 ipv4.hdr.dst_addr = conf->ipv4_addr; 518 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; 519 } 520 item.spec = &ipv4; 521 item.mask = &ipv4_mask; 522 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 523 MLX5_MODIFICATION_TYPE_SET, error); 524 } 525 526 /** 527 * Convert modify-header set IPv6 address action to DV specification. 528 * 529 * @param[in,out] resource 530 * Pointer to the modify-header resource. 531 * @param[in] action 532 * Pointer to action specification. 533 * @param[out] error 534 * Pointer to the error structure. 535 * 536 * @return 537 * 0 on success, a negative errno value otherwise and rte_errno is set. 538 */ 539 static int 540 flow_dv_convert_action_modify_ipv6 541 (struct mlx5_flow_dv_modify_hdr_resource *resource, 542 const struct rte_flow_action *action, 543 struct rte_flow_error *error) 544 { 545 const struct rte_flow_action_set_ipv6 *conf = 546 (const struct rte_flow_action_set_ipv6 *)(action->conf); 547 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 548 struct rte_flow_item_ipv6 ipv6; 549 struct rte_flow_item_ipv6 ipv6_mask; 550 551 memset(&ipv6, 0, sizeof(ipv6)); 552 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 553 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { 554 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, 555 sizeof(ipv6.hdr.src_addr)); 556 memcpy(&ipv6_mask.hdr.src_addr, 557 &rte_flow_item_ipv6_mask.hdr.src_addr, 558 sizeof(ipv6.hdr.src_addr)); 559 } else { 560 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, 561 sizeof(ipv6.hdr.dst_addr)); 562 memcpy(&ipv6_mask.hdr.dst_addr, 563 &rte_flow_item_ipv6_mask.hdr.dst_addr, 564 sizeof(ipv6.hdr.dst_addr)); 565 } 566 item.spec = &ipv6; 567 item.mask = &ipv6_mask; 568 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 569 MLX5_MODIFICATION_TYPE_SET, error); 570 } 571 572 /** 573 * Convert modify-header set MAC address action to DV specification. 574 * 575 * @param[in,out] resource 576 * Pointer to the modify-header resource. 577 * @param[in] action 578 * Pointer to action specification. 579 * @param[out] error 580 * Pointer to the error structure. 581 * 582 * @return 583 * 0 on success, a negative errno value otherwise and rte_errno is set. 584 */ 585 static int 586 flow_dv_convert_action_modify_mac 587 (struct mlx5_flow_dv_modify_hdr_resource *resource, 588 const struct rte_flow_action *action, 589 struct rte_flow_error *error) 590 { 591 const struct rte_flow_action_set_mac *conf = 592 (const struct rte_flow_action_set_mac *)(action->conf); 593 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; 594 struct rte_flow_item_eth eth; 595 struct rte_flow_item_eth eth_mask; 596 597 memset(ð, 0, sizeof(eth)); 598 memset(ð_mask, 0, sizeof(eth_mask)); 599 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { 600 memcpy(ð.src.addr_bytes, &conf->mac_addr, 601 sizeof(eth.src.addr_bytes)); 602 memcpy(ð_mask.src.addr_bytes, 603 &rte_flow_item_eth_mask.src.addr_bytes, 604 sizeof(eth_mask.src.addr_bytes)); 605 } else { 606 memcpy(ð.dst.addr_bytes, &conf->mac_addr, 607 sizeof(eth.dst.addr_bytes)); 608 memcpy(ð_mask.dst.addr_bytes, 609 &rte_flow_item_eth_mask.dst.addr_bytes, 610 sizeof(eth_mask.dst.addr_bytes)); 611 } 612 item.spec = ð 613 item.mask = ð_mask; 614 return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource, 615 MLX5_MODIFICATION_TYPE_SET, error); 616 } 617 618 /** 619 * Convert modify-header set VLAN VID action to DV specification. 620 * 621 * @param[in,out] resource 622 * Pointer to the modify-header resource. 623 * @param[in] action 624 * Pointer to action specification. 625 * @param[out] error 626 * Pointer to the error structure. 627 * 628 * @return 629 * 0 on success, a negative errno value otherwise and rte_errno is set. 630 */ 631 static int 632 flow_dv_convert_action_modify_vlan_vid 633 (struct mlx5_flow_dv_modify_hdr_resource *resource, 634 const struct rte_flow_action *action, 635 struct rte_flow_error *error) 636 { 637 const struct rte_flow_action_of_set_vlan_vid *conf = 638 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf); 639 int i = resource->actions_num; 640 struct mlx5_modification_cmd *actions = resource->actions; 641 struct field_modify_info *field = modify_vlan_out_first_vid; 642 643 if (i >= MLX5_MAX_MODIFY_NUM) 644 return rte_flow_error_set(error, EINVAL, 645 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 646 "too many items to modify"); 647 actions[i] = (struct mlx5_modification_cmd) { 648 .action_type = MLX5_MODIFICATION_TYPE_SET, 649 .field = field->id, 650 .length = field->size, 651 .offset = field->offset, 652 }; 653 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 654 actions[i].data1 = conf->vlan_vid; 655 actions[i].data1 = actions[i].data1 << 16; 656 resource->actions_num = ++i; 657 return 0; 658 } 659 660 /** 661 * Convert modify-header set TP action to DV specification. 662 * 663 * @param[in,out] resource 664 * Pointer to the modify-header resource. 665 * @param[in] action 666 * Pointer to action specification. 667 * @param[in] items 668 * Pointer to rte_flow_item objects list. 669 * @param[in] attr 670 * Pointer to flow attributes structure. 671 * @param[in] dev_flow 672 * Pointer to the sub flow. 673 * @param[in] tunnel_decap 674 * Whether action is after tunnel decapsulation. 675 * @param[out] error 676 * Pointer to the error structure. 677 * 678 * @return 679 * 0 on success, a negative errno value otherwise and rte_errno is set. 680 */ 681 static int 682 flow_dv_convert_action_modify_tp 683 (struct mlx5_flow_dv_modify_hdr_resource *resource, 684 const struct rte_flow_action *action, 685 const struct rte_flow_item *items, 686 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 687 bool tunnel_decap, struct rte_flow_error *error) 688 { 689 const struct rte_flow_action_set_tp *conf = 690 (const struct rte_flow_action_set_tp *)(action->conf); 691 struct rte_flow_item item; 692 struct rte_flow_item_udp udp; 693 struct rte_flow_item_udp udp_mask; 694 struct rte_flow_item_tcp tcp; 695 struct rte_flow_item_tcp tcp_mask; 696 struct field_modify_info *field; 697 698 if (!attr->valid) 699 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 700 if (attr->udp) { 701 memset(&udp, 0, sizeof(udp)); 702 memset(&udp_mask, 0, sizeof(udp_mask)); 703 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 704 udp.hdr.src_port = conf->port; 705 udp_mask.hdr.src_port = 706 rte_flow_item_udp_mask.hdr.src_port; 707 } else { 708 udp.hdr.dst_port = conf->port; 709 udp_mask.hdr.dst_port = 710 rte_flow_item_udp_mask.hdr.dst_port; 711 } 712 item.type = RTE_FLOW_ITEM_TYPE_UDP; 713 item.spec = &udp; 714 item.mask = &udp_mask; 715 field = modify_udp; 716 } else { 717 MLX5_ASSERT(attr->tcp); 718 memset(&tcp, 0, sizeof(tcp)); 719 memset(&tcp_mask, 0, sizeof(tcp_mask)); 720 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 721 tcp.hdr.src_port = conf->port; 722 tcp_mask.hdr.src_port = 723 rte_flow_item_tcp_mask.hdr.src_port; 724 } else { 725 tcp.hdr.dst_port = conf->port; 726 tcp_mask.hdr.dst_port = 727 rte_flow_item_tcp_mask.hdr.dst_port; 728 } 729 item.type = RTE_FLOW_ITEM_TYPE_TCP; 730 item.spec = &tcp; 731 item.mask = &tcp_mask; 732 field = modify_tcp; 733 } 734 return flow_dv_convert_modify_action(&item, field, NULL, resource, 735 MLX5_MODIFICATION_TYPE_SET, error); 736 } 737 738 /** 739 * Convert modify-header set TTL action to DV specification. 740 * 741 * @param[in,out] resource 742 * Pointer to the modify-header resource. 743 * @param[in] action 744 * Pointer to action specification. 745 * @param[in] items 746 * Pointer to rte_flow_item objects list. 747 * @param[in] attr 748 * Pointer to flow attributes structure. 749 * @param[in] dev_flow 750 * Pointer to the sub flow. 751 * @param[in] tunnel_decap 752 * Whether action is after tunnel decapsulation. 753 * @param[out] error 754 * Pointer to the error structure. 755 * 756 * @return 757 * 0 on success, a negative errno value otherwise and rte_errno is set. 758 */ 759 static int 760 flow_dv_convert_action_modify_ttl 761 (struct mlx5_flow_dv_modify_hdr_resource *resource, 762 const struct rte_flow_action *action, 763 const struct rte_flow_item *items, 764 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 765 bool tunnel_decap, struct rte_flow_error *error) 766 { 767 const struct rte_flow_action_set_ttl *conf = 768 (const struct rte_flow_action_set_ttl *)(action->conf); 769 struct rte_flow_item item; 770 struct rte_flow_item_ipv4 ipv4; 771 struct rte_flow_item_ipv4 ipv4_mask; 772 struct rte_flow_item_ipv6 ipv6; 773 struct rte_flow_item_ipv6 ipv6_mask; 774 struct field_modify_info *field; 775 776 if (!attr->valid) 777 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 778 if (attr->ipv4) { 779 memset(&ipv4, 0, sizeof(ipv4)); 780 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 781 ipv4.hdr.time_to_live = conf->ttl_value; 782 ipv4_mask.hdr.time_to_live = 0xFF; 783 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 784 item.spec = &ipv4; 785 item.mask = &ipv4_mask; 786 field = modify_ipv4; 787 } else { 788 MLX5_ASSERT(attr->ipv6); 789 memset(&ipv6, 0, sizeof(ipv6)); 790 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 791 ipv6.hdr.hop_limits = conf->ttl_value; 792 ipv6_mask.hdr.hop_limits = 0xFF; 793 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 794 item.spec = &ipv6; 795 item.mask = &ipv6_mask; 796 field = modify_ipv6; 797 } 798 return flow_dv_convert_modify_action(&item, field, NULL, resource, 799 MLX5_MODIFICATION_TYPE_SET, error); 800 } 801 802 /** 803 * Convert modify-header decrement TTL action to DV specification. 804 * 805 * @param[in,out] resource 806 * Pointer to the modify-header resource. 807 * @param[in] action 808 * Pointer to action specification. 809 * @param[in] items 810 * Pointer to rte_flow_item objects list. 811 * @param[in] attr 812 * Pointer to flow attributes structure. 813 * @param[in] dev_flow 814 * Pointer to the sub flow. 815 * @param[in] tunnel_decap 816 * Whether action is after tunnel decapsulation. 817 * @param[out] error 818 * Pointer to the error structure. 819 * 820 * @return 821 * 0 on success, a negative errno value otherwise and rte_errno is set. 822 */ 823 static int 824 flow_dv_convert_action_modify_dec_ttl 825 (struct mlx5_flow_dv_modify_hdr_resource *resource, 826 const struct rte_flow_item *items, 827 union flow_dv_attr *attr, struct mlx5_flow *dev_flow, 828 bool tunnel_decap, struct rte_flow_error *error) 829 { 830 struct rte_flow_item item; 831 struct rte_flow_item_ipv4 ipv4; 832 struct rte_flow_item_ipv4 ipv4_mask; 833 struct rte_flow_item_ipv6 ipv6; 834 struct rte_flow_item_ipv6 ipv6_mask; 835 struct field_modify_info *field; 836 837 if (!attr->valid) 838 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap); 839 if (attr->ipv4) { 840 memset(&ipv4, 0, sizeof(ipv4)); 841 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 842 ipv4.hdr.time_to_live = 0xFF; 843 ipv4_mask.hdr.time_to_live = 0xFF; 844 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 845 item.spec = &ipv4; 846 item.mask = &ipv4_mask; 847 field = modify_ipv4; 848 } else { 849 MLX5_ASSERT(attr->ipv6); 850 memset(&ipv6, 0, sizeof(ipv6)); 851 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 852 ipv6.hdr.hop_limits = 0xFF; 853 ipv6_mask.hdr.hop_limits = 0xFF; 854 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 855 item.spec = &ipv6; 856 item.mask = &ipv6_mask; 857 field = modify_ipv6; 858 } 859 return flow_dv_convert_modify_action(&item, field, NULL, resource, 860 MLX5_MODIFICATION_TYPE_ADD, error); 861 } 862 863 /** 864 * Convert modify-header increment/decrement TCP Sequence number 865 * to DV specification. 866 * 867 * @param[in,out] resource 868 * Pointer to the modify-header resource. 869 * @param[in] action 870 * Pointer to action specification. 871 * @param[out] error 872 * Pointer to the error structure. 873 * 874 * @return 875 * 0 on success, a negative errno value otherwise and rte_errno is set. 876 */ 877 static int 878 flow_dv_convert_action_modify_tcp_seq 879 (struct mlx5_flow_dv_modify_hdr_resource *resource, 880 const struct rte_flow_action *action, 881 struct rte_flow_error *error) 882 { 883 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 884 uint64_t value = rte_be_to_cpu_32(*conf); 885 struct rte_flow_item item; 886 struct rte_flow_item_tcp tcp; 887 struct rte_flow_item_tcp tcp_mask; 888 889 memset(&tcp, 0, sizeof(tcp)); 890 memset(&tcp_mask, 0, sizeof(tcp_mask)); 891 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) 892 /* 893 * The HW has no decrement operation, only increment operation. 894 * To simulate decrement X from Y using increment operation 895 * we need to add UINT32_MAX X times to Y. 896 * Each adding of UINT32_MAX decrements Y by 1. 897 */ 898 value *= UINT32_MAX; 899 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); 900 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); 901 item.type = RTE_FLOW_ITEM_TYPE_TCP; 902 item.spec = &tcp; 903 item.mask = &tcp_mask; 904 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 905 MLX5_MODIFICATION_TYPE_ADD, error); 906 } 907 908 /** 909 * Convert modify-header increment/decrement TCP Acknowledgment number 910 * to DV specification. 911 * 912 * @param[in,out] resource 913 * Pointer to the modify-header resource. 914 * @param[in] action 915 * Pointer to action specification. 916 * @param[out] error 917 * Pointer to the error structure. 918 * 919 * @return 920 * 0 on success, a negative errno value otherwise and rte_errno is set. 921 */ 922 static int 923 flow_dv_convert_action_modify_tcp_ack 924 (struct mlx5_flow_dv_modify_hdr_resource *resource, 925 const struct rte_flow_action *action, 926 struct rte_flow_error *error) 927 { 928 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 929 uint64_t value = rte_be_to_cpu_32(*conf); 930 struct rte_flow_item item; 931 struct rte_flow_item_tcp tcp; 932 struct rte_flow_item_tcp tcp_mask; 933 934 memset(&tcp, 0, sizeof(tcp)); 935 memset(&tcp_mask, 0, sizeof(tcp_mask)); 936 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) 937 /* 938 * The HW has no decrement operation, only increment operation. 939 * To simulate decrement X from Y using increment operation 940 * we need to add UINT32_MAX X times to Y. 941 * Each adding of UINT32_MAX decrements Y by 1. 942 */ 943 value *= UINT32_MAX; 944 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); 945 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); 946 item.type = RTE_FLOW_ITEM_TYPE_TCP; 947 item.spec = &tcp; 948 item.mask = &tcp_mask; 949 return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource, 950 MLX5_MODIFICATION_TYPE_ADD, error); 951 } 952 953 static enum mlx5_modification_field reg_to_field[] = { 954 [REG_NONE] = MLX5_MODI_OUT_NONE, 955 [REG_A] = MLX5_MODI_META_DATA_REG_A, 956 [REG_B] = MLX5_MODI_META_DATA_REG_B, 957 [REG_C_0] = MLX5_MODI_META_REG_C_0, 958 [REG_C_1] = MLX5_MODI_META_REG_C_1, 959 [REG_C_2] = MLX5_MODI_META_REG_C_2, 960 [REG_C_3] = MLX5_MODI_META_REG_C_3, 961 [REG_C_4] = MLX5_MODI_META_REG_C_4, 962 [REG_C_5] = MLX5_MODI_META_REG_C_5, 963 [REG_C_6] = MLX5_MODI_META_REG_C_6, 964 [REG_C_7] = MLX5_MODI_META_REG_C_7, 965 }; 966 967 /** 968 * Convert register set to DV specification. 969 * 970 * @param[in,out] resource 971 * Pointer to the modify-header resource. 972 * @param[in] action 973 * Pointer to action specification. 974 * @param[out] error 975 * Pointer to the error structure. 976 * 977 * @return 978 * 0 on success, a negative errno value otherwise and rte_errno is set. 979 */ 980 static int 981 flow_dv_convert_action_set_reg 982 (struct mlx5_flow_dv_modify_hdr_resource *resource, 983 const struct rte_flow_action *action, 984 struct rte_flow_error *error) 985 { 986 const struct mlx5_rte_flow_action_set_tag *conf = action->conf; 987 struct mlx5_modification_cmd *actions = resource->actions; 988 uint32_t i = resource->actions_num; 989 990 if (i >= MLX5_MAX_MODIFY_NUM) 991 return rte_flow_error_set(error, EINVAL, 992 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 993 "too many items to modify"); 994 MLX5_ASSERT(conf->id != REG_NONE); 995 MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field)); 996 actions[i] = (struct mlx5_modification_cmd) { 997 .action_type = MLX5_MODIFICATION_TYPE_SET, 998 .field = reg_to_field[conf->id], 999 }; 1000 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 1001 actions[i].data1 = rte_cpu_to_be_32(conf->data); 1002 ++i; 1003 resource->actions_num = i; 1004 return 0; 1005 } 1006 1007 /** 1008 * Convert SET_TAG action to DV specification. 1009 * 1010 * @param[in] dev 1011 * Pointer to the rte_eth_dev structure. 1012 * @param[in,out] resource 1013 * Pointer to the modify-header resource. 1014 * @param[in] conf 1015 * Pointer to action specification. 1016 * @param[out] error 1017 * Pointer to the error structure. 1018 * 1019 * @return 1020 * 0 on success, a negative errno value otherwise and rte_errno is set. 1021 */ 1022 static int 1023 flow_dv_convert_action_set_tag 1024 (struct rte_eth_dev *dev, 1025 struct mlx5_flow_dv_modify_hdr_resource *resource, 1026 const struct rte_flow_action_set_tag *conf, 1027 struct rte_flow_error *error) 1028 { 1029 rte_be32_t data = rte_cpu_to_be_32(conf->data); 1030 rte_be32_t mask = rte_cpu_to_be_32(conf->mask); 1031 struct rte_flow_item item = { 1032 .spec = &data, 1033 .mask = &mask, 1034 }; 1035 struct field_modify_info reg_c_x[] = { 1036 [1] = {0, 0, 0}, 1037 }; 1038 enum mlx5_modification_field reg_type; 1039 int ret; 1040 1041 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 1042 if (ret < 0) 1043 return ret; 1044 MLX5_ASSERT(ret != REG_NONE); 1045 MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field)); 1046 reg_type = reg_to_field[ret]; 1047 MLX5_ASSERT(reg_type > 0); 1048 reg_c_x[0] = (struct field_modify_info){4, 0, reg_type}; 1049 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1050 MLX5_MODIFICATION_TYPE_SET, error); 1051 } 1052 1053 /** 1054 * Convert internal COPY_REG action to DV specification. 1055 * 1056 * @param[in] dev 1057 * Pointer to the rte_eth_dev structure. 1058 * @param[in,out] res 1059 * Pointer to the modify-header resource. 1060 * @param[in] action 1061 * Pointer to action specification. 1062 * @param[out] error 1063 * Pointer to the error structure. 1064 * 1065 * @return 1066 * 0 on success, a negative errno value otherwise and rte_errno is set. 1067 */ 1068 static int 1069 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev, 1070 struct mlx5_flow_dv_modify_hdr_resource *res, 1071 const struct rte_flow_action *action, 1072 struct rte_flow_error *error) 1073 { 1074 const struct mlx5_flow_action_copy_mreg *conf = action->conf; 1075 rte_be32_t mask = RTE_BE32(UINT32_MAX); 1076 struct rte_flow_item item = { 1077 .spec = NULL, 1078 .mask = &mask, 1079 }; 1080 struct field_modify_info reg_src[] = { 1081 {4, 0, reg_to_field[conf->src]}, 1082 {0, 0, 0}, 1083 }; 1084 struct field_modify_info reg_dst = { 1085 .offset = 0, 1086 .id = reg_to_field[conf->dst], 1087 }; 1088 /* Adjust reg_c[0] usage according to reported mask. */ 1089 if (conf->dst == REG_C_0 || conf->src == REG_C_0) { 1090 struct mlx5_priv *priv = dev->data->dev_private; 1091 uint32_t reg_c0 = priv->sh->dv_regc0_mask; 1092 1093 MLX5_ASSERT(reg_c0); 1094 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY); 1095 if (conf->dst == REG_C_0) { 1096 /* Copy to reg_c[0], within mask only. */ 1097 reg_dst.offset = rte_bsf32(reg_c0); 1098 /* 1099 * Mask is ignoring the enianness, because 1100 * there is no conversion in datapath. 1101 */ 1102 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1103 /* Copy from destination lower bits to reg_c[0]. */ 1104 mask = reg_c0 >> reg_dst.offset; 1105 #else 1106 /* Copy from destination upper bits to reg_c[0]. */ 1107 mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT - 1108 rte_fls_u32(reg_c0)); 1109 #endif 1110 } else { 1111 mask = rte_cpu_to_be_32(reg_c0); 1112 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1113 /* Copy from reg_c[0] to destination lower bits. */ 1114 reg_dst.offset = 0; 1115 #else 1116 /* Copy from reg_c[0] to destination upper bits. */ 1117 reg_dst.offset = sizeof(reg_c0) * CHAR_BIT - 1118 (rte_fls_u32(reg_c0) - 1119 rte_bsf32(reg_c0)); 1120 #endif 1121 } 1122 } 1123 return flow_dv_convert_modify_action(&item, 1124 reg_src, ®_dst, res, 1125 MLX5_MODIFICATION_TYPE_COPY, 1126 error); 1127 } 1128 1129 /** 1130 * Convert MARK action to DV specification. This routine is used 1131 * in extensive metadata only and requires metadata register to be 1132 * handled. In legacy mode hardware tag resource is engaged. 1133 * 1134 * @param[in] dev 1135 * Pointer to the rte_eth_dev structure. 1136 * @param[in] conf 1137 * Pointer to MARK action specification. 1138 * @param[in,out] resource 1139 * Pointer to the modify-header resource. 1140 * @param[out] error 1141 * Pointer to the error structure. 1142 * 1143 * @return 1144 * 0 on success, a negative errno value otherwise and rte_errno is set. 1145 */ 1146 static int 1147 flow_dv_convert_action_mark(struct rte_eth_dev *dev, 1148 const struct rte_flow_action_mark *conf, 1149 struct mlx5_flow_dv_modify_hdr_resource *resource, 1150 struct rte_flow_error *error) 1151 { 1152 struct mlx5_priv *priv = dev->data->dev_private; 1153 rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK & 1154 priv->sh->dv_mark_mask); 1155 rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask; 1156 struct rte_flow_item item = { 1157 .spec = &data, 1158 .mask = &mask, 1159 }; 1160 struct field_modify_info reg_c_x[] = { 1161 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */ 1162 {0, 0, 0}, 1163 }; 1164 int reg; 1165 1166 if (!mask) 1167 return rte_flow_error_set(error, EINVAL, 1168 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1169 NULL, "zero mark action mask"); 1170 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1171 if (reg < 0) 1172 return reg; 1173 MLX5_ASSERT(reg > 0); 1174 if (reg == REG_C_0) { 1175 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1176 uint32_t shl_c0 = rte_bsf32(msk_c0); 1177 1178 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0); 1179 mask = rte_cpu_to_be_32(mask) & msk_c0; 1180 mask = rte_cpu_to_be_32(mask << shl_c0); 1181 } 1182 reg_c_x[0].id = reg_to_field[reg]; 1183 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1184 MLX5_MODIFICATION_TYPE_SET, error); 1185 } 1186 1187 /** 1188 * Get metadata register index for specified steering domain. 1189 * 1190 * @param[in] dev 1191 * Pointer to the rte_eth_dev structure. 1192 * @param[in] attr 1193 * Attributes of flow to determine steering domain. 1194 * @param[out] error 1195 * Pointer to the error structure. 1196 * 1197 * @return 1198 * positive index on success, a negative errno value otherwise 1199 * and rte_errno is set. 1200 */ 1201 static enum modify_reg 1202 flow_dv_get_metadata_reg(struct rte_eth_dev *dev, 1203 const struct rte_flow_attr *attr, 1204 struct rte_flow_error *error) 1205 { 1206 int reg = 1207 mlx5_flow_get_reg_id(dev, attr->transfer ? 1208 MLX5_METADATA_FDB : 1209 attr->egress ? 1210 MLX5_METADATA_TX : 1211 MLX5_METADATA_RX, 0, error); 1212 if (reg < 0) 1213 return rte_flow_error_set(error, 1214 ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 1215 NULL, "unavailable " 1216 "metadata register"); 1217 return reg; 1218 } 1219 1220 /** 1221 * Convert SET_META action to DV specification. 1222 * 1223 * @param[in] dev 1224 * Pointer to the rte_eth_dev structure. 1225 * @param[in,out] resource 1226 * Pointer to the modify-header resource. 1227 * @param[in] attr 1228 * Attributes of flow that includes this item. 1229 * @param[in] conf 1230 * Pointer to action specification. 1231 * @param[out] error 1232 * Pointer to the error structure. 1233 * 1234 * @return 1235 * 0 on success, a negative errno value otherwise and rte_errno is set. 1236 */ 1237 static int 1238 flow_dv_convert_action_set_meta 1239 (struct rte_eth_dev *dev, 1240 struct mlx5_flow_dv_modify_hdr_resource *resource, 1241 const struct rte_flow_attr *attr, 1242 const struct rte_flow_action_set_meta *conf, 1243 struct rte_flow_error *error) 1244 { 1245 uint32_t data = conf->data; 1246 uint32_t mask = conf->mask; 1247 struct rte_flow_item item = { 1248 .spec = &data, 1249 .mask = &mask, 1250 }; 1251 struct field_modify_info reg_c_x[] = { 1252 [1] = {0, 0, 0}, 1253 }; 1254 int reg = flow_dv_get_metadata_reg(dev, attr, error); 1255 1256 if (reg < 0) 1257 return reg; 1258 /* 1259 * In datapath code there is no endianness 1260 * coversions for perfromance reasons, all 1261 * pattern conversions are done in rte_flow. 1262 */ 1263 if (reg == REG_C_0) { 1264 struct mlx5_priv *priv = dev->data->dev_private; 1265 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 1266 uint32_t shl_c0; 1267 1268 MLX5_ASSERT(msk_c0); 1269 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 1270 shl_c0 = rte_bsf32(msk_c0); 1271 #else 1272 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0); 1273 #endif 1274 mask <<= shl_c0; 1275 data <<= shl_c0; 1276 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask))); 1277 } 1278 reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]}; 1279 /* The routine expects parameters in memory as big-endian ones. */ 1280 return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource, 1281 MLX5_MODIFICATION_TYPE_SET, error); 1282 } 1283 1284 /** 1285 * Convert modify-header set IPv4 DSCP action to DV specification. 1286 * 1287 * @param[in,out] resource 1288 * Pointer to the modify-header resource. 1289 * @param[in] action 1290 * Pointer to action specification. 1291 * @param[out] error 1292 * Pointer to the error structure. 1293 * 1294 * @return 1295 * 0 on success, a negative errno value otherwise and rte_errno is set. 1296 */ 1297 static int 1298 flow_dv_convert_action_modify_ipv4_dscp 1299 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1300 const struct rte_flow_action *action, 1301 struct rte_flow_error *error) 1302 { 1303 const struct rte_flow_action_set_dscp *conf = 1304 (const struct rte_flow_action_set_dscp *)(action->conf); 1305 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 1306 struct rte_flow_item_ipv4 ipv4; 1307 struct rte_flow_item_ipv4 ipv4_mask; 1308 1309 memset(&ipv4, 0, sizeof(ipv4)); 1310 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 1311 ipv4.hdr.type_of_service = conf->dscp; 1312 ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2; 1313 item.spec = &ipv4; 1314 item.mask = &ipv4_mask; 1315 return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource, 1316 MLX5_MODIFICATION_TYPE_SET, error); 1317 } 1318 1319 /** 1320 * Convert modify-header set IPv6 DSCP action to DV specification. 1321 * 1322 * @param[in,out] resource 1323 * Pointer to the modify-header resource. 1324 * @param[in] action 1325 * Pointer to action specification. 1326 * @param[out] error 1327 * Pointer to the error structure. 1328 * 1329 * @return 1330 * 0 on success, a negative errno value otherwise and rte_errno is set. 1331 */ 1332 static int 1333 flow_dv_convert_action_modify_ipv6_dscp 1334 (struct mlx5_flow_dv_modify_hdr_resource *resource, 1335 const struct rte_flow_action *action, 1336 struct rte_flow_error *error) 1337 { 1338 const struct rte_flow_action_set_dscp *conf = 1339 (const struct rte_flow_action_set_dscp *)(action->conf); 1340 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 1341 struct rte_flow_item_ipv6 ipv6; 1342 struct rte_flow_item_ipv6 ipv6_mask; 1343 1344 memset(&ipv6, 0, sizeof(ipv6)); 1345 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 1346 /* 1347 * Even though the DSCP bits offset of IPv6 is not byte aligned, 1348 * rdma-core only accept the DSCP bits byte aligned start from 1349 * bit 0 to 5 as to be compatible with IPv4. No need to shift the 1350 * bits in IPv6 case as rdma-core requires byte aligned value. 1351 */ 1352 ipv6.hdr.vtc_flow = conf->dscp; 1353 ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22; 1354 item.spec = &ipv6; 1355 item.mask = &ipv6_mask; 1356 return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource, 1357 MLX5_MODIFICATION_TYPE_SET, error); 1358 } 1359 1360 /** 1361 * Validate MARK item. 1362 * 1363 * @param[in] dev 1364 * Pointer to the rte_eth_dev structure. 1365 * @param[in] item 1366 * Item specification. 1367 * @param[in] attr 1368 * Attributes of flow that includes this item. 1369 * @param[out] error 1370 * Pointer to error structure. 1371 * 1372 * @return 1373 * 0 on success, a negative errno value otherwise and rte_errno is set. 1374 */ 1375 static int 1376 flow_dv_validate_item_mark(struct rte_eth_dev *dev, 1377 const struct rte_flow_item *item, 1378 const struct rte_flow_attr *attr __rte_unused, 1379 struct rte_flow_error *error) 1380 { 1381 struct mlx5_priv *priv = dev->data->dev_private; 1382 struct mlx5_dev_config *config = &priv->config; 1383 const struct rte_flow_item_mark *spec = item->spec; 1384 const struct rte_flow_item_mark *mask = item->mask; 1385 const struct rte_flow_item_mark nic_mask = { 1386 .id = priv->sh->dv_mark_mask, 1387 }; 1388 int ret; 1389 1390 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 1391 return rte_flow_error_set(error, ENOTSUP, 1392 RTE_FLOW_ERROR_TYPE_ITEM, item, 1393 "extended metadata feature" 1394 " isn't enabled"); 1395 if (!mlx5_flow_ext_mreg_supported(dev)) 1396 return rte_flow_error_set(error, ENOTSUP, 1397 RTE_FLOW_ERROR_TYPE_ITEM, item, 1398 "extended metadata register" 1399 " isn't supported"); 1400 if (!nic_mask.id) 1401 return rte_flow_error_set(error, ENOTSUP, 1402 RTE_FLOW_ERROR_TYPE_ITEM, item, 1403 "extended metadata register" 1404 " isn't available"); 1405 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 1406 if (ret < 0) 1407 return ret; 1408 if (!spec) 1409 return rte_flow_error_set(error, EINVAL, 1410 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1411 item->spec, 1412 "data cannot be empty"); 1413 if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id)) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1416 &spec->id, 1417 "mark id exceeds the limit"); 1418 if (!mask) 1419 mask = &nic_mask; 1420 if (!mask->id) 1421 return rte_flow_error_set(error, EINVAL, 1422 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1423 "mask cannot be zero"); 1424 1425 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1426 (const uint8_t *)&nic_mask, 1427 sizeof(struct rte_flow_item_mark), 1428 error); 1429 if (ret < 0) 1430 return ret; 1431 return 0; 1432 } 1433 1434 /** 1435 * Validate META item. 1436 * 1437 * @param[in] dev 1438 * Pointer to the rte_eth_dev structure. 1439 * @param[in] item 1440 * Item specification. 1441 * @param[in] attr 1442 * Attributes of flow that includes this item. 1443 * @param[out] error 1444 * Pointer to error structure. 1445 * 1446 * @return 1447 * 0 on success, a negative errno value otherwise and rte_errno is set. 1448 */ 1449 static int 1450 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused, 1451 const struct rte_flow_item *item, 1452 const struct rte_flow_attr *attr, 1453 struct rte_flow_error *error) 1454 { 1455 struct mlx5_priv *priv = dev->data->dev_private; 1456 struct mlx5_dev_config *config = &priv->config; 1457 const struct rte_flow_item_meta *spec = item->spec; 1458 const struct rte_flow_item_meta *mask = item->mask; 1459 struct rte_flow_item_meta nic_mask = { 1460 .data = UINT32_MAX 1461 }; 1462 int reg; 1463 int ret; 1464 1465 if (!spec) 1466 return rte_flow_error_set(error, EINVAL, 1467 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1468 item->spec, 1469 "data cannot be empty"); 1470 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 1471 if (!mlx5_flow_ext_mreg_supported(dev)) 1472 return rte_flow_error_set(error, ENOTSUP, 1473 RTE_FLOW_ERROR_TYPE_ITEM, item, 1474 "extended metadata register" 1475 " isn't supported"); 1476 reg = flow_dv_get_metadata_reg(dev, attr, error); 1477 if (reg < 0) 1478 return reg; 1479 if (reg == REG_B) 1480 return rte_flow_error_set(error, ENOTSUP, 1481 RTE_FLOW_ERROR_TYPE_ITEM, item, 1482 "match on reg_b " 1483 "isn't supported"); 1484 if (reg != REG_A) 1485 nic_mask.data = priv->sh->dv_meta_mask; 1486 } 1487 if (!mask) 1488 mask = &rte_flow_item_meta_mask; 1489 if (!mask->data) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1492 "mask cannot be zero"); 1493 1494 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1495 (const uint8_t *)&nic_mask, 1496 sizeof(struct rte_flow_item_meta), 1497 error); 1498 return ret; 1499 } 1500 1501 /** 1502 * Validate TAG item. 1503 * 1504 * @param[in] dev 1505 * Pointer to the rte_eth_dev structure. 1506 * @param[in] item 1507 * Item specification. 1508 * @param[in] attr 1509 * Attributes of flow that includes this item. 1510 * @param[out] error 1511 * Pointer to error structure. 1512 * 1513 * @return 1514 * 0 on success, a negative errno value otherwise and rte_errno is set. 1515 */ 1516 static int 1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev, 1518 const struct rte_flow_item *item, 1519 const struct rte_flow_attr *attr __rte_unused, 1520 struct rte_flow_error *error) 1521 { 1522 const struct rte_flow_item_tag *spec = item->spec; 1523 const struct rte_flow_item_tag *mask = item->mask; 1524 const struct rte_flow_item_tag nic_mask = { 1525 .data = RTE_BE32(UINT32_MAX), 1526 .index = 0xff, 1527 }; 1528 int ret; 1529 1530 if (!mlx5_flow_ext_mreg_supported(dev)) 1531 return rte_flow_error_set(error, ENOTSUP, 1532 RTE_FLOW_ERROR_TYPE_ITEM, item, 1533 "extensive metadata register" 1534 " isn't supported"); 1535 if (!spec) 1536 return rte_flow_error_set(error, EINVAL, 1537 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 1538 item->spec, 1539 "data cannot be empty"); 1540 if (!mask) 1541 mask = &rte_flow_item_tag_mask; 1542 if (!mask->data) 1543 return rte_flow_error_set(error, EINVAL, 1544 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1545 "mask cannot be zero"); 1546 1547 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1548 (const uint8_t *)&nic_mask, 1549 sizeof(struct rte_flow_item_tag), 1550 error); 1551 if (ret < 0) 1552 return ret; 1553 if (mask->index != 0xff) 1554 return rte_flow_error_set(error, EINVAL, 1555 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL, 1556 "partial mask for tag index" 1557 " is not supported"); 1558 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error); 1559 if (ret < 0) 1560 return ret; 1561 MLX5_ASSERT(ret != REG_NONE); 1562 return 0; 1563 } 1564 1565 /** 1566 * Validate vport item. 1567 * 1568 * @param[in] dev 1569 * Pointer to the rte_eth_dev structure. 1570 * @param[in] item 1571 * Item specification. 1572 * @param[in] attr 1573 * Attributes of flow that includes this item. 1574 * @param[in] item_flags 1575 * Bit-fields that holds the items detected until now. 1576 * @param[out] error 1577 * Pointer to error structure. 1578 * 1579 * @return 1580 * 0 on success, a negative errno value otherwise and rte_errno is set. 1581 */ 1582 static int 1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev, 1584 const struct rte_flow_item *item, 1585 const struct rte_flow_attr *attr, 1586 uint64_t item_flags, 1587 struct rte_flow_error *error) 1588 { 1589 const struct rte_flow_item_port_id *spec = item->spec; 1590 const struct rte_flow_item_port_id *mask = item->mask; 1591 const struct rte_flow_item_port_id switch_mask = { 1592 .id = 0xffffffff, 1593 }; 1594 struct mlx5_priv *esw_priv; 1595 struct mlx5_priv *dev_priv; 1596 int ret; 1597 1598 if (!attr->transfer) 1599 return rte_flow_error_set(error, EINVAL, 1600 RTE_FLOW_ERROR_TYPE_ITEM, 1601 NULL, 1602 "match on port id is valid only" 1603 " when transfer flag is enabled"); 1604 if (item_flags & MLX5_FLOW_ITEM_PORT_ID) 1605 return rte_flow_error_set(error, ENOTSUP, 1606 RTE_FLOW_ERROR_TYPE_ITEM, item, 1607 "multiple source ports are not" 1608 " supported"); 1609 if (!mask) 1610 mask = &switch_mask; 1611 if (mask->id != 0xffffffff) 1612 return rte_flow_error_set(error, ENOTSUP, 1613 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 1614 mask, 1615 "no support for partial mask on" 1616 " \"id\" field"); 1617 ret = mlx5_flow_item_acceptable 1618 (item, (const uint8_t *)mask, 1619 (const uint8_t *)&rte_flow_item_port_id_mask, 1620 sizeof(struct rte_flow_item_port_id), 1621 error); 1622 if (ret) 1623 return ret; 1624 if (!spec) 1625 return 0; 1626 esw_priv = mlx5_port_to_eswitch_info(spec->id, false); 1627 if (!esw_priv) 1628 return rte_flow_error_set(error, rte_errno, 1629 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1630 "failed to obtain E-Switch info for" 1631 " port"); 1632 dev_priv = mlx5_dev_to_eswitch_info(dev); 1633 if (!dev_priv) 1634 return rte_flow_error_set(error, rte_errno, 1635 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1636 NULL, 1637 "failed to obtain E-Switch info"); 1638 if (esw_priv->domain_id != dev_priv->domain_id) 1639 return rte_flow_error_set(error, EINVAL, 1640 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 1641 "cannot match on a port from a" 1642 " different E-Switch"); 1643 return 0; 1644 } 1645 1646 /** 1647 * Validate VLAN item. 1648 * 1649 * @param[in] item 1650 * Item specification. 1651 * @param[in] item_flags 1652 * Bit-fields that holds the items detected until now. 1653 * @param[in] dev 1654 * Ethernet device flow is being created on. 1655 * @param[out] error 1656 * Pointer to error structure. 1657 * 1658 * @return 1659 * 0 on success, a negative errno value otherwise and rte_errno is set. 1660 */ 1661 static int 1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item, 1663 uint64_t item_flags, 1664 struct rte_eth_dev *dev, 1665 struct rte_flow_error *error) 1666 { 1667 const struct rte_flow_item_vlan *mask = item->mask; 1668 const struct rte_flow_item_vlan nic_mask = { 1669 .tci = RTE_BE16(UINT16_MAX), 1670 .inner_type = RTE_BE16(UINT16_MAX), 1671 }; 1672 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1673 int ret; 1674 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 1675 MLX5_FLOW_LAYER_INNER_L4) : 1676 (MLX5_FLOW_LAYER_OUTER_L3 | 1677 MLX5_FLOW_LAYER_OUTER_L4); 1678 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 1679 MLX5_FLOW_LAYER_OUTER_VLAN; 1680 1681 if (item_flags & vlanm) 1682 return rte_flow_error_set(error, EINVAL, 1683 RTE_FLOW_ERROR_TYPE_ITEM, item, 1684 "multiple VLAN layers not supported"); 1685 else if ((item_flags & l34m) != 0) 1686 return rte_flow_error_set(error, EINVAL, 1687 RTE_FLOW_ERROR_TYPE_ITEM, item, 1688 "VLAN cannot follow L3/L4 layer"); 1689 if (!mask) 1690 mask = &rte_flow_item_vlan_mask; 1691 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 1692 (const uint8_t *)&nic_mask, 1693 sizeof(struct rte_flow_item_vlan), 1694 error); 1695 if (ret) 1696 return ret; 1697 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 1698 struct mlx5_priv *priv = dev->data->dev_private; 1699 1700 if (priv->vmwa_context) { 1701 /* 1702 * Non-NULL context means we have a virtual machine 1703 * and SR-IOV enabled, we have to create VLAN interface 1704 * to make hypervisor to setup E-Switch vport 1705 * context correctly. We avoid creating the multiple 1706 * VLAN interfaces, so we cannot support VLAN tag mask. 1707 */ 1708 return rte_flow_error_set(error, EINVAL, 1709 RTE_FLOW_ERROR_TYPE_ITEM, 1710 item, 1711 "VLAN tag mask is not" 1712 " supported in virtual" 1713 " environment"); 1714 } 1715 } 1716 return 0; 1717 } 1718 1719 /* 1720 * GTP flags are contained in 1 byte of the format: 1721 * ------------------------------------------- 1722 * | bit | 0 - 2 | 3 | 4 | 5 | 6 | 7 | 1723 * |-----------------------------------------| 1724 * | value | Version | PT | Res | E | S | PN | 1725 * ------------------------------------------- 1726 * 1727 * Matching is supported only for GTP flags E, S, PN. 1728 */ 1729 #define MLX5_GTP_FLAGS_MASK 0x07 1730 1731 /** 1732 * Validate GTP item. 1733 * 1734 * @param[in] dev 1735 * Pointer to the rte_eth_dev structure. 1736 * @param[in] item 1737 * Item specification. 1738 * @param[in] item_flags 1739 * Bit-fields that holds the items detected until now. 1740 * @param[out] error 1741 * Pointer to error structure. 1742 * 1743 * @return 1744 * 0 on success, a negative errno value otherwise and rte_errno is set. 1745 */ 1746 static int 1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev, 1748 const struct rte_flow_item *item, 1749 uint64_t item_flags, 1750 struct rte_flow_error *error) 1751 { 1752 struct mlx5_priv *priv = dev->data->dev_private; 1753 const struct rte_flow_item_gtp *spec = item->spec; 1754 const struct rte_flow_item_gtp *mask = item->mask; 1755 const struct rte_flow_item_gtp nic_mask = { 1756 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK, 1757 .msg_type = 0xff, 1758 .teid = RTE_BE32(0xffffffff), 1759 }; 1760 1761 if (!priv->config.hca_attr.tunnel_stateless_gtp) 1762 return rte_flow_error_set(error, ENOTSUP, 1763 RTE_FLOW_ERROR_TYPE_ITEM, item, 1764 "GTP support is not enabled"); 1765 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 1766 return rte_flow_error_set(error, ENOTSUP, 1767 RTE_FLOW_ERROR_TYPE_ITEM, item, 1768 "multiple tunnel layers not" 1769 " supported"); 1770 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 1771 return rte_flow_error_set(error, EINVAL, 1772 RTE_FLOW_ERROR_TYPE_ITEM, item, 1773 "no outer UDP layer found"); 1774 if (!mask) 1775 mask = &rte_flow_item_gtp_mask; 1776 if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK) 1777 return rte_flow_error_set(error, ENOTSUP, 1778 RTE_FLOW_ERROR_TYPE_ITEM, item, 1779 "Match is supported for GTP" 1780 " flags only"); 1781 return mlx5_flow_item_acceptable 1782 (item, (const uint8_t *)mask, 1783 (const uint8_t *)&nic_mask, 1784 sizeof(struct rte_flow_item_gtp), 1785 error); 1786 } 1787 1788 /** 1789 * Validate the pop VLAN action. 1790 * 1791 * @param[in] dev 1792 * Pointer to the rte_eth_dev structure. 1793 * @param[in] action_flags 1794 * Holds the actions detected until now. 1795 * @param[in] action 1796 * Pointer to the pop vlan action. 1797 * @param[in] item_flags 1798 * The items found in this flow rule. 1799 * @param[in] attr 1800 * Pointer to flow attributes. 1801 * @param[out] error 1802 * Pointer to error structure. 1803 * 1804 * @return 1805 * 0 on success, a negative errno value otherwise and rte_errno is set. 1806 */ 1807 static int 1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev, 1809 uint64_t action_flags, 1810 const struct rte_flow_action *action, 1811 uint64_t item_flags, 1812 const struct rte_flow_attr *attr, 1813 struct rte_flow_error *error) 1814 { 1815 const struct mlx5_priv *priv = dev->data->dev_private; 1816 1817 (void)action; 1818 (void)attr; 1819 if (!priv->sh->pop_vlan_action) 1820 return rte_flow_error_set(error, ENOTSUP, 1821 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1822 NULL, 1823 "pop vlan action is not supported"); 1824 if (attr->egress) 1825 return rte_flow_error_set(error, ENOTSUP, 1826 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 1827 NULL, 1828 "pop vlan action not supported for " 1829 "egress"); 1830 if (action_flags & MLX5_FLOW_VLAN_ACTIONS) 1831 return rte_flow_error_set(error, ENOTSUP, 1832 RTE_FLOW_ERROR_TYPE_ACTION, action, 1833 "no support for multiple VLAN " 1834 "actions"); 1835 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 1836 return rte_flow_error_set(error, ENOTSUP, 1837 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1838 NULL, 1839 "cannot pop vlan without a " 1840 "match on (outer) vlan in the flow"); 1841 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1842 return rte_flow_error_set(error, EINVAL, 1843 RTE_FLOW_ERROR_TYPE_ACTION, action, 1844 "wrong action order, port_id should " 1845 "be after pop VLAN action"); 1846 if (!attr->transfer && priv->representor) 1847 return rte_flow_error_set(error, ENOTSUP, 1848 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1849 "pop vlan action for VF representor " 1850 "not supported on NIC table"); 1851 return 0; 1852 } 1853 1854 /** 1855 * Get VLAN default info from vlan match info. 1856 * 1857 * @param[in] items 1858 * the list of item specifications. 1859 * @param[out] vlan 1860 * pointer VLAN info to fill to. 1861 * 1862 * @return 1863 * 0 on success, a negative errno value otherwise and rte_errno is set. 1864 */ 1865 static void 1866 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items, 1867 struct rte_vlan_hdr *vlan) 1868 { 1869 const struct rte_flow_item_vlan nic_mask = { 1870 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK | 1871 MLX5DV_FLOW_VLAN_VID_MASK), 1872 .inner_type = RTE_BE16(0xffff), 1873 }; 1874 1875 if (items == NULL) 1876 return; 1877 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1878 int type = items->type; 1879 1880 if (type == RTE_FLOW_ITEM_TYPE_VLAN || 1881 type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 1882 break; 1883 } 1884 if (items->type != RTE_FLOW_ITEM_TYPE_END) { 1885 const struct rte_flow_item_vlan *vlan_m = items->mask; 1886 const struct rte_flow_item_vlan *vlan_v = items->spec; 1887 1888 /* If VLAN item in pattern doesn't contain data, return here. */ 1889 if (!vlan_v) 1890 return; 1891 if (!vlan_m) 1892 vlan_m = &nic_mask; 1893 /* Only full match values are accepted */ 1894 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) == 1895 MLX5DV_FLOW_VLAN_PCP_MASK_BE) { 1896 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK; 1897 vlan->vlan_tci |= 1898 rte_be_to_cpu_16(vlan_v->tci & 1899 MLX5DV_FLOW_VLAN_PCP_MASK_BE); 1900 } 1901 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) == 1902 MLX5DV_FLOW_VLAN_VID_MASK_BE) { 1903 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK; 1904 vlan->vlan_tci |= 1905 rte_be_to_cpu_16(vlan_v->tci & 1906 MLX5DV_FLOW_VLAN_VID_MASK_BE); 1907 } 1908 if (vlan_m->inner_type == nic_mask.inner_type) 1909 vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type & 1910 vlan_m->inner_type); 1911 } 1912 } 1913 1914 /** 1915 * Validate the push VLAN action. 1916 * 1917 * @param[in] dev 1918 * Pointer to the rte_eth_dev structure. 1919 * @param[in] action_flags 1920 * Holds the actions detected until now. 1921 * @param[in] item_flags 1922 * The items found in this flow rule. 1923 * @param[in] action 1924 * Pointer to the action structure. 1925 * @param[in] attr 1926 * Pointer to flow attributes 1927 * @param[out] error 1928 * Pointer to error structure. 1929 * 1930 * @return 1931 * 0 on success, a negative errno value otherwise and rte_errno is set. 1932 */ 1933 static int 1934 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev, 1935 uint64_t action_flags, 1936 const struct rte_flow_item_vlan *vlan_m, 1937 const struct rte_flow_action *action, 1938 const struct rte_flow_attr *attr, 1939 struct rte_flow_error *error) 1940 { 1941 const struct rte_flow_action_of_push_vlan *push_vlan = action->conf; 1942 const struct mlx5_priv *priv = dev->data->dev_private; 1943 1944 if (!attr->transfer && attr->ingress) 1945 return rte_flow_error_set(error, ENOTSUP, 1946 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 1947 NULL, 1948 "push VLAN action not supported for " 1949 "ingress"); 1950 if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) && 1951 push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ)) 1952 return rte_flow_error_set(error, EINVAL, 1953 RTE_FLOW_ERROR_TYPE_ACTION, action, 1954 "invalid vlan ethertype"); 1955 if (action_flags & MLX5_FLOW_VLAN_ACTIONS) 1956 return rte_flow_error_set(error, ENOTSUP, 1957 RTE_FLOW_ERROR_TYPE_ACTION, action, 1958 "no support for multiple VLAN " 1959 "actions"); 1960 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 1961 return rte_flow_error_set(error, EINVAL, 1962 RTE_FLOW_ERROR_TYPE_ACTION, action, 1963 "wrong action order, port_id should " 1964 "be after push VLAN"); 1965 if (!attr->transfer && priv->representor) 1966 return rte_flow_error_set(error, ENOTSUP, 1967 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1968 "push vlan action for VF representor " 1969 "not supported on NIC table"); 1970 if (vlan_m && 1971 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) && 1972 (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) != 1973 MLX5DV_FLOW_VLAN_PCP_MASK_BE && 1974 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) && 1975 !(mlx5_flow_find_action 1976 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP))) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION, action, 1979 "not full match mask on VLAN PCP and " 1980 "there is no of_set_vlan_pcp action, " 1981 "push VLAN action cannot figure out " 1982 "PCP value"); 1983 if (vlan_m && 1984 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) && 1985 (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) != 1986 MLX5DV_FLOW_VLAN_VID_MASK_BE && 1987 !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) && 1988 !(mlx5_flow_find_action 1989 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID))) 1990 return rte_flow_error_set(error, EINVAL, 1991 RTE_FLOW_ERROR_TYPE_ACTION, action, 1992 "not full match mask on VLAN VID and " 1993 "there is no of_set_vlan_vid action, " 1994 "push VLAN action cannot figure out " 1995 "VID value"); 1996 (void)attr; 1997 return 0; 1998 } 1999 2000 /** 2001 * Validate the set VLAN PCP. 2002 * 2003 * @param[in] action_flags 2004 * Holds the actions detected until now. 2005 * @param[in] actions 2006 * Pointer to the list of actions remaining in the flow rule. 2007 * @param[out] error 2008 * Pointer to error structure. 2009 * 2010 * @return 2011 * 0 on success, a negative errno value otherwise and rte_errno is set. 2012 */ 2013 static int 2014 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags, 2015 const struct rte_flow_action actions[], 2016 struct rte_flow_error *error) 2017 { 2018 const struct rte_flow_action *action = actions; 2019 const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf; 2020 2021 if (conf->vlan_pcp > 7) 2022 return rte_flow_error_set(error, EINVAL, 2023 RTE_FLOW_ERROR_TYPE_ACTION, action, 2024 "VLAN PCP value is too big"); 2025 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)) 2026 return rte_flow_error_set(error, ENOTSUP, 2027 RTE_FLOW_ERROR_TYPE_ACTION, action, 2028 "set VLAN PCP action must follow " 2029 "the push VLAN action"); 2030 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) 2031 return rte_flow_error_set(error, ENOTSUP, 2032 RTE_FLOW_ERROR_TYPE_ACTION, action, 2033 "Multiple VLAN PCP modification are " 2034 "not supported"); 2035 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2036 return rte_flow_error_set(error, EINVAL, 2037 RTE_FLOW_ERROR_TYPE_ACTION, action, 2038 "wrong action order, port_id should " 2039 "be after set VLAN PCP"); 2040 return 0; 2041 } 2042 2043 /** 2044 * Validate the set VLAN VID. 2045 * 2046 * @param[in] item_flags 2047 * Holds the items detected in this rule. 2048 * @param[in] action_flags 2049 * Holds the actions detected until now. 2050 * @param[in] actions 2051 * Pointer to the list of actions remaining in the flow rule. 2052 * @param[out] error 2053 * Pointer to error structure. 2054 * 2055 * @return 2056 * 0 on success, a negative errno value otherwise and rte_errno is set. 2057 */ 2058 static int 2059 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags, 2060 uint64_t action_flags, 2061 const struct rte_flow_action actions[], 2062 struct rte_flow_error *error) 2063 { 2064 const struct rte_flow_action *action = actions; 2065 const struct rte_flow_action_of_set_vlan_vid *conf = action->conf; 2066 2067 if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE) 2068 return rte_flow_error_set(error, EINVAL, 2069 RTE_FLOW_ERROR_TYPE_ACTION, action, 2070 "VLAN VID value is too big"); 2071 if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && 2072 !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) 2073 return rte_flow_error_set(error, ENOTSUP, 2074 RTE_FLOW_ERROR_TYPE_ACTION, action, 2075 "set VLAN VID action must follow push" 2076 " VLAN action or match on VLAN item"); 2077 if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) 2078 return rte_flow_error_set(error, ENOTSUP, 2079 RTE_FLOW_ERROR_TYPE_ACTION, action, 2080 "Multiple VLAN VID modifications are " 2081 "not supported"); 2082 if (action_flags & MLX5_FLOW_ACTION_PORT_ID) 2083 return rte_flow_error_set(error, EINVAL, 2084 RTE_FLOW_ERROR_TYPE_ACTION, action, 2085 "wrong action order, port_id should " 2086 "be after set VLAN VID"); 2087 return 0; 2088 } 2089 2090 /* 2091 * Validate the FLAG action. 2092 * 2093 * @param[in] dev 2094 * Pointer to the rte_eth_dev structure. 2095 * @param[in] action_flags 2096 * Holds the actions detected until now. 2097 * @param[in] attr 2098 * Pointer to flow attributes 2099 * @param[out] error 2100 * Pointer to error structure. 2101 * 2102 * @return 2103 * 0 on success, a negative errno value otherwise and rte_errno is set. 2104 */ 2105 static int 2106 flow_dv_validate_action_flag(struct rte_eth_dev *dev, 2107 uint64_t action_flags, 2108 const struct rte_flow_attr *attr, 2109 struct rte_flow_error *error) 2110 { 2111 struct mlx5_priv *priv = dev->data->dev_private; 2112 struct mlx5_dev_config *config = &priv->config; 2113 int ret; 2114 2115 /* Fall back if no extended metadata register support. */ 2116 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2117 return mlx5_flow_validate_action_flag(action_flags, attr, 2118 error); 2119 /* Extensive metadata mode requires registers. */ 2120 if (!mlx5_flow_ext_mreg_supported(dev)) 2121 return rte_flow_error_set(error, ENOTSUP, 2122 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2123 "no metadata registers " 2124 "to support flag action"); 2125 if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT)) 2126 return rte_flow_error_set(error, ENOTSUP, 2127 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2128 "extended metadata register" 2129 " isn't available"); 2130 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2131 if (ret < 0) 2132 return ret; 2133 MLX5_ASSERT(ret > 0); 2134 if (action_flags & MLX5_FLOW_ACTION_MARK) 2135 return rte_flow_error_set(error, EINVAL, 2136 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2137 "can't mark and flag in same flow"); 2138 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2139 return rte_flow_error_set(error, EINVAL, 2140 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2141 "can't have 2 flag" 2142 " actions in same flow"); 2143 return 0; 2144 } 2145 2146 /** 2147 * Validate MARK action. 2148 * 2149 * @param[in] dev 2150 * Pointer to the rte_eth_dev structure. 2151 * @param[in] action 2152 * Pointer to action. 2153 * @param[in] action_flags 2154 * Holds the actions detected until now. 2155 * @param[in] attr 2156 * Pointer to flow attributes 2157 * @param[out] error 2158 * Pointer to error structure. 2159 * 2160 * @return 2161 * 0 on success, a negative errno value otherwise and rte_errno is set. 2162 */ 2163 static int 2164 flow_dv_validate_action_mark(struct rte_eth_dev *dev, 2165 const struct rte_flow_action *action, 2166 uint64_t action_flags, 2167 const struct rte_flow_attr *attr, 2168 struct rte_flow_error *error) 2169 { 2170 struct mlx5_priv *priv = dev->data->dev_private; 2171 struct mlx5_dev_config *config = &priv->config; 2172 const struct rte_flow_action_mark *mark = action->conf; 2173 int ret; 2174 2175 /* Fall back if no extended metadata register support. */ 2176 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY) 2177 return mlx5_flow_validate_action_mark(action, action_flags, 2178 attr, error); 2179 /* Extensive metadata mode requires registers. */ 2180 if (!mlx5_flow_ext_mreg_supported(dev)) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2183 "no metadata registers " 2184 "to support mark action"); 2185 if (!priv->sh->dv_mark_mask) 2186 return rte_flow_error_set(error, ENOTSUP, 2187 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2188 "extended metadata register" 2189 " isn't available"); 2190 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 2191 if (ret < 0) 2192 return ret; 2193 MLX5_ASSERT(ret > 0); 2194 if (!mark) 2195 return rte_flow_error_set(error, EINVAL, 2196 RTE_FLOW_ERROR_TYPE_ACTION, action, 2197 "configuration cannot be null"); 2198 if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask)) 2199 return rte_flow_error_set(error, EINVAL, 2200 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2201 &mark->id, 2202 "mark id exceeds the limit"); 2203 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2204 return rte_flow_error_set(error, EINVAL, 2205 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2206 "can't flag and mark in same flow"); 2207 if (action_flags & MLX5_FLOW_ACTION_MARK) 2208 return rte_flow_error_set(error, EINVAL, 2209 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2210 "can't have 2 mark actions in same" 2211 " flow"); 2212 return 0; 2213 } 2214 2215 /** 2216 * Validate SET_META action. 2217 * 2218 * @param[in] dev 2219 * Pointer to the rte_eth_dev structure. 2220 * @param[in] action 2221 * Pointer to the action structure. 2222 * @param[in] action_flags 2223 * Holds the actions detected until now. 2224 * @param[in] attr 2225 * Pointer to flow attributes 2226 * @param[out] error 2227 * Pointer to error structure. 2228 * 2229 * @return 2230 * 0 on success, a negative errno value otherwise and rte_errno is set. 2231 */ 2232 static int 2233 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev, 2234 const struct rte_flow_action *action, 2235 uint64_t action_flags __rte_unused, 2236 const struct rte_flow_attr *attr, 2237 struct rte_flow_error *error) 2238 { 2239 const struct rte_flow_action_set_meta *conf; 2240 uint32_t nic_mask = UINT32_MAX; 2241 int reg; 2242 2243 if (!mlx5_flow_ext_mreg_supported(dev)) 2244 return rte_flow_error_set(error, ENOTSUP, 2245 RTE_FLOW_ERROR_TYPE_ACTION, action, 2246 "extended metadata register" 2247 " isn't supported"); 2248 reg = flow_dv_get_metadata_reg(dev, attr, error); 2249 if (reg < 0) 2250 return reg; 2251 if (reg != REG_A && reg != REG_B) { 2252 struct mlx5_priv *priv = dev->data->dev_private; 2253 2254 nic_mask = priv->sh->dv_meta_mask; 2255 } 2256 if (!(action->conf)) 2257 return rte_flow_error_set(error, EINVAL, 2258 RTE_FLOW_ERROR_TYPE_ACTION, action, 2259 "configuration cannot be null"); 2260 conf = (const struct rte_flow_action_set_meta *)action->conf; 2261 if (!conf->mask) 2262 return rte_flow_error_set(error, EINVAL, 2263 RTE_FLOW_ERROR_TYPE_ACTION, action, 2264 "zero mask doesn't have any effect"); 2265 if (conf->mask & ~nic_mask) 2266 return rte_flow_error_set(error, EINVAL, 2267 RTE_FLOW_ERROR_TYPE_ACTION, action, 2268 "meta data must be within reg C0"); 2269 return 0; 2270 } 2271 2272 /** 2273 * Validate SET_TAG action. 2274 * 2275 * @param[in] dev 2276 * Pointer to the rte_eth_dev structure. 2277 * @param[in] action 2278 * Pointer to the action structure. 2279 * @param[in] action_flags 2280 * Holds the actions detected until now. 2281 * @param[in] attr 2282 * Pointer to flow attributes 2283 * @param[out] error 2284 * Pointer to error structure. 2285 * 2286 * @return 2287 * 0 on success, a negative errno value otherwise and rte_errno is set. 2288 */ 2289 static int 2290 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev, 2291 const struct rte_flow_action *action, 2292 uint64_t action_flags, 2293 const struct rte_flow_attr *attr, 2294 struct rte_flow_error *error) 2295 { 2296 const struct rte_flow_action_set_tag *conf; 2297 const uint64_t terminal_action_flags = 2298 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | 2299 MLX5_FLOW_ACTION_RSS; 2300 int ret; 2301 2302 if (!mlx5_flow_ext_mreg_supported(dev)) 2303 return rte_flow_error_set(error, ENOTSUP, 2304 RTE_FLOW_ERROR_TYPE_ACTION, action, 2305 "extensive metadata register" 2306 " isn't supported"); 2307 if (!(action->conf)) 2308 return rte_flow_error_set(error, EINVAL, 2309 RTE_FLOW_ERROR_TYPE_ACTION, action, 2310 "configuration cannot be null"); 2311 conf = (const struct rte_flow_action_set_tag *)action->conf; 2312 if (!conf->mask) 2313 return rte_flow_error_set(error, EINVAL, 2314 RTE_FLOW_ERROR_TYPE_ACTION, action, 2315 "zero mask doesn't have any effect"); 2316 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error); 2317 if (ret < 0) 2318 return ret; 2319 if (!attr->transfer && attr->ingress && 2320 (action_flags & terminal_action_flags)) 2321 return rte_flow_error_set(error, EINVAL, 2322 RTE_FLOW_ERROR_TYPE_ACTION, action, 2323 "set_tag has no effect" 2324 " with terminal actions"); 2325 return 0; 2326 } 2327 2328 /** 2329 * Validate count action. 2330 * 2331 * @param[in] dev 2332 * Pointer to rte_eth_dev structure. 2333 * @param[out] error 2334 * Pointer to error structure. 2335 * 2336 * @return 2337 * 0 on success, a negative errno value otherwise and rte_errno is set. 2338 */ 2339 static int 2340 flow_dv_validate_action_count(struct rte_eth_dev *dev, 2341 struct rte_flow_error *error) 2342 { 2343 struct mlx5_priv *priv = dev->data->dev_private; 2344 2345 if (!priv->config.devx) 2346 goto notsup_err; 2347 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS 2348 return 0; 2349 #endif 2350 notsup_err: 2351 return rte_flow_error_set 2352 (error, ENOTSUP, 2353 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2354 NULL, 2355 "count action not supported"); 2356 } 2357 2358 /** 2359 * Validate the L2 encap action. 2360 * 2361 * @param[in] dev 2362 * Pointer to the rte_eth_dev structure. 2363 * @param[in] action_flags 2364 * Holds the actions detected until now. 2365 * @param[in] action 2366 * Pointer to the action structure. 2367 * @param[in] attr 2368 * Pointer to flow attributes. 2369 * @param[out] error 2370 * Pointer to error structure. 2371 * 2372 * @return 2373 * 0 on success, a negative errno value otherwise and rte_errno is set. 2374 */ 2375 static int 2376 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev, 2377 uint64_t action_flags, 2378 const struct rte_flow_action *action, 2379 const struct rte_flow_attr *attr, 2380 struct rte_flow_error *error) 2381 { 2382 const struct mlx5_priv *priv = dev->data->dev_private; 2383 2384 if (!(action->conf)) 2385 return rte_flow_error_set(error, EINVAL, 2386 RTE_FLOW_ERROR_TYPE_ACTION, action, 2387 "configuration cannot be null"); 2388 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 2389 return rte_flow_error_set(error, EINVAL, 2390 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2391 "can only have a single encap action " 2392 "in a flow"); 2393 if (!attr->transfer && priv->representor) 2394 return rte_flow_error_set(error, ENOTSUP, 2395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2396 "encap action for VF representor " 2397 "not supported on NIC table"); 2398 return 0; 2399 } 2400 2401 /** 2402 * Validate a decap action. 2403 * 2404 * @param[in] dev 2405 * Pointer to the rte_eth_dev structure. 2406 * @param[in] action_flags 2407 * Holds the actions detected until now. 2408 * @param[in] attr 2409 * Pointer to flow attributes 2410 * @param[out] error 2411 * Pointer to error structure. 2412 * 2413 * @return 2414 * 0 on success, a negative errno value otherwise and rte_errno is set. 2415 */ 2416 static int 2417 flow_dv_validate_action_decap(struct rte_eth_dev *dev, 2418 uint64_t action_flags, 2419 const struct rte_flow_attr *attr, 2420 struct rte_flow_error *error) 2421 { 2422 const struct mlx5_priv *priv = dev->data->dev_private; 2423 2424 if (action_flags & MLX5_FLOW_XCAP_ACTIONS) 2425 return rte_flow_error_set(error, ENOTSUP, 2426 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2427 action_flags & 2428 MLX5_FLOW_ACTION_DECAP ? "can only " 2429 "have a single decap action" : "decap " 2430 "after encap is not supported"); 2431 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 2432 return rte_flow_error_set(error, EINVAL, 2433 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2434 "can't have decap action after" 2435 " modify action"); 2436 if (attr->egress) 2437 return rte_flow_error_set(error, ENOTSUP, 2438 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 2439 NULL, 2440 "decap action not supported for " 2441 "egress"); 2442 if (!attr->transfer && priv->representor) 2443 return rte_flow_error_set(error, ENOTSUP, 2444 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2445 "decap action for VF representor " 2446 "not supported on NIC table"); 2447 return 0; 2448 } 2449 2450 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,}; 2451 2452 /** 2453 * Validate the raw encap and decap actions. 2454 * 2455 * @param[in] dev 2456 * Pointer to the rte_eth_dev structure. 2457 * @param[in] decap 2458 * Pointer to the decap action. 2459 * @param[in] encap 2460 * Pointer to the encap action. 2461 * @param[in] attr 2462 * Pointer to flow attributes 2463 * @param[in/out] action_flags 2464 * Holds the actions detected until now. 2465 * @param[out] actions_n 2466 * pointer to the number of actions counter. 2467 * @param[out] error 2468 * Pointer to error structure. 2469 * 2470 * @return 2471 * 0 on success, a negative errno value otherwise and rte_errno is set. 2472 */ 2473 static int 2474 flow_dv_validate_action_raw_encap_decap 2475 (struct rte_eth_dev *dev, 2476 const struct rte_flow_action_raw_decap *decap, 2477 const struct rte_flow_action_raw_encap *encap, 2478 const struct rte_flow_attr *attr, uint64_t *action_flags, 2479 int *actions_n, struct rte_flow_error *error) 2480 { 2481 const struct mlx5_priv *priv = dev->data->dev_private; 2482 int ret; 2483 2484 if (encap && (!encap->size || !encap->data)) 2485 return rte_flow_error_set(error, EINVAL, 2486 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2487 "raw encap data cannot be empty"); 2488 if (decap && encap) { 2489 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE && 2490 encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 2491 /* L3 encap. */ 2492 decap = NULL; 2493 else if (encap->size <= 2494 MLX5_ENCAPSULATION_DECISION_SIZE && 2495 decap->size > 2496 MLX5_ENCAPSULATION_DECISION_SIZE) 2497 /* L3 decap. */ 2498 encap = NULL; 2499 else if (encap->size > 2500 MLX5_ENCAPSULATION_DECISION_SIZE && 2501 decap->size > 2502 MLX5_ENCAPSULATION_DECISION_SIZE) 2503 /* 2 L2 actions: encap and decap. */ 2504 ; 2505 else 2506 return rte_flow_error_set(error, 2507 ENOTSUP, 2508 RTE_FLOW_ERROR_TYPE_ACTION, 2509 NULL, "unsupported too small " 2510 "raw decap and too small raw " 2511 "encap combination"); 2512 } 2513 if (decap) { 2514 ret = flow_dv_validate_action_decap(dev, *action_flags, attr, 2515 error); 2516 if (ret < 0) 2517 return ret; 2518 *action_flags |= MLX5_FLOW_ACTION_DECAP; 2519 ++(*actions_n); 2520 } 2521 if (encap) { 2522 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE) 2523 return rte_flow_error_set(error, ENOTSUP, 2524 RTE_FLOW_ERROR_TYPE_ACTION, 2525 NULL, 2526 "small raw encap size"); 2527 if (*action_flags & MLX5_FLOW_ACTION_ENCAP) 2528 return rte_flow_error_set(error, EINVAL, 2529 RTE_FLOW_ERROR_TYPE_ACTION, 2530 NULL, 2531 "more than one encap action"); 2532 if (!attr->transfer && priv->representor) 2533 return rte_flow_error_set 2534 (error, ENOTSUP, 2535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2536 "encap action for VF representor " 2537 "not supported on NIC table"); 2538 *action_flags |= MLX5_FLOW_ACTION_ENCAP; 2539 ++(*actions_n); 2540 } 2541 return 0; 2542 } 2543 2544 /** 2545 * Find existing encap/decap resource or create and register a new one. 2546 * 2547 * @param[in, out] dev 2548 * Pointer to rte_eth_dev structure. 2549 * @param[in, out] resource 2550 * Pointer to encap/decap resource. 2551 * @parm[in, out] dev_flow 2552 * Pointer to the dev_flow. 2553 * @param[out] error 2554 * pointer to error structure. 2555 * 2556 * @return 2557 * 0 on success otherwise -errno and errno is set. 2558 */ 2559 static int 2560 flow_dv_encap_decap_resource_register 2561 (struct rte_eth_dev *dev, 2562 struct mlx5_flow_dv_encap_decap_resource *resource, 2563 struct mlx5_flow *dev_flow, 2564 struct rte_flow_error *error) 2565 { 2566 struct mlx5_priv *priv = dev->data->dev_private; 2567 struct mlx5_dev_ctx_shared *sh = priv->sh; 2568 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 2569 struct mlx5dv_dr_domain *domain; 2570 uint32_t idx = 0; 2571 2572 resource->flags = dev_flow->dv.group ? 0 : 1; 2573 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2574 domain = sh->fdb_domain; 2575 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2576 domain = sh->rx_domain; 2577 else 2578 domain = sh->tx_domain; 2579 /* Lookup a matching resource from cache. */ 2580 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx, 2581 cache_resource, next) { 2582 if (resource->reformat_type == cache_resource->reformat_type && 2583 resource->ft_type == cache_resource->ft_type && 2584 resource->flags == cache_resource->flags && 2585 resource->size == cache_resource->size && 2586 !memcmp((const void *)resource->buf, 2587 (const void *)cache_resource->buf, 2588 resource->size)) { 2589 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", 2590 (void *)cache_resource, 2591 rte_atomic32_read(&cache_resource->refcnt)); 2592 rte_atomic32_inc(&cache_resource->refcnt); 2593 dev_flow->handle->dvh.rix_encap_decap = idx; 2594 dev_flow->dv.encap_decap = cache_resource; 2595 return 0; 2596 } 2597 } 2598 /* Register new encap/decap resource. */ 2599 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 2600 &dev_flow->handle->dvh.rix_encap_decap); 2601 if (!cache_resource) 2602 return rte_flow_error_set(error, ENOMEM, 2603 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2604 "cannot allocate resource memory"); 2605 *cache_resource = *resource; 2606 cache_resource->verbs_action = 2607 mlx5_glue->dv_create_flow_action_packet_reformat 2608 (sh->ctx, cache_resource->reformat_type, 2609 cache_resource->ft_type, domain, cache_resource->flags, 2610 cache_resource->size, 2611 (cache_resource->size ? cache_resource->buf : NULL)); 2612 if (!cache_resource->verbs_action) { 2613 rte_free(cache_resource); 2614 return rte_flow_error_set(error, ENOMEM, 2615 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2616 NULL, "cannot create action"); 2617 } 2618 rte_atomic32_init(&cache_resource->refcnt); 2619 rte_atomic32_inc(&cache_resource->refcnt); 2620 ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps, 2621 dev_flow->handle->dvh.rix_encap_decap, cache_resource, 2622 next); 2623 dev_flow->dv.encap_decap = cache_resource; 2624 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", 2625 (void *)cache_resource, 2626 rte_atomic32_read(&cache_resource->refcnt)); 2627 return 0; 2628 } 2629 2630 /** 2631 * Find existing table jump resource or create and register a new one. 2632 * 2633 * @param[in, out] dev 2634 * Pointer to rte_eth_dev structure. 2635 * @param[in, out] tbl 2636 * Pointer to flow table resource. 2637 * @parm[in, out] dev_flow 2638 * Pointer to the dev_flow. 2639 * @param[out] error 2640 * pointer to error structure. 2641 * 2642 * @return 2643 * 0 on success otherwise -errno and errno is set. 2644 */ 2645 static int 2646 flow_dv_jump_tbl_resource_register 2647 (struct rte_eth_dev *dev __rte_unused, 2648 struct mlx5_flow_tbl_resource *tbl, 2649 struct mlx5_flow *dev_flow, 2650 struct rte_flow_error *error) 2651 { 2652 struct mlx5_flow_tbl_data_entry *tbl_data = 2653 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 2654 int cnt; 2655 2656 MLX5_ASSERT(tbl); 2657 cnt = rte_atomic32_read(&tbl_data->jump.refcnt); 2658 if (!cnt) { 2659 tbl_data->jump.action = 2660 mlx5_glue->dr_create_flow_action_dest_flow_tbl 2661 (tbl->obj); 2662 if (!tbl_data->jump.action) 2663 return rte_flow_error_set(error, ENOMEM, 2664 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2665 NULL, "cannot create jump action"); 2666 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", 2667 (void *)&tbl_data->jump, cnt); 2668 } else { 2669 /* old jump should not make the table ref++. */ 2670 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 2671 MLX5_ASSERT(tbl_data->jump.action); 2672 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++", 2673 (void *)&tbl_data->jump, cnt); 2674 } 2675 rte_atomic32_inc(&tbl_data->jump.refcnt); 2676 dev_flow->handle->rix_jump = tbl_data->idx; 2677 dev_flow->dv.jump = &tbl_data->jump; 2678 return 0; 2679 } 2680 2681 /** 2682 * Find existing default miss resource or create and register a new one. 2683 * 2684 * @param[in, out] dev 2685 * Pointer to rte_eth_dev structure. 2686 * @param[out] error 2687 * pointer to error structure. 2688 * 2689 * @return 2690 * 0 on success otherwise -errno and errno is set. 2691 */ 2692 static int 2693 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev, 2694 struct rte_flow_error *error) 2695 { 2696 struct mlx5_priv *priv = dev->data->dev_private; 2697 struct mlx5_dev_ctx_shared *sh = priv->sh; 2698 struct mlx5_flow_default_miss_resource *cache_resource = 2699 &sh->default_miss; 2700 int cnt = rte_atomic32_read(&cache_resource->refcnt); 2701 2702 if (!cnt) { 2703 MLX5_ASSERT(cache_resource->action); 2704 cache_resource->action = 2705 mlx5_glue->dr_create_flow_action_default_miss(); 2706 if (!cache_resource->action) 2707 return rte_flow_error_set(error, ENOMEM, 2708 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2709 "cannot create default miss action"); 2710 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++", 2711 (void *)cache_resource->action, cnt); 2712 } 2713 rte_atomic32_inc(&cache_resource->refcnt); 2714 return 0; 2715 } 2716 2717 /** 2718 * Find existing table port ID resource or create and register a new one. 2719 * 2720 * @param[in, out] dev 2721 * Pointer to rte_eth_dev structure. 2722 * @param[in, out] resource 2723 * Pointer to port ID action resource. 2724 * @parm[in, out] dev_flow 2725 * Pointer to the dev_flow. 2726 * @param[out] error 2727 * pointer to error structure. 2728 * 2729 * @return 2730 * 0 on success otherwise -errno and errno is set. 2731 */ 2732 static int 2733 flow_dv_port_id_action_resource_register 2734 (struct rte_eth_dev *dev, 2735 struct mlx5_flow_dv_port_id_action_resource *resource, 2736 struct mlx5_flow *dev_flow, 2737 struct rte_flow_error *error) 2738 { 2739 struct mlx5_priv *priv = dev->data->dev_private; 2740 struct mlx5_dev_ctx_shared *sh = priv->sh; 2741 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 2742 uint32_t idx = 0; 2743 2744 /* Lookup a matching resource from cache. */ 2745 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list, 2746 idx, cache_resource, next) { 2747 if (resource->port_id == cache_resource->port_id) { 2748 DRV_LOG(DEBUG, "port id action resource resource %p: " 2749 "refcnt %d++", 2750 (void *)cache_resource, 2751 rte_atomic32_read(&cache_resource->refcnt)); 2752 rte_atomic32_inc(&cache_resource->refcnt); 2753 dev_flow->handle->rix_port_id_action = idx; 2754 dev_flow->dv.port_id_action = cache_resource; 2755 return 0; 2756 } 2757 } 2758 /* Register new port id action resource. */ 2759 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], 2760 &dev_flow->handle->rix_port_id_action); 2761 if (!cache_resource) 2762 return rte_flow_error_set(error, ENOMEM, 2763 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2764 "cannot allocate resource memory"); 2765 *cache_resource = *resource; 2766 /* 2767 * Depending on rdma_core version the glue routine calls 2768 * either mlx5dv_dr_action_create_dest_ib_port(domain, dev_port) 2769 * or mlx5dv_dr_action_create_dest_vport(domain, vport_id). 2770 */ 2771 cache_resource->action = 2772 mlx5_glue->dr_create_flow_action_dest_port 2773 (priv->sh->fdb_domain, resource->port_id); 2774 if (!cache_resource->action) { 2775 rte_free(cache_resource); 2776 return rte_flow_error_set(error, ENOMEM, 2777 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2778 NULL, "cannot create action"); 2779 } 2780 rte_atomic32_init(&cache_resource->refcnt); 2781 rte_atomic32_inc(&cache_resource->refcnt); 2782 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list, 2783 dev_flow->handle->rix_port_id_action, cache_resource, 2784 next); 2785 dev_flow->dv.port_id_action = cache_resource; 2786 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", 2787 (void *)cache_resource, 2788 rte_atomic32_read(&cache_resource->refcnt)); 2789 return 0; 2790 } 2791 2792 /** 2793 * Find existing push vlan resource or create and register a new one. 2794 * 2795 * @param [in, out] dev 2796 * Pointer to rte_eth_dev structure. 2797 * @param[in, out] resource 2798 * Pointer to port ID action resource. 2799 * @parm[in, out] dev_flow 2800 * Pointer to the dev_flow. 2801 * @param[out] error 2802 * pointer to error structure. 2803 * 2804 * @return 2805 * 0 on success otherwise -errno and errno is set. 2806 */ 2807 static int 2808 flow_dv_push_vlan_action_resource_register 2809 (struct rte_eth_dev *dev, 2810 struct mlx5_flow_dv_push_vlan_action_resource *resource, 2811 struct mlx5_flow *dev_flow, 2812 struct rte_flow_error *error) 2813 { 2814 struct mlx5_priv *priv = dev->data->dev_private; 2815 struct mlx5_dev_ctx_shared *sh = priv->sh; 2816 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 2817 struct mlx5dv_dr_domain *domain; 2818 uint32_t idx = 0; 2819 2820 /* Lookup a matching resource from cache. */ 2821 ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2822 sh->push_vlan_action_list, idx, cache_resource, next) { 2823 if (resource->vlan_tag == cache_resource->vlan_tag && 2824 resource->ft_type == cache_resource->ft_type) { 2825 DRV_LOG(DEBUG, "push-VLAN action resource resource %p: " 2826 "refcnt %d++", 2827 (void *)cache_resource, 2828 rte_atomic32_read(&cache_resource->refcnt)); 2829 rte_atomic32_inc(&cache_resource->refcnt); 2830 dev_flow->handle->dvh.rix_push_vlan = idx; 2831 dev_flow->dv.push_vlan_res = cache_resource; 2832 return 0; 2833 } 2834 } 2835 /* Register new push_vlan action resource. */ 2836 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2837 &dev_flow->handle->dvh.rix_push_vlan); 2838 if (!cache_resource) 2839 return rte_flow_error_set(error, ENOMEM, 2840 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2841 "cannot allocate resource memory"); 2842 *cache_resource = *resource; 2843 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2844 domain = sh->fdb_domain; 2845 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 2846 domain = sh->rx_domain; 2847 else 2848 domain = sh->tx_domain; 2849 cache_resource->action = 2850 mlx5_glue->dr_create_flow_action_push_vlan(domain, 2851 resource->vlan_tag); 2852 if (!cache_resource->action) { 2853 rte_free(cache_resource); 2854 return rte_flow_error_set(error, ENOMEM, 2855 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2856 NULL, "cannot create action"); 2857 } 2858 rte_atomic32_init(&cache_resource->refcnt); 2859 rte_atomic32_inc(&cache_resource->refcnt); 2860 ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN], 2861 &sh->push_vlan_action_list, 2862 dev_flow->handle->dvh.rix_push_vlan, 2863 cache_resource, next); 2864 dev_flow->dv.push_vlan_res = cache_resource; 2865 DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++", 2866 (void *)cache_resource, 2867 rte_atomic32_read(&cache_resource->refcnt)); 2868 return 0; 2869 } 2870 /** 2871 * Get the size of specific rte_flow_item_type 2872 * 2873 * @param[in] item_type 2874 * Tested rte_flow_item_type. 2875 * 2876 * @return 2877 * sizeof struct item_type, 0 if void or irrelevant. 2878 */ 2879 static size_t 2880 flow_dv_get_item_len(const enum rte_flow_item_type item_type) 2881 { 2882 size_t retval; 2883 2884 switch (item_type) { 2885 case RTE_FLOW_ITEM_TYPE_ETH: 2886 retval = sizeof(struct rte_flow_item_eth); 2887 break; 2888 case RTE_FLOW_ITEM_TYPE_VLAN: 2889 retval = sizeof(struct rte_flow_item_vlan); 2890 break; 2891 case RTE_FLOW_ITEM_TYPE_IPV4: 2892 retval = sizeof(struct rte_flow_item_ipv4); 2893 break; 2894 case RTE_FLOW_ITEM_TYPE_IPV6: 2895 retval = sizeof(struct rte_flow_item_ipv6); 2896 break; 2897 case RTE_FLOW_ITEM_TYPE_UDP: 2898 retval = sizeof(struct rte_flow_item_udp); 2899 break; 2900 case RTE_FLOW_ITEM_TYPE_TCP: 2901 retval = sizeof(struct rte_flow_item_tcp); 2902 break; 2903 case RTE_FLOW_ITEM_TYPE_VXLAN: 2904 retval = sizeof(struct rte_flow_item_vxlan); 2905 break; 2906 case RTE_FLOW_ITEM_TYPE_GRE: 2907 retval = sizeof(struct rte_flow_item_gre); 2908 break; 2909 case RTE_FLOW_ITEM_TYPE_NVGRE: 2910 retval = sizeof(struct rte_flow_item_nvgre); 2911 break; 2912 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 2913 retval = sizeof(struct rte_flow_item_vxlan_gpe); 2914 break; 2915 case RTE_FLOW_ITEM_TYPE_MPLS: 2916 retval = sizeof(struct rte_flow_item_mpls); 2917 break; 2918 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 2919 default: 2920 retval = 0; 2921 break; 2922 } 2923 return retval; 2924 } 2925 2926 #define MLX5_ENCAP_IPV4_VERSION 0x40 2927 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 2928 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 2929 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 2930 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 2931 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 2932 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 2933 2934 /** 2935 * Convert the encap action data from list of rte_flow_item to raw buffer 2936 * 2937 * @param[in] items 2938 * Pointer to rte_flow_item objects list. 2939 * @param[out] buf 2940 * Pointer to the output buffer. 2941 * @param[out] size 2942 * Pointer to the output buffer size. 2943 * @param[out] error 2944 * Pointer to the error structure. 2945 * 2946 * @return 2947 * 0 on success, a negative errno value otherwise and rte_errno is set. 2948 */ 2949 static int 2950 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 2951 size_t *size, struct rte_flow_error *error) 2952 { 2953 struct rte_ether_hdr *eth = NULL; 2954 struct rte_vlan_hdr *vlan = NULL; 2955 struct rte_ipv4_hdr *ipv4 = NULL; 2956 struct rte_ipv6_hdr *ipv6 = NULL; 2957 struct rte_udp_hdr *udp = NULL; 2958 struct rte_vxlan_hdr *vxlan = NULL; 2959 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; 2960 struct rte_gre_hdr *gre = NULL; 2961 size_t len; 2962 size_t temp_size = 0; 2963 2964 if (!items) 2965 return rte_flow_error_set(error, EINVAL, 2966 RTE_FLOW_ERROR_TYPE_ACTION, 2967 NULL, "invalid empty data"); 2968 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2969 len = flow_dv_get_item_len(items->type); 2970 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 2971 return rte_flow_error_set(error, EINVAL, 2972 RTE_FLOW_ERROR_TYPE_ACTION, 2973 (void *)items->type, 2974 "items total size is too big" 2975 " for encap action"); 2976 rte_memcpy((void *)&buf[temp_size], items->spec, len); 2977 switch (items->type) { 2978 case RTE_FLOW_ITEM_TYPE_ETH: 2979 eth = (struct rte_ether_hdr *)&buf[temp_size]; 2980 break; 2981 case RTE_FLOW_ITEM_TYPE_VLAN: 2982 vlan = (struct rte_vlan_hdr *)&buf[temp_size]; 2983 if (!eth) 2984 return rte_flow_error_set(error, EINVAL, 2985 RTE_FLOW_ERROR_TYPE_ACTION, 2986 (void *)items->type, 2987 "eth header not found"); 2988 if (!eth->ether_type) 2989 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); 2990 break; 2991 case RTE_FLOW_ITEM_TYPE_IPV4: 2992 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; 2993 if (!vlan && !eth) 2994 return rte_flow_error_set(error, EINVAL, 2995 RTE_FLOW_ERROR_TYPE_ACTION, 2996 (void *)items->type, 2997 "neither eth nor vlan" 2998 " header found"); 2999 if (vlan && !vlan->eth_proto) 3000 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); 3001 else if (eth && !eth->ether_type) 3002 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); 3003 if (!ipv4->version_ihl) 3004 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 3005 MLX5_ENCAP_IPV4_IHL_MIN; 3006 if (!ipv4->time_to_live) 3007 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 3008 break; 3009 case RTE_FLOW_ITEM_TYPE_IPV6: 3010 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; 3011 if (!vlan && !eth) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ACTION, 3014 (void *)items->type, 3015 "neither eth nor vlan" 3016 " header found"); 3017 if (vlan && !vlan->eth_proto) 3018 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3019 else if (eth && !eth->ether_type) 3020 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); 3021 if (!ipv6->vtc_flow) 3022 ipv6->vtc_flow = 3023 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 3024 if (!ipv6->hop_limits) 3025 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 3026 break; 3027 case RTE_FLOW_ITEM_TYPE_UDP: 3028 udp = (struct rte_udp_hdr *)&buf[temp_size]; 3029 if (!ipv4 && !ipv6) 3030 return rte_flow_error_set(error, EINVAL, 3031 RTE_FLOW_ERROR_TYPE_ACTION, 3032 (void *)items->type, 3033 "ip header not found"); 3034 if (ipv4 && !ipv4->next_proto_id) 3035 ipv4->next_proto_id = IPPROTO_UDP; 3036 else if (ipv6 && !ipv6->proto) 3037 ipv6->proto = IPPROTO_UDP; 3038 break; 3039 case RTE_FLOW_ITEM_TYPE_VXLAN: 3040 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; 3041 if (!udp) 3042 return rte_flow_error_set(error, EINVAL, 3043 RTE_FLOW_ERROR_TYPE_ACTION, 3044 (void *)items->type, 3045 "udp header not found"); 3046 if (!udp->dst_port) 3047 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 3048 if (!vxlan->vx_flags) 3049 vxlan->vx_flags = 3050 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 3051 break; 3052 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 3053 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; 3054 if (!udp) 3055 return rte_flow_error_set(error, EINVAL, 3056 RTE_FLOW_ERROR_TYPE_ACTION, 3057 (void *)items->type, 3058 "udp header not found"); 3059 if (!vxlan_gpe->proto) 3060 return rte_flow_error_set(error, EINVAL, 3061 RTE_FLOW_ERROR_TYPE_ACTION, 3062 (void *)items->type, 3063 "next protocol not found"); 3064 if (!udp->dst_port) 3065 udp->dst_port = 3066 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 3067 if (!vxlan_gpe->vx_flags) 3068 vxlan_gpe->vx_flags = 3069 MLX5_ENCAP_VXLAN_GPE_FLAGS; 3070 break; 3071 case RTE_FLOW_ITEM_TYPE_GRE: 3072 case RTE_FLOW_ITEM_TYPE_NVGRE: 3073 gre = (struct rte_gre_hdr *)&buf[temp_size]; 3074 if (!gre->proto) 3075 return rte_flow_error_set(error, EINVAL, 3076 RTE_FLOW_ERROR_TYPE_ACTION, 3077 (void *)items->type, 3078 "next protocol not found"); 3079 if (!ipv4 && !ipv6) 3080 return rte_flow_error_set(error, EINVAL, 3081 RTE_FLOW_ERROR_TYPE_ACTION, 3082 (void *)items->type, 3083 "ip header not found"); 3084 if (ipv4 && !ipv4->next_proto_id) 3085 ipv4->next_proto_id = IPPROTO_GRE; 3086 else if (ipv6 && !ipv6->proto) 3087 ipv6->proto = IPPROTO_GRE; 3088 break; 3089 case RTE_FLOW_ITEM_TYPE_VOID: 3090 break; 3091 default: 3092 return rte_flow_error_set(error, EINVAL, 3093 RTE_FLOW_ERROR_TYPE_ACTION, 3094 (void *)items->type, 3095 "unsupported item type"); 3096 break; 3097 } 3098 temp_size += len; 3099 } 3100 *size = temp_size; 3101 return 0; 3102 } 3103 3104 static int 3105 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) 3106 { 3107 struct rte_ether_hdr *eth = NULL; 3108 struct rte_vlan_hdr *vlan = NULL; 3109 struct rte_ipv6_hdr *ipv6 = NULL; 3110 struct rte_udp_hdr *udp = NULL; 3111 char *next_hdr; 3112 uint16_t proto; 3113 3114 eth = (struct rte_ether_hdr *)data; 3115 next_hdr = (char *)(eth + 1); 3116 proto = RTE_BE16(eth->ether_type); 3117 3118 /* VLAN skipping */ 3119 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { 3120 vlan = (struct rte_vlan_hdr *)next_hdr; 3121 proto = RTE_BE16(vlan->eth_proto); 3122 next_hdr += sizeof(struct rte_vlan_hdr); 3123 } 3124 3125 /* HW calculates IPv4 csum. no need to proceed */ 3126 if (proto == RTE_ETHER_TYPE_IPV4) 3127 return 0; 3128 3129 /* non IPv4/IPv6 header. not supported */ 3130 if (proto != RTE_ETHER_TYPE_IPV6) { 3131 return rte_flow_error_set(error, ENOTSUP, 3132 RTE_FLOW_ERROR_TYPE_ACTION, 3133 NULL, "Cannot offload non IPv4/IPv6"); 3134 } 3135 3136 ipv6 = (struct rte_ipv6_hdr *)next_hdr; 3137 3138 /* ignore non UDP */ 3139 if (ipv6->proto != IPPROTO_UDP) 3140 return 0; 3141 3142 udp = (struct rte_udp_hdr *)(ipv6 + 1); 3143 udp->dgram_cksum = 0; 3144 3145 return 0; 3146 } 3147 3148 /** 3149 * Convert L2 encap action to DV specification. 3150 * 3151 * @param[in] dev 3152 * Pointer to rte_eth_dev structure. 3153 * @param[in] action 3154 * Pointer to action structure. 3155 * @param[in, out] dev_flow 3156 * Pointer to the mlx5_flow. 3157 * @param[in] transfer 3158 * Mark if the flow is E-Switch flow. 3159 * @param[out] error 3160 * Pointer to the error structure. 3161 * 3162 * @return 3163 * 0 on success, a negative errno value otherwise and rte_errno is set. 3164 */ 3165 static int 3166 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 3167 const struct rte_flow_action *action, 3168 struct mlx5_flow *dev_flow, 3169 uint8_t transfer, 3170 struct rte_flow_error *error) 3171 { 3172 const struct rte_flow_item *encap_data; 3173 const struct rte_flow_action_raw_encap *raw_encap_data; 3174 struct mlx5_flow_dv_encap_decap_resource res = { 3175 .reformat_type = 3176 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 3177 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3178 MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 3179 }; 3180 3181 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 3182 raw_encap_data = 3183 (const struct rte_flow_action_raw_encap *)action->conf; 3184 res.size = raw_encap_data->size; 3185 memcpy(res.buf, raw_encap_data->data, res.size); 3186 } else { 3187 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 3188 encap_data = 3189 ((const struct rte_flow_action_vxlan_encap *) 3190 action->conf)->definition; 3191 else 3192 encap_data = 3193 ((const struct rte_flow_action_nvgre_encap *) 3194 action->conf)->definition; 3195 if (flow_dv_convert_encap_data(encap_data, res.buf, 3196 &res.size, error)) 3197 return -rte_errno; 3198 } 3199 if (flow_dv_zero_encap_udp_csum(res.buf, error)) 3200 return -rte_errno; 3201 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3202 return rte_flow_error_set(error, EINVAL, 3203 RTE_FLOW_ERROR_TYPE_ACTION, 3204 NULL, "can't create L2 encap action"); 3205 return 0; 3206 } 3207 3208 /** 3209 * Convert L2 decap action to DV specification. 3210 * 3211 * @param[in] dev 3212 * Pointer to rte_eth_dev structure. 3213 * @param[in, out] dev_flow 3214 * Pointer to the mlx5_flow. 3215 * @param[in] transfer 3216 * Mark if the flow is E-Switch flow. 3217 * @param[out] error 3218 * Pointer to the error structure. 3219 * 3220 * @return 3221 * 0 on success, a negative errno value otherwise and rte_errno is set. 3222 */ 3223 static int 3224 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 3225 struct mlx5_flow *dev_flow, 3226 uint8_t transfer, 3227 struct rte_flow_error *error) 3228 { 3229 struct mlx5_flow_dv_encap_decap_resource res = { 3230 .size = 0, 3231 .reformat_type = 3232 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 3233 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 3234 MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 3235 }; 3236 3237 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3238 return rte_flow_error_set(error, EINVAL, 3239 RTE_FLOW_ERROR_TYPE_ACTION, 3240 NULL, "can't create L2 decap action"); 3241 return 0; 3242 } 3243 3244 /** 3245 * Convert raw decap/encap (L3 tunnel) action to DV specification. 3246 * 3247 * @param[in] dev 3248 * Pointer to rte_eth_dev structure. 3249 * @param[in] action 3250 * Pointer to action structure. 3251 * @param[in, out] dev_flow 3252 * Pointer to the mlx5_flow. 3253 * @param[in] attr 3254 * Pointer to the flow attributes. 3255 * @param[out] error 3256 * Pointer to the error structure. 3257 * 3258 * @return 3259 * 0 on success, a negative errno value otherwise and rte_errno is set. 3260 */ 3261 static int 3262 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 3263 const struct rte_flow_action *action, 3264 struct mlx5_flow *dev_flow, 3265 const struct rte_flow_attr *attr, 3266 struct rte_flow_error *error) 3267 { 3268 const struct rte_flow_action_raw_encap *encap_data; 3269 struct mlx5_flow_dv_encap_decap_resource res; 3270 3271 memset(&res, 0, sizeof(res)); 3272 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 3273 res.size = encap_data->size; 3274 memcpy(res.buf, encap_data->data, res.size); 3275 res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ? 3276 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 : 3277 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 3278 if (attr->transfer) 3279 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3280 else 3281 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3282 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3283 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 3284 return rte_flow_error_set(error, EINVAL, 3285 RTE_FLOW_ERROR_TYPE_ACTION, 3286 NULL, "can't create encap action"); 3287 return 0; 3288 } 3289 3290 /** 3291 * Create action push VLAN. 3292 * 3293 * @param[in] dev 3294 * Pointer to rte_eth_dev structure. 3295 * @param[in] attr 3296 * Pointer to the flow attributes. 3297 * @param[in] vlan 3298 * Pointer to the vlan to push to the Ethernet header. 3299 * @param[in, out] dev_flow 3300 * Pointer to the mlx5_flow. 3301 * @param[out] error 3302 * Pointer to the error structure. 3303 * 3304 * @return 3305 * 0 on success, a negative errno value otherwise and rte_errno is set. 3306 */ 3307 static int 3308 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev, 3309 const struct rte_flow_attr *attr, 3310 const struct rte_vlan_hdr *vlan, 3311 struct mlx5_flow *dev_flow, 3312 struct rte_flow_error *error) 3313 { 3314 struct mlx5_flow_dv_push_vlan_action_resource res; 3315 3316 memset(&res, 0, sizeof(res)); 3317 res.vlan_tag = 3318 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 | 3319 vlan->vlan_tci); 3320 if (attr->transfer) 3321 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 3322 else 3323 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 3324 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 3325 return flow_dv_push_vlan_action_resource_register 3326 (dev, &res, dev_flow, error); 3327 } 3328 3329 /** 3330 * Validate the modify-header actions. 3331 * 3332 * @param[in] action_flags 3333 * Holds the actions detected until now. 3334 * @param[in] action 3335 * Pointer to the modify action. 3336 * @param[out] error 3337 * Pointer to error structure. 3338 * 3339 * @return 3340 * 0 on success, a negative errno value otherwise and rte_errno is set. 3341 */ 3342 static int 3343 flow_dv_validate_action_modify_hdr(const uint64_t action_flags, 3344 const struct rte_flow_action *action, 3345 struct rte_flow_error *error) 3346 { 3347 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) 3348 return rte_flow_error_set(error, EINVAL, 3349 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3350 NULL, "action configuration not set"); 3351 if (action_flags & MLX5_FLOW_ACTION_ENCAP) 3352 return rte_flow_error_set(error, EINVAL, 3353 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3354 "can't have encap action before" 3355 " modify action"); 3356 return 0; 3357 } 3358 3359 /** 3360 * Validate the modify-header MAC address actions. 3361 * 3362 * @param[in] action_flags 3363 * Holds the actions detected until now. 3364 * @param[in] action 3365 * Pointer to the modify action. 3366 * @param[in] item_flags 3367 * Holds the items detected. 3368 * @param[out] error 3369 * Pointer to error structure. 3370 * 3371 * @return 3372 * 0 on success, a negative errno value otherwise and rte_errno is set. 3373 */ 3374 static int 3375 flow_dv_validate_action_modify_mac(const uint64_t action_flags, 3376 const struct rte_flow_action *action, 3377 const uint64_t item_flags, 3378 struct rte_flow_error *error) 3379 { 3380 int ret = 0; 3381 3382 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3383 if (!ret) { 3384 if (!(item_flags & MLX5_FLOW_LAYER_L2)) 3385 return rte_flow_error_set(error, EINVAL, 3386 RTE_FLOW_ERROR_TYPE_ACTION, 3387 NULL, 3388 "no L2 item in pattern"); 3389 } 3390 return ret; 3391 } 3392 3393 /** 3394 * Validate the modify-header IPv4 address actions. 3395 * 3396 * @param[in] action_flags 3397 * Holds the actions detected until now. 3398 * @param[in] action 3399 * Pointer to the modify action. 3400 * @param[in] item_flags 3401 * Holds the items detected. 3402 * @param[out] error 3403 * Pointer to error structure. 3404 * 3405 * @return 3406 * 0 on success, a negative errno value otherwise and rte_errno is set. 3407 */ 3408 static int 3409 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, 3410 const struct rte_flow_action *action, 3411 const uint64_t item_flags, 3412 struct rte_flow_error *error) 3413 { 3414 int ret = 0; 3415 uint64_t layer; 3416 3417 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3418 if (!ret) { 3419 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3420 MLX5_FLOW_LAYER_INNER_L3_IPV4 : 3421 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 3422 if (!(item_flags & layer)) 3423 return rte_flow_error_set(error, EINVAL, 3424 RTE_FLOW_ERROR_TYPE_ACTION, 3425 NULL, 3426 "no ipv4 item in pattern"); 3427 } 3428 return ret; 3429 } 3430 3431 /** 3432 * Validate the modify-header IPv6 address actions. 3433 * 3434 * @param[in] action_flags 3435 * Holds the actions detected until now. 3436 * @param[in] action 3437 * Pointer to the modify action. 3438 * @param[in] item_flags 3439 * Holds the items detected. 3440 * @param[out] error 3441 * Pointer to error structure. 3442 * 3443 * @return 3444 * 0 on success, a negative errno value otherwise and rte_errno is set. 3445 */ 3446 static int 3447 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, 3448 const struct rte_flow_action *action, 3449 const uint64_t item_flags, 3450 struct rte_flow_error *error) 3451 { 3452 int ret = 0; 3453 uint64_t layer; 3454 3455 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3456 if (!ret) { 3457 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3458 MLX5_FLOW_LAYER_INNER_L3_IPV6 : 3459 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 3460 if (!(item_flags & layer)) 3461 return rte_flow_error_set(error, EINVAL, 3462 RTE_FLOW_ERROR_TYPE_ACTION, 3463 NULL, 3464 "no ipv6 item in pattern"); 3465 } 3466 return ret; 3467 } 3468 3469 /** 3470 * Validate the modify-header TP actions. 3471 * 3472 * @param[in] action_flags 3473 * Holds the actions detected until now. 3474 * @param[in] action 3475 * Pointer to the modify action. 3476 * @param[in] item_flags 3477 * Holds the items detected. 3478 * @param[out] error 3479 * Pointer to error structure. 3480 * 3481 * @return 3482 * 0 on success, a negative errno value otherwise and rte_errno is set. 3483 */ 3484 static int 3485 flow_dv_validate_action_modify_tp(const uint64_t action_flags, 3486 const struct rte_flow_action *action, 3487 const uint64_t item_flags, 3488 struct rte_flow_error *error) 3489 { 3490 int ret = 0; 3491 uint64_t layer; 3492 3493 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3494 if (!ret) { 3495 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3496 MLX5_FLOW_LAYER_INNER_L4 : 3497 MLX5_FLOW_LAYER_OUTER_L4; 3498 if (!(item_flags & layer)) 3499 return rte_flow_error_set(error, EINVAL, 3500 RTE_FLOW_ERROR_TYPE_ACTION, 3501 NULL, "no transport layer " 3502 "in pattern"); 3503 } 3504 return ret; 3505 } 3506 3507 /** 3508 * Validate the modify-header actions of increment/decrement 3509 * TCP Sequence-number. 3510 * 3511 * @param[in] action_flags 3512 * Holds the actions detected until now. 3513 * @param[in] action 3514 * Pointer to the modify action. 3515 * @param[in] item_flags 3516 * Holds the items detected. 3517 * @param[out] error 3518 * Pointer to error structure. 3519 * 3520 * @return 3521 * 0 on success, a negative errno value otherwise and rte_errno is set. 3522 */ 3523 static int 3524 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, 3525 const struct rte_flow_action *action, 3526 const uint64_t item_flags, 3527 struct rte_flow_error *error) 3528 { 3529 int ret = 0; 3530 uint64_t layer; 3531 3532 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3533 if (!ret) { 3534 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3535 MLX5_FLOW_LAYER_INNER_L4_TCP : 3536 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3537 if (!(item_flags & layer)) 3538 return rte_flow_error_set(error, EINVAL, 3539 RTE_FLOW_ERROR_TYPE_ACTION, 3540 NULL, "no TCP item in" 3541 " pattern"); 3542 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && 3543 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || 3544 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && 3545 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) 3546 return rte_flow_error_set(error, EINVAL, 3547 RTE_FLOW_ERROR_TYPE_ACTION, 3548 NULL, 3549 "cannot decrease and increase" 3550 " TCP sequence number" 3551 " at the same time"); 3552 } 3553 return ret; 3554 } 3555 3556 /** 3557 * Validate the modify-header actions of increment/decrement 3558 * TCP Acknowledgment number. 3559 * 3560 * @param[in] action_flags 3561 * Holds the actions detected until now. 3562 * @param[in] action 3563 * Pointer to the modify action. 3564 * @param[in] item_flags 3565 * Holds the items detected. 3566 * @param[out] error 3567 * Pointer to error structure. 3568 * 3569 * @return 3570 * 0 on success, a negative errno value otherwise and rte_errno is set. 3571 */ 3572 static int 3573 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, 3574 const struct rte_flow_action *action, 3575 const uint64_t item_flags, 3576 struct rte_flow_error *error) 3577 { 3578 int ret = 0; 3579 uint64_t layer; 3580 3581 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3582 if (!ret) { 3583 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3584 MLX5_FLOW_LAYER_INNER_L4_TCP : 3585 MLX5_FLOW_LAYER_OUTER_L4_TCP; 3586 if (!(item_flags & layer)) 3587 return rte_flow_error_set(error, EINVAL, 3588 RTE_FLOW_ERROR_TYPE_ACTION, 3589 NULL, "no TCP item in" 3590 " pattern"); 3591 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && 3592 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || 3593 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && 3594 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) 3595 return rte_flow_error_set(error, EINVAL, 3596 RTE_FLOW_ERROR_TYPE_ACTION, 3597 NULL, 3598 "cannot decrease and increase" 3599 " TCP acknowledgment number" 3600 " at the same time"); 3601 } 3602 return ret; 3603 } 3604 3605 /** 3606 * Validate the modify-header TTL actions. 3607 * 3608 * @param[in] action_flags 3609 * Holds the actions detected until now. 3610 * @param[in] action 3611 * Pointer to the modify action. 3612 * @param[in] item_flags 3613 * Holds the items detected. 3614 * @param[out] error 3615 * Pointer to error structure. 3616 * 3617 * @return 3618 * 0 on success, a negative errno value otherwise and rte_errno is set. 3619 */ 3620 static int 3621 flow_dv_validate_action_modify_ttl(const uint64_t action_flags, 3622 const struct rte_flow_action *action, 3623 const uint64_t item_flags, 3624 struct rte_flow_error *error) 3625 { 3626 int ret = 0; 3627 uint64_t layer; 3628 3629 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3630 if (!ret) { 3631 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ? 3632 MLX5_FLOW_LAYER_INNER_L3 : 3633 MLX5_FLOW_LAYER_OUTER_L3; 3634 if (!(item_flags & layer)) 3635 return rte_flow_error_set(error, EINVAL, 3636 RTE_FLOW_ERROR_TYPE_ACTION, 3637 NULL, 3638 "no IP protocol in pattern"); 3639 } 3640 return ret; 3641 } 3642 3643 /** 3644 * Validate jump action. 3645 * 3646 * @param[in] action 3647 * Pointer to the jump action. 3648 * @param[in] action_flags 3649 * Holds the actions detected until now. 3650 * @param[in] attributes 3651 * Pointer to flow attributes 3652 * @param[in] external 3653 * Action belongs to flow rule created by request external to PMD. 3654 * @param[out] error 3655 * Pointer to error structure. 3656 * 3657 * @return 3658 * 0 on success, a negative errno value otherwise and rte_errno is set. 3659 */ 3660 static int 3661 flow_dv_validate_action_jump(const struct rte_flow_action *action, 3662 uint64_t action_flags, 3663 const struct rte_flow_attr *attributes, 3664 bool external, struct rte_flow_error *error) 3665 { 3666 uint32_t target_group, table; 3667 int ret = 0; 3668 3669 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3670 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3671 return rte_flow_error_set(error, EINVAL, 3672 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3673 "can't have 2 fate actions in" 3674 " same flow"); 3675 if (action_flags & MLX5_FLOW_ACTION_METER) 3676 return rte_flow_error_set(error, ENOTSUP, 3677 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3678 "jump with meter not support"); 3679 if (!action->conf) 3680 return rte_flow_error_set(error, EINVAL, 3681 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3682 NULL, "action configuration not set"); 3683 target_group = 3684 ((const struct rte_flow_action_jump *)action->conf)->group; 3685 ret = mlx5_flow_group_to_table(attributes, external, target_group, 3686 true, &table, error); 3687 if (ret) 3688 return ret; 3689 if (attributes->group == target_group) 3690 return rte_flow_error_set(error, EINVAL, 3691 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3692 "target group must be other than" 3693 " the current flow group"); 3694 return 0; 3695 } 3696 3697 /* 3698 * Validate the port_id action. 3699 * 3700 * @param[in] dev 3701 * Pointer to rte_eth_dev structure. 3702 * @param[in] action_flags 3703 * Bit-fields that holds the actions detected until now. 3704 * @param[in] action 3705 * Port_id RTE action structure. 3706 * @param[in] attr 3707 * Attributes of flow that includes this action. 3708 * @param[out] error 3709 * Pointer to error structure. 3710 * 3711 * @return 3712 * 0 on success, a negative errno value otherwise and rte_errno is set. 3713 */ 3714 static int 3715 flow_dv_validate_action_port_id(struct rte_eth_dev *dev, 3716 uint64_t action_flags, 3717 const struct rte_flow_action *action, 3718 const struct rte_flow_attr *attr, 3719 struct rte_flow_error *error) 3720 { 3721 const struct rte_flow_action_port_id *port_id; 3722 struct mlx5_priv *act_priv; 3723 struct mlx5_priv *dev_priv; 3724 uint16_t port; 3725 3726 if (!attr->transfer) 3727 return rte_flow_error_set(error, ENOTSUP, 3728 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3729 NULL, 3730 "port id action is valid in transfer" 3731 " mode only"); 3732 if (!action || !action->conf) 3733 return rte_flow_error_set(error, ENOTSUP, 3734 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 3735 NULL, 3736 "port id action parameters must be" 3737 " specified"); 3738 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 3739 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3740 return rte_flow_error_set(error, EINVAL, 3741 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3742 "can have only one fate actions in" 3743 " a flow"); 3744 dev_priv = mlx5_dev_to_eswitch_info(dev); 3745 if (!dev_priv) 3746 return rte_flow_error_set(error, rte_errno, 3747 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3748 NULL, 3749 "failed to obtain E-Switch info"); 3750 port_id = action->conf; 3751 port = port_id->original ? dev->data->port_id : port_id->id; 3752 act_priv = mlx5_port_to_eswitch_info(port, false); 3753 if (!act_priv) 3754 return rte_flow_error_set 3755 (error, rte_errno, 3756 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id, 3757 "failed to obtain E-Switch port id for port"); 3758 if (act_priv->domain_id != dev_priv->domain_id) 3759 return rte_flow_error_set 3760 (error, EINVAL, 3761 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3762 "port does not belong to" 3763 " E-Switch being configured"); 3764 return 0; 3765 } 3766 3767 /** 3768 * Get the maximum number of modify header actions. 3769 * 3770 * @param dev 3771 * Pointer to rte_eth_dev structure. 3772 * @param flags 3773 * Flags bits to check if root level. 3774 * 3775 * @return 3776 * Max number of modify header actions device can support. 3777 */ 3778 static inline unsigned int 3779 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused, 3780 uint64_t flags) 3781 { 3782 /* 3783 * There's no way to directly query the max capacity from FW. 3784 * The maximal value on root table should be assumed to be supported. 3785 */ 3786 if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL)) 3787 return MLX5_MAX_MODIFY_NUM; 3788 else 3789 return MLX5_ROOT_TBL_MODIFY_NUM; 3790 } 3791 3792 /** 3793 * Validate the meter action. 3794 * 3795 * @param[in] dev 3796 * Pointer to rte_eth_dev structure. 3797 * @param[in] action_flags 3798 * Bit-fields that holds the actions detected until now. 3799 * @param[in] action 3800 * Pointer to the meter action. 3801 * @param[in] attr 3802 * Attributes of flow that includes this action. 3803 * @param[out] error 3804 * Pointer to error structure. 3805 * 3806 * @return 3807 * 0 on success, a negative errno value otherwise and rte_ernno is set. 3808 */ 3809 static int 3810 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, 3811 uint64_t action_flags, 3812 const struct rte_flow_action *action, 3813 const struct rte_flow_attr *attr, 3814 struct rte_flow_error *error) 3815 { 3816 struct mlx5_priv *priv = dev->data->dev_private; 3817 const struct rte_flow_action_meter *am = action->conf; 3818 struct mlx5_flow_meter *fm; 3819 3820 if (!am) 3821 return rte_flow_error_set(error, EINVAL, 3822 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3823 "meter action conf is NULL"); 3824 3825 if (action_flags & MLX5_FLOW_ACTION_METER) 3826 return rte_flow_error_set(error, ENOTSUP, 3827 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3828 "meter chaining not support"); 3829 if (action_flags & MLX5_FLOW_ACTION_JUMP) 3830 return rte_flow_error_set(error, ENOTSUP, 3831 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3832 "meter with jump not support"); 3833 if (!priv->mtr_en) 3834 return rte_flow_error_set(error, ENOTSUP, 3835 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3836 NULL, 3837 "meter action not supported"); 3838 fm = mlx5_flow_meter_find(priv, am->mtr_id); 3839 if (!fm) 3840 return rte_flow_error_set(error, EINVAL, 3841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3842 "Meter not found"); 3843 if (fm->ref_cnt && (!(fm->transfer == attr->transfer || 3844 (!fm->ingress && !attr->ingress && attr->egress) || 3845 (!fm->egress && !attr->egress && attr->ingress)))) 3846 return rte_flow_error_set(error, EINVAL, 3847 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3848 "Flow attributes are either invalid " 3849 "or have a conflict with current " 3850 "meter attributes"); 3851 return 0; 3852 } 3853 3854 /** 3855 * Validate the age action. 3856 * 3857 * @param[in] action_flags 3858 * Holds the actions detected until now. 3859 * @param[in] action 3860 * Pointer to the age action. 3861 * @param[in] dev 3862 * Pointer to the Ethernet device structure. 3863 * @param[out] error 3864 * Pointer to error structure. 3865 * 3866 * @return 3867 * 0 on success, a negative errno value otherwise and rte_errno is set. 3868 */ 3869 static int 3870 flow_dv_validate_action_age(uint64_t action_flags, 3871 const struct rte_flow_action *action, 3872 struct rte_eth_dev *dev, 3873 struct rte_flow_error *error) 3874 { 3875 struct mlx5_priv *priv = dev->data->dev_private; 3876 const struct rte_flow_action_age *age = action->conf; 3877 3878 if (!priv->config.devx || priv->counter_fallback) 3879 return rte_flow_error_set(error, ENOTSUP, 3880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3881 NULL, 3882 "age action not supported"); 3883 if (!(action->conf)) 3884 return rte_flow_error_set(error, EINVAL, 3885 RTE_FLOW_ERROR_TYPE_ACTION, action, 3886 "configuration cannot be null"); 3887 if (age->timeout >= UINT16_MAX / 2 / 10) 3888 return rte_flow_error_set(error, ENOTSUP, 3889 RTE_FLOW_ERROR_TYPE_ACTION, action, 3890 "Max age time: 3275 seconds"); 3891 if (action_flags & MLX5_FLOW_ACTION_AGE) 3892 return rte_flow_error_set(error, EINVAL, 3893 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 3894 "Duplicate age ctions set"); 3895 return 0; 3896 } 3897 3898 /** 3899 * Validate the modify-header IPv4 DSCP actions. 3900 * 3901 * @param[in] action_flags 3902 * Holds the actions detected until now. 3903 * @param[in] action 3904 * Pointer to the modify action. 3905 * @param[in] item_flags 3906 * Holds the items detected. 3907 * @param[out] error 3908 * Pointer to error structure. 3909 * 3910 * @return 3911 * 0 on success, a negative errno value otherwise and rte_errno is set. 3912 */ 3913 static int 3914 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags, 3915 const struct rte_flow_action *action, 3916 const uint64_t item_flags, 3917 struct rte_flow_error *error) 3918 { 3919 int ret = 0; 3920 3921 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3922 if (!ret) { 3923 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) 3924 return rte_flow_error_set(error, EINVAL, 3925 RTE_FLOW_ERROR_TYPE_ACTION, 3926 NULL, 3927 "no ipv4 item in pattern"); 3928 } 3929 return ret; 3930 } 3931 3932 /** 3933 * Validate the modify-header IPv6 DSCP actions. 3934 * 3935 * @param[in] action_flags 3936 * Holds the actions detected until now. 3937 * @param[in] action 3938 * Pointer to the modify action. 3939 * @param[in] item_flags 3940 * Holds the items detected. 3941 * @param[out] error 3942 * Pointer to error structure. 3943 * 3944 * @return 3945 * 0 on success, a negative errno value otherwise and rte_errno is set. 3946 */ 3947 static int 3948 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags, 3949 const struct rte_flow_action *action, 3950 const uint64_t item_flags, 3951 struct rte_flow_error *error) 3952 { 3953 int ret = 0; 3954 3955 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 3956 if (!ret) { 3957 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) 3958 return rte_flow_error_set(error, EINVAL, 3959 RTE_FLOW_ERROR_TYPE_ACTION, 3960 NULL, 3961 "no ipv6 item in pattern"); 3962 } 3963 return ret; 3964 } 3965 3966 /** 3967 * Find existing modify-header resource or create and register a new one. 3968 * 3969 * @param dev[in, out] 3970 * Pointer to rte_eth_dev structure. 3971 * @param[in, out] resource 3972 * Pointer to modify-header resource. 3973 * @parm[in, out] dev_flow 3974 * Pointer to the dev_flow. 3975 * @param[out] error 3976 * pointer to error structure. 3977 * 3978 * @return 3979 * 0 on success otherwise -errno and errno is set. 3980 */ 3981 static int 3982 flow_dv_modify_hdr_resource_register 3983 (struct rte_eth_dev *dev, 3984 struct mlx5_flow_dv_modify_hdr_resource *resource, 3985 struct mlx5_flow *dev_flow, 3986 struct rte_flow_error *error) 3987 { 3988 struct mlx5_priv *priv = dev->data->dev_private; 3989 struct mlx5_dev_ctx_shared *sh = priv->sh; 3990 struct mlx5_flow_dv_modify_hdr_resource *cache_resource; 3991 struct mlx5dv_dr_domain *ns; 3992 uint32_t actions_len; 3993 3994 resource->flags = dev_flow->dv.group ? 0 : 3995 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 3996 if (resource->actions_num > flow_dv_modify_hdr_action_max(dev, 3997 resource->flags)) 3998 return rte_flow_error_set(error, EOVERFLOW, 3999 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 4000 "too many modify header items"); 4001 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 4002 ns = sh->fdb_domain; 4003 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 4004 ns = sh->tx_domain; 4005 else 4006 ns = sh->rx_domain; 4007 /* Lookup a matching resource from cache. */ 4008 actions_len = resource->actions_num * sizeof(resource->actions[0]); 4009 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { 4010 if (resource->ft_type == cache_resource->ft_type && 4011 resource->actions_num == cache_resource->actions_num && 4012 resource->flags == cache_resource->flags && 4013 !memcmp((const void *)resource->actions, 4014 (const void *)cache_resource->actions, 4015 actions_len)) { 4016 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", 4017 (void *)cache_resource, 4018 rte_atomic32_read(&cache_resource->refcnt)); 4019 rte_atomic32_inc(&cache_resource->refcnt); 4020 dev_flow->handle->dvh.modify_hdr = cache_resource; 4021 return 0; 4022 } 4023 } 4024 /* Register new modify-header resource. */ 4025 cache_resource = rte_calloc(__func__, 1, 4026 sizeof(*cache_resource) + actions_len, 0); 4027 if (!cache_resource) 4028 return rte_flow_error_set(error, ENOMEM, 4029 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4030 "cannot allocate resource memory"); 4031 *cache_resource = *resource; 4032 rte_memcpy(cache_resource->actions, resource->actions, actions_len); 4033 cache_resource->verbs_action = 4034 mlx5_glue->dv_create_flow_action_modify_header 4035 (sh->ctx, cache_resource->ft_type, ns, 4036 cache_resource->flags, actions_len, 4037 (uint64_t *)cache_resource->actions); 4038 if (!cache_resource->verbs_action) { 4039 rte_free(cache_resource); 4040 return rte_flow_error_set(error, ENOMEM, 4041 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4042 NULL, "cannot create action"); 4043 } 4044 rte_atomic32_init(&cache_resource->refcnt); 4045 rte_atomic32_inc(&cache_resource->refcnt); 4046 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); 4047 dev_flow->handle->dvh.modify_hdr = cache_resource; 4048 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", 4049 (void *)cache_resource, 4050 rte_atomic32_read(&cache_resource->refcnt)); 4051 return 0; 4052 } 4053 4054 /** 4055 * Get DV flow counter by index. 4056 * 4057 * @param[in] dev 4058 * Pointer to the Ethernet device structure. 4059 * @param[in] idx 4060 * mlx5 flow counter index in the container. 4061 * @param[out] ppool 4062 * mlx5 flow counter pool in the container, 4063 * 4064 * @return 4065 * Pointer to the counter, NULL otherwise. 4066 */ 4067 static struct mlx5_flow_counter * 4068 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev, 4069 uint32_t idx, 4070 struct mlx5_flow_counter_pool **ppool) 4071 { 4072 struct mlx5_priv *priv = dev->data->dev_private; 4073 struct mlx5_pools_container *cont; 4074 struct mlx5_flow_counter_pool *pool; 4075 uint32_t batch = 0, age = 0; 4076 4077 idx--; 4078 age = MLX_CNT_IS_AGE(idx); 4079 idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx; 4080 if (idx >= MLX5_CNT_BATCH_OFFSET) { 4081 idx -= MLX5_CNT_BATCH_OFFSET; 4082 batch = 1; 4083 } 4084 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4085 MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n); 4086 pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; 4087 MLX5_ASSERT(pool); 4088 if (ppool) 4089 *ppool = pool; 4090 return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); 4091 } 4092 4093 /** 4094 * Check the devx counter belongs to the pool. 4095 * 4096 * @param[in] pool 4097 * Pointer to the counter pool. 4098 * @param[in] id 4099 * The counter devx ID. 4100 * 4101 * @return 4102 * True if counter belongs to the pool, false otherwise. 4103 */ 4104 static bool 4105 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id) 4106 { 4107 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * 4108 MLX5_COUNTERS_PER_POOL; 4109 4110 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) 4111 return true; 4112 return false; 4113 } 4114 4115 /** 4116 * Get a pool by devx counter ID. 4117 * 4118 * @param[in] cont 4119 * Pointer to the counter container. 4120 * @param[in] id 4121 * The counter devx ID. 4122 * 4123 * @return 4124 * The counter pool pointer if exists, NULL otherwise, 4125 */ 4126 static struct mlx5_flow_counter_pool * 4127 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) 4128 { 4129 uint32_t i; 4130 4131 /* Check last used pool. */ 4132 if (cont->last_pool_idx != POOL_IDX_INVALID && 4133 flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id)) 4134 return cont->pools[cont->last_pool_idx]; 4135 /* ID out of range means no suitable pool in the container. */ 4136 if (id > cont->max_id || id < cont->min_id) 4137 return NULL; 4138 /* 4139 * Find the pool from the end of the container, since mostly counter 4140 * ID is sequence increasing, and the last pool should be the needed 4141 * one. 4142 */ 4143 i = rte_atomic16_read(&cont->n_valid); 4144 while (i--) { 4145 struct mlx5_flow_counter_pool *pool = cont->pools[i]; 4146 4147 if (flow_dv_is_counter_in_pool(pool, id)) 4148 return pool; 4149 } 4150 return NULL; 4151 } 4152 4153 /** 4154 * Allocate a new memory for the counter values wrapped by all the needed 4155 * management. 4156 * 4157 * @param[in] dev 4158 * Pointer to the Ethernet device structure. 4159 * @param[in] raws_n 4160 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters. 4161 * 4162 * @return 4163 * The new memory management pointer on success, otherwise NULL and rte_errno 4164 * is set. 4165 */ 4166 static struct mlx5_counter_stats_mem_mng * 4167 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) 4168 { 4169 struct mlx5_priv *priv = dev->data->dev_private; 4170 struct mlx5_dev_ctx_shared *sh = priv->sh; 4171 struct mlx5_devx_mkey_attr mkey_attr; 4172 struct mlx5_counter_stats_mem_mng *mem_mng; 4173 volatile struct flow_counter_stats *raw_data; 4174 int size = (sizeof(struct flow_counter_stats) * 4175 MLX5_COUNTERS_PER_POOL + 4176 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 4177 sizeof(struct mlx5_counter_stats_mem_mng); 4178 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE)); 4179 int i; 4180 4181 if (!mem) { 4182 rte_errno = ENOMEM; 4183 return NULL; 4184 } 4185 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 4186 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 4187 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 4188 IBV_ACCESS_LOCAL_WRITE); 4189 if (!mem_mng->umem) { 4190 rte_errno = errno; 4191 rte_free(mem); 4192 return NULL; 4193 } 4194 mkey_attr.addr = (uintptr_t)mem; 4195 mkey_attr.size = size; 4196 mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem); 4197 mkey_attr.pd = sh->pdn; 4198 mkey_attr.log_entity_size = 0; 4199 mkey_attr.pg_access = 0; 4200 mkey_attr.klm_array = NULL; 4201 mkey_attr.klm_num = 0; 4202 if (priv->config.hca_attr.relaxed_ordering_write && 4203 priv->config.hca_attr.relaxed_ordering_read && 4204 !haswell_broadwell_cpu) 4205 mkey_attr.relaxed_ordering = 1; 4206 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 4207 if (!mem_mng->dm) { 4208 mlx5_glue->devx_umem_dereg(mem_mng->umem); 4209 rte_errno = errno; 4210 rte_free(mem); 4211 return NULL; 4212 } 4213 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 4214 raw_data = (volatile struct flow_counter_stats *)mem; 4215 for (i = 0; i < raws_n; ++i) { 4216 mem_mng->raws[i].mem_mng = mem_mng; 4217 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 4218 } 4219 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 4220 return mem_mng; 4221 } 4222 4223 /** 4224 * Resize a counter container. 4225 * 4226 * @param[in] dev 4227 * Pointer to the Ethernet device structure. 4228 * @param[in] batch 4229 * Whether the pool is for counter that was allocated by batch command. 4230 * @param[in] age 4231 * Whether the pool is for Aging counter. 4232 * 4233 * @return 4234 * 0 on success, otherwise negative errno value and rte_errno is set. 4235 */ 4236 static int 4237 flow_dv_container_resize(struct rte_eth_dev *dev, 4238 uint32_t batch, uint32_t age) 4239 { 4240 struct mlx5_priv *priv = dev->data->dev_private; 4241 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4242 age); 4243 struct mlx5_counter_stats_mem_mng *mem_mng = NULL; 4244 void *old_pools = cont->pools; 4245 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; 4246 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; 4247 void *pools = rte_calloc(__func__, 1, mem_size, 0); 4248 4249 if (!pools) { 4250 rte_errno = ENOMEM; 4251 return -ENOMEM; 4252 } 4253 if (old_pools) 4254 memcpy(pools, old_pools, cont->n * 4255 sizeof(struct mlx5_flow_counter_pool *)); 4256 /* 4257 * Fallback mode query the counter directly, no background query 4258 * resources are needed. 4259 */ 4260 if (!priv->counter_fallback) { 4261 int i; 4262 4263 mem_mng = flow_dv_create_counter_stat_mem_mng(dev, 4264 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); 4265 if (!mem_mng) { 4266 rte_free(pools); 4267 return -ENOMEM; 4268 } 4269 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 4270 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, 4271 mem_mng->raws + 4272 MLX5_CNT_CONTAINER_RESIZE + 4273 i, next); 4274 } 4275 rte_spinlock_lock(&cont->resize_sl); 4276 cont->n = resize; 4277 cont->mem_mng = mem_mng; 4278 cont->pools = pools; 4279 rte_spinlock_unlock(&cont->resize_sl); 4280 if (old_pools) 4281 rte_free(old_pools); 4282 return 0; 4283 } 4284 4285 /** 4286 * Query a devx flow counter. 4287 * 4288 * @param[in] dev 4289 * Pointer to the Ethernet device structure. 4290 * @param[in] cnt 4291 * Index to the flow counter. 4292 * @param[out] pkts 4293 * The statistics value of packets. 4294 * @param[out] bytes 4295 * The statistics value of bytes. 4296 * 4297 * @return 4298 * 0 on success, otherwise a negative errno value and rte_errno is set. 4299 */ 4300 static inline int 4301 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts, 4302 uint64_t *bytes) 4303 { 4304 struct mlx5_priv *priv = dev->data->dev_private; 4305 struct mlx5_flow_counter_pool *pool = NULL; 4306 struct mlx5_flow_counter *cnt; 4307 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4308 int offset; 4309 4310 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4311 MLX5_ASSERT(pool); 4312 if (counter < MLX5_CNT_BATCH_OFFSET) { 4313 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4314 if (priv->counter_fallback) 4315 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0, 4316 0, pkts, bytes, 0, NULL, NULL, 0); 4317 } 4318 4319 rte_spinlock_lock(&pool->sl); 4320 /* 4321 * The single counters allocation may allocate smaller ID than the 4322 * current allocated in parallel to the host reading. 4323 * In this case the new counter values must be reported as 0. 4324 */ 4325 if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) { 4326 *pkts = 0; 4327 *bytes = 0; 4328 } else { 4329 offset = MLX5_CNT_ARRAY_IDX(pool, cnt); 4330 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); 4331 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); 4332 } 4333 rte_spinlock_unlock(&pool->sl); 4334 return 0; 4335 } 4336 4337 /** 4338 * Create and initialize a new counter pool. 4339 * 4340 * @param[in] dev 4341 * Pointer to the Ethernet device structure. 4342 * @param[out] dcs 4343 * The devX counter handle. 4344 * @param[in] batch 4345 * Whether the pool is for counter that was allocated by batch command. 4346 * @param[in] age 4347 * Whether the pool is for counter that was allocated for aging. 4348 * @param[in/out] cont_cur 4349 * Pointer to the container pointer, it will be update in pool resize. 4350 * 4351 * @return 4352 * The pool container pointer on success, NULL otherwise and rte_errno is set. 4353 */ 4354 static struct mlx5_flow_counter_pool * 4355 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, 4356 uint32_t batch, uint32_t age) 4357 { 4358 struct mlx5_priv *priv = dev->data->dev_private; 4359 struct mlx5_flow_counter_pool *pool; 4360 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4361 age); 4362 int16_t n_valid = rte_atomic16_read(&cont->n_valid); 4363 uint32_t size = sizeof(*pool); 4364 4365 if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age)) 4366 return NULL; 4367 size += MLX5_COUNTERS_PER_POOL * CNT_SIZE; 4368 size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE); 4369 size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE); 4370 pool = rte_calloc(__func__, 1, size, 0); 4371 if (!pool) { 4372 rte_errno = ENOMEM; 4373 return NULL; 4374 } 4375 pool->min_dcs = dcs; 4376 if (!priv->counter_fallback) 4377 pool->raw = cont->mem_mng->raws + n_valid % 4378 MLX5_CNT_CONTAINER_RESIZE; 4379 pool->raw_hw = NULL; 4380 pool->type = 0; 4381 pool->type |= (batch ? 0 : CNT_POOL_TYPE_EXT); 4382 pool->type |= (!age ? 0 : CNT_POOL_TYPE_AGE); 4383 pool->query_gen = 0; 4384 rte_spinlock_init(&pool->sl); 4385 TAILQ_INIT(&pool->counters[0]); 4386 TAILQ_INIT(&pool->counters[1]); 4387 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); 4388 pool->index = n_valid; 4389 cont->pools[n_valid] = pool; 4390 if (!batch) { 4391 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL); 4392 4393 if (base < cont->min_id) 4394 cont->min_id = base; 4395 if (base > cont->max_id) 4396 cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1; 4397 cont->last_pool_idx = pool->index; 4398 } 4399 /* Pool initialization must be updated before host thread access. */ 4400 rte_cio_wmb(); 4401 rte_atomic16_add(&cont->n_valid, 1); 4402 return pool; 4403 } 4404 4405 /** 4406 * Update the minimum dcs-id for aged or no-aged counter pool. 4407 * 4408 * @param[in] dev 4409 * Pointer to the Ethernet device structure. 4410 * @param[in] pool 4411 * Current counter pool. 4412 * @param[in] batch 4413 * Whether the pool is for counter that was allocated by batch command. 4414 * @param[in] age 4415 * Whether the counter is for aging. 4416 */ 4417 static void 4418 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev, 4419 struct mlx5_flow_counter_pool *pool, 4420 uint32_t batch, uint32_t age) 4421 { 4422 struct mlx5_priv *priv = dev->data->dev_private; 4423 struct mlx5_flow_counter_pool *other; 4424 struct mlx5_pools_container *cont; 4425 4426 cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1)); 4427 other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id); 4428 if (!other) 4429 return; 4430 if (pool->min_dcs->id < other->min_dcs->id) { 4431 rte_atomic64_set(&other->a64_dcs, 4432 rte_atomic64_read(&pool->a64_dcs)); 4433 } else { 4434 rte_atomic64_set(&pool->a64_dcs, 4435 rte_atomic64_read(&other->a64_dcs)); 4436 } 4437 } 4438 /** 4439 * Prepare a new counter and/or a new counter pool. 4440 * 4441 * @param[in] dev 4442 * Pointer to the Ethernet device structure. 4443 * @param[out] cnt_free 4444 * Where to put the pointer of a new counter. 4445 * @param[in] batch 4446 * Whether the pool is for counter that was allocated by batch command. 4447 * @param[in] age 4448 * Whether the pool is for counter that was allocated for aging. 4449 * 4450 * @return 4451 * The counter pool pointer and @p cnt_free is set on success, 4452 * NULL otherwise and rte_errno is set. 4453 */ 4454 static struct mlx5_flow_counter_pool * 4455 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, 4456 struct mlx5_flow_counter **cnt_free, 4457 uint32_t batch, uint32_t age) 4458 { 4459 struct mlx5_priv *priv = dev->data->dev_private; 4460 struct mlx5_pools_container *cont; 4461 struct mlx5_flow_counter_pool *pool; 4462 struct mlx5_counters tmp_tq; 4463 struct mlx5_devx_obj *dcs = NULL; 4464 struct mlx5_flow_counter *cnt; 4465 uint32_t i; 4466 4467 cont = MLX5_CNT_CONTAINER(priv->sh, batch, age); 4468 if (!batch) { 4469 /* bulk_bitmap must be 0 for single counter allocation. */ 4470 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); 4471 if (!dcs) 4472 return NULL; 4473 pool = flow_dv_find_pool_by_id(cont, dcs->id); 4474 if (!pool) { 4475 pool = flow_dv_pool_create(dev, dcs, batch, age); 4476 if (!pool) { 4477 mlx5_devx_cmd_destroy(dcs); 4478 return NULL; 4479 } 4480 } else if (dcs->id < pool->min_dcs->id) { 4481 rte_atomic64_set(&pool->a64_dcs, 4482 (int64_t)(uintptr_t)dcs); 4483 } 4484 flow_dv_counter_update_min_dcs(dev, 4485 pool, batch, age); 4486 i = dcs->id % MLX5_COUNTERS_PER_POOL; 4487 cnt = MLX5_POOL_GET_CNT(pool, i); 4488 cnt->pool = pool; 4489 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs; 4490 *cnt_free = cnt; 4491 return pool; 4492 } 4493 /* bulk_bitmap is in 128 counters units. */ 4494 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) 4495 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); 4496 if (!dcs) { 4497 rte_errno = ENODATA; 4498 return NULL; 4499 } 4500 pool = flow_dv_pool_create(dev, dcs, batch, age); 4501 if (!pool) { 4502 mlx5_devx_cmd_destroy(dcs); 4503 return NULL; 4504 } 4505 TAILQ_INIT(&tmp_tq); 4506 for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) { 4507 cnt = MLX5_POOL_GET_CNT(pool, i); 4508 cnt->pool = pool; 4509 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next); 4510 } 4511 rte_spinlock_lock(&cont->csl); 4512 TAILQ_CONCAT(&cont->counters, &tmp_tq, next); 4513 rte_spinlock_unlock(&cont->csl); 4514 *cnt_free = MLX5_POOL_GET_CNT(pool, 0); 4515 (*cnt_free)->pool = pool; 4516 return pool; 4517 } 4518 4519 /** 4520 * Search for existed shared counter. 4521 * 4522 * @param[in] dev 4523 * Pointer to the Ethernet device structure. 4524 * @param[in] id 4525 * The shared counter ID to search. 4526 * @param[out] ppool 4527 * mlx5 flow counter pool in the container, 4528 * 4529 * @return 4530 * NULL if not existed, otherwise pointer to the shared extend counter. 4531 */ 4532 static struct mlx5_flow_counter_ext * 4533 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id, 4534 struct mlx5_flow_counter_pool **ppool) 4535 { 4536 struct mlx5_priv *priv = dev->data->dev_private; 4537 union mlx5_l3t_data data; 4538 uint32_t cnt_idx; 4539 4540 if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword) 4541 return NULL; 4542 cnt_idx = data.dword; 4543 /* 4544 * Shared counters don't have age info. The counter extend is after 4545 * the counter datat structure. 4546 */ 4547 return (struct mlx5_flow_counter_ext *) 4548 ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1); 4549 } 4550 4551 /** 4552 * Allocate a flow counter. 4553 * 4554 * @param[in] dev 4555 * Pointer to the Ethernet device structure. 4556 * @param[in] shared 4557 * Indicate if this counter is shared with other flows. 4558 * @param[in] id 4559 * Counter identifier. 4560 * @param[in] group 4561 * Counter flow group. 4562 * @param[in] age 4563 * Whether the counter was allocated for aging. 4564 * 4565 * @return 4566 * Index to flow counter on success, 0 otherwise and rte_errno is set. 4567 */ 4568 static uint32_t 4569 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, 4570 uint16_t group, uint32_t age) 4571 { 4572 struct mlx5_priv *priv = dev->data->dev_private; 4573 struct mlx5_flow_counter_pool *pool = NULL; 4574 struct mlx5_flow_counter *cnt_free = NULL; 4575 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4576 /* 4577 * Currently group 0 flow counter cannot be assigned to a flow if it is 4578 * not the first one in the batch counter allocation, so it is better 4579 * to allocate counters one by one for these flows in a separate 4580 * container. 4581 * A counter can be shared between different groups so need to take 4582 * shared counters from the single container. 4583 */ 4584 uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0; 4585 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 4586 age); 4587 uint32_t cnt_idx; 4588 4589 if (!priv->config.devx) { 4590 rte_errno = ENOTSUP; 4591 return 0; 4592 } 4593 if (shared) { 4594 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool); 4595 if (cnt_ext) { 4596 if (cnt_ext->ref_cnt + 1 == 0) { 4597 rte_errno = E2BIG; 4598 return 0; 4599 } 4600 cnt_ext->ref_cnt++; 4601 cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL + 4602 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL) 4603 + 1; 4604 return cnt_idx; 4605 } 4606 } 4607 /* Get free counters from container. */ 4608 rte_spinlock_lock(&cont->csl); 4609 cnt_free = TAILQ_FIRST(&cont->counters); 4610 if (cnt_free) 4611 TAILQ_REMOVE(&cont->counters, cnt_free, next); 4612 rte_spinlock_unlock(&cont->csl); 4613 if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, 4614 batch, age)) 4615 goto err; 4616 pool = cnt_free->pool; 4617 if (!batch) 4618 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free); 4619 /* Create a DV counter action only in the first time usage. */ 4620 if (!cnt_free->action) { 4621 uint16_t offset; 4622 struct mlx5_devx_obj *dcs; 4623 4624 if (batch) { 4625 offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free); 4626 dcs = pool->min_dcs; 4627 } else { 4628 offset = 0; 4629 dcs = cnt_ext->dcs; 4630 } 4631 cnt_free->action = mlx5_glue->dv_create_flow_action_counter 4632 (dcs->obj, offset); 4633 if (!cnt_free->action) { 4634 rte_errno = errno; 4635 goto err; 4636 } 4637 } 4638 cnt_idx = MLX5_MAKE_CNT_IDX(pool->index, 4639 MLX5_CNT_ARRAY_IDX(pool, cnt_free)); 4640 cnt_idx += batch * MLX5_CNT_BATCH_OFFSET; 4641 cnt_idx += age * MLX5_CNT_AGE_OFFSET; 4642 /* Update the counter reset values. */ 4643 if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits, 4644 &cnt_free->bytes)) 4645 goto err; 4646 if (cnt_ext) { 4647 cnt_ext->shared = shared; 4648 cnt_ext->ref_cnt = 1; 4649 cnt_ext->id = id; 4650 if (shared) { 4651 union mlx5_l3t_data data; 4652 4653 data.dword = cnt_idx; 4654 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data)) 4655 return 0; 4656 } 4657 } 4658 if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on) 4659 /* Start the asynchronous batch query by the host thread. */ 4660 mlx5_set_query_alarm(priv->sh); 4661 return cnt_idx; 4662 err: 4663 if (cnt_free) { 4664 cnt_free->pool = pool; 4665 rte_spinlock_lock(&cont->csl); 4666 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next); 4667 rte_spinlock_unlock(&cont->csl); 4668 } 4669 return 0; 4670 } 4671 4672 /** 4673 * Get age param from counter index. 4674 * 4675 * @param[in] dev 4676 * Pointer to the Ethernet device structure. 4677 * @param[in] counter 4678 * Index to the counter handler. 4679 * 4680 * @return 4681 * The aging parameter specified for the counter index. 4682 */ 4683 static struct mlx5_age_param* 4684 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev, 4685 uint32_t counter) 4686 { 4687 struct mlx5_flow_counter *cnt; 4688 struct mlx5_flow_counter_pool *pool = NULL; 4689 4690 flow_dv_counter_get_by_idx(dev, counter, &pool); 4691 counter = (counter - 1) % MLX5_COUNTERS_PER_POOL; 4692 cnt = MLX5_POOL_GET_CNT(pool, counter); 4693 return MLX5_CNT_TO_AGE(cnt); 4694 } 4695 4696 /** 4697 * Remove a flow counter from aged counter list. 4698 * 4699 * @param[in] dev 4700 * Pointer to the Ethernet device structure. 4701 * @param[in] counter 4702 * Index to the counter handler. 4703 * @param[in] cnt 4704 * Pointer to the counter handler. 4705 */ 4706 static void 4707 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev, 4708 uint32_t counter, struct mlx5_flow_counter *cnt) 4709 { 4710 struct mlx5_age_info *age_info; 4711 struct mlx5_age_param *age_param; 4712 struct mlx5_priv *priv = dev->data->dev_private; 4713 4714 age_info = GET_PORT_AGE_INFO(priv); 4715 age_param = flow_dv_counter_idx_get_age(dev, counter); 4716 if (rte_atomic16_cmpset((volatile uint16_t *) 4717 &age_param->state, 4718 AGE_CANDIDATE, AGE_FREE) 4719 != AGE_CANDIDATE) { 4720 /** 4721 * We need the lock even it is age timeout, 4722 * since counter may still in process. 4723 */ 4724 rte_spinlock_lock(&age_info->aged_sl); 4725 TAILQ_REMOVE(&age_info->aged_counters, cnt, next); 4726 rte_spinlock_unlock(&age_info->aged_sl); 4727 } 4728 rte_atomic16_set(&age_param->state, AGE_FREE); 4729 } 4730 /** 4731 * Release a flow counter. 4732 * 4733 * @param[in] dev 4734 * Pointer to the Ethernet device structure. 4735 * @param[in] counter 4736 * Index to the counter handler. 4737 */ 4738 static void 4739 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter) 4740 { 4741 struct mlx5_priv *priv = dev->data->dev_private; 4742 struct mlx5_flow_counter_pool *pool = NULL; 4743 struct mlx5_flow_counter *cnt; 4744 struct mlx5_flow_counter_ext *cnt_ext = NULL; 4745 4746 if (!counter) 4747 return; 4748 cnt = flow_dv_counter_get_by_idx(dev, counter, &pool); 4749 MLX5_ASSERT(pool); 4750 if (counter < MLX5_CNT_BATCH_OFFSET) { 4751 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); 4752 if (cnt_ext) { 4753 if (--cnt_ext->ref_cnt) 4754 return; 4755 if (cnt_ext->shared) 4756 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, 4757 cnt_ext->id); 4758 } 4759 } 4760 if (IS_AGE_POOL(pool)) 4761 flow_dv_counter_remove_from_age(dev, counter, cnt); 4762 cnt->pool = pool; 4763 /* 4764 * Put the counter back to list to be updated in none fallback mode. 4765 * Currently, we are using two list alternately, while one is in query, 4766 * add the freed counter to the other list based on the pool query_gen 4767 * value. After query finishes, add counter the list to the global 4768 * container counter list. The list changes while query starts. In 4769 * this case, lock will not be needed as query callback and release 4770 * function both operate with the different list. 4771 * 4772 */ 4773 if (!priv->counter_fallback) 4774 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next); 4775 else 4776 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER 4777 (priv->sh, 0, 0))->counters), 4778 cnt, next); 4779 } 4780 4781 /** 4782 * Verify the @p attributes will be correctly understood by the NIC and store 4783 * them in the @p flow if everything is correct. 4784 * 4785 * @param[in] dev 4786 * Pointer to dev struct. 4787 * @param[in] attributes 4788 * Pointer to flow attributes 4789 * @param[in] external 4790 * This flow rule is created by request external to PMD. 4791 * @param[out] error 4792 * Pointer to error structure. 4793 * 4794 * @return 4795 * - 0 on success and non root table. 4796 * - 1 on success and root table. 4797 * - a negative errno value otherwise and rte_errno is set. 4798 */ 4799 static int 4800 flow_dv_validate_attributes(struct rte_eth_dev *dev, 4801 const struct rte_flow_attr *attributes, 4802 bool external __rte_unused, 4803 struct rte_flow_error *error) 4804 { 4805 struct mlx5_priv *priv = dev->data->dev_private; 4806 uint32_t priority_max = priv->config.flow_prio - 1; 4807 int ret = 0; 4808 4809 #ifndef HAVE_MLX5DV_DR 4810 if (attributes->group) 4811 return rte_flow_error_set(error, ENOTSUP, 4812 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 4813 NULL, 4814 "groups are not supported"); 4815 #else 4816 uint32_t table = 0; 4817 4818 ret = mlx5_flow_group_to_table(attributes, external, 4819 attributes->group, !!priv->fdb_def_rule, 4820 &table, error); 4821 if (ret) 4822 return ret; 4823 if (!table) 4824 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 4825 #endif 4826 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 4827 attributes->priority >= priority_max) 4828 return rte_flow_error_set(error, ENOTSUP, 4829 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 4830 NULL, 4831 "priority out of range"); 4832 if (attributes->transfer) { 4833 if (!priv->config.dv_esw_en) 4834 return rte_flow_error_set 4835 (error, ENOTSUP, 4836 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4837 "E-Switch dr is not supported"); 4838 if (!(priv->representor || priv->master)) 4839 return rte_flow_error_set 4840 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4841 NULL, "E-Switch configuration can only be" 4842 " done by a master or a representor device"); 4843 if (attributes->egress) 4844 return rte_flow_error_set 4845 (error, ENOTSUP, 4846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, 4847 "egress is not supported"); 4848 } 4849 if (!(attributes->egress ^ attributes->ingress)) 4850 return rte_flow_error_set(error, ENOTSUP, 4851 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 4852 "must specify exactly one of " 4853 "ingress or egress"); 4854 return ret; 4855 } 4856 4857 /** 4858 * Internal validation function. For validating both actions and items. 4859 * 4860 * @param[in] dev 4861 * Pointer to the rte_eth_dev structure. 4862 * @param[in] attr 4863 * Pointer to the flow attributes. 4864 * @param[in] items 4865 * Pointer to the list of items. 4866 * @param[in] actions 4867 * Pointer to the list of actions. 4868 * @param[in] external 4869 * This flow rule is created by request external to PMD. 4870 * @param[in] hairpin 4871 * Number of hairpin TX actions, 0 means classic flow. 4872 * @param[out] error 4873 * Pointer to the error structure. 4874 * 4875 * @return 4876 * 0 on success, a negative errno value otherwise and rte_errno is set. 4877 */ 4878 static int 4879 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 4880 const struct rte_flow_item items[], 4881 const struct rte_flow_action actions[], 4882 bool external, int hairpin, struct rte_flow_error *error) 4883 { 4884 int ret; 4885 uint64_t action_flags = 0; 4886 uint64_t item_flags = 0; 4887 uint64_t last_item = 0; 4888 uint8_t next_protocol = 0xff; 4889 uint16_t ether_type = 0; 4890 int actions_n = 0; 4891 uint8_t item_ipv6_proto = 0; 4892 const struct rte_flow_item *gre_item = NULL; 4893 const struct rte_flow_action_raw_decap *decap; 4894 const struct rte_flow_action_raw_encap *encap; 4895 const struct rte_flow_action_rss *rss; 4896 const struct rte_flow_item_tcp nic_tcp_mask = { 4897 .hdr = { 4898 .tcp_flags = 0xFF, 4899 .src_port = RTE_BE16(UINT16_MAX), 4900 .dst_port = RTE_BE16(UINT16_MAX), 4901 } 4902 }; 4903 const struct rte_flow_item_ipv4 nic_ipv4_mask = { 4904 .hdr = { 4905 .src_addr = RTE_BE32(0xffffffff), 4906 .dst_addr = RTE_BE32(0xffffffff), 4907 .type_of_service = 0xff, 4908 .next_proto_id = 0xff, 4909 .time_to_live = 0xff, 4910 }, 4911 }; 4912 const struct rte_flow_item_ipv6 nic_ipv6_mask = { 4913 .hdr = { 4914 .src_addr = 4915 "\xff\xff\xff\xff\xff\xff\xff\xff" 4916 "\xff\xff\xff\xff\xff\xff\xff\xff", 4917 .dst_addr = 4918 "\xff\xff\xff\xff\xff\xff\xff\xff" 4919 "\xff\xff\xff\xff\xff\xff\xff\xff", 4920 .vtc_flow = RTE_BE32(0xffffffff), 4921 .proto = 0xff, 4922 .hop_limits = 0xff, 4923 }, 4924 }; 4925 struct mlx5_priv *priv = dev->data->dev_private; 4926 struct mlx5_dev_config *dev_conf = &priv->config; 4927 uint16_t queue_index = 0xFFFF; 4928 const struct rte_flow_item_vlan *vlan_m = NULL; 4929 int16_t rw_act_num = 0; 4930 uint64_t is_root; 4931 4932 if (items == NULL) 4933 return -1; 4934 ret = flow_dv_validate_attributes(dev, attr, external, error); 4935 if (ret < 0) 4936 return ret; 4937 is_root = (uint64_t)ret; 4938 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4939 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 4940 int type = items->type; 4941 4942 switch (type) { 4943 case RTE_FLOW_ITEM_TYPE_VOID: 4944 break; 4945 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4946 ret = flow_dv_validate_item_port_id 4947 (dev, items, attr, item_flags, error); 4948 if (ret < 0) 4949 return ret; 4950 last_item = MLX5_FLOW_ITEM_PORT_ID; 4951 break; 4952 case RTE_FLOW_ITEM_TYPE_ETH: 4953 ret = mlx5_flow_validate_item_eth(items, item_flags, 4954 error); 4955 if (ret < 0) 4956 return ret; 4957 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 4958 MLX5_FLOW_LAYER_OUTER_L2; 4959 if (items->mask != NULL && items->spec != NULL) { 4960 ether_type = 4961 ((const struct rte_flow_item_eth *) 4962 items->spec)->type; 4963 ether_type &= 4964 ((const struct rte_flow_item_eth *) 4965 items->mask)->type; 4966 ether_type = rte_be_to_cpu_16(ether_type); 4967 } else { 4968 ether_type = 0; 4969 } 4970 break; 4971 case RTE_FLOW_ITEM_TYPE_VLAN: 4972 ret = flow_dv_validate_item_vlan(items, item_flags, 4973 dev, error); 4974 if (ret < 0) 4975 return ret; 4976 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 4977 MLX5_FLOW_LAYER_OUTER_VLAN; 4978 if (items->mask != NULL && items->spec != NULL) { 4979 ether_type = 4980 ((const struct rte_flow_item_vlan *) 4981 items->spec)->inner_type; 4982 ether_type &= 4983 ((const struct rte_flow_item_vlan *) 4984 items->mask)->inner_type; 4985 ether_type = rte_be_to_cpu_16(ether_type); 4986 } else { 4987 ether_type = 0; 4988 } 4989 /* Store outer VLAN mask for of_push_vlan action. */ 4990 if (!tunnel) 4991 vlan_m = items->mask; 4992 break; 4993 case RTE_FLOW_ITEM_TYPE_IPV4: 4994 mlx5_flow_tunnel_ip_check(items, next_protocol, 4995 &item_flags, &tunnel); 4996 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 4997 last_item, 4998 ether_type, 4999 &nic_ipv4_mask, 5000 error); 5001 if (ret < 0) 5002 return ret; 5003 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 5004 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 5005 if (items->mask != NULL && 5006 ((const struct rte_flow_item_ipv4 *) 5007 items->mask)->hdr.next_proto_id) { 5008 next_protocol = 5009 ((const struct rte_flow_item_ipv4 *) 5010 (items->spec))->hdr.next_proto_id; 5011 next_protocol &= 5012 ((const struct rte_flow_item_ipv4 *) 5013 (items->mask))->hdr.next_proto_id; 5014 } else { 5015 /* Reset for inner layer. */ 5016 next_protocol = 0xff; 5017 } 5018 break; 5019 case RTE_FLOW_ITEM_TYPE_IPV6: 5020 mlx5_flow_tunnel_ip_check(items, next_protocol, 5021 &item_flags, &tunnel); 5022 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 5023 last_item, 5024 ether_type, 5025 &nic_ipv6_mask, 5026 error); 5027 if (ret < 0) 5028 return ret; 5029 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 5030 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 5031 if (items->mask != NULL && 5032 ((const struct rte_flow_item_ipv6 *) 5033 items->mask)->hdr.proto) { 5034 item_ipv6_proto = 5035 ((const struct rte_flow_item_ipv6 *) 5036 items->spec)->hdr.proto; 5037 next_protocol = 5038 ((const struct rte_flow_item_ipv6 *) 5039 items->spec)->hdr.proto; 5040 next_protocol &= 5041 ((const struct rte_flow_item_ipv6 *) 5042 items->mask)->hdr.proto; 5043 } else { 5044 /* Reset for inner layer. */ 5045 next_protocol = 0xff; 5046 } 5047 break; 5048 case RTE_FLOW_ITEM_TYPE_TCP: 5049 ret = mlx5_flow_validate_item_tcp 5050 (items, item_flags, 5051 next_protocol, 5052 &nic_tcp_mask, 5053 error); 5054 if (ret < 0) 5055 return ret; 5056 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 5057 MLX5_FLOW_LAYER_OUTER_L4_TCP; 5058 break; 5059 case RTE_FLOW_ITEM_TYPE_UDP: 5060 ret = mlx5_flow_validate_item_udp(items, item_flags, 5061 next_protocol, 5062 error); 5063 if (ret < 0) 5064 return ret; 5065 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 5066 MLX5_FLOW_LAYER_OUTER_L4_UDP; 5067 break; 5068 case RTE_FLOW_ITEM_TYPE_GRE: 5069 ret = mlx5_flow_validate_item_gre(items, item_flags, 5070 next_protocol, error); 5071 if (ret < 0) 5072 return ret; 5073 gre_item = items; 5074 last_item = MLX5_FLOW_LAYER_GRE; 5075 break; 5076 case RTE_FLOW_ITEM_TYPE_NVGRE: 5077 ret = mlx5_flow_validate_item_nvgre(items, item_flags, 5078 next_protocol, 5079 error); 5080 if (ret < 0) 5081 return ret; 5082 last_item = MLX5_FLOW_LAYER_NVGRE; 5083 break; 5084 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 5085 ret = mlx5_flow_validate_item_gre_key 5086 (items, item_flags, gre_item, error); 5087 if (ret < 0) 5088 return ret; 5089 last_item = MLX5_FLOW_LAYER_GRE_KEY; 5090 break; 5091 case RTE_FLOW_ITEM_TYPE_VXLAN: 5092 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 5093 error); 5094 if (ret < 0) 5095 return ret; 5096 last_item = MLX5_FLOW_LAYER_VXLAN; 5097 break; 5098 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 5099 ret = mlx5_flow_validate_item_vxlan_gpe(items, 5100 item_flags, dev, 5101 error); 5102 if (ret < 0) 5103 return ret; 5104 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 5105 break; 5106 case RTE_FLOW_ITEM_TYPE_GENEVE: 5107 ret = mlx5_flow_validate_item_geneve(items, 5108 item_flags, dev, 5109 error); 5110 if (ret < 0) 5111 return ret; 5112 last_item = MLX5_FLOW_LAYER_GENEVE; 5113 break; 5114 case RTE_FLOW_ITEM_TYPE_MPLS: 5115 ret = mlx5_flow_validate_item_mpls(dev, items, 5116 item_flags, 5117 last_item, error); 5118 if (ret < 0) 5119 return ret; 5120 last_item = MLX5_FLOW_LAYER_MPLS; 5121 break; 5122 5123 case RTE_FLOW_ITEM_TYPE_MARK: 5124 ret = flow_dv_validate_item_mark(dev, items, attr, 5125 error); 5126 if (ret < 0) 5127 return ret; 5128 last_item = MLX5_FLOW_ITEM_MARK; 5129 break; 5130 case RTE_FLOW_ITEM_TYPE_META: 5131 ret = flow_dv_validate_item_meta(dev, items, attr, 5132 error); 5133 if (ret < 0) 5134 return ret; 5135 last_item = MLX5_FLOW_ITEM_METADATA; 5136 break; 5137 case RTE_FLOW_ITEM_TYPE_ICMP: 5138 ret = mlx5_flow_validate_item_icmp(items, item_flags, 5139 next_protocol, 5140 error); 5141 if (ret < 0) 5142 return ret; 5143 last_item = MLX5_FLOW_LAYER_ICMP; 5144 break; 5145 case RTE_FLOW_ITEM_TYPE_ICMP6: 5146 ret = mlx5_flow_validate_item_icmp6(items, item_flags, 5147 next_protocol, 5148 error); 5149 if (ret < 0) 5150 return ret; 5151 item_ipv6_proto = IPPROTO_ICMPV6; 5152 last_item = MLX5_FLOW_LAYER_ICMP6; 5153 break; 5154 case RTE_FLOW_ITEM_TYPE_TAG: 5155 ret = flow_dv_validate_item_tag(dev, items, 5156 attr, error); 5157 if (ret < 0) 5158 return ret; 5159 last_item = MLX5_FLOW_ITEM_TAG; 5160 break; 5161 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 5162 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 5163 break; 5164 case RTE_FLOW_ITEM_TYPE_GTP: 5165 ret = flow_dv_validate_item_gtp(dev, items, item_flags, 5166 error); 5167 if (ret < 0) 5168 return ret; 5169 last_item = MLX5_FLOW_LAYER_GTP; 5170 break; 5171 default: 5172 return rte_flow_error_set(error, ENOTSUP, 5173 RTE_FLOW_ERROR_TYPE_ITEM, 5174 NULL, "item not supported"); 5175 } 5176 item_flags |= last_item; 5177 } 5178 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5179 int type = actions->type; 5180 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 5181 return rte_flow_error_set(error, ENOTSUP, 5182 RTE_FLOW_ERROR_TYPE_ACTION, 5183 actions, "too many actions"); 5184 switch (type) { 5185 case RTE_FLOW_ACTION_TYPE_VOID: 5186 break; 5187 case RTE_FLOW_ACTION_TYPE_PORT_ID: 5188 ret = flow_dv_validate_action_port_id(dev, 5189 action_flags, 5190 actions, 5191 attr, 5192 error); 5193 if (ret) 5194 return ret; 5195 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 5196 ++actions_n; 5197 break; 5198 case RTE_FLOW_ACTION_TYPE_FLAG: 5199 ret = flow_dv_validate_action_flag(dev, action_flags, 5200 attr, error); 5201 if (ret < 0) 5202 return ret; 5203 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5204 /* Count all modify-header actions as one. */ 5205 if (!(action_flags & 5206 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5207 ++actions_n; 5208 action_flags |= MLX5_FLOW_ACTION_FLAG | 5209 MLX5_FLOW_ACTION_MARK_EXT; 5210 } else { 5211 action_flags |= MLX5_FLOW_ACTION_FLAG; 5212 ++actions_n; 5213 } 5214 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5215 break; 5216 case RTE_FLOW_ACTION_TYPE_MARK: 5217 ret = flow_dv_validate_action_mark(dev, actions, 5218 action_flags, 5219 attr, error); 5220 if (ret < 0) 5221 return ret; 5222 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 5223 /* Count all modify-header actions as one. */ 5224 if (!(action_flags & 5225 MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5226 ++actions_n; 5227 action_flags |= MLX5_FLOW_ACTION_MARK | 5228 MLX5_FLOW_ACTION_MARK_EXT; 5229 } else { 5230 action_flags |= MLX5_FLOW_ACTION_MARK; 5231 ++actions_n; 5232 } 5233 rw_act_num += MLX5_ACT_NUM_SET_MARK; 5234 break; 5235 case RTE_FLOW_ACTION_TYPE_SET_META: 5236 ret = flow_dv_validate_action_set_meta(dev, actions, 5237 action_flags, 5238 attr, error); 5239 if (ret < 0) 5240 return ret; 5241 /* Count all modify-header actions as one action. */ 5242 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5243 ++actions_n; 5244 action_flags |= MLX5_FLOW_ACTION_SET_META; 5245 rw_act_num += MLX5_ACT_NUM_SET_META; 5246 break; 5247 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5248 ret = flow_dv_validate_action_set_tag(dev, actions, 5249 action_flags, 5250 attr, error); 5251 if (ret < 0) 5252 return ret; 5253 /* Count all modify-header actions as one action. */ 5254 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5255 ++actions_n; 5256 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 5257 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5258 break; 5259 case RTE_FLOW_ACTION_TYPE_DROP: 5260 ret = mlx5_flow_validate_action_drop(action_flags, 5261 attr, error); 5262 if (ret < 0) 5263 return ret; 5264 action_flags |= MLX5_FLOW_ACTION_DROP; 5265 ++actions_n; 5266 break; 5267 case RTE_FLOW_ACTION_TYPE_QUEUE: 5268 ret = mlx5_flow_validate_action_queue(actions, 5269 action_flags, dev, 5270 attr, error); 5271 if (ret < 0) 5272 return ret; 5273 queue_index = ((const struct rte_flow_action_queue *) 5274 (actions->conf))->index; 5275 action_flags |= MLX5_FLOW_ACTION_QUEUE; 5276 ++actions_n; 5277 break; 5278 case RTE_FLOW_ACTION_TYPE_RSS: 5279 rss = actions->conf; 5280 ret = mlx5_flow_validate_action_rss(actions, 5281 action_flags, dev, 5282 attr, item_flags, 5283 error); 5284 if (ret < 0) 5285 return ret; 5286 if (rss != NULL && rss->queue_num) 5287 queue_index = rss->queue[0]; 5288 action_flags |= MLX5_FLOW_ACTION_RSS; 5289 ++actions_n; 5290 break; 5291 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 5292 ret = 5293 mlx5_flow_validate_action_default_miss(action_flags, 5294 attr, error); 5295 if (ret < 0) 5296 return ret; 5297 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 5298 ++actions_n; 5299 break; 5300 case RTE_FLOW_ACTION_TYPE_COUNT: 5301 ret = flow_dv_validate_action_count(dev, error); 5302 if (ret < 0) 5303 return ret; 5304 action_flags |= MLX5_FLOW_ACTION_COUNT; 5305 ++actions_n; 5306 break; 5307 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5308 if (flow_dv_validate_action_pop_vlan(dev, 5309 action_flags, 5310 actions, 5311 item_flags, attr, 5312 error)) 5313 return -rte_errno; 5314 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 5315 ++actions_n; 5316 break; 5317 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5318 ret = flow_dv_validate_action_push_vlan(dev, 5319 action_flags, 5320 vlan_m, 5321 actions, attr, 5322 error); 5323 if (ret < 0) 5324 return ret; 5325 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 5326 ++actions_n; 5327 break; 5328 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5329 ret = flow_dv_validate_action_set_vlan_pcp 5330 (action_flags, actions, error); 5331 if (ret < 0) 5332 return ret; 5333 /* Count PCP with push_vlan command. */ 5334 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP; 5335 break; 5336 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5337 ret = flow_dv_validate_action_set_vlan_vid 5338 (item_flags, action_flags, 5339 actions, error); 5340 if (ret < 0) 5341 return ret; 5342 /* Count VID with push_vlan command. */ 5343 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 5344 rw_act_num += MLX5_ACT_NUM_MDF_VID; 5345 break; 5346 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5347 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5348 ret = flow_dv_validate_action_l2_encap(dev, 5349 action_flags, 5350 actions, attr, 5351 error); 5352 if (ret < 0) 5353 return ret; 5354 action_flags |= MLX5_FLOW_ACTION_ENCAP; 5355 ++actions_n; 5356 break; 5357 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5358 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5359 ret = flow_dv_validate_action_decap(dev, action_flags, 5360 attr, error); 5361 if (ret < 0) 5362 return ret; 5363 action_flags |= MLX5_FLOW_ACTION_DECAP; 5364 ++actions_n; 5365 break; 5366 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5367 ret = flow_dv_validate_action_raw_encap_decap 5368 (dev, NULL, actions->conf, attr, &action_flags, 5369 &actions_n, error); 5370 if (ret < 0) 5371 return ret; 5372 break; 5373 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5374 decap = actions->conf; 5375 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5376 ; 5377 if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5378 encap = NULL; 5379 actions--; 5380 } else { 5381 encap = actions->conf; 5382 } 5383 ret = flow_dv_validate_action_raw_encap_decap 5384 (dev, 5385 decap ? decap : &empty_decap, encap, 5386 attr, &action_flags, &actions_n, 5387 error); 5388 if (ret < 0) 5389 return ret; 5390 break; 5391 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5392 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5393 ret = flow_dv_validate_action_modify_mac(action_flags, 5394 actions, 5395 item_flags, 5396 error); 5397 if (ret < 0) 5398 return ret; 5399 /* Count all modify-header actions as one action. */ 5400 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5401 ++actions_n; 5402 action_flags |= actions->type == 5403 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 5404 MLX5_FLOW_ACTION_SET_MAC_SRC : 5405 MLX5_FLOW_ACTION_SET_MAC_DST; 5406 /* 5407 * Even if the source and destination MAC addresses have 5408 * overlap in the header with 4B alignment, the convert 5409 * function will handle them separately and 4 SW actions 5410 * will be created. And 2 actions will be added each 5411 * time no matter how many bytes of address will be set. 5412 */ 5413 rw_act_num += MLX5_ACT_NUM_MDF_MAC; 5414 break; 5415 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5416 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5417 ret = flow_dv_validate_action_modify_ipv4(action_flags, 5418 actions, 5419 item_flags, 5420 error); 5421 if (ret < 0) 5422 return ret; 5423 /* Count all modify-header actions as one action. */ 5424 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5425 ++actions_n; 5426 action_flags |= actions->type == 5427 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 5428 MLX5_FLOW_ACTION_SET_IPV4_SRC : 5429 MLX5_FLOW_ACTION_SET_IPV4_DST; 5430 rw_act_num += MLX5_ACT_NUM_MDF_IPV4; 5431 break; 5432 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5433 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5434 ret = flow_dv_validate_action_modify_ipv6(action_flags, 5435 actions, 5436 item_flags, 5437 error); 5438 if (ret < 0) 5439 return ret; 5440 if (item_ipv6_proto == IPPROTO_ICMPV6) 5441 return rte_flow_error_set(error, ENOTSUP, 5442 RTE_FLOW_ERROR_TYPE_ACTION, 5443 actions, 5444 "Can't change header " 5445 "with ICMPv6 proto"); 5446 /* Count all modify-header actions as one action. */ 5447 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5448 ++actions_n; 5449 action_flags |= actions->type == 5450 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 5451 MLX5_FLOW_ACTION_SET_IPV6_SRC : 5452 MLX5_FLOW_ACTION_SET_IPV6_DST; 5453 rw_act_num += MLX5_ACT_NUM_MDF_IPV6; 5454 break; 5455 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5456 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5457 ret = flow_dv_validate_action_modify_tp(action_flags, 5458 actions, 5459 item_flags, 5460 error); 5461 if (ret < 0) 5462 return ret; 5463 /* Count all modify-header actions as one action. */ 5464 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5465 ++actions_n; 5466 action_flags |= actions->type == 5467 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 5468 MLX5_FLOW_ACTION_SET_TP_SRC : 5469 MLX5_FLOW_ACTION_SET_TP_DST; 5470 rw_act_num += MLX5_ACT_NUM_MDF_PORT; 5471 break; 5472 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5473 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5474 ret = flow_dv_validate_action_modify_ttl(action_flags, 5475 actions, 5476 item_flags, 5477 error); 5478 if (ret < 0) 5479 return ret; 5480 /* Count all modify-header actions as one action. */ 5481 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5482 ++actions_n; 5483 action_flags |= actions->type == 5484 RTE_FLOW_ACTION_TYPE_SET_TTL ? 5485 MLX5_FLOW_ACTION_SET_TTL : 5486 MLX5_FLOW_ACTION_DEC_TTL; 5487 rw_act_num += MLX5_ACT_NUM_MDF_TTL; 5488 break; 5489 case RTE_FLOW_ACTION_TYPE_JUMP: 5490 ret = flow_dv_validate_action_jump(actions, 5491 action_flags, 5492 attr, external, 5493 error); 5494 if (ret) 5495 return ret; 5496 ++actions_n; 5497 action_flags |= MLX5_FLOW_ACTION_JUMP; 5498 break; 5499 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5500 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5501 ret = flow_dv_validate_action_modify_tcp_seq 5502 (action_flags, 5503 actions, 5504 item_flags, 5505 error); 5506 if (ret < 0) 5507 return ret; 5508 /* Count all modify-header actions as one action. */ 5509 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5510 ++actions_n; 5511 action_flags |= actions->type == 5512 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 5513 MLX5_FLOW_ACTION_INC_TCP_SEQ : 5514 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 5515 rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ; 5516 break; 5517 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5518 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5519 ret = flow_dv_validate_action_modify_tcp_ack 5520 (action_flags, 5521 actions, 5522 item_flags, 5523 error); 5524 if (ret < 0) 5525 return ret; 5526 /* Count all modify-header actions as one action. */ 5527 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5528 ++actions_n; 5529 action_flags |= actions->type == 5530 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 5531 MLX5_FLOW_ACTION_INC_TCP_ACK : 5532 MLX5_FLOW_ACTION_DEC_TCP_ACK; 5533 rw_act_num += MLX5_ACT_NUM_MDF_TCPACK; 5534 break; 5535 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 5536 break; 5537 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 5538 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 5539 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5540 break; 5541 case RTE_FLOW_ACTION_TYPE_METER: 5542 ret = mlx5_flow_validate_action_meter(dev, 5543 action_flags, 5544 actions, attr, 5545 error); 5546 if (ret < 0) 5547 return ret; 5548 action_flags |= MLX5_FLOW_ACTION_METER; 5549 ++actions_n; 5550 /* Meter action will add one more TAG action. */ 5551 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5552 break; 5553 case RTE_FLOW_ACTION_TYPE_AGE: 5554 ret = flow_dv_validate_action_age(action_flags, 5555 actions, dev, 5556 error); 5557 if (ret < 0) 5558 return ret; 5559 action_flags |= MLX5_FLOW_ACTION_AGE; 5560 ++actions_n; 5561 break; 5562 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5563 ret = flow_dv_validate_action_modify_ipv4_dscp 5564 (action_flags, 5565 actions, 5566 item_flags, 5567 error); 5568 if (ret < 0) 5569 return ret; 5570 /* Count all modify-header actions as one action. */ 5571 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5572 ++actions_n; 5573 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 5574 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5575 break; 5576 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5577 ret = flow_dv_validate_action_modify_ipv6_dscp 5578 (action_flags, 5579 actions, 5580 item_flags, 5581 error); 5582 if (ret < 0) 5583 return ret; 5584 /* Count all modify-header actions as one action. */ 5585 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 5586 ++actions_n; 5587 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 5588 rw_act_num += MLX5_ACT_NUM_SET_DSCP; 5589 break; 5590 default: 5591 return rte_flow_error_set(error, ENOTSUP, 5592 RTE_FLOW_ERROR_TYPE_ACTION, 5593 actions, 5594 "action not supported"); 5595 } 5596 } 5597 /* 5598 * Validate the drop action mutual exclusion with other actions. 5599 * Drop action is mutually-exclusive with any other action, except for 5600 * Count action. 5601 */ 5602 if ((action_flags & MLX5_FLOW_ACTION_DROP) && 5603 (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT))) 5604 return rte_flow_error_set(error, EINVAL, 5605 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 5606 "Drop action is mutually-exclusive " 5607 "with any other action, except for " 5608 "Count action"); 5609 /* Eswitch has few restrictions on using items and actions */ 5610 if (attr->transfer) { 5611 if (!mlx5_flow_ext_mreg_supported(dev) && 5612 action_flags & MLX5_FLOW_ACTION_FLAG) 5613 return rte_flow_error_set(error, ENOTSUP, 5614 RTE_FLOW_ERROR_TYPE_ACTION, 5615 NULL, 5616 "unsupported action FLAG"); 5617 if (!mlx5_flow_ext_mreg_supported(dev) && 5618 action_flags & MLX5_FLOW_ACTION_MARK) 5619 return rte_flow_error_set(error, ENOTSUP, 5620 RTE_FLOW_ERROR_TYPE_ACTION, 5621 NULL, 5622 "unsupported action MARK"); 5623 if (action_flags & MLX5_FLOW_ACTION_QUEUE) 5624 return rte_flow_error_set(error, ENOTSUP, 5625 RTE_FLOW_ERROR_TYPE_ACTION, 5626 NULL, 5627 "unsupported action QUEUE"); 5628 if (action_flags & MLX5_FLOW_ACTION_RSS) 5629 return rte_flow_error_set(error, ENOTSUP, 5630 RTE_FLOW_ERROR_TYPE_ACTION, 5631 NULL, 5632 "unsupported action RSS"); 5633 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 5634 return rte_flow_error_set(error, EINVAL, 5635 RTE_FLOW_ERROR_TYPE_ACTION, 5636 actions, 5637 "no fate action is found"); 5638 } else { 5639 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 5640 return rte_flow_error_set(error, EINVAL, 5641 RTE_FLOW_ERROR_TYPE_ACTION, 5642 actions, 5643 "no fate action is found"); 5644 } 5645 /* Continue validation for Xcap actions.*/ 5646 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF || 5647 mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) { 5648 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) == 5649 MLX5_FLOW_XCAP_ACTIONS) 5650 return rte_flow_error_set(error, ENOTSUP, 5651 RTE_FLOW_ERROR_TYPE_ACTION, 5652 NULL, "encap and decap " 5653 "combination aren't supported"); 5654 if (!attr->transfer && attr->ingress && (action_flags & 5655 MLX5_FLOW_ACTION_ENCAP)) 5656 return rte_flow_error_set(error, ENOTSUP, 5657 RTE_FLOW_ERROR_TYPE_ACTION, 5658 NULL, "encap is not supported" 5659 " for ingress traffic"); 5660 } 5661 /* Hairpin flow will add one more TAG action. */ 5662 if (hairpin > 0) 5663 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5664 /* extra metadata enabled: one more TAG action will be add. */ 5665 if (dev_conf->dv_flow_en && 5666 dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 5667 mlx5_flow_ext_mreg_supported(dev)) 5668 rw_act_num += MLX5_ACT_NUM_SET_TAG; 5669 if ((uint32_t)rw_act_num > 5670 flow_dv_modify_hdr_action_max(dev, is_root)) { 5671 return rte_flow_error_set(error, ENOTSUP, 5672 RTE_FLOW_ERROR_TYPE_ACTION, 5673 NULL, "too many header modify" 5674 " actions to support"); 5675 } 5676 return 0; 5677 } 5678 5679 /** 5680 * Internal preparation function. Allocates the DV flow size, 5681 * this size is constant. 5682 * 5683 * @param[in] dev 5684 * Pointer to the rte_eth_dev structure. 5685 * @param[in] attr 5686 * Pointer to the flow attributes. 5687 * @param[in] items 5688 * Pointer to the list of items. 5689 * @param[in] actions 5690 * Pointer to the list of actions. 5691 * @param[out] error 5692 * Pointer to the error structure. 5693 * 5694 * @return 5695 * Pointer to mlx5_flow object on success, 5696 * otherwise NULL and rte_errno is set. 5697 */ 5698 static struct mlx5_flow * 5699 flow_dv_prepare(struct rte_eth_dev *dev, 5700 const struct rte_flow_attr *attr __rte_unused, 5701 const struct rte_flow_item items[] __rte_unused, 5702 const struct rte_flow_action actions[] __rte_unused, 5703 struct rte_flow_error *error) 5704 { 5705 uint32_t handle_idx = 0; 5706 struct mlx5_flow *dev_flow; 5707 struct mlx5_flow_handle *dev_handle; 5708 struct mlx5_priv *priv = dev->data->dev_private; 5709 5710 /* In case of corrupting the memory. */ 5711 if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { 5712 rte_flow_error_set(error, ENOSPC, 5713 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5714 "not free temporary device flow"); 5715 return NULL; 5716 } 5717 dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 5718 &handle_idx); 5719 if (!dev_handle) { 5720 rte_flow_error_set(error, ENOMEM, 5721 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5722 "not enough memory to create flow handle"); 5723 return NULL; 5724 } 5725 /* No multi-thread supporting. */ 5726 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; 5727 dev_flow->handle = dev_handle; 5728 dev_flow->handle_idx = handle_idx; 5729 dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); 5730 /* 5731 * The matching value needs to be cleared to 0 before using. In the 5732 * past, it will be automatically cleared when using rte_*alloc 5733 * API. The time consumption will be almost the same as before. 5734 */ 5735 memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param)); 5736 dev_flow->ingress = attr->ingress; 5737 dev_flow->dv.transfer = attr->transfer; 5738 return dev_flow; 5739 } 5740 5741 #ifdef RTE_LIBRTE_MLX5_DEBUG 5742 /** 5743 * Sanity check for match mask and value. Similar to check_valid_spec() in 5744 * kernel driver. If unmasked bit is present in value, it returns failure. 5745 * 5746 * @param match_mask 5747 * pointer to match mask buffer. 5748 * @param match_value 5749 * pointer to match value buffer. 5750 * 5751 * @return 5752 * 0 if valid, -EINVAL otherwise. 5753 */ 5754 static int 5755 flow_dv_check_valid_spec(void *match_mask, void *match_value) 5756 { 5757 uint8_t *m = match_mask; 5758 uint8_t *v = match_value; 5759 unsigned int i; 5760 5761 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { 5762 if (v[i] & ~m[i]) { 5763 DRV_LOG(ERR, 5764 "match_value differs from match_criteria" 5765 " %p[%u] != %p[%u]", 5766 match_value, i, match_mask, i); 5767 return -EINVAL; 5768 } 5769 } 5770 return 0; 5771 } 5772 #endif 5773 5774 /** 5775 * Add match of ip_version. 5776 * 5777 * @param[in] group 5778 * Flow group. 5779 * @param[in] headers_v 5780 * Values header pointer. 5781 * @param[in] headers_m 5782 * Masks header pointer. 5783 * @param[in] ip_version 5784 * The IP version to set. 5785 */ 5786 static inline void 5787 flow_dv_set_match_ip_version(uint32_t group, 5788 void *headers_v, 5789 void *headers_m, 5790 uint8_t ip_version) 5791 { 5792 if (group == 0) 5793 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 5794 else 5795 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 5796 ip_version); 5797 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version); 5798 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0); 5799 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0); 5800 } 5801 5802 /** 5803 * Add Ethernet item to matcher and to the value. 5804 * 5805 * @param[in, out] matcher 5806 * Flow matcher. 5807 * @param[in, out] key 5808 * Flow matcher value. 5809 * @param[in] item 5810 * Flow pattern to translate. 5811 * @param[in] inner 5812 * Item is inner pattern. 5813 */ 5814 static void 5815 flow_dv_translate_item_eth(void *matcher, void *key, 5816 const struct rte_flow_item *item, int inner, 5817 uint32_t group) 5818 { 5819 const struct rte_flow_item_eth *eth_m = item->mask; 5820 const struct rte_flow_item_eth *eth_v = item->spec; 5821 const struct rte_flow_item_eth nic_mask = { 5822 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 5823 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 5824 .type = RTE_BE16(0xffff), 5825 }; 5826 void *headers_m; 5827 void *headers_v; 5828 char *l24_v; 5829 unsigned int i; 5830 5831 if (!eth_v) 5832 return; 5833 if (!eth_m) 5834 eth_m = &nic_mask; 5835 if (inner) { 5836 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5837 inner_headers); 5838 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 5839 } else { 5840 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5841 outer_headers); 5842 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 5843 } 5844 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), 5845 ð_m->dst, sizeof(eth_m->dst)); 5846 /* The value must be in the range of the mask. */ 5847 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); 5848 for (i = 0; i < sizeof(eth_m->dst); ++i) 5849 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 5850 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), 5851 ð_m->src, sizeof(eth_m->src)); 5852 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); 5853 /* The value must be in the range of the mask. */ 5854 for (i = 0; i < sizeof(eth_m->dst); ++i) 5855 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 5856 if (eth_v->type) { 5857 /* When ethertype is present set mask for tagged VLAN. */ 5858 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 5859 /* Set value for tagged VLAN if ethertype is 802.1Q. */ 5860 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 5861 eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 5862 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 5863 1); 5864 /* Return here to avoid setting match on ethertype. */ 5865 return; 5866 } 5867 } 5868 /* 5869 * HW supports match on one Ethertype, the Ethertype following the last 5870 * VLAN tag of the packet (see PRM). 5871 * Set match on ethertype only if ETH header is not followed by VLAN. 5872 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 5873 * ethertype, and use ip_version field instead. 5874 */ 5875 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 5876 eth_m->type == 0xFFFF) { 5877 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 5878 } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 5879 eth_m->type == 0xFFFF) { 5880 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 5881 } else { 5882 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 5883 rte_be_to_cpu_16(eth_m->type)); 5884 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 5885 ethertype); 5886 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 5887 } 5888 } 5889 5890 /** 5891 * Add VLAN item to matcher and to the value. 5892 * 5893 * @param[in, out] dev_flow 5894 * Flow descriptor. 5895 * @param[in, out] matcher 5896 * Flow matcher. 5897 * @param[in, out] key 5898 * Flow matcher value. 5899 * @param[in] item 5900 * Flow pattern to translate. 5901 * @param[in] inner 5902 * Item is inner pattern. 5903 */ 5904 static void 5905 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow, 5906 void *matcher, void *key, 5907 const struct rte_flow_item *item, 5908 int inner, uint32_t group) 5909 { 5910 const struct rte_flow_item_vlan *vlan_m = item->mask; 5911 const struct rte_flow_item_vlan *vlan_v = item->spec; 5912 void *headers_m; 5913 void *headers_v; 5914 uint16_t tci_m; 5915 uint16_t tci_v; 5916 5917 if (inner) { 5918 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5919 inner_headers); 5920 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 5921 } else { 5922 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 5923 outer_headers); 5924 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 5925 /* 5926 * This is workaround, masks are not supported, 5927 * and pre-validated. 5928 */ 5929 if (vlan_v) 5930 dev_flow->handle->vf_vlan.tag = 5931 rte_be_to_cpu_16(vlan_v->tci) & 0x0fff; 5932 } 5933 /* 5934 * When VLAN item exists in flow, mark packet as tagged, 5935 * even if TCI is not specified. 5936 */ 5937 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 5938 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 5939 if (!vlan_v) 5940 return; 5941 if (!vlan_m) 5942 vlan_m = &rte_flow_item_vlan_mask; 5943 tci_m = rte_be_to_cpu_16(vlan_m->tci); 5944 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 5945 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); 5946 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); 5947 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); 5948 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); 5949 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); 5950 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); 5951 /* 5952 * HW is optimized for IPv4/IPv6. In such cases, avoid setting 5953 * ethertype, and use ip_version field instead. 5954 */ 5955 if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) && 5956 vlan_m->inner_type == 0xFFFF) { 5957 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 5958 } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) && 5959 vlan_m->inner_type == 0xFFFF) { 5960 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 5961 } else { 5962 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 5963 rte_be_to_cpu_16(vlan_m->inner_type)); 5964 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 5965 rte_be_to_cpu_16(vlan_m->inner_type & 5966 vlan_v->inner_type)); 5967 } 5968 } 5969 5970 /** 5971 * Add IPV4 item to matcher and to the value. 5972 * 5973 * @param[in, out] matcher 5974 * Flow matcher. 5975 * @param[in, out] key 5976 * Flow matcher value. 5977 * @param[in] item 5978 * Flow pattern to translate. 5979 * @param[in] item_flags 5980 * Bit-fields that holds the items detected until now. 5981 * @param[in] inner 5982 * Item is inner pattern. 5983 * @param[in] group 5984 * The group to insert the rule. 5985 */ 5986 static void 5987 flow_dv_translate_item_ipv4(void *matcher, void *key, 5988 const struct rte_flow_item *item, 5989 const uint64_t item_flags, 5990 int inner, uint32_t group) 5991 { 5992 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 5993 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 5994 const struct rte_flow_item_ipv4 nic_mask = { 5995 .hdr = { 5996 .src_addr = RTE_BE32(0xffffffff), 5997 .dst_addr = RTE_BE32(0xffffffff), 5998 .type_of_service = 0xff, 5999 .next_proto_id = 0xff, 6000 .time_to_live = 0xff, 6001 }, 6002 }; 6003 void *headers_m; 6004 void *headers_v; 6005 char *l24_m; 6006 char *l24_v; 6007 uint8_t tos; 6008 6009 if (inner) { 6010 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6011 inner_headers); 6012 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6013 } else { 6014 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6015 outer_headers); 6016 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6017 } 6018 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4); 6019 /* 6020 * On outer header (which must contains L2), or inner header with L2, 6021 * set cvlan_tag mask bit to mark this packet as untagged. 6022 * This should be done even if item->spec is empty. 6023 */ 6024 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6025 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6026 if (!ipv4_v) 6027 return; 6028 if (!ipv4_m) 6029 ipv4_m = &nic_mask; 6030 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6031 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6032 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6033 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 6034 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 6035 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 6036 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6037 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6038 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6039 src_ipv4_src_ipv6.ipv4_layout.ipv4); 6040 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 6041 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 6042 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 6043 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 6044 ipv4_m->hdr.type_of_service); 6045 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 6046 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 6047 ipv4_m->hdr.type_of_service >> 2); 6048 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 6049 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6050 ipv4_m->hdr.next_proto_id); 6051 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6052 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 6053 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6054 ipv4_m->hdr.time_to_live); 6055 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6056 ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live); 6057 } 6058 6059 /** 6060 * Add IPV6 item to matcher and to the value. 6061 * 6062 * @param[in, out] matcher 6063 * Flow matcher. 6064 * @param[in, out] key 6065 * Flow matcher value. 6066 * @param[in] item 6067 * Flow pattern to translate. 6068 * @param[in] item_flags 6069 * Bit-fields that holds the items detected until now. 6070 * @param[in] inner 6071 * Item is inner pattern. 6072 * @param[in] group 6073 * The group to insert the rule. 6074 */ 6075 static void 6076 flow_dv_translate_item_ipv6(void *matcher, void *key, 6077 const struct rte_flow_item *item, 6078 const uint64_t item_flags, 6079 int inner, uint32_t group) 6080 { 6081 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 6082 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 6083 const struct rte_flow_item_ipv6 nic_mask = { 6084 .hdr = { 6085 .src_addr = 6086 "\xff\xff\xff\xff\xff\xff\xff\xff" 6087 "\xff\xff\xff\xff\xff\xff\xff\xff", 6088 .dst_addr = 6089 "\xff\xff\xff\xff\xff\xff\xff\xff" 6090 "\xff\xff\xff\xff\xff\xff\xff\xff", 6091 .vtc_flow = RTE_BE32(0xffffffff), 6092 .proto = 0xff, 6093 .hop_limits = 0xff, 6094 }, 6095 }; 6096 void *headers_m; 6097 void *headers_v; 6098 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6099 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6100 char *l24_m; 6101 char *l24_v; 6102 uint32_t vtc_m; 6103 uint32_t vtc_v; 6104 int i; 6105 int size; 6106 6107 if (inner) { 6108 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6109 inner_headers); 6110 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6111 } else { 6112 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6113 outer_headers); 6114 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6115 } 6116 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6); 6117 /* 6118 * On outer header (which must contains L2), or inner header with L2, 6119 * set cvlan_tag mask bit to mark this packet as untagged. 6120 * This should be done even if item->spec is empty. 6121 */ 6122 if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2) 6123 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 6124 if (!ipv6_v) 6125 return; 6126 if (!ipv6_m) 6127 ipv6_m = &nic_mask; 6128 size = sizeof(ipv6_m->hdr.dst_addr); 6129 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6130 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6131 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6132 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 6133 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 6134 for (i = 0; i < size; ++i) 6135 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 6136 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 6137 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6138 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 6139 src_ipv4_src_ipv6.ipv6_layout.ipv6); 6140 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 6141 for (i = 0; i < size; ++i) 6142 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 6143 /* TOS. */ 6144 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 6145 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 6146 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 6147 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 6148 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 6149 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 6150 /* Label. */ 6151 if (inner) { 6152 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 6153 vtc_m); 6154 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 6155 vtc_v); 6156 } else { 6157 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 6158 vtc_m); 6159 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 6160 vtc_v); 6161 } 6162 /* Protocol. */ 6163 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 6164 ipv6_m->hdr.proto); 6165 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6166 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 6167 /* Hop limit. */ 6168 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit, 6169 ipv6_m->hdr.hop_limits); 6170 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit, 6171 ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits); 6172 } 6173 6174 /** 6175 * Add TCP item to matcher and to the value. 6176 * 6177 * @param[in, out] matcher 6178 * Flow matcher. 6179 * @param[in, out] key 6180 * Flow matcher value. 6181 * @param[in] item 6182 * Flow pattern to translate. 6183 * @param[in] inner 6184 * Item is inner pattern. 6185 */ 6186 static void 6187 flow_dv_translate_item_tcp(void *matcher, void *key, 6188 const struct rte_flow_item *item, 6189 int inner) 6190 { 6191 const struct rte_flow_item_tcp *tcp_m = item->mask; 6192 const struct rte_flow_item_tcp *tcp_v = item->spec; 6193 void *headers_m; 6194 void *headers_v; 6195 6196 if (inner) { 6197 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6198 inner_headers); 6199 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6200 } else { 6201 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6202 outer_headers); 6203 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6204 } 6205 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6206 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 6207 if (!tcp_v) 6208 return; 6209 if (!tcp_m) 6210 tcp_m = &rte_flow_item_tcp_mask; 6211 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 6212 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 6213 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 6214 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 6215 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 6216 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 6217 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 6218 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 6219 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, 6220 tcp_m->hdr.tcp_flags); 6221 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 6222 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); 6223 } 6224 6225 /** 6226 * Add UDP item to matcher and to the value. 6227 * 6228 * @param[in, out] matcher 6229 * Flow matcher. 6230 * @param[in, out] key 6231 * Flow matcher value. 6232 * @param[in] item 6233 * Flow pattern to translate. 6234 * @param[in] inner 6235 * Item is inner pattern. 6236 */ 6237 static void 6238 flow_dv_translate_item_udp(void *matcher, void *key, 6239 const struct rte_flow_item *item, 6240 int inner) 6241 { 6242 const struct rte_flow_item_udp *udp_m = item->mask; 6243 const struct rte_flow_item_udp *udp_v = item->spec; 6244 void *headers_m; 6245 void *headers_v; 6246 6247 if (inner) { 6248 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6249 inner_headers); 6250 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6251 } else { 6252 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6253 outer_headers); 6254 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6255 } 6256 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6257 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 6258 if (!udp_v) 6259 return; 6260 if (!udp_m) 6261 udp_m = &rte_flow_item_udp_mask; 6262 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 6263 rte_be_to_cpu_16(udp_m->hdr.src_port)); 6264 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 6265 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 6266 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 6267 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 6268 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6269 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 6270 } 6271 6272 /** 6273 * Add GRE optional Key item to matcher and to the value. 6274 * 6275 * @param[in, out] matcher 6276 * Flow matcher. 6277 * @param[in, out] key 6278 * Flow matcher value. 6279 * @param[in] item 6280 * Flow pattern to translate. 6281 * @param[in] inner 6282 * Item is inner pattern. 6283 */ 6284 static void 6285 flow_dv_translate_item_gre_key(void *matcher, void *key, 6286 const struct rte_flow_item *item) 6287 { 6288 const rte_be32_t *key_m = item->mask; 6289 const rte_be32_t *key_v = item->spec; 6290 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6291 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6292 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 6293 6294 /* GRE K bit must be on and should already be validated */ 6295 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); 6296 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); 6297 if (!key_v) 6298 return; 6299 if (!key_m) 6300 key_m = &gre_key_default_mask; 6301 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, 6302 rte_be_to_cpu_32(*key_m) >> 8); 6303 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, 6304 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); 6305 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, 6306 rte_be_to_cpu_32(*key_m) & 0xFF); 6307 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, 6308 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); 6309 } 6310 6311 /** 6312 * Add GRE item to matcher and to the value. 6313 * 6314 * @param[in, out] matcher 6315 * Flow matcher. 6316 * @param[in, out] key 6317 * Flow matcher value. 6318 * @param[in] item 6319 * Flow pattern to translate. 6320 * @param[in] inner 6321 * Item is inner pattern. 6322 */ 6323 static void 6324 flow_dv_translate_item_gre(void *matcher, void *key, 6325 const struct rte_flow_item *item, 6326 int inner) 6327 { 6328 const struct rte_flow_item_gre *gre_m = item->mask; 6329 const struct rte_flow_item_gre *gre_v = item->spec; 6330 void *headers_m; 6331 void *headers_v; 6332 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6333 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6334 struct { 6335 union { 6336 __extension__ 6337 struct { 6338 uint16_t version:3; 6339 uint16_t rsvd0:9; 6340 uint16_t s_present:1; 6341 uint16_t k_present:1; 6342 uint16_t rsvd_bit1:1; 6343 uint16_t c_present:1; 6344 }; 6345 uint16_t value; 6346 }; 6347 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; 6348 6349 if (inner) { 6350 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6351 inner_headers); 6352 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6353 } else { 6354 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6355 outer_headers); 6356 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6357 } 6358 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6359 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 6360 if (!gre_v) 6361 return; 6362 if (!gre_m) 6363 gre_m = &rte_flow_item_gre_mask; 6364 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 6365 rte_be_to_cpu_16(gre_m->protocol)); 6366 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6367 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); 6368 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); 6369 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); 6370 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, 6371 gre_crks_rsvd0_ver_m.c_present); 6372 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, 6373 gre_crks_rsvd0_ver_v.c_present & 6374 gre_crks_rsvd0_ver_m.c_present); 6375 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 6376 gre_crks_rsvd0_ver_m.k_present); 6377 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 6378 gre_crks_rsvd0_ver_v.k_present & 6379 gre_crks_rsvd0_ver_m.k_present); 6380 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, 6381 gre_crks_rsvd0_ver_m.s_present); 6382 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, 6383 gre_crks_rsvd0_ver_v.s_present & 6384 gre_crks_rsvd0_ver_m.s_present); 6385 } 6386 6387 /** 6388 * Add NVGRE item to matcher and to the value. 6389 * 6390 * @param[in, out] matcher 6391 * Flow matcher. 6392 * @param[in, out] key 6393 * Flow matcher value. 6394 * @param[in] item 6395 * Flow pattern to translate. 6396 * @param[in] inner 6397 * Item is inner pattern. 6398 */ 6399 static void 6400 flow_dv_translate_item_nvgre(void *matcher, void *key, 6401 const struct rte_flow_item *item, 6402 int inner) 6403 { 6404 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 6405 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 6406 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6407 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6408 const char *tni_flow_id_m = (const char *)nvgre_m->tni; 6409 const char *tni_flow_id_v = (const char *)nvgre_v->tni; 6410 char *gre_key_m; 6411 char *gre_key_v; 6412 int size; 6413 int i; 6414 6415 /* For NVGRE, GRE header fields must be set with defined values. */ 6416 const struct rte_flow_item_gre gre_spec = { 6417 .c_rsvd0_ver = RTE_BE16(0x2000), 6418 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB) 6419 }; 6420 const struct rte_flow_item_gre gre_mask = { 6421 .c_rsvd0_ver = RTE_BE16(0xB000), 6422 .protocol = RTE_BE16(UINT16_MAX), 6423 }; 6424 const struct rte_flow_item gre_item = { 6425 .spec = &gre_spec, 6426 .mask = &gre_mask, 6427 .last = NULL, 6428 }; 6429 flow_dv_translate_item_gre(matcher, key, &gre_item, inner); 6430 if (!nvgre_v) 6431 return; 6432 if (!nvgre_m) 6433 nvgre_m = &rte_flow_item_nvgre_mask; 6434 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 6435 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 6436 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 6437 memcpy(gre_key_m, tni_flow_id_m, size); 6438 for (i = 0; i < size; ++i) 6439 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 6440 } 6441 6442 /** 6443 * Add VXLAN item to matcher and to the value. 6444 * 6445 * @param[in, out] matcher 6446 * Flow matcher. 6447 * @param[in, out] key 6448 * Flow matcher value. 6449 * @param[in] item 6450 * Flow pattern to translate. 6451 * @param[in] inner 6452 * Item is inner pattern. 6453 */ 6454 static void 6455 flow_dv_translate_item_vxlan(void *matcher, void *key, 6456 const struct rte_flow_item *item, 6457 int inner) 6458 { 6459 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 6460 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 6461 void *headers_m; 6462 void *headers_v; 6463 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6464 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6465 char *vni_m; 6466 char *vni_v; 6467 uint16_t dport; 6468 int size; 6469 int i; 6470 6471 if (inner) { 6472 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6473 inner_headers); 6474 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6475 } else { 6476 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6477 outer_headers); 6478 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6479 } 6480 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6481 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6482 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6483 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6484 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6485 } 6486 if (!vxlan_v) 6487 return; 6488 if (!vxlan_m) 6489 vxlan_m = &rte_flow_item_vxlan_mask; 6490 size = sizeof(vxlan_m->vni); 6491 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 6492 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 6493 memcpy(vni_m, vxlan_m->vni, size); 6494 for (i = 0; i < size; ++i) 6495 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6496 } 6497 6498 /** 6499 * Add VXLAN-GPE item to matcher and to the value. 6500 * 6501 * @param[in, out] matcher 6502 * Flow matcher. 6503 * @param[in, out] key 6504 * Flow matcher value. 6505 * @param[in] item 6506 * Flow pattern to translate. 6507 * @param[in] inner 6508 * Item is inner pattern. 6509 */ 6510 6511 static void 6512 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key, 6513 const struct rte_flow_item *item, int inner) 6514 { 6515 const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask; 6516 const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec; 6517 void *headers_m; 6518 void *headers_v; 6519 void *misc_m = 6520 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3); 6521 void *misc_v = 6522 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 6523 char *vni_m; 6524 char *vni_v; 6525 uint16_t dport; 6526 int size; 6527 int i; 6528 uint8_t flags_m = 0xff; 6529 uint8_t flags_v = 0xc; 6530 6531 if (inner) { 6532 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6533 inner_headers); 6534 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6535 } else { 6536 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6537 outer_headers); 6538 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6539 } 6540 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 6541 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 6542 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6543 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6544 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6545 } 6546 if (!vxlan_v) 6547 return; 6548 if (!vxlan_m) 6549 vxlan_m = &rte_flow_item_vxlan_gpe_mask; 6550 size = sizeof(vxlan_m->vni); 6551 vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni); 6552 vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni); 6553 memcpy(vni_m, vxlan_m->vni, size); 6554 for (i = 0; i < size; ++i) 6555 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 6556 if (vxlan_m->flags) { 6557 flags_m = vxlan_m->flags; 6558 flags_v = vxlan_v->flags; 6559 } 6560 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m); 6561 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v); 6562 MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol, 6563 vxlan_m->protocol); 6564 MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol, 6565 vxlan_v->protocol); 6566 } 6567 6568 /** 6569 * Add Geneve item to matcher and to the value. 6570 * 6571 * @param[in, out] matcher 6572 * Flow matcher. 6573 * @param[in, out] key 6574 * Flow matcher value. 6575 * @param[in] item 6576 * Flow pattern to translate. 6577 * @param[in] inner 6578 * Item is inner pattern. 6579 */ 6580 6581 static void 6582 flow_dv_translate_item_geneve(void *matcher, void *key, 6583 const struct rte_flow_item *item, int inner) 6584 { 6585 const struct rte_flow_item_geneve *geneve_m = item->mask; 6586 const struct rte_flow_item_geneve *geneve_v = item->spec; 6587 void *headers_m; 6588 void *headers_v; 6589 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6590 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6591 uint16_t dport; 6592 uint16_t gbhdr_m; 6593 uint16_t gbhdr_v; 6594 char *vni_m; 6595 char *vni_v; 6596 size_t size, i; 6597 6598 if (inner) { 6599 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6600 inner_headers); 6601 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 6602 } else { 6603 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 6604 outer_headers); 6605 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6606 } 6607 dport = MLX5_UDP_PORT_GENEVE; 6608 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 6609 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 6610 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 6611 } 6612 if (!geneve_v) 6613 return; 6614 if (!geneve_m) 6615 geneve_m = &rte_flow_item_geneve_mask; 6616 size = sizeof(geneve_m->vni); 6617 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni); 6618 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni); 6619 memcpy(vni_m, geneve_m->vni, size); 6620 for (i = 0; i < size; ++i) 6621 vni_v[i] = vni_m[i] & geneve_v->vni[i]; 6622 MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, 6623 rte_be_to_cpu_16(geneve_m->protocol)); 6624 MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type, 6625 rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol)); 6626 gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0); 6627 gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0); 6628 MLX5_SET(fte_match_set_misc, misc_m, geneve_oam, 6629 MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6630 MLX5_SET(fte_match_set_misc, misc_v, geneve_oam, 6631 MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m)); 6632 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len, 6633 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6634 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len, 6635 MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) & 6636 MLX5_GENEVE_OPTLEN_VAL(gbhdr_m)); 6637 } 6638 6639 /** 6640 * Add MPLS item to matcher and to the value. 6641 * 6642 * @param[in, out] matcher 6643 * Flow matcher. 6644 * @param[in, out] key 6645 * Flow matcher value. 6646 * @param[in] item 6647 * Flow pattern to translate. 6648 * @param[in] prev_layer 6649 * The protocol layer indicated in previous item. 6650 * @param[in] inner 6651 * Item is inner pattern. 6652 */ 6653 static void 6654 flow_dv_translate_item_mpls(void *matcher, void *key, 6655 const struct rte_flow_item *item, 6656 uint64_t prev_layer, 6657 int inner) 6658 { 6659 const uint32_t *in_mpls_m = item->mask; 6660 const uint32_t *in_mpls_v = item->spec; 6661 uint32_t *out_mpls_m = 0; 6662 uint32_t *out_mpls_v = 0; 6663 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 6664 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 6665 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 6666 misc_parameters_2); 6667 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6668 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 6669 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 6670 6671 switch (prev_layer) { 6672 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6673 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff); 6674 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 6675 MLX5_UDP_PORT_MPLS); 6676 break; 6677 case MLX5_FLOW_LAYER_GRE: 6678 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); 6679 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 6680 RTE_ETHER_TYPE_MPLS); 6681 break; 6682 default: 6683 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 6684 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 6685 IPPROTO_MPLS); 6686 break; 6687 } 6688 if (!in_mpls_v) 6689 return; 6690 if (!in_mpls_m) 6691 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 6692 switch (prev_layer) { 6693 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 6694 out_mpls_m = 6695 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6696 outer_first_mpls_over_udp); 6697 out_mpls_v = 6698 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6699 outer_first_mpls_over_udp); 6700 break; 6701 case MLX5_FLOW_LAYER_GRE: 6702 out_mpls_m = 6703 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 6704 outer_first_mpls_over_gre); 6705 out_mpls_v = 6706 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 6707 outer_first_mpls_over_gre); 6708 break; 6709 default: 6710 /* Inner MPLS not over GRE is not supported. */ 6711 if (!inner) { 6712 out_mpls_m = 6713 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6714 misc2_m, 6715 outer_first_mpls); 6716 out_mpls_v = 6717 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 6718 misc2_v, 6719 outer_first_mpls); 6720 } 6721 break; 6722 } 6723 if (out_mpls_m && out_mpls_v) { 6724 *out_mpls_m = *in_mpls_m; 6725 *out_mpls_v = *in_mpls_v & *in_mpls_m; 6726 } 6727 } 6728 6729 /** 6730 * Add metadata register item to matcher 6731 * 6732 * @param[in, out] matcher 6733 * Flow matcher. 6734 * @param[in, out] key 6735 * Flow matcher value. 6736 * @param[in] reg_type 6737 * Type of device metadata register 6738 * @param[in] value 6739 * Register value 6740 * @param[in] mask 6741 * Register mask 6742 */ 6743 static void 6744 flow_dv_match_meta_reg(void *matcher, void *key, 6745 enum modify_reg reg_type, 6746 uint32_t data, uint32_t mask) 6747 { 6748 void *misc2_m = 6749 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 6750 void *misc2_v = 6751 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 6752 uint32_t temp; 6753 6754 data &= mask; 6755 switch (reg_type) { 6756 case REG_A: 6757 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask); 6758 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data); 6759 break; 6760 case REG_B: 6761 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask); 6762 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data); 6763 break; 6764 case REG_C_0: 6765 /* 6766 * The metadata register C0 field might be divided into 6767 * source vport index and META item value, we should set 6768 * this field according to specified mask, not as whole one. 6769 */ 6770 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0); 6771 temp |= mask; 6772 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp); 6773 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0); 6774 temp &= ~mask; 6775 temp |= data; 6776 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp); 6777 break; 6778 case REG_C_1: 6779 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask); 6780 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data); 6781 break; 6782 case REG_C_2: 6783 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask); 6784 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data); 6785 break; 6786 case REG_C_3: 6787 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask); 6788 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data); 6789 break; 6790 case REG_C_4: 6791 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask); 6792 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data); 6793 break; 6794 case REG_C_5: 6795 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask); 6796 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data); 6797 break; 6798 case REG_C_6: 6799 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask); 6800 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data); 6801 break; 6802 case REG_C_7: 6803 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask); 6804 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data); 6805 break; 6806 default: 6807 MLX5_ASSERT(false); 6808 break; 6809 } 6810 } 6811 6812 /** 6813 * Add MARK item to matcher 6814 * 6815 * @param[in] dev 6816 * The device to configure through. 6817 * @param[in, out] matcher 6818 * Flow matcher. 6819 * @param[in, out] key 6820 * Flow matcher value. 6821 * @param[in] item 6822 * Flow pattern to translate. 6823 */ 6824 static void 6825 flow_dv_translate_item_mark(struct rte_eth_dev *dev, 6826 void *matcher, void *key, 6827 const struct rte_flow_item *item) 6828 { 6829 struct mlx5_priv *priv = dev->data->dev_private; 6830 const struct rte_flow_item_mark *mark; 6831 uint32_t value; 6832 uint32_t mask; 6833 6834 mark = item->mask ? (const void *)item->mask : 6835 &rte_flow_item_mark_mask; 6836 mask = mark->id & priv->sh->dv_mark_mask; 6837 mark = (const void *)item->spec; 6838 MLX5_ASSERT(mark); 6839 value = mark->id & priv->sh->dv_mark_mask & mask; 6840 if (mask) { 6841 enum modify_reg reg; 6842 6843 /* Get the metadata register index for the mark. */ 6844 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL); 6845 MLX5_ASSERT(reg > 0); 6846 if (reg == REG_C_0) { 6847 struct mlx5_priv *priv = dev->data->dev_private; 6848 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 6849 uint32_t shl_c0 = rte_bsf32(msk_c0); 6850 6851 mask &= msk_c0; 6852 mask <<= shl_c0; 6853 value <<= shl_c0; 6854 } 6855 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 6856 } 6857 } 6858 6859 /** 6860 * Add META item to matcher 6861 * 6862 * @param[in] dev 6863 * The devich to configure through. 6864 * @param[in, out] matcher 6865 * Flow matcher. 6866 * @param[in, out] key 6867 * Flow matcher value. 6868 * @param[in] attr 6869 * Attributes of flow that includes this item. 6870 * @param[in] item 6871 * Flow pattern to translate. 6872 */ 6873 static void 6874 flow_dv_translate_item_meta(struct rte_eth_dev *dev, 6875 void *matcher, void *key, 6876 const struct rte_flow_attr *attr, 6877 const struct rte_flow_item *item) 6878 { 6879 const struct rte_flow_item_meta *meta_m; 6880 const struct rte_flow_item_meta *meta_v; 6881 6882 meta_m = (const void *)item->mask; 6883 if (!meta_m) 6884 meta_m = &rte_flow_item_meta_mask; 6885 meta_v = (const void *)item->spec; 6886 if (meta_v) { 6887 int reg; 6888 uint32_t value = meta_v->data; 6889 uint32_t mask = meta_m->data; 6890 6891 reg = flow_dv_get_metadata_reg(dev, attr, NULL); 6892 if (reg < 0) 6893 return; 6894 /* 6895 * In datapath code there is no endianness 6896 * coversions for perfromance reasons, all 6897 * pattern conversions are done in rte_flow. 6898 */ 6899 value = rte_cpu_to_be_32(value); 6900 mask = rte_cpu_to_be_32(mask); 6901 if (reg == REG_C_0) { 6902 struct mlx5_priv *priv = dev->data->dev_private; 6903 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 6904 uint32_t shl_c0 = rte_bsf32(msk_c0); 6905 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 6906 uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask); 6907 6908 value >>= shr_c0; 6909 mask >>= shr_c0; 6910 #endif 6911 value <<= shl_c0; 6912 mask <<= shl_c0; 6913 MLX5_ASSERT(msk_c0); 6914 MLX5_ASSERT(!(~msk_c0 & mask)); 6915 } 6916 flow_dv_match_meta_reg(matcher, key, reg, value, mask); 6917 } 6918 } 6919 6920 /** 6921 * Add vport metadata Reg C0 item to matcher 6922 * 6923 * @param[in, out] matcher 6924 * Flow matcher. 6925 * @param[in, out] key 6926 * Flow matcher value. 6927 * @param[in] reg 6928 * Flow pattern to translate. 6929 */ 6930 static void 6931 flow_dv_translate_item_meta_vport(void *matcher, void *key, 6932 uint32_t value, uint32_t mask) 6933 { 6934 flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask); 6935 } 6936 6937 /** 6938 * Add tag item to matcher 6939 * 6940 * @param[in] dev 6941 * The devich to configure through. 6942 * @param[in, out] matcher 6943 * Flow matcher. 6944 * @param[in, out] key 6945 * Flow matcher value. 6946 * @param[in] item 6947 * Flow pattern to translate. 6948 */ 6949 static void 6950 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev, 6951 void *matcher, void *key, 6952 const struct rte_flow_item *item) 6953 { 6954 const struct mlx5_rte_flow_item_tag *tag_v = item->spec; 6955 const struct mlx5_rte_flow_item_tag *tag_m = item->mask; 6956 uint32_t mask, value; 6957 6958 MLX5_ASSERT(tag_v); 6959 value = tag_v->data; 6960 mask = tag_m ? tag_m->data : UINT32_MAX; 6961 if (tag_v->id == REG_C_0) { 6962 struct mlx5_priv *priv = dev->data->dev_private; 6963 uint32_t msk_c0 = priv->sh->dv_regc0_mask; 6964 uint32_t shl_c0 = rte_bsf32(msk_c0); 6965 6966 mask &= msk_c0; 6967 mask <<= shl_c0; 6968 value <<= shl_c0; 6969 } 6970 flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask); 6971 } 6972 6973 /** 6974 * Add TAG item to matcher 6975 * 6976 * @param[in] dev 6977 * The devich to configure through. 6978 * @param[in, out] matcher 6979 * Flow matcher. 6980 * @param[in, out] key 6981 * Flow matcher value. 6982 * @param[in] item 6983 * Flow pattern to translate. 6984 */ 6985 static void 6986 flow_dv_translate_item_tag(struct rte_eth_dev *dev, 6987 void *matcher, void *key, 6988 const struct rte_flow_item *item) 6989 { 6990 const struct rte_flow_item_tag *tag_v = item->spec; 6991 const struct rte_flow_item_tag *tag_m = item->mask; 6992 enum modify_reg reg; 6993 6994 MLX5_ASSERT(tag_v); 6995 tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask; 6996 /* Get the metadata register index for the tag. */ 6997 reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL); 6998 MLX5_ASSERT(reg > 0); 6999 flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data); 7000 } 7001 7002 /** 7003 * Add source vport match to the specified matcher. 7004 * 7005 * @param[in, out] matcher 7006 * Flow matcher. 7007 * @param[in, out] key 7008 * Flow matcher value. 7009 * @param[in] port 7010 * Source vport value to match 7011 * @param[in] mask 7012 * Mask 7013 */ 7014 static void 7015 flow_dv_translate_item_source_vport(void *matcher, void *key, 7016 int16_t port, uint16_t mask) 7017 { 7018 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 7019 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 7020 7021 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); 7022 MLX5_SET(fte_match_set_misc, misc_v, source_port, port); 7023 } 7024 7025 /** 7026 * Translate port-id item to eswitch match on port-id. 7027 * 7028 * @param[in] dev 7029 * The devich to configure through. 7030 * @param[in, out] matcher 7031 * Flow matcher. 7032 * @param[in, out] key 7033 * Flow matcher value. 7034 * @param[in] item 7035 * Flow pattern to translate. 7036 * 7037 * @return 7038 * 0 on success, a negative errno value otherwise. 7039 */ 7040 static int 7041 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, 7042 void *key, const struct rte_flow_item *item) 7043 { 7044 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; 7045 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; 7046 struct mlx5_priv *priv; 7047 uint16_t mask, id; 7048 7049 mask = pid_m ? pid_m->id : 0xffff; 7050 id = pid_v ? pid_v->id : dev->data->port_id; 7051 priv = mlx5_port_to_eswitch_info(id, item == NULL); 7052 if (!priv) 7053 return -rte_errno; 7054 /* Translate to vport field or to metadata, depending on mode. */ 7055 if (priv->vport_meta_mask) 7056 flow_dv_translate_item_meta_vport(matcher, key, 7057 priv->vport_meta_tag, 7058 priv->vport_meta_mask); 7059 else 7060 flow_dv_translate_item_source_vport(matcher, key, 7061 priv->vport_id, mask); 7062 return 0; 7063 } 7064 7065 /** 7066 * Add ICMP6 item to matcher and to the value. 7067 * 7068 * @param[in, out] matcher 7069 * Flow matcher. 7070 * @param[in, out] key 7071 * Flow matcher value. 7072 * @param[in] item 7073 * Flow pattern to translate. 7074 * @param[in] inner 7075 * Item is inner pattern. 7076 */ 7077 static void 7078 flow_dv_translate_item_icmp6(void *matcher, void *key, 7079 const struct rte_flow_item *item, 7080 int inner) 7081 { 7082 const struct rte_flow_item_icmp6 *icmp6_m = item->mask; 7083 const struct rte_flow_item_icmp6 *icmp6_v = item->spec; 7084 void *headers_m; 7085 void *headers_v; 7086 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7087 misc_parameters_3); 7088 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7089 if (inner) { 7090 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7091 inner_headers); 7092 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7093 } else { 7094 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7095 outer_headers); 7096 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7097 } 7098 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7099 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); 7100 if (!icmp6_v) 7101 return; 7102 if (!icmp6_m) 7103 icmp6_m = &rte_flow_item_icmp6_mask; 7104 /* 7105 * Force flow only to match the non-fragmented IPv6 ICMPv6 packets. 7106 * If only the protocol is specified, no need to match the frag. 7107 */ 7108 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7109 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7110 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); 7111 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, 7112 icmp6_v->type & icmp6_m->type); 7113 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); 7114 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, 7115 icmp6_v->code & icmp6_m->code); 7116 } 7117 7118 /** 7119 * Add ICMP item to matcher and to the value. 7120 * 7121 * @param[in, out] matcher 7122 * Flow matcher. 7123 * @param[in, out] key 7124 * Flow matcher value. 7125 * @param[in] item 7126 * Flow pattern to translate. 7127 * @param[in] inner 7128 * Item is inner pattern. 7129 */ 7130 static void 7131 flow_dv_translate_item_icmp(void *matcher, void *key, 7132 const struct rte_flow_item *item, 7133 int inner) 7134 { 7135 const struct rte_flow_item_icmp *icmp_m = item->mask; 7136 const struct rte_flow_item_icmp *icmp_v = item->spec; 7137 void *headers_m; 7138 void *headers_v; 7139 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7140 misc_parameters_3); 7141 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7142 if (inner) { 7143 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7144 inner_headers); 7145 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7146 } else { 7147 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7148 outer_headers); 7149 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7150 } 7151 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 7152 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); 7153 if (!icmp_v) 7154 return; 7155 if (!icmp_m) 7156 icmp_m = &rte_flow_item_icmp_mask; 7157 /* 7158 * Force flow only to match the non-fragmented IPv4 ICMP packets. 7159 * If only the protocol is specified, no need to match the frag. 7160 */ 7161 MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1); 7162 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 7163 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, 7164 icmp_m->hdr.icmp_type); 7165 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, 7166 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); 7167 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, 7168 icmp_m->hdr.icmp_code); 7169 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, 7170 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); 7171 } 7172 7173 /** 7174 * Add GTP item to matcher and to the value. 7175 * 7176 * @param[in, out] matcher 7177 * Flow matcher. 7178 * @param[in, out] key 7179 * Flow matcher value. 7180 * @param[in] item 7181 * Flow pattern to translate. 7182 * @param[in] inner 7183 * Item is inner pattern. 7184 */ 7185 static void 7186 flow_dv_translate_item_gtp(void *matcher, void *key, 7187 const struct rte_flow_item *item, int inner) 7188 { 7189 const struct rte_flow_item_gtp *gtp_m = item->mask; 7190 const struct rte_flow_item_gtp *gtp_v = item->spec; 7191 void *headers_m; 7192 void *headers_v; 7193 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 7194 misc_parameters_3); 7195 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 7196 uint16_t dport = RTE_GTPU_UDP_PORT; 7197 7198 if (inner) { 7199 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7200 inner_headers); 7201 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 7202 } else { 7203 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 7204 outer_headers); 7205 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 7206 } 7207 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 7208 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 7209 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 7210 } 7211 if (!gtp_v) 7212 return; 7213 if (!gtp_m) 7214 gtp_m = &rte_flow_item_gtp_mask; 7215 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, 7216 gtp_m->v_pt_rsv_flags); 7217 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, 7218 gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags); 7219 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type); 7220 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type, 7221 gtp_v->msg_type & gtp_m->msg_type); 7222 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid, 7223 rte_be_to_cpu_32(gtp_m->teid)); 7224 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid, 7225 rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid)); 7226 } 7227 7228 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 7229 7230 #define HEADER_IS_ZERO(match_criteria, headers) \ 7231 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 7232 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 7233 7234 /** 7235 * Calculate flow matcher enable bitmap. 7236 * 7237 * @param match_criteria 7238 * Pointer to flow matcher criteria. 7239 * 7240 * @return 7241 * Bitmap of enabled fields. 7242 */ 7243 static uint8_t 7244 flow_dv_matcher_enable(uint32_t *match_criteria) 7245 { 7246 uint8_t match_criteria_enable; 7247 7248 match_criteria_enable = 7249 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 7250 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 7251 match_criteria_enable |= 7252 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 7253 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 7254 match_criteria_enable |= 7255 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 7256 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 7257 match_criteria_enable |= 7258 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 7259 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 7260 match_criteria_enable |= 7261 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << 7262 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; 7263 return match_criteria_enable; 7264 } 7265 7266 7267 /** 7268 * Get a flow table. 7269 * 7270 * @param[in, out] dev 7271 * Pointer to rte_eth_dev structure. 7272 * @param[in] table_id 7273 * Table id to use. 7274 * @param[in] egress 7275 * Direction of the table. 7276 * @param[in] transfer 7277 * E-Switch or NIC flow. 7278 * @param[out] error 7279 * pointer to error structure. 7280 * 7281 * @return 7282 * Returns tables resource based on the index, NULL in case of failed. 7283 */ 7284 static struct mlx5_flow_tbl_resource * 7285 flow_dv_tbl_resource_get(struct rte_eth_dev *dev, 7286 uint32_t table_id, uint8_t egress, 7287 uint8_t transfer, 7288 struct rte_flow_error *error) 7289 { 7290 struct mlx5_priv *priv = dev->data->dev_private; 7291 struct mlx5_dev_ctx_shared *sh = priv->sh; 7292 struct mlx5_flow_tbl_resource *tbl; 7293 union mlx5_flow_tbl_key table_key = { 7294 { 7295 .table_id = table_id, 7296 .reserved = 0, 7297 .domain = !!transfer, 7298 .direction = !!egress, 7299 } 7300 }; 7301 struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, 7302 table_key.v64); 7303 struct mlx5_flow_tbl_data_entry *tbl_data; 7304 uint32_t idx = 0; 7305 int ret; 7306 void *domain; 7307 7308 if (pos) { 7309 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, 7310 entry); 7311 tbl = &tbl_data->tbl; 7312 rte_atomic32_inc(&tbl->refcnt); 7313 return tbl; 7314 } 7315 tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); 7316 if (!tbl_data) { 7317 rte_flow_error_set(error, ENOMEM, 7318 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7319 NULL, 7320 "cannot allocate flow table data entry"); 7321 return NULL; 7322 } 7323 tbl_data->idx = idx; 7324 tbl = &tbl_data->tbl; 7325 pos = &tbl_data->entry; 7326 if (transfer) 7327 domain = sh->fdb_domain; 7328 else if (egress) 7329 domain = sh->tx_domain; 7330 else 7331 domain = sh->rx_domain; 7332 tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id); 7333 if (!tbl->obj) { 7334 rte_flow_error_set(error, ENOMEM, 7335 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7336 NULL, "cannot create flow table object"); 7337 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7338 return NULL; 7339 } 7340 /* 7341 * No multi-threads now, but still better to initialize the reference 7342 * count before insert it into the hash list. 7343 */ 7344 rte_atomic32_init(&tbl->refcnt); 7345 /* Jump action reference count is initialized here. */ 7346 rte_atomic32_init(&tbl_data->jump.refcnt); 7347 pos->key = table_key.v64; 7348 ret = mlx5_hlist_insert(sh->flow_tbls, pos); 7349 if (ret < 0) { 7350 rte_flow_error_set(error, -ret, 7351 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7352 "cannot insert flow table data entry"); 7353 mlx5_glue->dr_destroy_flow_tbl(tbl->obj); 7354 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); 7355 } 7356 rte_atomic32_inc(&tbl->refcnt); 7357 return tbl; 7358 } 7359 7360 /** 7361 * Release a flow table. 7362 * 7363 * @param[in] dev 7364 * Pointer to rte_eth_dev structure. 7365 * @param[in] tbl 7366 * Table resource to be released. 7367 * 7368 * @return 7369 * Returns 0 if table was released, else return 1; 7370 */ 7371 static int 7372 flow_dv_tbl_resource_release(struct rte_eth_dev *dev, 7373 struct mlx5_flow_tbl_resource *tbl) 7374 { 7375 struct mlx5_priv *priv = dev->data->dev_private; 7376 struct mlx5_dev_ctx_shared *sh = priv->sh; 7377 struct mlx5_flow_tbl_data_entry *tbl_data = 7378 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7379 7380 if (!tbl) 7381 return 0; 7382 if (rte_atomic32_dec_and_test(&tbl->refcnt)) { 7383 struct mlx5_hlist_entry *pos = &tbl_data->entry; 7384 7385 mlx5_glue->dr_destroy_flow_tbl(tbl->obj); 7386 tbl->obj = NULL; 7387 /* remove the entry from the hash list and free memory. */ 7388 mlx5_hlist_remove(sh->flow_tbls, pos); 7389 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], 7390 tbl_data->idx); 7391 return 0; 7392 } 7393 return 1; 7394 } 7395 7396 /** 7397 * Register the flow matcher. 7398 * 7399 * @param[in, out] dev 7400 * Pointer to rte_eth_dev structure. 7401 * @param[in, out] matcher 7402 * Pointer to flow matcher. 7403 * @param[in, out] key 7404 * Pointer to flow table key. 7405 * @parm[in, out] dev_flow 7406 * Pointer to the dev_flow. 7407 * @param[out] error 7408 * pointer to error structure. 7409 * 7410 * @return 7411 * 0 on success otherwise -errno and errno is set. 7412 */ 7413 static int 7414 flow_dv_matcher_register(struct rte_eth_dev *dev, 7415 struct mlx5_flow_dv_matcher *matcher, 7416 union mlx5_flow_tbl_key *key, 7417 struct mlx5_flow *dev_flow, 7418 struct rte_flow_error *error) 7419 { 7420 struct mlx5_priv *priv = dev->data->dev_private; 7421 struct mlx5_dev_ctx_shared *sh = priv->sh; 7422 struct mlx5_flow_dv_matcher *cache_matcher; 7423 struct mlx5dv_flow_matcher_attr dv_attr = { 7424 .type = IBV_FLOW_ATTR_NORMAL, 7425 .match_mask = (void *)&matcher->mask, 7426 }; 7427 struct mlx5_flow_tbl_resource *tbl; 7428 struct mlx5_flow_tbl_data_entry *tbl_data; 7429 7430 tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, 7431 key->domain, error); 7432 if (!tbl) 7433 return -rte_errno; /* No need to refill the error info */ 7434 tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); 7435 /* Lookup from cache. */ 7436 LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) { 7437 if (matcher->crc == cache_matcher->crc && 7438 matcher->priority == cache_matcher->priority && 7439 !memcmp((const void *)matcher->mask.buf, 7440 (const void *)cache_matcher->mask.buf, 7441 cache_matcher->mask.size)) { 7442 DRV_LOG(DEBUG, 7443 "%s group %u priority %hd use %s " 7444 "matcher %p: refcnt %d++", 7445 key->domain ? "FDB" : "NIC", key->table_id, 7446 cache_matcher->priority, 7447 key->direction ? "tx" : "rx", 7448 (void *)cache_matcher, 7449 rte_atomic32_read(&cache_matcher->refcnt)); 7450 rte_atomic32_inc(&cache_matcher->refcnt); 7451 dev_flow->handle->dvh.matcher = cache_matcher; 7452 /* old matcher should not make the table ref++. */ 7453 flow_dv_tbl_resource_release(dev, tbl); 7454 return 0; 7455 } 7456 } 7457 /* Register new matcher. */ 7458 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); 7459 if (!cache_matcher) { 7460 flow_dv_tbl_resource_release(dev, tbl); 7461 return rte_flow_error_set(error, ENOMEM, 7462 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7463 "cannot allocate matcher memory"); 7464 } 7465 *cache_matcher = *matcher; 7466 dv_attr.match_criteria_enable = 7467 flow_dv_matcher_enable(cache_matcher->mask.buf); 7468 dv_attr.priority = matcher->priority; 7469 if (key->direction) 7470 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 7471 cache_matcher->matcher_object = 7472 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); 7473 if (!cache_matcher->matcher_object) { 7474 rte_free(cache_matcher); 7475 #ifdef HAVE_MLX5DV_DR 7476 flow_dv_tbl_resource_release(dev, tbl); 7477 #endif 7478 return rte_flow_error_set(error, ENOMEM, 7479 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7480 NULL, "cannot create matcher"); 7481 } 7482 /* Save the table information */ 7483 cache_matcher->tbl = tbl; 7484 rte_atomic32_init(&cache_matcher->refcnt); 7485 /* only matcher ref++, table ref++ already done above in get API. */ 7486 rte_atomic32_inc(&cache_matcher->refcnt); 7487 LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next); 7488 dev_flow->handle->dvh.matcher = cache_matcher; 7489 DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d", 7490 key->domain ? "FDB" : "NIC", key->table_id, 7491 cache_matcher->priority, 7492 key->direction ? "tx" : "rx", (void *)cache_matcher, 7493 rte_atomic32_read(&cache_matcher->refcnt)); 7494 return 0; 7495 } 7496 7497 /** 7498 * Find existing tag resource or create and register a new one. 7499 * 7500 * @param dev[in, out] 7501 * Pointer to rte_eth_dev structure. 7502 * @param[in, out] tag_be24 7503 * Tag value in big endian then R-shift 8. 7504 * @parm[in, out] dev_flow 7505 * Pointer to the dev_flow. 7506 * @param[out] error 7507 * pointer to error structure. 7508 * 7509 * @return 7510 * 0 on success otherwise -errno and errno is set. 7511 */ 7512 static int 7513 flow_dv_tag_resource_register 7514 (struct rte_eth_dev *dev, 7515 uint32_t tag_be24, 7516 struct mlx5_flow *dev_flow, 7517 struct rte_flow_error *error) 7518 { 7519 struct mlx5_priv *priv = dev->data->dev_private; 7520 struct mlx5_dev_ctx_shared *sh = priv->sh; 7521 struct mlx5_flow_dv_tag_resource *cache_resource; 7522 struct mlx5_hlist_entry *entry; 7523 7524 /* Lookup a matching resource from cache. */ 7525 entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24); 7526 if (entry) { 7527 cache_resource = container_of 7528 (entry, struct mlx5_flow_dv_tag_resource, entry); 7529 rte_atomic32_inc(&cache_resource->refcnt); 7530 dev_flow->handle->dvh.rix_tag = cache_resource->idx; 7531 dev_flow->dv.tag_resource = cache_resource; 7532 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++", 7533 (void *)cache_resource, 7534 rte_atomic32_read(&cache_resource->refcnt)); 7535 return 0; 7536 } 7537 /* Register new resource. */ 7538 cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], 7539 &dev_flow->handle->dvh.rix_tag); 7540 if (!cache_resource) 7541 return rte_flow_error_set(error, ENOMEM, 7542 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 7543 "cannot allocate resource memory"); 7544 cache_resource->entry.key = (uint64_t)tag_be24; 7545 cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24); 7546 if (!cache_resource->action) { 7547 rte_free(cache_resource); 7548 return rte_flow_error_set(error, ENOMEM, 7549 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7550 NULL, "cannot create action"); 7551 } 7552 rte_atomic32_init(&cache_resource->refcnt); 7553 rte_atomic32_inc(&cache_resource->refcnt); 7554 if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) { 7555 mlx5_glue->destroy_flow_action(cache_resource->action); 7556 rte_free(cache_resource); 7557 return rte_flow_error_set(error, EEXIST, 7558 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7559 NULL, "cannot insert tag"); 7560 } 7561 dev_flow->dv.tag_resource = cache_resource; 7562 DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++", 7563 (void *)cache_resource, 7564 rte_atomic32_read(&cache_resource->refcnt)); 7565 return 0; 7566 } 7567 7568 /** 7569 * Release the tag. 7570 * 7571 * @param dev 7572 * Pointer to Ethernet device. 7573 * @param tag_idx 7574 * Tag index. 7575 * 7576 * @return 7577 * 1 while a reference on it exists, 0 when freed. 7578 */ 7579 static int 7580 flow_dv_tag_release(struct rte_eth_dev *dev, 7581 uint32_t tag_idx) 7582 { 7583 struct mlx5_priv *priv = dev->data->dev_private; 7584 struct mlx5_dev_ctx_shared *sh = priv->sh; 7585 struct mlx5_flow_dv_tag_resource *tag; 7586 7587 tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7588 if (!tag) 7589 return 0; 7590 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", 7591 dev->data->port_id, (void *)tag, 7592 rte_atomic32_read(&tag->refcnt)); 7593 if (rte_atomic32_dec_and_test(&tag->refcnt)) { 7594 claim_zero(mlx5_glue->destroy_flow_action(tag->action)); 7595 mlx5_hlist_remove(sh->tag_table, &tag->entry); 7596 DRV_LOG(DEBUG, "port %u tag %p: removed", 7597 dev->data->port_id, (void *)tag); 7598 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx); 7599 return 0; 7600 } 7601 return 1; 7602 } 7603 7604 /** 7605 * Translate port ID action to vport. 7606 * 7607 * @param[in] dev 7608 * Pointer to rte_eth_dev structure. 7609 * @param[in] action 7610 * Pointer to the port ID action. 7611 * @param[out] dst_port_id 7612 * The target port ID. 7613 * @param[out] error 7614 * Pointer to the error structure. 7615 * 7616 * @return 7617 * 0 on success, a negative errno value otherwise and rte_errno is set. 7618 */ 7619 static int 7620 flow_dv_translate_action_port_id(struct rte_eth_dev *dev, 7621 const struct rte_flow_action *action, 7622 uint32_t *dst_port_id, 7623 struct rte_flow_error *error) 7624 { 7625 uint32_t port; 7626 struct mlx5_priv *priv; 7627 const struct rte_flow_action_port_id *conf = 7628 (const struct rte_flow_action_port_id *)action->conf; 7629 7630 port = conf->original ? dev->data->port_id : conf->id; 7631 priv = mlx5_port_to_eswitch_info(port, false); 7632 if (!priv) 7633 return rte_flow_error_set(error, -rte_errno, 7634 RTE_FLOW_ERROR_TYPE_ACTION, 7635 NULL, 7636 "No eswitch info was found for port"); 7637 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 7638 /* 7639 * This parameter is transferred to 7640 * mlx5dv_dr_action_create_dest_ib_port(). 7641 */ 7642 *dst_port_id = priv->dev_port; 7643 #else 7644 /* 7645 * Legacy mode, no LAG configurations is supported. 7646 * This parameter is transferred to 7647 * mlx5dv_dr_action_create_dest_vport(). 7648 */ 7649 *dst_port_id = priv->vport_id; 7650 #endif 7651 return 0; 7652 } 7653 7654 /** 7655 * Create a counter with aging configuration. 7656 * 7657 * @param[in] dev 7658 * Pointer to rte_eth_dev structure. 7659 * @param[out] count 7660 * Pointer to the counter action configuration. 7661 * @param[in] age 7662 * Pointer to the aging action configuration. 7663 * 7664 * @return 7665 * Index to flow counter on success, 0 otherwise. 7666 */ 7667 static uint32_t 7668 flow_dv_translate_create_counter(struct rte_eth_dev *dev, 7669 struct mlx5_flow *dev_flow, 7670 const struct rte_flow_action_count *count, 7671 const struct rte_flow_action_age *age) 7672 { 7673 uint32_t counter; 7674 struct mlx5_age_param *age_param; 7675 7676 counter = flow_dv_counter_alloc(dev, 7677 count ? count->shared : 0, 7678 count ? count->id : 0, 7679 dev_flow->dv.group, !!age); 7680 if (!counter || age == NULL) 7681 return counter; 7682 age_param = flow_dv_counter_idx_get_age(dev, counter); 7683 /* 7684 * The counter age accuracy may have a bit delay. Have 3/4 7685 * second bias on the timeount in order to let it age in time. 7686 */ 7687 age_param->context = age->context ? age->context : 7688 (void *)(uintptr_t)(dev_flow->flow_idx); 7689 /* 7690 * The counter age accuracy may have a bit delay. Have 3/4 7691 * second bias on the timeount in order to let it age in time. 7692 */ 7693 age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY; 7694 /* Set expire time in unit of 0.1 sec. */ 7695 age_param->port_id = dev->data->port_id; 7696 age_param->expire = age_param->timeout + 7697 rte_rdtsc() / (rte_get_tsc_hz() / 10); 7698 rte_atomic16_set(&age_param->state, AGE_CANDIDATE); 7699 return counter; 7700 } 7701 /** 7702 * Add Tx queue matcher 7703 * 7704 * @param[in] dev 7705 * Pointer to the dev struct. 7706 * @param[in, out] matcher 7707 * Flow matcher. 7708 * @param[in, out] key 7709 * Flow matcher value. 7710 * @param[in] item 7711 * Flow pattern to translate. 7712 * @param[in] inner 7713 * Item is inner pattern. 7714 */ 7715 static void 7716 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev, 7717 void *matcher, void *key, 7718 const struct rte_flow_item *item) 7719 { 7720 const struct mlx5_rte_flow_item_tx_queue *queue_m; 7721 const struct mlx5_rte_flow_item_tx_queue *queue_v; 7722 void *misc_m = 7723 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 7724 void *misc_v = 7725 MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 7726 struct mlx5_txq_ctrl *txq; 7727 uint32_t queue; 7728 7729 7730 queue_m = (const void *)item->mask; 7731 if (!queue_m) 7732 return; 7733 queue_v = (const void *)item->spec; 7734 if (!queue_v) 7735 return; 7736 txq = mlx5_txq_get(dev, queue_v->queue); 7737 if (!txq) 7738 return; 7739 queue = txq->obj->sq->id; 7740 MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue); 7741 MLX5_SET(fte_match_set_misc, misc_v, source_sqn, 7742 queue & queue_m->queue); 7743 mlx5_txq_release(dev, queue_v->queue); 7744 } 7745 7746 /** 7747 * Set the hash fields according to the @p flow information. 7748 * 7749 * @param[in] dev_flow 7750 * Pointer to the mlx5_flow. 7751 * @param[in] rss_desc 7752 * Pointer to the mlx5_flow_rss_desc. 7753 */ 7754 static void 7755 flow_dv_hashfields_set(struct mlx5_flow *dev_flow, 7756 struct mlx5_flow_rss_desc *rss_desc) 7757 { 7758 uint64_t items = dev_flow->handle->layers; 7759 int rss_inner = 0; 7760 uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types); 7761 7762 dev_flow->hash_fields = 0; 7763 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 7764 if (rss_desc->level >= 2) { 7765 dev_flow->hash_fields |= IBV_RX_HASH_INNER; 7766 rss_inner = 1; 7767 } 7768 #endif 7769 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) || 7770 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) { 7771 if (rss_types & MLX5_IPV4_LAYER_TYPES) { 7772 if (rss_types & ETH_RSS_L3_SRC_ONLY) 7773 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4; 7774 else if (rss_types & ETH_RSS_L3_DST_ONLY) 7775 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4; 7776 else 7777 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH; 7778 } 7779 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) || 7780 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) { 7781 if (rss_types & MLX5_IPV6_LAYER_TYPES) { 7782 if (rss_types & ETH_RSS_L3_SRC_ONLY) 7783 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6; 7784 else if (rss_types & ETH_RSS_L3_DST_ONLY) 7785 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6; 7786 else 7787 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH; 7788 } 7789 } 7790 if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) || 7791 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) { 7792 if (rss_types & ETH_RSS_UDP) { 7793 if (rss_types & ETH_RSS_L4_SRC_ONLY) 7794 dev_flow->hash_fields |= 7795 IBV_RX_HASH_SRC_PORT_UDP; 7796 else if (rss_types & ETH_RSS_L4_DST_ONLY) 7797 dev_flow->hash_fields |= 7798 IBV_RX_HASH_DST_PORT_UDP; 7799 else 7800 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH; 7801 } 7802 } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) || 7803 (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) { 7804 if (rss_types & ETH_RSS_TCP) { 7805 if (rss_types & ETH_RSS_L4_SRC_ONLY) 7806 dev_flow->hash_fields |= 7807 IBV_RX_HASH_SRC_PORT_TCP; 7808 else if (rss_types & ETH_RSS_L4_DST_ONLY) 7809 dev_flow->hash_fields |= 7810 IBV_RX_HASH_DST_PORT_TCP; 7811 else 7812 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH; 7813 } 7814 } 7815 } 7816 7817 /** 7818 * Fill the flow with DV spec, lock free 7819 * (mutex should be acquired by caller). 7820 * 7821 * @param[in] dev 7822 * Pointer to rte_eth_dev structure. 7823 * @param[in, out] dev_flow 7824 * Pointer to the sub flow. 7825 * @param[in] attr 7826 * Pointer to the flow attributes. 7827 * @param[in] items 7828 * Pointer to the list of items. 7829 * @param[in] actions 7830 * Pointer to the list of actions. 7831 * @param[out] error 7832 * Pointer to the error structure. 7833 * 7834 * @return 7835 * 0 on success, a negative errno value otherwise and rte_errno is set. 7836 */ 7837 static int 7838 __flow_dv_translate(struct rte_eth_dev *dev, 7839 struct mlx5_flow *dev_flow, 7840 const struct rte_flow_attr *attr, 7841 const struct rte_flow_item items[], 7842 const struct rte_flow_action actions[], 7843 struct rte_flow_error *error) 7844 { 7845 struct mlx5_priv *priv = dev->data->dev_private; 7846 struct mlx5_dev_config *dev_conf = &priv->config; 7847 struct rte_flow *flow = dev_flow->flow; 7848 struct mlx5_flow_handle *handle = dev_flow->handle; 7849 struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *) 7850 priv->rss_desc) 7851 [!!priv->flow_nested_idx]; 7852 uint64_t item_flags = 0; 7853 uint64_t last_item = 0; 7854 uint64_t action_flags = 0; 7855 uint64_t priority = attr->priority; 7856 struct mlx5_flow_dv_matcher matcher = { 7857 .mask = { 7858 .size = sizeof(matcher.mask.buf), 7859 }, 7860 }; 7861 int actions_n = 0; 7862 bool actions_end = false; 7863 union { 7864 struct mlx5_flow_dv_modify_hdr_resource res; 7865 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) + 7866 sizeof(struct mlx5_modification_cmd) * 7867 (MLX5_MAX_MODIFY_NUM + 1)]; 7868 } mhdr_dummy; 7869 struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res; 7870 const struct rte_flow_action_count *count = NULL; 7871 const struct rte_flow_action_age *age = NULL; 7872 union flow_dv_attr flow_attr = { .attr = 0 }; 7873 uint32_t tag_be; 7874 union mlx5_flow_tbl_key tbl_key; 7875 uint32_t modify_action_position = UINT32_MAX; 7876 void *match_mask = matcher.mask.buf; 7877 void *match_value = dev_flow->dv.value.buf; 7878 uint8_t next_protocol = 0xff; 7879 struct rte_vlan_hdr vlan = { 0 }; 7880 uint32_t table; 7881 int ret = 0; 7882 7883 mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 7884 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 7885 ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group, 7886 !!priv->fdb_def_rule, &table, error); 7887 if (ret) 7888 return ret; 7889 dev_flow->dv.group = table; 7890 if (attr->transfer) 7891 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 7892 if (priority == MLX5_FLOW_PRIO_RSVD) 7893 priority = dev_conf->flow_prio - 1; 7894 /* number of actions must be set to 0 in case of dirty stack. */ 7895 mhdr_res->actions_num = 0; 7896 for (; !actions_end ; actions++) { 7897 const struct rte_flow_action_queue *queue; 7898 const struct rte_flow_action_rss *rss; 7899 const struct rte_flow_action *action = actions; 7900 const uint8_t *rss_key; 7901 const struct rte_flow_action_jump *jump_data; 7902 const struct rte_flow_action_meter *mtr; 7903 struct mlx5_flow_tbl_resource *tbl; 7904 uint32_t port_id = 0; 7905 struct mlx5_flow_dv_port_id_action_resource port_id_resource; 7906 int action_type = actions->type; 7907 const struct rte_flow_action *found_action = NULL; 7908 struct mlx5_flow_meter *fm = NULL; 7909 7910 switch (action_type) { 7911 case RTE_FLOW_ACTION_TYPE_VOID: 7912 break; 7913 case RTE_FLOW_ACTION_TYPE_PORT_ID: 7914 if (flow_dv_translate_action_port_id(dev, action, 7915 &port_id, error)) 7916 return -rte_errno; 7917 port_id_resource.port_id = port_id; 7918 MLX5_ASSERT(!handle->rix_port_id_action); 7919 if (flow_dv_port_id_action_resource_register 7920 (dev, &port_id_resource, dev_flow, error)) 7921 return -rte_errno; 7922 dev_flow->dv.actions[actions_n++] = 7923 dev_flow->dv.port_id_action->action; 7924 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 7925 dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID; 7926 break; 7927 case RTE_FLOW_ACTION_TYPE_FLAG: 7928 action_flags |= MLX5_FLOW_ACTION_FLAG; 7929 dev_flow->handle->mark = 1; 7930 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 7931 struct rte_flow_action_mark mark = { 7932 .id = MLX5_FLOW_MARK_DEFAULT, 7933 }; 7934 7935 if (flow_dv_convert_action_mark(dev, &mark, 7936 mhdr_res, 7937 error)) 7938 return -rte_errno; 7939 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 7940 break; 7941 } 7942 tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 7943 /* 7944 * Only one FLAG or MARK is supported per device flow 7945 * right now. So the pointer to the tag resource must be 7946 * zero before the register process. 7947 */ 7948 MLX5_ASSERT(!handle->dvh.rix_tag); 7949 if (flow_dv_tag_resource_register(dev, tag_be, 7950 dev_flow, error)) 7951 return -rte_errno; 7952 MLX5_ASSERT(dev_flow->dv.tag_resource); 7953 dev_flow->dv.actions[actions_n++] = 7954 dev_flow->dv.tag_resource->action; 7955 break; 7956 case RTE_FLOW_ACTION_TYPE_MARK: 7957 action_flags |= MLX5_FLOW_ACTION_MARK; 7958 dev_flow->handle->mark = 1; 7959 if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) { 7960 const struct rte_flow_action_mark *mark = 7961 (const struct rte_flow_action_mark *) 7962 actions->conf; 7963 7964 if (flow_dv_convert_action_mark(dev, mark, 7965 mhdr_res, 7966 error)) 7967 return -rte_errno; 7968 action_flags |= MLX5_FLOW_ACTION_MARK_EXT; 7969 break; 7970 } 7971 /* Fall-through */ 7972 case MLX5_RTE_FLOW_ACTION_TYPE_MARK: 7973 /* Legacy (non-extensive) MARK action. */ 7974 tag_be = mlx5_flow_mark_set 7975 (((const struct rte_flow_action_mark *) 7976 (actions->conf))->id); 7977 MLX5_ASSERT(!handle->dvh.rix_tag); 7978 if (flow_dv_tag_resource_register(dev, tag_be, 7979 dev_flow, error)) 7980 return -rte_errno; 7981 MLX5_ASSERT(dev_flow->dv.tag_resource); 7982 dev_flow->dv.actions[actions_n++] = 7983 dev_flow->dv.tag_resource->action; 7984 break; 7985 case RTE_FLOW_ACTION_TYPE_SET_META: 7986 if (flow_dv_convert_action_set_meta 7987 (dev, mhdr_res, attr, 7988 (const struct rte_flow_action_set_meta *) 7989 actions->conf, error)) 7990 return -rte_errno; 7991 action_flags |= MLX5_FLOW_ACTION_SET_META; 7992 break; 7993 case RTE_FLOW_ACTION_TYPE_SET_TAG: 7994 if (flow_dv_convert_action_set_tag 7995 (dev, mhdr_res, 7996 (const struct rte_flow_action_set_tag *) 7997 actions->conf, error)) 7998 return -rte_errno; 7999 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8000 break; 8001 case RTE_FLOW_ACTION_TYPE_DROP: 8002 action_flags |= MLX5_FLOW_ACTION_DROP; 8003 dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; 8004 break; 8005 case RTE_FLOW_ACTION_TYPE_QUEUE: 8006 queue = actions->conf; 8007 rss_desc->queue_num = 1; 8008 rss_desc->queue[0] = queue->index; 8009 action_flags |= MLX5_FLOW_ACTION_QUEUE; 8010 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8011 break; 8012 case RTE_FLOW_ACTION_TYPE_RSS: 8013 rss = actions->conf; 8014 memcpy(rss_desc->queue, rss->queue, 8015 rss->queue_num * sizeof(uint16_t)); 8016 rss_desc->queue_num = rss->queue_num; 8017 /* NULL RSS key indicates default RSS key. */ 8018 rss_key = !rss->key ? rss_hash_default_key : rss->key; 8019 memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 8020 /* 8021 * rss->level and rss.types should be set in advance 8022 * when expanding items for RSS. 8023 */ 8024 action_flags |= MLX5_FLOW_ACTION_RSS; 8025 dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; 8026 break; 8027 case RTE_FLOW_ACTION_TYPE_AGE: 8028 case RTE_FLOW_ACTION_TYPE_COUNT: 8029 if (!dev_conf->devx) { 8030 return rte_flow_error_set 8031 (error, ENOTSUP, 8032 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8033 NULL, 8034 "count action not supported"); 8035 } 8036 /* Save information first, will apply later. */ 8037 if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT) 8038 count = action->conf; 8039 else 8040 age = action->conf; 8041 action_flags |= MLX5_FLOW_ACTION_COUNT; 8042 break; 8043 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 8044 dev_flow->dv.actions[actions_n++] = 8045 priv->sh->pop_vlan_action; 8046 action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN; 8047 break; 8048 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 8049 if (!(action_flags & 8050 MLX5_FLOW_ACTION_OF_SET_VLAN_VID)) 8051 flow_dev_get_vlan_info_from_items(items, &vlan); 8052 vlan.eth_proto = rte_be_to_cpu_16 8053 ((((const struct rte_flow_action_of_push_vlan *) 8054 actions->conf)->ethertype)); 8055 found_action = mlx5_flow_find_action 8056 (actions + 1, 8057 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID); 8058 if (found_action) 8059 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8060 found_action = mlx5_flow_find_action 8061 (actions + 1, 8062 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP); 8063 if (found_action) 8064 mlx5_update_vlan_vid_pcp(found_action, &vlan); 8065 if (flow_dv_create_action_push_vlan 8066 (dev, attr, &vlan, dev_flow, error)) 8067 return -rte_errno; 8068 dev_flow->dv.actions[actions_n++] = 8069 dev_flow->dv.push_vlan_res->action; 8070 action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN; 8071 break; 8072 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 8073 /* of_vlan_push action handled this action */ 8074 MLX5_ASSERT(action_flags & 8075 MLX5_FLOW_ACTION_OF_PUSH_VLAN); 8076 break; 8077 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 8078 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) 8079 break; 8080 flow_dev_get_vlan_info_from_items(items, &vlan); 8081 mlx5_update_vlan_vid_pcp(actions, &vlan); 8082 /* If no VLAN push - this is a modify header action */ 8083 if (flow_dv_convert_action_modify_vlan_vid 8084 (mhdr_res, actions, error)) 8085 return -rte_errno; 8086 action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID; 8087 break; 8088 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 8089 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 8090 if (flow_dv_create_action_l2_encap(dev, actions, 8091 dev_flow, 8092 attr->transfer, 8093 error)) 8094 return -rte_errno; 8095 dev_flow->dv.actions[actions_n++] = 8096 dev_flow->dv.encap_decap->verbs_action; 8097 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8098 break; 8099 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 8100 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 8101 if (flow_dv_create_action_l2_decap(dev, dev_flow, 8102 attr->transfer, 8103 error)) 8104 return -rte_errno; 8105 dev_flow->dv.actions[actions_n++] = 8106 dev_flow->dv.encap_decap->verbs_action; 8107 action_flags |= MLX5_FLOW_ACTION_DECAP; 8108 break; 8109 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 8110 /* Handle encap with preceding decap. */ 8111 if (action_flags & MLX5_FLOW_ACTION_DECAP) { 8112 if (flow_dv_create_action_raw_encap 8113 (dev, actions, dev_flow, attr, error)) 8114 return -rte_errno; 8115 dev_flow->dv.actions[actions_n++] = 8116 dev_flow->dv.encap_decap->verbs_action; 8117 } else { 8118 /* Handle encap without preceding decap. */ 8119 if (flow_dv_create_action_l2_encap 8120 (dev, actions, dev_flow, attr->transfer, 8121 error)) 8122 return -rte_errno; 8123 dev_flow->dv.actions[actions_n++] = 8124 dev_flow->dv.encap_decap->verbs_action; 8125 } 8126 action_flags |= MLX5_FLOW_ACTION_ENCAP; 8127 break; 8128 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 8129 while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID) 8130 ; 8131 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 8132 if (flow_dv_create_action_l2_decap 8133 (dev, dev_flow, attr->transfer, error)) 8134 return -rte_errno; 8135 dev_flow->dv.actions[actions_n++] = 8136 dev_flow->dv.encap_decap->verbs_action; 8137 } 8138 /* If decap is followed by encap, handle it at encap. */ 8139 action_flags |= MLX5_FLOW_ACTION_DECAP; 8140 break; 8141 case RTE_FLOW_ACTION_TYPE_JUMP: 8142 jump_data = action->conf; 8143 ret = mlx5_flow_group_to_table(attr, dev_flow->external, 8144 jump_data->group, 8145 !!priv->fdb_def_rule, 8146 &table, error); 8147 if (ret) 8148 return ret; 8149 tbl = flow_dv_tbl_resource_get(dev, table, 8150 attr->egress, 8151 attr->transfer, error); 8152 if (!tbl) 8153 return rte_flow_error_set 8154 (error, errno, 8155 RTE_FLOW_ERROR_TYPE_ACTION, 8156 NULL, 8157 "cannot create jump action."); 8158 if (flow_dv_jump_tbl_resource_register 8159 (dev, tbl, dev_flow, error)) { 8160 flow_dv_tbl_resource_release(dev, tbl); 8161 return rte_flow_error_set 8162 (error, errno, 8163 RTE_FLOW_ERROR_TYPE_ACTION, 8164 NULL, 8165 "cannot create jump action."); 8166 } 8167 dev_flow->dv.actions[actions_n++] = 8168 dev_flow->dv.jump->action; 8169 action_flags |= MLX5_FLOW_ACTION_JUMP; 8170 dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP; 8171 break; 8172 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 8173 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 8174 if (flow_dv_convert_action_modify_mac 8175 (mhdr_res, actions, error)) 8176 return -rte_errno; 8177 action_flags |= actions->type == 8178 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 8179 MLX5_FLOW_ACTION_SET_MAC_SRC : 8180 MLX5_FLOW_ACTION_SET_MAC_DST; 8181 break; 8182 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 8183 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 8184 if (flow_dv_convert_action_modify_ipv4 8185 (mhdr_res, actions, error)) 8186 return -rte_errno; 8187 action_flags |= actions->type == 8188 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 8189 MLX5_FLOW_ACTION_SET_IPV4_SRC : 8190 MLX5_FLOW_ACTION_SET_IPV4_DST; 8191 break; 8192 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 8193 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 8194 if (flow_dv_convert_action_modify_ipv6 8195 (mhdr_res, actions, error)) 8196 return -rte_errno; 8197 action_flags |= actions->type == 8198 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 8199 MLX5_FLOW_ACTION_SET_IPV6_SRC : 8200 MLX5_FLOW_ACTION_SET_IPV6_DST; 8201 break; 8202 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 8203 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 8204 if (flow_dv_convert_action_modify_tp 8205 (mhdr_res, actions, items, 8206 &flow_attr, dev_flow, !!(action_flags & 8207 MLX5_FLOW_ACTION_DECAP), error)) 8208 return -rte_errno; 8209 action_flags |= actions->type == 8210 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 8211 MLX5_FLOW_ACTION_SET_TP_SRC : 8212 MLX5_FLOW_ACTION_SET_TP_DST; 8213 break; 8214 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 8215 if (flow_dv_convert_action_modify_dec_ttl 8216 (mhdr_res, items, &flow_attr, dev_flow, 8217 !!(action_flags & 8218 MLX5_FLOW_ACTION_DECAP), error)) 8219 return -rte_errno; 8220 action_flags |= MLX5_FLOW_ACTION_DEC_TTL; 8221 break; 8222 case RTE_FLOW_ACTION_TYPE_SET_TTL: 8223 if (flow_dv_convert_action_modify_ttl 8224 (mhdr_res, actions, items, &flow_attr, 8225 dev_flow, !!(action_flags & 8226 MLX5_FLOW_ACTION_DECAP), error)) 8227 return -rte_errno; 8228 action_flags |= MLX5_FLOW_ACTION_SET_TTL; 8229 break; 8230 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 8231 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 8232 if (flow_dv_convert_action_modify_tcp_seq 8233 (mhdr_res, actions, error)) 8234 return -rte_errno; 8235 action_flags |= actions->type == 8236 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 8237 MLX5_FLOW_ACTION_INC_TCP_SEQ : 8238 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 8239 break; 8240 8241 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 8242 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 8243 if (flow_dv_convert_action_modify_tcp_ack 8244 (mhdr_res, actions, error)) 8245 return -rte_errno; 8246 action_flags |= actions->type == 8247 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 8248 MLX5_FLOW_ACTION_INC_TCP_ACK : 8249 MLX5_FLOW_ACTION_DEC_TCP_ACK; 8250 break; 8251 case MLX5_RTE_FLOW_ACTION_TYPE_TAG: 8252 if (flow_dv_convert_action_set_reg 8253 (mhdr_res, actions, error)) 8254 return -rte_errno; 8255 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8256 break; 8257 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG: 8258 if (flow_dv_convert_action_copy_mreg 8259 (dev, mhdr_res, actions, error)) 8260 return -rte_errno; 8261 action_flags |= MLX5_FLOW_ACTION_SET_TAG; 8262 break; 8263 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS: 8264 action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS; 8265 dev_flow->handle->fate_action = 8266 MLX5_FLOW_FATE_DEFAULT_MISS; 8267 break; 8268 case RTE_FLOW_ACTION_TYPE_METER: 8269 mtr = actions->conf; 8270 if (!flow->meter) { 8271 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, 8272 attr, error); 8273 if (!fm) 8274 return rte_flow_error_set(error, 8275 rte_errno, 8276 RTE_FLOW_ERROR_TYPE_ACTION, 8277 NULL, 8278 "meter not found " 8279 "or invalid parameters"); 8280 flow->meter = fm->idx; 8281 } 8282 /* Set the meter action. */ 8283 if (!fm) { 8284 fm = mlx5_ipool_get(priv->sh->ipool 8285 [MLX5_IPOOL_MTR], flow->meter); 8286 if (!fm) 8287 return rte_flow_error_set(error, 8288 rte_errno, 8289 RTE_FLOW_ERROR_TYPE_ACTION, 8290 NULL, 8291 "meter not found " 8292 "or invalid parameters"); 8293 } 8294 dev_flow->dv.actions[actions_n++] = 8295 fm->mfts->meter_action; 8296 action_flags |= MLX5_FLOW_ACTION_METER; 8297 break; 8298 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 8299 if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res, 8300 actions, error)) 8301 return -rte_errno; 8302 action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP; 8303 break; 8304 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 8305 if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res, 8306 actions, error)) 8307 return -rte_errno; 8308 action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP; 8309 break; 8310 case RTE_FLOW_ACTION_TYPE_END: 8311 actions_end = true; 8312 if (mhdr_res->actions_num) { 8313 /* create modify action if needed. */ 8314 if (flow_dv_modify_hdr_resource_register 8315 (dev, mhdr_res, dev_flow, error)) 8316 return -rte_errno; 8317 dev_flow->dv.actions[modify_action_position] = 8318 handle->dvh.modify_hdr->verbs_action; 8319 } 8320 if (action_flags & MLX5_FLOW_ACTION_COUNT) { 8321 flow->counter = 8322 flow_dv_translate_create_counter(dev, 8323 dev_flow, count, age); 8324 8325 if (!flow->counter) 8326 return rte_flow_error_set 8327 (error, rte_errno, 8328 RTE_FLOW_ERROR_TYPE_ACTION, 8329 NULL, 8330 "cannot create counter" 8331 " object."); 8332 dev_flow->dv.actions[actions_n++] = 8333 (flow_dv_counter_get_by_idx(dev, 8334 flow->counter, NULL))->action; 8335 } 8336 break; 8337 default: 8338 break; 8339 } 8340 if (mhdr_res->actions_num && 8341 modify_action_position == UINT32_MAX) 8342 modify_action_position = actions_n++; 8343 } 8344 dev_flow->dv.actions_n = actions_n; 8345 dev_flow->act_flags = action_flags; 8346 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 8347 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 8348 int item_type = items->type; 8349 8350 switch (item_type) { 8351 case RTE_FLOW_ITEM_TYPE_PORT_ID: 8352 flow_dv_translate_item_port_id(dev, match_mask, 8353 match_value, items); 8354 last_item = MLX5_FLOW_ITEM_PORT_ID; 8355 break; 8356 case RTE_FLOW_ITEM_TYPE_ETH: 8357 flow_dv_translate_item_eth(match_mask, match_value, 8358 items, tunnel, 8359 dev_flow->dv.group); 8360 matcher.priority = action_flags & 8361 MLX5_FLOW_ACTION_DEFAULT_MISS && 8362 !dev_flow->external ? 8363 MLX5_PRIORITY_MAP_L3 : 8364 MLX5_PRIORITY_MAP_L2; 8365 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 8366 MLX5_FLOW_LAYER_OUTER_L2; 8367 break; 8368 case RTE_FLOW_ITEM_TYPE_VLAN: 8369 flow_dv_translate_item_vlan(dev_flow, 8370 match_mask, match_value, 8371 items, tunnel, 8372 dev_flow->dv.group); 8373 matcher.priority = MLX5_PRIORITY_MAP_L2; 8374 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 8375 MLX5_FLOW_LAYER_INNER_VLAN) : 8376 (MLX5_FLOW_LAYER_OUTER_L2 | 8377 MLX5_FLOW_LAYER_OUTER_VLAN); 8378 break; 8379 case RTE_FLOW_ITEM_TYPE_IPV4: 8380 mlx5_flow_tunnel_ip_check(items, next_protocol, 8381 &item_flags, &tunnel); 8382 flow_dv_translate_item_ipv4(match_mask, match_value, 8383 items, item_flags, tunnel, 8384 dev_flow->dv.group); 8385 matcher.priority = MLX5_PRIORITY_MAP_L3; 8386 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 8387 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 8388 if (items->mask != NULL && 8389 ((const struct rte_flow_item_ipv4 *) 8390 items->mask)->hdr.next_proto_id) { 8391 next_protocol = 8392 ((const struct rte_flow_item_ipv4 *) 8393 (items->spec))->hdr.next_proto_id; 8394 next_protocol &= 8395 ((const struct rte_flow_item_ipv4 *) 8396 (items->mask))->hdr.next_proto_id; 8397 } else { 8398 /* Reset for inner layer. */ 8399 next_protocol = 0xff; 8400 } 8401 break; 8402 case RTE_FLOW_ITEM_TYPE_IPV6: 8403 mlx5_flow_tunnel_ip_check(items, next_protocol, 8404 &item_flags, &tunnel); 8405 flow_dv_translate_item_ipv6(match_mask, match_value, 8406 items, item_flags, tunnel, 8407 dev_flow->dv.group); 8408 matcher.priority = MLX5_PRIORITY_MAP_L3; 8409 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 8410 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 8411 if (items->mask != NULL && 8412 ((const struct rte_flow_item_ipv6 *) 8413 items->mask)->hdr.proto) { 8414 next_protocol = 8415 ((const struct rte_flow_item_ipv6 *) 8416 items->spec)->hdr.proto; 8417 next_protocol &= 8418 ((const struct rte_flow_item_ipv6 *) 8419 items->mask)->hdr.proto; 8420 } else { 8421 /* Reset for inner layer. */ 8422 next_protocol = 0xff; 8423 } 8424 break; 8425 case RTE_FLOW_ITEM_TYPE_TCP: 8426 flow_dv_translate_item_tcp(match_mask, match_value, 8427 items, tunnel); 8428 matcher.priority = MLX5_PRIORITY_MAP_L4; 8429 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 8430 MLX5_FLOW_LAYER_OUTER_L4_TCP; 8431 break; 8432 case RTE_FLOW_ITEM_TYPE_UDP: 8433 flow_dv_translate_item_udp(match_mask, match_value, 8434 items, tunnel); 8435 matcher.priority = MLX5_PRIORITY_MAP_L4; 8436 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 8437 MLX5_FLOW_LAYER_OUTER_L4_UDP; 8438 break; 8439 case RTE_FLOW_ITEM_TYPE_GRE: 8440 flow_dv_translate_item_gre(match_mask, match_value, 8441 items, tunnel); 8442 matcher.priority = rss_desc->level >= 2 ? 8443 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8444 last_item = MLX5_FLOW_LAYER_GRE; 8445 break; 8446 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 8447 flow_dv_translate_item_gre_key(match_mask, 8448 match_value, items); 8449 last_item = MLX5_FLOW_LAYER_GRE_KEY; 8450 break; 8451 case RTE_FLOW_ITEM_TYPE_NVGRE: 8452 flow_dv_translate_item_nvgre(match_mask, match_value, 8453 items, tunnel); 8454 matcher.priority = rss_desc->level >= 2 ? 8455 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8456 last_item = MLX5_FLOW_LAYER_GRE; 8457 break; 8458 case RTE_FLOW_ITEM_TYPE_VXLAN: 8459 flow_dv_translate_item_vxlan(match_mask, match_value, 8460 items, tunnel); 8461 matcher.priority = rss_desc->level >= 2 ? 8462 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8463 last_item = MLX5_FLOW_LAYER_VXLAN; 8464 break; 8465 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 8466 flow_dv_translate_item_vxlan_gpe(match_mask, 8467 match_value, items, 8468 tunnel); 8469 matcher.priority = rss_desc->level >= 2 ? 8470 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8471 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 8472 break; 8473 case RTE_FLOW_ITEM_TYPE_GENEVE: 8474 flow_dv_translate_item_geneve(match_mask, match_value, 8475 items, tunnel); 8476 matcher.priority = rss_desc->level >= 2 ? 8477 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8478 last_item = MLX5_FLOW_LAYER_GENEVE; 8479 break; 8480 case RTE_FLOW_ITEM_TYPE_MPLS: 8481 flow_dv_translate_item_mpls(match_mask, match_value, 8482 items, last_item, tunnel); 8483 matcher.priority = rss_desc->level >= 2 ? 8484 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8485 last_item = MLX5_FLOW_LAYER_MPLS; 8486 break; 8487 case RTE_FLOW_ITEM_TYPE_MARK: 8488 flow_dv_translate_item_mark(dev, match_mask, 8489 match_value, items); 8490 last_item = MLX5_FLOW_ITEM_MARK; 8491 break; 8492 case RTE_FLOW_ITEM_TYPE_META: 8493 flow_dv_translate_item_meta(dev, match_mask, 8494 match_value, attr, items); 8495 last_item = MLX5_FLOW_ITEM_METADATA; 8496 break; 8497 case RTE_FLOW_ITEM_TYPE_ICMP: 8498 flow_dv_translate_item_icmp(match_mask, match_value, 8499 items, tunnel); 8500 last_item = MLX5_FLOW_LAYER_ICMP; 8501 break; 8502 case RTE_FLOW_ITEM_TYPE_ICMP6: 8503 flow_dv_translate_item_icmp6(match_mask, match_value, 8504 items, tunnel); 8505 last_item = MLX5_FLOW_LAYER_ICMP6; 8506 break; 8507 case RTE_FLOW_ITEM_TYPE_TAG: 8508 flow_dv_translate_item_tag(dev, match_mask, 8509 match_value, items); 8510 last_item = MLX5_FLOW_ITEM_TAG; 8511 break; 8512 case MLX5_RTE_FLOW_ITEM_TYPE_TAG: 8513 flow_dv_translate_mlx5_item_tag(dev, match_mask, 8514 match_value, items); 8515 last_item = MLX5_FLOW_ITEM_TAG; 8516 break; 8517 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE: 8518 flow_dv_translate_item_tx_queue(dev, match_mask, 8519 match_value, 8520 items); 8521 last_item = MLX5_FLOW_ITEM_TX_QUEUE; 8522 break; 8523 case RTE_FLOW_ITEM_TYPE_GTP: 8524 flow_dv_translate_item_gtp(match_mask, match_value, 8525 items, tunnel); 8526 matcher.priority = rss_desc->level >= 2 ? 8527 MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4; 8528 last_item = MLX5_FLOW_LAYER_GTP; 8529 break; 8530 default: 8531 break; 8532 } 8533 item_flags |= last_item; 8534 } 8535 /* 8536 * When E-Switch mode is enabled, we have two cases where we need to 8537 * set the source port manually. 8538 * The first one, is in case of Nic steering rule, and the second is 8539 * E-Switch rule where no port_id item was found. In both cases 8540 * the source port is set according the current port in use. 8541 */ 8542 if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && 8543 (priv->representor || priv->master)) { 8544 if (flow_dv_translate_item_port_id(dev, match_mask, 8545 match_value, NULL)) 8546 return -rte_errno; 8547 } 8548 #ifdef RTE_LIBRTE_MLX5_DEBUG 8549 MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf, 8550 dev_flow->dv.value.buf)); 8551 #endif 8552 /* 8553 * Layers may be already initialized from prefix flow if this dev_flow 8554 * is the suffix flow. 8555 */ 8556 handle->layers |= item_flags; 8557 if (action_flags & MLX5_FLOW_ACTION_RSS) 8558 flow_dv_hashfields_set(dev_flow, rss_desc); 8559 /* Register matcher. */ 8560 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 8561 matcher.mask.size); 8562 matcher.priority = mlx5_flow_adjust_priority(dev, priority, 8563 matcher.priority); 8564 /* reserved field no needs to be set to 0 here. */ 8565 tbl_key.domain = attr->transfer; 8566 tbl_key.direction = attr->egress; 8567 tbl_key.table_id = dev_flow->dv.group; 8568 if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error)) 8569 return -rte_errno; 8570 return 0; 8571 } 8572 8573 /** 8574 * Apply the flow to the NIC, lock free, 8575 * (mutex should be acquired by caller). 8576 * 8577 * @param[in] dev 8578 * Pointer to the Ethernet device structure. 8579 * @param[in, out] flow 8580 * Pointer to flow structure. 8581 * @param[out] error 8582 * Pointer to error structure. 8583 * 8584 * @return 8585 * 0 on success, a negative errno value otherwise and rte_errno is set. 8586 */ 8587 static int 8588 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 8589 struct rte_flow_error *error) 8590 { 8591 struct mlx5_flow_dv_workspace *dv; 8592 struct mlx5_flow_handle *dh; 8593 struct mlx5_flow_handle_dv *dv_h; 8594 struct mlx5_flow *dev_flow; 8595 struct mlx5_priv *priv = dev->data->dev_private; 8596 uint32_t handle_idx; 8597 int n; 8598 int err; 8599 int idx; 8600 8601 for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) { 8602 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; 8603 dv = &dev_flow->dv; 8604 dh = dev_flow->handle; 8605 dv_h = &dh->dvh; 8606 n = dv->actions_n; 8607 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 8608 if (dv->transfer) { 8609 dv->actions[n++] = priv->sh->esw_drop_action; 8610 } else { 8611 struct mlx5_hrxq *drop_hrxq; 8612 drop_hrxq = mlx5_hrxq_drop_new(dev); 8613 if (!drop_hrxq) { 8614 rte_flow_error_set 8615 (error, errno, 8616 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8617 NULL, 8618 "cannot get drop hash queue"); 8619 goto error; 8620 } 8621 /* 8622 * Drop queues will be released by the specify 8623 * mlx5_hrxq_drop_release() function. Assign 8624 * the special index to hrxq to mark the queue 8625 * has been allocated. 8626 */ 8627 dh->rix_hrxq = UINT32_MAX; 8628 dv->actions[n++] = drop_hrxq->action; 8629 } 8630 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 8631 struct mlx5_hrxq *hrxq; 8632 uint32_t hrxq_idx; 8633 struct mlx5_flow_rss_desc *rss_desc = 8634 &((struct mlx5_flow_rss_desc *)priv->rss_desc) 8635 [!!priv->flow_nested_idx]; 8636 8637 MLX5_ASSERT(rss_desc->queue_num); 8638 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, 8639 MLX5_RSS_HASH_KEY_LEN, 8640 dev_flow->hash_fields, 8641 rss_desc->queue, 8642 rss_desc->queue_num); 8643 if (!hrxq_idx) { 8644 hrxq_idx = mlx5_hrxq_new 8645 (dev, rss_desc->key, 8646 MLX5_RSS_HASH_KEY_LEN, 8647 dev_flow->hash_fields, 8648 rss_desc->queue, 8649 rss_desc->queue_num, 8650 !!(dh->layers & 8651 MLX5_FLOW_LAYER_TUNNEL)); 8652 } 8653 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 8654 hrxq_idx); 8655 if (!hrxq) { 8656 rte_flow_error_set 8657 (error, rte_errno, 8658 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8659 "cannot get hash queue"); 8660 goto error; 8661 } 8662 dh->rix_hrxq = hrxq_idx; 8663 dv->actions[n++] = hrxq->action; 8664 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) { 8665 if (flow_dv_default_miss_resource_register 8666 (dev, error)) { 8667 rte_flow_error_set 8668 (error, rte_errno, 8669 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8670 "cannot create default miss resource"); 8671 goto error_default_miss; 8672 } 8673 dh->rix_default_fate = MLX5_FLOW_FATE_DEFAULT_MISS; 8674 dv->actions[n++] = priv->sh->default_miss.action; 8675 } 8676 8677 dh->ib_flow = 8678 mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object, 8679 (void *)&dv->value, n, 8680 dv->actions); 8681 if (!dh->ib_flow) { 8682 rte_flow_error_set(error, errno, 8683 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8684 NULL, 8685 "hardware refuses to create flow"); 8686 goto error; 8687 } 8688 if (priv->vmwa_context && 8689 dh->vf_vlan.tag && !dh->vf_vlan.created) { 8690 /* 8691 * The rule contains the VLAN pattern. 8692 * For VF we are going to create VLAN 8693 * interface to make hypervisor set correct 8694 * e-Switch vport context. 8695 */ 8696 mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan); 8697 } 8698 } 8699 return 0; 8700 error: 8701 if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 8702 flow_dv_default_miss_resource_release(dev); 8703 error_default_miss: 8704 err = rte_errno; /* Save rte_errno before cleanup. */ 8705 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 8706 handle_idx, dh, next) { 8707 /* hrxq is union, don't clear it if the flag is not set. */ 8708 if (dh->rix_hrxq) { 8709 if (dh->fate_action == MLX5_FLOW_FATE_DROP) { 8710 mlx5_hrxq_drop_release(dev); 8711 dh->rix_hrxq = 0; 8712 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) { 8713 mlx5_hrxq_release(dev, dh->rix_hrxq); 8714 dh->rix_hrxq = 0; 8715 } 8716 } 8717 if (dh->vf_vlan.tag && dh->vf_vlan.created) 8718 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 8719 } 8720 rte_errno = err; /* Restore rte_errno. */ 8721 return -rte_errno; 8722 } 8723 8724 /** 8725 * Release the flow matcher. 8726 * 8727 * @param dev 8728 * Pointer to Ethernet device. 8729 * @param handle 8730 * Pointer to mlx5_flow_handle. 8731 * 8732 * @return 8733 * 1 while a reference on it exists, 0 when freed. 8734 */ 8735 static int 8736 flow_dv_matcher_release(struct rte_eth_dev *dev, 8737 struct mlx5_flow_handle *handle) 8738 { 8739 struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher; 8740 8741 MLX5_ASSERT(matcher->matcher_object); 8742 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", 8743 dev->data->port_id, (void *)matcher, 8744 rte_atomic32_read(&matcher->refcnt)); 8745 if (rte_atomic32_dec_and_test(&matcher->refcnt)) { 8746 claim_zero(mlx5_glue->dv_destroy_flow_matcher 8747 (matcher->matcher_object)); 8748 LIST_REMOVE(matcher, next); 8749 /* table ref-- in release interface. */ 8750 flow_dv_tbl_resource_release(dev, matcher->tbl); 8751 rte_free(matcher); 8752 DRV_LOG(DEBUG, "port %u matcher %p: removed", 8753 dev->data->port_id, (void *)matcher); 8754 return 0; 8755 } 8756 return 1; 8757 } 8758 8759 /** 8760 * Release an encap/decap resource. 8761 * 8762 * @param dev 8763 * Pointer to Ethernet device. 8764 * @param handle 8765 * Pointer to mlx5_flow_handle. 8766 * 8767 * @return 8768 * 1 while a reference on it exists, 0 when freed. 8769 */ 8770 static int 8771 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev, 8772 struct mlx5_flow_handle *handle) 8773 { 8774 struct mlx5_priv *priv = dev->data->dev_private; 8775 uint32_t idx = handle->dvh.rix_encap_decap; 8776 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 8777 8778 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 8779 idx); 8780 if (!cache_resource) 8781 return 0; 8782 MLX5_ASSERT(cache_resource->verbs_action); 8783 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", 8784 (void *)cache_resource, 8785 rte_atomic32_read(&cache_resource->refcnt)); 8786 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8787 claim_zero(mlx5_glue->destroy_flow_action 8788 (cache_resource->verbs_action)); 8789 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], 8790 &priv->sh->encaps_decaps, idx, 8791 cache_resource, next); 8792 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx); 8793 DRV_LOG(DEBUG, "encap/decap resource %p: removed", 8794 (void *)cache_resource); 8795 return 0; 8796 } 8797 return 1; 8798 } 8799 8800 /** 8801 * Release an jump to table action resource. 8802 * 8803 * @param dev 8804 * Pointer to Ethernet device. 8805 * @param handle 8806 * Pointer to mlx5_flow_handle. 8807 * 8808 * @return 8809 * 1 while a reference on it exists, 0 when freed. 8810 */ 8811 static int 8812 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev, 8813 struct mlx5_flow_handle *handle) 8814 { 8815 struct mlx5_priv *priv = dev->data->dev_private; 8816 struct mlx5_flow_dv_jump_tbl_resource *cache_resource; 8817 struct mlx5_flow_tbl_data_entry *tbl_data; 8818 8819 tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP], 8820 handle->rix_jump); 8821 if (!tbl_data) 8822 return 0; 8823 cache_resource = &tbl_data->jump; 8824 MLX5_ASSERT(cache_resource->action); 8825 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", 8826 (void *)cache_resource, 8827 rte_atomic32_read(&cache_resource->refcnt)); 8828 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8829 claim_zero(mlx5_glue->destroy_flow_action 8830 (cache_resource->action)); 8831 /* jump action memory free is inside the table release. */ 8832 flow_dv_tbl_resource_release(dev, &tbl_data->tbl); 8833 DRV_LOG(DEBUG, "jump table resource %p: removed", 8834 (void *)cache_resource); 8835 return 0; 8836 } 8837 return 1; 8838 } 8839 8840 /** 8841 * Release a default miss resource. 8842 * 8843 * @param dev 8844 * Pointer to Ethernet device. 8845 * @return 8846 * 1 while a reference on it exists, 0 when freed. 8847 */ 8848 static int 8849 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev) 8850 { 8851 struct mlx5_priv *priv = dev->data->dev_private; 8852 struct mlx5_dev_ctx_shared *sh = priv->sh; 8853 struct mlx5_flow_default_miss_resource *cache_resource = 8854 &sh->default_miss; 8855 8856 MLX5_ASSERT(cache_resource->action); 8857 DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--", 8858 (void *)cache_resource->action, 8859 rte_atomic32_read(&cache_resource->refcnt)); 8860 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8861 claim_zero(mlx5_glue->destroy_flow_action 8862 (cache_resource->action)); 8863 DRV_LOG(DEBUG, "default miss resource %p: removed", 8864 (void *)cache_resource->action); 8865 return 0; 8866 } 8867 return 1; 8868 } 8869 8870 /** 8871 * Release a modify-header resource. 8872 * 8873 * @param handle 8874 * Pointer to mlx5_flow_handle. 8875 * 8876 * @return 8877 * 1 while a reference on it exists, 0 when freed. 8878 */ 8879 static int 8880 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle) 8881 { 8882 struct mlx5_flow_dv_modify_hdr_resource *cache_resource = 8883 handle->dvh.modify_hdr; 8884 8885 MLX5_ASSERT(cache_resource->verbs_action); 8886 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", 8887 (void *)cache_resource, 8888 rte_atomic32_read(&cache_resource->refcnt)); 8889 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8890 claim_zero(mlx5_glue->destroy_flow_action 8891 (cache_resource->verbs_action)); 8892 LIST_REMOVE(cache_resource, next); 8893 rte_free(cache_resource); 8894 DRV_LOG(DEBUG, "modify-header resource %p: removed", 8895 (void *)cache_resource); 8896 return 0; 8897 } 8898 return 1; 8899 } 8900 8901 /** 8902 * Release port ID action resource. 8903 * 8904 * @param dev 8905 * Pointer to Ethernet device. 8906 * @param handle 8907 * Pointer to mlx5_flow_handle. 8908 * 8909 * @return 8910 * 1 while a reference on it exists, 0 when freed. 8911 */ 8912 static int 8913 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev, 8914 struct mlx5_flow_handle *handle) 8915 { 8916 struct mlx5_priv *priv = dev->data->dev_private; 8917 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 8918 uint32_t idx = handle->rix_port_id_action; 8919 8920 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 8921 idx); 8922 if (!cache_resource) 8923 return 0; 8924 MLX5_ASSERT(cache_resource->action); 8925 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", 8926 (void *)cache_resource, 8927 rte_atomic32_read(&cache_resource->refcnt)); 8928 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8929 claim_zero(mlx5_glue->destroy_flow_action 8930 (cache_resource->action)); 8931 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID], 8932 &priv->sh->port_id_action_list, idx, 8933 cache_resource, next); 8934 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx); 8935 DRV_LOG(DEBUG, "port id action resource %p: removed", 8936 (void *)cache_resource); 8937 return 0; 8938 } 8939 return 1; 8940 } 8941 8942 /** 8943 * Release push vlan action resource. 8944 * 8945 * @param dev 8946 * Pointer to Ethernet device. 8947 * @param handle 8948 * Pointer to mlx5_flow_handle. 8949 * 8950 * @return 8951 * 1 while a reference on it exists, 0 when freed. 8952 */ 8953 static int 8954 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev, 8955 struct mlx5_flow_handle *handle) 8956 { 8957 struct mlx5_priv *priv = dev->data->dev_private; 8958 uint32_t idx = handle->dvh.rix_push_vlan; 8959 struct mlx5_flow_dv_push_vlan_action_resource *cache_resource; 8960 8961 cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 8962 idx); 8963 if (!cache_resource) 8964 return 0; 8965 MLX5_ASSERT(cache_resource->action); 8966 DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--", 8967 (void *)cache_resource, 8968 rte_atomic32_read(&cache_resource->refcnt)); 8969 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 8970 claim_zero(mlx5_glue->destroy_flow_action 8971 (cache_resource->action)); 8972 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], 8973 &priv->sh->push_vlan_action_list, idx, 8974 cache_resource, next); 8975 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx); 8976 DRV_LOG(DEBUG, "push vlan action resource %p: removed", 8977 (void *)cache_resource); 8978 return 0; 8979 } 8980 return 1; 8981 } 8982 8983 /** 8984 * Release the fate resource. 8985 * 8986 * @param dev 8987 * Pointer to Ethernet device. 8988 * @param handle 8989 * Pointer to mlx5_flow_handle. 8990 */ 8991 static void 8992 flow_dv_fate_resource_release(struct rte_eth_dev *dev, 8993 struct mlx5_flow_handle *handle) 8994 { 8995 if (!handle->rix_fate) 8996 return; 8997 switch (handle->fate_action) { 8998 case MLX5_FLOW_FATE_DROP: 8999 mlx5_hrxq_drop_release(dev); 9000 break; 9001 case MLX5_FLOW_FATE_QUEUE: 9002 mlx5_hrxq_release(dev, handle->rix_hrxq); 9003 break; 9004 case MLX5_FLOW_FATE_JUMP: 9005 flow_dv_jump_tbl_resource_release(dev, handle); 9006 break; 9007 case MLX5_FLOW_FATE_PORT_ID: 9008 flow_dv_port_id_action_resource_release(dev, handle); 9009 break; 9010 case MLX5_FLOW_FATE_DEFAULT_MISS: 9011 flow_dv_default_miss_resource_release(dev); 9012 break; 9013 default: 9014 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action); 9015 break; 9016 } 9017 handle->rix_fate = 0; 9018 } 9019 9020 /** 9021 * Remove the flow from the NIC but keeps it in memory. 9022 * Lock free, (mutex should be acquired by caller). 9023 * 9024 * @param[in] dev 9025 * Pointer to Ethernet device. 9026 * @param[in, out] flow 9027 * Pointer to flow structure. 9028 */ 9029 static void 9030 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 9031 { 9032 struct mlx5_flow_handle *dh; 9033 uint32_t handle_idx; 9034 struct mlx5_priv *priv = dev->data->dev_private; 9035 9036 if (!flow) 9037 return; 9038 handle_idx = flow->dev_handles; 9039 while (handle_idx) { 9040 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9041 handle_idx); 9042 if (!dh) 9043 return; 9044 if (dh->ib_flow) { 9045 claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow)); 9046 dh->ib_flow = NULL; 9047 } 9048 if (dh->fate_action == MLX5_FLOW_FATE_DROP || 9049 dh->fate_action == MLX5_FLOW_FATE_QUEUE || 9050 dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) 9051 flow_dv_fate_resource_release(dev, dh); 9052 if (dh->vf_vlan.tag && dh->vf_vlan.created) 9053 mlx5_vlan_vmwa_release(dev, &dh->vf_vlan); 9054 handle_idx = dh->next.next; 9055 } 9056 } 9057 9058 /** 9059 * Remove the flow from the NIC and the memory. 9060 * Lock free, (mutex should be acquired by caller). 9061 * 9062 * @param[in] dev 9063 * Pointer to the Ethernet device structure. 9064 * @param[in, out] flow 9065 * Pointer to flow structure. 9066 */ 9067 static void 9068 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 9069 { 9070 struct mlx5_flow_handle *dev_handle; 9071 struct mlx5_priv *priv = dev->data->dev_private; 9072 9073 if (!flow) 9074 return; 9075 __flow_dv_remove(dev, flow); 9076 if (flow->counter) { 9077 flow_dv_counter_release(dev, flow->counter); 9078 flow->counter = 0; 9079 } 9080 if (flow->meter) { 9081 struct mlx5_flow_meter *fm; 9082 9083 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], 9084 flow->meter); 9085 if (fm) 9086 mlx5_flow_meter_detach(fm); 9087 flow->meter = 0; 9088 } 9089 while (flow->dev_handles) { 9090 uint32_t tmp_idx = flow->dev_handles; 9091 9092 dev_handle = mlx5_ipool_get(priv->sh->ipool 9093 [MLX5_IPOOL_MLX5_FLOW], tmp_idx); 9094 if (!dev_handle) 9095 return; 9096 flow->dev_handles = dev_handle->next.next; 9097 if (dev_handle->dvh.matcher) 9098 flow_dv_matcher_release(dev, dev_handle); 9099 if (dev_handle->dvh.rix_encap_decap) 9100 flow_dv_encap_decap_resource_release(dev, dev_handle); 9101 if (dev_handle->dvh.modify_hdr) 9102 flow_dv_modify_hdr_resource_release(dev_handle); 9103 if (dev_handle->dvh.rix_push_vlan) 9104 flow_dv_push_vlan_action_resource_release(dev, 9105 dev_handle); 9106 if (dev_handle->dvh.rix_tag) 9107 flow_dv_tag_release(dev, 9108 dev_handle->dvh.rix_tag); 9109 flow_dv_fate_resource_release(dev, dev_handle); 9110 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9111 tmp_idx); 9112 } 9113 } 9114 9115 /** 9116 * Query a dv flow rule for its statistics via devx. 9117 * 9118 * @param[in] dev 9119 * Pointer to Ethernet device. 9120 * @param[in] flow 9121 * Pointer to the sub flow. 9122 * @param[out] data 9123 * data retrieved by the query. 9124 * @param[out] error 9125 * Perform verbose error reporting if not NULL. 9126 * 9127 * @return 9128 * 0 on success, a negative errno value otherwise and rte_errno is set. 9129 */ 9130 static int 9131 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, 9132 void *data, struct rte_flow_error *error) 9133 { 9134 struct mlx5_priv *priv = dev->data->dev_private; 9135 struct rte_flow_query_count *qc = data; 9136 9137 if (!priv->config.devx) 9138 return rte_flow_error_set(error, ENOTSUP, 9139 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9140 NULL, 9141 "counters are not supported"); 9142 if (flow->counter) { 9143 uint64_t pkts, bytes; 9144 struct mlx5_flow_counter *cnt; 9145 9146 cnt = flow_dv_counter_get_by_idx(dev, flow->counter, 9147 NULL); 9148 int err = _flow_dv_query_count(dev, flow->counter, &pkts, 9149 &bytes); 9150 9151 if (err) 9152 return rte_flow_error_set(error, -err, 9153 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9154 NULL, "cannot read counters"); 9155 qc->hits_set = 1; 9156 qc->bytes_set = 1; 9157 qc->hits = pkts - cnt->hits; 9158 qc->bytes = bytes - cnt->bytes; 9159 if (qc->reset) { 9160 cnt->hits = pkts; 9161 cnt->bytes = bytes; 9162 } 9163 return 0; 9164 } 9165 return rte_flow_error_set(error, EINVAL, 9166 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9167 NULL, 9168 "counters are not available"); 9169 } 9170 9171 /** 9172 * Query a flow. 9173 * 9174 * @see rte_flow_query() 9175 * @see rte_flow_ops 9176 */ 9177 static int 9178 flow_dv_query(struct rte_eth_dev *dev, 9179 struct rte_flow *flow __rte_unused, 9180 const struct rte_flow_action *actions __rte_unused, 9181 void *data __rte_unused, 9182 struct rte_flow_error *error __rte_unused) 9183 { 9184 int ret = -EINVAL; 9185 9186 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 9187 switch (actions->type) { 9188 case RTE_FLOW_ACTION_TYPE_VOID: 9189 break; 9190 case RTE_FLOW_ACTION_TYPE_COUNT: 9191 ret = flow_dv_query_count(dev, flow, data, error); 9192 break; 9193 default: 9194 return rte_flow_error_set(error, ENOTSUP, 9195 RTE_FLOW_ERROR_TYPE_ACTION, 9196 actions, 9197 "action not supported"); 9198 } 9199 } 9200 return ret; 9201 } 9202 9203 /** 9204 * Destroy the meter table set. 9205 * Lock free, (mutex should be acquired by caller). 9206 * 9207 * @param[in] dev 9208 * Pointer to Ethernet device. 9209 * @param[in] tbl 9210 * Pointer to the meter table set. 9211 * 9212 * @return 9213 * Always 0. 9214 */ 9215 static int 9216 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, 9217 struct mlx5_meter_domains_infos *tbl) 9218 { 9219 struct mlx5_priv *priv = dev->data->dev_private; 9220 struct mlx5_meter_domains_infos *mtd = 9221 (struct mlx5_meter_domains_infos *)tbl; 9222 9223 if (!mtd || !priv->config.dv_flow_en) 9224 return 0; 9225 if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) 9226 claim_zero(mlx5_glue->dv_destroy_flow 9227 (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); 9228 if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) 9229 claim_zero(mlx5_glue->dv_destroy_flow 9230 (mtd->egress.policer_rules[RTE_MTR_DROPPED])); 9231 if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) 9232 claim_zero(mlx5_glue->dv_destroy_flow 9233 (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); 9234 if (mtd->egress.color_matcher) 9235 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9236 (mtd->egress.color_matcher)); 9237 if (mtd->egress.any_matcher) 9238 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9239 (mtd->egress.any_matcher)); 9240 if (mtd->egress.tbl) 9241 flow_dv_tbl_resource_release(dev, mtd->egress.tbl); 9242 if (mtd->egress.sfx_tbl) 9243 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl); 9244 if (mtd->ingress.color_matcher) 9245 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9246 (mtd->ingress.color_matcher)); 9247 if (mtd->ingress.any_matcher) 9248 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9249 (mtd->ingress.any_matcher)); 9250 if (mtd->ingress.tbl) 9251 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl); 9252 if (mtd->ingress.sfx_tbl) 9253 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl); 9254 if (mtd->transfer.color_matcher) 9255 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9256 (mtd->transfer.color_matcher)); 9257 if (mtd->transfer.any_matcher) 9258 claim_zero(mlx5_glue->dv_destroy_flow_matcher 9259 (mtd->transfer.any_matcher)); 9260 if (mtd->transfer.tbl) 9261 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl); 9262 if (mtd->transfer.sfx_tbl) 9263 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl); 9264 if (mtd->drop_actn) 9265 claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn)); 9266 rte_free(mtd); 9267 return 0; 9268 } 9269 9270 /* Number of meter flow actions, count and jump or count and drop. */ 9271 #define METER_ACTIONS 2 9272 9273 /** 9274 * Create specify domain meter table and suffix table. 9275 * 9276 * @param[in] dev 9277 * Pointer to Ethernet device. 9278 * @param[in,out] mtb 9279 * Pointer to DV meter table set. 9280 * @param[in] egress 9281 * Table attribute. 9282 * @param[in] transfer 9283 * Table attribute. 9284 * @param[in] color_reg_c_idx 9285 * Reg C index for color match. 9286 * 9287 * @return 9288 * 0 on success, -1 otherwise and rte_errno is set. 9289 */ 9290 static int 9291 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, 9292 struct mlx5_meter_domains_infos *mtb, 9293 uint8_t egress, uint8_t transfer, 9294 uint32_t color_reg_c_idx) 9295 { 9296 struct mlx5_priv *priv = dev->data->dev_private; 9297 struct mlx5_dev_ctx_shared *sh = priv->sh; 9298 struct mlx5_flow_dv_match_params mask = { 9299 .size = sizeof(mask.buf), 9300 }; 9301 struct mlx5_flow_dv_match_params value = { 9302 .size = sizeof(value.buf), 9303 }; 9304 struct mlx5dv_flow_matcher_attr dv_attr = { 9305 .type = IBV_FLOW_ATTR_NORMAL, 9306 .priority = 0, 9307 .match_criteria_enable = 0, 9308 .match_mask = (void *)&mask, 9309 }; 9310 void *actions[METER_ACTIONS]; 9311 struct mlx5_meter_domain_info *dtb; 9312 struct rte_flow_error error; 9313 int i = 0; 9314 9315 if (transfer) 9316 dtb = &mtb->transfer; 9317 else if (egress) 9318 dtb = &mtb->egress; 9319 else 9320 dtb = &mtb->ingress; 9321 /* Create the meter table with METER level. */ 9322 dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, 9323 egress, transfer, &error); 9324 if (!dtb->tbl) { 9325 DRV_LOG(ERR, "Failed to create meter policer table."); 9326 return -1; 9327 } 9328 /* Create the meter suffix table with SUFFIX level. */ 9329 dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, 9330 MLX5_FLOW_TABLE_LEVEL_SUFFIX, 9331 egress, transfer, &error); 9332 if (!dtb->sfx_tbl) { 9333 DRV_LOG(ERR, "Failed to create meter suffix table."); 9334 return -1; 9335 } 9336 /* Create matchers, Any and Color. */ 9337 dv_attr.priority = 3; 9338 dv_attr.match_criteria_enable = 0; 9339 dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, 9340 &dv_attr, 9341 dtb->tbl->obj); 9342 if (!dtb->any_matcher) { 9343 DRV_LOG(ERR, "Failed to create meter" 9344 " policer default matcher."); 9345 goto error_exit; 9346 } 9347 dv_attr.priority = 0; 9348 dv_attr.match_criteria_enable = 9349 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 9350 flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, 9351 rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); 9352 dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx, 9353 &dv_attr, 9354 dtb->tbl->obj); 9355 if (!dtb->color_matcher) { 9356 DRV_LOG(ERR, "Failed to create meter policer color matcher."); 9357 goto error_exit; 9358 } 9359 if (mtb->count_actns[RTE_MTR_DROPPED]) 9360 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; 9361 actions[i++] = mtb->drop_actn; 9362 /* Default rule: lowest priority, match any, actions: drop. */ 9363 dtb->policer_rules[RTE_MTR_DROPPED] = 9364 mlx5_glue->dv_create_flow(dtb->any_matcher, 9365 (void *)&value, i, actions); 9366 if (!dtb->policer_rules[RTE_MTR_DROPPED]) { 9367 DRV_LOG(ERR, "Failed to create meter policer drop rule."); 9368 goto error_exit; 9369 } 9370 return 0; 9371 error_exit: 9372 return -1; 9373 } 9374 9375 /** 9376 * Create the needed meter and suffix tables. 9377 * Lock free, (mutex should be acquired by caller). 9378 * 9379 * @param[in] dev 9380 * Pointer to Ethernet device. 9381 * @param[in] fm 9382 * Pointer to the flow meter. 9383 * 9384 * @return 9385 * Pointer to table set on success, NULL otherwise and rte_errno is set. 9386 */ 9387 static struct mlx5_meter_domains_infos * 9388 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, 9389 const struct mlx5_flow_meter *fm) 9390 { 9391 struct mlx5_priv *priv = dev->data->dev_private; 9392 struct mlx5_meter_domains_infos *mtb; 9393 int ret; 9394 int i; 9395 9396 if (!priv->mtr_en) { 9397 rte_errno = ENOTSUP; 9398 return NULL; 9399 } 9400 mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0); 9401 if (!mtb) { 9402 DRV_LOG(ERR, "Failed to allocate memory for meter."); 9403 return NULL; 9404 } 9405 /* Create meter count actions */ 9406 for (i = 0; i <= RTE_MTR_DROPPED; i++) { 9407 struct mlx5_flow_counter *cnt; 9408 if (!fm->policer_stats.cnt[i]) 9409 continue; 9410 cnt = flow_dv_counter_get_by_idx(dev, 9411 fm->policer_stats.cnt[i], NULL); 9412 mtb->count_actns[i] = cnt->action; 9413 } 9414 /* Create drop action. */ 9415 mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop(); 9416 if (!mtb->drop_actn) { 9417 DRV_LOG(ERR, "Failed to create drop action."); 9418 goto error_exit; 9419 } 9420 /* Egress meter table. */ 9421 ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); 9422 if (ret) { 9423 DRV_LOG(ERR, "Failed to prepare egress meter table."); 9424 goto error_exit; 9425 } 9426 /* Ingress meter table. */ 9427 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); 9428 if (ret) { 9429 DRV_LOG(ERR, "Failed to prepare ingress meter table."); 9430 goto error_exit; 9431 } 9432 /* FDB meter table. */ 9433 if (priv->config.dv_esw_en) { 9434 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, 9435 priv->mtr_color_reg); 9436 if (ret) { 9437 DRV_LOG(ERR, "Failed to prepare fdb meter table."); 9438 goto error_exit; 9439 } 9440 } 9441 return mtb; 9442 error_exit: 9443 flow_dv_destroy_mtr_tbl(dev, mtb); 9444 return NULL; 9445 } 9446 9447 /** 9448 * Destroy domain policer rule. 9449 * 9450 * @param[in] dt 9451 * Pointer to domain table. 9452 */ 9453 static void 9454 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) 9455 { 9456 int i; 9457 9458 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9459 if (dt->policer_rules[i]) { 9460 claim_zero(mlx5_glue->dv_destroy_flow 9461 (dt->policer_rules[i])); 9462 dt->policer_rules[i] = NULL; 9463 } 9464 } 9465 if (dt->jump_actn) { 9466 claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn)); 9467 dt->jump_actn = NULL; 9468 } 9469 } 9470 9471 /** 9472 * Destroy policer rules. 9473 * 9474 * @param[in] dev 9475 * Pointer to Ethernet device. 9476 * @param[in] fm 9477 * Pointer to flow meter structure. 9478 * @param[in] attr 9479 * Pointer to flow attributes. 9480 * 9481 * @return 9482 * Always 0. 9483 */ 9484 static int 9485 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, 9486 const struct mlx5_flow_meter *fm, 9487 const struct rte_flow_attr *attr) 9488 { 9489 struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; 9490 9491 if (!mtb) 9492 return 0; 9493 if (attr->egress) 9494 flow_dv_destroy_domain_policer_rule(&mtb->egress); 9495 if (attr->ingress) 9496 flow_dv_destroy_domain_policer_rule(&mtb->ingress); 9497 if (attr->transfer) 9498 flow_dv_destroy_domain_policer_rule(&mtb->transfer); 9499 return 0; 9500 } 9501 9502 /** 9503 * Create specify domain meter policer rule. 9504 * 9505 * @param[in] fm 9506 * Pointer to flow meter structure. 9507 * @param[in] mtb 9508 * Pointer to DV meter table set. 9509 * @param[in] mtr_reg_c 9510 * Color match REG_C. 9511 * 9512 * @return 9513 * 0 on success, -1 otherwise. 9514 */ 9515 static int 9516 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, 9517 struct mlx5_meter_domain_info *dtb, 9518 uint8_t mtr_reg_c) 9519 { 9520 struct mlx5_flow_dv_match_params matcher = { 9521 .size = sizeof(matcher.buf), 9522 }; 9523 struct mlx5_flow_dv_match_params value = { 9524 .size = sizeof(value.buf), 9525 }; 9526 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9527 void *actions[METER_ACTIONS]; 9528 int i; 9529 9530 /* Create jump action. */ 9531 if (!dtb->jump_actn) 9532 dtb->jump_actn = 9533 mlx5_glue->dr_create_flow_action_dest_flow_tbl 9534 (dtb->sfx_tbl->obj); 9535 if (!dtb->jump_actn) { 9536 DRV_LOG(ERR, "Failed to create policer jump action."); 9537 goto error; 9538 } 9539 for (i = 0; i < RTE_MTR_DROPPED; i++) { 9540 int j = 0; 9541 9542 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, 9543 rte_col_2_mlx5_col(i), UINT8_MAX); 9544 if (mtb->count_actns[i]) 9545 actions[j++] = mtb->count_actns[i]; 9546 if (fm->action[i] == MTR_POLICER_ACTION_DROP) 9547 actions[j++] = mtb->drop_actn; 9548 else 9549 actions[j++] = dtb->jump_actn; 9550 dtb->policer_rules[i] = 9551 mlx5_glue->dv_create_flow(dtb->color_matcher, 9552 (void *)&value, 9553 j, actions); 9554 if (!dtb->policer_rules[i]) { 9555 DRV_LOG(ERR, "Failed to create policer rule."); 9556 goto error; 9557 } 9558 } 9559 return 0; 9560 error: 9561 rte_errno = errno; 9562 return -1; 9563 } 9564 9565 /** 9566 * Create policer rules. 9567 * 9568 * @param[in] dev 9569 * Pointer to Ethernet device. 9570 * @param[in] fm 9571 * Pointer to flow meter structure. 9572 * @param[in] attr 9573 * Pointer to flow attributes. 9574 * 9575 * @return 9576 * 0 on success, -1 otherwise. 9577 */ 9578 static int 9579 flow_dv_create_policer_rules(struct rte_eth_dev *dev, 9580 struct mlx5_flow_meter *fm, 9581 const struct rte_flow_attr *attr) 9582 { 9583 struct mlx5_priv *priv = dev->data->dev_private; 9584 struct mlx5_meter_domains_infos *mtb = fm->mfts; 9585 int ret; 9586 9587 if (attr->egress) { 9588 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, 9589 priv->mtr_color_reg); 9590 if (ret) { 9591 DRV_LOG(ERR, "Failed to create egress policer."); 9592 goto error; 9593 } 9594 } 9595 if (attr->ingress) { 9596 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, 9597 priv->mtr_color_reg); 9598 if (ret) { 9599 DRV_LOG(ERR, "Failed to create ingress policer."); 9600 goto error; 9601 } 9602 } 9603 if (attr->transfer) { 9604 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, 9605 priv->mtr_color_reg); 9606 if (ret) { 9607 DRV_LOG(ERR, "Failed to create transfer policer."); 9608 goto error; 9609 } 9610 } 9611 return 0; 9612 error: 9613 flow_dv_destroy_policer_rules(dev, fm, attr); 9614 return -1; 9615 } 9616 9617 /** 9618 * Query a devx counter. 9619 * 9620 * @param[in] dev 9621 * Pointer to the Ethernet device structure. 9622 * @param[in] cnt 9623 * Index to the flow counter. 9624 * @param[in] clear 9625 * Set to clear the counter statistics. 9626 * @param[out] pkts 9627 * The statistics value of packets. 9628 * @param[out] bytes 9629 * The statistics value of bytes. 9630 * 9631 * @return 9632 * 0 on success, otherwise return -1. 9633 */ 9634 static int 9635 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear, 9636 uint64_t *pkts, uint64_t *bytes) 9637 { 9638 struct mlx5_priv *priv = dev->data->dev_private; 9639 struct mlx5_flow_counter *cnt; 9640 uint64_t inn_pkts, inn_bytes; 9641 int ret; 9642 9643 if (!priv->config.devx) 9644 return -1; 9645 9646 ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes); 9647 if (ret) 9648 return -1; 9649 cnt = flow_dv_counter_get_by_idx(dev, counter, NULL); 9650 *pkts = inn_pkts - cnt->hits; 9651 *bytes = inn_bytes - cnt->bytes; 9652 if (clear) { 9653 cnt->hits = inn_pkts; 9654 cnt->bytes = inn_bytes; 9655 } 9656 return 0; 9657 } 9658 9659 /** 9660 * Get aged-out flows. 9661 * 9662 * @param[in] dev 9663 * Pointer to the Ethernet device structure. 9664 * @param[in] context 9665 * The address of an array of pointers to the aged-out flows contexts. 9666 * @param[in] nb_contexts 9667 * The length of context array pointers. 9668 * @param[out] error 9669 * Perform verbose error reporting if not NULL. Initialized in case of 9670 * error only. 9671 * 9672 * @return 9673 * how many contexts get in success, otherwise negative errno value. 9674 * if nb_contexts is 0, return the amount of all aged contexts. 9675 * if nb_contexts is not 0 , return the amount of aged flows reported 9676 * in the context array. 9677 * @note: only stub for now 9678 */ 9679 static int 9680 flow_get_aged_flows(struct rte_eth_dev *dev, 9681 void **context, 9682 uint32_t nb_contexts, 9683 struct rte_flow_error *error) 9684 { 9685 struct mlx5_priv *priv = dev->data->dev_private; 9686 struct mlx5_age_info *age_info; 9687 struct mlx5_age_param *age_param; 9688 struct mlx5_flow_counter *counter; 9689 int nb_flows = 0; 9690 9691 if (nb_contexts && !context) 9692 return rte_flow_error_set(error, EINVAL, 9693 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9694 NULL, 9695 "Should assign at least one flow or" 9696 " context to get if nb_contexts != 0"); 9697 age_info = GET_PORT_AGE_INFO(priv); 9698 rte_spinlock_lock(&age_info->aged_sl); 9699 TAILQ_FOREACH(counter, &age_info->aged_counters, next) { 9700 nb_flows++; 9701 if (nb_contexts) { 9702 age_param = MLX5_CNT_TO_AGE(counter); 9703 context[nb_flows - 1] = age_param->context; 9704 if (!(--nb_contexts)) 9705 break; 9706 } 9707 } 9708 rte_spinlock_unlock(&age_info->aged_sl); 9709 MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER); 9710 return nb_flows; 9711 } 9712 9713 /* 9714 * Mutex-protected thunk to lock-free __flow_dv_translate(). 9715 */ 9716 static int 9717 flow_dv_translate(struct rte_eth_dev *dev, 9718 struct mlx5_flow *dev_flow, 9719 const struct rte_flow_attr *attr, 9720 const struct rte_flow_item items[], 9721 const struct rte_flow_action actions[], 9722 struct rte_flow_error *error) 9723 { 9724 int ret; 9725 9726 flow_dv_shared_lock(dev); 9727 ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error); 9728 flow_dv_shared_unlock(dev); 9729 return ret; 9730 } 9731 9732 /* 9733 * Mutex-protected thunk to lock-free __flow_dv_apply(). 9734 */ 9735 static int 9736 flow_dv_apply(struct rte_eth_dev *dev, 9737 struct rte_flow *flow, 9738 struct rte_flow_error *error) 9739 { 9740 int ret; 9741 9742 flow_dv_shared_lock(dev); 9743 ret = __flow_dv_apply(dev, flow, error); 9744 flow_dv_shared_unlock(dev); 9745 return ret; 9746 } 9747 9748 /* 9749 * Mutex-protected thunk to lock-free __flow_dv_remove(). 9750 */ 9751 static void 9752 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 9753 { 9754 flow_dv_shared_lock(dev); 9755 __flow_dv_remove(dev, flow); 9756 flow_dv_shared_unlock(dev); 9757 } 9758 9759 /* 9760 * Mutex-protected thunk to lock-free __flow_dv_destroy(). 9761 */ 9762 static void 9763 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 9764 { 9765 flow_dv_shared_lock(dev); 9766 __flow_dv_destroy(dev, flow); 9767 flow_dv_shared_unlock(dev); 9768 } 9769 9770 /* 9771 * Mutex-protected thunk to lock-free flow_dv_counter_alloc(). 9772 */ 9773 static uint32_t 9774 flow_dv_counter_allocate(struct rte_eth_dev *dev) 9775 { 9776 uint32_t cnt; 9777 9778 flow_dv_shared_lock(dev); 9779 cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0); 9780 flow_dv_shared_unlock(dev); 9781 return cnt; 9782 } 9783 9784 /* 9785 * Mutex-protected thunk to lock-free flow_dv_counter_release(). 9786 */ 9787 static void 9788 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 9789 { 9790 flow_dv_shared_lock(dev); 9791 flow_dv_counter_release(dev, cnt); 9792 flow_dv_shared_unlock(dev); 9793 } 9794 9795 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 9796 .validate = flow_dv_validate, 9797 .prepare = flow_dv_prepare, 9798 .translate = flow_dv_translate, 9799 .apply = flow_dv_apply, 9800 .remove = flow_dv_remove, 9801 .destroy = flow_dv_destroy, 9802 .query = flow_dv_query, 9803 .create_mtr_tbls = flow_dv_create_mtr_tbl, 9804 .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, 9805 .create_policer_rules = flow_dv_create_policer_rules, 9806 .destroy_policer_rules = flow_dv_destroy_policer_rules, 9807 .counter_alloc = flow_dv_counter_allocate, 9808 .counter_free = flow_dv_counter_free, 9809 .counter_query = flow_dv_counter_query, 9810 .get_aged_flows = flow_get_aged_flows, 9811 }; 9812 9813 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 9814