1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <unistd.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_common.h> 22 #include <rte_ether.h> 23 #include <rte_ethdev_driver.h> 24 #include <rte_flow.h> 25 #include <rte_flow_driver.h> 26 #include <rte_malloc.h> 27 #include <rte_ip.h> 28 #include <rte_gre.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_glue.h" 33 #include "mlx5_flow.h" 34 #include "mlx5_prm.h" 35 #include "mlx5_rxtx.h" 36 37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 38 39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS 40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0 41 #endif 42 43 #ifndef HAVE_MLX5DV_DR_ESWITCH 44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB 45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0 46 #endif 47 #endif 48 49 #ifndef HAVE_MLX5DV_DR 50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1 51 #endif 52 53 union flow_dv_attr { 54 struct { 55 uint32_t valid:1; 56 uint32_t ipv4:1; 57 uint32_t ipv6:1; 58 uint32_t tcp:1; 59 uint32_t udp:1; 60 uint32_t reserved:27; 61 }; 62 uint32_t attr; 63 }; 64 65 /** 66 * Initialize flow attributes structure according to flow items' types. 67 * 68 * @param[in] item 69 * Pointer to item specification. 70 * @param[out] attr 71 * Pointer to flow attributes structure. 72 */ 73 static void 74 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr) 75 { 76 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 77 switch (item->type) { 78 case RTE_FLOW_ITEM_TYPE_IPV4: 79 attr->ipv4 = 1; 80 break; 81 case RTE_FLOW_ITEM_TYPE_IPV6: 82 attr->ipv6 = 1; 83 break; 84 case RTE_FLOW_ITEM_TYPE_UDP: 85 attr->udp = 1; 86 break; 87 case RTE_FLOW_ITEM_TYPE_TCP: 88 attr->tcp = 1; 89 break; 90 default: 91 break; 92 } 93 } 94 attr->valid = 1; 95 } 96 97 struct field_modify_info { 98 uint32_t size; /* Size of field in protocol header, in bytes. */ 99 uint32_t offset; /* Offset of field in protocol header, in bytes. */ 100 enum mlx5_modification_field id; 101 }; 102 103 struct field_modify_info modify_eth[] = { 104 {4, 0, MLX5_MODI_OUT_DMAC_47_16}, 105 {2, 4, MLX5_MODI_OUT_DMAC_15_0}, 106 {4, 6, MLX5_MODI_OUT_SMAC_47_16}, 107 {2, 10, MLX5_MODI_OUT_SMAC_15_0}, 108 {0, 0, 0}, 109 }; 110 111 struct field_modify_info modify_ipv4[] = { 112 {1, 8, MLX5_MODI_OUT_IPV4_TTL}, 113 {4, 12, MLX5_MODI_OUT_SIPV4}, 114 {4, 16, MLX5_MODI_OUT_DIPV4}, 115 {0, 0, 0}, 116 }; 117 118 struct field_modify_info modify_ipv6[] = { 119 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT}, 120 {4, 8, MLX5_MODI_OUT_SIPV6_127_96}, 121 {4, 12, MLX5_MODI_OUT_SIPV6_95_64}, 122 {4, 16, MLX5_MODI_OUT_SIPV6_63_32}, 123 {4, 20, MLX5_MODI_OUT_SIPV6_31_0}, 124 {4, 24, MLX5_MODI_OUT_DIPV6_127_96}, 125 {4, 28, MLX5_MODI_OUT_DIPV6_95_64}, 126 {4, 32, MLX5_MODI_OUT_DIPV6_63_32}, 127 {4, 36, MLX5_MODI_OUT_DIPV6_31_0}, 128 {0, 0, 0}, 129 }; 130 131 struct field_modify_info modify_udp[] = { 132 {2, 0, MLX5_MODI_OUT_UDP_SPORT}, 133 {2, 2, MLX5_MODI_OUT_UDP_DPORT}, 134 {0, 0, 0}, 135 }; 136 137 struct field_modify_info modify_tcp[] = { 138 {2, 0, MLX5_MODI_OUT_TCP_SPORT}, 139 {2, 2, MLX5_MODI_OUT_TCP_DPORT}, 140 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM}, 141 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM}, 142 {0, 0, 0}, 143 }; 144 145 static void 146 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item, uint64_t *flags) 147 { 148 uint8_t next_protocol = 0xFF; 149 150 if (item->mask != NULL) { 151 switch (item->type) { 152 case RTE_FLOW_ITEM_TYPE_IPV4: 153 next_protocol = 154 ((const struct rte_flow_item_ipv4 *) 155 (item->spec))->hdr.next_proto_id; 156 next_protocol &= 157 ((const struct rte_flow_item_ipv4 *) 158 (item->mask))->hdr.next_proto_id; 159 break; 160 case RTE_FLOW_ITEM_TYPE_IPV6: 161 next_protocol = 162 ((const struct rte_flow_item_ipv6 *) 163 (item->spec))->hdr.proto; 164 next_protocol &= 165 ((const struct rte_flow_item_ipv6 *) 166 (item->mask))->hdr.proto; 167 break; 168 default: 169 break; 170 } 171 } 172 if (next_protocol == IPPROTO_IPIP) 173 *flags |= MLX5_FLOW_LAYER_IPIP; 174 if (next_protocol == IPPROTO_IPV6) 175 *flags |= MLX5_FLOW_LAYER_IPV6_ENCAP; 176 } 177 178 /** 179 * Acquire the synchronizing object to protect multithreaded access 180 * to shared dv context. Lock occurs only if context is actually 181 * shared, i.e. we have multiport IB device and representors are 182 * created. 183 * 184 * @param[in] dev 185 * Pointer to the rte_eth_dev structure. 186 */ 187 static void 188 flow_d_shared_lock(struct rte_eth_dev *dev) 189 { 190 struct mlx5_priv *priv = dev->data->dev_private; 191 struct mlx5_ibv_shared *sh = priv->sh; 192 193 if (sh->dv_refcnt > 1) { 194 int ret; 195 196 ret = pthread_mutex_lock(&sh->dv_mutex); 197 assert(!ret); 198 (void)ret; 199 } 200 } 201 202 static void 203 flow_d_shared_unlock(struct rte_eth_dev *dev) 204 { 205 struct mlx5_priv *priv = dev->data->dev_private; 206 struct mlx5_ibv_shared *sh = priv->sh; 207 208 if (sh->dv_refcnt > 1) { 209 int ret; 210 211 ret = pthread_mutex_unlock(&sh->dv_mutex); 212 assert(!ret); 213 (void)ret; 214 } 215 } 216 217 /** 218 * Convert modify-header action to DV specification. 219 * 220 * @param[in] item 221 * Pointer to item specification. 222 * @param[in] field 223 * Pointer to field modification information. 224 * @param[in,out] resource 225 * Pointer to the modify-header resource. 226 * @param[in] type 227 * Type of modification. 228 * @param[out] error 229 * Pointer to the error structure. 230 * 231 * @return 232 * 0 on success, a negative errno value otherwise and rte_errno is set. 233 */ 234 static int 235 flow_dv_convert_modify_action(struct rte_flow_item *item, 236 struct field_modify_info *field, 237 struct mlx5_flow_dv_modify_hdr_resource *resource, 238 uint32_t type, 239 struct rte_flow_error *error) 240 { 241 uint32_t i = resource->actions_num; 242 struct mlx5_modification_cmd *actions = resource->actions; 243 const uint8_t *spec = item->spec; 244 const uint8_t *mask = item->mask; 245 uint32_t set; 246 247 while (field->size) { 248 set = 0; 249 /* Generate modify command for each mask segment. */ 250 memcpy(&set, &mask[field->offset], field->size); 251 if (set) { 252 if (i >= MLX5_MODIFY_NUM) 253 return rte_flow_error_set(error, EINVAL, 254 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 255 "too many items to modify"); 256 actions[i].action_type = type; 257 actions[i].field = field->id; 258 actions[i].length = field->size == 259 4 ? 0 : field->size * 8; 260 rte_memcpy(&actions[i].data[4 - field->size], 261 &spec[field->offset], field->size); 262 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); 263 ++i; 264 } 265 if (resource->actions_num != i) 266 resource->actions_num = i; 267 field++; 268 } 269 if (!resource->actions_num) 270 return rte_flow_error_set(error, EINVAL, 271 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 272 "invalid modification flow item"); 273 return 0; 274 } 275 276 /** 277 * Convert modify-header set IPv4 address action to DV specification. 278 * 279 * @param[in,out] resource 280 * Pointer to the modify-header resource. 281 * @param[in] action 282 * Pointer to action specification. 283 * @param[out] error 284 * Pointer to the error structure. 285 * 286 * @return 287 * 0 on success, a negative errno value otherwise and rte_errno is set. 288 */ 289 static int 290 flow_dv_convert_action_modify_ipv4 291 (struct mlx5_flow_dv_modify_hdr_resource *resource, 292 const struct rte_flow_action *action, 293 struct rte_flow_error *error) 294 { 295 const struct rte_flow_action_set_ipv4 *conf = 296 (const struct rte_flow_action_set_ipv4 *)(action->conf); 297 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 }; 298 struct rte_flow_item_ipv4 ipv4; 299 struct rte_flow_item_ipv4 ipv4_mask; 300 301 memset(&ipv4, 0, sizeof(ipv4)); 302 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 303 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) { 304 ipv4.hdr.src_addr = conf->ipv4_addr; 305 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr; 306 } else { 307 ipv4.hdr.dst_addr = conf->ipv4_addr; 308 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr; 309 } 310 item.spec = &ipv4; 311 item.mask = &ipv4_mask; 312 return flow_dv_convert_modify_action(&item, modify_ipv4, resource, 313 MLX5_MODIFICATION_TYPE_SET, error); 314 } 315 316 /** 317 * Convert modify-header set IPv6 address action to DV specification. 318 * 319 * @param[in,out] resource 320 * Pointer to the modify-header resource. 321 * @param[in] action 322 * Pointer to action specification. 323 * @param[out] error 324 * Pointer to the error structure. 325 * 326 * @return 327 * 0 on success, a negative errno value otherwise and rte_errno is set. 328 */ 329 static int 330 flow_dv_convert_action_modify_ipv6 331 (struct mlx5_flow_dv_modify_hdr_resource *resource, 332 const struct rte_flow_action *action, 333 struct rte_flow_error *error) 334 { 335 const struct rte_flow_action_set_ipv6 *conf = 336 (const struct rte_flow_action_set_ipv6 *)(action->conf); 337 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 }; 338 struct rte_flow_item_ipv6 ipv6; 339 struct rte_flow_item_ipv6 ipv6_mask; 340 341 memset(&ipv6, 0, sizeof(ipv6)); 342 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 343 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) { 344 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr, 345 sizeof(ipv6.hdr.src_addr)); 346 memcpy(&ipv6_mask.hdr.src_addr, 347 &rte_flow_item_ipv6_mask.hdr.src_addr, 348 sizeof(ipv6.hdr.src_addr)); 349 } else { 350 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr, 351 sizeof(ipv6.hdr.dst_addr)); 352 memcpy(&ipv6_mask.hdr.dst_addr, 353 &rte_flow_item_ipv6_mask.hdr.dst_addr, 354 sizeof(ipv6.hdr.dst_addr)); 355 } 356 item.spec = &ipv6; 357 item.mask = &ipv6_mask; 358 return flow_dv_convert_modify_action(&item, modify_ipv6, resource, 359 MLX5_MODIFICATION_TYPE_SET, error); 360 } 361 362 /** 363 * Convert modify-header set MAC address action to DV specification. 364 * 365 * @param[in,out] resource 366 * Pointer to the modify-header resource. 367 * @param[in] action 368 * Pointer to action specification. 369 * @param[out] error 370 * Pointer to the error structure. 371 * 372 * @return 373 * 0 on success, a negative errno value otherwise and rte_errno is set. 374 */ 375 static int 376 flow_dv_convert_action_modify_mac 377 (struct mlx5_flow_dv_modify_hdr_resource *resource, 378 const struct rte_flow_action *action, 379 struct rte_flow_error *error) 380 { 381 const struct rte_flow_action_set_mac *conf = 382 (const struct rte_flow_action_set_mac *)(action->conf); 383 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH }; 384 struct rte_flow_item_eth eth; 385 struct rte_flow_item_eth eth_mask; 386 387 memset(ð, 0, sizeof(eth)); 388 memset(ð_mask, 0, sizeof(eth_mask)); 389 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) { 390 memcpy(ð.src.addr_bytes, &conf->mac_addr, 391 sizeof(eth.src.addr_bytes)); 392 memcpy(ð_mask.src.addr_bytes, 393 &rte_flow_item_eth_mask.src.addr_bytes, 394 sizeof(eth_mask.src.addr_bytes)); 395 } else { 396 memcpy(ð.dst.addr_bytes, &conf->mac_addr, 397 sizeof(eth.dst.addr_bytes)); 398 memcpy(ð_mask.dst.addr_bytes, 399 &rte_flow_item_eth_mask.dst.addr_bytes, 400 sizeof(eth_mask.dst.addr_bytes)); 401 } 402 item.spec = ð 403 item.mask = ð_mask; 404 return flow_dv_convert_modify_action(&item, modify_eth, resource, 405 MLX5_MODIFICATION_TYPE_SET, error); 406 } 407 408 /** 409 * Convert modify-header set TP action to DV specification. 410 * 411 * @param[in,out] resource 412 * Pointer to the modify-header resource. 413 * @param[in] action 414 * Pointer to action specification. 415 * @param[in] items 416 * Pointer to rte_flow_item objects list. 417 * @param[in] attr 418 * Pointer to flow attributes structure. 419 * @param[out] error 420 * Pointer to the error structure. 421 * 422 * @return 423 * 0 on success, a negative errno value otherwise and rte_errno is set. 424 */ 425 static int 426 flow_dv_convert_action_modify_tp 427 (struct mlx5_flow_dv_modify_hdr_resource *resource, 428 const struct rte_flow_action *action, 429 const struct rte_flow_item *items, 430 union flow_dv_attr *attr, 431 struct rte_flow_error *error) 432 { 433 const struct rte_flow_action_set_tp *conf = 434 (const struct rte_flow_action_set_tp *)(action->conf); 435 struct rte_flow_item item; 436 struct rte_flow_item_udp udp; 437 struct rte_flow_item_udp udp_mask; 438 struct rte_flow_item_tcp tcp; 439 struct rte_flow_item_tcp tcp_mask; 440 struct field_modify_info *field; 441 442 if (!attr->valid) 443 flow_dv_attr_init(items, attr); 444 if (attr->udp) { 445 memset(&udp, 0, sizeof(udp)); 446 memset(&udp_mask, 0, sizeof(udp_mask)); 447 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 448 udp.hdr.src_port = conf->port; 449 udp_mask.hdr.src_port = 450 rte_flow_item_udp_mask.hdr.src_port; 451 } else { 452 udp.hdr.dst_port = conf->port; 453 udp_mask.hdr.dst_port = 454 rte_flow_item_udp_mask.hdr.dst_port; 455 } 456 item.type = RTE_FLOW_ITEM_TYPE_UDP; 457 item.spec = &udp; 458 item.mask = &udp_mask; 459 field = modify_udp; 460 } 461 if (attr->tcp) { 462 memset(&tcp, 0, sizeof(tcp)); 463 memset(&tcp_mask, 0, sizeof(tcp_mask)); 464 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) { 465 tcp.hdr.src_port = conf->port; 466 tcp_mask.hdr.src_port = 467 rte_flow_item_tcp_mask.hdr.src_port; 468 } else { 469 tcp.hdr.dst_port = conf->port; 470 tcp_mask.hdr.dst_port = 471 rte_flow_item_tcp_mask.hdr.dst_port; 472 } 473 item.type = RTE_FLOW_ITEM_TYPE_TCP; 474 item.spec = &tcp; 475 item.mask = &tcp_mask; 476 field = modify_tcp; 477 } 478 return flow_dv_convert_modify_action(&item, field, resource, 479 MLX5_MODIFICATION_TYPE_SET, error); 480 } 481 482 /** 483 * Convert modify-header set TTL action to DV specification. 484 * 485 * @param[in,out] resource 486 * Pointer to the modify-header resource. 487 * @param[in] action 488 * Pointer to action specification. 489 * @param[in] items 490 * Pointer to rte_flow_item objects list. 491 * @param[in] attr 492 * Pointer to flow attributes structure. 493 * @param[out] error 494 * Pointer to the error structure. 495 * 496 * @return 497 * 0 on success, a negative errno value otherwise and rte_errno is set. 498 */ 499 static int 500 flow_dv_convert_action_modify_ttl 501 (struct mlx5_flow_dv_modify_hdr_resource *resource, 502 const struct rte_flow_action *action, 503 const struct rte_flow_item *items, 504 union flow_dv_attr *attr, 505 struct rte_flow_error *error) 506 { 507 const struct rte_flow_action_set_ttl *conf = 508 (const struct rte_flow_action_set_ttl *)(action->conf); 509 struct rte_flow_item item; 510 struct rte_flow_item_ipv4 ipv4; 511 struct rte_flow_item_ipv4 ipv4_mask; 512 struct rte_flow_item_ipv6 ipv6; 513 struct rte_flow_item_ipv6 ipv6_mask; 514 struct field_modify_info *field; 515 516 if (!attr->valid) 517 flow_dv_attr_init(items, attr); 518 if (attr->ipv4) { 519 memset(&ipv4, 0, sizeof(ipv4)); 520 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 521 ipv4.hdr.time_to_live = conf->ttl_value; 522 ipv4_mask.hdr.time_to_live = 0xFF; 523 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 524 item.spec = &ipv4; 525 item.mask = &ipv4_mask; 526 field = modify_ipv4; 527 } 528 if (attr->ipv6) { 529 memset(&ipv6, 0, sizeof(ipv6)); 530 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 531 ipv6.hdr.hop_limits = conf->ttl_value; 532 ipv6_mask.hdr.hop_limits = 0xFF; 533 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 534 item.spec = &ipv6; 535 item.mask = &ipv6_mask; 536 field = modify_ipv6; 537 } 538 return flow_dv_convert_modify_action(&item, field, resource, 539 MLX5_MODIFICATION_TYPE_SET, error); 540 } 541 542 /** 543 * Convert modify-header decrement TTL action to DV specification. 544 * 545 * @param[in,out] resource 546 * Pointer to the modify-header resource. 547 * @param[in] action 548 * Pointer to action specification. 549 * @param[in] items 550 * Pointer to rte_flow_item objects list. 551 * @param[in] attr 552 * Pointer to flow attributes structure. 553 * @param[out] error 554 * Pointer to the error structure. 555 * 556 * @return 557 * 0 on success, a negative errno value otherwise and rte_errno is set. 558 */ 559 static int 560 flow_dv_convert_action_modify_dec_ttl 561 (struct mlx5_flow_dv_modify_hdr_resource *resource, 562 const struct rte_flow_item *items, 563 union flow_dv_attr *attr, 564 struct rte_flow_error *error) 565 { 566 struct rte_flow_item item; 567 struct rte_flow_item_ipv4 ipv4; 568 struct rte_flow_item_ipv4 ipv4_mask; 569 struct rte_flow_item_ipv6 ipv6; 570 struct rte_flow_item_ipv6 ipv6_mask; 571 struct field_modify_info *field; 572 573 if (!attr->valid) 574 flow_dv_attr_init(items, attr); 575 if (attr->ipv4) { 576 memset(&ipv4, 0, sizeof(ipv4)); 577 memset(&ipv4_mask, 0, sizeof(ipv4_mask)); 578 ipv4.hdr.time_to_live = 0xFF; 579 ipv4_mask.hdr.time_to_live = 0xFF; 580 item.type = RTE_FLOW_ITEM_TYPE_IPV4; 581 item.spec = &ipv4; 582 item.mask = &ipv4_mask; 583 field = modify_ipv4; 584 } 585 if (attr->ipv6) { 586 memset(&ipv6, 0, sizeof(ipv6)); 587 memset(&ipv6_mask, 0, sizeof(ipv6_mask)); 588 ipv6.hdr.hop_limits = 0xFF; 589 ipv6_mask.hdr.hop_limits = 0xFF; 590 item.type = RTE_FLOW_ITEM_TYPE_IPV6; 591 item.spec = &ipv6; 592 item.mask = &ipv6_mask; 593 field = modify_ipv6; 594 } 595 return flow_dv_convert_modify_action(&item, field, resource, 596 MLX5_MODIFICATION_TYPE_ADD, error); 597 } 598 599 /** 600 * Convert modify-header increment/decrement TCP Sequence number 601 * to DV specification. 602 * 603 * @param[in,out] resource 604 * Pointer to the modify-header resource. 605 * @param[in] action 606 * Pointer to action specification. 607 * @param[out] error 608 * Pointer to the error structure. 609 * 610 * @return 611 * 0 on success, a negative errno value otherwise and rte_errno is set. 612 */ 613 static int 614 flow_dv_convert_action_modify_tcp_seq 615 (struct mlx5_flow_dv_modify_hdr_resource *resource, 616 const struct rte_flow_action *action, 617 struct rte_flow_error *error) 618 { 619 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 620 uint64_t value = rte_be_to_cpu_32(*conf); 621 struct rte_flow_item item; 622 struct rte_flow_item_tcp tcp; 623 struct rte_flow_item_tcp tcp_mask; 624 625 memset(&tcp, 0, sizeof(tcp)); 626 memset(&tcp_mask, 0, sizeof(tcp_mask)); 627 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ) 628 /* 629 * The HW has no decrement operation, only increment operation. 630 * To simulate decrement X from Y using increment operation 631 * we need to add UINT32_MAX X times to Y. 632 * Each adding of UINT32_MAX decrements Y by 1. 633 */ 634 value *= UINT32_MAX; 635 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value); 636 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX); 637 item.type = RTE_FLOW_ITEM_TYPE_TCP; 638 item.spec = &tcp; 639 item.mask = &tcp_mask; 640 return flow_dv_convert_modify_action(&item, modify_tcp, resource, 641 MLX5_MODIFICATION_TYPE_ADD, error); 642 } 643 644 /** 645 * Convert modify-header increment/decrement TCP Acknowledgment number 646 * to DV specification. 647 * 648 * @param[in,out] resource 649 * Pointer to the modify-header resource. 650 * @param[in] action 651 * Pointer to action specification. 652 * @param[out] error 653 * Pointer to the error structure. 654 * 655 * @return 656 * 0 on success, a negative errno value otherwise and rte_errno is set. 657 */ 658 static int 659 flow_dv_convert_action_modify_tcp_ack 660 (struct mlx5_flow_dv_modify_hdr_resource *resource, 661 const struct rte_flow_action *action, 662 struct rte_flow_error *error) 663 { 664 const rte_be32_t *conf = (const rte_be32_t *)(action->conf); 665 uint64_t value = rte_be_to_cpu_32(*conf); 666 struct rte_flow_item item; 667 struct rte_flow_item_tcp tcp; 668 struct rte_flow_item_tcp tcp_mask; 669 670 memset(&tcp, 0, sizeof(tcp)); 671 memset(&tcp_mask, 0, sizeof(tcp_mask)); 672 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK) 673 /* 674 * The HW has no decrement operation, only increment operation. 675 * To simulate decrement X from Y using increment operation 676 * we need to add UINT32_MAX X times to Y. 677 * Each adding of UINT32_MAX decrements Y by 1. 678 */ 679 value *= UINT32_MAX; 680 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value); 681 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX); 682 item.type = RTE_FLOW_ITEM_TYPE_TCP; 683 item.spec = &tcp; 684 item.mask = &tcp_mask; 685 return flow_dv_convert_modify_action(&item, modify_tcp, resource, 686 MLX5_MODIFICATION_TYPE_ADD, error); 687 } 688 689 /** 690 * Validate META item. 691 * 692 * @param[in] dev 693 * Pointer to the rte_eth_dev structure. 694 * @param[in] item 695 * Item specification. 696 * @param[in] attr 697 * Attributes of flow that includes this item. 698 * @param[out] error 699 * Pointer to error structure. 700 * 701 * @return 702 * 0 on success, a negative errno value otherwise and rte_errno is set. 703 */ 704 static int 705 flow_dv_validate_item_meta(struct rte_eth_dev *dev, 706 const struct rte_flow_item *item, 707 const struct rte_flow_attr *attr, 708 struct rte_flow_error *error) 709 { 710 const struct rte_flow_item_meta *spec = item->spec; 711 const struct rte_flow_item_meta *mask = item->mask; 712 const struct rte_flow_item_meta nic_mask = { 713 .data = RTE_BE32(UINT32_MAX) 714 }; 715 int ret; 716 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 717 718 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA)) 719 return rte_flow_error_set(error, EPERM, 720 RTE_FLOW_ERROR_TYPE_ITEM, 721 NULL, 722 "match on metadata offload " 723 "configuration is off for this port"); 724 if (!spec) 725 return rte_flow_error_set(error, EINVAL, 726 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 727 item->spec, 728 "data cannot be empty"); 729 if (!spec->data) 730 return rte_flow_error_set(error, EINVAL, 731 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 732 NULL, 733 "data cannot be zero"); 734 if (!mask) 735 mask = &rte_flow_item_meta_mask; 736 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 737 (const uint8_t *)&nic_mask, 738 sizeof(struct rte_flow_item_meta), 739 error); 740 if (ret < 0) 741 return ret; 742 if (attr->ingress) 743 return rte_flow_error_set(error, ENOTSUP, 744 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 745 NULL, 746 "pattern not supported for ingress"); 747 return 0; 748 } 749 750 /** 751 * Validate vport item. 752 * 753 * @param[in] dev 754 * Pointer to the rte_eth_dev structure. 755 * @param[in] item 756 * Item specification. 757 * @param[in] attr 758 * Attributes of flow that includes this item. 759 * @param[in] item_flags 760 * Bit-fields that holds the items detected until now. 761 * @param[out] error 762 * Pointer to error structure. 763 * 764 * @return 765 * 0 on success, a negative errno value otherwise and rte_errno is set. 766 */ 767 static int 768 flow_dv_validate_item_port_id(struct rte_eth_dev *dev, 769 const struct rte_flow_item *item, 770 const struct rte_flow_attr *attr, 771 uint64_t item_flags, 772 struct rte_flow_error *error) 773 { 774 const struct rte_flow_item_port_id *spec = item->spec; 775 const struct rte_flow_item_port_id *mask = item->mask; 776 const struct rte_flow_item_port_id switch_mask = { 777 .id = 0xffffffff, 778 }; 779 uint16_t esw_domain_id; 780 uint16_t item_port_esw_domain_id; 781 int ret; 782 783 if (!attr->transfer) 784 return rte_flow_error_set(error, EINVAL, 785 RTE_FLOW_ERROR_TYPE_ITEM, 786 NULL, 787 "match on port id is valid only" 788 " when transfer flag is enabled"); 789 if (item_flags & MLX5_FLOW_ITEM_PORT_ID) 790 return rte_flow_error_set(error, ENOTSUP, 791 RTE_FLOW_ERROR_TYPE_ITEM, item, 792 "multiple source ports are not" 793 " supported"); 794 if (!mask) 795 mask = &switch_mask; 796 if (mask->id != 0xffffffff) 797 return rte_flow_error_set(error, ENOTSUP, 798 RTE_FLOW_ERROR_TYPE_ITEM_MASK, 799 mask, 800 "no support for partial mask on" 801 " \"id\" field"); 802 ret = mlx5_flow_item_acceptable 803 (item, (const uint8_t *)mask, 804 (const uint8_t *)&rte_flow_item_port_id_mask, 805 sizeof(struct rte_flow_item_port_id), 806 error); 807 if (ret) 808 return ret; 809 if (!spec) 810 return 0; 811 ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id, 812 NULL); 813 if (ret) 814 return rte_flow_error_set(error, -ret, 815 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 816 "failed to obtain E-Switch info for" 817 " port"); 818 ret = mlx5_port_to_eswitch_info(dev->data->port_id, 819 &esw_domain_id, NULL); 820 if (ret < 0) 821 return rte_flow_error_set(error, -ret, 822 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 823 NULL, 824 "failed to obtain E-Switch info"); 825 if (item_port_esw_domain_id != esw_domain_id) 826 return rte_flow_error_set(error, -ret, 827 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec, 828 "cannot match on a port from a" 829 " different E-Switch"); 830 return 0; 831 } 832 833 /** 834 * Validate count action. 835 * 836 * @param[in] dev 837 * device otr. 838 * @param[out] error 839 * Pointer to error structure. 840 * 841 * @return 842 * 0 on success, a negative errno value otherwise and rte_errno is set. 843 */ 844 static int 845 flow_dv_validate_action_count(struct rte_eth_dev *dev, 846 struct rte_flow_error *error) 847 { 848 struct mlx5_priv *priv = dev->data->dev_private; 849 850 if (!priv->config.devx) 851 goto notsup_err; 852 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS 853 return 0; 854 #endif 855 notsup_err: 856 return rte_flow_error_set 857 (error, ENOTSUP, 858 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 859 NULL, 860 "count action not supported"); 861 } 862 863 /** 864 * Validate the L2 encap action. 865 * 866 * @param[in] action_flags 867 * Holds the actions detected until now. 868 * @param[in] action 869 * Pointer to the encap action. 870 * @param[in] attr 871 * Pointer to flow attributes 872 * @param[out] error 873 * Pointer to error structure. 874 * 875 * @return 876 * 0 on success, a negative errno value otherwise and rte_errno is set. 877 */ 878 static int 879 flow_dv_validate_action_l2_encap(uint64_t action_flags, 880 const struct rte_flow_action *action, 881 const struct rte_flow_attr *attr, 882 struct rte_flow_error *error) 883 { 884 if (!(action->conf)) 885 return rte_flow_error_set(error, EINVAL, 886 RTE_FLOW_ERROR_TYPE_ACTION, action, 887 "configuration cannot be null"); 888 if (action_flags & MLX5_FLOW_ACTION_DROP) 889 return rte_flow_error_set(error, EINVAL, 890 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 891 "can't drop and encap in same flow"); 892 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) 893 return rte_flow_error_set(error, EINVAL, 894 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 895 "can only have a single encap or" 896 " decap action in a flow"); 897 if (!attr->transfer && attr->ingress) 898 return rte_flow_error_set(error, ENOTSUP, 899 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 900 NULL, 901 "encap action not supported for " 902 "ingress"); 903 return 0; 904 } 905 906 /** 907 * Validate the L2 decap action. 908 * 909 * @param[in] action_flags 910 * Holds the actions detected until now. 911 * @param[in] attr 912 * Pointer to flow attributes 913 * @param[out] error 914 * Pointer to error structure. 915 * 916 * @return 917 * 0 on success, a negative errno value otherwise and rte_errno is set. 918 */ 919 static int 920 flow_dv_validate_action_l2_decap(uint64_t action_flags, 921 const struct rte_flow_attr *attr, 922 struct rte_flow_error *error) 923 { 924 if (action_flags & MLX5_FLOW_ACTION_DROP) 925 return rte_flow_error_set(error, EINVAL, 926 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 927 "can't drop and decap in same flow"); 928 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) 929 return rte_flow_error_set(error, EINVAL, 930 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 931 "can only have a single encap or" 932 " decap action in a flow"); 933 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 934 return rte_flow_error_set(error, EINVAL, 935 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 936 "can't have decap action after" 937 " modify action"); 938 if (attr->egress) 939 return rte_flow_error_set(error, ENOTSUP, 940 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 941 NULL, 942 "decap action not supported for " 943 "egress"); 944 return 0; 945 } 946 947 /** 948 * Validate the raw encap action. 949 * 950 * @param[in] action_flags 951 * Holds the actions detected until now. 952 * @param[in] action 953 * Pointer to the encap action. 954 * @param[in] attr 955 * Pointer to flow attributes 956 * @param[out] error 957 * Pointer to error structure. 958 * 959 * @return 960 * 0 on success, a negative errno value otherwise and rte_errno is set. 961 */ 962 static int 963 flow_dv_validate_action_raw_encap(uint64_t action_flags, 964 const struct rte_flow_action *action, 965 const struct rte_flow_attr *attr, 966 struct rte_flow_error *error) 967 { 968 if (!(action->conf)) 969 return rte_flow_error_set(error, EINVAL, 970 RTE_FLOW_ERROR_TYPE_ACTION, action, 971 "configuration cannot be null"); 972 if (action_flags & MLX5_FLOW_ACTION_DROP) 973 return rte_flow_error_set(error, EINVAL, 974 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 975 "can't drop and encap in same flow"); 976 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) 977 return rte_flow_error_set(error, EINVAL, 978 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 979 "can only have a single encap" 980 " action in a flow"); 981 /* encap without preceding decap is not supported for ingress */ 982 if (!attr->transfer && attr->ingress && 983 !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP)) 984 return rte_flow_error_set(error, ENOTSUP, 985 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 986 NULL, 987 "encap action not supported for " 988 "ingress"); 989 return 0; 990 } 991 992 /** 993 * Validate the raw decap action. 994 * 995 * @param[in] action_flags 996 * Holds the actions detected until now. 997 * @param[in] action 998 * Pointer to the encap action. 999 * @param[in] attr 1000 * Pointer to flow attributes 1001 * @param[out] error 1002 * Pointer to error structure. 1003 * 1004 * @return 1005 * 0 on success, a negative errno value otherwise and rte_errno is set. 1006 */ 1007 static int 1008 flow_dv_validate_action_raw_decap(uint64_t action_flags, 1009 const struct rte_flow_action *action, 1010 const struct rte_flow_attr *attr, 1011 struct rte_flow_error *error) 1012 { 1013 if (action_flags & MLX5_FLOW_ACTION_DROP) 1014 return rte_flow_error_set(error, EINVAL, 1015 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1016 "can't drop and decap in same flow"); 1017 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) 1018 return rte_flow_error_set(error, EINVAL, 1019 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1020 "can't have encap action before" 1021 " decap action"); 1022 if (action_flags & MLX5_FLOW_DECAP_ACTIONS) 1023 return rte_flow_error_set(error, EINVAL, 1024 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1025 "can only have a single decap" 1026 " action in a flow"); 1027 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) 1028 return rte_flow_error_set(error, EINVAL, 1029 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1030 "can't have decap action after" 1031 " modify action"); 1032 /* decap action is valid on egress only if it is followed by encap */ 1033 if (attr->egress) { 1034 for (; action->type != RTE_FLOW_ACTION_TYPE_END && 1035 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; 1036 action++) { 1037 } 1038 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) 1039 return rte_flow_error_set 1040 (error, ENOTSUP, 1041 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 1042 NULL, "decap action not supported" 1043 " for egress"); 1044 } 1045 return 0; 1046 } 1047 1048 /** 1049 * Find existing encap/decap resource or create and register a new one. 1050 * 1051 * @param dev[in, out] 1052 * Pointer to rte_eth_dev structure. 1053 * @param[in, out] resource 1054 * Pointer to encap/decap resource. 1055 * @parm[in, out] dev_flow 1056 * Pointer to the dev_flow. 1057 * @param[out] error 1058 * pointer to error structure. 1059 * 1060 * @return 1061 * 0 on success otherwise -errno and errno is set. 1062 */ 1063 static int 1064 flow_dv_encap_decap_resource_register 1065 (struct rte_eth_dev *dev, 1066 struct mlx5_flow_dv_encap_decap_resource *resource, 1067 struct mlx5_flow *dev_flow, 1068 struct rte_flow_error *error) 1069 { 1070 struct mlx5_priv *priv = dev->data->dev_private; 1071 struct mlx5_ibv_shared *sh = priv->sh; 1072 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 1073 struct rte_flow *flow = dev_flow->flow; 1074 struct mlx5dv_dr_domain *domain; 1075 1076 resource->flags = flow->group ? 0 : 1; 1077 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 1078 domain = sh->fdb_domain; 1079 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX) 1080 domain = sh->rx_domain; 1081 else 1082 domain = sh->tx_domain; 1083 1084 /* Lookup a matching resource from cache. */ 1085 LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) { 1086 if (resource->reformat_type == cache_resource->reformat_type && 1087 resource->ft_type == cache_resource->ft_type && 1088 resource->flags == cache_resource->flags && 1089 resource->size == cache_resource->size && 1090 !memcmp((const void *)resource->buf, 1091 (const void *)cache_resource->buf, 1092 resource->size)) { 1093 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", 1094 (void *)cache_resource, 1095 rte_atomic32_read(&cache_resource->refcnt)); 1096 rte_atomic32_inc(&cache_resource->refcnt); 1097 dev_flow->dv.encap_decap = cache_resource; 1098 return 0; 1099 } 1100 } 1101 /* Register new encap/decap resource. */ 1102 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 1103 if (!cache_resource) 1104 return rte_flow_error_set(error, ENOMEM, 1105 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1106 "cannot allocate resource memory"); 1107 *cache_resource = *resource; 1108 cache_resource->verbs_action = 1109 mlx5_glue->dv_create_flow_action_packet_reformat 1110 (sh->ctx, cache_resource->reformat_type, 1111 cache_resource->ft_type, domain, cache_resource->flags, 1112 cache_resource->size, 1113 (cache_resource->size ? cache_resource->buf : NULL)); 1114 if (!cache_resource->verbs_action) { 1115 rte_free(cache_resource); 1116 return rte_flow_error_set(error, ENOMEM, 1117 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1118 NULL, "cannot create action"); 1119 } 1120 rte_atomic32_init(&cache_resource->refcnt); 1121 rte_atomic32_inc(&cache_resource->refcnt); 1122 LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next); 1123 dev_flow->dv.encap_decap = cache_resource; 1124 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", 1125 (void *)cache_resource, 1126 rte_atomic32_read(&cache_resource->refcnt)); 1127 return 0; 1128 } 1129 1130 /** 1131 * Find existing table jump resource or create and register a new one. 1132 * 1133 * @param dev[in, out] 1134 * Pointer to rte_eth_dev structure. 1135 * @param[in, out] resource 1136 * Pointer to jump table resource. 1137 * @parm[in, out] dev_flow 1138 * Pointer to the dev_flow. 1139 * @param[out] error 1140 * pointer to error structure. 1141 * 1142 * @return 1143 * 0 on success otherwise -errno and errno is set. 1144 */ 1145 static int 1146 flow_dv_jump_tbl_resource_register 1147 (struct rte_eth_dev *dev, 1148 struct mlx5_flow_dv_jump_tbl_resource *resource, 1149 struct mlx5_flow *dev_flow, 1150 struct rte_flow_error *error) 1151 { 1152 struct mlx5_priv *priv = dev->data->dev_private; 1153 struct mlx5_ibv_shared *sh = priv->sh; 1154 struct mlx5_flow_dv_jump_tbl_resource *cache_resource; 1155 1156 /* Lookup a matching resource from cache. */ 1157 LIST_FOREACH(cache_resource, &sh->jump_tbl, next) { 1158 if (resource->tbl == cache_resource->tbl) { 1159 DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++", 1160 (void *)cache_resource, 1161 rte_atomic32_read(&cache_resource->refcnt)); 1162 rte_atomic32_inc(&cache_resource->refcnt); 1163 dev_flow->dv.jump = cache_resource; 1164 return 0; 1165 } 1166 } 1167 /* Register new jump table resource. */ 1168 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 1169 if (!cache_resource) 1170 return rte_flow_error_set(error, ENOMEM, 1171 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1172 "cannot allocate resource memory"); 1173 *cache_resource = *resource; 1174 cache_resource->action = 1175 mlx5_glue->dr_create_flow_action_dest_flow_tbl 1176 (resource->tbl->obj); 1177 if (!cache_resource->action) { 1178 rte_free(cache_resource); 1179 return rte_flow_error_set(error, ENOMEM, 1180 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1181 NULL, "cannot create action"); 1182 } 1183 rte_atomic32_init(&cache_resource->refcnt); 1184 rte_atomic32_inc(&cache_resource->refcnt); 1185 LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next); 1186 dev_flow->dv.jump = cache_resource; 1187 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++", 1188 (void *)cache_resource, 1189 rte_atomic32_read(&cache_resource->refcnt)); 1190 return 0; 1191 } 1192 1193 /** 1194 * Find existing table port ID resource or create and register a new one. 1195 * 1196 * @param dev[in, out] 1197 * Pointer to rte_eth_dev structure. 1198 * @param[in, out] resource 1199 * Pointer to port ID action resource. 1200 * @parm[in, out] dev_flow 1201 * Pointer to the dev_flow. 1202 * @param[out] error 1203 * pointer to error structure. 1204 * 1205 * @return 1206 * 0 on success otherwise -errno and errno is set. 1207 */ 1208 static int 1209 flow_dv_port_id_action_resource_register 1210 (struct rte_eth_dev *dev, 1211 struct mlx5_flow_dv_port_id_action_resource *resource, 1212 struct mlx5_flow *dev_flow, 1213 struct rte_flow_error *error) 1214 { 1215 struct mlx5_priv *priv = dev->data->dev_private; 1216 struct mlx5_ibv_shared *sh = priv->sh; 1217 struct mlx5_flow_dv_port_id_action_resource *cache_resource; 1218 1219 /* Lookup a matching resource from cache. */ 1220 LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) { 1221 if (resource->port_id == cache_resource->port_id) { 1222 DRV_LOG(DEBUG, "port id action resource resource %p: " 1223 "refcnt %d++", 1224 (void *)cache_resource, 1225 rte_atomic32_read(&cache_resource->refcnt)); 1226 rte_atomic32_inc(&cache_resource->refcnt); 1227 dev_flow->dv.port_id_action = cache_resource; 1228 return 0; 1229 } 1230 } 1231 /* Register new port id action resource. */ 1232 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 1233 if (!cache_resource) 1234 return rte_flow_error_set(error, ENOMEM, 1235 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1236 "cannot allocate resource memory"); 1237 *cache_resource = *resource; 1238 cache_resource->action = 1239 mlx5_glue->dr_create_flow_action_dest_vport 1240 (priv->sh->fdb_domain, resource->port_id); 1241 if (!cache_resource->action) { 1242 rte_free(cache_resource); 1243 return rte_flow_error_set(error, ENOMEM, 1244 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1245 NULL, "cannot create action"); 1246 } 1247 rte_atomic32_init(&cache_resource->refcnt); 1248 rte_atomic32_inc(&cache_resource->refcnt); 1249 LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next); 1250 dev_flow->dv.port_id_action = cache_resource; 1251 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++", 1252 (void *)cache_resource, 1253 rte_atomic32_read(&cache_resource->refcnt)); 1254 return 0; 1255 } 1256 1257 /** 1258 * Get the size of specific rte_flow_item_type 1259 * 1260 * @param[in] item_type 1261 * Tested rte_flow_item_type. 1262 * 1263 * @return 1264 * sizeof struct item_type, 0 if void or irrelevant. 1265 */ 1266 static size_t 1267 flow_dv_get_item_len(const enum rte_flow_item_type item_type) 1268 { 1269 size_t retval; 1270 1271 switch (item_type) { 1272 case RTE_FLOW_ITEM_TYPE_ETH: 1273 retval = sizeof(struct rte_flow_item_eth); 1274 break; 1275 case RTE_FLOW_ITEM_TYPE_VLAN: 1276 retval = sizeof(struct rte_flow_item_vlan); 1277 break; 1278 case RTE_FLOW_ITEM_TYPE_IPV4: 1279 retval = sizeof(struct rte_flow_item_ipv4); 1280 break; 1281 case RTE_FLOW_ITEM_TYPE_IPV6: 1282 retval = sizeof(struct rte_flow_item_ipv6); 1283 break; 1284 case RTE_FLOW_ITEM_TYPE_UDP: 1285 retval = sizeof(struct rte_flow_item_udp); 1286 break; 1287 case RTE_FLOW_ITEM_TYPE_TCP: 1288 retval = sizeof(struct rte_flow_item_tcp); 1289 break; 1290 case RTE_FLOW_ITEM_TYPE_VXLAN: 1291 retval = sizeof(struct rte_flow_item_vxlan); 1292 break; 1293 case RTE_FLOW_ITEM_TYPE_GRE: 1294 retval = sizeof(struct rte_flow_item_gre); 1295 break; 1296 case RTE_FLOW_ITEM_TYPE_NVGRE: 1297 retval = sizeof(struct rte_flow_item_nvgre); 1298 break; 1299 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1300 retval = sizeof(struct rte_flow_item_vxlan_gpe); 1301 break; 1302 case RTE_FLOW_ITEM_TYPE_MPLS: 1303 retval = sizeof(struct rte_flow_item_mpls); 1304 break; 1305 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 1306 default: 1307 retval = 0; 1308 break; 1309 } 1310 return retval; 1311 } 1312 1313 #define MLX5_ENCAP_IPV4_VERSION 0x40 1314 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 1315 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 1316 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 1317 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 1318 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 1319 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 1320 1321 /** 1322 * Convert the encap action data from list of rte_flow_item to raw buffer 1323 * 1324 * @param[in] items 1325 * Pointer to rte_flow_item objects list. 1326 * @param[out] buf 1327 * Pointer to the output buffer. 1328 * @param[out] size 1329 * Pointer to the output buffer size. 1330 * @param[out] error 1331 * Pointer to the error structure. 1332 * 1333 * @return 1334 * 0 on success, a negative errno value otherwise and rte_errno is set. 1335 */ 1336 static int 1337 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 1338 size_t *size, struct rte_flow_error *error) 1339 { 1340 struct rte_ether_hdr *eth = NULL; 1341 struct rte_vlan_hdr *vlan = NULL; 1342 struct rte_ipv4_hdr *ipv4 = NULL; 1343 struct rte_ipv6_hdr *ipv6 = NULL; 1344 struct rte_udp_hdr *udp = NULL; 1345 struct rte_vxlan_hdr *vxlan = NULL; 1346 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL; 1347 struct rte_gre_hdr *gre = NULL; 1348 size_t len; 1349 size_t temp_size = 0; 1350 1351 if (!items) 1352 return rte_flow_error_set(error, EINVAL, 1353 RTE_FLOW_ERROR_TYPE_ACTION, 1354 NULL, "invalid empty data"); 1355 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 1356 len = flow_dv_get_item_len(items->type); 1357 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 1358 return rte_flow_error_set(error, EINVAL, 1359 RTE_FLOW_ERROR_TYPE_ACTION, 1360 (void *)items->type, 1361 "items total size is too big" 1362 " for encap action"); 1363 rte_memcpy((void *)&buf[temp_size], items->spec, len); 1364 switch (items->type) { 1365 case RTE_FLOW_ITEM_TYPE_ETH: 1366 eth = (struct rte_ether_hdr *)&buf[temp_size]; 1367 break; 1368 case RTE_FLOW_ITEM_TYPE_VLAN: 1369 vlan = (struct rte_vlan_hdr *)&buf[temp_size]; 1370 if (!eth) 1371 return rte_flow_error_set(error, EINVAL, 1372 RTE_FLOW_ERROR_TYPE_ACTION, 1373 (void *)items->type, 1374 "eth header not found"); 1375 if (!eth->ether_type) 1376 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN); 1377 break; 1378 case RTE_FLOW_ITEM_TYPE_IPV4: 1379 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size]; 1380 if (!vlan && !eth) 1381 return rte_flow_error_set(error, EINVAL, 1382 RTE_FLOW_ERROR_TYPE_ACTION, 1383 (void *)items->type, 1384 "neither eth nor vlan" 1385 " header found"); 1386 if (vlan && !vlan->eth_proto) 1387 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4); 1388 else if (eth && !eth->ether_type) 1389 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4); 1390 if (!ipv4->version_ihl) 1391 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 1392 MLX5_ENCAP_IPV4_IHL_MIN; 1393 if (!ipv4->time_to_live) 1394 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 1395 break; 1396 case RTE_FLOW_ITEM_TYPE_IPV6: 1397 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size]; 1398 if (!vlan && !eth) 1399 return rte_flow_error_set(error, EINVAL, 1400 RTE_FLOW_ERROR_TYPE_ACTION, 1401 (void *)items->type, 1402 "neither eth nor vlan" 1403 " header found"); 1404 if (vlan && !vlan->eth_proto) 1405 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6); 1406 else if (eth && !eth->ether_type) 1407 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6); 1408 if (!ipv6->vtc_flow) 1409 ipv6->vtc_flow = 1410 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 1411 if (!ipv6->hop_limits) 1412 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 1413 break; 1414 case RTE_FLOW_ITEM_TYPE_UDP: 1415 udp = (struct rte_udp_hdr *)&buf[temp_size]; 1416 if (!ipv4 && !ipv6) 1417 return rte_flow_error_set(error, EINVAL, 1418 RTE_FLOW_ERROR_TYPE_ACTION, 1419 (void *)items->type, 1420 "ip header not found"); 1421 if (ipv4 && !ipv4->next_proto_id) 1422 ipv4->next_proto_id = IPPROTO_UDP; 1423 else if (ipv6 && !ipv6->proto) 1424 ipv6->proto = IPPROTO_UDP; 1425 break; 1426 case RTE_FLOW_ITEM_TYPE_VXLAN: 1427 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size]; 1428 if (!udp) 1429 return rte_flow_error_set(error, EINVAL, 1430 RTE_FLOW_ERROR_TYPE_ACTION, 1431 (void *)items->type, 1432 "udp header not found"); 1433 if (!udp->dst_port) 1434 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 1435 if (!vxlan->vx_flags) 1436 vxlan->vx_flags = 1437 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 1438 break; 1439 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 1440 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size]; 1441 if (!udp) 1442 return rte_flow_error_set(error, EINVAL, 1443 RTE_FLOW_ERROR_TYPE_ACTION, 1444 (void *)items->type, 1445 "udp header not found"); 1446 if (!vxlan_gpe->proto) 1447 return rte_flow_error_set(error, EINVAL, 1448 RTE_FLOW_ERROR_TYPE_ACTION, 1449 (void *)items->type, 1450 "next protocol not found"); 1451 if (!udp->dst_port) 1452 udp->dst_port = 1453 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 1454 if (!vxlan_gpe->vx_flags) 1455 vxlan_gpe->vx_flags = 1456 MLX5_ENCAP_VXLAN_GPE_FLAGS; 1457 break; 1458 case RTE_FLOW_ITEM_TYPE_GRE: 1459 case RTE_FLOW_ITEM_TYPE_NVGRE: 1460 gre = (struct rte_gre_hdr *)&buf[temp_size]; 1461 if (!gre->proto) 1462 return rte_flow_error_set(error, EINVAL, 1463 RTE_FLOW_ERROR_TYPE_ACTION, 1464 (void *)items->type, 1465 "next protocol not found"); 1466 if (!ipv4 && !ipv6) 1467 return rte_flow_error_set(error, EINVAL, 1468 RTE_FLOW_ERROR_TYPE_ACTION, 1469 (void *)items->type, 1470 "ip header not found"); 1471 if (ipv4 && !ipv4->next_proto_id) 1472 ipv4->next_proto_id = IPPROTO_GRE; 1473 else if (ipv6 && !ipv6->proto) 1474 ipv6->proto = IPPROTO_GRE; 1475 break; 1476 case RTE_FLOW_ITEM_TYPE_VOID: 1477 break; 1478 default: 1479 return rte_flow_error_set(error, EINVAL, 1480 RTE_FLOW_ERROR_TYPE_ACTION, 1481 (void *)items->type, 1482 "unsupported item type"); 1483 break; 1484 } 1485 temp_size += len; 1486 } 1487 *size = temp_size; 1488 return 0; 1489 } 1490 1491 static int 1492 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error) 1493 { 1494 struct rte_ether_hdr *eth = NULL; 1495 struct rte_vlan_hdr *vlan = NULL; 1496 struct rte_ipv6_hdr *ipv6 = NULL; 1497 struct rte_udp_hdr *udp = NULL; 1498 char *next_hdr; 1499 uint16_t proto; 1500 1501 eth = (struct rte_ether_hdr *)data; 1502 next_hdr = (char *)(eth + 1); 1503 proto = RTE_BE16(eth->ether_type); 1504 1505 /* VLAN skipping */ 1506 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) { 1507 next_hdr += sizeof(struct rte_vlan_hdr); 1508 vlan = (struct rte_vlan_hdr *)next_hdr; 1509 proto = RTE_BE16(vlan->eth_proto); 1510 } 1511 1512 /* HW calculates IPv4 csum. no need to proceed */ 1513 if (proto == RTE_ETHER_TYPE_IPV4) 1514 return 0; 1515 1516 /* non IPv4/IPv6 header. not supported */ 1517 if (proto != RTE_ETHER_TYPE_IPV6) { 1518 return rte_flow_error_set(error, ENOTSUP, 1519 RTE_FLOW_ERROR_TYPE_ACTION, 1520 NULL, "Cannot offload non IPv4/IPv6"); 1521 } 1522 1523 ipv6 = (struct rte_ipv6_hdr *)next_hdr; 1524 1525 /* ignore non UDP */ 1526 if (ipv6->proto != IPPROTO_UDP) 1527 return 0; 1528 1529 udp = (struct rte_udp_hdr *)(ipv6 + 1); 1530 udp->dgram_cksum = 0; 1531 1532 return 0; 1533 } 1534 1535 /** 1536 * Convert L2 encap action to DV specification. 1537 * 1538 * @param[in] dev 1539 * Pointer to rte_eth_dev structure. 1540 * @param[in] action 1541 * Pointer to action structure. 1542 * @param[in, out] dev_flow 1543 * Pointer to the mlx5_flow. 1544 * @param[in] transfer 1545 * Mark if the flow is E-Switch flow. 1546 * @param[out] error 1547 * Pointer to the error structure. 1548 * 1549 * @return 1550 * 0 on success, a negative errno value otherwise and rte_errno is set. 1551 */ 1552 static int 1553 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 1554 const struct rte_flow_action *action, 1555 struct mlx5_flow *dev_flow, 1556 uint8_t transfer, 1557 struct rte_flow_error *error) 1558 { 1559 const struct rte_flow_item *encap_data; 1560 const struct rte_flow_action_raw_encap *raw_encap_data; 1561 struct mlx5_flow_dv_encap_decap_resource res = { 1562 .reformat_type = 1563 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 1564 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 1565 MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 1566 }; 1567 1568 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 1569 raw_encap_data = 1570 (const struct rte_flow_action_raw_encap *)action->conf; 1571 res.size = raw_encap_data->size; 1572 memcpy(res.buf, raw_encap_data->data, res.size); 1573 if (flow_dv_zero_encap_udp_csum(res.buf, error)) 1574 return -rte_errno; 1575 } else { 1576 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 1577 encap_data = 1578 ((const struct rte_flow_action_vxlan_encap *) 1579 action->conf)->definition; 1580 else 1581 encap_data = 1582 ((const struct rte_flow_action_nvgre_encap *) 1583 action->conf)->definition; 1584 if (flow_dv_convert_encap_data(encap_data, res.buf, 1585 &res.size, error)) 1586 return -rte_errno; 1587 } 1588 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 1589 return rte_flow_error_set(error, EINVAL, 1590 RTE_FLOW_ERROR_TYPE_ACTION, 1591 NULL, "can't create L2 encap action"); 1592 return 0; 1593 } 1594 1595 /** 1596 * Convert L2 decap action to DV specification. 1597 * 1598 * @param[in] dev 1599 * Pointer to rte_eth_dev structure. 1600 * @param[in, out] dev_flow 1601 * Pointer to the mlx5_flow. 1602 * @param[in] transfer 1603 * Mark if the flow is E-Switch flow. 1604 * @param[out] error 1605 * Pointer to the error structure. 1606 * 1607 * @return 1608 * 0 on success, a negative errno value otherwise and rte_errno is set. 1609 */ 1610 static int 1611 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 1612 struct mlx5_flow *dev_flow, 1613 uint8_t transfer, 1614 struct rte_flow_error *error) 1615 { 1616 struct mlx5_flow_dv_encap_decap_resource res = { 1617 .size = 0, 1618 .reformat_type = 1619 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 1620 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB : 1621 MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 1622 }; 1623 1624 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 1625 return rte_flow_error_set(error, EINVAL, 1626 RTE_FLOW_ERROR_TYPE_ACTION, 1627 NULL, "can't create L2 decap action"); 1628 return 0; 1629 } 1630 1631 /** 1632 * Convert raw decap/encap (L3 tunnel) action to DV specification. 1633 * 1634 * @param[in] dev 1635 * Pointer to rte_eth_dev structure. 1636 * @param[in] action 1637 * Pointer to action structure. 1638 * @param[in, out] dev_flow 1639 * Pointer to the mlx5_flow. 1640 * @param[in] attr 1641 * Pointer to the flow attributes. 1642 * @param[out] error 1643 * Pointer to the error structure. 1644 * 1645 * @return 1646 * 0 on success, a negative errno value otherwise and rte_errno is set. 1647 */ 1648 static int 1649 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 1650 const struct rte_flow_action *action, 1651 struct mlx5_flow *dev_flow, 1652 const struct rte_flow_attr *attr, 1653 struct rte_flow_error *error) 1654 { 1655 const struct rte_flow_action_raw_encap *encap_data; 1656 struct mlx5_flow_dv_encap_decap_resource res; 1657 1658 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 1659 res.size = encap_data->size; 1660 memcpy(res.buf, encap_data->data, res.size); 1661 res.reformat_type = attr->egress ? 1662 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : 1663 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 1664 if (attr->transfer) 1665 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 1666 else 1667 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 1668 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 1669 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 1670 return rte_flow_error_set(error, EINVAL, 1671 RTE_FLOW_ERROR_TYPE_ACTION, 1672 NULL, "can't create encap action"); 1673 return 0; 1674 } 1675 1676 /** 1677 * Validate the modify-header actions. 1678 * 1679 * @param[in] action_flags 1680 * Holds the actions detected until now. 1681 * @param[in] action 1682 * Pointer to the modify action. 1683 * @param[out] error 1684 * Pointer to error structure. 1685 * 1686 * @return 1687 * 0 on success, a negative errno value otherwise and rte_errno is set. 1688 */ 1689 static int 1690 flow_dv_validate_action_modify_hdr(const uint64_t action_flags, 1691 const struct rte_flow_action *action, 1692 struct rte_flow_error *error) 1693 { 1694 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf) 1695 return rte_flow_error_set(error, EINVAL, 1696 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1697 NULL, "action configuration not set"); 1698 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) 1699 return rte_flow_error_set(error, EINVAL, 1700 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1701 "can't have encap action before" 1702 " modify action"); 1703 return 0; 1704 } 1705 1706 /** 1707 * Validate the modify-header MAC address actions. 1708 * 1709 * @param[in] action_flags 1710 * Holds the actions detected until now. 1711 * @param[in] action 1712 * Pointer to the modify action. 1713 * @param[in] item_flags 1714 * Holds the items detected. 1715 * @param[out] error 1716 * Pointer to error structure. 1717 * 1718 * @return 1719 * 0 on success, a negative errno value otherwise and rte_errno is set. 1720 */ 1721 static int 1722 flow_dv_validate_action_modify_mac(const uint64_t action_flags, 1723 const struct rte_flow_action *action, 1724 const uint64_t item_flags, 1725 struct rte_flow_error *error) 1726 { 1727 int ret = 0; 1728 1729 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1730 if (!ret) { 1731 if (!(item_flags & MLX5_FLOW_LAYER_L2)) 1732 return rte_flow_error_set(error, EINVAL, 1733 RTE_FLOW_ERROR_TYPE_ACTION, 1734 NULL, 1735 "no L2 item in pattern"); 1736 } 1737 return ret; 1738 } 1739 1740 /** 1741 * Validate the modify-header IPv4 address actions. 1742 * 1743 * @param[in] action_flags 1744 * Holds the actions detected until now. 1745 * @param[in] action 1746 * Pointer to the modify action. 1747 * @param[in] item_flags 1748 * Holds the items detected. 1749 * @param[out] error 1750 * Pointer to error structure. 1751 * 1752 * @return 1753 * 0 on success, a negative errno value otherwise and rte_errno is set. 1754 */ 1755 static int 1756 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags, 1757 const struct rte_flow_action *action, 1758 const uint64_t item_flags, 1759 struct rte_flow_error *error) 1760 { 1761 int ret = 0; 1762 1763 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1764 if (!ret) { 1765 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4)) 1766 return rte_flow_error_set(error, EINVAL, 1767 RTE_FLOW_ERROR_TYPE_ACTION, 1768 NULL, 1769 "no ipv4 item in pattern"); 1770 } 1771 return ret; 1772 } 1773 1774 /** 1775 * Validate the modify-header IPv6 address actions. 1776 * 1777 * @param[in] action_flags 1778 * Holds the actions detected until now. 1779 * @param[in] action 1780 * Pointer to the modify action. 1781 * @param[in] item_flags 1782 * Holds the items detected. 1783 * @param[out] error 1784 * Pointer to error structure. 1785 * 1786 * @return 1787 * 0 on success, a negative errno value otherwise and rte_errno is set. 1788 */ 1789 static int 1790 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags, 1791 const struct rte_flow_action *action, 1792 const uint64_t item_flags, 1793 struct rte_flow_error *error) 1794 { 1795 int ret = 0; 1796 1797 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1798 if (!ret) { 1799 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6)) 1800 return rte_flow_error_set(error, EINVAL, 1801 RTE_FLOW_ERROR_TYPE_ACTION, 1802 NULL, 1803 "no ipv6 item in pattern"); 1804 } 1805 return ret; 1806 } 1807 1808 /** 1809 * Validate the modify-header TP actions. 1810 * 1811 * @param[in] action_flags 1812 * Holds the actions detected until now. 1813 * @param[in] action 1814 * Pointer to the modify action. 1815 * @param[in] item_flags 1816 * Holds the items detected. 1817 * @param[out] error 1818 * Pointer to error structure. 1819 * 1820 * @return 1821 * 0 on success, a negative errno value otherwise and rte_errno is set. 1822 */ 1823 static int 1824 flow_dv_validate_action_modify_tp(const uint64_t action_flags, 1825 const struct rte_flow_action *action, 1826 const uint64_t item_flags, 1827 struct rte_flow_error *error) 1828 { 1829 int ret = 0; 1830 1831 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1832 if (!ret) { 1833 if (!(item_flags & MLX5_FLOW_LAYER_L4)) 1834 return rte_flow_error_set(error, EINVAL, 1835 RTE_FLOW_ERROR_TYPE_ACTION, 1836 NULL, "no transport layer " 1837 "in pattern"); 1838 } 1839 return ret; 1840 } 1841 1842 /** 1843 * Validate the modify-header actions of increment/decrement 1844 * TCP Sequence-number. 1845 * 1846 * @param[in] action_flags 1847 * Holds the actions detected until now. 1848 * @param[in] action 1849 * Pointer to the modify action. 1850 * @param[in] item_flags 1851 * Holds the items detected. 1852 * @param[out] error 1853 * Pointer to error structure. 1854 * 1855 * @return 1856 * 0 on success, a negative errno value otherwise and rte_errno is set. 1857 */ 1858 static int 1859 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags, 1860 const struct rte_flow_action *action, 1861 const uint64_t item_flags, 1862 struct rte_flow_error *error) 1863 { 1864 int ret = 0; 1865 1866 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1867 if (!ret) { 1868 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) 1869 return rte_flow_error_set(error, EINVAL, 1870 RTE_FLOW_ERROR_TYPE_ACTION, 1871 NULL, "no TCP item in" 1872 " pattern"); 1873 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ && 1874 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) || 1875 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ && 1876 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ))) 1877 return rte_flow_error_set(error, EINVAL, 1878 RTE_FLOW_ERROR_TYPE_ACTION, 1879 NULL, 1880 "cannot decrease and increase" 1881 " TCP sequence number" 1882 " at the same time"); 1883 } 1884 return ret; 1885 } 1886 1887 /** 1888 * Validate the modify-header actions of increment/decrement 1889 * TCP Acknowledgment number. 1890 * 1891 * @param[in] action_flags 1892 * Holds the actions detected until now. 1893 * @param[in] action 1894 * Pointer to the modify action. 1895 * @param[in] item_flags 1896 * Holds the items detected. 1897 * @param[out] error 1898 * Pointer to error structure. 1899 * 1900 * @return 1901 * 0 on success, a negative errno value otherwise and rte_errno is set. 1902 */ 1903 static int 1904 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags, 1905 const struct rte_flow_action *action, 1906 const uint64_t item_flags, 1907 struct rte_flow_error *error) 1908 { 1909 int ret = 0; 1910 1911 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1912 if (!ret) { 1913 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)) 1914 return rte_flow_error_set(error, EINVAL, 1915 RTE_FLOW_ERROR_TYPE_ACTION, 1916 NULL, "no TCP item in" 1917 " pattern"); 1918 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK && 1919 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) || 1920 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK && 1921 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK))) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ACTION, 1924 NULL, 1925 "cannot decrease and increase" 1926 " TCP acknowledgment number" 1927 " at the same time"); 1928 } 1929 return ret; 1930 } 1931 1932 /** 1933 * Validate the modify-header TTL actions. 1934 * 1935 * @param[in] action_flags 1936 * Holds the actions detected until now. 1937 * @param[in] action 1938 * Pointer to the modify action. 1939 * @param[in] item_flags 1940 * Holds the items detected. 1941 * @param[out] error 1942 * Pointer to error structure. 1943 * 1944 * @return 1945 * 0 on success, a negative errno value otherwise and rte_errno is set. 1946 */ 1947 static int 1948 flow_dv_validate_action_modify_ttl(const uint64_t action_flags, 1949 const struct rte_flow_action *action, 1950 const uint64_t item_flags, 1951 struct rte_flow_error *error) 1952 { 1953 int ret = 0; 1954 1955 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error); 1956 if (!ret) { 1957 if (!(item_flags & MLX5_FLOW_LAYER_L3)) 1958 return rte_flow_error_set(error, EINVAL, 1959 RTE_FLOW_ERROR_TYPE_ACTION, 1960 NULL, 1961 "no IP protocol in pattern"); 1962 } 1963 return ret; 1964 } 1965 1966 /** 1967 * Validate jump action. 1968 * 1969 * @param[in] action 1970 * Pointer to the modify action. 1971 * @param[in] group 1972 * The group of the current flow. 1973 * @param[out] error 1974 * Pointer to error structure. 1975 * 1976 * @return 1977 * 0 on success, a negative errno value otherwise and rte_errno is set. 1978 */ 1979 static int 1980 flow_dv_validate_action_jump(const struct rte_flow_action *action, 1981 uint32_t group, 1982 struct rte_flow_error *error) 1983 { 1984 if (action->type != RTE_FLOW_ACTION_TYPE_JUMP && !action->conf) 1985 return rte_flow_error_set(error, EINVAL, 1986 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1987 NULL, "action configuration not set"); 1988 if (group >= ((const struct rte_flow_action_jump *)action->conf)->group) 1989 return rte_flow_error_set(error, EINVAL, 1990 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1991 "target group must be higher then" 1992 " the current flow group"); 1993 return 0; 1994 } 1995 1996 /* 1997 * Validate the port_id action. 1998 * 1999 * @param[in] dev 2000 * Pointer to rte_eth_dev structure. 2001 * @param[in] action_flags 2002 * Bit-fields that holds the actions detected until now. 2003 * @param[in] action 2004 * Port_id RTE action structure. 2005 * @param[in] attr 2006 * Attributes of flow that includes this action. 2007 * @param[out] error 2008 * Pointer to error structure. 2009 * 2010 * @return 2011 * 0 on success, a negative errno value otherwise and rte_errno is set. 2012 */ 2013 static int 2014 flow_dv_validate_action_port_id(struct rte_eth_dev *dev, 2015 uint64_t action_flags, 2016 const struct rte_flow_action *action, 2017 const struct rte_flow_attr *attr, 2018 struct rte_flow_error *error) 2019 { 2020 const struct rte_flow_action_port_id *port_id; 2021 uint16_t port; 2022 uint16_t esw_domain_id; 2023 uint16_t act_port_domain_id; 2024 int ret; 2025 2026 if (!attr->transfer) 2027 return rte_flow_error_set(error, ENOTSUP, 2028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2029 NULL, 2030 "port id action is valid in transfer" 2031 " mode only"); 2032 if (!action || !action->conf) 2033 return rte_flow_error_set(error, ENOTSUP, 2034 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2035 NULL, 2036 "port id action parameters must be" 2037 " specified"); 2038 if (action_flags & (MLX5_FLOW_FATE_ACTIONS | 2039 MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 2040 return rte_flow_error_set(error, EINVAL, 2041 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2042 "can have only one fate actions in" 2043 " a flow"); 2044 ret = mlx5_port_to_eswitch_info(dev->data->port_id, 2045 &esw_domain_id, NULL); 2046 if (ret < 0) 2047 return rte_flow_error_set(error, -ret, 2048 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2049 NULL, 2050 "failed to obtain E-Switch info"); 2051 port_id = action->conf; 2052 port = port_id->original ? dev->data->port_id : port_id->id; 2053 ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL); 2054 if (ret) 2055 return rte_flow_error_set 2056 (error, -ret, 2057 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id, 2058 "failed to obtain E-Switch port id for port"); 2059 if (act_port_domain_id != esw_domain_id) 2060 return rte_flow_error_set 2061 (error, -ret, 2062 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2063 "port does not belong to" 2064 " E-Switch being configured"); 2065 return 0; 2066 } 2067 2068 /** 2069 * Find existing modify-header resource or create and register a new one. 2070 * 2071 * @param dev[in, out] 2072 * Pointer to rte_eth_dev structure. 2073 * @param[in, out] resource 2074 * Pointer to modify-header resource. 2075 * @parm[in, out] dev_flow 2076 * Pointer to the dev_flow. 2077 * @param[out] error 2078 * pointer to error structure. 2079 * 2080 * @return 2081 * 0 on success otherwise -errno and errno is set. 2082 */ 2083 static int 2084 flow_dv_modify_hdr_resource_register 2085 (struct rte_eth_dev *dev, 2086 struct mlx5_flow_dv_modify_hdr_resource *resource, 2087 struct mlx5_flow *dev_flow, 2088 struct rte_flow_error *error) 2089 { 2090 struct mlx5_priv *priv = dev->data->dev_private; 2091 struct mlx5_ibv_shared *sh = priv->sh; 2092 struct mlx5_flow_dv_modify_hdr_resource *cache_resource; 2093 struct mlx5dv_dr_domain *ns; 2094 2095 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) 2096 ns = sh->fdb_domain; 2097 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX) 2098 ns = sh->tx_domain; 2099 else 2100 ns = sh->rx_domain; 2101 resource->flags = 2102 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL; 2103 /* Lookup a matching resource from cache. */ 2104 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) { 2105 if (resource->ft_type == cache_resource->ft_type && 2106 resource->actions_num == cache_resource->actions_num && 2107 resource->flags == cache_resource->flags && 2108 !memcmp((const void *)resource->actions, 2109 (const void *)cache_resource->actions, 2110 (resource->actions_num * 2111 sizeof(resource->actions[0])))) { 2112 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++", 2113 (void *)cache_resource, 2114 rte_atomic32_read(&cache_resource->refcnt)); 2115 rte_atomic32_inc(&cache_resource->refcnt); 2116 dev_flow->dv.modify_hdr = cache_resource; 2117 return 0; 2118 } 2119 } 2120 /* Register new modify-header resource. */ 2121 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 2122 if (!cache_resource) 2123 return rte_flow_error_set(error, ENOMEM, 2124 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2125 "cannot allocate resource memory"); 2126 *cache_resource = *resource; 2127 cache_resource->verbs_action = 2128 mlx5_glue->dv_create_flow_action_modify_header 2129 (sh->ctx, cache_resource->ft_type, 2130 ns, cache_resource->flags, 2131 cache_resource->actions_num * 2132 sizeof(cache_resource->actions[0]), 2133 (uint64_t *)cache_resource->actions); 2134 if (!cache_resource->verbs_action) { 2135 rte_free(cache_resource); 2136 return rte_flow_error_set(error, ENOMEM, 2137 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2138 NULL, "cannot create action"); 2139 } 2140 rte_atomic32_init(&cache_resource->refcnt); 2141 rte_atomic32_inc(&cache_resource->refcnt); 2142 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next); 2143 dev_flow->dv.modify_hdr = cache_resource; 2144 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++", 2145 (void *)cache_resource, 2146 rte_atomic32_read(&cache_resource->refcnt)); 2147 return 0; 2148 } 2149 2150 #define MLX5_CNT_CONTAINER_RESIZE 64 2151 2152 /** 2153 * Get or create a flow counter. 2154 * 2155 * @param[in] dev 2156 * Pointer to the Ethernet device structure. 2157 * @param[in] shared 2158 * Indicate if this counter is shared with other flows. 2159 * @param[in] id 2160 * Counter identifier. 2161 * 2162 * @return 2163 * pointer to flow counter on success, NULL otherwise and rte_errno is set. 2164 */ 2165 static struct mlx5_flow_counter * 2166 flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared, 2167 uint32_t id) 2168 { 2169 struct mlx5_priv *priv = dev->data->dev_private; 2170 struct mlx5_flow_counter *cnt = NULL; 2171 struct mlx5_devx_obj *dcs = NULL; 2172 2173 if (!priv->config.devx) { 2174 rte_errno = ENOTSUP; 2175 return NULL; 2176 } 2177 if (shared) { 2178 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) { 2179 if (cnt->shared && cnt->id == id) { 2180 cnt->ref_cnt++; 2181 return cnt; 2182 } 2183 } 2184 } 2185 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); 2186 if (!dcs) 2187 return NULL; 2188 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0); 2189 if (!cnt) { 2190 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); 2191 rte_errno = ENOMEM; 2192 return NULL; 2193 } 2194 struct mlx5_flow_counter tmpl = { 2195 .shared = shared, 2196 .ref_cnt = 1, 2197 .id = id, 2198 .dcs = dcs, 2199 }; 2200 tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0); 2201 if (!tmpl.action) { 2202 claim_zero(mlx5_devx_cmd_destroy(cnt->dcs)); 2203 rte_errno = errno; 2204 rte_free(cnt); 2205 return NULL; 2206 } 2207 *cnt = tmpl; 2208 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next); 2209 return cnt; 2210 } 2211 2212 /** 2213 * Release a flow counter. 2214 * 2215 * @param[in] dev 2216 * Pointer to the Ethernet device structure. 2217 * @param[in] counter 2218 * Pointer to the counter handler. 2219 */ 2220 static void 2221 flow_dv_counter_release_fallback(struct rte_eth_dev *dev, 2222 struct mlx5_flow_counter *counter) 2223 { 2224 struct mlx5_priv *priv = dev->data->dev_private; 2225 2226 if (!counter) 2227 return; 2228 if (--counter->ref_cnt == 0) { 2229 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next); 2230 claim_zero(mlx5_devx_cmd_destroy(counter->dcs)); 2231 rte_free(counter); 2232 } 2233 } 2234 2235 /** 2236 * Query a devx flow counter. 2237 * 2238 * @param[in] dev 2239 * Pointer to the Ethernet device structure. 2240 * @param[in] cnt 2241 * Pointer to the flow counter. 2242 * @param[out] pkts 2243 * The statistics value of packets. 2244 * @param[out] bytes 2245 * The statistics value of bytes. 2246 * 2247 * @return 2248 * 0 on success, otherwise a negative errno value and rte_errno is set. 2249 */ 2250 static inline int 2251 _flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused, 2252 struct mlx5_flow_counter *cnt, uint64_t *pkts, 2253 uint64_t *bytes) 2254 { 2255 return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes, 2256 0, NULL, NULL, 0); 2257 } 2258 2259 /** 2260 * Get a pool by a counter. 2261 * 2262 * @param[in] cnt 2263 * Pointer to the counter. 2264 * 2265 * @return 2266 * The counter pool. 2267 */ 2268 static struct mlx5_flow_counter_pool * 2269 flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt) 2270 { 2271 if (!cnt->batch) { 2272 cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL; 2273 return (struct mlx5_flow_counter_pool *)cnt - 1; 2274 } 2275 return cnt->pool; 2276 } 2277 2278 /** 2279 * Get a pool by devx counter ID. 2280 * 2281 * @param[in] cont 2282 * Pointer to the counter container. 2283 * @param[in] id 2284 * The counter devx ID. 2285 * 2286 * @return 2287 * The counter pool pointer if exists, NULL otherwise, 2288 */ 2289 static struct mlx5_flow_counter_pool * 2290 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id) 2291 { 2292 struct mlx5_flow_counter_pool *pool; 2293 2294 TAILQ_FOREACH(pool, &cont->pool_list, next) { 2295 int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) * 2296 MLX5_COUNTERS_PER_POOL; 2297 2298 if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) 2299 return pool; 2300 }; 2301 return NULL; 2302 } 2303 2304 /** 2305 * Allocate a new memory for the counter values wrapped by all the needed 2306 * management. 2307 * 2308 * @param[in] dev 2309 * Pointer to the Ethernet device structure. 2310 * @param[in] raws_n 2311 * The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters. 2312 * 2313 * @return 2314 * The new memory management pointer on success, otherwise NULL and rte_errno 2315 * is set. 2316 */ 2317 static struct mlx5_counter_stats_mem_mng * 2318 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n) 2319 { 2320 struct mlx5_ibv_shared *sh = ((struct mlx5_priv *) 2321 (dev->data->dev_private))->sh; 2322 struct mlx5_devx_mkey_attr mkey_attr; 2323 struct mlx5_counter_stats_mem_mng *mem_mng; 2324 volatile struct flow_counter_stats *raw_data; 2325 int size = (sizeof(struct flow_counter_stats) * 2326 MLX5_COUNTERS_PER_POOL + 2327 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 2328 sizeof(struct mlx5_counter_stats_mem_mng); 2329 uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE)); 2330 int i; 2331 2332 if (!mem) { 2333 rte_errno = ENOMEM; 2334 return NULL; 2335 } 2336 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 2337 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 2338 mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size, 2339 IBV_ACCESS_LOCAL_WRITE); 2340 if (!mem_mng->umem) { 2341 rte_errno = errno; 2342 rte_free(mem); 2343 return NULL; 2344 } 2345 mkey_attr.addr = (uintptr_t)mem; 2346 mkey_attr.size = size; 2347 mkey_attr.umem_id = mem_mng->umem->umem_id; 2348 mkey_attr.pd = sh->pdn; 2349 mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); 2350 if (!mem_mng->dm) { 2351 mlx5_glue->devx_umem_dereg(mem_mng->umem); 2352 rte_errno = errno; 2353 rte_free(mem); 2354 return NULL; 2355 } 2356 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 2357 raw_data = (volatile struct flow_counter_stats *)mem; 2358 for (i = 0; i < raws_n; ++i) { 2359 mem_mng->raws[i].mem_mng = mem_mng; 2360 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 2361 } 2362 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 2363 return mem_mng; 2364 } 2365 2366 /** 2367 * Resize a counter container. 2368 * 2369 * @param[in] dev 2370 * Pointer to the Ethernet device structure. 2371 * @param[in] batch 2372 * Whether the pool is for counter that was allocated by batch command. 2373 * 2374 * @return 2375 * The new container pointer on success, otherwise NULL and rte_errno is set. 2376 */ 2377 static struct mlx5_pools_container * 2378 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch) 2379 { 2380 struct mlx5_priv *priv = dev->data->dev_private; 2381 struct mlx5_pools_container *cont = 2382 MLX5_CNT_CONTAINER(priv->sh, batch, 0); 2383 struct mlx5_pools_container *new_cont = 2384 MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0); 2385 struct mlx5_counter_stats_mem_mng *mem_mng; 2386 uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE; 2387 uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize; 2388 int i; 2389 2390 if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) { 2391 /* The last resize still hasn't detected by the host thread. */ 2392 rte_errno = EAGAIN; 2393 return NULL; 2394 } 2395 new_cont->pools = rte_calloc(__func__, 1, mem_size, 0); 2396 if (!new_cont->pools) { 2397 rte_errno = ENOMEM; 2398 return NULL; 2399 } 2400 if (cont->n) 2401 memcpy(new_cont->pools, cont->pools, cont->n * 2402 sizeof(struct mlx5_flow_counter_pool *)); 2403 mem_mng = flow_dv_create_counter_stat_mem_mng(dev, 2404 MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES); 2405 if (!mem_mng) { 2406 rte_free(new_cont->pools); 2407 return NULL; 2408 } 2409 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 2410 LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws, 2411 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + 2412 i, next); 2413 new_cont->n = resize; 2414 rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid)); 2415 TAILQ_INIT(&new_cont->pool_list); 2416 TAILQ_CONCAT(&new_cont->pool_list, &cont->pool_list, next); 2417 new_cont->init_mem_mng = mem_mng; 2418 rte_cio_wmb(); 2419 /* Flip the master container. */ 2420 priv->sh->cmng.mhi[batch] ^= (uint8_t)1; 2421 return new_cont; 2422 } 2423 2424 /** 2425 * Query a devx flow counter. 2426 * 2427 * @param[in] dev 2428 * Pointer to the Ethernet device structure. 2429 * @param[in] cnt 2430 * Pointer to the flow counter. 2431 * @param[out] pkts 2432 * The statistics value of packets. 2433 * @param[out] bytes 2434 * The statistics value of bytes. 2435 * 2436 * @return 2437 * 0 on success, otherwise a negative errno value and rte_errno is set. 2438 */ 2439 static inline int 2440 _flow_dv_query_count(struct rte_eth_dev *dev, 2441 struct mlx5_flow_counter *cnt, uint64_t *pkts, 2442 uint64_t *bytes) 2443 { 2444 struct mlx5_priv *priv = dev->data->dev_private; 2445 struct mlx5_flow_counter_pool *pool = 2446 flow_dv_counter_pool_get(cnt); 2447 int offset = cnt - &pool->counters_raw[0]; 2448 2449 if (priv->counter_fallback) 2450 return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes); 2451 2452 rte_spinlock_lock(&pool->sl); 2453 /* 2454 * The single counters allocation may allocate smaller ID than the 2455 * current allocated in parallel to the host reading. 2456 * In this case the new counter values must be reported as 0. 2457 */ 2458 if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) { 2459 *pkts = 0; 2460 *bytes = 0; 2461 } else { 2462 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits); 2463 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes); 2464 } 2465 rte_spinlock_unlock(&pool->sl); 2466 return 0; 2467 } 2468 2469 /** 2470 * Create and initialize a new counter pool. 2471 * 2472 * @param[in] dev 2473 * Pointer to the Ethernet device structure. 2474 * @param[out] dcs 2475 * The devX counter handle. 2476 * @param[in] batch 2477 * Whether the pool is for counter that was allocated by batch command. 2478 * 2479 * @return 2480 * A new pool pointer on success, NULL otherwise and rte_errno is set. 2481 */ 2482 static struct mlx5_flow_counter_pool * 2483 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs, 2484 uint32_t batch) 2485 { 2486 struct mlx5_priv *priv = dev->data->dev_private; 2487 struct mlx5_flow_counter_pool *pool; 2488 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 2489 0); 2490 int16_t n_valid = rte_atomic16_read(&cont->n_valid); 2491 uint32_t size; 2492 2493 if (cont->n == n_valid) { 2494 cont = flow_dv_container_resize(dev, batch); 2495 if (!cont) 2496 return NULL; 2497 } 2498 size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL * 2499 sizeof(struct mlx5_flow_counter); 2500 pool = rte_calloc(__func__, 1, size, 0); 2501 if (!pool) { 2502 rte_errno = ENOMEM; 2503 return NULL; 2504 } 2505 pool->min_dcs = dcs; 2506 pool->raw = cont->init_mem_mng->raws + n_valid % 2507 MLX5_CNT_CONTAINER_RESIZE; 2508 pool->raw_hw = NULL; 2509 rte_spinlock_init(&pool->sl); 2510 /* 2511 * The generation of the new allocated counters in this pool is 0, 2 in 2512 * the pool generation makes all the counters valid for allocation. 2513 */ 2514 rte_atomic64_set(&pool->query_gen, 0x2); 2515 TAILQ_INIT(&pool->counters); 2516 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); 2517 cont->pools[n_valid] = pool; 2518 /* Pool initialization must be updated before host thread access. */ 2519 rte_cio_wmb(); 2520 rte_atomic16_add(&cont->n_valid, 1); 2521 return pool; 2522 } 2523 2524 /** 2525 * Prepare a new counter and/or a new counter pool. 2526 * 2527 * @param[in] dev 2528 * Pointer to the Ethernet device structure. 2529 * @param[out] cnt_free 2530 * Where to put the pointer of a new counter. 2531 * @param[in] batch 2532 * Whether the pool is for counter that was allocated by batch command. 2533 * 2534 * @return 2535 * The free counter pool pointer and @p cnt_free is set on success, 2536 * NULL otherwise and rte_errno is set. 2537 */ 2538 static struct mlx5_flow_counter_pool * 2539 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev, 2540 struct mlx5_flow_counter **cnt_free, 2541 uint32_t batch) 2542 { 2543 struct mlx5_priv *priv = dev->data->dev_private; 2544 struct mlx5_flow_counter_pool *pool; 2545 struct mlx5_devx_obj *dcs = NULL; 2546 struct mlx5_flow_counter *cnt; 2547 uint32_t i; 2548 2549 if (!batch) { 2550 /* bulk_bitmap must be 0 for single counter allocation. */ 2551 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0); 2552 if (!dcs) 2553 return NULL; 2554 pool = flow_dv_find_pool_by_id 2555 (MLX5_CNT_CONTAINER(priv->sh, batch, 0), dcs->id); 2556 if (!pool) { 2557 pool = flow_dv_pool_create(dev, dcs, batch); 2558 if (!pool) { 2559 mlx5_devx_cmd_destroy(dcs); 2560 return NULL; 2561 } 2562 } else if (dcs->id < pool->min_dcs->id) { 2563 rte_atomic64_set(&pool->a64_dcs, 2564 (int64_t)(uintptr_t)dcs); 2565 } 2566 cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL]; 2567 TAILQ_INSERT_HEAD(&pool->counters, cnt, next); 2568 cnt->dcs = dcs; 2569 *cnt_free = cnt; 2570 return pool; 2571 } 2572 /* bulk_bitmap is in 128 counters units. */ 2573 if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4) 2574 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); 2575 if (!dcs) { 2576 rte_errno = ENODATA; 2577 return NULL; 2578 } 2579 pool = flow_dv_pool_create(dev, dcs, batch); 2580 if (!pool) { 2581 mlx5_devx_cmd_destroy(dcs); 2582 return NULL; 2583 } 2584 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 2585 cnt = &pool->counters_raw[i]; 2586 cnt->pool = pool; 2587 TAILQ_INSERT_HEAD(&pool->counters, cnt, next); 2588 } 2589 *cnt_free = &pool->counters_raw[0]; 2590 return pool; 2591 } 2592 2593 /** 2594 * Search for existed shared counter. 2595 * 2596 * @param[in] cont 2597 * Pointer to the relevant counter pool container. 2598 * @param[in] id 2599 * The shared counter ID to search. 2600 * 2601 * @return 2602 * NULL if not existed, otherwise pointer to the shared counter. 2603 */ 2604 static struct mlx5_flow_counter * 2605 flow_dv_counter_shared_search(struct mlx5_pools_container *cont, 2606 uint32_t id) 2607 { 2608 static struct mlx5_flow_counter *cnt; 2609 struct mlx5_flow_counter_pool *pool; 2610 int i; 2611 2612 TAILQ_FOREACH(pool, &cont->pool_list, next) { 2613 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 2614 cnt = &pool->counters_raw[i]; 2615 if (cnt->ref_cnt && cnt->shared && cnt->id == id) 2616 return cnt; 2617 } 2618 } 2619 return NULL; 2620 } 2621 2622 /** 2623 * Allocate a flow counter. 2624 * 2625 * @param[in] dev 2626 * Pointer to the Ethernet device structure. 2627 * @param[in] shared 2628 * Indicate if this counter is shared with other flows. 2629 * @param[in] id 2630 * Counter identifier. 2631 * @param[in] group 2632 * Counter flow group. 2633 * 2634 * @return 2635 * pointer to flow counter on success, NULL otherwise and rte_errno is set. 2636 */ 2637 static struct mlx5_flow_counter * 2638 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id, 2639 uint16_t group) 2640 { 2641 struct mlx5_priv *priv = dev->data->dev_private; 2642 struct mlx5_flow_counter_pool *pool = NULL; 2643 struct mlx5_flow_counter *cnt_free = NULL; 2644 /* 2645 * Currently group 0 flow counter cannot be assigned to a flow if it is 2646 * not the first one in the batch counter allocation, so it is better 2647 * to allocate counters one by one for these flows in a separate 2648 * container. 2649 * A counter can be shared between different groups so need to take 2650 * shared counters from the single container. 2651 */ 2652 uint32_t batch = (group && !shared) ? 1 : 0; 2653 struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch, 2654 0); 2655 2656 if (priv->counter_fallback) 2657 return flow_dv_counter_alloc_fallback(dev, shared, id); 2658 if (!priv->config.devx) { 2659 rte_errno = ENOTSUP; 2660 return NULL; 2661 } 2662 if (shared) { 2663 cnt_free = flow_dv_counter_shared_search(cont, id); 2664 if (cnt_free) { 2665 if (cnt_free->ref_cnt + 1 == 0) { 2666 rte_errno = E2BIG; 2667 return NULL; 2668 } 2669 cnt_free->ref_cnt++; 2670 return cnt_free; 2671 } 2672 } 2673 /* Pools which has a free counters are in the start. */ 2674 TAILQ_FOREACH(pool, &cont->pool_list, next) { 2675 /* 2676 * The free counter reset values must be updated between the 2677 * counter release to the counter allocation, so, at least one 2678 * query must be done in this time. ensure it by saving the 2679 * query generation in the release time. 2680 * The free list is sorted according to the generation - so if 2681 * the first one is not updated, all the others are not 2682 * updated too. 2683 */ 2684 cnt_free = TAILQ_FIRST(&pool->counters); 2685 if (cnt_free && cnt_free->query_gen + 1 < 2686 rte_atomic64_read(&pool->query_gen)) 2687 break; 2688 cnt_free = NULL; 2689 } 2690 if (!cnt_free) { 2691 pool = flow_dv_counter_pool_prepare(dev, &cnt_free, batch); 2692 if (!pool) 2693 return NULL; 2694 } 2695 cnt_free->batch = batch; 2696 /* Create a DV counter action only in the first time usage. */ 2697 if (!cnt_free->action) { 2698 uint16_t offset; 2699 struct mlx5_devx_obj *dcs; 2700 2701 if (batch) { 2702 offset = cnt_free - &pool->counters_raw[0]; 2703 dcs = pool->min_dcs; 2704 } else { 2705 offset = 0; 2706 dcs = cnt_free->dcs; 2707 } 2708 cnt_free->action = mlx5_glue->dv_create_flow_action_counter 2709 (dcs->obj, offset); 2710 if (!cnt_free->action) { 2711 rte_errno = errno; 2712 return NULL; 2713 } 2714 } 2715 /* Update the counter reset values. */ 2716 if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits, 2717 &cnt_free->bytes)) 2718 return NULL; 2719 cnt_free->shared = shared; 2720 cnt_free->ref_cnt = 1; 2721 cnt_free->id = id; 2722 if (!priv->sh->cmng.query_thread_on) 2723 /* Start the asynchronous batch query by the host thread. */ 2724 mlx5_set_query_alarm(priv->sh); 2725 TAILQ_REMOVE(&pool->counters, cnt_free, next); 2726 if (TAILQ_EMPTY(&pool->counters)) { 2727 /* Move the pool to the end of the container pool list. */ 2728 TAILQ_REMOVE(&cont->pool_list, pool, next); 2729 TAILQ_INSERT_TAIL(&cont->pool_list, pool, next); 2730 } 2731 return cnt_free; 2732 } 2733 2734 /** 2735 * Release a flow counter. 2736 * 2737 * @param[in] dev 2738 * Pointer to the Ethernet device structure. 2739 * @param[in] counter 2740 * Pointer to the counter handler. 2741 */ 2742 static void 2743 flow_dv_counter_release(struct rte_eth_dev *dev, 2744 struct mlx5_flow_counter *counter) 2745 { 2746 struct mlx5_priv *priv = dev->data->dev_private; 2747 2748 if (!counter) 2749 return; 2750 if (priv->counter_fallback) { 2751 flow_dv_counter_release_fallback(dev, counter); 2752 return; 2753 } 2754 if (--counter->ref_cnt == 0) { 2755 struct mlx5_flow_counter_pool *pool = 2756 flow_dv_counter_pool_get(counter); 2757 2758 /* Put the counter in the end - the last updated one. */ 2759 TAILQ_INSERT_TAIL(&pool->counters, counter, next); 2760 counter->query_gen = rte_atomic64_read(&pool->query_gen); 2761 } 2762 } 2763 2764 /** 2765 * Verify the @p attributes will be correctly understood by the NIC and store 2766 * them in the @p flow if everything is correct. 2767 * 2768 * @param[in] dev 2769 * Pointer to dev struct. 2770 * @param[in] attributes 2771 * Pointer to flow attributes 2772 * @param[out] error 2773 * Pointer to error structure. 2774 * 2775 * @return 2776 * 0 on success, a negative errno value otherwise and rte_errno is set. 2777 */ 2778 static int 2779 flow_dv_validate_attributes(struct rte_eth_dev *dev, 2780 const struct rte_flow_attr *attributes, 2781 struct rte_flow_error *error) 2782 { 2783 struct mlx5_priv *priv = dev->data->dev_private; 2784 uint32_t priority_max = priv->config.flow_prio - 1; 2785 2786 #ifndef HAVE_MLX5DV_DR 2787 if (attributes->group) 2788 return rte_flow_error_set(error, ENOTSUP, 2789 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2790 NULL, 2791 "groups is not supported"); 2792 #endif 2793 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 2794 attributes->priority >= priority_max) 2795 return rte_flow_error_set(error, ENOTSUP, 2796 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2797 NULL, 2798 "priority out of range"); 2799 if (attributes->transfer) { 2800 if (!priv->config.dv_esw_en) 2801 return rte_flow_error_set 2802 (error, ENOTSUP, 2803 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2804 "E-Switch dr is not supported"); 2805 if (!(priv->representor || priv->master)) 2806 return rte_flow_error_set 2807 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2808 NULL, "E-Switch configurationd can only be" 2809 " done by a master or a representor device"); 2810 if (attributes->egress) 2811 return rte_flow_error_set 2812 (error, ENOTSUP, 2813 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes, 2814 "egress is not supported"); 2815 if (attributes->group >= MLX5_MAX_TABLES_FDB) 2816 return rte_flow_error_set 2817 (error, EINVAL, 2818 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2819 NULL, "group must be smaller than " 2820 RTE_STR(MLX5_MAX_FDB_TABLES)); 2821 } 2822 if (!(attributes->egress ^ attributes->ingress)) 2823 return rte_flow_error_set(error, ENOTSUP, 2824 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2825 "must specify exactly one of " 2826 "ingress or egress"); 2827 return 0; 2828 } 2829 2830 /** 2831 * Internal validation function. For validating both actions and items. 2832 * 2833 * @param[in] dev 2834 * Pointer to the rte_eth_dev structure. 2835 * @param[in] attr 2836 * Pointer to the flow attributes. 2837 * @param[in] items 2838 * Pointer to the list of items. 2839 * @param[in] actions 2840 * Pointer to the list of actions. 2841 * @param[out] error 2842 * Pointer to the error structure. 2843 * 2844 * @return 2845 * 0 on success, a negative errno value otherwise and rte_errno is set. 2846 */ 2847 static int 2848 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 2849 const struct rte_flow_item items[], 2850 const struct rte_flow_action actions[], 2851 struct rte_flow_error *error) 2852 { 2853 int ret; 2854 uint64_t action_flags = 0; 2855 uint64_t item_flags = 0; 2856 uint64_t last_item = 0; 2857 uint8_t next_protocol = 0xff; 2858 int actions_n = 0; 2859 const struct rte_flow_item *gre_item = NULL; 2860 struct rte_flow_item_tcp nic_tcp_mask = { 2861 .hdr = { 2862 .tcp_flags = 0xFF, 2863 .src_port = RTE_BE16(UINT16_MAX), 2864 .dst_port = RTE_BE16(UINT16_MAX), 2865 } 2866 }; 2867 2868 if (items == NULL) 2869 return -1; 2870 ret = flow_dv_validate_attributes(dev, attr, error); 2871 if (ret < 0) 2872 return ret; 2873 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2874 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2875 switch (items->type) { 2876 case RTE_FLOW_ITEM_TYPE_VOID: 2877 break; 2878 case RTE_FLOW_ITEM_TYPE_PORT_ID: 2879 ret = flow_dv_validate_item_port_id 2880 (dev, items, attr, item_flags, error); 2881 if (ret < 0) 2882 return ret; 2883 last_item = MLX5_FLOW_ITEM_PORT_ID; 2884 break; 2885 case RTE_FLOW_ITEM_TYPE_ETH: 2886 ret = mlx5_flow_validate_item_eth(items, item_flags, 2887 error); 2888 if (ret < 0) 2889 return ret; 2890 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2891 MLX5_FLOW_LAYER_OUTER_L2; 2892 break; 2893 case RTE_FLOW_ITEM_TYPE_VLAN: 2894 ret = mlx5_flow_validate_item_vlan(items, item_flags, 2895 error); 2896 if (ret < 0) 2897 return ret; 2898 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2899 MLX5_FLOW_LAYER_OUTER_VLAN; 2900 break; 2901 case RTE_FLOW_ITEM_TYPE_IPV4: 2902 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 2903 NULL, error); 2904 if (ret < 0) 2905 return ret; 2906 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2907 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2908 if (items->mask != NULL && 2909 ((const struct rte_flow_item_ipv4 *) 2910 items->mask)->hdr.next_proto_id) { 2911 next_protocol = 2912 ((const struct rte_flow_item_ipv4 *) 2913 (items->spec))->hdr.next_proto_id; 2914 next_protocol &= 2915 ((const struct rte_flow_item_ipv4 *) 2916 (items->mask))->hdr.next_proto_id; 2917 } else { 2918 /* Reset for inner layer. */ 2919 next_protocol = 0xff; 2920 } 2921 mlx5_flow_tunnel_ip_check(items, &last_item); 2922 break; 2923 case RTE_FLOW_ITEM_TYPE_IPV6: 2924 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 2925 NULL, error); 2926 if (ret < 0) 2927 return ret; 2928 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2929 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2930 if (items->mask != NULL && 2931 ((const struct rte_flow_item_ipv6 *) 2932 items->mask)->hdr.proto) { 2933 next_protocol = 2934 ((const struct rte_flow_item_ipv6 *) 2935 items->spec)->hdr.proto; 2936 next_protocol &= 2937 ((const struct rte_flow_item_ipv6 *) 2938 items->mask)->hdr.proto; 2939 } else { 2940 /* Reset for inner layer. */ 2941 next_protocol = 0xff; 2942 } 2943 mlx5_flow_tunnel_ip_check(items, &last_item); 2944 break; 2945 case RTE_FLOW_ITEM_TYPE_TCP: 2946 ret = mlx5_flow_validate_item_tcp 2947 (items, item_flags, 2948 next_protocol, 2949 &nic_tcp_mask, 2950 error); 2951 if (ret < 0) 2952 return ret; 2953 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 2954 MLX5_FLOW_LAYER_OUTER_L4_TCP; 2955 break; 2956 case RTE_FLOW_ITEM_TYPE_UDP: 2957 ret = mlx5_flow_validate_item_udp(items, item_flags, 2958 next_protocol, 2959 error); 2960 if (ret < 0) 2961 return ret; 2962 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 2963 MLX5_FLOW_LAYER_OUTER_L4_UDP; 2964 break; 2965 case RTE_FLOW_ITEM_TYPE_GRE: 2966 ret = mlx5_flow_validate_item_gre(items, item_flags, 2967 next_protocol, error); 2968 if (ret < 0) 2969 return ret; 2970 gre_item = items; 2971 last_item = MLX5_FLOW_LAYER_GRE; 2972 break; 2973 case RTE_FLOW_ITEM_TYPE_NVGRE: 2974 ret = mlx5_flow_validate_item_nvgre(items, item_flags, 2975 next_protocol, 2976 error); 2977 if (ret < 0) 2978 return ret; 2979 last_item = MLX5_FLOW_LAYER_NVGRE; 2980 break; 2981 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 2982 ret = mlx5_flow_validate_item_gre_key 2983 (items, item_flags, gre_item, error); 2984 if (ret < 0) 2985 return ret; 2986 last_item = MLX5_FLOW_LAYER_GRE_KEY; 2987 break; 2988 case RTE_FLOW_ITEM_TYPE_VXLAN: 2989 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 2990 error); 2991 if (ret < 0) 2992 return ret; 2993 last_item = MLX5_FLOW_LAYER_VXLAN; 2994 break; 2995 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 2996 ret = mlx5_flow_validate_item_vxlan_gpe(items, 2997 item_flags, dev, 2998 error); 2999 if (ret < 0) 3000 return ret; 3001 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 3002 break; 3003 case RTE_FLOW_ITEM_TYPE_MPLS: 3004 ret = mlx5_flow_validate_item_mpls(dev, items, 3005 item_flags, 3006 last_item, error); 3007 if (ret < 0) 3008 return ret; 3009 last_item = MLX5_FLOW_LAYER_MPLS; 3010 break; 3011 case RTE_FLOW_ITEM_TYPE_META: 3012 ret = flow_dv_validate_item_meta(dev, items, attr, 3013 error); 3014 if (ret < 0) 3015 return ret; 3016 last_item = MLX5_FLOW_ITEM_METADATA; 3017 break; 3018 case RTE_FLOW_ITEM_TYPE_ICMP: 3019 ret = mlx5_flow_validate_item_icmp(items, item_flags, 3020 next_protocol, 3021 error); 3022 if (ret < 0) 3023 return ret; 3024 last_item = MLX5_FLOW_LAYER_ICMP; 3025 break; 3026 case RTE_FLOW_ITEM_TYPE_ICMP6: 3027 ret = mlx5_flow_validate_item_icmp6(items, item_flags, 3028 next_protocol, 3029 error); 3030 if (ret < 0) 3031 return ret; 3032 last_item = MLX5_FLOW_LAYER_ICMP6; 3033 break; 3034 default: 3035 return rte_flow_error_set(error, ENOTSUP, 3036 RTE_FLOW_ERROR_TYPE_ITEM, 3037 NULL, "item not supported"); 3038 } 3039 item_flags |= last_item; 3040 } 3041 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3042 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 3043 return rte_flow_error_set(error, ENOTSUP, 3044 RTE_FLOW_ERROR_TYPE_ACTION, 3045 actions, "too many actions"); 3046 switch (actions->type) { 3047 case RTE_FLOW_ACTION_TYPE_VOID: 3048 break; 3049 case RTE_FLOW_ACTION_TYPE_PORT_ID: 3050 ret = flow_dv_validate_action_port_id(dev, 3051 action_flags, 3052 actions, 3053 attr, 3054 error); 3055 if (ret) 3056 return ret; 3057 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 3058 ++actions_n; 3059 break; 3060 case RTE_FLOW_ACTION_TYPE_FLAG: 3061 ret = mlx5_flow_validate_action_flag(action_flags, 3062 attr, error); 3063 if (ret < 0) 3064 return ret; 3065 action_flags |= MLX5_FLOW_ACTION_FLAG; 3066 ++actions_n; 3067 break; 3068 case RTE_FLOW_ACTION_TYPE_MARK: 3069 ret = mlx5_flow_validate_action_mark(actions, 3070 action_flags, 3071 attr, error); 3072 if (ret < 0) 3073 return ret; 3074 action_flags |= MLX5_FLOW_ACTION_MARK; 3075 ++actions_n; 3076 break; 3077 case RTE_FLOW_ACTION_TYPE_DROP: 3078 ret = mlx5_flow_validate_action_drop(action_flags, 3079 attr, error); 3080 if (ret < 0) 3081 return ret; 3082 action_flags |= MLX5_FLOW_ACTION_DROP; 3083 ++actions_n; 3084 break; 3085 case RTE_FLOW_ACTION_TYPE_QUEUE: 3086 ret = mlx5_flow_validate_action_queue(actions, 3087 action_flags, dev, 3088 attr, error); 3089 if (ret < 0) 3090 return ret; 3091 action_flags |= MLX5_FLOW_ACTION_QUEUE; 3092 ++actions_n; 3093 break; 3094 case RTE_FLOW_ACTION_TYPE_RSS: 3095 ret = mlx5_flow_validate_action_rss(actions, 3096 action_flags, dev, 3097 attr, item_flags, 3098 error); 3099 if (ret < 0) 3100 return ret; 3101 action_flags |= MLX5_FLOW_ACTION_RSS; 3102 ++actions_n; 3103 break; 3104 case RTE_FLOW_ACTION_TYPE_COUNT: 3105 ret = flow_dv_validate_action_count(dev, error); 3106 if (ret < 0) 3107 return ret; 3108 action_flags |= MLX5_FLOW_ACTION_COUNT; 3109 ++actions_n; 3110 break; 3111 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 3112 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 3113 ret = flow_dv_validate_action_l2_encap(action_flags, 3114 actions, attr, 3115 error); 3116 if (ret < 0) 3117 return ret; 3118 action_flags |= actions->type == 3119 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? 3120 MLX5_FLOW_ACTION_VXLAN_ENCAP : 3121 MLX5_FLOW_ACTION_NVGRE_ENCAP; 3122 ++actions_n; 3123 break; 3124 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 3125 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 3126 ret = flow_dv_validate_action_l2_decap(action_flags, 3127 attr, error); 3128 if (ret < 0) 3129 return ret; 3130 action_flags |= actions->type == 3131 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? 3132 MLX5_FLOW_ACTION_VXLAN_DECAP : 3133 MLX5_FLOW_ACTION_NVGRE_DECAP; 3134 ++actions_n; 3135 break; 3136 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 3137 ret = flow_dv_validate_action_raw_encap(action_flags, 3138 actions, attr, 3139 error); 3140 if (ret < 0) 3141 return ret; 3142 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; 3143 ++actions_n; 3144 break; 3145 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 3146 ret = flow_dv_validate_action_raw_decap(action_flags, 3147 actions, attr, 3148 error); 3149 if (ret < 0) 3150 return ret; 3151 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; 3152 ++actions_n; 3153 break; 3154 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 3155 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 3156 ret = flow_dv_validate_action_modify_mac(action_flags, 3157 actions, 3158 item_flags, 3159 error); 3160 if (ret < 0) 3161 return ret; 3162 /* Count all modify-header actions as one action. */ 3163 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3164 ++actions_n; 3165 action_flags |= actions->type == 3166 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 3167 MLX5_FLOW_ACTION_SET_MAC_SRC : 3168 MLX5_FLOW_ACTION_SET_MAC_DST; 3169 break; 3170 3171 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 3172 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 3173 ret = flow_dv_validate_action_modify_ipv4(action_flags, 3174 actions, 3175 item_flags, 3176 error); 3177 if (ret < 0) 3178 return ret; 3179 /* Count all modify-header actions as one action. */ 3180 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3181 ++actions_n; 3182 action_flags |= actions->type == 3183 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 3184 MLX5_FLOW_ACTION_SET_IPV4_SRC : 3185 MLX5_FLOW_ACTION_SET_IPV4_DST; 3186 break; 3187 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 3188 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 3189 ret = flow_dv_validate_action_modify_ipv6(action_flags, 3190 actions, 3191 item_flags, 3192 error); 3193 if (ret < 0) 3194 return ret; 3195 /* Count all modify-header actions as one action. */ 3196 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3197 ++actions_n; 3198 action_flags |= actions->type == 3199 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 3200 MLX5_FLOW_ACTION_SET_IPV6_SRC : 3201 MLX5_FLOW_ACTION_SET_IPV6_DST; 3202 break; 3203 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 3204 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 3205 ret = flow_dv_validate_action_modify_tp(action_flags, 3206 actions, 3207 item_flags, 3208 error); 3209 if (ret < 0) 3210 return ret; 3211 /* Count all modify-header actions as one action. */ 3212 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3213 ++actions_n; 3214 action_flags |= actions->type == 3215 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 3216 MLX5_FLOW_ACTION_SET_TP_SRC : 3217 MLX5_FLOW_ACTION_SET_TP_DST; 3218 break; 3219 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 3220 case RTE_FLOW_ACTION_TYPE_SET_TTL: 3221 ret = flow_dv_validate_action_modify_ttl(action_flags, 3222 actions, 3223 item_flags, 3224 error); 3225 if (ret < 0) 3226 return ret; 3227 /* Count all modify-header actions as one action. */ 3228 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3229 ++actions_n; 3230 action_flags |= actions->type == 3231 RTE_FLOW_ACTION_TYPE_SET_TTL ? 3232 MLX5_FLOW_ACTION_SET_TTL : 3233 MLX5_FLOW_ACTION_DEC_TTL; 3234 break; 3235 case RTE_FLOW_ACTION_TYPE_JUMP: 3236 ret = flow_dv_validate_action_jump(actions, 3237 attr->group, error); 3238 if (ret) 3239 return ret; 3240 ++actions_n; 3241 action_flags |= MLX5_FLOW_ACTION_JUMP; 3242 break; 3243 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 3244 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 3245 ret = flow_dv_validate_action_modify_tcp_seq 3246 (action_flags, 3247 actions, 3248 item_flags, 3249 error); 3250 if (ret < 0) 3251 return ret; 3252 /* Count all modify-header actions as one action. */ 3253 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3254 ++actions_n; 3255 action_flags |= actions->type == 3256 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 3257 MLX5_FLOW_ACTION_INC_TCP_SEQ : 3258 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 3259 break; 3260 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 3261 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 3262 ret = flow_dv_validate_action_modify_tcp_ack 3263 (action_flags, 3264 actions, 3265 item_flags, 3266 error); 3267 if (ret < 0) 3268 return ret; 3269 /* Count all modify-header actions as one action. */ 3270 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)) 3271 ++actions_n; 3272 action_flags |= actions->type == 3273 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 3274 MLX5_FLOW_ACTION_INC_TCP_ACK : 3275 MLX5_FLOW_ACTION_DEC_TCP_ACK; 3276 break; 3277 default: 3278 return rte_flow_error_set(error, ENOTSUP, 3279 RTE_FLOW_ERROR_TYPE_ACTION, 3280 actions, 3281 "action not supported"); 3282 } 3283 } 3284 /* Eswitch has few restrictions on using items and actions */ 3285 if (attr->transfer) { 3286 if (action_flags & MLX5_FLOW_ACTION_FLAG) 3287 return rte_flow_error_set(error, ENOTSUP, 3288 RTE_FLOW_ERROR_TYPE_ACTION, 3289 NULL, 3290 "unsupported action FLAG"); 3291 if (action_flags & MLX5_FLOW_ACTION_MARK) 3292 return rte_flow_error_set(error, ENOTSUP, 3293 RTE_FLOW_ERROR_TYPE_ACTION, 3294 NULL, 3295 "unsupported action MARK"); 3296 if (action_flags & MLX5_FLOW_ACTION_QUEUE) 3297 return rte_flow_error_set(error, ENOTSUP, 3298 RTE_FLOW_ERROR_TYPE_ACTION, 3299 NULL, 3300 "unsupported action QUEUE"); 3301 if (action_flags & MLX5_FLOW_ACTION_RSS) 3302 return rte_flow_error_set(error, ENOTSUP, 3303 RTE_FLOW_ERROR_TYPE_ACTION, 3304 NULL, 3305 "unsupported action RSS"); 3306 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS)) 3307 return rte_flow_error_set(error, EINVAL, 3308 RTE_FLOW_ERROR_TYPE_ACTION, 3309 actions, 3310 "no fate action is found"); 3311 } else { 3312 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 3313 return rte_flow_error_set(error, EINVAL, 3314 RTE_FLOW_ERROR_TYPE_ACTION, 3315 actions, 3316 "no fate action is found"); 3317 } 3318 return 0; 3319 } 3320 3321 /** 3322 * Internal preparation function. Allocates the DV flow size, 3323 * this size is constant. 3324 * 3325 * @param[in] attr 3326 * Pointer to the flow attributes. 3327 * @param[in] items 3328 * Pointer to the list of items. 3329 * @param[in] actions 3330 * Pointer to the list of actions. 3331 * @param[out] error 3332 * Pointer to the error structure. 3333 * 3334 * @return 3335 * Pointer to mlx5_flow object on success, 3336 * otherwise NULL and rte_errno is set. 3337 */ 3338 static struct mlx5_flow * 3339 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, 3340 const struct rte_flow_item items[] __rte_unused, 3341 const struct rte_flow_action actions[] __rte_unused, 3342 struct rte_flow_error *error) 3343 { 3344 uint32_t size = sizeof(struct mlx5_flow); 3345 struct mlx5_flow *flow; 3346 3347 flow = rte_calloc(__func__, 1, size, 0); 3348 if (!flow) { 3349 rte_flow_error_set(error, ENOMEM, 3350 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3351 "not enough memory to create flow"); 3352 return NULL; 3353 } 3354 flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param); 3355 return flow; 3356 } 3357 3358 #ifndef NDEBUG 3359 /** 3360 * Sanity check for match mask and value. Similar to check_valid_spec() in 3361 * kernel driver. If unmasked bit is present in value, it returns failure. 3362 * 3363 * @param match_mask 3364 * pointer to match mask buffer. 3365 * @param match_value 3366 * pointer to match value buffer. 3367 * 3368 * @return 3369 * 0 if valid, -EINVAL otherwise. 3370 */ 3371 static int 3372 flow_dv_check_valid_spec(void *match_mask, void *match_value) 3373 { 3374 uint8_t *m = match_mask; 3375 uint8_t *v = match_value; 3376 unsigned int i; 3377 3378 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) { 3379 if (v[i] & ~m[i]) { 3380 DRV_LOG(ERR, 3381 "match_value differs from match_criteria" 3382 " %p[%u] != %p[%u]", 3383 match_value, i, match_mask, i); 3384 return -EINVAL; 3385 } 3386 } 3387 return 0; 3388 } 3389 #endif 3390 3391 /** 3392 * Add Ethernet item to matcher and to the value. 3393 * 3394 * @param[in, out] matcher 3395 * Flow matcher. 3396 * @param[in, out] key 3397 * Flow matcher value. 3398 * @param[in] item 3399 * Flow pattern to translate. 3400 * @param[in] inner 3401 * Item is inner pattern. 3402 */ 3403 static void 3404 flow_dv_translate_item_eth(void *matcher, void *key, 3405 const struct rte_flow_item *item, int inner) 3406 { 3407 const struct rte_flow_item_eth *eth_m = item->mask; 3408 const struct rte_flow_item_eth *eth_v = item->spec; 3409 const struct rte_flow_item_eth nic_mask = { 3410 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 3411 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 3412 .type = RTE_BE16(0xffff), 3413 }; 3414 void *headers_m; 3415 void *headers_v; 3416 char *l24_v; 3417 unsigned int i; 3418 3419 if (!eth_v) 3420 return; 3421 if (!eth_m) 3422 eth_m = &nic_mask; 3423 if (inner) { 3424 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3425 inner_headers); 3426 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3427 } else { 3428 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3429 outer_headers); 3430 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3431 } 3432 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), 3433 ð_m->dst, sizeof(eth_m->dst)); 3434 /* The value must be in the range of the mask. */ 3435 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); 3436 for (i = 0; i < sizeof(eth_m->dst); ++i) 3437 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 3438 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), 3439 ð_m->src, sizeof(eth_m->src)); 3440 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); 3441 /* The value must be in the range of the mask. */ 3442 for (i = 0; i < sizeof(eth_m->dst); ++i) 3443 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 3444 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 3445 rte_be_to_cpu_16(eth_m->type)); 3446 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype); 3447 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 3448 } 3449 3450 /** 3451 * Add VLAN item to matcher and to the value. 3452 * 3453 * @param[in, out] matcher 3454 * Flow matcher. 3455 * @param[in, out] key 3456 * Flow matcher value. 3457 * @param[in] item 3458 * Flow pattern to translate. 3459 * @param[in] inner 3460 * Item is inner pattern. 3461 */ 3462 static void 3463 flow_dv_translate_item_vlan(void *matcher, void *key, 3464 const struct rte_flow_item *item, 3465 int inner) 3466 { 3467 const struct rte_flow_item_vlan *vlan_m = item->mask; 3468 const struct rte_flow_item_vlan *vlan_v = item->spec; 3469 const struct rte_flow_item_vlan nic_mask = { 3470 .tci = RTE_BE16(0x0fff), 3471 .inner_type = RTE_BE16(0xffff), 3472 }; 3473 void *headers_m; 3474 void *headers_v; 3475 uint16_t tci_m; 3476 uint16_t tci_v; 3477 3478 if (!vlan_v) 3479 return; 3480 if (!vlan_m) 3481 vlan_m = &nic_mask; 3482 if (inner) { 3483 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3484 inner_headers); 3485 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3486 } else { 3487 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3488 outer_headers); 3489 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3490 } 3491 tci_m = rte_be_to_cpu_16(vlan_m->tci); 3492 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 3493 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 3494 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 3495 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); 3496 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); 3497 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); 3498 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); 3499 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); 3500 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); 3501 } 3502 3503 /** 3504 * Add IPV4 item to matcher and to the value. 3505 * 3506 * @param[in, out] matcher 3507 * Flow matcher. 3508 * @param[in, out] key 3509 * Flow matcher value. 3510 * @param[in] item 3511 * Flow pattern to translate. 3512 * @param[in] inner 3513 * Item is inner pattern. 3514 * @param[in] group 3515 * The group to insert the rule. 3516 */ 3517 static void 3518 flow_dv_translate_item_ipv4(void *matcher, void *key, 3519 const struct rte_flow_item *item, 3520 int inner, uint32_t group) 3521 { 3522 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 3523 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 3524 const struct rte_flow_item_ipv4 nic_mask = { 3525 .hdr = { 3526 .src_addr = RTE_BE32(0xffffffff), 3527 .dst_addr = RTE_BE32(0xffffffff), 3528 .type_of_service = 0xff, 3529 .next_proto_id = 0xff, 3530 }, 3531 }; 3532 void *headers_m; 3533 void *headers_v; 3534 char *l24_m; 3535 char *l24_v; 3536 uint8_t tos; 3537 3538 if (inner) { 3539 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3540 inner_headers); 3541 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3542 } else { 3543 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3544 outer_headers); 3545 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3546 } 3547 if (group == 0) 3548 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 3549 else 3550 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4); 3551 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4); 3552 if (!ipv4_v) 3553 return; 3554 if (!ipv4_m) 3555 ipv4_m = &nic_mask; 3556 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 3557 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 3558 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 3559 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 3560 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 3561 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 3562 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 3563 src_ipv4_src_ipv6.ipv4_layout.ipv4); 3564 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 3565 src_ipv4_src_ipv6.ipv4_layout.ipv4); 3566 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 3567 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 3568 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 3569 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 3570 ipv4_m->hdr.type_of_service); 3571 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 3572 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 3573 ipv4_m->hdr.type_of_service >> 2); 3574 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 3575 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 3576 ipv4_m->hdr.next_proto_id); 3577 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 3578 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 3579 } 3580 3581 /** 3582 * Add IPV6 item to matcher and to the value. 3583 * 3584 * @param[in, out] matcher 3585 * Flow matcher. 3586 * @param[in, out] key 3587 * Flow matcher value. 3588 * @param[in] item 3589 * Flow pattern to translate. 3590 * @param[in] inner 3591 * Item is inner pattern. 3592 * @param[in] group 3593 * The group to insert the rule. 3594 */ 3595 static void 3596 flow_dv_translate_item_ipv6(void *matcher, void *key, 3597 const struct rte_flow_item *item, 3598 int inner, uint32_t group) 3599 { 3600 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 3601 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 3602 const struct rte_flow_item_ipv6 nic_mask = { 3603 .hdr = { 3604 .src_addr = 3605 "\xff\xff\xff\xff\xff\xff\xff\xff" 3606 "\xff\xff\xff\xff\xff\xff\xff\xff", 3607 .dst_addr = 3608 "\xff\xff\xff\xff\xff\xff\xff\xff" 3609 "\xff\xff\xff\xff\xff\xff\xff\xff", 3610 .vtc_flow = RTE_BE32(0xffffffff), 3611 .proto = 0xff, 3612 .hop_limits = 0xff, 3613 }, 3614 }; 3615 void *headers_m; 3616 void *headers_v; 3617 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 3618 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 3619 char *l24_m; 3620 char *l24_v; 3621 uint32_t vtc_m; 3622 uint32_t vtc_v; 3623 int i; 3624 int size; 3625 3626 if (inner) { 3627 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3628 inner_headers); 3629 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3630 } else { 3631 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3632 outer_headers); 3633 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3634 } 3635 if (group == 0) 3636 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 3637 else 3638 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6); 3639 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6); 3640 if (!ipv6_v) 3641 return; 3642 if (!ipv6_m) 3643 ipv6_m = &nic_mask; 3644 size = sizeof(ipv6_m->hdr.dst_addr); 3645 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 3646 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 3647 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 3648 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 3649 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 3650 for (i = 0; i < size; ++i) 3651 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 3652 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 3653 src_ipv4_src_ipv6.ipv6_layout.ipv6); 3654 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 3655 src_ipv4_src_ipv6.ipv6_layout.ipv6); 3656 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 3657 for (i = 0; i < size; ++i) 3658 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 3659 /* TOS. */ 3660 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 3661 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 3662 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 3663 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 3664 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 3665 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 3666 /* Label. */ 3667 if (inner) { 3668 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 3669 vtc_m); 3670 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 3671 vtc_v); 3672 } else { 3673 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 3674 vtc_m); 3675 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 3676 vtc_v); 3677 } 3678 /* Protocol. */ 3679 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 3680 ipv6_m->hdr.proto); 3681 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 3682 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 3683 } 3684 3685 /** 3686 * Add TCP item to matcher and to the value. 3687 * 3688 * @param[in, out] matcher 3689 * Flow matcher. 3690 * @param[in, out] key 3691 * Flow matcher value. 3692 * @param[in] item 3693 * Flow pattern to translate. 3694 * @param[in] inner 3695 * Item is inner pattern. 3696 */ 3697 static void 3698 flow_dv_translate_item_tcp(void *matcher, void *key, 3699 const struct rte_flow_item *item, 3700 int inner) 3701 { 3702 const struct rte_flow_item_tcp *tcp_m = item->mask; 3703 const struct rte_flow_item_tcp *tcp_v = item->spec; 3704 void *headers_m; 3705 void *headers_v; 3706 3707 if (inner) { 3708 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3709 inner_headers); 3710 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3711 } else { 3712 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3713 outer_headers); 3714 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3715 } 3716 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 3717 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 3718 if (!tcp_v) 3719 return; 3720 if (!tcp_m) 3721 tcp_m = &rte_flow_item_tcp_mask; 3722 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 3723 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 3724 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 3725 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 3726 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 3727 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 3728 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 3729 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 3730 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags, 3731 tcp_m->hdr.tcp_flags); 3732 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 3733 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags)); 3734 } 3735 3736 /** 3737 * Add UDP item to matcher and to the value. 3738 * 3739 * @param[in, out] matcher 3740 * Flow matcher. 3741 * @param[in, out] key 3742 * Flow matcher value. 3743 * @param[in] item 3744 * Flow pattern to translate. 3745 * @param[in] inner 3746 * Item is inner pattern. 3747 */ 3748 static void 3749 flow_dv_translate_item_udp(void *matcher, void *key, 3750 const struct rte_flow_item *item, 3751 int inner) 3752 { 3753 const struct rte_flow_item_udp *udp_m = item->mask; 3754 const struct rte_flow_item_udp *udp_v = item->spec; 3755 void *headers_m; 3756 void *headers_v; 3757 3758 if (inner) { 3759 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3760 inner_headers); 3761 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3762 } else { 3763 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3764 outer_headers); 3765 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3766 } 3767 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 3768 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 3769 if (!udp_v) 3770 return; 3771 if (!udp_m) 3772 udp_m = &rte_flow_item_udp_mask; 3773 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 3774 rte_be_to_cpu_16(udp_m->hdr.src_port)); 3775 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 3776 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 3777 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 3778 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 3779 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 3780 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 3781 } 3782 3783 /** 3784 * Add GRE optional Key item to matcher and to the value. 3785 * 3786 * @param[in, out] matcher 3787 * Flow matcher. 3788 * @param[in, out] key 3789 * Flow matcher value. 3790 * @param[in] item 3791 * Flow pattern to translate. 3792 * @param[in] inner 3793 * Item is inner pattern. 3794 */ 3795 static void 3796 flow_dv_translate_item_gre_key(void *matcher, void *key, 3797 const struct rte_flow_item *item) 3798 { 3799 const rte_be32_t *key_m = item->mask; 3800 const rte_be32_t *key_v = item->spec; 3801 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 3802 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 3803 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3804 3805 if (!key_v) 3806 return; 3807 if (!key_m) 3808 key_m = &gre_key_default_mask; 3809 /* GRE K bit must be on and should already be validated */ 3810 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1); 3811 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1); 3812 MLX5_SET(fte_match_set_misc, misc_m, gre_key_h, 3813 rte_be_to_cpu_32(*key_m) >> 8); 3814 MLX5_SET(fte_match_set_misc, misc_v, gre_key_h, 3815 rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8); 3816 MLX5_SET(fte_match_set_misc, misc_m, gre_key_l, 3817 rte_be_to_cpu_32(*key_m) & 0xFF); 3818 MLX5_SET(fte_match_set_misc, misc_v, gre_key_l, 3819 rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF); 3820 } 3821 3822 /** 3823 * Add GRE item to matcher and to the value. 3824 * 3825 * @param[in, out] matcher 3826 * Flow matcher. 3827 * @param[in, out] key 3828 * Flow matcher value. 3829 * @param[in] item 3830 * Flow pattern to translate. 3831 * @param[in] inner 3832 * Item is inner pattern. 3833 */ 3834 static void 3835 flow_dv_translate_item_gre(void *matcher, void *key, 3836 const struct rte_flow_item *item, 3837 int inner) 3838 { 3839 const struct rte_flow_item_gre *gre_m = item->mask; 3840 const struct rte_flow_item_gre *gre_v = item->spec; 3841 void *headers_m; 3842 void *headers_v; 3843 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 3844 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 3845 struct { 3846 union { 3847 __extension__ 3848 struct { 3849 uint16_t version:3; 3850 uint16_t rsvd0:9; 3851 uint16_t s_present:1; 3852 uint16_t k_present:1; 3853 uint16_t rsvd_bit1:1; 3854 uint16_t c_present:1; 3855 }; 3856 uint16_t value; 3857 }; 3858 } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v; 3859 3860 if (inner) { 3861 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3862 inner_headers); 3863 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3864 } else { 3865 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3866 outer_headers); 3867 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3868 } 3869 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 3870 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 3871 if (!gre_v) 3872 return; 3873 if (!gre_m) 3874 gre_m = &rte_flow_item_gre_mask; 3875 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 3876 rte_be_to_cpu_16(gre_m->protocol)); 3877 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 3878 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); 3879 gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver); 3880 gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver); 3881 MLX5_SET(fte_match_set_misc, misc_m, gre_c_present, 3882 gre_crks_rsvd0_ver_m.c_present); 3883 MLX5_SET(fte_match_set_misc, misc_v, gre_c_present, 3884 gre_crks_rsvd0_ver_v.c_present & 3885 gre_crks_rsvd0_ver_m.c_present); 3886 MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 3887 gre_crks_rsvd0_ver_m.k_present); 3888 MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 3889 gre_crks_rsvd0_ver_v.k_present & 3890 gre_crks_rsvd0_ver_m.k_present); 3891 MLX5_SET(fte_match_set_misc, misc_m, gre_s_present, 3892 gre_crks_rsvd0_ver_m.s_present); 3893 MLX5_SET(fte_match_set_misc, misc_v, gre_s_present, 3894 gre_crks_rsvd0_ver_v.s_present & 3895 gre_crks_rsvd0_ver_m.s_present); 3896 } 3897 3898 /** 3899 * Add NVGRE item to matcher and to the value. 3900 * 3901 * @param[in, out] matcher 3902 * Flow matcher. 3903 * @param[in, out] key 3904 * Flow matcher value. 3905 * @param[in] item 3906 * Flow pattern to translate. 3907 * @param[in] inner 3908 * Item is inner pattern. 3909 */ 3910 static void 3911 flow_dv_translate_item_nvgre(void *matcher, void *key, 3912 const struct rte_flow_item *item, 3913 int inner) 3914 { 3915 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 3916 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 3917 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 3918 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 3919 const char *tni_flow_id_m = (const char *)nvgre_m->tni; 3920 const char *tni_flow_id_v = (const char *)nvgre_v->tni; 3921 char *gre_key_m; 3922 char *gre_key_v; 3923 int size; 3924 int i; 3925 3926 /* For NVGRE, GRE header fields must be set with defined values. */ 3927 const struct rte_flow_item_gre gre_spec = { 3928 .c_rsvd0_ver = RTE_BE16(0x2000), 3929 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB) 3930 }; 3931 const struct rte_flow_item_gre gre_mask = { 3932 .c_rsvd0_ver = RTE_BE16(0xB000), 3933 .protocol = RTE_BE16(UINT16_MAX), 3934 }; 3935 const struct rte_flow_item gre_item = { 3936 .spec = &gre_spec, 3937 .mask = &gre_mask, 3938 .last = NULL, 3939 }; 3940 flow_dv_translate_item_gre(matcher, key, &gre_item, inner); 3941 if (!nvgre_v) 3942 return; 3943 if (!nvgre_m) 3944 nvgre_m = &rte_flow_item_nvgre_mask; 3945 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 3946 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 3947 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 3948 memcpy(gre_key_m, tni_flow_id_m, size); 3949 for (i = 0; i < size; ++i) 3950 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 3951 } 3952 3953 /** 3954 * Add VXLAN item to matcher and to the value. 3955 * 3956 * @param[in, out] matcher 3957 * Flow matcher. 3958 * @param[in, out] key 3959 * Flow matcher value. 3960 * @param[in] item 3961 * Flow pattern to translate. 3962 * @param[in] inner 3963 * Item is inner pattern. 3964 */ 3965 static void 3966 flow_dv_translate_item_vxlan(void *matcher, void *key, 3967 const struct rte_flow_item *item, 3968 int inner) 3969 { 3970 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 3971 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 3972 void *headers_m; 3973 void *headers_v; 3974 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 3975 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 3976 char *vni_m; 3977 char *vni_v; 3978 uint16_t dport; 3979 int size; 3980 int i; 3981 3982 if (inner) { 3983 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3984 inner_headers); 3985 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 3986 } else { 3987 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 3988 outer_headers); 3989 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 3990 } 3991 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 3992 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 3993 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 3994 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 3995 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 3996 } 3997 if (!vxlan_v) 3998 return; 3999 if (!vxlan_m) 4000 vxlan_m = &rte_flow_item_vxlan_mask; 4001 size = sizeof(vxlan_m->vni); 4002 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 4003 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 4004 memcpy(vni_m, vxlan_m->vni, size); 4005 for (i = 0; i < size; ++i) 4006 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 4007 } 4008 4009 /** 4010 * Add MPLS item to matcher and to the value. 4011 * 4012 * @param[in, out] matcher 4013 * Flow matcher. 4014 * @param[in, out] key 4015 * Flow matcher value. 4016 * @param[in] item 4017 * Flow pattern to translate. 4018 * @param[in] prev_layer 4019 * The protocol layer indicated in previous item. 4020 * @param[in] inner 4021 * Item is inner pattern. 4022 */ 4023 static void 4024 flow_dv_translate_item_mpls(void *matcher, void *key, 4025 const struct rte_flow_item *item, 4026 uint64_t prev_layer, 4027 int inner) 4028 { 4029 const uint32_t *in_mpls_m = item->mask; 4030 const uint32_t *in_mpls_v = item->spec; 4031 uint32_t *out_mpls_m = 0; 4032 uint32_t *out_mpls_v = 0; 4033 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 4034 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 4035 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 4036 misc_parameters_2); 4037 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 4038 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 4039 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 4040 4041 switch (prev_layer) { 4042 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 4043 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff); 4044 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 4045 MLX5_UDP_PORT_MPLS); 4046 break; 4047 case MLX5_FLOW_LAYER_GRE: 4048 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); 4049 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 4050 RTE_ETHER_TYPE_MPLS); 4051 break; 4052 default: 4053 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 4054 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 4055 IPPROTO_MPLS); 4056 break; 4057 } 4058 if (!in_mpls_v) 4059 return; 4060 if (!in_mpls_m) 4061 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 4062 switch (prev_layer) { 4063 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 4064 out_mpls_m = 4065 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 4066 outer_first_mpls_over_udp); 4067 out_mpls_v = 4068 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 4069 outer_first_mpls_over_udp); 4070 break; 4071 case MLX5_FLOW_LAYER_GRE: 4072 out_mpls_m = 4073 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 4074 outer_first_mpls_over_gre); 4075 out_mpls_v = 4076 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 4077 outer_first_mpls_over_gre); 4078 break; 4079 default: 4080 /* Inner MPLS not over GRE is not supported. */ 4081 if (!inner) { 4082 out_mpls_m = 4083 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 4084 misc2_m, 4085 outer_first_mpls); 4086 out_mpls_v = 4087 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 4088 misc2_v, 4089 outer_first_mpls); 4090 } 4091 break; 4092 } 4093 if (out_mpls_m && out_mpls_v) { 4094 *out_mpls_m = *in_mpls_m; 4095 *out_mpls_v = *in_mpls_v & *in_mpls_m; 4096 } 4097 } 4098 4099 /** 4100 * Add META item to matcher 4101 * 4102 * @param[in, out] matcher 4103 * Flow matcher. 4104 * @param[in, out] key 4105 * Flow matcher value. 4106 * @param[in] item 4107 * Flow pattern to translate. 4108 * @param[in] inner 4109 * Item is inner pattern. 4110 */ 4111 static void 4112 flow_dv_translate_item_meta(void *matcher, void *key, 4113 const struct rte_flow_item *item) 4114 { 4115 const struct rte_flow_item_meta *meta_m; 4116 const struct rte_flow_item_meta *meta_v; 4117 void *misc2_m = 4118 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 4119 void *misc2_v = 4120 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 4121 4122 meta_m = (const void *)item->mask; 4123 if (!meta_m) 4124 meta_m = &rte_flow_item_meta_mask; 4125 meta_v = (const void *)item->spec; 4126 if (meta_v) { 4127 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, 4128 rte_be_to_cpu_32(meta_m->data)); 4129 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, 4130 rte_be_to_cpu_32(meta_v->data & meta_m->data)); 4131 } 4132 } 4133 4134 /** 4135 * Add source vport match to the specified matcher. 4136 * 4137 * @param[in, out] matcher 4138 * Flow matcher. 4139 * @param[in, out] key 4140 * Flow matcher value. 4141 * @param[in] port 4142 * Source vport value to match 4143 * @param[in] mask 4144 * Mask 4145 */ 4146 static void 4147 flow_dv_translate_item_source_vport(void *matcher, void *key, 4148 int16_t port, uint16_t mask) 4149 { 4150 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 4151 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 4152 4153 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask); 4154 MLX5_SET(fte_match_set_misc, misc_v, source_port, port); 4155 } 4156 4157 /** 4158 * Translate port-id item to eswitch match on port-id. 4159 * 4160 * @param[in] dev 4161 * The devich to configure through. 4162 * @param[in, out] matcher 4163 * Flow matcher. 4164 * @param[in, out] key 4165 * Flow matcher value. 4166 * @param[in] item 4167 * Flow pattern to translate. 4168 * 4169 * @return 4170 * 0 on success, a negative errno value otherwise. 4171 */ 4172 static int 4173 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher, 4174 void *key, const struct rte_flow_item *item) 4175 { 4176 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL; 4177 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL; 4178 uint16_t mask, val, id; 4179 int ret; 4180 4181 mask = pid_m ? pid_m->id : 0xffff; 4182 id = pid_v ? pid_v->id : dev->data->port_id; 4183 ret = mlx5_port_to_eswitch_info(id, NULL, &val); 4184 if (ret) 4185 return ret; 4186 flow_dv_translate_item_source_vport(matcher, key, val, mask); 4187 return 0; 4188 } 4189 4190 /** 4191 * Add ICMP6 item to matcher and to the value. 4192 * 4193 * @param[in, out] matcher 4194 * Flow matcher. 4195 * @param[in, out] key 4196 * Flow matcher value. 4197 * @param[in] item 4198 * Flow pattern to translate. 4199 * @param[in] inner 4200 * Item is inner pattern. 4201 */ 4202 static void 4203 flow_dv_translate_item_icmp6(void *matcher, void *key, 4204 const struct rte_flow_item *item, 4205 int inner) 4206 { 4207 const struct rte_flow_item_icmp6 *icmp6_m = item->mask; 4208 const struct rte_flow_item_icmp6 *icmp6_v = item->spec; 4209 void *headers_m; 4210 void *headers_v; 4211 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 4212 misc_parameters_3); 4213 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 4214 if (inner) { 4215 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 4216 inner_headers); 4217 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 4218 } else { 4219 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 4220 outer_headers); 4221 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 4222 } 4223 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 4224 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6); 4225 if (!icmp6_v) 4226 return; 4227 if (!icmp6_m) 4228 icmp6_m = &rte_flow_item_icmp6_mask; 4229 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type); 4230 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type, 4231 icmp6_v->type & icmp6_m->type); 4232 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code); 4233 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code, 4234 icmp6_v->code & icmp6_m->code); 4235 } 4236 4237 /** 4238 * Add ICMP item to matcher and to the value. 4239 * 4240 * @param[in, out] matcher 4241 * Flow matcher. 4242 * @param[in, out] key 4243 * Flow matcher value. 4244 * @param[in] item 4245 * Flow pattern to translate. 4246 * @param[in] inner 4247 * Item is inner pattern. 4248 */ 4249 static void 4250 flow_dv_translate_item_icmp(void *matcher, void *key, 4251 const struct rte_flow_item *item, 4252 int inner) 4253 { 4254 const struct rte_flow_item_icmp *icmp_m = item->mask; 4255 const struct rte_flow_item_icmp *icmp_v = item->spec; 4256 void *headers_m; 4257 void *headers_v; 4258 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher, 4259 misc_parameters_3); 4260 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3); 4261 if (inner) { 4262 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 4263 inner_headers); 4264 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 4265 } else { 4266 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 4267 outer_headers); 4268 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 4269 } 4270 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF); 4271 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP); 4272 if (!icmp_v) 4273 return; 4274 if (!icmp_m) 4275 icmp_m = &rte_flow_item_icmp_mask; 4276 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type, 4277 icmp_m->hdr.icmp_type); 4278 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type, 4279 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type); 4280 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code, 4281 icmp_m->hdr.icmp_code); 4282 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code, 4283 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code); 4284 } 4285 4286 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 4287 4288 #define HEADER_IS_ZERO(match_criteria, headers) \ 4289 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 4290 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 4291 4292 /** 4293 * Calculate flow matcher enable bitmap. 4294 * 4295 * @param match_criteria 4296 * Pointer to flow matcher criteria. 4297 * 4298 * @return 4299 * Bitmap of enabled fields. 4300 */ 4301 static uint8_t 4302 flow_dv_matcher_enable(uint32_t *match_criteria) 4303 { 4304 uint8_t match_criteria_enable; 4305 4306 match_criteria_enable = 4307 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 4308 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 4309 match_criteria_enable |= 4310 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 4311 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 4312 match_criteria_enable |= 4313 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 4314 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 4315 match_criteria_enable |= 4316 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 4317 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 4318 #ifdef HAVE_MLX5DV_DR 4319 match_criteria_enable |= 4320 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) << 4321 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT; 4322 #endif 4323 return match_criteria_enable; 4324 } 4325 4326 4327 /** 4328 * Get a flow table. 4329 * 4330 * @param dev[in, out] 4331 * Pointer to rte_eth_dev structure. 4332 * @param[in] table_id 4333 * Table id to use. 4334 * @param[in] egress 4335 * Direction of the table. 4336 * @param[in] transfer 4337 * E-Switch or NIC flow. 4338 * @param[out] error 4339 * pointer to error structure. 4340 * 4341 * @return 4342 * Returns tables resource based on the index, NULL in case of failed. 4343 */ 4344 static struct mlx5_flow_tbl_resource * 4345 flow_dv_tbl_resource_get(struct rte_eth_dev *dev, 4346 uint32_t table_id, uint8_t egress, 4347 uint8_t transfer, 4348 struct rte_flow_error *error) 4349 { 4350 struct mlx5_priv *priv = dev->data->dev_private; 4351 struct mlx5_ibv_shared *sh = priv->sh; 4352 struct mlx5_flow_tbl_resource *tbl; 4353 4354 #ifdef HAVE_MLX5DV_DR 4355 if (transfer) { 4356 tbl = &sh->fdb_tbl[table_id]; 4357 if (!tbl->obj) 4358 tbl->obj = mlx5_glue->dr_create_flow_tbl 4359 (sh->fdb_domain, table_id); 4360 } else if (egress) { 4361 tbl = &sh->tx_tbl[table_id]; 4362 if (!tbl->obj) 4363 tbl->obj = mlx5_glue->dr_create_flow_tbl 4364 (sh->tx_domain, table_id); 4365 } else { 4366 tbl = &sh->rx_tbl[table_id]; 4367 if (!tbl->obj) 4368 tbl->obj = mlx5_glue->dr_create_flow_tbl 4369 (sh->rx_domain, table_id); 4370 } 4371 if (!tbl->obj) { 4372 rte_flow_error_set(error, ENOMEM, 4373 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4374 NULL, "cannot create table"); 4375 return NULL; 4376 } 4377 rte_atomic32_inc(&tbl->refcnt); 4378 return tbl; 4379 #else 4380 (void)error; 4381 (void)tbl; 4382 if (transfer) 4383 return &sh->fdb_tbl[table_id]; 4384 else if (egress) 4385 return &sh->tx_tbl[table_id]; 4386 else 4387 return &sh->rx_tbl[table_id]; 4388 #endif 4389 } 4390 4391 /** 4392 * Release a flow table. 4393 * 4394 * @param[in] tbl 4395 * Table resource to be released. 4396 * 4397 * @return 4398 * Returns 0 if table was released, else return 1; 4399 */ 4400 static int 4401 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl) 4402 { 4403 if (!tbl) 4404 return 0; 4405 if (rte_atomic32_dec_and_test(&tbl->refcnt)) { 4406 mlx5_glue->dr_destroy_flow_tbl(tbl->obj); 4407 tbl->obj = NULL; 4408 return 0; 4409 } 4410 return 1; 4411 } 4412 4413 /** 4414 * Register the flow matcher. 4415 * 4416 * @param dev[in, out] 4417 * Pointer to rte_eth_dev structure. 4418 * @param[in, out] matcher 4419 * Pointer to flow matcher. 4420 * @parm[in, out] dev_flow 4421 * Pointer to the dev_flow. 4422 * @param[out] error 4423 * pointer to error structure. 4424 * 4425 * @return 4426 * 0 on success otherwise -errno and errno is set. 4427 */ 4428 static int 4429 flow_dv_matcher_register(struct rte_eth_dev *dev, 4430 struct mlx5_flow_dv_matcher *matcher, 4431 struct mlx5_flow *dev_flow, 4432 struct rte_flow_error *error) 4433 { 4434 struct mlx5_priv *priv = dev->data->dev_private; 4435 struct mlx5_ibv_shared *sh = priv->sh; 4436 struct mlx5_flow_dv_matcher *cache_matcher; 4437 struct mlx5dv_flow_matcher_attr dv_attr = { 4438 .type = IBV_FLOW_ATTR_NORMAL, 4439 .match_mask = (void *)&matcher->mask, 4440 }; 4441 struct mlx5_flow_tbl_resource *tbl = NULL; 4442 4443 /* Lookup from cache. */ 4444 LIST_FOREACH(cache_matcher, &sh->matchers, next) { 4445 if (matcher->crc == cache_matcher->crc && 4446 matcher->priority == cache_matcher->priority && 4447 matcher->egress == cache_matcher->egress && 4448 matcher->group == cache_matcher->group && 4449 matcher->transfer == cache_matcher->transfer && 4450 !memcmp((const void *)matcher->mask.buf, 4451 (const void *)cache_matcher->mask.buf, 4452 cache_matcher->mask.size)) { 4453 DRV_LOG(DEBUG, 4454 "priority %hd use %s matcher %p: refcnt %d++", 4455 cache_matcher->priority, 4456 cache_matcher->egress ? "tx" : "rx", 4457 (void *)cache_matcher, 4458 rte_atomic32_read(&cache_matcher->refcnt)); 4459 rte_atomic32_inc(&cache_matcher->refcnt); 4460 dev_flow->dv.matcher = cache_matcher; 4461 return 0; 4462 } 4463 } 4464 /* Register new matcher. */ 4465 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); 4466 if (!cache_matcher) 4467 return rte_flow_error_set(error, ENOMEM, 4468 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4469 "cannot allocate matcher memory"); 4470 tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR, 4471 matcher->egress, matcher->transfer, 4472 error); 4473 if (!tbl) { 4474 rte_free(cache_matcher); 4475 return rte_flow_error_set(error, ENOMEM, 4476 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4477 NULL, "cannot create table"); 4478 } 4479 *cache_matcher = *matcher; 4480 dv_attr.match_criteria_enable = 4481 flow_dv_matcher_enable(cache_matcher->mask.buf); 4482 dv_attr.priority = matcher->priority; 4483 if (matcher->egress) 4484 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 4485 cache_matcher->matcher_object = 4486 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj); 4487 if (!cache_matcher->matcher_object) { 4488 rte_free(cache_matcher); 4489 #ifdef HAVE_MLX5DV_DR 4490 flow_dv_tbl_resource_release(tbl); 4491 #endif 4492 return rte_flow_error_set(error, ENOMEM, 4493 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4494 NULL, "cannot create matcher"); 4495 } 4496 rte_atomic32_inc(&cache_matcher->refcnt); 4497 LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next); 4498 dev_flow->dv.matcher = cache_matcher; 4499 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", 4500 cache_matcher->priority, 4501 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, 4502 rte_atomic32_read(&cache_matcher->refcnt)); 4503 rte_atomic32_inc(&tbl->refcnt); 4504 return 0; 4505 } 4506 4507 /** 4508 * Find existing tag resource or create and register a new one. 4509 * 4510 * @param dev[in, out] 4511 * Pointer to rte_eth_dev structure. 4512 * @param[in, out] resource 4513 * Pointer to tag resource. 4514 * @parm[in, out] dev_flow 4515 * Pointer to the dev_flow. 4516 * @param[out] error 4517 * pointer to error structure. 4518 * 4519 * @return 4520 * 0 on success otherwise -errno and errno is set. 4521 */ 4522 static int 4523 flow_dv_tag_resource_register 4524 (struct rte_eth_dev *dev, 4525 struct mlx5_flow_dv_tag_resource *resource, 4526 struct mlx5_flow *dev_flow, 4527 struct rte_flow_error *error) 4528 { 4529 struct mlx5_priv *priv = dev->data->dev_private; 4530 struct mlx5_ibv_shared *sh = priv->sh; 4531 struct mlx5_flow_dv_tag_resource *cache_resource; 4532 4533 /* Lookup a matching resource from cache. */ 4534 LIST_FOREACH(cache_resource, &sh->tags, next) { 4535 if (resource->tag == cache_resource->tag) { 4536 DRV_LOG(DEBUG, "tag resource %p: refcnt %d++", 4537 (void *)cache_resource, 4538 rte_atomic32_read(&cache_resource->refcnt)); 4539 rte_atomic32_inc(&cache_resource->refcnt); 4540 dev_flow->flow->tag_resource = cache_resource; 4541 return 0; 4542 } 4543 } 4544 /* Register new resource. */ 4545 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 4546 if (!cache_resource) 4547 return rte_flow_error_set(error, ENOMEM, 4548 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4549 "cannot allocate resource memory"); 4550 *cache_resource = *resource; 4551 cache_resource->action = mlx5_glue->dv_create_flow_action_tag 4552 (resource->tag); 4553 if (!cache_resource->action) { 4554 rte_free(cache_resource); 4555 return rte_flow_error_set(error, ENOMEM, 4556 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4557 NULL, "cannot create action"); 4558 } 4559 rte_atomic32_init(&cache_resource->refcnt); 4560 rte_atomic32_inc(&cache_resource->refcnt); 4561 LIST_INSERT_HEAD(&sh->tags, cache_resource, next); 4562 dev_flow->flow->tag_resource = cache_resource; 4563 DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++", 4564 (void *)cache_resource, 4565 rte_atomic32_read(&cache_resource->refcnt)); 4566 return 0; 4567 } 4568 4569 /** 4570 * Release the tag. 4571 * 4572 * @param dev 4573 * Pointer to Ethernet device. 4574 * @param flow 4575 * Pointer to mlx5_flow. 4576 * 4577 * @return 4578 * 1 while a reference on it exists, 0 when freed. 4579 */ 4580 static int 4581 flow_dv_tag_release(struct rte_eth_dev *dev, 4582 struct mlx5_flow_dv_tag_resource *tag) 4583 { 4584 assert(tag); 4585 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--", 4586 dev->data->port_id, (void *)tag, 4587 rte_atomic32_read(&tag->refcnt)); 4588 if (rte_atomic32_dec_and_test(&tag->refcnt)) { 4589 claim_zero(mlx5_glue->destroy_flow_action(tag->action)); 4590 LIST_REMOVE(tag, next); 4591 DRV_LOG(DEBUG, "port %u tag %p: removed", 4592 dev->data->port_id, (void *)tag); 4593 rte_free(tag); 4594 return 0; 4595 } 4596 return 1; 4597 } 4598 4599 /** 4600 * Translate port ID action to vport. 4601 * 4602 * @param[in] dev 4603 * Pointer to rte_eth_dev structure. 4604 * @param[in] action 4605 * Pointer to the port ID action. 4606 * @param[out] dst_port_id 4607 * The target port ID. 4608 * @param[out] error 4609 * Pointer to the error structure. 4610 * 4611 * @return 4612 * 0 on success, a negative errno value otherwise and rte_errno is set. 4613 */ 4614 static int 4615 flow_dv_translate_action_port_id(struct rte_eth_dev *dev, 4616 const struct rte_flow_action *action, 4617 uint32_t *dst_port_id, 4618 struct rte_flow_error *error) 4619 { 4620 uint32_t port; 4621 uint16_t port_id; 4622 int ret; 4623 const struct rte_flow_action_port_id *conf = 4624 (const struct rte_flow_action_port_id *)action->conf; 4625 4626 port = conf->original ? dev->data->port_id : conf->id; 4627 ret = mlx5_port_to_eswitch_info(port, NULL, &port_id); 4628 if (ret) 4629 return rte_flow_error_set(error, -ret, 4630 RTE_FLOW_ERROR_TYPE_ACTION, 4631 NULL, 4632 "No eswitch info was found for port"); 4633 *dst_port_id = port_id; 4634 return 0; 4635 } 4636 4637 /** 4638 * Fill the flow with DV spec. 4639 * 4640 * @param[in] dev 4641 * Pointer to rte_eth_dev structure. 4642 * @param[in, out] dev_flow 4643 * Pointer to the sub flow. 4644 * @param[in] attr 4645 * Pointer to the flow attributes. 4646 * @param[in] items 4647 * Pointer to the list of items. 4648 * @param[in] actions 4649 * Pointer to the list of actions. 4650 * @param[out] error 4651 * Pointer to the error structure. 4652 * 4653 * @return 4654 * 0 on success, a negative errno value otherwise and rte_errno is set. 4655 */ 4656 static int 4657 flow_dv_translate(struct rte_eth_dev *dev, 4658 struct mlx5_flow *dev_flow, 4659 const struct rte_flow_attr *attr, 4660 const struct rte_flow_item items[], 4661 const struct rte_flow_action actions[], 4662 struct rte_flow_error *error) 4663 { 4664 struct mlx5_priv *priv = dev->data->dev_private; 4665 struct rte_flow *flow = dev_flow->flow; 4666 uint64_t item_flags = 0; 4667 uint64_t last_item = 0; 4668 uint64_t action_flags = 0; 4669 uint64_t priority = attr->priority; 4670 struct mlx5_flow_dv_matcher matcher = { 4671 .mask = { 4672 .size = sizeof(matcher.mask.buf), 4673 }, 4674 }; 4675 int actions_n = 0; 4676 bool actions_end = false; 4677 struct mlx5_flow_dv_modify_hdr_resource res = { 4678 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 4679 MLX5DV_FLOW_TABLE_TYPE_NIC_RX 4680 }; 4681 union flow_dv_attr flow_attr = { .attr = 0 }; 4682 struct mlx5_flow_dv_tag_resource tag_resource; 4683 uint32_t modify_action_position = UINT32_MAX; 4684 void *match_mask = matcher.mask.buf; 4685 void *match_value = dev_flow->dv.value.buf; 4686 4687 flow->group = attr->group; 4688 if (attr->transfer) 4689 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; 4690 if (priority == MLX5_FLOW_PRIO_RSVD) 4691 priority = priv->config.flow_prio - 1; 4692 for (; !actions_end ; actions++) { 4693 const struct rte_flow_action_queue *queue; 4694 const struct rte_flow_action_rss *rss; 4695 const struct rte_flow_action *action = actions; 4696 const struct rte_flow_action_count *count = action->conf; 4697 const uint8_t *rss_key; 4698 const struct rte_flow_action_jump *jump_data; 4699 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource; 4700 struct mlx5_flow_tbl_resource *tbl; 4701 uint32_t port_id = 0; 4702 struct mlx5_flow_dv_port_id_action_resource port_id_resource; 4703 4704 switch (actions->type) { 4705 case RTE_FLOW_ACTION_TYPE_VOID: 4706 break; 4707 case RTE_FLOW_ACTION_TYPE_PORT_ID: 4708 if (flow_dv_translate_action_port_id(dev, action, 4709 &port_id, error)) 4710 return -rte_errno; 4711 port_id_resource.port_id = port_id; 4712 if (flow_dv_port_id_action_resource_register 4713 (dev, &port_id_resource, dev_flow, error)) 4714 return -rte_errno; 4715 dev_flow->dv.actions[actions_n++] = 4716 dev_flow->dv.port_id_action->action; 4717 action_flags |= MLX5_FLOW_ACTION_PORT_ID; 4718 break; 4719 case RTE_FLOW_ACTION_TYPE_FLAG: 4720 tag_resource.tag = 4721 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 4722 if (!flow->tag_resource) 4723 if (flow_dv_tag_resource_register 4724 (dev, &tag_resource, dev_flow, error)) 4725 return errno; 4726 dev_flow->dv.actions[actions_n++] = 4727 flow->tag_resource->action; 4728 action_flags |= MLX5_FLOW_ACTION_FLAG; 4729 break; 4730 case RTE_FLOW_ACTION_TYPE_MARK: 4731 tag_resource.tag = mlx5_flow_mark_set 4732 (((const struct rte_flow_action_mark *) 4733 (actions->conf))->id); 4734 if (!flow->tag_resource) 4735 if (flow_dv_tag_resource_register 4736 (dev, &tag_resource, dev_flow, error)) 4737 return errno; 4738 dev_flow->dv.actions[actions_n++] = 4739 flow->tag_resource->action; 4740 action_flags |= MLX5_FLOW_ACTION_MARK; 4741 break; 4742 case RTE_FLOW_ACTION_TYPE_DROP: 4743 action_flags |= MLX5_FLOW_ACTION_DROP; 4744 break; 4745 case RTE_FLOW_ACTION_TYPE_QUEUE: 4746 queue = actions->conf; 4747 flow->rss.queue_num = 1; 4748 (*flow->queue)[0] = queue->index; 4749 action_flags |= MLX5_FLOW_ACTION_QUEUE; 4750 break; 4751 case RTE_FLOW_ACTION_TYPE_RSS: 4752 rss = actions->conf; 4753 if (flow->queue) 4754 memcpy((*flow->queue), rss->queue, 4755 rss->queue_num * sizeof(uint16_t)); 4756 flow->rss.queue_num = rss->queue_num; 4757 /* NULL RSS key indicates default RSS key. */ 4758 rss_key = !rss->key ? rss_hash_default_key : rss->key; 4759 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 4760 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */ 4761 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 4762 flow->rss.level = rss->level; 4763 action_flags |= MLX5_FLOW_ACTION_RSS; 4764 break; 4765 case RTE_FLOW_ACTION_TYPE_COUNT: 4766 if (!priv->config.devx) { 4767 rte_errno = ENOTSUP; 4768 goto cnt_err; 4769 } 4770 flow->counter = flow_dv_counter_alloc(dev, 4771 count->shared, 4772 count->id, 4773 attr->group); 4774 if (flow->counter == NULL) 4775 goto cnt_err; 4776 dev_flow->dv.actions[actions_n++] = 4777 flow->counter->action; 4778 action_flags |= MLX5_FLOW_ACTION_COUNT; 4779 break; 4780 cnt_err: 4781 if (rte_errno == ENOTSUP) 4782 return rte_flow_error_set 4783 (error, ENOTSUP, 4784 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4785 NULL, 4786 "count action not supported"); 4787 else 4788 return rte_flow_error_set 4789 (error, rte_errno, 4790 RTE_FLOW_ERROR_TYPE_ACTION, 4791 action, 4792 "cannot create counter" 4793 " object."); 4794 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4795 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4796 if (flow_dv_create_action_l2_encap(dev, actions, 4797 dev_flow, 4798 attr->transfer, 4799 error)) 4800 return -rte_errno; 4801 dev_flow->dv.actions[actions_n++] = 4802 dev_flow->dv.encap_decap->verbs_action; 4803 action_flags |= actions->type == 4804 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? 4805 MLX5_FLOW_ACTION_VXLAN_ENCAP : 4806 MLX5_FLOW_ACTION_NVGRE_ENCAP; 4807 break; 4808 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4809 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4810 if (flow_dv_create_action_l2_decap(dev, dev_flow, 4811 attr->transfer, 4812 error)) 4813 return -rte_errno; 4814 dev_flow->dv.actions[actions_n++] = 4815 dev_flow->dv.encap_decap->verbs_action; 4816 action_flags |= actions->type == 4817 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? 4818 MLX5_FLOW_ACTION_VXLAN_DECAP : 4819 MLX5_FLOW_ACTION_NVGRE_DECAP; 4820 break; 4821 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4822 /* Handle encap with preceding decap. */ 4823 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) { 4824 if (flow_dv_create_action_raw_encap 4825 (dev, actions, dev_flow, attr, error)) 4826 return -rte_errno; 4827 dev_flow->dv.actions[actions_n++] = 4828 dev_flow->dv.encap_decap->verbs_action; 4829 } else { 4830 /* Handle encap without preceding decap. */ 4831 if (flow_dv_create_action_l2_encap 4832 (dev, actions, dev_flow, attr->transfer, 4833 error)) 4834 return -rte_errno; 4835 dev_flow->dv.actions[actions_n++] = 4836 dev_flow->dv.encap_decap->verbs_action; 4837 } 4838 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; 4839 break; 4840 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4841 /* Check if this decap is followed by encap. */ 4842 for (; action->type != RTE_FLOW_ACTION_TYPE_END && 4843 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; 4844 action++) { 4845 } 4846 /* Handle decap only if it isn't followed by encap. */ 4847 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 4848 if (flow_dv_create_action_l2_decap 4849 (dev, dev_flow, attr->transfer, error)) 4850 return -rte_errno; 4851 dev_flow->dv.actions[actions_n++] = 4852 dev_flow->dv.encap_decap->verbs_action; 4853 } 4854 /* If decap is followed by encap, handle it at encap. */ 4855 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; 4856 break; 4857 case RTE_FLOW_ACTION_TYPE_JUMP: 4858 jump_data = action->conf; 4859 tbl = flow_dv_tbl_resource_get(dev, jump_data->group * 4860 MLX5_GROUP_FACTOR, 4861 attr->egress, 4862 attr->transfer, error); 4863 if (!tbl) 4864 return rte_flow_error_set 4865 (error, errno, 4866 RTE_FLOW_ERROR_TYPE_ACTION, 4867 NULL, 4868 "cannot create jump action."); 4869 jump_tbl_resource.tbl = tbl; 4870 if (flow_dv_jump_tbl_resource_register 4871 (dev, &jump_tbl_resource, dev_flow, error)) { 4872 flow_dv_tbl_resource_release(tbl); 4873 return rte_flow_error_set 4874 (error, errno, 4875 RTE_FLOW_ERROR_TYPE_ACTION, 4876 NULL, 4877 "cannot create jump action."); 4878 } 4879 dev_flow->dv.actions[actions_n++] = 4880 dev_flow->dv.jump->action; 4881 action_flags |= MLX5_FLOW_ACTION_JUMP; 4882 break; 4883 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4884 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4885 if (flow_dv_convert_action_modify_mac(&res, actions, 4886 error)) 4887 return -rte_errno; 4888 action_flags |= actions->type == 4889 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ? 4890 MLX5_FLOW_ACTION_SET_MAC_SRC : 4891 MLX5_FLOW_ACTION_SET_MAC_DST; 4892 break; 4893 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4894 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4895 if (flow_dv_convert_action_modify_ipv4(&res, actions, 4896 error)) 4897 return -rte_errno; 4898 action_flags |= actions->type == 4899 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ? 4900 MLX5_FLOW_ACTION_SET_IPV4_SRC : 4901 MLX5_FLOW_ACTION_SET_IPV4_DST; 4902 break; 4903 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4904 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4905 if (flow_dv_convert_action_modify_ipv6(&res, actions, 4906 error)) 4907 return -rte_errno; 4908 action_flags |= actions->type == 4909 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ? 4910 MLX5_FLOW_ACTION_SET_IPV6_SRC : 4911 MLX5_FLOW_ACTION_SET_IPV6_DST; 4912 break; 4913 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4914 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4915 if (flow_dv_convert_action_modify_tp(&res, actions, 4916 items, &flow_attr, 4917 error)) 4918 return -rte_errno; 4919 action_flags |= actions->type == 4920 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ? 4921 MLX5_FLOW_ACTION_SET_TP_SRC : 4922 MLX5_FLOW_ACTION_SET_TP_DST; 4923 break; 4924 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4925 if (flow_dv_convert_action_modify_dec_ttl(&res, items, 4926 &flow_attr, 4927 error)) 4928 return -rte_errno; 4929 action_flags |= MLX5_FLOW_ACTION_DEC_TTL; 4930 break; 4931 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4932 if (flow_dv_convert_action_modify_ttl(&res, actions, 4933 items, &flow_attr, 4934 error)) 4935 return -rte_errno; 4936 action_flags |= MLX5_FLOW_ACTION_SET_TTL; 4937 break; 4938 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4939 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4940 if (flow_dv_convert_action_modify_tcp_seq(&res, actions, 4941 error)) 4942 return -rte_errno; 4943 action_flags |= actions->type == 4944 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ? 4945 MLX5_FLOW_ACTION_INC_TCP_SEQ : 4946 MLX5_FLOW_ACTION_DEC_TCP_SEQ; 4947 break; 4948 4949 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4950 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4951 if (flow_dv_convert_action_modify_tcp_ack(&res, actions, 4952 error)) 4953 return -rte_errno; 4954 action_flags |= actions->type == 4955 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ? 4956 MLX5_FLOW_ACTION_INC_TCP_ACK : 4957 MLX5_FLOW_ACTION_DEC_TCP_ACK; 4958 break; 4959 case RTE_FLOW_ACTION_TYPE_END: 4960 actions_end = true; 4961 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) { 4962 /* create modify action if needed. */ 4963 if (flow_dv_modify_hdr_resource_register 4964 (dev, &res, 4965 dev_flow, 4966 error)) 4967 return -rte_errno; 4968 dev_flow->dv.actions[modify_action_position] = 4969 dev_flow->dv.modify_hdr->verbs_action; 4970 } 4971 break; 4972 default: 4973 break; 4974 } 4975 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) && 4976 modify_action_position == UINT32_MAX) 4977 modify_action_position = actions_n++; 4978 } 4979 dev_flow->dv.actions_n = actions_n; 4980 flow->actions = action_flags; 4981 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 4982 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 4983 4984 switch (items->type) { 4985 case RTE_FLOW_ITEM_TYPE_PORT_ID: 4986 flow_dv_translate_item_port_id(dev, match_mask, 4987 match_value, items); 4988 last_item = MLX5_FLOW_ITEM_PORT_ID; 4989 break; 4990 case RTE_FLOW_ITEM_TYPE_ETH: 4991 flow_dv_translate_item_eth(match_mask, match_value, 4992 items, tunnel); 4993 matcher.priority = MLX5_PRIORITY_MAP_L2; 4994 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 4995 MLX5_FLOW_LAYER_OUTER_L2; 4996 break; 4997 case RTE_FLOW_ITEM_TYPE_VLAN: 4998 flow_dv_translate_item_vlan(match_mask, match_value, 4999 items, tunnel); 5000 matcher.priority = MLX5_PRIORITY_MAP_L2; 5001 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 5002 MLX5_FLOW_LAYER_INNER_VLAN) : 5003 (MLX5_FLOW_LAYER_OUTER_L2 | 5004 MLX5_FLOW_LAYER_OUTER_VLAN); 5005 break; 5006 case RTE_FLOW_ITEM_TYPE_IPV4: 5007 flow_dv_translate_item_ipv4(match_mask, match_value, 5008 items, tunnel, attr->group); 5009 matcher.priority = MLX5_PRIORITY_MAP_L3; 5010 dev_flow->dv.hash_fields |= 5011 mlx5_flow_hashfields_adjust 5012 (dev_flow, tunnel, 5013 MLX5_IPV4_LAYER_TYPES, 5014 MLX5_IPV4_IBV_RX_HASH); 5015 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 5016 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 5017 mlx5_flow_tunnel_ip_check(items, &last_item); 5018 break; 5019 case RTE_FLOW_ITEM_TYPE_IPV6: 5020 flow_dv_translate_item_ipv6(match_mask, match_value, 5021 items, tunnel, attr->group); 5022 matcher.priority = MLX5_PRIORITY_MAP_L3; 5023 dev_flow->dv.hash_fields |= 5024 mlx5_flow_hashfields_adjust 5025 (dev_flow, tunnel, 5026 MLX5_IPV6_LAYER_TYPES, 5027 MLX5_IPV6_IBV_RX_HASH); 5028 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 5029 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 5030 mlx5_flow_tunnel_ip_check(items, &last_item); 5031 break; 5032 case RTE_FLOW_ITEM_TYPE_TCP: 5033 flow_dv_translate_item_tcp(match_mask, match_value, 5034 items, tunnel); 5035 matcher.priority = MLX5_PRIORITY_MAP_L4; 5036 dev_flow->dv.hash_fields |= 5037 mlx5_flow_hashfields_adjust 5038 (dev_flow, tunnel, ETH_RSS_TCP, 5039 IBV_RX_HASH_SRC_PORT_TCP | 5040 IBV_RX_HASH_DST_PORT_TCP); 5041 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 5042 MLX5_FLOW_LAYER_OUTER_L4_TCP; 5043 break; 5044 case RTE_FLOW_ITEM_TYPE_UDP: 5045 flow_dv_translate_item_udp(match_mask, match_value, 5046 items, tunnel); 5047 matcher.priority = MLX5_PRIORITY_MAP_L4; 5048 dev_flow->dv.hash_fields |= 5049 mlx5_flow_hashfields_adjust 5050 (dev_flow, tunnel, ETH_RSS_UDP, 5051 IBV_RX_HASH_SRC_PORT_UDP | 5052 IBV_RX_HASH_DST_PORT_UDP); 5053 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 5054 MLX5_FLOW_LAYER_OUTER_L4_UDP; 5055 break; 5056 case RTE_FLOW_ITEM_TYPE_GRE: 5057 flow_dv_translate_item_gre(match_mask, match_value, 5058 items, tunnel); 5059 last_item = MLX5_FLOW_LAYER_GRE; 5060 break; 5061 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 5062 flow_dv_translate_item_gre_key(match_mask, 5063 match_value, items); 5064 last_item = MLX5_FLOW_LAYER_GRE_KEY; 5065 break; 5066 case RTE_FLOW_ITEM_TYPE_NVGRE: 5067 flow_dv_translate_item_nvgre(match_mask, match_value, 5068 items, tunnel); 5069 last_item = MLX5_FLOW_LAYER_GRE; 5070 break; 5071 case RTE_FLOW_ITEM_TYPE_VXLAN: 5072 flow_dv_translate_item_vxlan(match_mask, match_value, 5073 items, tunnel); 5074 last_item = MLX5_FLOW_LAYER_VXLAN; 5075 break; 5076 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 5077 flow_dv_translate_item_vxlan(match_mask, match_value, 5078 items, tunnel); 5079 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 5080 break; 5081 case RTE_FLOW_ITEM_TYPE_MPLS: 5082 flow_dv_translate_item_mpls(match_mask, match_value, 5083 items, last_item, tunnel); 5084 last_item = MLX5_FLOW_LAYER_MPLS; 5085 break; 5086 case RTE_FLOW_ITEM_TYPE_META: 5087 flow_dv_translate_item_meta(match_mask, match_value, 5088 items); 5089 last_item = MLX5_FLOW_ITEM_METADATA; 5090 break; 5091 case RTE_FLOW_ITEM_TYPE_ICMP: 5092 flow_dv_translate_item_icmp(match_mask, match_value, 5093 items, tunnel); 5094 last_item = MLX5_FLOW_LAYER_ICMP; 5095 break; 5096 case RTE_FLOW_ITEM_TYPE_ICMP6: 5097 flow_dv_translate_item_icmp6(match_mask, match_value, 5098 items, tunnel); 5099 last_item = MLX5_FLOW_LAYER_ICMP6; 5100 break; 5101 default: 5102 break; 5103 } 5104 item_flags |= last_item; 5105 } 5106 /* 5107 * In case of ingress traffic when E-Switch mode is enabled, 5108 * we have two cases where we need to set the source port manually. 5109 * The first one, is in case of Nic steering rule, and the second is 5110 * E-Switch rule where no port_id item was found. In both cases 5111 * the source port is set according the current port in use. 5112 */ 5113 if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) && 5114 (priv->representor || priv->master)) { 5115 if (flow_dv_translate_item_port_id(dev, match_mask, 5116 match_value, NULL)) 5117 return -rte_errno; 5118 } 5119 assert(!flow_dv_check_valid_spec(matcher.mask.buf, 5120 dev_flow->dv.value.buf)); 5121 dev_flow->layers = item_flags; 5122 /* Register matcher. */ 5123 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 5124 matcher.mask.size); 5125 matcher.priority = mlx5_flow_adjust_priority(dev, priority, 5126 matcher.priority); 5127 matcher.egress = attr->egress; 5128 matcher.group = attr->group; 5129 matcher.transfer = attr->transfer; 5130 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) 5131 return -rte_errno; 5132 return 0; 5133 } 5134 5135 /** 5136 * Apply the flow to the NIC. 5137 * 5138 * @param[in] dev 5139 * Pointer to the Ethernet device structure. 5140 * @param[in, out] flow 5141 * Pointer to flow structure. 5142 * @param[out] error 5143 * Pointer to error structure. 5144 * 5145 * @return 5146 * 0 on success, a negative errno value otherwise and rte_errno is set. 5147 */ 5148 static int 5149 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 5150 struct rte_flow_error *error) 5151 { 5152 struct mlx5_flow_dv *dv; 5153 struct mlx5_flow *dev_flow; 5154 struct mlx5_priv *priv = dev->data->dev_private; 5155 int n; 5156 int err; 5157 5158 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 5159 dv = &dev_flow->dv; 5160 n = dv->actions_n; 5161 if (flow->actions & MLX5_FLOW_ACTION_DROP) { 5162 if (flow->transfer) { 5163 dv->actions[n++] = priv->sh->esw_drop_action; 5164 } else { 5165 dv->hrxq = mlx5_hrxq_drop_new(dev); 5166 if (!dv->hrxq) { 5167 rte_flow_error_set 5168 (error, errno, 5169 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5170 NULL, 5171 "cannot get drop hash queue"); 5172 goto error; 5173 } 5174 dv->actions[n++] = dv->hrxq->action; 5175 } 5176 } else if (flow->actions & 5177 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { 5178 struct mlx5_hrxq *hrxq; 5179 5180 hrxq = mlx5_hrxq_get(dev, flow->key, 5181 MLX5_RSS_HASH_KEY_LEN, 5182 dv->hash_fields, 5183 (*flow->queue), 5184 flow->rss.queue_num); 5185 if (!hrxq) { 5186 hrxq = mlx5_hrxq_new 5187 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN, 5188 dv->hash_fields, (*flow->queue), 5189 flow->rss.queue_num, 5190 !!(dev_flow->layers & 5191 MLX5_FLOW_LAYER_TUNNEL)); 5192 } 5193 if (!hrxq) { 5194 rte_flow_error_set 5195 (error, rte_errno, 5196 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5197 "cannot get hash queue"); 5198 goto error; 5199 } 5200 dv->hrxq = hrxq; 5201 dv->actions[n++] = dv->hrxq->action; 5202 } 5203 dv->flow = 5204 mlx5_glue->dv_create_flow(dv->matcher->matcher_object, 5205 (void *)&dv->value, n, 5206 dv->actions); 5207 if (!dv->flow) { 5208 rte_flow_error_set(error, errno, 5209 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5210 NULL, 5211 "hardware refuses to create flow"); 5212 goto error; 5213 } 5214 } 5215 return 0; 5216 error: 5217 err = rte_errno; /* Save rte_errno before cleanup. */ 5218 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 5219 struct mlx5_flow_dv *dv = &dev_flow->dv; 5220 if (dv->hrxq) { 5221 if (flow->actions & MLX5_FLOW_ACTION_DROP) 5222 mlx5_hrxq_drop_release(dev); 5223 else 5224 mlx5_hrxq_release(dev, dv->hrxq); 5225 dv->hrxq = NULL; 5226 } 5227 } 5228 rte_errno = err; /* Restore rte_errno. */ 5229 return -rte_errno; 5230 } 5231 5232 /** 5233 * Release the flow matcher. 5234 * 5235 * @param dev 5236 * Pointer to Ethernet device. 5237 * @param flow 5238 * Pointer to mlx5_flow. 5239 * 5240 * @return 5241 * 1 while a reference on it exists, 0 when freed. 5242 */ 5243 static int 5244 flow_dv_matcher_release(struct rte_eth_dev *dev, 5245 struct mlx5_flow *flow) 5246 { 5247 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; 5248 struct mlx5_priv *priv = dev->data->dev_private; 5249 struct mlx5_ibv_shared *sh = priv->sh; 5250 struct mlx5_flow_tbl_resource *tbl; 5251 5252 assert(matcher->matcher_object); 5253 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", 5254 dev->data->port_id, (void *)matcher, 5255 rte_atomic32_read(&matcher->refcnt)); 5256 if (rte_atomic32_dec_and_test(&matcher->refcnt)) { 5257 claim_zero(mlx5_glue->dv_destroy_flow_matcher 5258 (matcher->matcher_object)); 5259 LIST_REMOVE(matcher, next); 5260 if (matcher->egress) 5261 tbl = &sh->tx_tbl[matcher->group]; 5262 else 5263 tbl = &sh->rx_tbl[matcher->group]; 5264 flow_dv_tbl_resource_release(tbl); 5265 rte_free(matcher); 5266 DRV_LOG(DEBUG, "port %u matcher %p: removed", 5267 dev->data->port_id, (void *)matcher); 5268 return 0; 5269 } 5270 return 1; 5271 } 5272 5273 /** 5274 * Release an encap/decap resource. 5275 * 5276 * @param flow 5277 * Pointer to mlx5_flow. 5278 * 5279 * @return 5280 * 1 while a reference on it exists, 0 when freed. 5281 */ 5282 static int 5283 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) 5284 { 5285 struct mlx5_flow_dv_encap_decap_resource *cache_resource = 5286 flow->dv.encap_decap; 5287 5288 assert(cache_resource->verbs_action); 5289 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", 5290 (void *)cache_resource, 5291 rte_atomic32_read(&cache_resource->refcnt)); 5292 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 5293 claim_zero(mlx5_glue->destroy_flow_action 5294 (cache_resource->verbs_action)); 5295 LIST_REMOVE(cache_resource, next); 5296 rte_free(cache_resource); 5297 DRV_LOG(DEBUG, "encap/decap resource %p: removed", 5298 (void *)cache_resource); 5299 return 0; 5300 } 5301 return 1; 5302 } 5303 5304 /** 5305 * Release an jump to table action resource. 5306 * 5307 * @param flow 5308 * Pointer to mlx5_flow. 5309 * 5310 * @return 5311 * 1 while a reference on it exists, 0 when freed. 5312 */ 5313 static int 5314 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow) 5315 { 5316 struct mlx5_flow_dv_jump_tbl_resource *cache_resource = 5317 flow->dv.jump; 5318 5319 assert(cache_resource->action); 5320 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--", 5321 (void *)cache_resource, 5322 rte_atomic32_read(&cache_resource->refcnt)); 5323 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 5324 claim_zero(mlx5_glue->destroy_flow_action 5325 (cache_resource->action)); 5326 LIST_REMOVE(cache_resource, next); 5327 flow_dv_tbl_resource_release(cache_resource->tbl); 5328 rte_free(cache_resource); 5329 DRV_LOG(DEBUG, "jump table resource %p: removed", 5330 (void *)cache_resource); 5331 return 0; 5332 } 5333 return 1; 5334 } 5335 5336 /** 5337 * Release a modify-header resource. 5338 * 5339 * @param flow 5340 * Pointer to mlx5_flow. 5341 * 5342 * @return 5343 * 1 while a reference on it exists, 0 when freed. 5344 */ 5345 static int 5346 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow) 5347 { 5348 struct mlx5_flow_dv_modify_hdr_resource *cache_resource = 5349 flow->dv.modify_hdr; 5350 5351 assert(cache_resource->verbs_action); 5352 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--", 5353 (void *)cache_resource, 5354 rte_atomic32_read(&cache_resource->refcnt)); 5355 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 5356 claim_zero(mlx5_glue->destroy_flow_action 5357 (cache_resource->verbs_action)); 5358 LIST_REMOVE(cache_resource, next); 5359 rte_free(cache_resource); 5360 DRV_LOG(DEBUG, "modify-header resource %p: removed", 5361 (void *)cache_resource); 5362 return 0; 5363 } 5364 return 1; 5365 } 5366 5367 /** 5368 * Release port ID action resource. 5369 * 5370 * @param flow 5371 * Pointer to mlx5_flow. 5372 * 5373 * @return 5374 * 1 while a reference on it exists, 0 when freed. 5375 */ 5376 static int 5377 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow) 5378 { 5379 struct mlx5_flow_dv_port_id_action_resource *cache_resource = 5380 flow->dv.port_id_action; 5381 5382 assert(cache_resource->action); 5383 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--", 5384 (void *)cache_resource, 5385 rte_atomic32_read(&cache_resource->refcnt)); 5386 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 5387 claim_zero(mlx5_glue->destroy_flow_action 5388 (cache_resource->action)); 5389 LIST_REMOVE(cache_resource, next); 5390 rte_free(cache_resource); 5391 DRV_LOG(DEBUG, "port id action resource %p: removed", 5392 (void *)cache_resource); 5393 return 0; 5394 } 5395 return 1; 5396 } 5397 5398 /** 5399 * Remove the flow from the NIC but keeps it in memory. 5400 * 5401 * @param[in] dev 5402 * Pointer to Ethernet device. 5403 * @param[in, out] flow 5404 * Pointer to flow structure. 5405 */ 5406 static void 5407 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 5408 { 5409 struct mlx5_flow_dv *dv; 5410 struct mlx5_flow *dev_flow; 5411 5412 if (!flow) 5413 return; 5414 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 5415 dv = &dev_flow->dv; 5416 if (dv->flow) { 5417 claim_zero(mlx5_glue->dv_destroy_flow(dv->flow)); 5418 dv->flow = NULL; 5419 } 5420 if (dv->hrxq) { 5421 if (flow->actions & MLX5_FLOW_ACTION_DROP) 5422 mlx5_hrxq_drop_release(dev); 5423 else 5424 mlx5_hrxq_release(dev, dv->hrxq); 5425 dv->hrxq = NULL; 5426 } 5427 } 5428 } 5429 5430 /** 5431 * Remove the flow from the NIC and the memory. 5432 * 5433 * @param[in] dev 5434 * Pointer to the Ethernet device structure. 5435 * @param[in, out] flow 5436 * Pointer to flow structure. 5437 */ 5438 static void 5439 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 5440 { 5441 struct mlx5_flow *dev_flow; 5442 5443 if (!flow) 5444 return; 5445 flow_dv_remove(dev, flow); 5446 if (flow->counter) { 5447 flow_dv_counter_release(dev, flow->counter); 5448 flow->counter = NULL; 5449 } 5450 if (flow->tag_resource) { 5451 flow_dv_tag_release(dev, flow->tag_resource); 5452 flow->tag_resource = NULL; 5453 } 5454 while (!LIST_EMPTY(&flow->dev_flows)) { 5455 dev_flow = LIST_FIRST(&flow->dev_flows); 5456 LIST_REMOVE(dev_flow, next); 5457 if (dev_flow->dv.matcher) 5458 flow_dv_matcher_release(dev, dev_flow); 5459 if (dev_flow->dv.encap_decap) 5460 flow_dv_encap_decap_resource_release(dev_flow); 5461 if (dev_flow->dv.modify_hdr) 5462 flow_dv_modify_hdr_resource_release(dev_flow); 5463 if (dev_flow->dv.jump) 5464 flow_dv_jump_tbl_resource_release(dev_flow); 5465 if (dev_flow->dv.port_id_action) 5466 flow_dv_port_id_action_resource_release(dev_flow); 5467 rte_free(dev_flow); 5468 } 5469 } 5470 5471 /** 5472 * Query a dv flow rule for its statistics via devx. 5473 * 5474 * @param[in] dev 5475 * Pointer to Ethernet device. 5476 * @param[in] flow 5477 * Pointer to the sub flow. 5478 * @param[out] data 5479 * data retrieved by the query. 5480 * @param[out] error 5481 * Perform verbose error reporting if not NULL. 5482 * 5483 * @return 5484 * 0 on success, a negative errno value otherwise and rte_errno is set. 5485 */ 5486 static int 5487 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow, 5488 void *data, struct rte_flow_error *error) 5489 { 5490 struct mlx5_priv *priv = dev->data->dev_private; 5491 struct rte_flow_query_count *qc = data; 5492 5493 if (!priv->config.devx) 5494 return rte_flow_error_set(error, ENOTSUP, 5495 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5496 NULL, 5497 "counters are not supported"); 5498 if (flow->counter) { 5499 uint64_t pkts, bytes; 5500 int err = _flow_dv_query_count(dev, flow->counter, &pkts, 5501 &bytes); 5502 5503 if (err) 5504 return rte_flow_error_set(error, -err, 5505 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5506 NULL, "cannot read counters"); 5507 qc->hits_set = 1; 5508 qc->bytes_set = 1; 5509 qc->hits = pkts - flow->counter->hits; 5510 qc->bytes = bytes - flow->counter->bytes; 5511 if (qc->reset) { 5512 flow->counter->hits = pkts; 5513 flow->counter->bytes = bytes; 5514 } 5515 return 0; 5516 } 5517 return rte_flow_error_set(error, EINVAL, 5518 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5519 NULL, 5520 "counters are not available"); 5521 } 5522 5523 /** 5524 * Query a flow. 5525 * 5526 * @see rte_flow_query() 5527 * @see rte_flow_ops 5528 */ 5529 static int 5530 flow_dv_query(struct rte_eth_dev *dev, 5531 struct rte_flow *flow __rte_unused, 5532 const struct rte_flow_action *actions __rte_unused, 5533 void *data __rte_unused, 5534 struct rte_flow_error *error __rte_unused) 5535 { 5536 int ret = -EINVAL; 5537 5538 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5539 switch (actions->type) { 5540 case RTE_FLOW_ACTION_TYPE_VOID: 5541 break; 5542 case RTE_FLOW_ACTION_TYPE_COUNT: 5543 ret = flow_dv_query_count(dev, flow, data, error); 5544 break; 5545 default: 5546 return rte_flow_error_set(error, ENOTSUP, 5547 RTE_FLOW_ERROR_TYPE_ACTION, 5548 actions, 5549 "action not supported"); 5550 } 5551 } 5552 return ret; 5553 } 5554 5555 /* 5556 * Mutex-protected thunk to flow_dv_translate(). 5557 */ 5558 static int 5559 flow_d_translate(struct rte_eth_dev *dev, 5560 struct mlx5_flow *dev_flow, 5561 const struct rte_flow_attr *attr, 5562 const struct rte_flow_item items[], 5563 const struct rte_flow_action actions[], 5564 struct rte_flow_error *error) 5565 { 5566 int ret; 5567 5568 flow_d_shared_lock(dev); 5569 ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error); 5570 flow_d_shared_unlock(dev); 5571 return ret; 5572 } 5573 5574 /* 5575 * Mutex-protected thunk to flow_dv_apply(). 5576 */ 5577 static int 5578 flow_d_apply(struct rte_eth_dev *dev, 5579 struct rte_flow *flow, 5580 struct rte_flow_error *error) 5581 { 5582 int ret; 5583 5584 flow_d_shared_lock(dev); 5585 ret = flow_dv_apply(dev, flow, error); 5586 flow_d_shared_unlock(dev); 5587 return ret; 5588 } 5589 5590 /* 5591 * Mutex-protected thunk to flow_dv_remove(). 5592 */ 5593 static void 5594 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 5595 { 5596 flow_d_shared_lock(dev); 5597 flow_dv_remove(dev, flow); 5598 flow_d_shared_unlock(dev); 5599 } 5600 5601 /* 5602 * Mutex-protected thunk to flow_dv_destroy(). 5603 */ 5604 static void 5605 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 5606 { 5607 flow_d_shared_lock(dev); 5608 flow_dv_destroy(dev, flow); 5609 flow_d_shared_unlock(dev); 5610 } 5611 5612 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 5613 .validate = flow_dv_validate, 5614 .prepare = flow_dv_prepare, 5615 .translate = flow_d_translate, 5616 .apply = flow_d_apply, 5617 .remove = flow_d_remove, 5618 .destroy = flow_d_destroy, 5619 .query = flow_dv_query, 5620 }; 5621 5622 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 5623