1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #ifdef PEDANTIC 17 #pragma GCC diagnostic error "-Wpedantic" 18 #endif 19 20 #include <rte_common.h> 21 #include <rte_ether.h> 22 #include <rte_eth_ctrl.h> 23 #include <rte_ethdev_driver.h> 24 #include <rte_flow.h> 25 #include <rte_flow_driver.h> 26 #include <rte_malloc.h> 27 #include <rte_ip.h> 28 #include <rte_gre.h> 29 30 #include "mlx5.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_prm.h" 33 #include "mlx5_glue.h" 34 #include "mlx5_flow.h" 35 36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 37 38 /** 39 * Validate META item. 40 * 41 * @param[in] dev 42 * Pointer to the rte_eth_dev structure. 43 * @param[in] item 44 * Item specification. 45 * @param[in] attr 46 * Attributes of flow that includes this item. 47 * @param[out] error 48 * Pointer to error structure. 49 * 50 * @return 51 * 0 on success, a negative errno value otherwise and rte_errno is set. 52 */ 53 static int 54 flow_dv_validate_item_meta(struct rte_eth_dev *dev, 55 const struct rte_flow_item *item, 56 const struct rte_flow_attr *attr, 57 struct rte_flow_error *error) 58 { 59 const struct rte_flow_item_meta *spec = item->spec; 60 const struct rte_flow_item_meta *mask = item->mask; 61 const struct rte_flow_item_meta nic_mask = { 62 .data = RTE_BE32(UINT32_MAX) 63 }; 64 int ret; 65 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 66 67 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA)) 68 return rte_flow_error_set(error, EPERM, 69 RTE_FLOW_ERROR_TYPE_ITEM, 70 NULL, 71 "match on metadata offload " 72 "configuration is off for this port"); 73 if (!spec) 74 return rte_flow_error_set(error, EINVAL, 75 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 76 item->spec, 77 "data cannot be empty"); 78 if (!spec->data) 79 return rte_flow_error_set(error, EINVAL, 80 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 81 NULL, 82 "data cannot be zero"); 83 if (!mask) 84 mask = &rte_flow_item_meta_mask; 85 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 86 (const uint8_t *)&nic_mask, 87 sizeof(struct rte_flow_item_meta), 88 error); 89 if (ret < 0) 90 return ret; 91 if (attr->ingress) 92 return rte_flow_error_set(error, ENOTSUP, 93 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 94 NULL, 95 "pattern not supported for ingress"); 96 return 0; 97 } 98 99 /** 100 * Validate the L2 encap action. 101 * 102 * @param[in] action_flags 103 * Holds the actions detected until now. 104 * @param[in] action 105 * Pointer to the encap action. 106 * @param[in] attr 107 * Pointer to flow attributes 108 * @param[out] error 109 * Pointer to error structure. 110 * 111 * @return 112 * 0 on success, a negative errno value otherwise and rte_errno is set. 113 */ 114 static int 115 flow_dv_validate_action_l2_encap(uint64_t action_flags, 116 const struct rte_flow_action *action, 117 const struct rte_flow_attr *attr, 118 struct rte_flow_error *error) 119 { 120 if (!(action->conf)) 121 return rte_flow_error_set(error, EINVAL, 122 RTE_FLOW_ERROR_TYPE_ACTION, action, 123 "configuration cannot be null"); 124 if (action_flags & MLX5_FLOW_ACTION_DROP) 125 return rte_flow_error_set(error, EINVAL, 126 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 127 "can't drop and encap in same flow"); 128 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) 129 return rte_flow_error_set(error, EINVAL, 130 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 131 "can only have a single encap or" 132 " decap action in a flow"); 133 if (attr->ingress) 134 return rte_flow_error_set(error, ENOTSUP, 135 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 136 NULL, 137 "encap action not supported for " 138 "ingress"); 139 return 0; 140 } 141 142 /** 143 * Validate the L2 decap action. 144 * 145 * @param[in] action_flags 146 * Holds the actions detected until now. 147 * @param[in] attr 148 * Pointer to flow attributes 149 * @param[out] error 150 * Pointer to error structure. 151 * 152 * @return 153 * 0 on success, a negative errno value otherwise and rte_errno is set. 154 */ 155 static int 156 flow_dv_validate_action_l2_decap(uint64_t action_flags, 157 const struct rte_flow_attr *attr, 158 struct rte_flow_error *error) 159 { 160 if (action_flags & MLX5_FLOW_ACTION_DROP) 161 return rte_flow_error_set(error, EINVAL, 162 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 163 "can't drop and decap in same flow"); 164 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS)) 165 return rte_flow_error_set(error, EINVAL, 166 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 167 "can only have a single encap or" 168 " decap action in a flow"); 169 if (attr->egress) 170 return rte_flow_error_set(error, ENOTSUP, 171 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 172 NULL, 173 "decap action not supported for " 174 "egress"); 175 return 0; 176 } 177 178 /** 179 * Validate the raw encap action. 180 * 181 * @param[in] action_flags 182 * Holds the actions detected until now. 183 * @param[in] action 184 * Pointer to the encap action. 185 * @param[in] attr 186 * Pointer to flow attributes 187 * @param[out] error 188 * Pointer to error structure. 189 * 190 * @return 191 * 0 on success, a negative errno value otherwise and rte_errno is set. 192 */ 193 static int 194 flow_dv_validate_action_raw_encap(uint64_t action_flags, 195 const struct rte_flow_action *action, 196 const struct rte_flow_attr *attr, 197 struct rte_flow_error *error) 198 { 199 if (!(action->conf)) 200 return rte_flow_error_set(error, EINVAL, 201 RTE_FLOW_ERROR_TYPE_ACTION, action, 202 "configuration cannot be null"); 203 if (action_flags & MLX5_FLOW_ACTION_DROP) 204 return rte_flow_error_set(error, EINVAL, 205 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 206 "can't drop and encap in same flow"); 207 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) 208 return rte_flow_error_set(error, EINVAL, 209 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 210 "can only have a single encap" 211 " action in a flow"); 212 /* encap without preceding decap is not supported for ingress */ 213 if (attr->ingress && !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP)) 214 return rte_flow_error_set(error, ENOTSUP, 215 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 216 NULL, 217 "encap action not supported for " 218 "ingress"); 219 return 0; 220 } 221 222 /** 223 * Validate the raw decap action. 224 * 225 * @param[in] action_flags 226 * Holds the actions detected until now. 227 * @param[in] action 228 * Pointer to the encap action. 229 * @param[in] attr 230 * Pointer to flow attributes 231 * @param[out] error 232 * Pointer to error structure. 233 * 234 * @return 235 * 0 on success, a negative errno value otherwise and rte_errno is set. 236 */ 237 static int 238 flow_dv_validate_action_raw_decap(uint64_t action_flags, 239 const struct rte_flow_action *action, 240 const struct rte_flow_attr *attr, 241 struct rte_flow_error *error) 242 { 243 if (action_flags & MLX5_FLOW_ACTION_DROP) 244 return rte_flow_error_set(error, EINVAL, 245 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 246 "can't drop and decap in same flow"); 247 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS) 248 return rte_flow_error_set(error, EINVAL, 249 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 250 "can't have encap action before" 251 " decap action"); 252 if (action_flags & MLX5_FLOW_DECAP_ACTIONS) 253 return rte_flow_error_set(error, EINVAL, 254 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 255 "can only have a single decap" 256 " action in a flow"); 257 /* decap action is valid on egress only if it is followed by encap */ 258 if (attr->egress) { 259 for (; action->type != RTE_FLOW_ACTION_TYPE_END && 260 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; 261 action++) { 262 } 263 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) 264 return rte_flow_error_set 265 (error, ENOTSUP, 266 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, 267 NULL, "decap action not supported" 268 " for egress"); 269 } 270 return 0; 271 } 272 273 274 /** 275 * Find existing encap/decap resource or create and register a new one. 276 * 277 * @param dev[in, out] 278 * Pointer to rte_eth_dev structure. 279 * @param[in, out] resource 280 * Pointer to encap/decap resource. 281 * @parm[in, out] dev_flow 282 * Pointer to the dev_flow. 283 * @param[out] error 284 * pointer to error structure. 285 * 286 * @return 287 * 0 on success otherwise -errno and errno is set. 288 */ 289 static int 290 flow_dv_encap_decap_resource_register 291 (struct rte_eth_dev *dev, 292 struct mlx5_flow_dv_encap_decap_resource *resource, 293 struct mlx5_flow *dev_flow, 294 struct rte_flow_error *error) 295 { 296 struct mlx5_priv *priv = dev->data->dev_private; 297 struct mlx5_flow_dv_encap_decap_resource *cache_resource; 298 299 /* Lookup a matching resource from cache. */ 300 LIST_FOREACH(cache_resource, &priv->encaps_decaps, next) { 301 if (resource->reformat_type == cache_resource->reformat_type && 302 resource->ft_type == cache_resource->ft_type && 303 resource->size == cache_resource->size && 304 !memcmp((const void *)resource->buf, 305 (const void *)cache_resource->buf, 306 resource->size)) { 307 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++", 308 (void *)cache_resource, 309 rte_atomic32_read(&cache_resource->refcnt)); 310 rte_atomic32_inc(&cache_resource->refcnt); 311 dev_flow->dv.encap_decap = cache_resource; 312 return 0; 313 } 314 } 315 /* Register new encap/decap resource. */ 316 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0); 317 if (!cache_resource) 318 return rte_flow_error_set(error, ENOMEM, 319 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 320 "cannot allocate resource memory"); 321 *cache_resource = *resource; 322 cache_resource->verbs_action = 323 mlx5_glue->dv_create_flow_action_packet_reformat 324 (priv->ctx, cache_resource->size, 325 (cache_resource->size ? cache_resource->buf : NULL), 326 cache_resource->reformat_type, 327 cache_resource->ft_type); 328 if (!cache_resource->verbs_action) { 329 rte_free(cache_resource); 330 return rte_flow_error_set(error, ENOMEM, 331 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 332 NULL, "cannot create action"); 333 } 334 rte_atomic32_init(&cache_resource->refcnt); 335 rte_atomic32_inc(&cache_resource->refcnt); 336 LIST_INSERT_HEAD(&priv->encaps_decaps, cache_resource, next); 337 dev_flow->dv.encap_decap = cache_resource; 338 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++", 339 (void *)cache_resource, 340 rte_atomic32_read(&cache_resource->refcnt)); 341 return 0; 342 } 343 344 /** 345 * Get the size of specific rte_flow_item_type 346 * 347 * @param[in] item_type 348 * Tested rte_flow_item_type. 349 * 350 * @return 351 * sizeof struct item_type, 0 if void or irrelevant. 352 */ 353 static size_t 354 flow_dv_get_item_len(const enum rte_flow_item_type item_type) 355 { 356 size_t retval; 357 358 switch (item_type) { 359 case RTE_FLOW_ITEM_TYPE_ETH: 360 retval = sizeof(struct rte_flow_item_eth); 361 break; 362 case RTE_FLOW_ITEM_TYPE_VLAN: 363 retval = sizeof(struct rte_flow_item_vlan); 364 break; 365 case RTE_FLOW_ITEM_TYPE_IPV4: 366 retval = sizeof(struct rte_flow_item_ipv4); 367 break; 368 case RTE_FLOW_ITEM_TYPE_IPV6: 369 retval = sizeof(struct rte_flow_item_ipv6); 370 break; 371 case RTE_FLOW_ITEM_TYPE_UDP: 372 retval = sizeof(struct rte_flow_item_udp); 373 break; 374 case RTE_FLOW_ITEM_TYPE_TCP: 375 retval = sizeof(struct rte_flow_item_tcp); 376 break; 377 case RTE_FLOW_ITEM_TYPE_VXLAN: 378 retval = sizeof(struct rte_flow_item_vxlan); 379 break; 380 case RTE_FLOW_ITEM_TYPE_GRE: 381 retval = sizeof(struct rte_flow_item_gre); 382 break; 383 case RTE_FLOW_ITEM_TYPE_NVGRE: 384 retval = sizeof(struct rte_flow_item_nvgre); 385 break; 386 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 387 retval = sizeof(struct rte_flow_item_vxlan_gpe); 388 break; 389 case RTE_FLOW_ITEM_TYPE_MPLS: 390 retval = sizeof(struct rte_flow_item_mpls); 391 break; 392 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */ 393 default: 394 retval = 0; 395 break; 396 } 397 return retval; 398 } 399 400 #define MLX5_ENCAP_IPV4_VERSION 0x40 401 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05 402 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40 403 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000 404 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff 405 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000 406 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04 407 408 /** 409 * Convert the encap action data from list of rte_flow_item to raw buffer 410 * 411 * @param[in] items 412 * Pointer to rte_flow_item objects list. 413 * @param[out] buf 414 * Pointer to the output buffer. 415 * @param[out] size 416 * Pointer to the output buffer size. 417 * @param[out] error 418 * Pointer to the error structure. 419 * 420 * @return 421 * 0 on success, a negative errno value otherwise and rte_errno is set. 422 */ 423 static int 424 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf, 425 size_t *size, struct rte_flow_error *error) 426 { 427 struct ether_hdr *eth = NULL; 428 struct vlan_hdr *vlan = NULL; 429 struct ipv4_hdr *ipv4 = NULL; 430 struct ipv6_hdr *ipv6 = NULL; 431 struct udp_hdr *udp = NULL; 432 struct vxlan_hdr *vxlan = NULL; 433 struct vxlan_gpe_hdr *vxlan_gpe = NULL; 434 struct gre_hdr *gre = NULL; 435 size_t len; 436 size_t temp_size = 0; 437 438 if (!items) 439 return rte_flow_error_set(error, EINVAL, 440 RTE_FLOW_ERROR_TYPE_ACTION, 441 NULL, "invalid empty data"); 442 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 443 len = flow_dv_get_item_len(items->type); 444 if (len + temp_size > MLX5_ENCAP_MAX_LEN) 445 return rte_flow_error_set(error, EINVAL, 446 RTE_FLOW_ERROR_TYPE_ACTION, 447 (void *)items->type, 448 "items total size is too big" 449 " for encap action"); 450 rte_memcpy((void *)&buf[temp_size], items->spec, len); 451 switch (items->type) { 452 case RTE_FLOW_ITEM_TYPE_ETH: 453 eth = (struct ether_hdr *)&buf[temp_size]; 454 break; 455 case RTE_FLOW_ITEM_TYPE_VLAN: 456 vlan = (struct vlan_hdr *)&buf[temp_size]; 457 if (!eth) 458 return rte_flow_error_set(error, EINVAL, 459 RTE_FLOW_ERROR_TYPE_ACTION, 460 (void *)items->type, 461 "eth header not found"); 462 if (!eth->ether_type) 463 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN); 464 break; 465 case RTE_FLOW_ITEM_TYPE_IPV4: 466 ipv4 = (struct ipv4_hdr *)&buf[temp_size]; 467 if (!vlan && !eth) 468 return rte_flow_error_set(error, EINVAL, 469 RTE_FLOW_ERROR_TYPE_ACTION, 470 (void *)items->type, 471 "neither eth nor vlan" 472 " header found"); 473 if (vlan && !vlan->eth_proto) 474 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4); 475 else if (eth && !eth->ether_type) 476 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4); 477 if (!ipv4->version_ihl) 478 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION | 479 MLX5_ENCAP_IPV4_IHL_MIN; 480 if (!ipv4->time_to_live) 481 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF; 482 break; 483 case RTE_FLOW_ITEM_TYPE_IPV6: 484 ipv6 = (struct ipv6_hdr *)&buf[temp_size]; 485 if (!vlan && !eth) 486 return rte_flow_error_set(error, EINVAL, 487 RTE_FLOW_ERROR_TYPE_ACTION, 488 (void *)items->type, 489 "neither eth nor vlan" 490 " header found"); 491 if (vlan && !vlan->eth_proto) 492 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6); 493 else if (eth && !eth->ether_type) 494 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6); 495 if (!ipv6->vtc_flow) 496 ipv6->vtc_flow = 497 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW); 498 if (!ipv6->hop_limits) 499 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT; 500 break; 501 case RTE_FLOW_ITEM_TYPE_UDP: 502 udp = (struct udp_hdr *)&buf[temp_size]; 503 if (!ipv4 && !ipv6) 504 return rte_flow_error_set(error, EINVAL, 505 RTE_FLOW_ERROR_TYPE_ACTION, 506 (void *)items->type, 507 "ip header not found"); 508 if (ipv4 && !ipv4->next_proto_id) 509 ipv4->next_proto_id = IPPROTO_UDP; 510 else if (ipv6 && !ipv6->proto) 511 ipv6->proto = IPPROTO_UDP; 512 break; 513 case RTE_FLOW_ITEM_TYPE_VXLAN: 514 vxlan = (struct vxlan_hdr *)&buf[temp_size]; 515 if (!udp) 516 return rte_flow_error_set(error, EINVAL, 517 RTE_FLOW_ERROR_TYPE_ACTION, 518 (void *)items->type, 519 "udp header not found"); 520 if (!udp->dst_port) 521 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN); 522 if (!vxlan->vx_flags) 523 vxlan->vx_flags = 524 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS); 525 break; 526 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 527 vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size]; 528 if (!udp) 529 return rte_flow_error_set(error, EINVAL, 530 RTE_FLOW_ERROR_TYPE_ACTION, 531 (void *)items->type, 532 "udp header not found"); 533 if (!vxlan_gpe->proto) 534 return rte_flow_error_set(error, EINVAL, 535 RTE_FLOW_ERROR_TYPE_ACTION, 536 (void *)items->type, 537 "next protocol not found"); 538 if (!udp->dst_port) 539 udp->dst_port = 540 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE); 541 if (!vxlan_gpe->vx_flags) 542 vxlan_gpe->vx_flags = 543 MLX5_ENCAP_VXLAN_GPE_FLAGS; 544 break; 545 case RTE_FLOW_ITEM_TYPE_GRE: 546 case RTE_FLOW_ITEM_TYPE_NVGRE: 547 gre = (struct gre_hdr *)&buf[temp_size]; 548 if (!gre->proto) 549 return rte_flow_error_set(error, EINVAL, 550 RTE_FLOW_ERROR_TYPE_ACTION, 551 (void *)items->type, 552 "next protocol not found"); 553 if (!ipv4 && !ipv6) 554 return rte_flow_error_set(error, EINVAL, 555 RTE_FLOW_ERROR_TYPE_ACTION, 556 (void *)items->type, 557 "ip header not found"); 558 if (ipv4 && !ipv4->next_proto_id) 559 ipv4->next_proto_id = IPPROTO_GRE; 560 else if (ipv6 && !ipv6->proto) 561 ipv6->proto = IPPROTO_GRE; 562 break; 563 case RTE_FLOW_ITEM_TYPE_VOID: 564 break; 565 default: 566 return rte_flow_error_set(error, EINVAL, 567 RTE_FLOW_ERROR_TYPE_ACTION, 568 (void *)items->type, 569 "unsupported item type"); 570 break; 571 } 572 temp_size += len; 573 } 574 *size = temp_size; 575 return 0; 576 } 577 578 /** 579 * Convert L2 encap action to DV specification. 580 * 581 * @param[in] dev 582 * Pointer to rte_eth_dev structure. 583 * @param[in] action 584 * Pointer to action structure. 585 * @param[in, out] dev_flow 586 * Pointer to the mlx5_flow. 587 * @param[out] error 588 * Pointer to the error structure. 589 * 590 * @return 591 * 0 on success, a negative errno value otherwise and rte_errno is set. 592 */ 593 static int 594 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev, 595 const struct rte_flow_action *action, 596 struct mlx5_flow *dev_flow, 597 struct rte_flow_error *error) 598 { 599 const struct rte_flow_item *encap_data; 600 const struct rte_flow_action_raw_encap *raw_encap_data; 601 struct mlx5_flow_dv_encap_decap_resource res = { 602 .reformat_type = 603 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL, 604 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX, 605 }; 606 607 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 608 raw_encap_data = 609 (const struct rte_flow_action_raw_encap *)action->conf; 610 res.size = raw_encap_data->size; 611 memcpy(res.buf, raw_encap_data->data, res.size); 612 } else { 613 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP) 614 encap_data = 615 ((const struct rte_flow_action_vxlan_encap *) 616 action->conf)->definition; 617 else 618 encap_data = 619 ((const struct rte_flow_action_nvgre_encap *) 620 action->conf)->definition; 621 if (flow_dv_convert_encap_data(encap_data, res.buf, 622 &res.size, error)) 623 return -rte_errno; 624 } 625 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 626 return rte_flow_error_set(error, EINVAL, 627 RTE_FLOW_ERROR_TYPE_ACTION, 628 NULL, "can't create L2 encap action"); 629 return 0; 630 } 631 632 /** 633 * Convert L2 decap action to DV specification. 634 * 635 * @param[in] dev 636 * Pointer to rte_eth_dev structure. 637 * @param[in, out] dev_flow 638 * Pointer to the mlx5_flow. 639 * @param[out] error 640 * Pointer to the error structure. 641 * 642 * @return 643 * 0 on success, a negative errno value otherwise and rte_errno is set. 644 */ 645 static int 646 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev, 647 struct mlx5_flow *dev_flow, 648 struct rte_flow_error *error) 649 { 650 struct mlx5_flow_dv_encap_decap_resource res = { 651 .size = 0, 652 .reformat_type = 653 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2, 654 .ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX, 655 }; 656 657 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 658 return rte_flow_error_set(error, EINVAL, 659 RTE_FLOW_ERROR_TYPE_ACTION, 660 NULL, "can't create L2 decap action"); 661 return 0; 662 } 663 664 /** 665 * Convert raw decap/encap (L3 tunnel) action to DV specification. 666 * 667 * @param[in] dev 668 * Pointer to rte_eth_dev structure. 669 * @param[in] action 670 * Pointer to action structure. 671 * @param[in, out] dev_flow 672 * Pointer to the mlx5_flow. 673 * @param[in] attr 674 * Pointer to the flow attributes. 675 * @param[out] error 676 * Pointer to the error structure. 677 * 678 * @return 679 * 0 on success, a negative errno value otherwise and rte_errno is set. 680 */ 681 static int 682 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev, 683 const struct rte_flow_action *action, 684 struct mlx5_flow *dev_flow, 685 const struct rte_flow_attr *attr, 686 struct rte_flow_error *error) 687 { 688 const struct rte_flow_action_raw_encap *encap_data; 689 struct mlx5_flow_dv_encap_decap_resource res; 690 691 encap_data = (const struct rte_flow_action_raw_encap *)action->conf; 692 res.size = encap_data->size; 693 memcpy(res.buf, encap_data->data, res.size); 694 res.reformat_type = attr->egress ? 695 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL : 696 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 697 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : 698 MLX5DV_FLOW_TABLE_TYPE_NIC_RX; 699 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error)) 700 return rte_flow_error_set(error, EINVAL, 701 RTE_FLOW_ERROR_TYPE_ACTION, 702 NULL, "can't create encap action"); 703 return 0; 704 } 705 706 /** 707 * Verify the @p attributes will be correctly understood by the NIC and store 708 * them in the @p flow if everything is correct. 709 * 710 * @param[in] dev 711 * Pointer to dev struct. 712 * @param[in] attributes 713 * Pointer to flow attributes 714 * @param[out] error 715 * Pointer to error structure. 716 * 717 * @return 718 * 0 on success, a negative errno value otherwise and rte_errno is set. 719 */ 720 static int 721 flow_dv_validate_attributes(struct rte_eth_dev *dev, 722 const struct rte_flow_attr *attributes, 723 struct rte_flow_error *error) 724 { 725 struct mlx5_priv *priv = dev->data->dev_private; 726 uint32_t priority_max = priv->config.flow_prio - 1; 727 728 if (attributes->group) 729 return rte_flow_error_set(error, ENOTSUP, 730 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 731 NULL, 732 "groups is not supported"); 733 if (attributes->priority != MLX5_FLOW_PRIO_RSVD && 734 attributes->priority >= priority_max) 735 return rte_flow_error_set(error, ENOTSUP, 736 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 737 NULL, 738 "priority out of range"); 739 if (attributes->transfer) 740 return rte_flow_error_set(error, ENOTSUP, 741 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 742 NULL, 743 "transfer is not supported"); 744 if (!(attributes->egress ^ attributes->ingress)) 745 return rte_flow_error_set(error, ENOTSUP, 746 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 747 "must specify exactly one of " 748 "ingress or egress"); 749 return 0; 750 } 751 752 /** 753 * Internal validation function. For validating both actions and items. 754 * 755 * @param[in] dev 756 * Pointer to the rte_eth_dev structure. 757 * @param[in] attr 758 * Pointer to the flow attributes. 759 * @param[in] items 760 * Pointer to the list of items. 761 * @param[in] actions 762 * Pointer to the list of actions. 763 * @param[out] error 764 * Pointer to the error structure. 765 * 766 * @return 767 * 0 on success, a negative errno value otherwise and rte_errno is set. 768 */ 769 static int 770 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, 771 const struct rte_flow_item items[], 772 const struct rte_flow_action actions[], 773 struct rte_flow_error *error) 774 { 775 int ret; 776 uint64_t action_flags = 0; 777 uint64_t item_flags = 0; 778 uint64_t last_item = 0; 779 uint8_t next_protocol = 0xff; 780 int actions_n = 0; 781 782 if (items == NULL) 783 return -1; 784 ret = flow_dv_validate_attributes(dev, attr, error); 785 if (ret < 0) 786 return ret; 787 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 788 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 789 switch (items->type) { 790 case RTE_FLOW_ITEM_TYPE_VOID: 791 break; 792 case RTE_FLOW_ITEM_TYPE_ETH: 793 ret = mlx5_flow_validate_item_eth(items, item_flags, 794 error); 795 if (ret < 0) 796 return ret; 797 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 798 MLX5_FLOW_LAYER_OUTER_L2; 799 break; 800 case RTE_FLOW_ITEM_TYPE_VLAN: 801 ret = mlx5_flow_validate_item_vlan(items, item_flags, 802 error); 803 if (ret < 0) 804 return ret; 805 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 806 MLX5_FLOW_LAYER_OUTER_VLAN; 807 break; 808 case RTE_FLOW_ITEM_TYPE_IPV4: 809 ret = mlx5_flow_validate_item_ipv4(items, item_flags, 810 error); 811 if (ret < 0) 812 return ret; 813 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 814 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 815 if (items->mask != NULL && 816 ((const struct rte_flow_item_ipv4 *) 817 items->mask)->hdr.next_proto_id) { 818 next_protocol = 819 ((const struct rte_flow_item_ipv4 *) 820 (items->spec))->hdr.next_proto_id; 821 next_protocol &= 822 ((const struct rte_flow_item_ipv4 *) 823 (items->mask))->hdr.next_proto_id; 824 } else { 825 /* Reset for inner layer. */ 826 next_protocol = 0xff; 827 } 828 break; 829 case RTE_FLOW_ITEM_TYPE_IPV6: 830 ret = mlx5_flow_validate_item_ipv6(items, item_flags, 831 error); 832 if (ret < 0) 833 return ret; 834 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 835 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 836 if (items->mask != NULL && 837 ((const struct rte_flow_item_ipv6 *) 838 items->mask)->hdr.proto) { 839 next_protocol = 840 ((const struct rte_flow_item_ipv6 *) 841 items->spec)->hdr.proto; 842 next_protocol &= 843 ((const struct rte_flow_item_ipv6 *) 844 items->mask)->hdr.proto; 845 } else { 846 /* Reset for inner layer. */ 847 next_protocol = 0xff; 848 } 849 break; 850 case RTE_FLOW_ITEM_TYPE_TCP: 851 ret = mlx5_flow_validate_item_tcp 852 (items, item_flags, 853 next_protocol, 854 &rte_flow_item_tcp_mask, 855 error); 856 if (ret < 0) 857 return ret; 858 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 859 MLX5_FLOW_LAYER_OUTER_L4_TCP; 860 break; 861 case RTE_FLOW_ITEM_TYPE_UDP: 862 ret = mlx5_flow_validate_item_udp(items, item_flags, 863 next_protocol, 864 error); 865 if (ret < 0) 866 return ret; 867 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 868 MLX5_FLOW_LAYER_OUTER_L4_UDP; 869 break; 870 case RTE_FLOW_ITEM_TYPE_GRE: 871 case RTE_FLOW_ITEM_TYPE_NVGRE: 872 ret = mlx5_flow_validate_item_gre(items, item_flags, 873 next_protocol, error); 874 if (ret < 0) 875 return ret; 876 last_item = MLX5_FLOW_LAYER_GRE; 877 break; 878 case RTE_FLOW_ITEM_TYPE_VXLAN: 879 ret = mlx5_flow_validate_item_vxlan(items, item_flags, 880 error); 881 if (ret < 0) 882 return ret; 883 last_item = MLX5_FLOW_LAYER_VXLAN; 884 break; 885 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 886 ret = mlx5_flow_validate_item_vxlan_gpe(items, 887 item_flags, dev, 888 error); 889 if (ret < 0) 890 return ret; 891 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 892 break; 893 case RTE_FLOW_ITEM_TYPE_MPLS: 894 ret = mlx5_flow_validate_item_mpls(dev, items, 895 item_flags, 896 last_item, error); 897 if (ret < 0) 898 return ret; 899 last_item = MLX5_FLOW_LAYER_MPLS; 900 break; 901 case RTE_FLOW_ITEM_TYPE_META: 902 ret = flow_dv_validate_item_meta(dev, items, attr, 903 error); 904 if (ret < 0) 905 return ret; 906 last_item = MLX5_FLOW_ITEM_METADATA; 907 break; 908 default: 909 return rte_flow_error_set(error, ENOTSUP, 910 RTE_FLOW_ERROR_TYPE_ITEM, 911 NULL, "item not supported"); 912 } 913 item_flags |= last_item; 914 } 915 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 916 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS) 917 return rte_flow_error_set(error, ENOTSUP, 918 RTE_FLOW_ERROR_TYPE_ACTION, 919 actions, "too many actions"); 920 switch (actions->type) { 921 case RTE_FLOW_ACTION_TYPE_VOID: 922 break; 923 case RTE_FLOW_ACTION_TYPE_FLAG: 924 ret = mlx5_flow_validate_action_flag(action_flags, 925 attr, error); 926 if (ret < 0) 927 return ret; 928 action_flags |= MLX5_FLOW_ACTION_FLAG; 929 ++actions_n; 930 break; 931 case RTE_FLOW_ACTION_TYPE_MARK: 932 ret = mlx5_flow_validate_action_mark(actions, 933 action_flags, 934 attr, error); 935 if (ret < 0) 936 return ret; 937 action_flags |= MLX5_FLOW_ACTION_MARK; 938 ++actions_n; 939 break; 940 case RTE_FLOW_ACTION_TYPE_DROP: 941 ret = mlx5_flow_validate_action_drop(action_flags, 942 attr, error); 943 if (ret < 0) 944 return ret; 945 action_flags |= MLX5_FLOW_ACTION_DROP; 946 ++actions_n; 947 break; 948 case RTE_FLOW_ACTION_TYPE_QUEUE: 949 ret = mlx5_flow_validate_action_queue(actions, 950 action_flags, dev, 951 attr, error); 952 if (ret < 0) 953 return ret; 954 action_flags |= MLX5_FLOW_ACTION_QUEUE; 955 ++actions_n; 956 break; 957 case RTE_FLOW_ACTION_TYPE_RSS: 958 ret = mlx5_flow_validate_action_rss(actions, 959 action_flags, dev, 960 attr, item_flags, 961 error); 962 if (ret < 0) 963 return ret; 964 action_flags |= MLX5_FLOW_ACTION_RSS; 965 ++actions_n; 966 break; 967 case RTE_FLOW_ACTION_TYPE_COUNT: 968 ret = mlx5_flow_validate_action_count(dev, attr, error); 969 if (ret < 0) 970 return ret; 971 action_flags |= MLX5_FLOW_ACTION_COUNT; 972 ++actions_n; 973 break; 974 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 975 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 976 ret = flow_dv_validate_action_l2_encap(action_flags, 977 actions, attr, 978 error); 979 if (ret < 0) 980 return ret; 981 action_flags |= actions->type == 982 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? 983 MLX5_FLOW_ACTION_VXLAN_ENCAP : 984 MLX5_FLOW_ACTION_NVGRE_ENCAP; 985 ++actions_n; 986 break; 987 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 988 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 989 ret = flow_dv_validate_action_l2_decap(action_flags, 990 attr, error); 991 if (ret < 0) 992 return ret; 993 action_flags |= actions->type == 994 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? 995 MLX5_FLOW_ACTION_VXLAN_DECAP : 996 MLX5_FLOW_ACTION_NVGRE_DECAP; 997 ++actions_n; 998 break; 999 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 1000 ret = flow_dv_validate_action_raw_encap(action_flags, 1001 actions, attr, 1002 error); 1003 if (ret < 0) 1004 return ret; 1005 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; 1006 ++actions_n; 1007 break; 1008 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 1009 ret = flow_dv_validate_action_raw_decap(action_flags, 1010 actions, attr, 1011 error); 1012 if (ret < 0) 1013 return ret; 1014 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; 1015 ++actions_n; 1016 break; 1017 default: 1018 return rte_flow_error_set(error, ENOTSUP, 1019 RTE_FLOW_ERROR_TYPE_ACTION, 1020 actions, 1021 "action not supported"); 1022 } 1023 } 1024 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress) 1025 return rte_flow_error_set(error, EINVAL, 1026 RTE_FLOW_ERROR_TYPE_ACTION, actions, 1027 "no fate action is found"); 1028 return 0; 1029 } 1030 1031 /** 1032 * Internal preparation function. Allocates the DV flow size, 1033 * this size is constant. 1034 * 1035 * @param[in] attr 1036 * Pointer to the flow attributes. 1037 * @param[in] items 1038 * Pointer to the list of items. 1039 * @param[in] actions 1040 * Pointer to the list of actions. 1041 * @param[out] error 1042 * Pointer to the error structure. 1043 * 1044 * @return 1045 * Pointer to mlx5_flow object on success, 1046 * otherwise NULL and rte_errno is set. 1047 */ 1048 static struct mlx5_flow * 1049 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, 1050 const struct rte_flow_item items[] __rte_unused, 1051 const struct rte_flow_action actions[] __rte_unused, 1052 struct rte_flow_error *error) 1053 { 1054 uint32_t size = sizeof(struct mlx5_flow); 1055 struct mlx5_flow *flow; 1056 1057 flow = rte_calloc(__func__, 1, size, 0); 1058 if (!flow) { 1059 rte_flow_error_set(error, ENOMEM, 1060 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1061 "not enough memory to create flow"); 1062 return NULL; 1063 } 1064 flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param); 1065 return flow; 1066 } 1067 1068 #ifndef NDEBUG 1069 /** 1070 * Sanity check for match mask and value. Similar to check_valid_spec() in 1071 * kernel driver. If unmasked bit is present in value, it returns failure. 1072 * 1073 * @param match_mask 1074 * pointer to match mask buffer. 1075 * @param match_value 1076 * pointer to match value buffer. 1077 * 1078 * @return 1079 * 0 if valid, -EINVAL otherwise. 1080 */ 1081 static int 1082 flow_dv_check_valid_spec(void *match_mask, void *match_value) 1083 { 1084 uint8_t *m = match_mask; 1085 uint8_t *v = match_value; 1086 unsigned int i; 1087 1088 for (i = 0; i < MLX5_ST_SZ_DB(fte_match_param); ++i) { 1089 if (v[i] & ~m[i]) { 1090 DRV_LOG(ERR, 1091 "match_value differs from match_criteria" 1092 " %p[%u] != %p[%u]", 1093 match_value, i, match_mask, i); 1094 return -EINVAL; 1095 } 1096 } 1097 return 0; 1098 } 1099 #endif 1100 1101 /** 1102 * Add Ethernet item to matcher and to the value. 1103 * 1104 * @param[in, out] matcher 1105 * Flow matcher. 1106 * @param[in, out] key 1107 * Flow matcher value. 1108 * @param[in] item 1109 * Flow pattern to translate. 1110 * @param[in] inner 1111 * Item is inner pattern. 1112 */ 1113 static void 1114 flow_dv_translate_item_eth(void *matcher, void *key, 1115 const struct rte_flow_item *item, int inner) 1116 { 1117 const struct rte_flow_item_eth *eth_m = item->mask; 1118 const struct rte_flow_item_eth *eth_v = item->spec; 1119 const struct rte_flow_item_eth nic_mask = { 1120 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1121 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1122 .type = RTE_BE16(0xffff), 1123 }; 1124 void *headers_m; 1125 void *headers_v; 1126 char *l24_v; 1127 unsigned int i; 1128 1129 if (!eth_v) 1130 return; 1131 if (!eth_m) 1132 eth_m = &nic_mask; 1133 if (inner) { 1134 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1135 inner_headers); 1136 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1137 } else { 1138 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1139 outer_headers); 1140 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1141 } 1142 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16), 1143 ð_m->dst, sizeof(eth_m->dst)); 1144 /* The value must be in the range of the mask. */ 1145 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16); 1146 for (i = 0; i < sizeof(eth_m->dst); ++i) 1147 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i]; 1148 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16), 1149 ð_m->src, sizeof(eth_m->src)); 1150 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16); 1151 /* The value must be in the range of the mask. */ 1152 for (i = 0; i < sizeof(eth_m->dst); ++i) 1153 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i]; 1154 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 1155 rte_be_to_cpu_16(eth_m->type)); 1156 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype); 1157 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type; 1158 } 1159 1160 /** 1161 * Add VLAN item to matcher and to the value. 1162 * 1163 * @param[in, out] matcher 1164 * Flow matcher. 1165 * @param[in, out] key 1166 * Flow matcher value. 1167 * @param[in] item 1168 * Flow pattern to translate. 1169 * @param[in] inner 1170 * Item is inner pattern. 1171 */ 1172 static void 1173 flow_dv_translate_item_vlan(void *matcher, void *key, 1174 const struct rte_flow_item *item, 1175 int inner) 1176 { 1177 const struct rte_flow_item_vlan *vlan_m = item->mask; 1178 const struct rte_flow_item_vlan *vlan_v = item->spec; 1179 const struct rte_flow_item_vlan nic_mask = { 1180 .tci = RTE_BE16(0x0fff), 1181 .inner_type = RTE_BE16(0xffff), 1182 }; 1183 void *headers_m; 1184 void *headers_v; 1185 uint16_t tci_m; 1186 uint16_t tci_v; 1187 1188 if (!vlan_v) 1189 return; 1190 if (!vlan_m) 1191 vlan_m = &nic_mask; 1192 if (inner) { 1193 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1194 inner_headers); 1195 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1196 } else { 1197 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1198 outer_headers); 1199 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1200 } 1201 tci_m = rte_be_to_cpu_16(vlan_m->tci); 1202 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci); 1203 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1); 1204 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1205 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m); 1206 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v); 1207 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12); 1208 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12); 1209 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13); 1210 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13); 1211 } 1212 1213 /** 1214 * Add IPV4 item to matcher and to the value. 1215 * 1216 * @param[in, out] matcher 1217 * Flow matcher. 1218 * @param[in, out] key 1219 * Flow matcher value. 1220 * @param[in] item 1221 * Flow pattern to translate. 1222 * @param[in] inner 1223 * Item is inner pattern. 1224 */ 1225 static void 1226 flow_dv_translate_item_ipv4(void *matcher, void *key, 1227 const struct rte_flow_item *item, 1228 int inner) 1229 { 1230 const struct rte_flow_item_ipv4 *ipv4_m = item->mask; 1231 const struct rte_flow_item_ipv4 *ipv4_v = item->spec; 1232 const struct rte_flow_item_ipv4 nic_mask = { 1233 .hdr = { 1234 .src_addr = RTE_BE32(0xffffffff), 1235 .dst_addr = RTE_BE32(0xffffffff), 1236 .type_of_service = 0xff, 1237 .next_proto_id = 0xff, 1238 }, 1239 }; 1240 void *headers_m; 1241 void *headers_v; 1242 char *l24_m; 1243 char *l24_v; 1244 uint8_t tos; 1245 1246 if (inner) { 1247 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1248 inner_headers); 1249 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1250 } else { 1251 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1252 outer_headers); 1253 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1254 } 1255 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 1256 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4); 1257 if (!ipv4_v) 1258 return; 1259 if (!ipv4_m) 1260 ipv4_m = &nic_mask; 1261 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 1262 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1263 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1264 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1265 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr; 1266 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr; 1267 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 1268 src_ipv4_src_ipv6.ipv4_layout.ipv4); 1269 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1270 src_ipv4_src_ipv6.ipv4_layout.ipv4); 1271 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr; 1272 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr; 1273 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service; 1274 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, 1275 ipv4_m->hdr.type_of_service); 1276 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos); 1277 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, 1278 ipv4_m->hdr.type_of_service >> 2); 1279 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2); 1280 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 1281 ipv4_m->hdr.next_proto_id); 1282 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 1283 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id); 1284 } 1285 1286 /** 1287 * Add IPV6 item to matcher and to the value. 1288 * 1289 * @param[in, out] matcher 1290 * Flow matcher. 1291 * @param[in, out] key 1292 * Flow matcher value. 1293 * @param[in] item 1294 * Flow pattern to translate. 1295 * @param[in] inner 1296 * Item is inner pattern. 1297 */ 1298 static void 1299 flow_dv_translate_item_ipv6(void *matcher, void *key, 1300 const struct rte_flow_item *item, 1301 int inner) 1302 { 1303 const struct rte_flow_item_ipv6 *ipv6_m = item->mask; 1304 const struct rte_flow_item_ipv6 *ipv6_v = item->spec; 1305 const struct rte_flow_item_ipv6 nic_mask = { 1306 .hdr = { 1307 .src_addr = 1308 "\xff\xff\xff\xff\xff\xff\xff\xff" 1309 "\xff\xff\xff\xff\xff\xff\xff\xff", 1310 .dst_addr = 1311 "\xff\xff\xff\xff\xff\xff\xff\xff" 1312 "\xff\xff\xff\xff\xff\xff\xff\xff", 1313 .vtc_flow = RTE_BE32(0xffffffff), 1314 .proto = 0xff, 1315 .hop_limits = 0xff, 1316 }, 1317 }; 1318 void *headers_m; 1319 void *headers_v; 1320 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 1321 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 1322 char *l24_m; 1323 char *l24_v; 1324 uint32_t vtc_m; 1325 uint32_t vtc_v; 1326 int i; 1327 int size; 1328 1329 if (inner) { 1330 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1331 inner_headers); 1332 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1333 } else { 1334 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1335 outer_headers); 1336 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1337 } 1338 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf); 1339 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6); 1340 if (!ipv6_v) 1341 return; 1342 if (!ipv6_m) 1343 ipv6_m = &nic_mask; 1344 size = sizeof(ipv6_m->hdr.dst_addr); 1345 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 1346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 1347 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1348 dst_ipv4_dst_ipv6.ipv6_layout.ipv6); 1349 memcpy(l24_m, ipv6_m->hdr.dst_addr, size); 1350 for (i = 0; i < size; ++i) 1351 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i]; 1352 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, 1353 src_ipv4_src_ipv6.ipv6_layout.ipv6); 1354 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1355 src_ipv4_src_ipv6.ipv6_layout.ipv6); 1356 memcpy(l24_m, ipv6_m->hdr.src_addr, size); 1357 for (i = 0; i < size; ++i) 1358 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i]; 1359 /* TOS. */ 1360 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow); 1361 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow); 1362 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20); 1363 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20); 1364 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22); 1365 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22); 1366 /* Label. */ 1367 if (inner) { 1368 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label, 1369 vtc_m); 1370 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label, 1371 vtc_v); 1372 } else { 1373 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label, 1374 vtc_m); 1375 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label, 1376 vtc_v); 1377 } 1378 /* Protocol. */ 1379 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 1380 ipv6_m->hdr.proto); 1381 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 1382 ipv6_v->hdr.proto & ipv6_m->hdr.proto); 1383 } 1384 1385 /** 1386 * Add TCP item to matcher and to the value. 1387 * 1388 * @param[in, out] matcher 1389 * Flow matcher. 1390 * @param[in, out] key 1391 * Flow matcher value. 1392 * @param[in] item 1393 * Flow pattern to translate. 1394 * @param[in] inner 1395 * Item is inner pattern. 1396 */ 1397 static void 1398 flow_dv_translate_item_tcp(void *matcher, void *key, 1399 const struct rte_flow_item *item, 1400 int inner) 1401 { 1402 const struct rte_flow_item_tcp *tcp_m = item->mask; 1403 const struct rte_flow_item_tcp *tcp_v = item->spec; 1404 void *headers_m; 1405 void *headers_v; 1406 1407 if (inner) { 1408 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1409 inner_headers); 1410 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1411 } else { 1412 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1413 outer_headers); 1414 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1415 } 1416 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 1417 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP); 1418 if (!tcp_v) 1419 return; 1420 if (!tcp_m) 1421 tcp_m = &rte_flow_item_tcp_mask; 1422 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport, 1423 rte_be_to_cpu_16(tcp_m->hdr.src_port)); 1424 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 1425 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port)); 1426 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport, 1427 rte_be_to_cpu_16(tcp_m->hdr.dst_port)); 1428 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 1429 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port)); 1430 } 1431 1432 /** 1433 * Add UDP item to matcher and to the value. 1434 * 1435 * @param[in, out] matcher 1436 * Flow matcher. 1437 * @param[in, out] key 1438 * Flow matcher value. 1439 * @param[in] item 1440 * Flow pattern to translate. 1441 * @param[in] inner 1442 * Item is inner pattern. 1443 */ 1444 static void 1445 flow_dv_translate_item_udp(void *matcher, void *key, 1446 const struct rte_flow_item *item, 1447 int inner) 1448 { 1449 const struct rte_flow_item_udp *udp_m = item->mask; 1450 const struct rte_flow_item_udp *udp_v = item->spec; 1451 void *headers_m; 1452 void *headers_v; 1453 1454 if (inner) { 1455 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1456 inner_headers); 1457 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1458 } else { 1459 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1460 outer_headers); 1461 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1462 } 1463 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 1464 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP); 1465 if (!udp_v) 1466 return; 1467 if (!udp_m) 1468 udp_m = &rte_flow_item_udp_mask; 1469 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport, 1470 rte_be_to_cpu_16(udp_m->hdr.src_port)); 1471 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 1472 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port)); 1473 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 1474 rte_be_to_cpu_16(udp_m->hdr.dst_port)); 1475 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 1476 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port)); 1477 } 1478 1479 /** 1480 * Add GRE item to matcher and to the value. 1481 * 1482 * @param[in, out] matcher 1483 * Flow matcher. 1484 * @param[in, out] key 1485 * Flow matcher value. 1486 * @param[in] item 1487 * Flow pattern to translate. 1488 * @param[in] inner 1489 * Item is inner pattern. 1490 */ 1491 static void 1492 flow_dv_translate_item_gre(void *matcher, void *key, 1493 const struct rte_flow_item *item, 1494 int inner) 1495 { 1496 const struct rte_flow_item_gre *gre_m = item->mask; 1497 const struct rte_flow_item_gre *gre_v = item->spec; 1498 void *headers_m; 1499 void *headers_v; 1500 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 1501 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 1502 1503 if (inner) { 1504 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1505 inner_headers); 1506 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1507 } else { 1508 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1509 outer_headers); 1510 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1511 } 1512 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 1513 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE); 1514 if (!gre_v) 1515 return; 1516 if (!gre_m) 1517 gre_m = &rte_flow_item_gre_mask; 1518 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 1519 rte_be_to_cpu_16(gre_m->protocol)); 1520 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 1521 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol)); 1522 } 1523 1524 /** 1525 * Add NVGRE item to matcher and to the value. 1526 * 1527 * @param[in, out] matcher 1528 * Flow matcher. 1529 * @param[in, out] key 1530 * Flow matcher value. 1531 * @param[in] item 1532 * Flow pattern to translate. 1533 * @param[in] inner 1534 * Item is inner pattern. 1535 */ 1536 static void 1537 flow_dv_translate_item_nvgre(void *matcher, void *key, 1538 const struct rte_flow_item *item, 1539 int inner) 1540 { 1541 const struct rte_flow_item_nvgre *nvgre_m = item->mask; 1542 const struct rte_flow_item_nvgre *nvgre_v = item->spec; 1543 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 1544 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 1545 const char *tni_flow_id_m = (const char *)nvgre_m->tni; 1546 const char *tni_flow_id_v = (const char *)nvgre_v->tni; 1547 char *gre_key_m; 1548 char *gre_key_v; 1549 int size; 1550 int i; 1551 1552 flow_dv_translate_item_gre(matcher, key, item, inner); 1553 if (!nvgre_v) 1554 return; 1555 if (!nvgre_m) 1556 nvgre_m = &rte_flow_item_nvgre_mask; 1557 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id); 1558 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h); 1559 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h); 1560 memcpy(gre_key_m, tni_flow_id_m, size); 1561 for (i = 0; i < size; ++i) 1562 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i]; 1563 } 1564 1565 /** 1566 * Add VXLAN item to matcher and to the value. 1567 * 1568 * @param[in, out] matcher 1569 * Flow matcher. 1570 * @param[in, out] key 1571 * Flow matcher value. 1572 * @param[in] item 1573 * Flow pattern to translate. 1574 * @param[in] inner 1575 * Item is inner pattern. 1576 */ 1577 static void 1578 flow_dv_translate_item_vxlan(void *matcher, void *key, 1579 const struct rte_flow_item *item, 1580 int inner) 1581 { 1582 const struct rte_flow_item_vxlan *vxlan_m = item->mask; 1583 const struct rte_flow_item_vxlan *vxlan_v = item->spec; 1584 void *headers_m; 1585 void *headers_v; 1586 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 1587 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 1588 char *vni_m; 1589 char *vni_v; 1590 uint16_t dport; 1591 int size; 1592 int i; 1593 1594 if (inner) { 1595 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1596 inner_headers); 1597 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers); 1598 } else { 1599 headers_m = MLX5_ADDR_OF(fte_match_param, matcher, 1600 outer_headers); 1601 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1602 } 1603 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ? 1604 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE; 1605 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) { 1606 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF); 1607 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport); 1608 } 1609 if (!vxlan_v) 1610 return; 1611 if (!vxlan_m) 1612 vxlan_m = &rte_flow_item_vxlan_mask; 1613 size = sizeof(vxlan_m->vni); 1614 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni); 1615 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni); 1616 memcpy(vni_m, vxlan_m->vni, size); 1617 for (i = 0; i < size; ++i) 1618 vni_v[i] = vni_m[i] & vxlan_v->vni[i]; 1619 } 1620 1621 /** 1622 * Add MPLS item to matcher and to the value. 1623 * 1624 * @param[in, out] matcher 1625 * Flow matcher. 1626 * @param[in, out] key 1627 * Flow matcher value. 1628 * @param[in] item 1629 * Flow pattern to translate. 1630 * @param[in] prev_layer 1631 * The protocol layer indicated in previous item. 1632 * @param[in] inner 1633 * Item is inner pattern. 1634 */ 1635 static void 1636 flow_dv_translate_item_mpls(void *matcher, void *key, 1637 const struct rte_flow_item *item, 1638 uint64_t prev_layer, 1639 int inner) 1640 { 1641 const uint32_t *in_mpls_m = item->mask; 1642 const uint32_t *in_mpls_v = item->spec; 1643 uint32_t *out_mpls_m = 0; 1644 uint32_t *out_mpls_v = 0; 1645 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters); 1646 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters); 1647 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher, 1648 misc_parameters_2); 1649 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 1650 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers); 1651 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers); 1652 1653 switch (prev_layer) { 1654 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 1655 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff); 1656 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 1657 MLX5_UDP_PORT_MPLS); 1658 break; 1659 case MLX5_FLOW_LAYER_GRE: 1660 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff); 1661 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, 1662 ETHER_TYPE_MPLS); 1663 break; 1664 default: 1665 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff); 1666 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 1667 IPPROTO_MPLS); 1668 break; 1669 } 1670 if (!in_mpls_v) 1671 return; 1672 if (!in_mpls_m) 1673 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask; 1674 switch (prev_layer) { 1675 case MLX5_FLOW_LAYER_OUTER_L4_UDP: 1676 out_mpls_m = 1677 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 1678 outer_first_mpls_over_udp); 1679 out_mpls_v = 1680 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 1681 outer_first_mpls_over_udp); 1682 break; 1683 case MLX5_FLOW_LAYER_GRE: 1684 out_mpls_m = 1685 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m, 1686 outer_first_mpls_over_gre); 1687 out_mpls_v = 1688 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v, 1689 outer_first_mpls_over_gre); 1690 break; 1691 default: 1692 /* Inner MPLS not over GRE is not supported. */ 1693 if (!inner) { 1694 out_mpls_m = 1695 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 1696 misc2_m, 1697 outer_first_mpls); 1698 out_mpls_v = 1699 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, 1700 misc2_v, 1701 outer_first_mpls); 1702 } 1703 break; 1704 } 1705 if (out_mpls_m && out_mpls_v) { 1706 *out_mpls_m = *in_mpls_m; 1707 *out_mpls_v = *in_mpls_v & *in_mpls_m; 1708 } 1709 } 1710 1711 /** 1712 * Add META item to matcher 1713 * 1714 * @param[in, out] matcher 1715 * Flow matcher. 1716 * @param[in, out] key 1717 * Flow matcher value. 1718 * @param[in] item 1719 * Flow pattern to translate. 1720 * @param[in] inner 1721 * Item is inner pattern. 1722 */ 1723 static void 1724 flow_dv_translate_item_meta(void *matcher, void *key, 1725 const struct rte_flow_item *item) 1726 { 1727 const struct rte_flow_item_meta *meta_m; 1728 const struct rte_flow_item_meta *meta_v; 1729 void *misc2_m = 1730 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2); 1731 void *misc2_v = 1732 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2); 1733 1734 meta_m = (const void *)item->mask; 1735 if (!meta_m) 1736 meta_m = &rte_flow_item_meta_mask; 1737 meta_v = (const void *)item->spec; 1738 if (meta_v) { 1739 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, 1740 rte_be_to_cpu_32(meta_m->data)); 1741 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, 1742 rte_be_to_cpu_32(meta_v->data & meta_m->data)); 1743 } 1744 } 1745 1746 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 }; 1747 1748 #define HEADER_IS_ZERO(match_criteria, headers) \ 1749 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 1750 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 1751 1752 /** 1753 * Calculate flow matcher enable bitmap. 1754 * 1755 * @param match_criteria 1756 * Pointer to flow matcher criteria. 1757 * 1758 * @return 1759 * Bitmap of enabled fields. 1760 */ 1761 static uint8_t 1762 flow_dv_matcher_enable(uint32_t *match_criteria) 1763 { 1764 uint8_t match_criteria_enable; 1765 1766 match_criteria_enable = 1767 (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 1768 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT; 1769 match_criteria_enable |= 1770 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 1771 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT; 1772 match_criteria_enable |= 1773 (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 1774 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT; 1775 match_criteria_enable |= 1776 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 1777 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; 1778 1779 return match_criteria_enable; 1780 } 1781 1782 /** 1783 * Register the flow matcher. 1784 * 1785 * @param dev[in, out] 1786 * Pointer to rte_eth_dev structure. 1787 * @param[in, out] matcher 1788 * Pointer to flow matcher. 1789 * @parm[in, out] dev_flow 1790 * Pointer to the dev_flow. 1791 * @param[out] error 1792 * pointer to error structure. 1793 * 1794 * @return 1795 * 0 on success otherwise -errno and errno is set. 1796 */ 1797 static int 1798 flow_dv_matcher_register(struct rte_eth_dev *dev, 1799 struct mlx5_flow_dv_matcher *matcher, 1800 struct mlx5_flow *dev_flow, 1801 struct rte_flow_error *error) 1802 { 1803 struct mlx5_priv *priv = dev->data->dev_private; 1804 struct mlx5_flow_dv_matcher *cache_matcher; 1805 struct mlx5dv_flow_matcher_attr dv_attr = { 1806 .type = IBV_FLOW_ATTR_NORMAL, 1807 .match_mask = (void *)&matcher->mask, 1808 }; 1809 1810 /* Lookup from cache. */ 1811 LIST_FOREACH(cache_matcher, &priv->matchers, next) { 1812 if (matcher->crc == cache_matcher->crc && 1813 matcher->priority == cache_matcher->priority && 1814 matcher->egress == cache_matcher->egress && 1815 !memcmp((const void *)matcher->mask.buf, 1816 (const void *)cache_matcher->mask.buf, 1817 cache_matcher->mask.size)) { 1818 DRV_LOG(DEBUG, 1819 "priority %hd use %s matcher %p: refcnt %d++", 1820 cache_matcher->priority, 1821 cache_matcher->egress ? "tx" : "rx", 1822 (void *)cache_matcher, 1823 rte_atomic32_read(&cache_matcher->refcnt)); 1824 rte_atomic32_inc(&cache_matcher->refcnt); 1825 dev_flow->dv.matcher = cache_matcher; 1826 return 0; 1827 } 1828 } 1829 /* Register new matcher. */ 1830 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0); 1831 if (!cache_matcher) 1832 return rte_flow_error_set(error, ENOMEM, 1833 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 1834 "cannot allocate matcher memory"); 1835 *cache_matcher = *matcher; 1836 dv_attr.match_criteria_enable = 1837 flow_dv_matcher_enable(cache_matcher->mask.buf); 1838 dv_attr.priority = matcher->priority; 1839 if (matcher->egress) 1840 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; 1841 cache_matcher->matcher_object = 1842 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr); 1843 if (!cache_matcher->matcher_object) { 1844 rte_free(cache_matcher); 1845 return rte_flow_error_set(error, ENOMEM, 1846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1847 NULL, "cannot create matcher"); 1848 } 1849 rte_atomic32_inc(&cache_matcher->refcnt); 1850 LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next); 1851 dev_flow->dv.matcher = cache_matcher; 1852 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d", 1853 cache_matcher->priority, 1854 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher, 1855 rte_atomic32_read(&cache_matcher->refcnt)); 1856 return 0; 1857 } 1858 1859 /** 1860 * Fill the flow with DV spec. 1861 * 1862 * @param[in] dev 1863 * Pointer to rte_eth_dev structure. 1864 * @param[in, out] dev_flow 1865 * Pointer to the sub flow. 1866 * @param[in] attr 1867 * Pointer to the flow attributes. 1868 * @param[in] items 1869 * Pointer to the list of items. 1870 * @param[in] actions 1871 * Pointer to the list of actions. 1872 * @param[out] error 1873 * Pointer to the error structure. 1874 * 1875 * @return 1876 * 0 on success, a negative errno value otherwise and rte_errno is set. 1877 */ 1878 static int 1879 flow_dv_translate(struct rte_eth_dev *dev, 1880 struct mlx5_flow *dev_flow, 1881 const struct rte_flow_attr *attr, 1882 const struct rte_flow_item items[], 1883 const struct rte_flow_action actions[], 1884 struct rte_flow_error *error) 1885 { 1886 struct mlx5_priv *priv = dev->data->dev_private; 1887 struct rte_flow *flow = dev_flow->flow; 1888 uint64_t item_flags = 0; 1889 uint64_t last_item = 0; 1890 uint64_t action_flags = 0; 1891 uint64_t priority = attr->priority; 1892 struct mlx5_flow_dv_matcher matcher = { 1893 .mask = { 1894 .size = sizeof(matcher.mask.buf), 1895 }, 1896 }; 1897 int actions_n = 0; 1898 1899 if (priority == MLX5_FLOW_PRIO_RSVD) 1900 priority = priv->config.flow_prio - 1; 1901 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 1902 const struct rte_flow_action_queue *queue; 1903 const struct rte_flow_action_rss *rss; 1904 const struct rte_flow_action *action = actions; 1905 const uint8_t *rss_key; 1906 1907 switch (actions->type) { 1908 case RTE_FLOW_ACTION_TYPE_VOID: 1909 break; 1910 case RTE_FLOW_ACTION_TYPE_FLAG: 1911 dev_flow->dv.actions[actions_n].type = 1912 MLX5DV_FLOW_ACTION_TAG; 1913 dev_flow->dv.actions[actions_n].tag_value = 1914 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT); 1915 actions_n++; 1916 action_flags |= MLX5_FLOW_ACTION_FLAG; 1917 break; 1918 case RTE_FLOW_ACTION_TYPE_MARK: 1919 dev_flow->dv.actions[actions_n].type = 1920 MLX5DV_FLOW_ACTION_TAG; 1921 dev_flow->dv.actions[actions_n].tag_value = 1922 mlx5_flow_mark_set 1923 (((const struct rte_flow_action_mark *) 1924 (actions->conf))->id); 1925 actions_n++; 1926 action_flags |= MLX5_FLOW_ACTION_MARK; 1927 break; 1928 case RTE_FLOW_ACTION_TYPE_DROP: 1929 dev_flow->dv.actions[actions_n].type = 1930 MLX5DV_FLOW_ACTION_DROP; 1931 action_flags |= MLX5_FLOW_ACTION_DROP; 1932 break; 1933 case RTE_FLOW_ACTION_TYPE_QUEUE: 1934 queue = actions->conf; 1935 flow->rss.queue_num = 1; 1936 (*flow->queue)[0] = queue->index; 1937 action_flags |= MLX5_FLOW_ACTION_QUEUE; 1938 break; 1939 case RTE_FLOW_ACTION_TYPE_RSS: 1940 rss = actions->conf; 1941 if (flow->queue) 1942 memcpy((*flow->queue), rss->queue, 1943 rss->queue_num * sizeof(uint16_t)); 1944 flow->rss.queue_num = rss->queue_num; 1945 /* NULL RSS key indicates default RSS key. */ 1946 rss_key = !rss->key ? rss_hash_default_key : rss->key; 1947 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN); 1948 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */ 1949 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types; 1950 flow->rss.level = rss->level; 1951 action_flags |= MLX5_FLOW_ACTION_RSS; 1952 break; 1953 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 1954 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 1955 if (flow_dv_create_action_l2_encap(dev, actions, 1956 dev_flow, error)) 1957 return -rte_errno; 1958 dev_flow->dv.actions[actions_n].type = 1959 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 1960 dev_flow->dv.actions[actions_n].action = 1961 dev_flow->dv.encap_decap->verbs_action; 1962 actions_n++; 1963 action_flags |= actions->type == 1964 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ? 1965 MLX5_FLOW_ACTION_VXLAN_ENCAP : 1966 MLX5_FLOW_ACTION_NVGRE_ENCAP; 1967 break; 1968 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 1969 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 1970 if (flow_dv_create_action_l2_decap(dev, dev_flow, 1971 error)) 1972 return -rte_errno; 1973 dev_flow->dv.actions[actions_n].type = 1974 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 1975 dev_flow->dv.actions[actions_n].action = 1976 dev_flow->dv.encap_decap->verbs_action; 1977 actions_n++; 1978 action_flags |= actions->type == 1979 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ? 1980 MLX5_FLOW_ACTION_VXLAN_DECAP : 1981 MLX5_FLOW_ACTION_NVGRE_DECAP; 1982 break; 1983 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 1984 /* Handle encap with preceding decap. */ 1985 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) { 1986 if (flow_dv_create_action_raw_encap 1987 (dev, actions, dev_flow, attr, error)) 1988 return -rte_errno; 1989 dev_flow->dv.actions[actions_n].type = 1990 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 1991 dev_flow->dv.actions[actions_n].action = 1992 dev_flow->dv.encap_decap->verbs_action; 1993 } else { 1994 /* Handle encap without preceding decap. */ 1995 if (flow_dv_create_action_l2_encap(dev, actions, 1996 dev_flow, 1997 error)) 1998 return -rte_errno; 1999 dev_flow->dv.actions[actions_n].type = 2000 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 2001 dev_flow->dv.actions[actions_n].action = 2002 dev_flow->dv.encap_decap->verbs_action; 2003 } 2004 actions_n++; 2005 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP; 2006 break; 2007 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 2008 /* Check if this decap is followed by encap. */ 2009 for (; action->type != RTE_FLOW_ACTION_TYPE_END && 2010 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP; 2011 action++) { 2012 } 2013 /* Handle decap only if it isn't followed by encap. */ 2014 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 2015 if (flow_dv_create_action_l2_decap(dev, 2016 dev_flow, 2017 error)) 2018 return -rte_errno; 2019 dev_flow->dv.actions[actions_n].type = 2020 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 2021 dev_flow->dv.actions[actions_n].action = 2022 dev_flow->dv.encap_decap->verbs_action; 2023 actions_n++; 2024 } 2025 /* If decap is followed by encap, handle it at encap. */ 2026 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP; 2027 break; 2028 default: 2029 break; 2030 } 2031 } 2032 dev_flow->dv.actions_n = actions_n; 2033 flow->actions = action_flags; 2034 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 2035 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2036 void *match_mask = matcher.mask.buf; 2037 void *match_value = dev_flow->dv.value.buf; 2038 2039 switch (items->type) { 2040 case RTE_FLOW_ITEM_TYPE_ETH: 2041 flow_dv_translate_item_eth(match_mask, match_value, 2042 items, tunnel); 2043 matcher.priority = MLX5_PRIORITY_MAP_L2; 2044 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2045 MLX5_FLOW_LAYER_OUTER_L2; 2046 break; 2047 case RTE_FLOW_ITEM_TYPE_VLAN: 2048 flow_dv_translate_item_vlan(match_mask, match_value, 2049 items, tunnel); 2050 matcher.priority = MLX5_PRIORITY_MAP_L2; 2051 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 | 2052 MLX5_FLOW_LAYER_INNER_VLAN) : 2053 (MLX5_FLOW_LAYER_OUTER_L2 | 2054 MLX5_FLOW_LAYER_OUTER_VLAN); 2055 break; 2056 case RTE_FLOW_ITEM_TYPE_IPV4: 2057 flow_dv_translate_item_ipv4(match_mask, match_value, 2058 items, tunnel); 2059 matcher.priority = MLX5_PRIORITY_MAP_L3; 2060 dev_flow->dv.hash_fields |= 2061 mlx5_flow_hashfields_adjust 2062 (dev_flow, tunnel, 2063 MLX5_IPV4_LAYER_TYPES, 2064 MLX5_IPV4_IBV_RX_HASH); 2065 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2066 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2067 break; 2068 case RTE_FLOW_ITEM_TYPE_IPV6: 2069 flow_dv_translate_item_ipv6(match_mask, match_value, 2070 items, tunnel); 2071 matcher.priority = MLX5_PRIORITY_MAP_L3; 2072 dev_flow->dv.hash_fields |= 2073 mlx5_flow_hashfields_adjust 2074 (dev_flow, tunnel, 2075 MLX5_IPV6_LAYER_TYPES, 2076 MLX5_IPV6_IBV_RX_HASH); 2077 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2078 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2079 break; 2080 case RTE_FLOW_ITEM_TYPE_TCP: 2081 flow_dv_translate_item_tcp(match_mask, match_value, 2082 items, tunnel); 2083 matcher.priority = MLX5_PRIORITY_MAP_L4; 2084 dev_flow->dv.hash_fields |= 2085 mlx5_flow_hashfields_adjust 2086 (dev_flow, tunnel, ETH_RSS_TCP, 2087 IBV_RX_HASH_SRC_PORT_TCP | 2088 IBV_RX_HASH_DST_PORT_TCP); 2089 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : 2090 MLX5_FLOW_LAYER_OUTER_L4_TCP; 2091 break; 2092 case RTE_FLOW_ITEM_TYPE_UDP: 2093 flow_dv_translate_item_udp(match_mask, match_value, 2094 items, tunnel); 2095 matcher.priority = MLX5_PRIORITY_MAP_L4; 2096 dev_flow->dv.hash_fields |= 2097 mlx5_flow_hashfields_adjust 2098 (dev_flow, tunnel, ETH_RSS_UDP, 2099 IBV_RX_HASH_SRC_PORT_UDP | 2100 IBV_RX_HASH_DST_PORT_UDP); 2101 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : 2102 MLX5_FLOW_LAYER_OUTER_L4_UDP; 2103 break; 2104 case RTE_FLOW_ITEM_TYPE_GRE: 2105 flow_dv_translate_item_gre(match_mask, match_value, 2106 items, tunnel); 2107 last_item = MLX5_FLOW_LAYER_GRE; 2108 break; 2109 case RTE_FLOW_ITEM_TYPE_NVGRE: 2110 flow_dv_translate_item_nvgre(match_mask, match_value, 2111 items, tunnel); 2112 last_item = MLX5_FLOW_LAYER_GRE; 2113 break; 2114 case RTE_FLOW_ITEM_TYPE_VXLAN: 2115 flow_dv_translate_item_vxlan(match_mask, match_value, 2116 items, tunnel); 2117 last_item = MLX5_FLOW_LAYER_VXLAN; 2118 break; 2119 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 2120 flow_dv_translate_item_vxlan(match_mask, match_value, 2121 items, tunnel); 2122 last_item = MLX5_FLOW_LAYER_VXLAN_GPE; 2123 break; 2124 case RTE_FLOW_ITEM_TYPE_MPLS: 2125 flow_dv_translate_item_mpls(match_mask, match_value, 2126 items, last_item, tunnel); 2127 last_item = MLX5_FLOW_LAYER_MPLS; 2128 break; 2129 case RTE_FLOW_ITEM_TYPE_META: 2130 flow_dv_translate_item_meta(match_mask, match_value, 2131 items); 2132 last_item = MLX5_FLOW_ITEM_METADATA; 2133 break; 2134 default: 2135 break; 2136 } 2137 item_flags |= last_item; 2138 } 2139 assert(!flow_dv_check_valid_spec(matcher.mask.buf, 2140 dev_flow->dv.value.buf)); 2141 dev_flow->layers = item_flags; 2142 /* Register matcher. */ 2143 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, 2144 matcher.mask.size); 2145 matcher.priority = mlx5_flow_adjust_priority(dev, priority, 2146 matcher.priority); 2147 matcher.egress = attr->egress; 2148 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error)) 2149 return -rte_errno; 2150 return 0; 2151 } 2152 2153 /** 2154 * Apply the flow to the NIC. 2155 * 2156 * @param[in] dev 2157 * Pointer to the Ethernet device structure. 2158 * @param[in, out] flow 2159 * Pointer to flow structure. 2160 * @param[out] error 2161 * Pointer to error structure. 2162 * 2163 * @return 2164 * 0 on success, a negative errno value otherwise and rte_errno is set. 2165 */ 2166 static int 2167 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 2168 struct rte_flow_error *error) 2169 { 2170 struct mlx5_flow_dv *dv; 2171 struct mlx5_flow *dev_flow; 2172 int n; 2173 int err; 2174 2175 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 2176 dv = &dev_flow->dv; 2177 n = dv->actions_n; 2178 if (flow->actions & MLX5_FLOW_ACTION_DROP) { 2179 dv->hrxq = mlx5_hrxq_drop_new(dev); 2180 if (!dv->hrxq) { 2181 rte_flow_error_set 2182 (error, errno, 2183 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2184 "cannot get drop hash queue"); 2185 goto error; 2186 } 2187 dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP; 2188 dv->actions[n].qp = dv->hrxq->qp; 2189 n++; 2190 } else if (flow->actions & 2191 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) { 2192 struct mlx5_hrxq *hrxq; 2193 2194 hrxq = mlx5_hrxq_get(dev, flow->key, 2195 MLX5_RSS_HASH_KEY_LEN, 2196 dv->hash_fields, 2197 (*flow->queue), 2198 flow->rss.queue_num); 2199 if (!hrxq) 2200 hrxq = mlx5_hrxq_new 2201 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN, 2202 dv->hash_fields, (*flow->queue), 2203 flow->rss.queue_num, 2204 !!(dev_flow->layers & 2205 MLX5_FLOW_LAYER_TUNNEL)); 2206 if (!hrxq) { 2207 rte_flow_error_set 2208 (error, rte_errno, 2209 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2210 "cannot get hash queue"); 2211 goto error; 2212 } 2213 dv->hrxq = hrxq; 2214 dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP; 2215 dv->actions[n].qp = hrxq->qp; 2216 n++; 2217 } 2218 dv->flow = 2219 mlx5_glue->dv_create_flow(dv->matcher->matcher_object, 2220 (void *)&dv->value, n, 2221 dv->actions); 2222 if (!dv->flow) { 2223 rte_flow_error_set(error, errno, 2224 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2225 NULL, 2226 "hardware refuses to create flow"); 2227 goto error; 2228 } 2229 } 2230 return 0; 2231 error: 2232 err = rte_errno; /* Save rte_errno before cleanup. */ 2233 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 2234 struct mlx5_flow_dv *dv = &dev_flow->dv; 2235 if (dv->hrxq) { 2236 if (flow->actions & MLX5_FLOW_ACTION_DROP) 2237 mlx5_hrxq_drop_release(dev); 2238 else 2239 mlx5_hrxq_release(dev, dv->hrxq); 2240 dv->hrxq = NULL; 2241 } 2242 } 2243 rte_errno = err; /* Restore rte_errno. */ 2244 return -rte_errno; 2245 } 2246 2247 /** 2248 * Release the flow matcher. 2249 * 2250 * @param dev 2251 * Pointer to Ethernet device. 2252 * @param flow 2253 * Pointer to mlx5_flow. 2254 * 2255 * @return 2256 * 1 while a reference on it exists, 0 when freed. 2257 */ 2258 static int 2259 flow_dv_matcher_release(struct rte_eth_dev *dev, 2260 struct mlx5_flow *flow) 2261 { 2262 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher; 2263 2264 assert(matcher->matcher_object); 2265 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--", 2266 dev->data->port_id, (void *)matcher, 2267 rte_atomic32_read(&matcher->refcnt)); 2268 if (rte_atomic32_dec_and_test(&matcher->refcnt)) { 2269 claim_zero(mlx5_glue->dv_destroy_flow_matcher 2270 (matcher->matcher_object)); 2271 LIST_REMOVE(matcher, next); 2272 rte_free(matcher); 2273 DRV_LOG(DEBUG, "port %u matcher %p: removed", 2274 dev->data->port_id, (void *)matcher); 2275 return 0; 2276 } 2277 return 1; 2278 } 2279 2280 /** 2281 * Release an encap/decap resource. 2282 * 2283 * @param flow 2284 * Pointer to mlx5_flow. 2285 * 2286 * @return 2287 * 1 while a reference on it exists, 0 when freed. 2288 */ 2289 static int 2290 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow) 2291 { 2292 struct mlx5_flow_dv_encap_decap_resource *cache_resource = 2293 flow->dv.encap_decap; 2294 2295 assert(cache_resource->verbs_action); 2296 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--", 2297 (void *)cache_resource, 2298 rte_atomic32_read(&cache_resource->refcnt)); 2299 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) { 2300 claim_zero(mlx5_glue->destroy_flow_action 2301 (cache_resource->verbs_action)); 2302 LIST_REMOVE(cache_resource, next); 2303 rte_free(cache_resource); 2304 DRV_LOG(DEBUG, "encap/decap resource %p: removed", 2305 (void *)cache_resource); 2306 return 0; 2307 } 2308 return 1; 2309 } 2310 2311 /** 2312 * Remove the flow from the NIC but keeps it in memory. 2313 * 2314 * @param[in] dev 2315 * Pointer to Ethernet device. 2316 * @param[in, out] flow 2317 * Pointer to flow structure. 2318 */ 2319 static void 2320 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow) 2321 { 2322 struct mlx5_flow_dv *dv; 2323 struct mlx5_flow *dev_flow; 2324 2325 if (!flow) 2326 return; 2327 LIST_FOREACH(dev_flow, &flow->dev_flows, next) { 2328 dv = &dev_flow->dv; 2329 if (dv->flow) { 2330 claim_zero(mlx5_glue->destroy_flow(dv->flow)); 2331 dv->flow = NULL; 2332 } 2333 if (dv->hrxq) { 2334 if (flow->actions & MLX5_FLOW_ACTION_DROP) 2335 mlx5_hrxq_drop_release(dev); 2336 else 2337 mlx5_hrxq_release(dev, dv->hrxq); 2338 dv->hrxq = NULL; 2339 } 2340 } 2341 if (flow->counter) 2342 flow->counter = NULL; 2343 } 2344 2345 /** 2346 * Remove the flow from the NIC and the memory. 2347 * 2348 * @param[in] dev 2349 * Pointer to the Ethernet device structure. 2350 * @param[in, out] flow 2351 * Pointer to flow structure. 2352 */ 2353 static void 2354 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 2355 { 2356 struct mlx5_flow *dev_flow; 2357 2358 if (!flow) 2359 return; 2360 flow_dv_remove(dev, flow); 2361 while (!LIST_EMPTY(&flow->dev_flows)) { 2362 dev_flow = LIST_FIRST(&flow->dev_flows); 2363 LIST_REMOVE(dev_flow, next); 2364 if (dev_flow->dv.matcher) 2365 flow_dv_matcher_release(dev, dev_flow); 2366 if (dev_flow->dv.encap_decap) 2367 flow_dv_encap_decap_resource_release(dev_flow); 2368 rte_free(dev_flow); 2369 } 2370 } 2371 2372 /** 2373 * Query a flow. 2374 * 2375 * @see rte_flow_query() 2376 * @see rte_flow_ops 2377 */ 2378 static int 2379 flow_dv_query(struct rte_eth_dev *dev __rte_unused, 2380 struct rte_flow *flow __rte_unused, 2381 const struct rte_flow_action *actions __rte_unused, 2382 void *data __rte_unused, 2383 struct rte_flow_error *error __rte_unused) 2384 { 2385 return rte_flow_error_set(error, ENOTSUP, 2386 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2387 NULL, 2388 "flow query with DV is not supported"); 2389 } 2390 2391 2392 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { 2393 .validate = flow_dv_validate, 2394 .prepare = flow_dv_prepare, 2395 .translate = flow_dv_translate, 2396 .apply = flow_dv_apply, 2397 .remove = flow_dv_remove, 2398 .destroy = flow_dv_destroy, 2399 .query = flow_dv_query, 2400 }; 2401 2402 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 2403