1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_log.h> 16 #include <rte_memzone.h> 17 #include <rte_malloc.h> 18 #include <rte_arp.h> 19 #include <rte_ip.h> 20 #include <rte_udp.h> 21 #include <rte_tcp.h> 22 #include <rte_sctp.h> 23 #include <rte_hash_crc.h> 24 #include <rte_bitmap.h> 25 #include <rte_os_shim.h> 26 27 #include "i40e_logs.h" 28 #include "base/i40e_type.h" 29 #include "base/i40e_prototype.h" 30 #include "i40e_ethdev.h" 31 #include "i40e_rxtx.h" 32 33 #define I40E_FDIR_MZ_NAME "FDIR_MEMZONE" 34 #ifndef IPV6_ADDR_LEN 35 #define IPV6_ADDR_LEN 16 36 #endif 37 38 #ifndef IPPROTO_L2TP 39 #define IPPROTO_L2TP 115 40 #endif 41 42 #define I40E_FDIR_PKT_LEN 512 43 #define I40E_FDIR_IP_DEFAULT_LEN 420 44 #define I40E_FDIR_IP_DEFAULT_TTL 0x40 45 #define I40E_FDIR_IP_DEFAULT_VERSION_IHL 0x45 46 #define I40E_FDIR_TCP_DEFAULT_DATAOFF 0x50 47 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW 0x60000000 48 49 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS 0xFF 50 #define I40E_FDIR_IPv6_PAYLOAD_LEN 380 51 #define I40E_FDIR_UDP_DEFAULT_LEN 400 52 #define I40E_FDIR_GTP_DEFAULT_LEN 384 53 #define I40E_FDIR_INNER_IP_DEFAULT_LEN 384 54 #define I40E_FDIR_INNER_IPV6_DEFAULT_LEN 344 55 56 #define I40E_FDIR_GTPC_DST_PORT 2123 57 #define I40E_FDIR_GTPU_DST_PORT 2152 58 #define I40E_FDIR_GTP_VER_FLAG_0X30 0x30 59 #define I40E_FDIR_GTP_VER_FLAG_0X32 0x32 60 #define I40E_FDIR_GTP_MSG_TYPE_0X01 0x01 61 #define I40E_FDIR_GTP_MSG_TYPE_0XFF 0xFF 62 63 #define I40E_FDIR_ESP_DST_PORT 4500 64 65 /* Wait time for fdir filter programming */ 66 #define I40E_FDIR_MAX_WAIT_US 10000 67 68 /* Wait count and interval for fdir filter flush */ 69 #define I40E_FDIR_FLUSH_RETRY 50 70 #define I40E_FDIR_FLUSH_INTERVAL_MS 5 71 72 #define I40E_COUNTER_PF 2 73 /* Statistic counter index for one pf */ 74 #define I40E_COUNTER_INDEX_FDIR(pf_id) (0 + (pf_id) * I40E_COUNTER_PF) 75 76 #define I40E_FDIR_FLOWS ( \ 77 (1ULL << RTE_ETH_FLOW_FRAG_IPV4) | \ 78 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \ 79 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \ 80 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \ 81 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \ 82 (1ULL << RTE_ETH_FLOW_FRAG_IPV6) | \ 83 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \ 84 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \ 85 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \ 86 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \ 87 (1ULL << RTE_ETH_FLOW_L2_PAYLOAD)) 88 89 static int i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input, 90 struct i40e_fdir_filter *filter); 91 static struct i40e_fdir_filter * 92 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info, 93 const struct i40e_fdir_input *input); 94 static int i40e_sw_fdir_filter_insert(struct i40e_pf *pf, 95 struct i40e_fdir_filter *filter); 96 static int 97 i40e_flow_fdir_filter_programming(struct i40e_pf *pf, 98 enum i40e_filter_pctype pctype, 99 const struct i40e_fdir_filter_conf *filter, 100 bool add, bool wait_status); 101 102 static int 103 i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq) 104 { 105 struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi); 106 struct i40e_hmc_obj_rxq rx_ctx; 107 int err = I40E_SUCCESS; 108 109 memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 110 /* Init the RX queue in hardware */ 111 rx_ctx.dbuff = I40E_RXBUF_SZ_1024 >> I40E_RXQ_CTX_DBUFF_SHIFT; 112 rx_ctx.hbuff = 0; 113 rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT; 114 rx_ctx.qlen = rxq->nb_rx_desc; 115 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC 116 rx_ctx.dsize = 1; 117 #endif 118 rx_ctx.dtype = i40e_header_split_none; 119 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE; 120 rx_ctx.rxmax = I40E_ETH_MAX_LEN; 121 rx_ctx.tphrdesc_ena = 1; 122 rx_ctx.tphwdesc_ena = 1; 123 rx_ctx.tphdata_ena = 1; 124 rx_ctx.tphhead_ena = 1; 125 rx_ctx.lrxqthresh = 2; 126 rx_ctx.crcstrip = 0; 127 rx_ctx.l2tsel = 1; 128 rx_ctx.showiv = 0; 129 rx_ctx.prefena = 1; 130 131 err = i40e_clear_lan_rx_queue_context(hw, rxq->reg_idx); 132 if (err != I40E_SUCCESS) { 133 PMD_DRV_LOG(ERR, "Failed to clear FDIR RX queue context."); 134 return err; 135 } 136 err = i40e_set_lan_rx_queue_context(hw, rxq->reg_idx, &rx_ctx); 137 if (err != I40E_SUCCESS) { 138 PMD_DRV_LOG(ERR, "Failed to set FDIR RX queue context."); 139 return err; 140 } 141 rxq->qrx_tail = hw->hw_addr + 142 I40E_QRX_TAIL(rxq->vsi->base_queue); 143 144 rte_wmb(); 145 /* Init the RX tail regieter. */ 146 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 147 148 return err; 149 } 150 151 /* 152 * i40e_fdir_setup - reserve and initialize the Flow Director resources 153 * @pf: board private structure 154 */ 155 int 156 i40e_fdir_setup(struct i40e_pf *pf) 157 { 158 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 159 struct i40e_vsi *vsi; 160 int err = I40E_SUCCESS; 161 char z_name[RTE_MEMZONE_NAMESIZE]; 162 const struct rte_memzone *mz = NULL; 163 struct rte_eth_dev *eth_dev = pf->adapter->eth_dev; 164 uint16_t i; 165 166 if ((pf->flags & I40E_FLAG_FDIR) == 0) { 167 PMD_INIT_LOG(ERR, "HW doesn't support FDIR"); 168 return I40E_NOT_SUPPORTED; 169 } 170 171 PMD_DRV_LOG(INFO, "FDIR HW Capabilities: num_filters_guaranteed = %u," 172 " num_filters_best_effort = %u.", 173 hw->func_caps.fd_filters_guaranteed, 174 hw->func_caps.fd_filters_best_effort); 175 176 vsi = pf->fdir.fdir_vsi; 177 if (vsi) { 178 PMD_DRV_LOG(INFO, "FDIR initialization has been done."); 179 return I40E_SUCCESS; 180 } 181 182 /* make new FDIR VSI */ 183 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0); 184 if (!vsi) { 185 PMD_DRV_LOG(ERR, "Couldn't create FDIR VSI."); 186 return I40E_ERR_NO_AVAILABLE_VSI; 187 } 188 pf->fdir.fdir_vsi = vsi; 189 190 /*Fdir tx queue setup*/ 191 err = i40e_fdir_setup_tx_resources(pf); 192 if (err) { 193 PMD_DRV_LOG(ERR, "Failed to setup FDIR TX resources."); 194 goto fail_setup_tx; 195 } 196 197 /*Fdir rx queue setup*/ 198 err = i40e_fdir_setup_rx_resources(pf); 199 if (err) { 200 PMD_DRV_LOG(ERR, "Failed to setup FDIR RX resources."); 201 goto fail_setup_rx; 202 } 203 204 err = i40e_tx_queue_init(pf->fdir.txq); 205 if (err) { 206 PMD_DRV_LOG(ERR, "Failed to do FDIR TX initialization."); 207 goto fail_mem; 208 } 209 210 /* need switch on before dev start*/ 211 err = i40e_switch_tx_queue(hw, vsi->base_queue, TRUE); 212 if (err) { 213 PMD_DRV_LOG(ERR, "Failed to do fdir TX switch on."); 214 goto fail_mem; 215 } 216 217 /* Init the rx queue in hardware */ 218 err = i40e_fdir_rx_queue_init(pf->fdir.rxq); 219 if (err) { 220 PMD_DRV_LOG(ERR, "Failed to do FDIR RX initialization."); 221 goto fail_mem; 222 } 223 224 /* switch on rx queue */ 225 err = i40e_switch_rx_queue(hw, vsi->base_queue, TRUE); 226 if (err) { 227 PMD_DRV_LOG(ERR, "Failed to do FDIR RX switch on."); 228 goto fail_mem; 229 } 230 231 /* enable FDIR MSIX interrupt */ 232 vsi->nb_used_qps = 1; 233 i40e_vsi_queues_bind_intr(vsi, I40E_ITR_INDEX_NONE); 234 i40e_vsi_enable_queues_intr(vsi); 235 236 /* reserve memory for the fdir programming packet */ 237 snprintf(z_name, sizeof(z_name), "%s_%s_%d", 238 eth_dev->device->driver->name, 239 I40E_FDIR_MZ_NAME, 240 eth_dev->data->port_id); 241 mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN * 242 I40E_FDIR_PRG_PKT_CNT, SOCKET_ID_ANY); 243 if (!mz) { 244 PMD_DRV_LOG(ERR, "Cannot init memzone for " 245 "flow director program packet."); 246 err = I40E_ERR_NO_MEMORY; 247 goto fail_mem; 248 } 249 250 for (i = 0; i < I40E_FDIR_PRG_PKT_CNT; i++) { 251 pf->fdir.prg_pkt[i] = (uint8_t *)mz->addr + 252 I40E_FDIR_PKT_LEN * i; 253 pf->fdir.dma_addr[i] = mz->iova + 254 I40E_FDIR_PKT_LEN * i; 255 } 256 257 pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id); 258 pf->fdir.fdir_actual_cnt = 0; 259 pf->fdir.fdir_guarantee_free_space = 260 pf->fdir.fdir_guarantee_total_space; 261 262 PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.", 263 vsi->base_queue); 264 return I40E_SUCCESS; 265 266 fail_mem: 267 i40e_dev_rx_queue_release(pf->fdir.rxq); 268 pf->fdir.rxq = NULL; 269 fail_setup_rx: 270 i40e_dev_tx_queue_release(pf->fdir.txq); 271 pf->fdir.txq = NULL; 272 fail_setup_tx: 273 i40e_vsi_release(vsi); 274 pf->fdir.fdir_vsi = NULL; 275 return err; 276 } 277 278 /* 279 * i40e_fdir_teardown - release the Flow Director resources 280 * @pf: board private structure 281 */ 282 void 283 i40e_fdir_teardown(struct i40e_pf *pf) 284 { 285 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 286 struct i40e_vsi *vsi; 287 struct rte_eth_dev *dev = pf->adapter->eth_dev; 288 289 vsi = pf->fdir.fdir_vsi; 290 if (!vsi) 291 return; 292 293 /* disable FDIR MSIX interrupt */ 294 i40e_vsi_queues_unbind_intr(vsi); 295 i40e_vsi_disable_queues_intr(vsi); 296 297 int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE); 298 if (err) 299 PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off"); 300 err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE); 301 if (err) 302 PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off"); 303 304 i40e_dev_rx_queue_release(pf->fdir.rxq); 305 rte_eth_dma_zone_free(dev, "fdir_rx_ring", pf->fdir.rxq->queue_id); 306 pf->fdir.rxq = NULL; 307 i40e_dev_tx_queue_release(pf->fdir.txq); 308 rte_eth_dma_zone_free(dev, "fdir_tx_ring", pf->fdir.txq->queue_id); 309 pf->fdir.txq = NULL; 310 i40e_vsi_release(vsi); 311 pf->fdir.fdir_vsi = NULL; 312 } 313 314 /* check whether the flow director table in empty */ 315 static inline int 316 i40e_fdir_empty(struct i40e_hw *hw) 317 { 318 uint32_t guarant_cnt, best_cnt; 319 320 guarant_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) & 321 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >> 322 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT); 323 best_cnt = (uint32_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) & 324 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 325 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 326 if (best_cnt + guarant_cnt > 0) 327 return -1; 328 329 return 0; 330 } 331 332 /* 333 * Initialize the configuration about bytes stream extracted as flexible payload 334 * and mask setting 335 */ 336 static inline void 337 i40e_init_flx_pld(struct i40e_pf *pf) 338 { 339 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 340 uint8_t pctype; 341 int i, index; 342 uint16_t flow_type; 343 344 /* 345 * Define the bytes stream extracted as flexible payload in 346 * field vector. By default, select 8 words from the beginning 347 * of payload as flexible payload. 348 */ 349 for (i = I40E_FLXPLD_L2_IDX; i < I40E_MAX_FLXPLD_LAYER; i++) { 350 index = i * I40E_MAX_FLXPLD_FIED; 351 pf->fdir.flex_set[index].src_offset = 0; 352 pf->fdir.flex_set[index].size = I40E_FDIR_MAX_FLEXWORD_NUM; 353 pf->fdir.flex_set[index].dst_offset = 0; 354 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(index), 0x0000C900); 355 I40E_WRITE_REG(hw, 356 I40E_PRTQF_FLX_PIT(index + 1), 0x0000FC29);/*non-used*/ 357 I40E_WRITE_REG(hw, 358 I40E_PRTQF_FLX_PIT(index + 2), 0x0000FC2A);/*non-used*/ 359 pf->fdir.flex_pit_flag[i] = 0; 360 } 361 362 /* initialize the masks */ 363 for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 364 pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) { 365 flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype); 366 367 if (flow_type == RTE_ETH_FLOW_UNKNOWN) 368 continue; 369 pf->fdir.flex_mask[pctype].word_mask = 0; 370 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0); 371 for (i = 0; i < I40E_FDIR_BITMASK_NUM_WORD; i++) { 372 pf->fdir.flex_mask[pctype].bitmask[i].offset = 0; 373 pf->fdir.flex_mask[pctype].bitmask[i].mask = 0; 374 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), 0); 375 } 376 } 377 } 378 379 #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \ 380 if ((flex_pit2).src_offset < \ 381 (flex_pit1).src_offset + (flex_pit1).size) { \ 382 PMD_DRV_LOG(ERR, "src_offset should be not" \ 383 " less than than previous offset" \ 384 " + previous FSIZE."); \ 385 return -EINVAL; \ 386 } \ 387 } while (0) 388 389 /* 390 * i40e_srcoff_to_flx_pit - transform the src_offset into flex_pit structure, 391 * and the flex_pit will be sorted by it's src_offset value 392 */ 393 static inline uint16_t 394 i40e_srcoff_to_flx_pit(const uint16_t *src_offset, 395 struct i40e_fdir_flex_pit *flex_pit) 396 { 397 uint16_t src_tmp, size, num = 0; 398 uint16_t i, k, j = 0; 399 400 while (j < I40E_FDIR_MAX_FLEX_LEN) { 401 size = 1; 402 for (; j < I40E_FDIR_MAX_FLEX_LEN - 1; j++) { 403 if (src_offset[j + 1] == src_offset[j] + 1) 404 size++; 405 else 406 break; 407 } 408 src_tmp = src_offset[j] + 1 - size; 409 /* the flex_pit need to be sort by src_offset */ 410 for (i = 0; i < num; i++) { 411 if (src_tmp < flex_pit[i].src_offset) 412 break; 413 } 414 /* if insert required, move backward */ 415 for (k = num; k > i; k--) 416 flex_pit[k] = flex_pit[k - 1]; 417 /* insert */ 418 flex_pit[i].dst_offset = j + 1 - size; 419 flex_pit[i].src_offset = src_tmp; 420 flex_pit[i].size = size; 421 j++; 422 num++; 423 } 424 return num; 425 } 426 427 /* i40e_check_fdir_flex_payload -check flex payload configuration arguments */ 428 static inline int 429 i40e_check_fdir_flex_payload(const struct rte_eth_flex_payload_cfg *flex_cfg) 430 { 431 struct i40e_fdir_flex_pit flex_pit[I40E_FDIR_MAX_FLEX_LEN]; 432 uint16_t num, i; 433 434 for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i++) { 435 if (flex_cfg->src_offset[i] >= I40E_MAX_FLX_SOURCE_OFF) { 436 PMD_DRV_LOG(ERR, "exceeds maxmial payload limit."); 437 return -EINVAL; 438 } 439 } 440 441 memset(flex_pit, 0, sizeof(flex_pit)); 442 num = i40e_srcoff_to_flx_pit(flex_cfg->src_offset, flex_pit); 443 if (num > I40E_MAX_FLXPLD_FIED) { 444 PMD_DRV_LOG(ERR, "exceeds maxmial number of flex fields."); 445 return -EINVAL; 446 } 447 for (i = 0; i < num; i++) { 448 if (flex_pit[i].size & 0x01 || flex_pit[i].dst_offset & 0x01 || 449 flex_pit[i].src_offset & 0x01) { 450 PMD_DRV_LOG(ERR, "flexpayload should be measured" 451 " in word"); 452 return -EINVAL; 453 } 454 if (i != num - 1) 455 I40E_VALIDATE_FLEX_PIT(flex_pit[i], flex_pit[i + 1]); 456 } 457 return 0; 458 } 459 460 /* 461 * i40e_check_fdir_flex_conf -check if the flex payload and mask configuration 462 * arguments are valid 463 */ 464 static int 465 i40e_check_fdir_flex_conf(const struct i40e_adapter *adapter, 466 const struct rte_eth_fdir_flex_conf *conf) 467 { 468 const struct rte_eth_flex_payload_cfg *flex_cfg; 469 const struct rte_eth_fdir_flex_mask *flex_mask; 470 uint16_t mask_tmp; 471 uint8_t nb_bitmask; 472 uint16_t i, j; 473 int ret = 0; 474 enum i40e_filter_pctype pctype; 475 476 if (conf == NULL) { 477 PMD_DRV_LOG(INFO, "NULL pointer."); 478 return -EINVAL; 479 } 480 /* check flexible payload setting configuration */ 481 if (conf->nb_payloads > RTE_ETH_L4_PAYLOAD) { 482 PMD_DRV_LOG(ERR, "invalid number of payload setting."); 483 return -EINVAL; 484 } 485 for (i = 0; i < conf->nb_payloads; i++) { 486 flex_cfg = &conf->flex_set[i]; 487 if (flex_cfg->type > RTE_ETH_L4_PAYLOAD) { 488 PMD_DRV_LOG(ERR, "invalid payload type."); 489 return -EINVAL; 490 } 491 ret = i40e_check_fdir_flex_payload(flex_cfg); 492 if (ret < 0) { 493 PMD_DRV_LOG(ERR, "invalid flex payload arguments."); 494 return -EINVAL; 495 } 496 } 497 498 /* check flex mask setting configuration */ 499 if (conf->nb_flexmasks >= RTE_ETH_FLOW_MAX) { 500 PMD_DRV_LOG(ERR, "invalid number of flex masks."); 501 return -EINVAL; 502 } 503 for (i = 0; i < conf->nb_flexmasks; i++) { 504 flex_mask = &conf->flex_mask[i]; 505 pctype = i40e_flowtype_to_pctype(adapter, flex_mask->flow_type); 506 if (pctype == I40E_FILTER_PCTYPE_INVALID) { 507 PMD_DRV_LOG(WARNING, "invalid flow type."); 508 return -EINVAL; 509 } 510 nb_bitmask = 0; 511 for (j = 0; j < I40E_FDIR_MAX_FLEX_LEN; j += sizeof(uint16_t)) { 512 mask_tmp = I40E_WORD(flex_mask->mask[j], 513 flex_mask->mask[j + 1]); 514 if (mask_tmp != 0x0 && mask_tmp != UINT16_MAX) { 515 nb_bitmask++; 516 if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) { 517 PMD_DRV_LOG(ERR, " exceed maximal" 518 " number of bitmasks."); 519 return -EINVAL; 520 } 521 } 522 } 523 } 524 return 0; 525 } 526 527 /* 528 * i40e_set_flx_pld_cfg -configure the rule how bytes stream is extracted as flexible payload 529 * @pf: board private structure 530 * @cfg: the rule how bytes stream is extracted as flexible payload 531 */ 532 static void 533 i40e_set_flx_pld_cfg(struct i40e_pf *pf, 534 const struct rte_eth_flex_payload_cfg *cfg) 535 { 536 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 537 struct i40e_fdir_flex_pit flex_pit[I40E_MAX_FLXPLD_FIED]; 538 uint32_t flx_pit, flx_ort; 539 uint16_t num, min_next_off; /* in words */ 540 uint8_t field_idx = 0; 541 uint8_t layer_idx = 0; 542 uint16_t i; 543 544 if (cfg->type == RTE_ETH_L2_PAYLOAD) 545 layer_idx = I40E_FLXPLD_L2_IDX; 546 else if (cfg->type == RTE_ETH_L3_PAYLOAD) 547 layer_idx = I40E_FLXPLD_L3_IDX; 548 else if (cfg->type == RTE_ETH_L4_PAYLOAD) 549 layer_idx = I40E_FLXPLD_L4_IDX; 550 551 memset(flex_pit, 0, sizeof(flex_pit)); 552 num = RTE_MIN(i40e_srcoff_to_flx_pit(cfg->src_offset, flex_pit), 553 RTE_DIM(flex_pit)); 554 555 if (num) { 556 flx_ort = (1 << I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) | 557 (num << I40E_GLQF_ORT_FIELD_CNT_SHIFT) | 558 (layer_idx * I40E_MAX_FLXPLD_FIED); 559 I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort); 560 } 561 562 for (i = 0; i < num; i++) { 563 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; 564 /* record the info in fdir structure */ 565 pf->fdir.flex_set[field_idx].src_offset = 566 flex_pit[i].src_offset / sizeof(uint16_t); 567 pf->fdir.flex_set[field_idx].size = 568 flex_pit[i].size / sizeof(uint16_t); 569 pf->fdir.flex_set[field_idx].dst_offset = 570 flex_pit[i].dst_offset / sizeof(uint16_t); 571 flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset, 572 pf->fdir.flex_set[field_idx].size, 573 pf->fdir.flex_set[field_idx].dst_offset); 574 575 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); 576 } 577 min_next_off = pf->fdir.flex_set[field_idx].src_offset + 578 pf->fdir.flex_set[field_idx].size; 579 580 for (; i < I40E_MAX_FLXPLD_FIED; i++) { 581 /* set the non-used register obeying register's constrain */ 582 flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE, 583 NONUSE_FLX_PIT_DEST_OFF); 584 I40E_WRITE_REG(hw, 585 I40E_PRTQF_FLX_PIT(layer_idx * I40E_MAX_FLXPLD_FIED + i), 586 flx_pit); 587 min_next_off++; 588 } 589 } 590 591 /* 592 * i40e_set_flex_mask_on_pctype - configure the mask on flexible payload 593 * @pf: board private structure 594 * @pctype: packet classify type 595 * @flex_masks: mask for flexible payload 596 */ 597 static void 598 i40e_set_flex_mask_on_pctype(struct i40e_pf *pf, 599 enum i40e_filter_pctype pctype, 600 const struct rte_eth_fdir_flex_mask *mask_cfg) 601 { 602 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 603 struct i40e_fdir_flex_mask *flex_mask; 604 uint32_t flxinset, fd_mask; 605 uint16_t mask_tmp; 606 uint8_t i, nb_bitmask = 0; 607 608 flex_mask = &pf->fdir.flex_mask[pctype]; 609 memset(flex_mask, 0, sizeof(struct i40e_fdir_flex_mask)); 610 for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) { 611 mask_tmp = I40E_WORD(mask_cfg->mask[i], mask_cfg->mask[i + 1]); 612 if (mask_tmp != 0x0) { 613 flex_mask->word_mask |= 614 I40E_FLEX_WORD_MASK(i / sizeof(uint16_t)); 615 if (mask_tmp != UINT16_MAX) { 616 /* set bit mask */ 617 flex_mask->bitmask[nb_bitmask].mask = ~mask_tmp; 618 flex_mask->bitmask[nb_bitmask].offset = 619 i / sizeof(uint16_t); 620 nb_bitmask++; 621 } 622 } 623 } 624 /* write mask to hw */ 625 flxinset = (flex_mask->word_mask << 626 I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) & 627 I40E_PRTQF_FD_FLXINSET_INSET_MASK; 628 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset); 629 630 for (i = 0; i < nb_bitmask; i++) { 631 fd_mask = (flex_mask->bitmask[i].mask << 632 I40E_PRTQF_FD_MSK_MASK_SHIFT) & 633 I40E_PRTQF_FD_MSK_MASK_MASK; 634 fd_mask |= ((flex_mask->bitmask[i].offset + 635 I40E_FLX_OFFSET_IN_FIELD_VECTOR) << 636 I40E_PRTQF_FD_MSK_OFFSET_SHIFT) & 637 I40E_PRTQF_FD_MSK_OFFSET_MASK; 638 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask); 639 } 640 } 641 642 /* 643 * Enable/disable flow director RX processing in vector routines. 644 */ 645 void 646 i40e_fdir_rx_proc_enable(struct rte_eth_dev *dev, bool on) 647 { 648 int32_t i; 649 650 for (i = 0; i < dev->data->nb_rx_queues; i++) { 651 struct i40e_rx_queue *rxq = dev->data->rx_queues[i]; 652 if (!rxq) 653 continue; 654 rxq->fdir_enabled = on; 655 } 656 PMD_DRV_LOG(DEBUG, "Flow Director processing on RX set to %d", on); 657 } 658 659 /* 660 * Configure flow director related setting 661 */ 662 int 663 i40e_fdir_configure(struct rte_eth_dev *dev) 664 { 665 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 666 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 667 struct rte_eth_fdir_flex_conf *conf; 668 enum i40e_filter_pctype pctype; 669 uint32_t val; 670 uint8_t i; 671 int ret = 0; 672 673 /* 674 * configuration need to be done before 675 * flow director filters are added 676 * If filters exist, flush them. 677 */ 678 if (i40e_fdir_empty(hw) < 0) { 679 ret = i40e_fdir_flush(dev); 680 if (ret) { 681 PMD_DRV_LOG(ERR, "failed to flush fdir table."); 682 return ret; 683 } 684 } 685 686 /* enable FDIR filter */ 687 val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 688 val |= I40E_PFQF_CTL_0_FD_ENA_MASK; 689 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, val); 690 691 i40e_init_flx_pld(pf); /* set flex config to default value */ 692 693 conf = &dev->data->dev_conf.fdir_conf.flex_conf; 694 ret = i40e_check_fdir_flex_conf(pf->adapter, conf); 695 if (ret < 0) { 696 PMD_DRV_LOG(ERR, " invalid configuration arguments."); 697 return -EINVAL; 698 } 699 700 if (!pf->support_multi_driver) { 701 /* configure flex payload */ 702 for (i = 0; i < conf->nb_payloads; i++) 703 i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]); 704 /* configure flex mask*/ 705 for (i = 0; i < conf->nb_flexmasks; i++) { 706 if (hw->mac.type == I40E_MAC_X722) { 707 /* get pctype value in fd pctype register */ 708 pctype = (enum i40e_filter_pctype) 709 i40e_read_rx_ctl(hw, 710 I40E_GLQF_FD_PCTYPES( 711 (int)i40e_flowtype_to_pctype( 712 pf->adapter, 713 conf->flex_mask[i].flow_type))); 714 } else { 715 pctype = i40e_flowtype_to_pctype(pf->adapter, 716 conf->flex_mask[i].flow_type); 717 } 718 719 i40e_set_flex_mask_on_pctype(pf, pctype, 720 &conf->flex_mask[i]); 721 } 722 } else { 723 PMD_DRV_LOG(ERR, "Not support flexible payload."); 724 } 725 726 /* Enable FDIR processing in RX routines */ 727 i40e_fdir_rx_proc_enable(dev, 1); 728 729 return ret; 730 } 731 732 733 static struct i40e_customized_pctype * 734 i40e_flow_fdir_find_customized_pctype(struct i40e_pf *pf, uint8_t pctype) 735 { 736 struct i40e_customized_pctype *cus_pctype; 737 enum i40e_new_pctype i = I40E_CUSTOMIZED_GTPC; 738 739 for (; i < I40E_CUSTOMIZED_MAX; i++) { 740 cus_pctype = &pf->customized_pctype[i]; 741 if (pctype == cus_pctype->pctype) 742 return cus_pctype; 743 } 744 return NULL; 745 } 746 747 static inline int 748 fill_ip6_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt, 749 uint8_t next_proto, uint8_t len, uint16_t *ether_type) 750 { 751 struct rte_ipv6_hdr *ip6; 752 753 ip6 = (struct rte_ipv6_hdr *)raw_pkt; 754 755 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6); 756 ip6->vtc_flow = rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW | 757 (fdir_input->flow.ipv6_flow.tc << I40E_FDIR_IPv6_TC_OFFSET)); 758 ip6->payload_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN); 759 ip6->proto = fdir_input->flow.ipv6_flow.proto ? 760 fdir_input->flow.ipv6_flow.proto : next_proto; 761 ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ? 762 fdir_input->flow.ipv6_flow.hop_limits : 763 I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS; 764 /** 765 * The source and destination fields in the transmitted packet 766 * need to be presented in a reversed order with respect 767 * to the expected received packets. 768 */ 769 rte_memcpy(&ip6->src_addr, &fdir_input->flow.ipv6_flow.dst_ip, 770 IPV6_ADDR_LEN); 771 rte_memcpy(&ip6->dst_addr, &fdir_input->flow.ipv6_flow.src_ip, 772 IPV6_ADDR_LEN); 773 len += sizeof(struct rte_ipv6_hdr); 774 775 return len; 776 } 777 778 static inline int 779 fill_ip4_head(const struct i40e_fdir_input *fdir_input, unsigned char *raw_pkt, 780 uint8_t next_proto, uint8_t len, uint16_t *ether_type) 781 { 782 struct rte_ipv4_hdr *ip4; 783 784 ip4 = (struct rte_ipv4_hdr *)raw_pkt; 785 786 *ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 787 ip4->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL; 788 /* set len to by default */ 789 ip4->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN); 790 ip4->time_to_live = fdir_input->flow.ip4_flow.ttl ? 791 fdir_input->flow.ip4_flow.ttl : 792 I40E_FDIR_IP_DEFAULT_TTL; 793 ip4->type_of_service = fdir_input->flow.ip4_flow.tos; 794 ip4->next_proto_id = fdir_input->flow.ip4_flow.proto ? 795 fdir_input->flow.ip4_flow.proto : next_proto; 796 /** 797 * The source and destination fields in the transmitted packet 798 * need to be presented in a reversed order with respect 799 * to the expected received packets. 800 */ 801 ip4->src_addr = fdir_input->flow.ip4_flow.dst_ip; 802 ip4->dst_addr = fdir_input->flow.ip4_flow.src_ip; 803 len += sizeof(struct rte_ipv4_hdr); 804 805 return len; 806 } 807 808 static inline int 809 i40e_flow_fdir_fill_eth_ip_head(struct i40e_pf *pf, 810 const struct i40e_fdir_input *fdir_input, 811 unsigned char *raw_pkt, 812 bool vlan) 813 { 814 struct i40e_customized_pctype *cus_pctype = NULL; 815 static uint8_t vlan_frame[] = {0x81, 0, 0, 0}; 816 uint16_t *ether_type; 817 uint8_t len = 2 * sizeof(struct rte_ether_addr); 818 uint8_t pctype = fdir_input->pctype; 819 bool is_customized_pctype = fdir_input->flow_ext.customized_pctype; 820 static const uint8_t next_proto[] = { 821 [I40E_FILTER_PCTYPE_FRAG_IPV4] = IPPROTO_IP, 822 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = IPPROTO_TCP, 823 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = IPPROTO_UDP, 824 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = IPPROTO_SCTP, 825 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = IPPROTO_IP, 826 [I40E_FILTER_PCTYPE_FRAG_IPV6] = IPPROTO_NONE, 827 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = IPPROTO_TCP, 828 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = IPPROTO_UDP, 829 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = IPPROTO_SCTP, 830 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = IPPROTO_NONE, 831 }; 832 833 rte_memcpy(raw_pkt, &fdir_input->flow.l2_flow.dst, 834 sizeof(struct rte_ether_addr)); 835 rte_memcpy(raw_pkt + sizeof(struct rte_ether_addr), 836 &fdir_input->flow.l2_flow.src, 837 sizeof(struct rte_ether_addr)); 838 raw_pkt += 2 * sizeof(struct rte_ether_addr); 839 840 if (vlan && fdir_input->flow_ext.vlan_tci) { 841 rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame)); 842 rte_memcpy(raw_pkt + sizeof(uint16_t), 843 &fdir_input->flow_ext.vlan_tci, 844 sizeof(uint16_t)); 845 raw_pkt += sizeof(vlan_frame); 846 len += sizeof(vlan_frame); 847 } 848 ether_type = (uint16_t *)raw_pkt; 849 raw_pkt += sizeof(uint16_t); 850 len += sizeof(uint16_t); 851 852 if (is_customized_pctype) { 853 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype); 854 if (!cus_pctype) { 855 PMD_DRV_LOG(ERR, "unknown pctype %u.", 856 fdir_input->pctype); 857 return -1; 858 } 859 } 860 861 if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) 862 *ether_type = fdir_input->flow.l2_flow.ether_type; 863 else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP || 864 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP || 865 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP || 866 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || 867 pctype == I40E_FILTER_PCTYPE_FRAG_IPV4 || 868 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP || 869 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP || 870 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP || 871 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || 872 pctype == I40E_FILTER_PCTYPE_FRAG_IPV6 || 873 is_customized_pctype) { 874 if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP || 875 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP || 876 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP || 877 pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || 878 pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) { 879 len = fill_ip4_head(fdir_input, raw_pkt, 880 next_proto[pctype], len, ether_type); 881 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP || 882 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP || 883 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP || 884 pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || 885 pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) { 886 len = fill_ip6_head(fdir_input, raw_pkt, 887 next_proto[pctype], len, 888 ether_type); 889 } else if (cus_pctype->index == I40E_CUSTOMIZED_GTPC || 890 cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 || 891 cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 || 892 cus_pctype->index == I40E_CUSTOMIZED_GTPU) { 893 len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP, 894 len, ether_type); 895 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3) { 896 len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_L2TP, 897 len, ether_type); 898 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) { 899 len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_ESP, 900 len, ether_type); 901 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) { 902 len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP, 903 len, ether_type); 904 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP) { 905 len = fill_ip4_head(fdir_input, raw_pkt, IPPROTO_UDP, 906 len, ether_type); 907 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6) 908 len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_ESP, 909 len, ether_type); 910 else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP) 911 len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_UDP, 912 len, ether_type); 913 else if (cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3) 914 len = fill_ip6_head(fdir_input, raw_pkt, IPPROTO_L2TP, 915 len, ether_type); 916 } else { 917 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype); 918 return -1; 919 } 920 921 return len; 922 } 923 924 /** 925 * i40e_flow_fdir_construct_pkt - construct packet based on fields in input 926 * @pf: board private structure 927 * @fdir_input: input set of the flow director entry 928 * @raw_pkt: a packet to be constructed 929 */ 930 static int 931 i40e_flow_fdir_construct_pkt(struct i40e_pf *pf, 932 const struct i40e_fdir_input *fdir_input, 933 unsigned char *raw_pkt) 934 { 935 unsigned char *payload = NULL; 936 unsigned char *ptr; 937 struct rte_udp_hdr *udp; 938 struct rte_tcp_hdr *tcp; 939 struct rte_sctp_hdr *sctp; 940 struct rte_flow_item_gtp *gtp; 941 struct rte_ipv4_hdr *gtp_ipv4; 942 struct rte_ipv6_hdr *gtp_ipv6; 943 struct rte_flow_item_l2tpv3oip *l2tpv3oip; 944 struct rte_flow_item_esp *esp; 945 struct rte_ipv4_hdr *esp_ipv4; 946 struct rte_ipv6_hdr *esp_ipv6; 947 948 uint8_t size, dst = 0; 949 uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/ 950 int len; 951 uint8_t pctype = fdir_input->pctype; 952 struct i40e_customized_pctype *cus_pctype; 953 954 /* raw pcket template - just copy contents of the raw packet */ 955 if (fdir_input->flow_ext.pkt_template) { 956 memcpy(raw_pkt, fdir_input->flow.raw_flow.packet, 957 fdir_input->flow.raw_flow.length); 958 return 0; 959 } 960 961 /* fill the ethernet and IP head */ 962 len = i40e_flow_fdir_fill_eth_ip_head(pf, fdir_input, raw_pkt, 963 !!fdir_input->flow_ext.vlan_tci); 964 if (len < 0) 965 return -EINVAL; 966 967 /* fill the L4 head */ 968 if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) { 969 udp = (struct rte_udp_hdr *)(raw_pkt + len); 970 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr); 971 /** 972 * The source and destination fields in the transmitted packet 973 * need to be presented in a reversed order with respect 974 * to the expected received packets. 975 */ 976 udp->src_port = fdir_input->flow.udp4_flow.dst_port; 977 udp->dst_port = fdir_input->flow.udp4_flow.src_port; 978 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN); 979 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) { 980 tcp = (struct rte_tcp_hdr *)(raw_pkt + len); 981 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr); 982 /** 983 * The source and destination fields in the transmitted packet 984 * need to be presented in a reversed order with respect 985 * to the expected received packets. 986 */ 987 tcp->src_port = fdir_input->flow.tcp4_flow.dst_port; 988 tcp->dst_port = fdir_input->flow.tcp4_flow.src_port; 989 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF; 990 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) { 991 sctp = (struct rte_sctp_hdr *)(raw_pkt + len); 992 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr); 993 /** 994 * The source and destination fields in the transmitted packet 995 * need to be presented in a reversed order with respect 996 * to the expected received packets. 997 */ 998 sctp->src_port = fdir_input->flow.sctp4_flow.dst_port; 999 sctp->dst_port = fdir_input->flow.sctp4_flow.src_port; 1000 sctp->tag = fdir_input->flow.sctp4_flow.verify_tag; 1001 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || 1002 pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) { 1003 payload = raw_pkt + len; 1004 set_idx = I40E_FLXPLD_L3_IDX; 1005 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) { 1006 udp = (struct rte_udp_hdr *)(raw_pkt + len); 1007 payload = (unsigned char *)udp + sizeof(struct rte_udp_hdr); 1008 /** 1009 * The source and destination fields in the transmitted packet 1010 * need to be presented in a reversed order with respect 1011 * to the expected received packets. 1012 */ 1013 udp->src_port = fdir_input->flow.udp6_flow.dst_port; 1014 udp->dst_port = fdir_input->flow.udp6_flow.src_port; 1015 udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN); 1016 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) { 1017 tcp = (struct rte_tcp_hdr *)(raw_pkt + len); 1018 payload = (unsigned char *)tcp + sizeof(struct rte_tcp_hdr); 1019 /** 1020 * The source and destination fields in the transmitted packet 1021 * need to be presented in a reversed order with respect 1022 * to the expected received packets. 1023 */ 1024 tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF; 1025 tcp->src_port = fdir_input->flow.udp6_flow.dst_port; 1026 tcp->dst_port = fdir_input->flow.udp6_flow.src_port; 1027 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) { 1028 sctp = (struct rte_sctp_hdr *)(raw_pkt + len); 1029 payload = (unsigned char *)sctp + sizeof(struct rte_sctp_hdr); 1030 /** 1031 * The source and destination fields in the transmitted packet 1032 * need to be presented in a reversed order with respect 1033 * to the expected received packets. 1034 */ 1035 sctp->src_port = fdir_input->flow.sctp6_flow.dst_port; 1036 sctp->dst_port = fdir_input->flow.sctp6_flow.src_port; 1037 sctp->tag = fdir_input->flow.sctp6_flow.verify_tag; 1038 } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || 1039 pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) { 1040 payload = raw_pkt + len; 1041 set_idx = I40E_FLXPLD_L3_IDX; 1042 } else if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) { 1043 payload = raw_pkt + len; 1044 /** 1045 * ARP packet is a special case on which the payload 1046 * starts after the whole ARP header 1047 */ 1048 if (fdir_input->flow.l2_flow.ether_type == 1049 rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 1050 payload += sizeof(struct rte_arp_hdr); 1051 set_idx = I40E_FLXPLD_L2_IDX; 1052 } else if (fdir_input->flow_ext.customized_pctype) { 1053 /* If customized pctype is used */ 1054 cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype); 1055 if (cus_pctype->index == I40E_CUSTOMIZED_GTPC || 1056 cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 || 1057 cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 || 1058 cus_pctype->index == I40E_CUSTOMIZED_GTPU) { 1059 udp = (struct rte_udp_hdr *)(raw_pkt + len); 1060 udp->dgram_len = 1061 rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN); 1062 1063 gtp = (struct rte_flow_item_gtp *) 1064 ((unsigned char *)udp + 1065 sizeof(struct rte_udp_hdr)); 1066 gtp->msg_len = 1067 rte_cpu_to_be_16(I40E_FDIR_GTP_DEFAULT_LEN); 1068 gtp->teid = fdir_input->flow.gtp_flow.teid; 1069 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0X01; 1070 1071 /* GTP-C message type is not supported. */ 1072 if (cus_pctype->index == I40E_CUSTOMIZED_GTPC) { 1073 udp->dst_port = 1074 rte_cpu_to_be_16(I40E_FDIR_GTPC_DST_PORT); 1075 gtp->v_pt_rsv_flags = 1076 I40E_FDIR_GTP_VER_FLAG_0X32; 1077 } else { 1078 udp->dst_port = 1079 rte_cpu_to_be_16(I40E_FDIR_GTPU_DST_PORT); 1080 gtp->v_pt_rsv_flags = 1081 I40E_FDIR_GTP_VER_FLAG_0X30; 1082 } 1083 1084 if (cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4) { 1085 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF; 1086 gtp_ipv4 = (struct rte_ipv4_hdr *) 1087 ((unsigned char *)gtp + 1088 sizeof(struct rte_flow_item_gtp)); 1089 gtp_ipv4->version_ihl = 1090 I40E_FDIR_IP_DEFAULT_VERSION_IHL; 1091 gtp_ipv4->next_proto_id = IPPROTO_IP; 1092 gtp_ipv4->total_length = 1093 rte_cpu_to_be_16( 1094 I40E_FDIR_INNER_IP_DEFAULT_LEN); 1095 payload = (unsigned char *)gtp_ipv4 + 1096 sizeof(struct rte_ipv4_hdr); 1097 } else if (cus_pctype->index == 1098 I40E_CUSTOMIZED_GTPU_IPV6) { 1099 gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF; 1100 gtp_ipv6 = (struct rte_ipv6_hdr *) 1101 ((unsigned char *)gtp + 1102 sizeof(struct rte_flow_item_gtp)); 1103 gtp_ipv6->vtc_flow = 1104 rte_cpu_to_be_32( 1105 I40E_FDIR_IPv6_DEFAULT_VTC_FLOW | 1106 (0 << I40E_FDIR_IPv6_TC_OFFSET)); 1107 gtp_ipv6->proto = IPPROTO_NONE; 1108 gtp_ipv6->payload_len = 1109 rte_cpu_to_be_16( 1110 I40E_FDIR_INNER_IPV6_DEFAULT_LEN); 1111 gtp_ipv6->hop_limits = 1112 I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS; 1113 payload = (unsigned char *)gtp_ipv6 + 1114 sizeof(struct rte_ipv6_hdr); 1115 } else 1116 payload = (unsigned char *)gtp + 1117 sizeof(struct rte_flow_item_gtp); 1118 } else if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3 || 1119 cus_pctype->index == I40E_CUSTOMIZED_IPV6_L2TPV3) { 1120 l2tpv3oip = (struct rte_flow_item_l2tpv3oip *)(raw_pkt 1121 + len); 1122 1123 if (cus_pctype->index == I40E_CUSTOMIZED_IPV4_L2TPV3) 1124 l2tpv3oip->session_id = 1125 fdir_input->flow.ip4_l2tpv3oip_flow.session_id; 1126 else 1127 l2tpv3oip->session_id = 1128 fdir_input->flow.ip6_l2tpv3oip_flow.session_id; 1129 payload = (unsigned char *)l2tpv3oip + 1130 sizeof(struct rte_flow_item_l2tpv3oip); 1131 } else if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4 || 1132 cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6 || 1133 cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4_UDP || 1134 cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV6_UDP) { 1135 if (cus_pctype->index == I40E_CUSTOMIZED_ESP_IPV4) { 1136 esp_ipv4 = (struct rte_ipv4_hdr *) 1137 (raw_pkt + len); 1138 esp = (struct rte_flow_item_esp *)esp_ipv4; 1139 esp->hdr.spi = 1140 fdir_input->flow.esp_ipv4_flow.spi; 1141 payload = (unsigned char *)esp + 1142 sizeof(struct rte_esp_hdr); 1143 len += sizeof(struct rte_esp_hdr); 1144 } else if (cus_pctype->index == 1145 I40E_CUSTOMIZED_ESP_IPV4_UDP) { 1146 esp_ipv4 = (struct rte_ipv4_hdr *) 1147 (raw_pkt + len); 1148 udp = (struct rte_udp_hdr *)esp_ipv4; 1149 udp->dst_port = rte_cpu_to_be_16 1150 (I40E_FDIR_ESP_DST_PORT); 1151 1152 udp->dgram_len = rte_cpu_to_be_16 1153 (I40E_FDIR_UDP_DEFAULT_LEN); 1154 esp = (struct rte_flow_item_esp *) 1155 ((unsigned char *)esp_ipv4 + 1156 sizeof(struct rte_udp_hdr)); 1157 esp->hdr.spi = 1158 fdir_input->flow.esp_ipv4_udp_flow.spi; 1159 payload = (unsigned char *)esp + 1160 sizeof(struct rte_esp_hdr); 1161 len += sizeof(struct rte_udp_hdr) + 1162 sizeof(struct rte_esp_hdr); 1163 } else if (cus_pctype->index == 1164 I40E_CUSTOMIZED_ESP_IPV6) { 1165 esp_ipv6 = (struct rte_ipv6_hdr *) 1166 (raw_pkt + len); 1167 esp = (struct rte_flow_item_esp *)esp_ipv6; 1168 esp->hdr.spi = 1169 fdir_input->flow.esp_ipv6_flow.spi; 1170 payload = (unsigned char *)esp + 1171 sizeof(struct rte_esp_hdr); 1172 len += sizeof(struct rte_esp_hdr); 1173 } else if (cus_pctype->index == 1174 I40E_CUSTOMIZED_ESP_IPV6_UDP) { 1175 esp_ipv6 = (struct rte_ipv6_hdr *) 1176 (raw_pkt + len); 1177 udp = (struct rte_udp_hdr *)esp_ipv6; 1178 udp->dst_port = rte_cpu_to_be_16 1179 (I40E_FDIR_ESP_DST_PORT); 1180 1181 udp->dgram_len = rte_cpu_to_be_16 1182 (I40E_FDIR_UDP_DEFAULT_LEN); 1183 esp = (struct rte_flow_item_esp *) 1184 ((unsigned char *)esp_ipv6 + 1185 sizeof(struct rte_udp_hdr)); 1186 esp->hdr.spi = 1187 fdir_input->flow.esp_ipv6_udp_flow.spi; 1188 payload = (unsigned char *)esp + 1189 sizeof(struct rte_esp_hdr); 1190 len += sizeof(struct rte_udp_hdr) + 1191 sizeof(struct rte_esp_hdr); 1192 } 1193 } 1194 } else { 1195 PMD_DRV_LOG(ERR, "unknown pctype %u.", fdir_input->pctype); 1196 return -1; 1197 } 1198 1199 /* fill the flexbytes to payload */ 1200 for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) { 1201 pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i; 1202 size = pf->fdir.flex_set[pit_idx].size; 1203 if (size == 0) 1204 continue; 1205 dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t); 1206 ptr = payload + 1207 pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t); 1208 (void)rte_memcpy(ptr, 1209 &fdir_input->flow_ext.flexbytes[dst], 1210 size * sizeof(uint16_t)); 1211 } 1212 1213 return 0; 1214 } 1215 1216 /* Construct the tx flags */ 1217 static inline uint64_t 1218 i40e_build_ctob(uint32_t td_cmd, 1219 uint32_t td_offset, 1220 unsigned int size, 1221 uint32_t td_tag) 1222 { 1223 return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA | 1224 ((uint64_t)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | 1225 ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | 1226 ((uint64_t)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 1227 ((uint64_t)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); 1228 } 1229 1230 /* 1231 * check the programming status descriptor in rx queue. 1232 * done after Programming Flow Director is programmed on 1233 * tx queue 1234 */ 1235 static inline int 1236 i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq) 1237 { 1238 volatile union i40e_rx_desc *rxdp; 1239 uint64_t qword1; 1240 uint32_t rx_status; 1241 uint32_t len, id; 1242 uint32_t error; 1243 int ret = 0; 1244 1245 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1246 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 1247 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) 1248 >> I40E_RXD_QW1_STATUS_SHIFT; 1249 1250 if (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { 1251 len = qword1 >> I40E_RX_PROG_STATUS_DESC_LENGTH_SHIFT; 1252 id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 1253 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 1254 1255 if (len == I40E_RX_PROG_STATUS_DESC_LENGTH && 1256 id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) { 1257 error = (qword1 & 1258 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 1259 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 1260 if (error == (0x1 << 1261 I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { 1262 PMD_DRV_LOG(ERR, "Failed to add FDIR filter" 1263 " (FD_ID %u): programming status" 1264 " reported.", 1265 rxdp->wb.qword0.hi_dword.fd_id); 1266 ret = -1; 1267 } else if (error == (0x1 << 1268 I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { 1269 PMD_DRV_LOG(ERR, "Failed to delete FDIR filter" 1270 " (FD_ID %u): programming status" 1271 " reported.", 1272 rxdp->wb.qword0.hi_dword.fd_id); 1273 ret = -1; 1274 } else 1275 PMD_DRV_LOG(ERR, "invalid programming status" 1276 " reported, error = %u.", error); 1277 } else 1278 PMD_DRV_LOG(INFO, "unknown programming status" 1279 " reported, len = %d, id = %u.", len, id); 1280 rxdp->wb.qword1.status_error_len = 0; 1281 rxq->rx_tail++; 1282 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc)) 1283 rxq->rx_tail = 0; 1284 if (rxq->rx_tail == 0) 1285 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 1286 else 1287 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1); 1288 } 1289 1290 return ret; 1291 } 1292 1293 static inline void 1294 i40e_fdir_programming_status_cleanup(struct i40e_rx_queue *rxq) 1295 { 1296 uint16_t retry_count = 0; 1297 1298 /* capture the previous error report(if any) from rx ring */ 1299 while ((i40e_check_fdir_programming_status(rxq) < 0) && 1300 (++retry_count < I40E_FDIR_NUM_RX_DESC)) 1301 PMD_DRV_LOG(INFO, "error report captured."); 1302 } 1303 1304 static int 1305 i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input, 1306 struct i40e_fdir_filter *filter) 1307 { 1308 rte_memcpy(&filter->fdir, input, sizeof(struct i40e_fdir_filter_conf)); 1309 if (input->input.flow_ext.pkt_template) { 1310 filter->fdir.input.flow.raw_flow.packet = NULL; 1311 filter->fdir.input.flow.raw_flow.length = 1312 rte_hash_crc(input->input.flow.raw_flow.packet, 1313 input->input.flow.raw_flow.length, 1314 input->input.flow.raw_flow.pctype); 1315 } 1316 return 0; 1317 } 1318 1319 /* Check if there exists the flow director filter */ 1320 static struct i40e_fdir_filter * 1321 i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info, 1322 const struct i40e_fdir_input *input) 1323 { 1324 int ret; 1325 1326 if (input->flow_ext.pkt_template) 1327 ret = rte_hash_lookup_with_hash(fdir_info->hash_table, 1328 (const void *)input, 1329 input->flow.raw_flow.length); 1330 else 1331 ret = rte_hash_lookup(fdir_info->hash_table, 1332 (const void *)input); 1333 if (ret < 0) 1334 return NULL; 1335 1336 return fdir_info->hash_map[ret]; 1337 } 1338 1339 /* Add a flow director filter into the SW list */ 1340 static int 1341 i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter) 1342 { 1343 struct i40e_fdir_info *fdir_info = &pf->fdir; 1344 struct i40e_fdir_filter *hash_filter; 1345 int ret; 1346 1347 if (filter->fdir.input.flow_ext.pkt_template) 1348 ret = rte_hash_add_key_with_hash(fdir_info->hash_table, 1349 &filter->fdir.input, 1350 filter->fdir.input.flow.raw_flow.length); 1351 else 1352 ret = rte_hash_add_key(fdir_info->hash_table, 1353 &filter->fdir.input); 1354 if (ret < 0) { 1355 PMD_DRV_LOG(ERR, 1356 "Failed to insert fdir filter to hash table %d!", 1357 ret); 1358 return ret; 1359 } 1360 1361 if (fdir_info->hash_map[ret]) 1362 return -1; 1363 1364 hash_filter = &fdir_info->fdir_filter_array[ret]; 1365 rte_memcpy(hash_filter, filter, sizeof(*filter)); 1366 fdir_info->hash_map[ret] = hash_filter; 1367 TAILQ_INSERT_TAIL(&fdir_info->fdir_list, hash_filter, rules); 1368 1369 return 0; 1370 } 1371 1372 /* Delete a flow director filter from the SW list */ 1373 int 1374 i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input) 1375 { 1376 struct i40e_fdir_info *fdir_info = &pf->fdir; 1377 struct i40e_fdir_filter *filter; 1378 int ret; 1379 1380 if (input->flow_ext.pkt_template) 1381 ret = rte_hash_del_key_with_hash(fdir_info->hash_table, 1382 input, 1383 input->flow.raw_flow.length); 1384 else 1385 ret = rte_hash_del_key(fdir_info->hash_table, input); 1386 if (ret < 0) { 1387 PMD_DRV_LOG(ERR, 1388 "Failed to delete fdir filter to hash table %d!", 1389 ret); 1390 return ret; 1391 } 1392 filter = fdir_info->hash_map[ret]; 1393 fdir_info->hash_map[ret] = NULL; 1394 1395 TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules); 1396 1397 return 0; 1398 } 1399 1400 struct rte_flow * 1401 i40e_fdir_entry_pool_get(struct i40e_fdir_info *fdir_info) 1402 { 1403 struct rte_flow *flow = NULL; 1404 uint64_t slab = 0; 1405 uint32_t pos = 0; 1406 uint32_t i = 0; 1407 int ret; 1408 1409 if (fdir_info->fdir_actual_cnt >= 1410 fdir_info->fdir_space_size) { 1411 PMD_DRV_LOG(ERR, "Fdir space full"); 1412 return NULL; 1413 } 1414 1415 ret = rte_bitmap_scan(fdir_info->fdir_flow_pool.bitmap, &pos, 1416 &slab); 1417 1418 /* normally this won't happen as the fdir_actual_cnt should be 1419 * same with the number of the set bits in fdir_flow_pool, 1420 * but anyway handle this error condition here for safe 1421 */ 1422 if (ret == 0) { 1423 PMD_DRV_LOG(ERR, "fdir_actual_cnt out of sync"); 1424 return NULL; 1425 } 1426 1427 i = rte_bsf64(slab); 1428 pos += i; 1429 rte_bitmap_clear(fdir_info->fdir_flow_pool.bitmap, pos); 1430 flow = &fdir_info->fdir_flow_pool.pool[pos].flow; 1431 1432 memset(flow, 0, sizeof(struct rte_flow)); 1433 1434 return flow; 1435 } 1436 1437 void 1438 i40e_fdir_entry_pool_put(struct i40e_fdir_info *fdir_info, 1439 struct rte_flow *flow) 1440 { 1441 struct i40e_fdir_entry *f; 1442 1443 f = FLOW_TO_FLOW_BITMAP(flow); 1444 rte_bitmap_set(fdir_info->fdir_flow_pool.bitmap, f->idx); 1445 } 1446 1447 static int 1448 i40e_flow_store_flex_pit(struct i40e_pf *pf, 1449 struct i40e_fdir_flex_pit *flex_pit, 1450 enum i40e_flxpld_layer_idx layer_idx, 1451 uint8_t raw_id) 1452 { 1453 uint8_t field_idx; 1454 1455 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + raw_id; 1456 /* Check if the configuration is conflicted */ 1457 if (pf->fdir.flex_pit_flag[layer_idx] && 1458 (pf->fdir.flex_set[field_idx].src_offset != flex_pit->src_offset || 1459 pf->fdir.flex_set[field_idx].size != flex_pit->size || 1460 pf->fdir.flex_set[field_idx].dst_offset != flex_pit->dst_offset)) 1461 return -1; 1462 1463 /* Check if the configuration exists. */ 1464 if (pf->fdir.flex_pit_flag[layer_idx] && 1465 (pf->fdir.flex_set[field_idx].src_offset == flex_pit->src_offset && 1466 pf->fdir.flex_set[field_idx].size == flex_pit->size && 1467 pf->fdir.flex_set[field_idx].dst_offset == flex_pit->dst_offset)) 1468 return 1; 1469 1470 pf->fdir.flex_set[field_idx].src_offset = 1471 flex_pit->src_offset; 1472 pf->fdir.flex_set[field_idx].size = 1473 flex_pit->size; 1474 pf->fdir.flex_set[field_idx].dst_offset = 1475 flex_pit->dst_offset; 1476 1477 return 0; 1478 } 1479 1480 static void 1481 i40e_flow_set_fdir_flex_pit(struct i40e_pf *pf, 1482 enum i40e_flxpld_layer_idx layer_idx, 1483 uint8_t raw_id) 1484 { 1485 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 1486 uint32_t flx_pit, flx_ort; 1487 uint16_t min_next_off = 0; 1488 uint8_t field_idx; 1489 uint8_t i; 1490 1491 if (raw_id) { 1492 flx_ort = (1 << I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) | 1493 (raw_id << I40E_GLQF_ORT_FIELD_CNT_SHIFT) | 1494 (layer_idx * I40E_MAX_FLXPLD_FIED); 1495 I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort); 1496 } 1497 1498 /* Set flex pit */ 1499 for (i = 0; i < raw_id; i++) { 1500 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; 1501 flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset, 1502 pf->fdir.flex_set[field_idx].size, 1503 pf->fdir.flex_set[field_idx].dst_offset); 1504 1505 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); 1506 min_next_off = pf->fdir.flex_set[field_idx].src_offset + 1507 pf->fdir.flex_set[field_idx].size; 1508 } 1509 1510 for (; i < I40E_MAX_FLXPLD_FIED; i++) { 1511 /* set the non-used register obeying register's constrain */ 1512 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; 1513 flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE, 1514 NONUSE_FLX_PIT_DEST_OFF); 1515 I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); 1516 min_next_off++; 1517 } 1518 } 1519 1520 static int 1521 i40e_flow_store_flex_mask(struct i40e_pf *pf, 1522 enum i40e_filter_pctype pctype, 1523 uint8_t *mask) 1524 { 1525 struct i40e_fdir_flex_mask flex_mask; 1526 uint8_t nb_bitmask = 0; 1527 uint16_t mask_tmp; 1528 uint8_t i; 1529 1530 memset(&flex_mask, 0, sizeof(struct i40e_fdir_flex_mask)); 1531 for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) { 1532 mask_tmp = I40E_WORD(mask[i], mask[i + 1]); 1533 if (mask_tmp) { 1534 flex_mask.word_mask |= 1535 I40E_FLEX_WORD_MASK(i / sizeof(uint16_t)); 1536 if (mask_tmp != UINT16_MAX) { 1537 flex_mask.bitmask[nb_bitmask].mask = ~mask_tmp; 1538 flex_mask.bitmask[nb_bitmask].offset = 1539 i / sizeof(uint16_t); 1540 nb_bitmask++; 1541 if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) 1542 return -1; 1543 } 1544 } 1545 } 1546 flex_mask.nb_bitmask = nb_bitmask; 1547 1548 if (pf->fdir.flex_mask_flag[pctype] && 1549 (memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], 1550 sizeof(struct i40e_fdir_flex_mask)))) 1551 return -2; 1552 else if (pf->fdir.flex_mask_flag[pctype] && 1553 !(memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], 1554 sizeof(struct i40e_fdir_flex_mask)))) 1555 return 1; 1556 1557 memcpy(&pf->fdir.flex_mask[pctype], &flex_mask, 1558 sizeof(struct i40e_fdir_flex_mask)); 1559 return 0; 1560 } 1561 1562 static void 1563 i40e_flow_set_fdir_flex_msk(struct i40e_pf *pf, 1564 enum i40e_filter_pctype pctype) 1565 { 1566 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 1567 struct i40e_fdir_flex_mask *flex_mask; 1568 uint32_t flxinset, fd_mask; 1569 uint8_t i; 1570 1571 /* Set flex mask */ 1572 flex_mask = &pf->fdir.flex_mask[pctype]; 1573 flxinset = (flex_mask->word_mask << 1574 I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) & 1575 I40E_PRTQF_FD_FLXINSET_INSET_MASK; 1576 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset); 1577 1578 for (i = 0; i < flex_mask->nb_bitmask; i++) { 1579 fd_mask = (flex_mask->bitmask[i].mask << 1580 I40E_PRTQF_FD_MSK_MASK_SHIFT) & 1581 I40E_PRTQF_FD_MSK_MASK_MASK; 1582 fd_mask |= ((flex_mask->bitmask[i].offset + 1583 I40E_FLX_OFFSET_IN_FIELD_VECTOR) << 1584 I40E_PRTQF_FD_MSK_OFFSET_SHIFT) & 1585 I40E_PRTQF_FD_MSK_OFFSET_MASK; 1586 i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask); 1587 } 1588 1589 pf->fdir.flex_mask_flag[pctype] = 1; 1590 } 1591 1592 static int 1593 i40e_flow_set_fdir_inset(struct i40e_pf *pf, 1594 enum i40e_filter_pctype pctype, 1595 uint64_t input_set) 1596 { 1597 uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; 1598 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 1599 uint64_t inset_reg = 0; 1600 int i, num; 1601 1602 /* Check if the input set is valid */ 1603 if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR, 1604 input_set) != 0) { 1605 PMD_DRV_LOG(ERR, "Invalid input set"); 1606 return -EINVAL; 1607 } 1608 1609 /* Check if the configuration is conflicted */ 1610 if (pf->fdir.inset_flag[pctype] && 1611 memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) { 1612 PMD_DRV_LOG(ERR, "Conflict with the first rule's input set."); 1613 return -EINVAL; 1614 } 1615 1616 if (pf->fdir.inset_flag[pctype] && 1617 !memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) 1618 return 0; 1619 1620 num = i40e_generate_inset_mask_reg(hw, input_set, mask_reg, 1621 I40E_INSET_MASK_NUM_REG); 1622 if (num < 0) { 1623 PMD_DRV_LOG(ERR, "Invalid pattern mask."); 1624 return -EINVAL; 1625 } 1626 1627 if (pf->support_multi_driver) { 1628 for (i = 0; i < num; i++) 1629 if (i40e_read_rx_ctl(hw, 1630 I40E_GLQF_FD_MSK(i, pctype)) != 1631 mask_reg[i]) { 1632 PMD_DRV_LOG(ERR, "Input set setting is not" 1633 " supported with" 1634 " `support-multi-driver`" 1635 " enabled!"); 1636 return -EPERM; 1637 } 1638 for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) 1639 if (i40e_read_rx_ctl(hw, 1640 I40E_GLQF_FD_MSK(i, pctype)) != 0) { 1641 PMD_DRV_LOG(ERR, "Input set setting is not" 1642 " supported with" 1643 " `support-multi-driver`" 1644 " enabled!"); 1645 return -EPERM; 1646 } 1647 1648 } else { 1649 for (i = 0; i < num; i++) 1650 i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), 1651 mask_reg[i]); 1652 /*clear unused mask registers of the pctype */ 1653 for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) 1654 i40e_check_write_reg(hw, 1655 I40E_GLQF_FD_MSK(i, pctype), 0); 1656 } 1657 1658 inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); 1659 1660 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), 1661 (uint32_t)(inset_reg & UINT32_MAX)); 1662 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), 1663 (uint32_t)((inset_reg >> 1664 I40E_32_BIT_WIDTH) & UINT32_MAX)); 1665 1666 I40E_WRITE_FLUSH(hw); 1667 1668 pf->fdir.input_set[pctype] = input_set; 1669 pf->fdir.inset_flag[pctype] = 1; 1670 return 0; 1671 } 1672 1673 static inline unsigned char * 1674 i40e_find_available_buffer(struct rte_eth_dev *dev) 1675 { 1676 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1677 struct i40e_fdir_info *fdir_info = &pf->fdir; 1678 struct i40e_tx_queue *txq = pf->fdir.txq; 1679 1680 /* no available buffer 1681 * search for more available buffers from the current 1682 * descriptor, until an unavailable one 1683 */ 1684 if (fdir_info->txq_available_buf_count <= 0) { 1685 uint16_t tmp_tail; 1686 volatile struct i40e_tx_desc *tmp_txdp; 1687 1688 tmp_tail = txq->tx_tail; 1689 tmp_txdp = &txq->tx_ring[tmp_tail + 1]; 1690 1691 do { 1692 if ((tmp_txdp->cmd_type_offset_bsz & 1693 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) == 1694 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) 1695 fdir_info->txq_available_buf_count++; 1696 else 1697 break; 1698 1699 tmp_tail += 2; 1700 if (tmp_tail >= txq->nb_tx_desc) 1701 tmp_tail = 0; 1702 } while (tmp_tail != txq->tx_tail); 1703 } 1704 1705 if (fdir_info->txq_available_buf_count > 0) 1706 fdir_info->txq_available_buf_count--; 1707 else 1708 return NULL; 1709 return (unsigned char *)fdir_info->prg_pkt[txq->tx_tail >> 1]; 1710 } 1711 1712 /** 1713 * i40e_flow_add_del_fdir_filter - add or remove a flow director filter. 1714 * @pf: board private structure 1715 * @filter: fdir filter entry 1716 * @add: 0 - delete, 1 - add 1717 */ 1718 int 1719 i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev, 1720 const struct i40e_fdir_filter_conf *filter, 1721 bool add) 1722 { 1723 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1724 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1725 enum i40e_flxpld_layer_idx layer_idx = I40E_FLXPLD_L2_IDX; 1726 struct i40e_fdir_info *fdir_info = &pf->fdir; 1727 uint8_t flex_mask[I40E_FDIR_MAX_FLEX_LEN]; 1728 struct i40e_fdir_filter check_filter; /* Check if the filter exists */ 1729 struct i40e_fdir_flex_pit flex_pit; 1730 enum i40e_filter_pctype pctype; 1731 struct i40e_fdir_filter *node; 1732 unsigned char *pkt = NULL; 1733 bool cfg_flex_pit = true; 1734 bool wait_status = true; 1735 uint8_t field_idx; 1736 int ret = 0; 1737 int i; 1738 1739 if (pf->fdir.fdir_vsi == NULL) { 1740 PMD_DRV_LOG(ERR, "FDIR is not enabled"); 1741 return -ENOTSUP; 1742 } 1743 1744 if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) { 1745 PMD_DRV_LOG(ERR, "Invalid queue ID"); 1746 return -EINVAL; 1747 } 1748 if (filter->input.flow_ext.is_vf && 1749 filter->input.flow_ext.dst_id >= pf->vf_num) { 1750 PMD_DRV_LOG(ERR, "Invalid VF ID"); 1751 return -EINVAL; 1752 } 1753 if (filter->input.flow_ext.pkt_template) { 1754 if (filter->input.flow.raw_flow.length > I40E_FDIR_PKT_LEN || 1755 !filter->input.flow.raw_flow.packet) { 1756 PMD_DRV_LOG(ERR, "Invalid raw packet template" 1757 " flow filter parameters!"); 1758 return -EINVAL; 1759 } 1760 pctype = filter->input.flow.raw_flow.pctype; 1761 } else { 1762 pctype = filter->input.pctype; 1763 } 1764 1765 /* Check if there is the filter in SW list */ 1766 memset(&check_filter, 0, sizeof(check_filter)); 1767 i40e_fdir_filter_convert(filter, &check_filter); 1768 1769 if (add) { 1770 /* configure the input set for common PCTYPEs*/ 1771 if (!filter->input.flow_ext.customized_pctype) { 1772 ret = i40e_flow_set_fdir_inset(pf, pctype, 1773 filter->input.flow_ext.input_set); 1774 if (ret < 0) 1775 return ret; 1776 } 1777 1778 if (filter->input.flow_ext.is_flex_flow) { 1779 for (i = 0; i < filter->input.flow_ext.raw_id; i++) { 1780 layer_idx = filter->input.flow_ext.layer_idx; 1781 field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; 1782 flex_pit = filter->input.flow_ext.flex_pit[field_idx]; 1783 1784 /* Store flex pit to SW */ 1785 ret = i40e_flow_store_flex_pit(pf, &flex_pit, 1786 layer_idx, i); 1787 if (ret < 0) { 1788 PMD_DRV_LOG(ERR, "Conflict with the" 1789 " first flexible rule."); 1790 return -EINVAL; 1791 } else if (ret > 0) { 1792 cfg_flex_pit = false; 1793 } 1794 } 1795 1796 if (cfg_flex_pit) 1797 i40e_flow_set_fdir_flex_pit(pf, layer_idx, 1798 filter->input.flow_ext.raw_id); 1799 1800 /* Store flex mask to SW */ 1801 for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i++) 1802 flex_mask[i] = 1803 filter->input.flow_ext.flex_mask[i]; 1804 1805 ret = i40e_flow_store_flex_mask(pf, pctype, flex_mask); 1806 if (ret == -1) { 1807 PMD_DRV_LOG(ERR, "Exceed maximal" 1808 " number of bitmasks"); 1809 return -EINVAL; 1810 } else if (ret == -2) { 1811 PMD_DRV_LOG(ERR, "Conflict with the" 1812 " first flexible rule"); 1813 return -EINVAL; 1814 } else if (ret == 0) { 1815 i40e_flow_set_fdir_flex_msk(pf, pctype); 1816 } 1817 } 1818 1819 ret = i40e_sw_fdir_filter_insert(pf, &check_filter); 1820 if (ret < 0) { 1821 PMD_DRV_LOG(ERR, 1822 "Conflict with existing flow director rules!"); 1823 return -EINVAL; 1824 } 1825 1826 if (fdir_info->fdir_invalprio == 1 && 1827 fdir_info->fdir_guarantee_free_space > 0) 1828 wait_status = false; 1829 } else { 1830 if (filter->input.flow_ext.is_flex_flow) 1831 layer_idx = filter->input.flow_ext.layer_idx; 1832 1833 node = i40e_sw_fdir_filter_lookup(fdir_info, 1834 &check_filter.fdir.input); 1835 if (!node) { 1836 PMD_DRV_LOG(ERR, 1837 "There's no corresponding flow firector filter!"); 1838 return -EINVAL; 1839 } 1840 1841 ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input); 1842 if (ret < 0) { 1843 PMD_DRV_LOG(ERR, 1844 "Error deleting fdir rule from hash table!"); 1845 return -EINVAL; 1846 } 1847 1848 pf->fdir.flex_mask_flag[pctype] = 0; 1849 1850 if (fdir_info->fdir_invalprio == 1) 1851 wait_status = false; 1852 } 1853 1854 /* find a buffer to store the pkt */ 1855 pkt = i40e_find_available_buffer(dev); 1856 if (pkt == NULL) 1857 goto error_op; 1858 1859 memset(pkt, 0, I40E_FDIR_PKT_LEN); 1860 ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt); 1861 if (ret < 0) { 1862 PMD_DRV_LOG(ERR, "construct packet for fdir fails."); 1863 goto error_op; 1864 } 1865 1866 if (hw->mac.type == I40E_MAC_X722) { 1867 /* get translated pctype value in fd pctype register */ 1868 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl( 1869 hw, I40E_GLQF_FD_PCTYPES((int)pctype)); 1870 } 1871 1872 ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add, 1873 wait_status); 1874 if (ret < 0) { 1875 PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).", 1876 pctype); 1877 goto error_op; 1878 } 1879 1880 if (filter->input.flow_ext.is_flex_flow) { 1881 if (add) { 1882 fdir_info->flex_flow_count[layer_idx]++; 1883 pf->fdir.flex_pit_flag[layer_idx] = 1; 1884 } else { 1885 fdir_info->flex_flow_count[layer_idx]--; 1886 if (!fdir_info->flex_flow_count[layer_idx]) 1887 pf->fdir.flex_pit_flag[layer_idx] = 0; 1888 } 1889 } 1890 1891 if (add) { 1892 fdir_info->fdir_actual_cnt++; 1893 if (fdir_info->fdir_invalprio == 1 && 1894 fdir_info->fdir_guarantee_free_space > 0) 1895 fdir_info->fdir_guarantee_free_space--; 1896 } else { 1897 fdir_info->fdir_actual_cnt--; 1898 if (fdir_info->fdir_invalprio == 1 && 1899 fdir_info->fdir_guarantee_free_space < 1900 fdir_info->fdir_guarantee_total_space) 1901 fdir_info->fdir_guarantee_free_space++; 1902 } 1903 1904 return ret; 1905 1906 error_op: 1907 /* roll back */ 1908 if (add) 1909 i40e_sw_fdir_filter_del(pf, &check_filter.fdir.input); 1910 else 1911 i40e_sw_fdir_filter_insert(pf, &check_filter); 1912 1913 return ret; 1914 } 1915 1916 /* 1917 * i40e_flow_fdir_filter_programming - Program a flow director filter rule. 1918 * Is done by Flow Director Programming Descriptor followed by packet 1919 * structure that contains the filter fields need to match. 1920 * @pf: board private structure 1921 * @pctype: pctype 1922 * @filter: fdir filter entry 1923 * @add: 0 - delete, 1 - add 1924 */ 1925 static int 1926 i40e_flow_fdir_filter_programming(struct i40e_pf *pf, 1927 enum i40e_filter_pctype pctype, 1928 const struct i40e_fdir_filter_conf *filter, 1929 bool add, bool wait_status) 1930 { 1931 struct i40e_tx_queue *txq = pf->fdir.txq; 1932 struct i40e_rx_queue *rxq = pf->fdir.rxq; 1933 const struct i40e_fdir_action *fdir_action = &filter->action; 1934 volatile struct i40e_tx_desc *txdp; 1935 volatile struct i40e_filter_program_desc *fdirdp; 1936 uint32_t td_cmd; 1937 uint16_t vsi_id; 1938 uint8_t dest; 1939 uint32_t i; 1940 1941 PMD_DRV_LOG(INFO, "filling filter programming descriptor."); 1942 fdirdp = (volatile struct i40e_filter_program_desc *) 1943 (&txq->tx_ring[txq->tx_tail]); 1944 1945 fdirdp->qindex_flex_ptype_vsi = 1946 rte_cpu_to_le_32((fdir_action->rx_queue << 1947 I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 1948 I40E_TXD_FLTR_QW0_QINDEX_MASK); 1949 1950 fdirdp->qindex_flex_ptype_vsi |= 1951 rte_cpu_to_le_32((fdir_action->flex_off << 1952 I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) & 1953 I40E_TXD_FLTR_QW0_FLEXOFF_MASK); 1954 1955 fdirdp->qindex_flex_ptype_vsi |= 1956 rte_cpu_to_le_32((pctype << 1957 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) & 1958 I40E_TXD_FLTR_QW0_PCTYPE_MASK); 1959 1960 if (filter->input.flow_ext.is_vf) 1961 vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id; 1962 else 1963 /* Use LAN VSI Id by default */ 1964 vsi_id = pf->main_vsi->vsi_id; 1965 fdirdp->qindex_flex_ptype_vsi |= 1966 rte_cpu_to_le_32(((uint32_t)vsi_id << 1967 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) & 1968 I40E_TXD_FLTR_QW0_DEST_VSI_MASK); 1969 1970 fdirdp->dtype_cmd_cntindex = 1971 rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG); 1972 1973 if (add) 1974 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32( 1975 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 1976 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1977 else 1978 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32( 1979 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 1980 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1981 1982 if (fdir_action->behavior == I40E_FDIR_REJECT) 1983 dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET; 1984 else if (fdir_action->behavior == I40E_FDIR_ACCEPT) 1985 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX; 1986 else if (fdir_action->behavior == I40E_FDIR_PASSTHRU) 1987 dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER; 1988 else { 1989 PMD_DRV_LOG(ERR, "Failed to program FDIR filter: unsupported fdir behavior."); 1990 return -EINVAL; 1991 } 1992 1993 fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest << 1994 I40E_TXD_FLTR_QW1_DEST_SHIFT) & 1995 I40E_TXD_FLTR_QW1_DEST_MASK); 1996 1997 fdirdp->dtype_cmd_cntindex |= 1998 rte_cpu_to_le_32((fdir_action->report_status << 1999 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) & 2000 I40E_TXD_FLTR_QW1_FD_STATUS_MASK); 2001 2002 fdirdp->dtype_cmd_cntindex |= 2003 rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK); 2004 fdirdp->dtype_cmd_cntindex |= 2005 rte_cpu_to_le_32( 2006 ((uint32_t)pf->fdir.match_counter_index << 2007 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2008 I40E_TXD_FLTR_QW1_CNTINDEX_MASK); 2009 2010 fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id); 2011 2012 PMD_DRV_LOG(INFO, "filling transmit descriptor."); 2013 txdp = &txq->tx_ring[txq->tx_tail + 1]; 2014 txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr[txq->tx_tail >> 1]); 2015 2016 td_cmd = I40E_TX_DESC_CMD_EOP | 2017 I40E_TX_DESC_CMD_RS | 2018 I40E_TX_DESC_CMD_DUMMY; 2019 2020 txdp->cmd_type_offset_bsz = 2021 i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0); 2022 2023 txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */ 2024 if (txq->tx_tail >= txq->nb_tx_desc) 2025 txq->tx_tail = 0; 2026 /* Update the tx tail register */ 2027 rte_wmb(); 2028 2029 /* fdir program rx queue cleanup */ 2030 i40e_fdir_programming_status_cleanup(rxq); 2031 2032 I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); 2033 2034 if (wait_status) { 2035 for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) { 2036 if ((txdp->cmd_type_offset_bsz & 2037 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) == 2038 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) 2039 break; 2040 rte_delay_us(1); 2041 } 2042 if (i >= I40E_FDIR_MAX_WAIT_US) { 2043 PMD_DRV_LOG(ERR, 2044 "Failed to program FDIR filter: time out to get DD on tx queue."); 2045 return -ETIMEDOUT; 2046 } 2047 /* totally delay 10 ms to check programming status*/ 2048 rte_delay_us(I40E_FDIR_MAX_WAIT_US); 2049 if (i40e_check_fdir_programming_status(rxq) < 0) { 2050 PMD_DRV_LOG(ERR, 2051 "Failed to program FDIR filter: programming status reported."); 2052 return -ETIMEDOUT; 2053 } 2054 } 2055 2056 return 0; 2057 } 2058 2059 /* 2060 * i40e_fdir_flush - clear all filters of Flow Director table 2061 * @pf: board private structure 2062 */ 2063 int 2064 i40e_fdir_flush(struct rte_eth_dev *dev) 2065 { 2066 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2067 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 2068 uint32_t reg; 2069 uint16_t guarant_cnt, best_cnt; 2070 uint16_t i; 2071 2072 I40E_WRITE_REG(hw, I40E_PFQF_CTL_1, I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 2073 I40E_WRITE_FLUSH(hw); 2074 2075 for (i = 0; i < I40E_FDIR_FLUSH_RETRY; i++) { 2076 rte_delay_ms(I40E_FDIR_FLUSH_INTERVAL_MS); 2077 reg = I40E_READ_REG(hw, I40E_PFQF_CTL_1); 2078 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 2079 break; 2080 } 2081 if (i >= I40E_FDIR_FLUSH_RETRY) { 2082 PMD_DRV_LOG(ERR, "FD table did not flush, may need more time."); 2083 return -ETIMEDOUT; 2084 } 2085 guarant_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) & 2086 I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >> 2087 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT); 2088 best_cnt = (uint16_t)((I40E_READ_REG(hw, I40E_PFQF_FDSTAT) & 2089 I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 2090 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 2091 if (guarant_cnt != 0 || best_cnt != 0) { 2092 PMD_DRV_LOG(ERR, "Failed to flush FD table."); 2093 return -ENOSYS; 2094 } else 2095 PMD_DRV_LOG(INFO, "FD table Flush success."); 2096 return 0; 2097 } 2098 2099 static inline void 2100 i40e_fdir_info_get_flex_set(struct i40e_pf *pf, 2101 struct rte_eth_flex_payload_cfg *flex_set, 2102 uint16_t *num) 2103 { 2104 struct i40e_fdir_flex_pit *flex_pit; 2105 struct rte_eth_flex_payload_cfg *ptr = flex_set; 2106 uint16_t src, dst, size, j, k; 2107 uint8_t i, layer_idx; 2108 2109 for (layer_idx = I40E_FLXPLD_L2_IDX; 2110 layer_idx <= I40E_FLXPLD_L4_IDX; 2111 layer_idx++) { 2112 if (layer_idx == I40E_FLXPLD_L2_IDX) 2113 ptr->type = RTE_ETH_L2_PAYLOAD; 2114 else if (layer_idx == I40E_FLXPLD_L3_IDX) 2115 ptr->type = RTE_ETH_L3_PAYLOAD; 2116 else if (layer_idx == I40E_FLXPLD_L4_IDX) 2117 ptr->type = RTE_ETH_L4_PAYLOAD; 2118 2119 for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) { 2120 flex_pit = &pf->fdir.flex_set[layer_idx * 2121 I40E_MAX_FLXPLD_FIED + i]; 2122 if (flex_pit->size == 0) 2123 continue; 2124 src = flex_pit->src_offset * sizeof(uint16_t); 2125 dst = flex_pit->dst_offset * sizeof(uint16_t); 2126 size = flex_pit->size * sizeof(uint16_t); 2127 for (j = src, k = dst; j < src + size; j++, k++) 2128 ptr->src_offset[k] = j; 2129 } 2130 (*num)++; 2131 ptr++; 2132 } 2133 } 2134 2135 static inline void 2136 i40e_fdir_info_get_flex_mask(struct i40e_pf *pf, 2137 struct rte_eth_fdir_flex_mask *flex_mask, 2138 uint16_t *num) 2139 { 2140 struct i40e_fdir_flex_mask *mask; 2141 struct rte_eth_fdir_flex_mask *ptr = flex_mask; 2142 uint16_t flow_type; 2143 uint8_t i, j; 2144 uint16_t off_bytes, mask_tmp; 2145 2146 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 2147 i <= I40E_FILTER_PCTYPE_L2_PAYLOAD; 2148 i++) { 2149 mask = &pf->fdir.flex_mask[i]; 2150 flow_type = i40e_pctype_to_flowtype(pf->adapter, 2151 (enum i40e_filter_pctype)i); 2152 if (flow_type == RTE_ETH_FLOW_UNKNOWN) 2153 continue; 2154 2155 for (j = 0; j < I40E_FDIR_MAX_FLEXWORD_NUM; j++) { 2156 if (mask->word_mask & I40E_FLEX_WORD_MASK(j)) { 2157 ptr->mask[j * sizeof(uint16_t)] = UINT8_MAX; 2158 ptr->mask[j * sizeof(uint16_t) + 1] = UINT8_MAX; 2159 } else { 2160 ptr->mask[j * sizeof(uint16_t)] = 0x0; 2161 ptr->mask[j * sizeof(uint16_t) + 1] = 0x0; 2162 } 2163 } 2164 for (j = 0; j < I40E_FDIR_BITMASK_NUM_WORD; j++) { 2165 off_bytes = mask->bitmask[j].offset * sizeof(uint16_t); 2166 mask_tmp = ~mask->bitmask[j].mask; 2167 ptr->mask[off_bytes] &= I40E_HI_BYTE(mask_tmp); 2168 ptr->mask[off_bytes + 1] &= I40E_LO_BYTE(mask_tmp); 2169 } 2170 ptr->flow_type = flow_type; 2171 ptr++; 2172 (*num)++; 2173 } 2174 } 2175 2176 /* 2177 * i40e_fdir_info_get - get information of Flow Director 2178 * @pf: ethernet device to get info from 2179 * @fdir: a pointer to a structure of type *rte_eth_fdir_info* to be filled with 2180 * the flow director information. 2181 */ 2182 void 2183 i40e_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir) 2184 { 2185 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2186 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 2187 uint16_t num_flex_set = 0; 2188 uint16_t num_flex_mask = 0; 2189 uint16_t i; 2190 2191 if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) 2192 fdir->mode = RTE_FDIR_MODE_PERFECT; 2193 else 2194 fdir->mode = RTE_FDIR_MODE_NONE; 2195 2196 fdir->guarant_spc = 2197 (uint32_t)hw->func_caps.fd_filters_guaranteed; 2198 fdir->best_spc = 2199 (uint32_t)hw->func_caps.fd_filters_best_effort; 2200 fdir->max_flexpayload = I40E_FDIR_MAX_FLEX_LEN; 2201 fdir->flow_types_mask[0] = I40E_FDIR_FLOWS; 2202 for (i = 1; i < RTE_FLOW_MASK_ARRAY_SIZE; i++) 2203 fdir->flow_types_mask[i] = 0ULL; 2204 fdir->flex_payload_unit = sizeof(uint16_t); 2205 fdir->flex_bitmask_unit = sizeof(uint16_t); 2206 fdir->max_flex_payload_segment_num = I40E_MAX_FLXPLD_FIED; 2207 fdir->flex_payload_limit = I40E_MAX_FLX_SOURCE_OFF; 2208 fdir->max_flex_bitmask_num = I40E_FDIR_BITMASK_NUM_WORD; 2209 2210 i40e_fdir_info_get_flex_set(pf, 2211 fdir->flex_conf.flex_set, 2212 &num_flex_set); 2213 i40e_fdir_info_get_flex_mask(pf, 2214 fdir->flex_conf.flex_mask, 2215 &num_flex_mask); 2216 2217 fdir->flex_conf.nb_payloads = num_flex_set; 2218 fdir->flex_conf.nb_flexmasks = num_flex_mask; 2219 } 2220 2221 /* 2222 * i40e_fdir_stat_get - get statistics of Flow Director 2223 * @pf: ethernet device to get info from 2224 * @stat: a pointer to a structure of type *rte_eth_fdir_stats* to be filled with 2225 * the flow director statistics. 2226 */ 2227 void 2228 i40e_fdir_stats_get(struct rte_eth_dev *dev, struct rte_eth_fdir_stats *stat) 2229 { 2230 struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 2231 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 2232 uint32_t fdstat; 2233 2234 fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT); 2235 stat->guarant_cnt = 2236 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >> 2237 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT); 2238 stat->best_cnt = 2239 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 2240 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 2241 } 2242 2243 /* Restore flow director filter */ 2244 void 2245 i40e_fdir_filter_restore(struct i40e_pf *pf) 2246 { 2247 struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(pf->main_vsi); 2248 struct i40e_fdir_filter_list *fdir_list = &pf->fdir.fdir_list; 2249 struct i40e_fdir_filter *f; 2250 struct i40e_hw *hw = I40E_PF_TO_HW(pf); 2251 uint32_t fdstat; 2252 uint32_t guarant_cnt; /**< Number of filters in guaranteed spaces. */ 2253 uint32_t best_cnt; /**< Number of filters in best effort spaces. */ 2254 2255 TAILQ_FOREACH(f, fdir_list, rules) 2256 i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE); 2257 2258 fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT); 2259 guarant_cnt = 2260 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >> 2261 I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT); 2262 best_cnt = 2263 (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 2264 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 2265 2266 PMD_DRV_LOG(INFO, "FDIR: Guarant count: %d, Best count: %d", 2267 guarant_cnt, best_cnt); 2268 } 2269