1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2020 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <stdio.h> 7 #include <stdint.h> 8 #include <stdarg.h> 9 #include <errno.h> 10 #include <sys/queue.h> 11 #include <rte_malloc.h> 12 13 #include "txgbe_logs.h" 14 #include "base/txgbe.h" 15 #include "txgbe_ethdev.h" 16 17 #define TXGBE_DEFAULT_FLEXBYTES_OFFSET 12 /*default flexbytes offset in bytes*/ 18 #define TXGBE_MAX_FLX_SOURCE_OFF 62 19 #define TXGBE_FDIRCMD_CMD_INTERVAL_US 10 20 21 #define IPV6_ADDR_TO_MASK(ipaddr, ipv6m) do { \ 22 uint8_t ipv6_addr[16]; \ 23 uint8_t i; \ 24 rte_memcpy(ipv6_addr, (ipaddr), sizeof(ipv6_addr));\ 25 (ipv6m) = 0; \ 26 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 27 if (ipv6_addr[i] == UINT8_MAX) \ 28 (ipv6m) |= 1 << i; \ 29 else if (ipv6_addr[i] != 0) { \ 30 PMD_DRV_LOG(ERR, " invalid IPv6 address mask."); \ 31 return -EINVAL; \ 32 } \ 33 } \ 34 } while (0) 35 36 #define IPV6_MASK_TO_ADDR(ipv6m, ipaddr) do { \ 37 uint8_t ipv6_addr[16]; \ 38 uint8_t i; \ 39 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 40 if ((ipv6m) & (1 << i)) \ 41 ipv6_addr[i] = UINT8_MAX; \ 42 else \ 43 ipv6_addr[i] = 0; \ 44 } \ 45 rte_memcpy((ipaddr), ipv6_addr, sizeof(ipv6_addr));\ 46 } while (0) 47 48 /** 49 * Initialize Flow Director control registers 50 * @hw: pointer to hardware structure 51 * @fdirctrl: value to write to flow director control register 52 **/ 53 static int 54 txgbe_fdir_enable(struct txgbe_hw *hw, uint32_t fdirctrl) 55 { 56 int i; 57 58 PMD_INIT_FUNC_TRACE(); 59 60 /* Prime the keys for hashing */ 61 wr32(hw, TXGBE_FDIRBKTHKEY, TXGBE_ATR_BUCKET_HASH_KEY); 62 wr32(hw, TXGBE_FDIRSIGHKEY, TXGBE_ATR_SIGNATURE_HASH_KEY); 63 64 /* 65 * Continue setup of fdirctrl register bits: 66 * Set the maximum length per hash bucket to 0xA filters 67 * Send interrupt when 64 filters are left 68 */ 69 fdirctrl |= TXGBE_FDIRCTL_MAXLEN(0xA) | 70 TXGBE_FDIRCTL_FULLTHR(4); 71 72 /* 73 * Poll init-done after we write the register. Estimated times: 74 * 10G: PBALLOC = 11b, timing is 60us 75 * 1G: PBALLOC = 11b, timing is 600us 76 * 100M: PBALLOC = 11b, timing is 6ms 77 * 78 * Multiple these timings by 4 if under full Rx load 79 * 80 * So we'll poll for TXGBE_FDIR_INIT_DONE_POLL times, sleeping for 81 * 1 msec per poll time. If we're at line rate and drop to 100M, then 82 * this might not finish in our poll time, but we can live with that 83 * for now. 84 */ 85 wr32(hw, TXGBE_FDIRCTL, fdirctrl); 86 txgbe_flush(hw); 87 for (i = 0; i < TXGBE_FDIR_INIT_DONE_POLL; i++) { 88 if (rd32(hw, TXGBE_FDIRCTL) & TXGBE_FDIRCTL_INITDONE) 89 break; 90 msec_delay(1); 91 } 92 93 if (i >= TXGBE_FDIR_INIT_DONE_POLL) { 94 PMD_INIT_LOG(ERR, "Flow Director poll time exceeded during enabling!"); 95 return -ETIMEDOUT; 96 } 97 return 0; 98 } 99 100 /* 101 * Set appropriate bits in fdirctrl for: variable reporting levels, moving 102 * flexbytes matching field, and drop queue (only for perfect matching mode). 103 */ 104 static inline int 105 configure_fdir_flags(const struct rte_fdir_conf *conf, 106 uint32_t *fdirctrl, uint32_t *flex) 107 { 108 *fdirctrl = 0; 109 *flex = 0; 110 111 switch (conf->pballoc) { 112 case RTE_FDIR_PBALLOC_64K: 113 /* 8k - 1 signature filters */ 114 *fdirctrl |= TXGBE_FDIRCTL_BUF_64K; 115 break; 116 case RTE_FDIR_PBALLOC_128K: 117 /* 16k - 1 signature filters */ 118 *fdirctrl |= TXGBE_FDIRCTL_BUF_128K; 119 break; 120 case RTE_FDIR_PBALLOC_256K: 121 /* 32k - 1 signature filters */ 122 *fdirctrl |= TXGBE_FDIRCTL_BUF_256K; 123 break; 124 default: 125 /* bad value */ 126 PMD_INIT_LOG(ERR, "Invalid fdir_conf->pballoc value"); 127 return -EINVAL; 128 }; 129 130 /* status flags: write hash & swindex in the rx descriptor */ 131 switch (conf->status) { 132 case RTE_FDIR_NO_REPORT_STATUS: 133 /* do nothing, default mode */ 134 break; 135 case RTE_FDIR_REPORT_STATUS: 136 /* report status when the packet matches a fdir rule */ 137 *fdirctrl |= TXGBE_FDIRCTL_REPORT_MATCH; 138 break; 139 case RTE_FDIR_REPORT_STATUS_ALWAYS: 140 /* always report status */ 141 *fdirctrl |= TXGBE_FDIRCTL_REPORT_ALWAYS; 142 break; 143 default: 144 /* bad value */ 145 PMD_INIT_LOG(ERR, "Invalid fdir_conf->status value"); 146 return -EINVAL; 147 }; 148 149 *flex |= TXGBE_FDIRFLEXCFG_BASE_MAC; 150 *flex |= TXGBE_FDIRFLEXCFG_OFST(TXGBE_DEFAULT_FLEXBYTES_OFFSET / 2); 151 152 switch (conf->mode) { 153 case RTE_FDIR_MODE_SIGNATURE: 154 break; 155 case RTE_FDIR_MODE_PERFECT: 156 *fdirctrl |= TXGBE_FDIRCTL_PERFECT; 157 *fdirctrl |= TXGBE_FDIRCTL_DROPQP(conf->drop_queue); 158 break; 159 default: 160 /* bad value */ 161 PMD_INIT_LOG(ERR, "Invalid fdir_conf->mode value"); 162 return -EINVAL; 163 } 164 165 return 0; 166 } 167 168 static inline uint32_t 169 reverse_fdir_bmks(uint16_t hi_dword, uint16_t lo_dword) 170 { 171 uint32_t mask = hi_dword << 16; 172 173 mask |= lo_dword; 174 mask = ((mask & 0x55555555) << 1) | ((mask & 0xAAAAAAAA) >> 1); 175 mask = ((mask & 0x33333333) << 2) | ((mask & 0xCCCCCCCC) >> 2); 176 mask = ((mask & 0x0F0F0F0F) << 4) | ((mask & 0xF0F0F0F0) >> 4); 177 return ((mask & 0x00FF00FF) << 8) | ((mask & 0xFF00FF00) >> 8); 178 } 179 180 int 181 txgbe_fdir_set_input_mask(struct rte_eth_dev *dev) 182 { 183 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 184 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 185 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 186 /* 187 * mask VM pool and DIPv6 since there are currently not supported 188 * mask FLEX byte, it will be set in flex_conf 189 */ 190 uint32_t fdirm = TXGBE_FDIRMSK_POOL; 191 uint32_t fdirtcpm; /* TCP source and destination port masks. */ 192 uint32_t fdiripv6m; /* IPv6 source and destination masks. */ 193 194 PMD_INIT_FUNC_TRACE(); 195 196 if (mode != RTE_FDIR_MODE_SIGNATURE && 197 mode != RTE_FDIR_MODE_PERFECT) { 198 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 199 return -ENOTSUP; 200 } 201 202 /* 203 * Program the relevant mask registers. If src/dst_port or src/dst_addr 204 * are zero, then assume a full mask for that field. Also assume that 205 * a VLAN of 0 is unspecified, so mask that out as well. L4type 206 * cannot be masked out in this implementation. 207 */ 208 if (info->mask.dst_port_mask == 0 && info->mask.src_port_mask == 0) { 209 /* use the L4 protocol mask for raw IPv4/IPv6 traffic */ 210 fdirm |= TXGBE_FDIRMSK_L4P; 211 } 212 213 /* TBD: don't support encapsulation yet */ 214 wr32(hw, TXGBE_FDIRMSK, fdirm); 215 216 /* store the TCP/UDP port masks, bit reversed from port layout */ 217 fdirtcpm = reverse_fdir_bmks(rte_be_to_cpu_16(info->mask.dst_port_mask), 218 rte_be_to_cpu_16(info->mask.src_port_mask)); 219 220 /* write all the same so that UDP, TCP and SCTP use the same mask 221 * (little-endian) 222 */ 223 wr32(hw, TXGBE_FDIRTCPMSK, ~fdirtcpm); 224 wr32(hw, TXGBE_FDIRUDPMSK, ~fdirtcpm); 225 wr32(hw, TXGBE_FDIRSCTPMSK, ~fdirtcpm); 226 227 /* Store source and destination IPv4 masks (big-endian) */ 228 wr32(hw, TXGBE_FDIRSIP4MSK, ~info->mask.src_ipv4_mask); 229 wr32(hw, TXGBE_FDIRDIP4MSK, ~info->mask.dst_ipv4_mask); 230 231 if (mode == RTE_FDIR_MODE_SIGNATURE) { 232 /* 233 * Store source and destination IPv6 masks (bit reversed) 234 */ 235 fdiripv6m = TXGBE_FDIRIP6MSK_DST(info->mask.dst_ipv6_mask) | 236 TXGBE_FDIRIP6MSK_SRC(info->mask.src_ipv6_mask); 237 238 wr32(hw, TXGBE_FDIRIP6MSK, ~fdiripv6m); 239 } 240 241 return 0; 242 } 243 244 static int 245 txgbe_fdir_store_input_mask(struct rte_eth_dev *dev) 246 { 247 struct rte_eth_fdir_masks *input_mask = 248 &dev->data->dev_conf.fdir_conf.mask; 249 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 250 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 251 uint16_t dst_ipv6m = 0; 252 uint16_t src_ipv6m = 0; 253 254 if (mode != RTE_FDIR_MODE_SIGNATURE && 255 mode != RTE_FDIR_MODE_PERFECT) { 256 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 257 return -ENOTSUP; 258 } 259 260 memset(&info->mask, 0, sizeof(struct txgbe_hw_fdir_mask)); 261 info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; 262 info->mask.src_port_mask = input_mask->src_port_mask; 263 info->mask.dst_port_mask = input_mask->dst_port_mask; 264 info->mask.src_ipv4_mask = input_mask->ipv4_mask.src_ip; 265 info->mask.dst_ipv4_mask = input_mask->ipv4_mask.dst_ip; 266 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.src_ip, src_ipv6m); 267 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.dst_ip, dst_ipv6m); 268 info->mask.src_ipv6_mask = src_ipv6m; 269 info->mask.dst_ipv6_mask = dst_ipv6m; 270 271 return 0; 272 } 273 274 int 275 txgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, 276 uint16_t offset) 277 { 278 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 279 int i; 280 281 for (i = 0; i < 64; i++) { 282 uint32_t flexreg, flex; 283 flexreg = rd32(hw, TXGBE_FDIRFLEXCFG(i / 4)); 284 flex = TXGBE_FDIRFLEXCFG_BASE_MAC; 285 flex |= TXGBE_FDIRFLEXCFG_OFST(offset / 2); 286 flexreg &= ~(TXGBE_FDIRFLEXCFG_ALL(~0UL, i % 4)); 287 flexreg |= TXGBE_FDIRFLEXCFG_ALL(flex, i % 4); 288 wr32(hw, TXGBE_FDIRFLEXCFG(i / 4), flexreg); 289 } 290 291 txgbe_flush(hw); 292 for (i = 0; i < TXGBE_FDIR_INIT_DONE_POLL; i++) { 293 if (rd32(hw, TXGBE_FDIRCTL) & 294 TXGBE_FDIRCTL_INITDONE) 295 break; 296 msec_delay(1); 297 } 298 return 0; 299 } 300 301 /* 302 * txgbe_check_fdir_flex_conf -check if the flex payload and mask configuration 303 * arguments are valid 304 */ 305 static int 306 txgbe_set_fdir_flex_conf(struct rte_eth_dev *dev, uint32_t flex) 307 { 308 const struct rte_eth_fdir_flex_conf *conf = 309 &dev->data->dev_conf.fdir_conf.flex_conf; 310 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 311 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 312 const struct rte_eth_flex_payload_cfg *flex_cfg; 313 const struct rte_eth_fdir_flex_mask *flex_mask; 314 uint16_t flexbytes = 0; 315 uint16_t i; 316 317 if (conf == NULL) { 318 PMD_DRV_LOG(ERR, "NULL pointer."); 319 return -EINVAL; 320 } 321 322 flex |= TXGBE_FDIRFLEXCFG_DIA; 323 324 for (i = 0; i < conf->nb_payloads; i++) { 325 flex_cfg = &conf->flex_set[i]; 326 if (flex_cfg->type != RTE_ETH_RAW_PAYLOAD) { 327 PMD_DRV_LOG(ERR, "unsupported payload type."); 328 return -EINVAL; 329 } 330 if (((flex_cfg->src_offset[0] & 0x1) == 0) && 331 (flex_cfg->src_offset[1] == flex_cfg->src_offset[0] + 1) && 332 flex_cfg->src_offset[0] <= TXGBE_MAX_FLX_SOURCE_OFF) { 333 flex &= ~TXGBE_FDIRFLEXCFG_OFST_MASK; 334 flex |= 335 TXGBE_FDIRFLEXCFG_OFST(flex_cfg->src_offset[0] / 2); 336 } else { 337 PMD_DRV_LOG(ERR, "invalid flexbytes arguments."); 338 return -EINVAL; 339 } 340 } 341 342 for (i = 0; i < conf->nb_flexmasks; i++) { 343 flex_mask = &conf->flex_mask[i]; 344 if (flex_mask->flow_type != RTE_ETH_FLOW_UNKNOWN) { 345 PMD_DRV_LOG(ERR, "flexmask should be set globally."); 346 return -EINVAL; 347 } 348 flexbytes = (uint16_t)(((flex_mask->mask[1] << 8) & 0xFF00) | 349 ((flex_mask->mask[0]) & 0xFF)); 350 if (flexbytes == UINT16_MAX) { 351 flex &= ~TXGBE_FDIRFLEXCFG_DIA; 352 } else if (flexbytes != 0) { 353 /* TXGBE_FDIRFLEXCFG_DIA is set by default when set mask */ 354 PMD_DRV_LOG(ERR, " invalid flexbytes mask arguments."); 355 return -EINVAL; 356 } 357 } 358 359 info->mask.flex_bytes_mask = flexbytes ? UINT16_MAX : 0; 360 info->flex_bytes_offset = (uint8_t)(TXGBD_FDIRFLEXCFG_OFST(flex) * 2); 361 362 for (i = 0; i < 64; i++) { 363 uint32_t flexreg; 364 flexreg = rd32(hw, TXGBE_FDIRFLEXCFG(i / 4)); 365 flexreg &= ~(TXGBE_FDIRFLEXCFG_ALL(~0UL, i % 4)); 366 flexreg |= TXGBE_FDIRFLEXCFG_ALL(flex, i % 4); 367 wr32(hw, TXGBE_FDIRFLEXCFG(i / 4), flexreg); 368 } 369 return 0; 370 } 371 372 int 373 txgbe_fdir_configure(struct rte_eth_dev *dev) 374 { 375 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 376 int err; 377 uint32_t fdirctrl, flex, pbsize; 378 int i; 379 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 380 381 PMD_INIT_FUNC_TRACE(); 382 383 /* supports mac-vlan and tunnel mode */ 384 if (mode != RTE_FDIR_MODE_SIGNATURE && 385 mode != RTE_FDIR_MODE_PERFECT) 386 return -ENOSYS; 387 388 err = configure_fdir_flags(&dev->data->dev_conf.fdir_conf, 389 &fdirctrl, &flex); 390 if (err) 391 return err; 392 393 /* 394 * Before enabling Flow Director, the Rx Packet Buffer size 395 * must be reduced. The new value is the current size minus 396 * flow director memory usage size. 397 */ 398 pbsize = rd32(hw, TXGBE_PBRXSIZE(0)); 399 pbsize -= TXGBD_FDIRCTL_BUF_BYTE(fdirctrl); 400 wr32(hw, TXGBE_PBRXSIZE(0), pbsize); 401 402 /* 403 * The defaults in the HW for RX PB 1-7 are not zero and so should be 404 * initialized to zero for non DCB mode otherwise actual total RX PB 405 * would be bigger than programmed and filter space would run into 406 * the PB 0 region. 407 */ 408 for (i = 1; i < 8; i++) 409 wr32(hw, TXGBE_PBRXSIZE(i), 0); 410 411 err = txgbe_fdir_store_input_mask(dev); 412 if (err < 0) { 413 PMD_INIT_LOG(ERR, " Error on setting FD mask"); 414 return err; 415 } 416 417 err = txgbe_fdir_set_input_mask(dev); 418 if (err < 0) { 419 PMD_INIT_LOG(ERR, " Error on setting FD mask"); 420 return err; 421 } 422 423 err = txgbe_set_fdir_flex_conf(dev, flex); 424 if (err < 0) { 425 PMD_INIT_LOG(ERR, " Error on setting FD flexible arguments."); 426 return err; 427 } 428 429 err = txgbe_fdir_enable(hw, fdirctrl); 430 if (err < 0) { 431 PMD_INIT_LOG(ERR, " Error on enabling FD."); 432 return err; 433 } 434 return 0; 435 } 436 437 /* 438 * Note that the bkt_hash field in the txgbe_atr_input structure is also never 439 * set. 440 * 441 * Compute the hashes for SW ATR 442 * @stream: input bitstream to compute the hash on 443 * @key: 32-bit hash key 444 **/ 445 static uint32_t 446 txgbe_atr_compute_hash(struct txgbe_atr_input *atr_input, 447 uint32_t key) 448 { 449 /* 450 * The algorithm is as follows: 451 * Hash[15:0] = Sum { S[n] x K[n+16] }, n = 0...350 452 * where Sum {A[n]}, n = 0...n is bitwise XOR of A[0], A[1]...A[n] 453 * and A[n] x B[n] is bitwise AND between same length strings 454 * 455 * K[n] is 16 bits, defined as: 456 * for n modulo 32 >= 15, K[n] = K[n % 32 : (n % 32) - 15] 457 * for n modulo 32 < 15, K[n] = 458 * K[(n % 32:0) | (31:31 - (14 - (n % 32)))] 459 * 460 * S[n] is 16 bits, defined as: 461 * for n >= 15, S[n] = S[n:n - 15] 462 * for n < 15, S[n] = S[(n:0) | (350:350 - (14 - n))] 463 * 464 * To simplify for programming, the algorithm is implemented 465 * in software this way: 466 * 467 * key[31:0], hi_hash_dword[31:0], lo_hash_dword[31:0], hash[15:0] 468 * 469 * for (i = 0; i < 352; i+=32) 470 * hi_hash_dword[31:0] ^= Stream[(i+31):i]; 471 * 472 * lo_hash_dword[15:0] ^= Stream[15:0]; 473 * lo_hash_dword[15:0] ^= hi_hash_dword[31:16]; 474 * lo_hash_dword[31:16] ^= hi_hash_dword[15:0]; 475 * 476 * hi_hash_dword[31:0] ^= Stream[351:320]; 477 * 478 * if (key[0]) 479 * hash[15:0] ^= Stream[15:0]; 480 * 481 * for (i = 0; i < 16; i++) { 482 * if (key[i]) 483 * hash[15:0] ^= lo_hash_dword[(i+15):i]; 484 * if (key[i + 16]) 485 * hash[15:0] ^= hi_hash_dword[(i+15):i]; 486 * } 487 * 488 */ 489 __be32 *dword_stream = (__be32 *)atr_input; 490 __be32 common_hash_dword = 0; 491 u32 hi_hash_dword, lo_hash_dword, flow_pool_ptid; 492 u32 hash_result = 0; 493 u8 i; 494 495 /* record the flow_vm_vlan bits as they are a key part to the hash */ 496 flow_pool_ptid = be_to_cpu32(dword_stream[0]); 497 498 /* generate common hash dword */ 499 for (i = 1; i <= 10; i++) 500 common_hash_dword ^= dword_stream[i]; 501 502 hi_hash_dword = be_to_cpu32(common_hash_dword); 503 504 /* low dword is word swapped version of common */ 505 lo_hash_dword = (hi_hash_dword >> 16) | (hi_hash_dword << 16); 506 507 /* apply (Flow ID/VM Pool/Packet Type) bits to hash words */ 508 hi_hash_dword ^= flow_pool_ptid ^ (flow_pool_ptid >> 16); 509 510 /* Process bits 0 and 16 */ 511 if (key & 0x0001) 512 hash_result ^= lo_hash_dword; 513 if (key & 0x00010000) 514 hash_result ^= hi_hash_dword; 515 516 /* 517 * apply flow ID/VM pool/VLAN ID bits to lo hash dword, we had to 518 * delay this because bit 0 of the stream should not be processed 519 * so we do not add the vlan until after bit 0 was processed 520 */ 521 lo_hash_dword ^= flow_pool_ptid ^ (flow_pool_ptid << 16); 522 523 /* process the remaining 30 bits in the key 2 bits at a time */ 524 for (i = 15; i; i--) { 525 if (key & (0x0001 << i)) 526 hash_result ^= lo_hash_dword >> i; 527 if (key & (0x00010000 << i)) 528 hash_result ^= hi_hash_dword >> i; 529 } 530 531 return hash_result; 532 } 533 534 static uint32_t 535 atr_compute_perfect_hash(struct txgbe_atr_input *input, 536 enum rte_fdir_pballoc_type pballoc) 537 { 538 uint32_t bucket_hash; 539 540 bucket_hash = txgbe_atr_compute_hash(input, 541 TXGBE_ATR_BUCKET_HASH_KEY); 542 if (pballoc == RTE_FDIR_PBALLOC_256K) 543 bucket_hash &= PERFECT_BUCKET_256KB_HASH_MASK; 544 else if (pballoc == RTE_FDIR_PBALLOC_128K) 545 bucket_hash &= PERFECT_BUCKET_128KB_HASH_MASK; 546 else 547 bucket_hash &= PERFECT_BUCKET_64KB_HASH_MASK; 548 549 return TXGBE_FDIRPIHASH_BKT(bucket_hash); 550 } 551 552 /** 553 * txgbe_fdir_check_cmd_complete - poll to check whether FDIRPICMD is complete 554 * @hw: pointer to hardware structure 555 */ 556 static inline int 557 txgbe_fdir_check_cmd_complete(struct txgbe_hw *hw, uint32_t *fdircmd) 558 { 559 int i; 560 561 for (i = 0; i < TXGBE_FDIRCMD_CMD_POLL; i++) { 562 *fdircmd = rd32(hw, TXGBE_FDIRPICMD); 563 if (!(*fdircmd & TXGBE_FDIRPICMD_OP_MASK)) 564 return 0; 565 rte_delay_us(TXGBE_FDIRCMD_CMD_INTERVAL_US); 566 } 567 568 return -ETIMEDOUT; 569 } 570 571 /* 572 * Calculate the hash value needed for signature-match filters. In the FreeBSD 573 * driver, this is done by the optimised function 574 * txgbe_atr_compute_sig_hash_raptor(). However that can't be used here as it 575 * doesn't support calculating a hash for an IPv6 filter. 576 */ 577 static uint32_t 578 atr_compute_signature_hash(struct txgbe_atr_input *input, 579 enum rte_fdir_pballoc_type pballoc) 580 { 581 uint32_t bucket_hash, sig_hash; 582 583 bucket_hash = txgbe_atr_compute_hash(input, 584 TXGBE_ATR_BUCKET_HASH_KEY); 585 if (pballoc == RTE_FDIR_PBALLOC_256K) 586 bucket_hash &= SIG_BUCKET_256KB_HASH_MASK; 587 else if (pballoc == RTE_FDIR_PBALLOC_128K) 588 bucket_hash &= SIG_BUCKET_128KB_HASH_MASK; 589 else 590 bucket_hash &= SIG_BUCKET_64KB_HASH_MASK; 591 592 sig_hash = txgbe_atr_compute_hash(input, 593 TXGBE_ATR_SIGNATURE_HASH_KEY); 594 595 return TXGBE_FDIRPIHASH_SIG(sig_hash) | 596 TXGBE_FDIRPIHASH_BKT(bucket_hash); 597 } 598 599 /** 600 * With the ability to set extra flags in FDIRPICMD register 601 * added, and IPv6 support also added. The hash value is also pre-calculated 602 * as the pballoc value is needed to do it. 603 */ 604 static int 605 fdir_write_perfect_filter(struct txgbe_hw *hw, 606 struct txgbe_atr_input *input, uint8_t queue, 607 uint32_t fdircmd, uint32_t fdirhash, 608 enum rte_fdir_mode mode) 609 { 610 uint32_t fdirport, fdirflex; 611 int err = 0; 612 613 UNREFERENCED_PARAMETER(mode); 614 615 /* record the IPv4 address (little-endian) 616 * can not use wr32. 617 */ 618 wr32(hw, TXGBE_FDIRPISIP4, be_to_le32(input->src_ip[0])); 619 wr32(hw, TXGBE_FDIRPIDIP4, be_to_le32(input->dst_ip[0])); 620 621 /* record source and destination port (little-endian)*/ 622 fdirport = TXGBE_FDIRPIPORT_DST(be_to_le16(input->dst_port)); 623 fdirport |= TXGBE_FDIRPIPORT_SRC(be_to_le16(input->src_port)); 624 wr32(hw, TXGBE_FDIRPIPORT, fdirport); 625 626 /* record pkt_type (little-endian) and flex_bytes(big-endian) */ 627 fdirflex = TXGBE_FDIRPIFLEX_FLEX(be_to_npu16(input->flex_bytes)); 628 fdirflex |= TXGBE_FDIRPIFLEX_PTYPE(be_to_le16(input->pkt_type)); 629 wr32(hw, TXGBE_FDIRPIFLEX, fdirflex); 630 631 /* configure FDIRHASH register */ 632 fdirhash |= TXGBE_FDIRPIHASH_VLD; 633 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 634 635 /* 636 * flush all previous writes to make certain registers are 637 * programmed prior to issuing the command 638 */ 639 txgbe_flush(hw); 640 641 /* configure FDIRPICMD register */ 642 fdircmd |= TXGBE_FDIRPICMD_OP_ADD | 643 TXGBE_FDIRPICMD_UPD | 644 TXGBE_FDIRPICMD_LAST | 645 TXGBE_FDIRPICMD_QPENA; 646 fdircmd |= TXGBE_FDIRPICMD_FT(input->flow_type); 647 fdircmd |= TXGBE_FDIRPICMD_QP(queue); 648 fdircmd |= TXGBE_FDIRPICMD_POOL(input->vm_pool); 649 650 wr32(hw, TXGBE_FDIRPICMD, fdircmd); 651 652 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 653 654 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 655 if (err < 0) 656 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 657 658 return err; 659 } 660 661 /** 662 * This function supports setting extra fields in the FDIRPICMD register, and 663 * removes the code that was verifying the flow_type field. According to the 664 * documentation, a flow type of 00 (i.e. not TCP, UDP, or SCTP) is not 665 * supported, however it appears to work ok... 666 * Adds a signature hash filter 667 * @hw: pointer to hardware structure 668 * @input: unique input dword 669 * @queue: queue index to direct traffic to 670 * @fdircmd: any extra flags to set in fdircmd register 671 * @fdirhash: pre-calculated hash value for the filter 672 **/ 673 static int 674 fdir_add_signature_filter(struct txgbe_hw *hw, 675 struct txgbe_atr_input *input, uint8_t queue, uint32_t fdircmd, 676 uint32_t fdirhash) 677 { 678 int err = 0; 679 680 PMD_INIT_FUNC_TRACE(); 681 682 /* configure FDIRPICMD register */ 683 fdircmd |= TXGBE_FDIRPICMD_OP_ADD | 684 TXGBE_FDIRPICMD_UPD | 685 TXGBE_FDIRPICMD_LAST | 686 TXGBE_FDIRPICMD_QPENA; 687 fdircmd |= TXGBE_FDIRPICMD_FT(input->flow_type); 688 fdircmd |= TXGBE_FDIRPICMD_QP(queue); 689 690 fdirhash |= TXGBE_FDIRPIHASH_VLD; 691 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 692 wr32(hw, TXGBE_FDIRPICMD, fdircmd); 693 694 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 695 696 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 697 if (err < 0) 698 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 699 700 return err; 701 } 702 703 /* 704 * This is modified to take in the hash as a parameter so that 705 * it can be used for removing signature and perfect filters. 706 */ 707 static int 708 fdir_erase_filter_raptor(struct txgbe_hw *hw, uint32_t fdirhash) 709 { 710 uint32_t fdircmd = 0; 711 int err = 0; 712 713 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 714 715 /* flush hash to HW */ 716 txgbe_flush(hw); 717 718 /* Query if filter is present */ 719 wr32(hw, TXGBE_FDIRPICMD, TXGBE_FDIRPICMD_OP_QRY); 720 721 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 722 if (err < 0) { 723 PMD_INIT_LOG(ERR, "Timeout querying for flow director filter."); 724 return err; 725 } 726 727 /* if filter exists in hardware then remove it */ 728 if (fdircmd & TXGBE_FDIRPICMD_VLD) { 729 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 730 txgbe_flush(hw); 731 wr32(hw, TXGBE_FDIRPICMD, TXGBE_FDIRPICMD_OP_REM); 732 } 733 734 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 735 if (err < 0) 736 PMD_INIT_LOG(ERR, "Timeout erasing flow director filter."); 737 738 return err; 739 } 740 741 static inline struct txgbe_fdir_filter * 742 txgbe_fdir_filter_lookup(struct txgbe_hw_fdir_info *fdir_info, 743 struct txgbe_atr_input *input) 744 { 745 int ret; 746 747 ret = rte_hash_lookup(fdir_info->hash_handle, (const void *)input); 748 if (ret < 0) 749 return NULL; 750 751 return fdir_info->hash_map[ret]; 752 } 753 754 static inline int 755 txgbe_insert_fdir_filter(struct txgbe_hw_fdir_info *fdir_info, 756 struct txgbe_fdir_filter *fdir_filter) 757 { 758 int ret; 759 760 ret = rte_hash_add_key(fdir_info->hash_handle, &fdir_filter->input); 761 if (ret < 0) { 762 PMD_DRV_LOG(ERR, 763 "Failed to insert fdir filter to hash table %d!", 764 ret); 765 return ret; 766 } 767 768 fdir_info->hash_map[ret] = fdir_filter; 769 770 TAILQ_INSERT_TAIL(&fdir_info->fdir_list, fdir_filter, entries); 771 772 return 0; 773 } 774 775 static inline int 776 txgbe_remove_fdir_filter(struct txgbe_hw_fdir_info *fdir_info, 777 struct txgbe_atr_input *input) 778 { 779 int ret; 780 struct txgbe_fdir_filter *fdir_filter; 781 782 ret = rte_hash_del_key(fdir_info->hash_handle, input); 783 if (ret < 0) 784 return ret; 785 786 fdir_filter = fdir_info->hash_map[ret]; 787 fdir_info->hash_map[ret] = NULL; 788 789 TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries); 790 rte_free(fdir_filter); 791 792 return 0; 793 } 794 795 int 796 txgbe_fdir_filter_program(struct rte_eth_dev *dev, 797 struct txgbe_fdir_rule *rule, 798 bool del, 799 bool update) 800 { 801 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 802 uint32_t fdirhash; 803 uint8_t queue; 804 bool is_perfect = FALSE; 805 int err; 806 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 807 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 808 struct txgbe_fdir_filter *node; 809 810 if (fdir_mode == RTE_FDIR_MODE_NONE || 811 fdir_mode != rule->mode) 812 return -ENOTSUP; 813 814 if (fdir_mode >= RTE_FDIR_MODE_PERFECT) 815 is_perfect = TRUE; 816 817 if (is_perfect) { 818 if (rule->input.flow_type & TXGBE_ATR_L3TYPE_IPV6) { 819 PMD_DRV_LOG(ERR, "IPv6 is not supported in" 820 " perfect mode!"); 821 return -ENOTSUP; 822 } 823 fdirhash = atr_compute_perfect_hash(&rule->input, 824 dev->data->dev_conf.fdir_conf.pballoc); 825 fdirhash |= TXGBE_FDIRPIHASH_IDX(rule->soft_id); 826 } else { 827 fdirhash = atr_compute_signature_hash(&rule->input, 828 dev->data->dev_conf.fdir_conf.pballoc); 829 } 830 831 if (del) { 832 err = txgbe_remove_fdir_filter(info, &rule->input); 833 if (err < 0) { 834 PMD_DRV_LOG(ERR, 835 "No such fdir filter to delete %d!", err); 836 return err; 837 } 838 839 err = fdir_erase_filter_raptor(hw, fdirhash); 840 if (err < 0) 841 PMD_DRV_LOG(ERR, "Fail to delete FDIR filter!"); 842 else 843 PMD_DRV_LOG(DEBUG, "Success to delete FDIR filter!"); 844 return err; 845 } 846 847 /* add or update an fdir filter*/ 848 if (rule->fdirflags & TXGBE_FDIRPICMD_DROP) { 849 if (!is_perfect) { 850 PMD_DRV_LOG(ERR, "Drop option is not supported in" 851 " signature mode."); 852 return -EINVAL; 853 } 854 queue = dev->data->dev_conf.fdir_conf.drop_queue; 855 } else if (rule->queue < TXGBE_MAX_RX_QUEUE_NUM) { 856 queue = rule->queue; 857 } else { 858 return -EINVAL; 859 } 860 861 node = txgbe_fdir_filter_lookup(info, &rule->input); 862 if (node) { 863 if (!update) { 864 PMD_DRV_LOG(ERR, "Conflict with existing fdir filter!"); 865 return -EINVAL; 866 } 867 node->fdirflags = rule->fdirflags; 868 node->fdirhash = fdirhash; 869 node->queue = queue; 870 } else { 871 node = rte_zmalloc("txgbe_fdir", 872 sizeof(struct txgbe_fdir_filter), 0); 873 if (!node) 874 return -ENOMEM; 875 rte_memcpy(&node->input, &rule->input, 876 sizeof(struct txgbe_atr_input)); 877 node->fdirflags = rule->fdirflags; 878 node->fdirhash = fdirhash; 879 node->queue = queue; 880 881 err = txgbe_insert_fdir_filter(info, node); 882 if (err < 0) { 883 rte_free(node); 884 return err; 885 } 886 } 887 888 if (is_perfect) 889 err = fdir_write_perfect_filter(hw, &node->input, 890 node->queue, node->fdirflags, 891 node->fdirhash, fdir_mode); 892 else 893 err = fdir_add_signature_filter(hw, &node->input, 894 node->queue, node->fdirflags, 895 node->fdirhash); 896 if (err < 0) { 897 PMD_DRV_LOG(ERR, "Fail to add FDIR filter!"); 898 txgbe_remove_fdir_filter(info, &rule->input); 899 } else { 900 PMD_DRV_LOG(DEBUG, "Success to add FDIR filter"); 901 } 902 903 return err; 904 } 905 906 static int 907 txgbe_fdir_flush(struct rte_eth_dev *dev) 908 { 909 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 910 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 911 int ret; 912 913 ret = txgbe_reinit_fdir_tables(hw); 914 if (ret < 0) { 915 PMD_INIT_LOG(ERR, "Failed to re-initialize FD table."); 916 return ret; 917 } 918 919 info->f_add = 0; 920 info->f_remove = 0; 921 info->add = 0; 922 info->remove = 0; 923 924 return ret; 925 } 926 927 /* restore flow director filter */ 928 void 929 txgbe_fdir_filter_restore(struct rte_eth_dev *dev) 930 { 931 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 932 struct txgbe_hw_fdir_info *fdir_info = TXGBE_DEV_FDIR(dev); 933 struct txgbe_fdir_filter *node; 934 bool is_perfect = FALSE; 935 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 936 937 if (fdir_mode >= RTE_FDIR_MODE_PERFECT && 938 fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 939 is_perfect = TRUE; 940 941 if (is_perfect) { 942 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 943 (void)fdir_write_perfect_filter(hw, 944 &node->input, 945 node->queue, 946 node->fdirflags, 947 node->fdirhash, 948 fdir_mode); 949 } 950 } else { 951 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 952 (void)fdir_add_signature_filter(hw, 953 &node->input, 954 node->queue, 955 node->fdirflags, 956 node->fdirhash); 957 } 958 } 959 } 960 961 /* remove all the flow director filters */ 962 int 963 txgbe_clear_all_fdir_filter(struct rte_eth_dev *dev) 964 { 965 struct txgbe_hw_fdir_info *fdir_info = TXGBE_DEV_FDIR(dev); 966 struct txgbe_fdir_filter *fdir_filter; 967 struct txgbe_fdir_filter *filter_flag; 968 int ret = 0; 969 970 /* flush flow director */ 971 rte_hash_reset(fdir_info->hash_handle); 972 memset(fdir_info->hash_map, 0, 973 sizeof(struct txgbe_fdir_filter *) * TXGBE_MAX_FDIR_FILTER_NUM); 974 filter_flag = TAILQ_FIRST(&fdir_info->fdir_list); 975 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 976 TAILQ_REMOVE(&fdir_info->fdir_list, 977 fdir_filter, 978 entries); 979 rte_free(fdir_filter); 980 } 981 982 if (filter_flag != NULL) 983 ret = txgbe_fdir_flush(dev); 984 985 return ret; 986 } 987