1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2020 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <stdarg.h> 8 #include <errno.h> 9 #include <sys/queue.h> 10 #include <rte_malloc.h> 11 12 #include "txgbe_logs.h" 13 #include "base/txgbe.h" 14 #include "txgbe_ethdev.h" 15 16 #define TXGBE_DEFAULT_FLEXBYTES_OFFSET 12 /*default flexbytes offset in bytes*/ 17 #define TXGBE_MAX_FLX_SOURCE_OFF 62 18 #define TXGBE_FDIRCMD_CMD_INTERVAL_US 10 19 20 #define IPV6_ADDR_TO_MASK(ipaddr, ipv6m) do { \ 21 uint8_t ipv6_addr[16]; \ 22 uint8_t i; \ 23 rte_memcpy(ipv6_addr, (ipaddr), sizeof(ipv6_addr));\ 24 (ipv6m) = 0; \ 25 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 26 if (ipv6_addr[i] == UINT8_MAX) \ 27 (ipv6m) |= 1 << i; \ 28 else if (ipv6_addr[i] != 0) { \ 29 PMD_DRV_LOG(ERR, " invalid IPv6 address mask."); \ 30 return -EINVAL; \ 31 } \ 32 } \ 33 } while (0) 34 35 #define IPV6_MASK_TO_ADDR(ipv6m, ipaddr) do { \ 36 uint8_t ipv6_addr[16]; \ 37 uint8_t i; \ 38 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 39 if ((ipv6m) & (1 << i)) \ 40 ipv6_addr[i] = UINT8_MAX; \ 41 else \ 42 ipv6_addr[i] = 0; \ 43 } \ 44 rte_memcpy((ipaddr), ipv6_addr, sizeof(ipv6_addr));\ 45 } while (0) 46 47 /** 48 * Initialize Flow Director control registers 49 * @hw: pointer to hardware structure 50 * @fdirctrl: value to write to flow director control register 51 **/ 52 static int 53 txgbe_fdir_enable(struct txgbe_hw *hw, uint32_t fdirctrl) 54 { 55 int i; 56 57 PMD_INIT_FUNC_TRACE(); 58 59 /* Prime the keys for hashing */ 60 wr32(hw, TXGBE_FDIRBKTHKEY, TXGBE_ATR_BUCKET_HASH_KEY); 61 wr32(hw, TXGBE_FDIRSIGHKEY, TXGBE_ATR_SIGNATURE_HASH_KEY); 62 63 /* 64 * Continue setup of fdirctrl register bits: 65 * Set the maximum length per hash bucket to 0xA filters 66 * Send interrupt when 64 filters are left 67 */ 68 fdirctrl |= TXGBE_FDIRCTL_MAXLEN(0xA) | 69 TXGBE_FDIRCTL_FULLTHR(4); 70 71 /* 72 * Poll init-done after we write the register. Estimated times: 73 * 10G: PBALLOC = 11b, timing is 60us 74 * 1G: PBALLOC = 11b, timing is 600us 75 * 100M: PBALLOC = 11b, timing is 6ms 76 * 77 * Multiple these timings by 4 if under full Rx load 78 * 79 * So we'll poll for TXGBE_FDIR_INIT_DONE_POLL times, sleeping for 80 * 1 msec per poll time. If we're at line rate and drop to 100M, then 81 * this might not finish in our poll time, but we can live with that 82 * for now. 83 */ 84 wr32(hw, TXGBE_FDIRCTL, fdirctrl); 85 txgbe_flush(hw); 86 for (i = 0; i < TXGBE_FDIR_INIT_DONE_POLL; i++) { 87 if (rd32(hw, TXGBE_FDIRCTL) & TXGBE_FDIRCTL_INITDONE) 88 break; 89 msec_delay(1); 90 } 91 92 if (i >= TXGBE_FDIR_INIT_DONE_POLL) { 93 PMD_INIT_LOG(ERR, "Flow Director poll time exceeded during enabling!"); 94 return -ETIMEDOUT; 95 } 96 return 0; 97 } 98 99 /* 100 * Set appropriate bits in fdirctrl for: variable reporting levels, moving 101 * flexbytes matching field, and drop queue (only for perfect matching mode). 102 */ 103 static inline int 104 configure_fdir_flags(const struct rte_fdir_conf *conf, 105 uint32_t *fdirctrl, uint32_t *flex) 106 { 107 *fdirctrl = 0; 108 *flex = 0; 109 110 switch (conf->pballoc) { 111 case RTE_FDIR_PBALLOC_64K: 112 /* 8k - 1 signature filters */ 113 *fdirctrl |= TXGBE_FDIRCTL_BUF_64K; 114 break; 115 case RTE_FDIR_PBALLOC_128K: 116 /* 16k - 1 signature filters */ 117 *fdirctrl |= TXGBE_FDIRCTL_BUF_128K; 118 break; 119 case RTE_FDIR_PBALLOC_256K: 120 /* 32k - 1 signature filters */ 121 *fdirctrl |= TXGBE_FDIRCTL_BUF_256K; 122 break; 123 default: 124 /* bad value */ 125 PMD_INIT_LOG(ERR, "Invalid fdir_conf->pballoc value"); 126 return -EINVAL; 127 }; 128 129 /* status flags: write hash & swindex in the rx descriptor */ 130 switch (conf->status) { 131 case RTE_FDIR_NO_REPORT_STATUS: 132 /* do nothing, default mode */ 133 break; 134 case RTE_FDIR_REPORT_STATUS: 135 /* report status when the packet matches a fdir rule */ 136 *fdirctrl |= TXGBE_FDIRCTL_REPORT_MATCH; 137 break; 138 case RTE_FDIR_REPORT_STATUS_ALWAYS: 139 /* always report status */ 140 *fdirctrl |= TXGBE_FDIRCTL_REPORT_ALWAYS; 141 break; 142 default: 143 /* bad value */ 144 PMD_INIT_LOG(ERR, "Invalid fdir_conf->status value"); 145 return -EINVAL; 146 }; 147 148 *flex |= TXGBE_FDIRFLEXCFG_BASE_MAC; 149 *flex |= TXGBE_FDIRFLEXCFG_OFST(TXGBE_DEFAULT_FLEXBYTES_OFFSET / 2); 150 151 switch (conf->mode) { 152 case RTE_FDIR_MODE_SIGNATURE: 153 break; 154 case RTE_FDIR_MODE_PERFECT: 155 *fdirctrl |= TXGBE_FDIRCTL_PERFECT; 156 *fdirctrl |= TXGBE_FDIRCTL_DROPQP(conf->drop_queue); 157 break; 158 default: 159 /* bad value */ 160 PMD_INIT_LOG(ERR, "Invalid fdir_conf->mode value"); 161 return -EINVAL; 162 } 163 164 return 0; 165 } 166 167 static inline uint32_t 168 reverse_fdir_bmks(uint16_t hi_dword, uint16_t lo_dword) 169 { 170 uint32_t mask = hi_dword << 16; 171 172 mask |= lo_dword; 173 mask = ((mask & 0x55555555) << 1) | ((mask & 0xAAAAAAAA) >> 1); 174 mask = ((mask & 0x33333333) << 2) | ((mask & 0xCCCCCCCC) >> 2); 175 mask = ((mask & 0x0F0F0F0F) << 4) | ((mask & 0xF0F0F0F0) >> 4); 176 return ((mask & 0x00FF00FF) << 8) | ((mask & 0xFF00FF00) >> 8); 177 } 178 179 int 180 txgbe_fdir_set_input_mask(struct rte_eth_dev *dev) 181 { 182 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 183 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 184 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 185 /* 186 * mask VM pool and DIPv6 since there are currently not supported 187 * mask FLEX byte, it will be set in flex_conf 188 */ 189 uint32_t fdirm = TXGBE_FDIRMSK_POOL; 190 uint32_t fdirtcpm; /* TCP source and destination port masks. */ 191 uint32_t fdiripv6m; /* IPv6 source and destination masks. */ 192 193 PMD_INIT_FUNC_TRACE(); 194 195 if (mode != RTE_FDIR_MODE_SIGNATURE && 196 mode != RTE_FDIR_MODE_PERFECT) { 197 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 198 return -ENOTSUP; 199 } 200 201 /* 202 * Program the relevant mask registers. If src/dst_port or src/dst_addr 203 * are zero, then assume a full mask for that field. Also assume that 204 * a VLAN of 0 is unspecified, so mask that out as well. L4type 205 * cannot be masked out in this implementation. 206 */ 207 if (info->mask.dst_port_mask == 0 && info->mask.src_port_mask == 0) { 208 /* use the L4 protocol mask for raw IPv4/IPv6 traffic */ 209 fdirm |= TXGBE_FDIRMSK_L4P; 210 } 211 212 /* TBD: don't support encapsulation yet */ 213 wr32(hw, TXGBE_FDIRMSK, fdirm); 214 215 /* store the TCP/UDP port masks, bit reversed from port layout */ 216 fdirtcpm = reverse_fdir_bmks(rte_be_to_cpu_16(info->mask.dst_port_mask), 217 rte_be_to_cpu_16(info->mask.src_port_mask)); 218 219 /* write all the same so that UDP, TCP and SCTP use the same mask 220 * (little-endian) 221 */ 222 wr32(hw, TXGBE_FDIRTCPMSK, ~fdirtcpm); 223 wr32(hw, TXGBE_FDIRUDPMSK, ~fdirtcpm); 224 wr32(hw, TXGBE_FDIRSCTPMSK, ~fdirtcpm); 225 226 /* Store source and destination IPv4 masks (big-endian) */ 227 wr32(hw, TXGBE_FDIRSIP4MSK, ~info->mask.src_ipv4_mask); 228 wr32(hw, TXGBE_FDIRDIP4MSK, ~info->mask.dst_ipv4_mask); 229 230 if (mode == RTE_FDIR_MODE_SIGNATURE) { 231 /* 232 * Store source and destination IPv6 masks (bit reversed) 233 */ 234 fdiripv6m = TXGBE_FDIRIP6MSK_DST(info->mask.dst_ipv6_mask) | 235 TXGBE_FDIRIP6MSK_SRC(info->mask.src_ipv6_mask); 236 237 wr32(hw, TXGBE_FDIRIP6MSK, ~fdiripv6m); 238 } 239 240 return 0; 241 } 242 243 static int 244 txgbe_fdir_store_input_mask(struct rte_eth_dev *dev) 245 { 246 struct rte_eth_fdir_masks *input_mask = 247 &dev->data->dev_conf.fdir_conf.mask; 248 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 249 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 250 uint16_t dst_ipv6m = 0; 251 uint16_t src_ipv6m = 0; 252 253 if (mode != RTE_FDIR_MODE_SIGNATURE && 254 mode != RTE_FDIR_MODE_PERFECT) { 255 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 256 return -ENOTSUP; 257 } 258 259 memset(&info->mask, 0, sizeof(struct txgbe_hw_fdir_mask)); 260 info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; 261 info->mask.src_port_mask = input_mask->src_port_mask; 262 info->mask.dst_port_mask = input_mask->dst_port_mask; 263 info->mask.src_ipv4_mask = input_mask->ipv4_mask.src_ip; 264 info->mask.dst_ipv4_mask = input_mask->ipv4_mask.dst_ip; 265 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.src_ip, src_ipv6m); 266 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.dst_ip, dst_ipv6m); 267 info->mask.src_ipv6_mask = src_ipv6m; 268 info->mask.dst_ipv6_mask = dst_ipv6m; 269 270 return 0; 271 } 272 273 int 274 txgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, 275 uint16_t offset) 276 { 277 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 278 int i; 279 280 for (i = 0; i < 64; i++) { 281 uint32_t flexreg, flex; 282 flexreg = rd32(hw, TXGBE_FDIRFLEXCFG(i / 4)); 283 flex = TXGBE_FDIRFLEXCFG_BASE_MAC; 284 flex |= TXGBE_FDIRFLEXCFG_OFST(offset / 2); 285 flexreg &= ~(TXGBE_FDIRFLEXCFG_ALL(~0UL, i % 4)); 286 flexreg |= TXGBE_FDIRFLEXCFG_ALL(flex, i % 4); 287 wr32(hw, TXGBE_FDIRFLEXCFG(i / 4), flexreg); 288 } 289 290 txgbe_flush(hw); 291 for (i = 0; i < TXGBE_FDIR_INIT_DONE_POLL; i++) { 292 if (rd32(hw, TXGBE_FDIRCTL) & 293 TXGBE_FDIRCTL_INITDONE) 294 break; 295 msec_delay(1); 296 } 297 return 0; 298 } 299 300 /* 301 * txgbe_check_fdir_flex_conf -check if the flex payload and mask configuration 302 * arguments are valid 303 */ 304 static int 305 txgbe_set_fdir_flex_conf(struct rte_eth_dev *dev, uint32_t flex) 306 { 307 const struct rte_eth_fdir_flex_conf *conf = 308 &dev->data->dev_conf.fdir_conf.flex_conf; 309 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 310 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 311 const struct rte_eth_flex_payload_cfg *flex_cfg; 312 const struct rte_eth_fdir_flex_mask *flex_mask; 313 uint16_t flexbytes = 0; 314 uint16_t i; 315 316 if (conf == NULL) { 317 PMD_DRV_LOG(ERR, "NULL pointer."); 318 return -EINVAL; 319 } 320 321 flex |= TXGBE_FDIRFLEXCFG_DIA; 322 323 for (i = 0; i < conf->nb_payloads; i++) { 324 flex_cfg = &conf->flex_set[i]; 325 if (flex_cfg->type != RTE_ETH_RAW_PAYLOAD) { 326 PMD_DRV_LOG(ERR, "unsupported payload type."); 327 return -EINVAL; 328 } 329 if (((flex_cfg->src_offset[0] & 0x1) == 0) && 330 (flex_cfg->src_offset[1] == flex_cfg->src_offset[0] + 1) && 331 flex_cfg->src_offset[0] <= TXGBE_MAX_FLX_SOURCE_OFF) { 332 flex &= ~TXGBE_FDIRFLEXCFG_OFST_MASK; 333 flex |= 334 TXGBE_FDIRFLEXCFG_OFST(flex_cfg->src_offset[0] / 2); 335 } else { 336 PMD_DRV_LOG(ERR, "invalid flexbytes arguments."); 337 return -EINVAL; 338 } 339 } 340 341 for (i = 0; i < conf->nb_flexmasks; i++) { 342 flex_mask = &conf->flex_mask[i]; 343 if (flex_mask->flow_type != RTE_ETH_FLOW_UNKNOWN) { 344 PMD_DRV_LOG(ERR, "flexmask should be set globally."); 345 return -EINVAL; 346 } 347 flexbytes = (uint16_t)(((flex_mask->mask[1] << 8) & 0xFF00) | 348 ((flex_mask->mask[0]) & 0xFF)); 349 if (flexbytes == UINT16_MAX) { 350 flex &= ~TXGBE_FDIRFLEXCFG_DIA; 351 } else if (flexbytes != 0) { 352 /* TXGBE_FDIRFLEXCFG_DIA is set by default when set mask */ 353 PMD_DRV_LOG(ERR, " invalid flexbytes mask arguments."); 354 return -EINVAL; 355 } 356 } 357 358 info->mask.flex_bytes_mask = flexbytes ? UINT16_MAX : 0; 359 info->flex_bytes_offset = (uint8_t)(TXGBD_FDIRFLEXCFG_OFST(flex) * 2); 360 361 for (i = 0; i < 64; i++) { 362 uint32_t flexreg; 363 flexreg = rd32(hw, TXGBE_FDIRFLEXCFG(i / 4)); 364 flexreg &= ~(TXGBE_FDIRFLEXCFG_ALL(~0UL, i % 4)); 365 flexreg |= TXGBE_FDIRFLEXCFG_ALL(flex, i % 4); 366 wr32(hw, TXGBE_FDIRFLEXCFG(i / 4), flexreg); 367 } 368 return 0; 369 } 370 371 int 372 txgbe_fdir_configure(struct rte_eth_dev *dev) 373 { 374 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 375 int err; 376 uint32_t fdirctrl, flex, pbsize; 377 int i; 378 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 379 380 PMD_INIT_FUNC_TRACE(); 381 382 /* supports mac-vlan and tunnel mode */ 383 if (mode != RTE_FDIR_MODE_SIGNATURE && 384 mode != RTE_FDIR_MODE_PERFECT) 385 return -ENOSYS; 386 387 err = configure_fdir_flags(&dev->data->dev_conf.fdir_conf, 388 &fdirctrl, &flex); 389 if (err) 390 return err; 391 392 /* 393 * Before enabling Flow Director, the Rx Packet Buffer size 394 * must be reduced. The new value is the current size minus 395 * flow director memory usage size. 396 */ 397 pbsize = rd32(hw, TXGBE_PBRXSIZE(0)); 398 pbsize -= TXGBD_FDIRCTL_BUF_BYTE(fdirctrl); 399 wr32(hw, TXGBE_PBRXSIZE(0), pbsize); 400 401 /* 402 * The defaults in the HW for RX PB 1-7 are not zero and so should be 403 * initialized to zero for non DCB mode otherwise actual total RX PB 404 * would be bigger than programmed and filter space would run into 405 * the PB 0 region. 406 */ 407 for (i = 1; i < 8; i++) 408 wr32(hw, TXGBE_PBRXSIZE(i), 0); 409 410 err = txgbe_fdir_store_input_mask(dev); 411 if (err < 0) { 412 PMD_INIT_LOG(ERR, " Error on setting FD mask"); 413 return err; 414 } 415 416 err = txgbe_fdir_set_input_mask(dev); 417 if (err < 0) { 418 PMD_INIT_LOG(ERR, " Error on setting FD mask"); 419 return err; 420 } 421 422 err = txgbe_set_fdir_flex_conf(dev, flex); 423 if (err < 0) { 424 PMD_INIT_LOG(ERR, " Error on setting FD flexible arguments."); 425 return err; 426 } 427 428 err = txgbe_fdir_enable(hw, fdirctrl); 429 if (err < 0) { 430 PMD_INIT_LOG(ERR, " Error on enabling FD."); 431 return err; 432 } 433 return 0; 434 } 435 436 /* 437 * Note that the bkt_hash field in the txgbe_atr_input structure is also never 438 * set. 439 * 440 * Compute the hashes for SW ATR 441 * @stream: input bitstream to compute the hash on 442 * @key: 32-bit hash key 443 **/ 444 static uint32_t 445 txgbe_atr_compute_hash(struct txgbe_atr_input *atr_input, 446 uint32_t key) 447 { 448 /* 449 * The algorithm is as follows: 450 * Hash[15:0] = Sum { S[n] x K[n+16] }, n = 0...350 451 * where Sum {A[n]}, n = 0...n is bitwise XOR of A[0], A[1]...A[n] 452 * and A[n] x B[n] is bitwise AND between same length strings 453 * 454 * K[n] is 16 bits, defined as: 455 * for n modulo 32 >= 15, K[n] = K[n % 32 : (n % 32) - 15] 456 * for n modulo 32 < 15, K[n] = 457 * K[(n % 32:0) | (31:31 - (14 - (n % 32)))] 458 * 459 * S[n] is 16 bits, defined as: 460 * for n >= 15, S[n] = S[n:n - 15] 461 * for n < 15, S[n] = S[(n:0) | (350:350 - (14 - n))] 462 * 463 * To simplify for programming, the algorithm is implemented 464 * in software this way: 465 * 466 * key[31:0], hi_hash_dword[31:0], lo_hash_dword[31:0], hash[15:0] 467 * 468 * for (i = 0; i < 352; i+=32) 469 * hi_hash_dword[31:0] ^= Stream[(i+31):i]; 470 * 471 * lo_hash_dword[15:0] ^= Stream[15:0]; 472 * lo_hash_dword[15:0] ^= hi_hash_dword[31:16]; 473 * lo_hash_dword[31:16] ^= hi_hash_dword[15:0]; 474 * 475 * hi_hash_dword[31:0] ^= Stream[351:320]; 476 * 477 * if (key[0]) 478 * hash[15:0] ^= Stream[15:0]; 479 * 480 * for (i = 0; i < 16; i++) { 481 * if (key[i]) 482 * hash[15:0] ^= lo_hash_dword[(i+15):i]; 483 * if (key[i + 16]) 484 * hash[15:0] ^= hi_hash_dword[(i+15):i]; 485 * } 486 * 487 */ 488 __be32 *dword_stream = (__be32 *)atr_input; 489 __be32 common_hash_dword = 0; 490 u32 hi_hash_dword, lo_hash_dword, flow_pool_ptid; 491 u32 hash_result = 0; 492 u8 i; 493 494 /* record the flow_vm_vlan bits as they are a key part to the hash */ 495 flow_pool_ptid = be_to_cpu32(dword_stream[0]); 496 497 /* generate common hash dword */ 498 for (i = 1; i <= 10; i++) 499 common_hash_dword ^= dword_stream[i]; 500 501 hi_hash_dword = be_to_cpu32(common_hash_dword); 502 503 /* low dword is word swapped version of common */ 504 lo_hash_dword = (hi_hash_dword >> 16) | (hi_hash_dword << 16); 505 506 /* apply (Flow ID/VM Pool/Packet Type) bits to hash words */ 507 hi_hash_dword ^= flow_pool_ptid ^ (flow_pool_ptid >> 16); 508 509 /* Process bits 0 and 16 */ 510 if (key & 0x0001) 511 hash_result ^= lo_hash_dword; 512 if (key & 0x00010000) 513 hash_result ^= hi_hash_dword; 514 515 /* 516 * apply flow ID/VM pool/VLAN ID bits to lo hash dword, we had to 517 * delay this because bit 0 of the stream should not be processed 518 * so we do not add the vlan until after bit 0 was processed 519 */ 520 lo_hash_dword ^= flow_pool_ptid ^ (flow_pool_ptid << 16); 521 522 /* process the remaining 30 bits in the key 2 bits at a time */ 523 for (i = 15; i; i--) { 524 if (key & (0x0001 << i)) 525 hash_result ^= lo_hash_dword >> i; 526 if (key & (0x00010000 << i)) 527 hash_result ^= hi_hash_dword >> i; 528 } 529 530 return hash_result; 531 } 532 533 static uint32_t 534 atr_compute_perfect_hash(struct txgbe_atr_input *input, 535 enum rte_fdir_pballoc_type pballoc) 536 { 537 uint32_t bucket_hash; 538 539 bucket_hash = txgbe_atr_compute_hash(input, 540 TXGBE_ATR_BUCKET_HASH_KEY); 541 if (pballoc == RTE_FDIR_PBALLOC_256K) 542 bucket_hash &= PERFECT_BUCKET_256KB_HASH_MASK; 543 else if (pballoc == RTE_FDIR_PBALLOC_128K) 544 bucket_hash &= PERFECT_BUCKET_128KB_HASH_MASK; 545 else 546 bucket_hash &= PERFECT_BUCKET_64KB_HASH_MASK; 547 548 return TXGBE_FDIRPIHASH_BKT(bucket_hash); 549 } 550 551 /** 552 * txgbe_fdir_check_cmd_complete - poll to check whether FDIRPICMD is complete 553 * @hw: pointer to hardware structure 554 */ 555 static inline int 556 txgbe_fdir_check_cmd_complete(struct txgbe_hw *hw, uint32_t *fdircmd) 557 { 558 int i; 559 560 for (i = 0; i < TXGBE_FDIRCMD_CMD_POLL; i++) { 561 *fdircmd = rd32(hw, TXGBE_FDIRPICMD); 562 if (!(*fdircmd & TXGBE_FDIRPICMD_OP_MASK)) 563 return 0; 564 rte_delay_us(TXGBE_FDIRCMD_CMD_INTERVAL_US); 565 } 566 567 return -ETIMEDOUT; 568 } 569 570 /* 571 * Calculate the hash value needed for signature-match filters. In the FreeBSD 572 * driver, this is done by the optimised function 573 * txgbe_atr_compute_sig_hash_raptor(). However that can't be used here as it 574 * doesn't support calculating a hash for an IPv6 filter. 575 */ 576 static uint32_t 577 atr_compute_signature_hash(struct txgbe_atr_input *input, 578 enum rte_fdir_pballoc_type pballoc) 579 { 580 uint32_t bucket_hash, sig_hash; 581 582 bucket_hash = txgbe_atr_compute_hash(input, 583 TXGBE_ATR_BUCKET_HASH_KEY); 584 if (pballoc == RTE_FDIR_PBALLOC_256K) 585 bucket_hash &= SIG_BUCKET_256KB_HASH_MASK; 586 else if (pballoc == RTE_FDIR_PBALLOC_128K) 587 bucket_hash &= SIG_BUCKET_128KB_HASH_MASK; 588 else 589 bucket_hash &= SIG_BUCKET_64KB_HASH_MASK; 590 591 sig_hash = txgbe_atr_compute_hash(input, 592 TXGBE_ATR_SIGNATURE_HASH_KEY); 593 594 return TXGBE_FDIRPIHASH_SIG(sig_hash) | 595 TXGBE_FDIRPIHASH_BKT(bucket_hash); 596 } 597 598 /** 599 * With the ability to set extra flags in FDIRPICMD register 600 * added, and IPv6 support also added. The hash value is also pre-calculated 601 * as the pballoc value is needed to do it. 602 */ 603 static int 604 fdir_write_perfect_filter(struct txgbe_hw *hw, 605 struct txgbe_atr_input *input, uint8_t queue, 606 uint32_t fdircmd, uint32_t fdirhash, 607 enum rte_fdir_mode mode) 608 { 609 uint32_t fdirport, fdirflex; 610 int err = 0; 611 612 UNREFERENCED_PARAMETER(mode); 613 614 /* record the IPv4 address (little-endian) 615 * can not use wr32. 616 */ 617 wr32(hw, TXGBE_FDIRPISIP4, be_to_le32(input->src_ip[0])); 618 wr32(hw, TXGBE_FDIRPIDIP4, be_to_le32(input->dst_ip[0])); 619 620 /* record source and destination port (little-endian)*/ 621 fdirport = TXGBE_FDIRPIPORT_DST(be_to_le16(input->dst_port)); 622 fdirport |= TXGBE_FDIRPIPORT_SRC(be_to_le16(input->src_port)); 623 wr32(hw, TXGBE_FDIRPIPORT, fdirport); 624 625 /* record pkt_type (little-endian) and flex_bytes(big-endian) */ 626 fdirflex = TXGBE_FDIRPIFLEX_FLEX(be_to_npu16(input->flex_bytes)); 627 fdirflex |= TXGBE_FDIRPIFLEX_PTYPE(be_to_le16(input->pkt_type)); 628 wr32(hw, TXGBE_FDIRPIFLEX, fdirflex); 629 630 /* configure FDIRHASH register */ 631 fdirhash |= TXGBE_FDIRPIHASH_VLD; 632 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 633 634 /* 635 * flush all previous writes to make certain registers are 636 * programmed prior to issuing the command 637 */ 638 txgbe_flush(hw); 639 640 /* configure FDIRPICMD register */ 641 fdircmd |= TXGBE_FDIRPICMD_OP_ADD | 642 TXGBE_FDIRPICMD_UPD | 643 TXGBE_FDIRPICMD_LAST | 644 TXGBE_FDIRPICMD_QPENA; 645 fdircmd |= TXGBE_FDIRPICMD_FT(input->flow_type); 646 fdircmd |= TXGBE_FDIRPICMD_QP(queue); 647 fdircmd |= TXGBE_FDIRPICMD_POOL(input->vm_pool); 648 649 wr32(hw, TXGBE_FDIRPICMD, fdircmd); 650 651 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 652 653 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 654 if (err < 0) 655 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 656 657 return err; 658 } 659 660 /** 661 * This function supports setting extra fields in the FDIRPICMD register, and 662 * removes the code that was verifying the flow_type field. According to the 663 * documentation, a flow type of 00 (i.e. not TCP, UDP, or SCTP) is not 664 * supported, however it appears to work ok... 665 * Adds a signature hash filter 666 * @hw: pointer to hardware structure 667 * @input: unique input dword 668 * @queue: queue index to direct traffic to 669 * @fdircmd: any extra flags to set in fdircmd register 670 * @fdirhash: pre-calculated hash value for the filter 671 **/ 672 static int 673 fdir_add_signature_filter(struct txgbe_hw *hw, 674 struct txgbe_atr_input *input, uint8_t queue, uint32_t fdircmd, 675 uint32_t fdirhash) 676 { 677 int err = 0; 678 679 PMD_INIT_FUNC_TRACE(); 680 681 /* configure FDIRPICMD register */ 682 fdircmd |= TXGBE_FDIRPICMD_OP_ADD | 683 TXGBE_FDIRPICMD_UPD | 684 TXGBE_FDIRPICMD_LAST | 685 TXGBE_FDIRPICMD_QPENA; 686 fdircmd |= TXGBE_FDIRPICMD_FT(input->flow_type); 687 fdircmd |= TXGBE_FDIRPICMD_QP(queue); 688 689 fdirhash |= TXGBE_FDIRPIHASH_VLD; 690 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 691 wr32(hw, TXGBE_FDIRPICMD, fdircmd); 692 693 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 694 695 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 696 if (err < 0) 697 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 698 699 return err; 700 } 701 702 /* 703 * This is modified to take in the hash as a parameter so that 704 * it can be used for removing signature and perfect filters. 705 */ 706 static int 707 fdir_erase_filter_raptor(struct txgbe_hw *hw, uint32_t fdirhash) 708 { 709 uint32_t fdircmd = 0; 710 int err = 0; 711 712 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 713 714 /* flush hash to HW */ 715 txgbe_flush(hw); 716 717 /* Query if filter is present */ 718 wr32(hw, TXGBE_FDIRPICMD, TXGBE_FDIRPICMD_OP_QRY); 719 720 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 721 if (err < 0) { 722 PMD_INIT_LOG(ERR, "Timeout querying for flow director filter."); 723 return err; 724 } 725 726 /* if filter exists in hardware then remove it */ 727 if (fdircmd & TXGBE_FDIRPICMD_VLD) { 728 wr32(hw, TXGBE_FDIRPIHASH, fdirhash); 729 txgbe_flush(hw); 730 wr32(hw, TXGBE_FDIRPICMD, TXGBE_FDIRPICMD_OP_REM); 731 } 732 733 err = txgbe_fdir_check_cmd_complete(hw, &fdircmd); 734 if (err < 0) 735 PMD_INIT_LOG(ERR, "Timeout erasing flow director filter."); 736 737 return err; 738 } 739 740 static inline struct txgbe_fdir_filter * 741 txgbe_fdir_filter_lookup(struct txgbe_hw_fdir_info *fdir_info, 742 struct txgbe_atr_input *input) 743 { 744 int ret; 745 746 ret = rte_hash_lookup(fdir_info->hash_handle, (const void *)input); 747 if (ret < 0) 748 return NULL; 749 750 return fdir_info->hash_map[ret]; 751 } 752 753 static inline int 754 txgbe_insert_fdir_filter(struct txgbe_hw_fdir_info *fdir_info, 755 struct txgbe_fdir_filter *fdir_filter) 756 { 757 int ret; 758 759 ret = rte_hash_add_key(fdir_info->hash_handle, &fdir_filter->input); 760 if (ret < 0) { 761 PMD_DRV_LOG(ERR, 762 "Failed to insert fdir filter to hash table %d!", 763 ret); 764 return ret; 765 } 766 767 fdir_info->hash_map[ret] = fdir_filter; 768 769 TAILQ_INSERT_TAIL(&fdir_info->fdir_list, fdir_filter, entries); 770 771 return 0; 772 } 773 774 static inline int 775 txgbe_remove_fdir_filter(struct txgbe_hw_fdir_info *fdir_info, 776 struct txgbe_atr_input *input) 777 { 778 int ret; 779 struct txgbe_fdir_filter *fdir_filter; 780 781 ret = rte_hash_del_key(fdir_info->hash_handle, input); 782 if (ret < 0) 783 return ret; 784 785 fdir_filter = fdir_info->hash_map[ret]; 786 fdir_info->hash_map[ret] = NULL; 787 788 TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries); 789 rte_free(fdir_filter); 790 791 return 0; 792 } 793 794 int 795 txgbe_fdir_filter_program(struct rte_eth_dev *dev, 796 struct txgbe_fdir_rule *rule, 797 bool del, 798 bool update) 799 { 800 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 801 uint32_t fdirhash; 802 uint8_t queue; 803 bool is_perfect = FALSE; 804 int err; 805 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 806 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 807 struct txgbe_fdir_filter *node; 808 809 if (fdir_mode == RTE_FDIR_MODE_NONE || 810 fdir_mode != rule->mode) 811 return -ENOTSUP; 812 813 if (fdir_mode >= RTE_FDIR_MODE_PERFECT) 814 is_perfect = TRUE; 815 816 if (is_perfect) { 817 if (rule->input.flow_type & TXGBE_ATR_L3TYPE_IPV6) { 818 PMD_DRV_LOG(ERR, "IPv6 is not supported in" 819 " perfect mode!"); 820 return -ENOTSUP; 821 } 822 fdirhash = atr_compute_perfect_hash(&rule->input, 823 dev->data->dev_conf.fdir_conf.pballoc); 824 fdirhash |= TXGBE_FDIRPIHASH_IDX(rule->soft_id); 825 } else { 826 fdirhash = atr_compute_signature_hash(&rule->input, 827 dev->data->dev_conf.fdir_conf.pballoc); 828 } 829 830 if (del) { 831 err = txgbe_remove_fdir_filter(info, &rule->input); 832 if (err < 0) { 833 PMD_DRV_LOG(ERR, 834 "No such fdir filter to delete %d!", err); 835 return err; 836 } 837 838 err = fdir_erase_filter_raptor(hw, fdirhash); 839 if (err < 0) 840 PMD_DRV_LOG(ERR, "Fail to delete FDIR filter!"); 841 else 842 PMD_DRV_LOG(DEBUG, "Success to delete FDIR filter!"); 843 return err; 844 } 845 846 /* add or update an fdir filter*/ 847 if (rule->fdirflags & TXGBE_FDIRPICMD_DROP) { 848 if (!is_perfect) { 849 PMD_DRV_LOG(ERR, "Drop option is not supported in" 850 " signature mode."); 851 return -EINVAL; 852 } 853 queue = dev->data->dev_conf.fdir_conf.drop_queue; 854 } else if (rule->queue < TXGBE_MAX_RX_QUEUE_NUM) { 855 queue = rule->queue; 856 } else { 857 return -EINVAL; 858 } 859 860 node = txgbe_fdir_filter_lookup(info, &rule->input); 861 if (node) { 862 if (!update) { 863 PMD_DRV_LOG(ERR, "Conflict with existing fdir filter!"); 864 return -EINVAL; 865 } 866 node->fdirflags = rule->fdirflags; 867 node->fdirhash = fdirhash; 868 node->queue = queue; 869 } else { 870 node = rte_zmalloc("txgbe_fdir", 871 sizeof(struct txgbe_fdir_filter), 0); 872 if (!node) 873 return -ENOMEM; 874 rte_memcpy(&node->input, &rule->input, 875 sizeof(struct txgbe_atr_input)); 876 node->fdirflags = rule->fdirflags; 877 node->fdirhash = fdirhash; 878 node->queue = queue; 879 880 err = txgbe_insert_fdir_filter(info, node); 881 if (err < 0) { 882 rte_free(node); 883 return err; 884 } 885 } 886 887 if (is_perfect) 888 err = fdir_write_perfect_filter(hw, &node->input, 889 node->queue, node->fdirflags, 890 node->fdirhash, fdir_mode); 891 else 892 err = fdir_add_signature_filter(hw, &node->input, 893 node->queue, node->fdirflags, 894 node->fdirhash); 895 if (err < 0) { 896 PMD_DRV_LOG(ERR, "Fail to add FDIR filter!"); 897 txgbe_remove_fdir_filter(info, &rule->input); 898 } else { 899 PMD_DRV_LOG(DEBUG, "Success to add FDIR filter"); 900 } 901 902 return err; 903 } 904 905 static int 906 txgbe_fdir_flush(struct rte_eth_dev *dev) 907 { 908 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 909 struct txgbe_hw_fdir_info *info = TXGBE_DEV_FDIR(dev); 910 int ret; 911 912 ret = txgbe_reinit_fdir_tables(hw); 913 if (ret < 0) { 914 PMD_INIT_LOG(ERR, "Failed to re-initialize FD table."); 915 return ret; 916 } 917 918 info->f_add = 0; 919 info->f_remove = 0; 920 info->add = 0; 921 info->remove = 0; 922 923 return ret; 924 } 925 926 /* restore flow director filter */ 927 void 928 txgbe_fdir_filter_restore(struct rte_eth_dev *dev) 929 { 930 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 931 struct txgbe_hw_fdir_info *fdir_info = TXGBE_DEV_FDIR(dev); 932 struct txgbe_fdir_filter *node; 933 bool is_perfect = FALSE; 934 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 935 936 if (fdir_mode >= RTE_FDIR_MODE_PERFECT && 937 fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 938 is_perfect = TRUE; 939 940 if (is_perfect) { 941 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 942 (void)fdir_write_perfect_filter(hw, 943 &node->input, 944 node->queue, 945 node->fdirflags, 946 node->fdirhash, 947 fdir_mode); 948 } 949 } else { 950 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 951 (void)fdir_add_signature_filter(hw, 952 &node->input, 953 node->queue, 954 node->fdirflags, 955 node->fdirhash); 956 } 957 } 958 } 959 960 /* remove all the flow director filters */ 961 int 962 txgbe_clear_all_fdir_filter(struct rte_eth_dev *dev) 963 { 964 struct txgbe_hw_fdir_info *fdir_info = TXGBE_DEV_FDIR(dev); 965 struct txgbe_fdir_filter *fdir_filter; 966 struct txgbe_fdir_filter *filter_flag; 967 int ret = 0; 968 969 /* flush flow director */ 970 rte_hash_reset(fdir_info->hash_handle); 971 memset(fdir_info->hash_map, 0, 972 sizeof(struct txgbe_fdir_filter *) * TXGBE_MAX_FDIR_FILTER_NUM); 973 filter_flag = TAILQ_FIRST(&fdir_info->fdir_list); 974 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 975 TAILQ_REMOVE(&fdir_info->fdir_list, 976 fdir_filter, 977 entries); 978 rte_free(fdir_filter); 979 } 980 981 if (filter_flag != NULL) 982 ret = txgbe_fdir_flush(dev); 983 984 return ret; 985 } 986