1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 #include <stdarg.h> 8 #include <errno.h> 9 #include <sys/queue.h> 10 11 #include <rte_interrupts.h> 12 #include <rte_log.h> 13 #include <rte_debug.h> 14 #include <rte_pci.h> 15 #include <rte_vxlan.h> 16 #include <rte_ethdev_driver.h> 17 #include <rte_malloc.h> 18 19 #include "ixgbe_logs.h" 20 #include "base/ixgbe_api.h" 21 #include "base/ixgbe_common.h" 22 #include "ixgbe_ethdev.h" 23 24 /* To get PBALLOC (Packet Buffer Allocation) bits from FDIRCTRL value */ 25 #define FDIRCTRL_PBALLOC_MASK 0x03 26 27 /* For calculating memory required for FDIR filters */ 28 #define PBALLOC_SIZE_SHIFT 15 29 30 /* Number of bits used to mask bucket hash for different pballoc sizes */ 31 #define PERFECT_BUCKET_64KB_HASH_MASK 0x07FF /* 11 bits */ 32 #define PERFECT_BUCKET_128KB_HASH_MASK 0x0FFF /* 12 bits */ 33 #define PERFECT_BUCKET_256KB_HASH_MASK 0x1FFF /* 13 bits */ 34 #define SIG_BUCKET_64KB_HASH_MASK 0x1FFF /* 13 bits */ 35 #define SIG_BUCKET_128KB_HASH_MASK 0x3FFF /* 14 bits */ 36 #define SIG_BUCKET_256KB_HASH_MASK 0x7FFF /* 15 bits */ 37 #define IXGBE_DEFAULT_FLEXBYTES_OFFSET 12 /* default flexbytes offset in bytes */ 38 #define IXGBE_FDIR_MAX_FLEX_LEN 2 /* len in bytes of flexbytes */ 39 #define IXGBE_MAX_FLX_SOURCE_OFF 62 40 #define IXGBE_FDIRCTRL_FLEX_MASK (0x1F << IXGBE_FDIRCTRL_FLEX_SHIFT) 41 #define IXGBE_FDIRCMD_CMD_INTERVAL_US 10 42 43 #define IXGBE_FDIR_FLOW_TYPES ( \ 44 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \ 45 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \ 46 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \ 47 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \ 48 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \ 49 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \ 50 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \ 51 (1ULL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER)) 52 53 #define IPV6_ADDR_TO_MASK(ipaddr, ipv6m) do { \ 54 uint8_t ipv6_addr[16]; \ 55 uint8_t i; \ 56 rte_memcpy(ipv6_addr, (ipaddr), sizeof(ipv6_addr));\ 57 (ipv6m) = 0; \ 58 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 59 if (ipv6_addr[i] == UINT8_MAX) \ 60 (ipv6m) |= 1 << i; \ 61 else if (ipv6_addr[i] != 0) { \ 62 PMD_DRV_LOG(ERR, " invalid IPv6 address mask."); \ 63 return -EINVAL; \ 64 } \ 65 } \ 66 } while (0) 67 68 #define IPV6_MASK_TO_ADDR(ipv6m, ipaddr) do { \ 69 uint8_t ipv6_addr[16]; \ 70 uint8_t i; \ 71 for (i = 0; i < sizeof(ipv6_addr); i++) { \ 72 if ((ipv6m) & (1 << i)) \ 73 ipv6_addr[i] = UINT8_MAX; \ 74 else \ 75 ipv6_addr[i] = 0; \ 76 } \ 77 rte_memcpy((ipaddr), ipv6_addr, sizeof(ipv6_addr));\ 78 } while (0) 79 80 #define IXGBE_FDIRIP6M_INNER_MAC_SHIFT 4 81 82 static int fdir_erase_filter_82599(struct ixgbe_hw *hw, uint32_t fdirhash); 83 static int fdir_set_input_mask(struct rte_eth_dev *dev, 84 const struct rte_eth_fdir_masks *input_mask); 85 static int fdir_set_input_mask_82599(struct rte_eth_dev *dev); 86 static int fdir_set_input_mask_x550(struct rte_eth_dev *dev); 87 static int ixgbe_set_fdir_flex_conf(struct rte_eth_dev *dev, 88 const struct rte_eth_fdir_flex_conf *conf, uint32_t *fdirctrl); 89 static int fdir_enable_82599(struct ixgbe_hw *hw, uint32_t fdirctrl); 90 static uint32_t ixgbe_atr_compute_hash_82599(union ixgbe_atr_input *atr_input, 91 uint32_t key); 92 static uint32_t atr_compute_sig_hash_82599(union ixgbe_atr_input *input, 93 enum rte_fdir_pballoc_type pballoc); 94 static uint32_t atr_compute_perfect_hash_82599(union ixgbe_atr_input *input, 95 enum rte_fdir_pballoc_type pballoc); 96 static int fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, 97 union ixgbe_atr_input *input, uint8_t queue, 98 uint32_t fdircmd, uint32_t fdirhash, 99 enum rte_fdir_mode mode); 100 static int fdir_add_signature_filter_82599(struct ixgbe_hw *hw, 101 union ixgbe_atr_input *input, u8 queue, uint32_t fdircmd, 102 uint32_t fdirhash); 103 static int ixgbe_fdir_flush(struct rte_eth_dev *dev); 104 105 /** 106 * This function is based on ixgbe_fdir_enable_82599() in base/ixgbe_82599.c. 107 * It adds extra configuration of fdirctrl that is common for all filter types. 108 * 109 * Initialize Flow Director control registers 110 * @hw: pointer to hardware structure 111 * @fdirctrl: value to write to flow director control register 112 **/ 113 static int 114 fdir_enable_82599(struct ixgbe_hw *hw, uint32_t fdirctrl) 115 { 116 int i; 117 118 PMD_INIT_FUNC_TRACE(); 119 120 /* Prime the keys for hashing */ 121 IXGBE_WRITE_REG(hw, IXGBE_FDIRHKEY, IXGBE_ATR_BUCKET_HASH_KEY); 122 IXGBE_WRITE_REG(hw, IXGBE_FDIRSKEY, IXGBE_ATR_SIGNATURE_HASH_KEY); 123 124 /* 125 * Continue setup of fdirctrl register bits: 126 * Set the maximum length per hash bucket to 0xA filters 127 * Send interrupt when 64 filters are left 128 */ 129 fdirctrl |= (0xA << IXGBE_FDIRCTRL_MAX_LENGTH_SHIFT) | 130 (4 << IXGBE_FDIRCTRL_FULL_THRESH_SHIFT); 131 132 /* 133 * Poll init-done after we write the register. Estimated times: 134 * 10G: PBALLOC = 11b, timing is 60us 135 * 1G: PBALLOC = 11b, timing is 600us 136 * 100M: PBALLOC = 11b, timing is 6ms 137 * 138 * Multiple these timings by 4 if under full Rx load 139 * 140 * So we'll poll for IXGBE_FDIR_INIT_DONE_POLL times, sleeping for 141 * 1 msec per poll time. If we're at line rate and drop to 100M, then 142 * this might not finish in our poll time, but we can live with that 143 * for now. 144 */ 145 IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl); 146 IXGBE_WRITE_FLUSH(hw); 147 for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) { 148 if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) & 149 IXGBE_FDIRCTRL_INIT_DONE) 150 break; 151 msec_delay(1); 152 } 153 154 if (i >= IXGBE_FDIR_INIT_DONE_POLL) { 155 PMD_INIT_LOG(ERR, "Flow Director poll time exceeded during enabling!"); 156 return -ETIMEDOUT; 157 } 158 return 0; 159 } 160 161 /* 162 * Set appropriate bits in fdirctrl for: variable reporting levels, moving 163 * flexbytes matching field, and drop queue (only for perfect matching mode). 164 */ 165 static inline int 166 configure_fdir_flags(const struct rte_fdir_conf *conf, uint32_t *fdirctrl) 167 { 168 *fdirctrl = 0; 169 170 switch (conf->pballoc) { 171 case RTE_FDIR_PBALLOC_64K: 172 /* 8k - 1 signature filters */ 173 *fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_64K; 174 break; 175 case RTE_FDIR_PBALLOC_128K: 176 /* 16k - 1 signature filters */ 177 *fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_128K; 178 break; 179 case RTE_FDIR_PBALLOC_256K: 180 /* 32k - 1 signature filters */ 181 *fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_256K; 182 break; 183 default: 184 /* bad value */ 185 PMD_INIT_LOG(ERR, "Invalid fdir_conf->pballoc value"); 186 return -EINVAL; 187 }; 188 189 /* status flags: write hash & swindex in the rx descriptor */ 190 switch (conf->status) { 191 case RTE_FDIR_NO_REPORT_STATUS: 192 /* do nothing, default mode */ 193 break; 194 case RTE_FDIR_REPORT_STATUS: 195 /* report status when the packet matches a fdir rule */ 196 *fdirctrl |= IXGBE_FDIRCTRL_REPORT_STATUS; 197 break; 198 case RTE_FDIR_REPORT_STATUS_ALWAYS: 199 /* always report status */ 200 *fdirctrl |= IXGBE_FDIRCTRL_REPORT_STATUS_ALWAYS; 201 break; 202 default: 203 /* bad value */ 204 PMD_INIT_LOG(ERR, "Invalid fdir_conf->status value"); 205 return -EINVAL; 206 }; 207 208 *fdirctrl |= (IXGBE_DEFAULT_FLEXBYTES_OFFSET / sizeof(uint16_t)) << 209 IXGBE_FDIRCTRL_FLEX_SHIFT; 210 211 if (conf->mode >= RTE_FDIR_MODE_PERFECT && 212 conf->mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) { 213 *fdirctrl |= IXGBE_FDIRCTRL_PERFECT_MATCH; 214 *fdirctrl |= (conf->drop_queue << IXGBE_FDIRCTRL_DROP_Q_SHIFT); 215 if (conf->mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) 216 *fdirctrl |= (IXGBE_FDIRCTRL_FILTERMODE_MACVLAN 217 << IXGBE_FDIRCTRL_FILTERMODE_SHIFT); 218 else if (conf->mode == RTE_FDIR_MODE_PERFECT_TUNNEL) 219 *fdirctrl |= (IXGBE_FDIRCTRL_FILTERMODE_CLOUD 220 << IXGBE_FDIRCTRL_FILTERMODE_SHIFT); 221 } 222 223 return 0; 224 } 225 226 /** 227 * Reverse the bits in FDIR registers that store 2 x 16 bit masks. 228 * 229 * @hi_dword: Bits 31:16 mask to be bit swapped. 230 * @lo_dword: Bits 15:0 mask to be bit swapped. 231 * 232 * Flow director uses several registers to store 2 x 16 bit masks with the 233 * bits reversed such as FDIRTCPM, FDIRUDPM. The LS bit of the 234 * mask affects the MS bit/byte of the target. This function reverses the 235 * bits in these masks. 236 * **/ 237 static inline uint32_t 238 reverse_fdir_bitmasks(uint16_t hi_dword, uint16_t lo_dword) 239 { 240 uint32_t mask = hi_dword << 16; 241 242 mask |= lo_dword; 243 mask = ((mask & 0x55555555) << 1) | ((mask & 0xAAAAAAAA) >> 1); 244 mask = ((mask & 0x33333333) << 2) | ((mask & 0xCCCCCCCC) >> 2); 245 mask = ((mask & 0x0F0F0F0F) << 4) | ((mask & 0xF0F0F0F0) >> 4); 246 return ((mask & 0x00FF00FF) << 8) | ((mask & 0xFF00FF00) >> 8); 247 } 248 249 /* 250 * This references ixgbe_fdir_set_input_mask_82599() in base/ixgbe_82599.c, 251 * but makes use of the rte_fdir_masks structure to see which bits to set. 252 */ 253 static int 254 fdir_set_input_mask_82599(struct rte_eth_dev *dev) 255 { 256 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 257 struct ixgbe_hw_fdir_info *info = 258 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 259 /* 260 * mask VM pool and DIPv6 since there are currently not supported 261 * mask FLEX byte, it will be set in flex_conf 262 */ 263 uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6; 264 uint32_t fdirtcpm; /* TCP source and destination port masks. */ 265 uint32_t fdiripv6m; /* IPv6 source and destination masks. */ 266 volatile uint32_t *reg; 267 268 PMD_INIT_FUNC_TRACE(); 269 270 /* 271 * Program the relevant mask registers. If src/dst_port or src/dst_addr 272 * are zero, then assume a full mask for that field. Also assume that 273 * a VLAN of 0 is unspecified, so mask that out as well. L4type 274 * cannot be masked out in this implementation. 275 */ 276 if (info->mask.dst_port_mask == 0 && info->mask.src_port_mask == 0) 277 /* use the L4 protocol mask for raw IPv4/IPv6 traffic */ 278 fdirm |= IXGBE_FDIRM_L4P; 279 280 if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) 281 /* mask VLAN Priority */ 282 fdirm |= IXGBE_FDIRM_VLANP; 283 else if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0xE000)) 284 /* mask VLAN ID */ 285 fdirm |= IXGBE_FDIRM_VLANID; 286 else if (info->mask.vlan_tci_mask == 0) 287 /* mask VLAN ID and Priority */ 288 fdirm |= IXGBE_FDIRM_VLANID | IXGBE_FDIRM_VLANP; 289 else if (info->mask.vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { 290 PMD_INIT_LOG(ERR, "invalid vlan_tci_mask"); 291 return -EINVAL; 292 } 293 294 /* flex byte mask */ 295 if (info->mask.flex_bytes_mask == 0) 296 fdirm |= IXGBE_FDIRM_FLEX; 297 298 IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); 299 300 /* store the TCP/UDP port masks, bit reversed from port layout */ 301 fdirtcpm = reverse_fdir_bitmasks( 302 rte_be_to_cpu_16(info->mask.dst_port_mask), 303 rte_be_to_cpu_16(info->mask.src_port_mask)); 304 305 /* write all the same so that UDP, TCP and SCTP use the same mask 306 * (little-endian) 307 */ 308 IXGBE_WRITE_REG(hw, IXGBE_FDIRTCPM, ~fdirtcpm); 309 IXGBE_WRITE_REG(hw, IXGBE_FDIRUDPM, ~fdirtcpm); 310 IXGBE_WRITE_REG(hw, IXGBE_FDIRSCTPM, ~fdirtcpm); 311 312 /* Store source and destination IPv4 masks (big-endian), 313 * can not use IXGBE_WRITE_REG. 314 */ 315 reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRSIP4M); 316 *reg = ~(info->mask.src_ipv4_mask); 317 reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRDIP4M); 318 *reg = ~(info->mask.dst_ipv4_mask); 319 320 if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_SIGNATURE) { 321 /* 322 * Store source and destination IPv6 masks (bit reversed) 323 */ 324 fdiripv6m = (info->mask.dst_ipv6_mask << 16) | 325 info->mask.src_ipv6_mask; 326 327 IXGBE_WRITE_REG(hw, IXGBE_FDIRIP6M, ~fdiripv6m); 328 } 329 330 return IXGBE_SUCCESS; 331 } 332 333 /* 334 * This references ixgbe_fdir_set_input_mask_82599() in base/ixgbe_82599.c, 335 * but makes use of the rte_fdir_masks structure to see which bits to set. 336 */ 337 static int 338 fdir_set_input_mask_x550(struct rte_eth_dev *dev) 339 { 340 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 341 struct ixgbe_hw_fdir_info *info = 342 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 343 /* mask VM pool and DIPv6 since there are currently not supported 344 * mask FLEX byte, it will be set in flex_conf 345 */ 346 uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6 | 347 IXGBE_FDIRM_FLEX; 348 uint32_t fdiripv6m; 349 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 350 uint16_t mac_mask; 351 352 PMD_INIT_FUNC_TRACE(); 353 354 /* set the default UDP port for VxLAN */ 355 if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) 356 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, RTE_VXLAN_DEFAULT_PORT); 357 358 /* some bits must be set for mac vlan or tunnel mode */ 359 fdirm |= IXGBE_FDIRM_L4P | IXGBE_FDIRM_L3P; 360 361 if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) 362 /* mask VLAN Priority */ 363 fdirm |= IXGBE_FDIRM_VLANP; 364 else if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0xE000)) 365 /* mask VLAN ID */ 366 fdirm |= IXGBE_FDIRM_VLANID; 367 else if (info->mask.vlan_tci_mask == 0) 368 /* mask VLAN ID and Priority */ 369 fdirm |= IXGBE_FDIRM_VLANID | IXGBE_FDIRM_VLANP; 370 else if (info->mask.vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { 371 PMD_INIT_LOG(ERR, "invalid vlan_tci_mask"); 372 return -EINVAL; 373 } 374 375 IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); 376 377 fdiripv6m = ((u32)0xFFFFU << IXGBE_FDIRIP6M_DIPM_SHIFT); 378 fdiripv6m |= IXGBE_FDIRIP6M_ALWAYS_MASK; 379 if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) 380 fdiripv6m |= IXGBE_FDIRIP6M_TUNNEL_TYPE | 381 IXGBE_FDIRIP6M_TNI_VNI; 382 383 if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) { 384 fdiripv6m |= IXGBE_FDIRIP6M_INNER_MAC; 385 mac_mask = info->mask.mac_addr_byte_mask & 386 (IXGBE_FDIRIP6M_INNER_MAC >> 387 IXGBE_FDIRIP6M_INNER_MAC_SHIFT); 388 fdiripv6m &= ~((mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) & 389 IXGBE_FDIRIP6M_INNER_MAC); 390 391 switch (info->mask.tunnel_type_mask) { 392 case 0: 393 /* Mask turnnel type */ 394 fdiripv6m |= IXGBE_FDIRIP6M_TUNNEL_TYPE; 395 break; 396 case 1: 397 break; 398 default: 399 PMD_INIT_LOG(ERR, "invalid tunnel_type_mask"); 400 return -EINVAL; 401 } 402 403 switch (rte_be_to_cpu_32(info->mask.tunnel_id_mask)) { 404 case 0x0: 405 /* Mask vxlan id */ 406 fdiripv6m |= IXGBE_FDIRIP6M_TNI_VNI; 407 break; 408 case 0x00FFFFFF: 409 fdiripv6m |= IXGBE_FDIRIP6M_TNI_VNI_24; 410 break; 411 case 0xFFFFFFFF: 412 break; 413 default: 414 PMD_INIT_LOG(ERR, "invalid tunnel_id_mask"); 415 return -EINVAL; 416 } 417 } 418 419 IXGBE_WRITE_REG(hw, IXGBE_FDIRIP6M, fdiripv6m); 420 IXGBE_WRITE_REG(hw, IXGBE_FDIRTCPM, 0xFFFFFFFF); 421 IXGBE_WRITE_REG(hw, IXGBE_FDIRUDPM, 0xFFFFFFFF); 422 IXGBE_WRITE_REG(hw, IXGBE_FDIRSCTPM, 0xFFFFFFFF); 423 IXGBE_WRITE_REG(hw, IXGBE_FDIRDIP4M, 0xFFFFFFFF); 424 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIP4M, 0xFFFFFFFF); 425 426 return IXGBE_SUCCESS; 427 } 428 429 static int 430 ixgbe_fdir_store_input_mask_82599(struct rte_eth_dev *dev, 431 const struct rte_eth_fdir_masks *input_mask) 432 { 433 struct ixgbe_hw_fdir_info *info = 434 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 435 uint16_t dst_ipv6m = 0; 436 uint16_t src_ipv6m = 0; 437 438 memset(&info->mask, 0, sizeof(struct ixgbe_hw_fdir_mask)); 439 info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; 440 info->mask.src_port_mask = input_mask->src_port_mask; 441 info->mask.dst_port_mask = input_mask->dst_port_mask; 442 info->mask.src_ipv4_mask = input_mask->ipv4_mask.src_ip; 443 info->mask.dst_ipv4_mask = input_mask->ipv4_mask.dst_ip; 444 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.src_ip, src_ipv6m); 445 IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.dst_ip, dst_ipv6m); 446 info->mask.src_ipv6_mask = src_ipv6m; 447 info->mask.dst_ipv6_mask = dst_ipv6m; 448 449 return IXGBE_SUCCESS; 450 } 451 452 static int 453 ixgbe_fdir_store_input_mask_x550(struct rte_eth_dev *dev, 454 const struct rte_eth_fdir_masks *input_mask) 455 { 456 struct ixgbe_hw_fdir_info *info = 457 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 458 459 memset(&info->mask, 0, sizeof(struct ixgbe_hw_fdir_mask)); 460 info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; 461 info->mask.mac_addr_byte_mask = input_mask->mac_addr_byte_mask; 462 info->mask.tunnel_type_mask = input_mask->tunnel_type_mask; 463 info->mask.tunnel_id_mask = input_mask->tunnel_id_mask; 464 465 return IXGBE_SUCCESS; 466 } 467 468 static int 469 ixgbe_fdir_store_input_mask(struct rte_eth_dev *dev, 470 const struct rte_eth_fdir_masks *input_mask) 471 { 472 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 473 474 if (mode >= RTE_FDIR_MODE_SIGNATURE && 475 mode <= RTE_FDIR_MODE_PERFECT) 476 return ixgbe_fdir_store_input_mask_82599(dev, input_mask); 477 else if (mode >= RTE_FDIR_MODE_PERFECT_MAC_VLAN && 478 mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 479 return ixgbe_fdir_store_input_mask_x550(dev, input_mask); 480 481 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 482 return -ENOTSUP; 483 } 484 485 int 486 ixgbe_fdir_set_input_mask(struct rte_eth_dev *dev) 487 { 488 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 489 490 if (mode >= RTE_FDIR_MODE_SIGNATURE && 491 mode <= RTE_FDIR_MODE_PERFECT) 492 return fdir_set_input_mask_82599(dev); 493 else if (mode >= RTE_FDIR_MODE_PERFECT_MAC_VLAN && 494 mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 495 return fdir_set_input_mask_x550(dev); 496 497 PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); 498 return -ENOTSUP; 499 } 500 501 int 502 ixgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, 503 uint16_t offset) 504 { 505 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 506 uint32_t fdirctrl; 507 int i; 508 509 fdirctrl = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL); 510 511 fdirctrl &= ~IXGBE_FDIRCTRL_FLEX_MASK; 512 fdirctrl |= ((offset >> 1) /* convert to word offset */ 513 << IXGBE_FDIRCTRL_FLEX_SHIFT); 514 515 IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl); 516 IXGBE_WRITE_FLUSH(hw); 517 for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) { 518 if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) & 519 IXGBE_FDIRCTRL_INIT_DONE) 520 break; 521 msec_delay(1); 522 } 523 return 0; 524 } 525 526 static int 527 fdir_set_input_mask(struct rte_eth_dev *dev, 528 const struct rte_eth_fdir_masks *input_mask) 529 { 530 int ret; 531 532 ret = ixgbe_fdir_store_input_mask(dev, input_mask); 533 if (ret) 534 return ret; 535 536 return ixgbe_fdir_set_input_mask(dev); 537 } 538 539 /* 540 * ixgbe_check_fdir_flex_conf -check if the flex payload and mask configuration 541 * arguments are valid 542 */ 543 static int 544 ixgbe_set_fdir_flex_conf(struct rte_eth_dev *dev, 545 const struct rte_eth_fdir_flex_conf *conf, uint32_t *fdirctrl) 546 { 547 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 548 struct ixgbe_hw_fdir_info *info = 549 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 550 const struct rte_eth_flex_payload_cfg *flex_cfg; 551 const struct rte_eth_fdir_flex_mask *flex_mask; 552 uint32_t fdirm; 553 uint16_t flexbytes = 0; 554 uint16_t i; 555 556 fdirm = IXGBE_READ_REG(hw, IXGBE_FDIRM); 557 558 if (conf == NULL) { 559 PMD_DRV_LOG(ERR, "NULL pointer."); 560 return -EINVAL; 561 } 562 563 for (i = 0; i < conf->nb_payloads; i++) { 564 flex_cfg = &conf->flex_set[i]; 565 if (flex_cfg->type != RTE_ETH_RAW_PAYLOAD) { 566 PMD_DRV_LOG(ERR, "unsupported payload type."); 567 return -EINVAL; 568 } 569 if (((flex_cfg->src_offset[0] & 0x1) == 0) && 570 (flex_cfg->src_offset[1] == flex_cfg->src_offset[0] + 1) && 571 (flex_cfg->src_offset[0] <= IXGBE_MAX_FLX_SOURCE_OFF)) { 572 *fdirctrl &= ~IXGBE_FDIRCTRL_FLEX_MASK; 573 *fdirctrl |= 574 (flex_cfg->src_offset[0] / sizeof(uint16_t)) << 575 IXGBE_FDIRCTRL_FLEX_SHIFT; 576 } else { 577 PMD_DRV_LOG(ERR, "invalid flexbytes arguments."); 578 return -EINVAL; 579 } 580 } 581 582 for (i = 0; i < conf->nb_flexmasks; i++) { 583 flex_mask = &conf->flex_mask[i]; 584 if (flex_mask->flow_type != RTE_ETH_FLOW_UNKNOWN) { 585 PMD_DRV_LOG(ERR, "flexmask should be set globally."); 586 return -EINVAL; 587 } 588 flexbytes = (uint16_t)(((flex_mask->mask[0] << 8) & 0xFF00) | 589 ((flex_mask->mask[1]) & 0xFF)); 590 if (flexbytes == UINT16_MAX) 591 fdirm &= ~IXGBE_FDIRM_FLEX; 592 else if (flexbytes != 0) { 593 /* IXGBE_FDIRM_FLEX is set by default when set mask */ 594 PMD_DRV_LOG(ERR, " invalid flexbytes mask arguments."); 595 return -EINVAL; 596 } 597 } 598 IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); 599 info->mask.flex_bytes_mask = flexbytes ? UINT16_MAX : 0; 600 info->flex_bytes_offset = (uint8_t)((*fdirctrl & 601 IXGBE_FDIRCTRL_FLEX_MASK) >> 602 IXGBE_FDIRCTRL_FLEX_SHIFT); 603 return 0; 604 } 605 606 int 607 ixgbe_fdir_configure(struct rte_eth_dev *dev) 608 { 609 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 610 int err; 611 uint32_t fdirctrl, pbsize; 612 int i; 613 enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; 614 615 PMD_INIT_FUNC_TRACE(); 616 617 if (hw->mac.type != ixgbe_mac_82599EB && 618 hw->mac.type != ixgbe_mac_X540 && 619 hw->mac.type != ixgbe_mac_X550 && 620 hw->mac.type != ixgbe_mac_X550EM_x && 621 hw->mac.type != ixgbe_mac_X550EM_a) 622 return -ENOSYS; 623 624 /* x550 supports mac-vlan and tunnel mode but other NICs not */ 625 if (hw->mac.type != ixgbe_mac_X550 && 626 hw->mac.type != ixgbe_mac_X550EM_x && 627 hw->mac.type != ixgbe_mac_X550EM_a && 628 mode != RTE_FDIR_MODE_SIGNATURE && 629 mode != RTE_FDIR_MODE_PERFECT) 630 return -ENOSYS; 631 632 err = configure_fdir_flags(&dev->data->dev_conf.fdir_conf, &fdirctrl); 633 if (err) 634 return err; 635 636 /* 637 * Before enabling Flow Director, the Rx Packet Buffer size 638 * must be reduced. The new value is the current size minus 639 * flow director memory usage size. 640 */ 641 pbsize = (1 << (PBALLOC_SIZE_SHIFT + (fdirctrl & FDIRCTRL_PBALLOC_MASK))); 642 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(0), 643 (IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) - pbsize)); 644 645 /* 646 * The defaults in the HW for RX PB 1-7 are not zero and so should be 647 * initialized to zero for non DCB mode otherwise actual total RX PB 648 * would be bigger than programmed and filter space would run into 649 * the PB 0 region. 650 */ 651 for (i = 1; i < 8; i++) 652 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0); 653 654 err = fdir_set_input_mask(dev, &dev->data->dev_conf.fdir_conf.mask); 655 if (err < 0) { 656 PMD_INIT_LOG(ERR, " Error on setting FD mask"); 657 return err; 658 } 659 err = ixgbe_set_fdir_flex_conf(dev, 660 &dev->data->dev_conf.fdir_conf.flex_conf, &fdirctrl); 661 if (err < 0) { 662 PMD_INIT_LOG(ERR, " Error on setting FD flexible arguments."); 663 return err; 664 } 665 666 err = fdir_enable_82599(hw, fdirctrl); 667 if (err < 0) { 668 PMD_INIT_LOG(ERR, " Error on enabling FD."); 669 return err; 670 } 671 return 0; 672 } 673 674 /* 675 * The below function is taken from the FreeBSD IXGBE drivers release 676 * 2.3.8. The only change is not to mask hash_result with IXGBE_ATR_HASH_MASK 677 * before returning, as the signature hash can use 16bits. 678 * 679 * The newer driver has optimised functions for calculating bucket and 680 * signature hashes. However they don't support IPv6 type packets for signature 681 * filters so are not used here. 682 * 683 * Note that the bkt_hash field in the ixgbe_atr_input structure is also never 684 * set. 685 * 686 * Compute the hashes for SW ATR 687 * @stream: input bitstream to compute the hash on 688 * @key: 32-bit hash key 689 **/ 690 static uint32_t 691 ixgbe_atr_compute_hash_82599(union ixgbe_atr_input *atr_input, 692 uint32_t key) 693 { 694 /* 695 * The algorithm is as follows: 696 * Hash[15:0] = Sum { S[n] x K[n+16] }, n = 0...350 697 * where Sum {A[n]}, n = 0...n is bitwise XOR of A[0], A[1]...A[n] 698 * and A[n] x B[n] is bitwise AND between same length strings 699 * 700 * K[n] is 16 bits, defined as: 701 * for n modulo 32 >= 15, K[n] = K[n % 32 : (n % 32) - 15] 702 * for n modulo 32 < 15, K[n] = 703 * K[(n % 32:0) | (31:31 - (14 - (n % 32)))] 704 * 705 * S[n] is 16 bits, defined as: 706 * for n >= 15, S[n] = S[n:n - 15] 707 * for n < 15, S[n] = S[(n:0) | (350:350 - (14 - n))] 708 * 709 * To simplify for programming, the algorithm is implemented 710 * in software this way: 711 * 712 * key[31:0], hi_hash_dword[31:0], lo_hash_dword[31:0], hash[15:0] 713 * 714 * for (i = 0; i < 352; i+=32) 715 * hi_hash_dword[31:0] ^= Stream[(i+31):i]; 716 * 717 * lo_hash_dword[15:0] ^= Stream[15:0]; 718 * lo_hash_dword[15:0] ^= hi_hash_dword[31:16]; 719 * lo_hash_dword[31:16] ^= hi_hash_dword[15:0]; 720 * 721 * hi_hash_dword[31:0] ^= Stream[351:320]; 722 * 723 * if (key[0]) 724 * hash[15:0] ^= Stream[15:0]; 725 * 726 * for (i = 0; i < 16; i++) { 727 * if (key[i]) 728 * hash[15:0] ^= lo_hash_dword[(i+15):i]; 729 * if (key[i + 16]) 730 * hash[15:0] ^= hi_hash_dword[(i+15):i]; 731 * } 732 * 733 */ 734 __be32 common_hash_dword = 0; 735 u32 hi_hash_dword, lo_hash_dword, flow_vm_vlan; 736 u32 hash_result = 0; 737 u8 i; 738 739 /* record the flow_vm_vlan bits as they are a key part to the hash */ 740 flow_vm_vlan = IXGBE_NTOHL(atr_input->dword_stream[0]); 741 742 /* generate common hash dword */ 743 for (i = 1; i <= 13; i++) 744 common_hash_dword ^= atr_input->dword_stream[i]; 745 746 hi_hash_dword = IXGBE_NTOHL(common_hash_dword); 747 748 /* low dword is word swapped version of common */ 749 lo_hash_dword = (hi_hash_dword >> 16) | (hi_hash_dword << 16); 750 751 /* apply flow ID/VM pool/VLAN ID bits to hash words */ 752 hi_hash_dword ^= flow_vm_vlan ^ (flow_vm_vlan >> 16); 753 754 /* Process bits 0 and 16 */ 755 if (key & 0x0001) 756 hash_result ^= lo_hash_dword; 757 if (key & 0x00010000) 758 hash_result ^= hi_hash_dword; 759 760 /* 761 * apply flow ID/VM pool/VLAN ID bits to lo hash dword, we had to 762 * delay this because bit 0 of the stream should not be processed 763 * so we do not add the vlan until after bit 0 was processed 764 */ 765 lo_hash_dword ^= flow_vm_vlan ^ (flow_vm_vlan << 16); 766 767 768 /* process the remaining 30 bits in the key 2 bits at a time */ 769 for (i = 15; i; i--) { 770 if (key & (0x0001 << i)) 771 hash_result ^= lo_hash_dword >> i; 772 if (key & (0x00010000 << i)) 773 hash_result ^= hi_hash_dword >> i; 774 } 775 776 return hash_result; 777 } 778 779 static uint32_t 780 atr_compute_perfect_hash_82599(union ixgbe_atr_input *input, 781 enum rte_fdir_pballoc_type pballoc) 782 { 783 if (pballoc == RTE_FDIR_PBALLOC_256K) 784 return ixgbe_atr_compute_hash_82599(input, 785 IXGBE_ATR_BUCKET_HASH_KEY) & 786 PERFECT_BUCKET_256KB_HASH_MASK; 787 else if (pballoc == RTE_FDIR_PBALLOC_128K) 788 return ixgbe_atr_compute_hash_82599(input, 789 IXGBE_ATR_BUCKET_HASH_KEY) & 790 PERFECT_BUCKET_128KB_HASH_MASK; 791 else 792 return ixgbe_atr_compute_hash_82599(input, 793 IXGBE_ATR_BUCKET_HASH_KEY) & 794 PERFECT_BUCKET_64KB_HASH_MASK; 795 } 796 797 /** 798 * ixgbe_fdir_check_cmd_complete - poll to check whether FDIRCMD is complete 799 * @hw: pointer to hardware structure 800 */ 801 static inline int 802 ixgbe_fdir_check_cmd_complete(struct ixgbe_hw *hw, uint32_t *fdircmd) 803 { 804 int i; 805 806 for (i = 0; i < IXGBE_FDIRCMD_CMD_POLL; i++) { 807 *fdircmd = IXGBE_READ_REG(hw, IXGBE_FDIRCMD); 808 if (!(*fdircmd & IXGBE_FDIRCMD_CMD_MASK)) 809 return 0; 810 rte_delay_us(IXGBE_FDIRCMD_CMD_INTERVAL_US); 811 } 812 813 return -ETIMEDOUT; 814 } 815 816 /* 817 * Calculate the hash value needed for signature-match filters. In the FreeBSD 818 * driver, this is done by the optimised function 819 * ixgbe_atr_compute_sig_hash_82599(). However that can't be used here as it 820 * doesn't support calculating a hash for an IPv6 filter. 821 */ 822 static uint32_t 823 atr_compute_sig_hash_82599(union ixgbe_atr_input *input, 824 enum rte_fdir_pballoc_type pballoc) 825 { 826 uint32_t bucket_hash, sig_hash; 827 828 if (pballoc == RTE_FDIR_PBALLOC_256K) 829 bucket_hash = ixgbe_atr_compute_hash_82599(input, 830 IXGBE_ATR_BUCKET_HASH_KEY) & 831 SIG_BUCKET_256KB_HASH_MASK; 832 else if (pballoc == RTE_FDIR_PBALLOC_128K) 833 bucket_hash = ixgbe_atr_compute_hash_82599(input, 834 IXGBE_ATR_BUCKET_HASH_KEY) & 835 SIG_BUCKET_128KB_HASH_MASK; 836 else 837 bucket_hash = ixgbe_atr_compute_hash_82599(input, 838 IXGBE_ATR_BUCKET_HASH_KEY) & 839 SIG_BUCKET_64KB_HASH_MASK; 840 841 sig_hash = ixgbe_atr_compute_hash_82599(input, 842 IXGBE_ATR_SIGNATURE_HASH_KEY); 843 844 return (sig_hash << IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT) | bucket_hash; 845 } 846 847 /* 848 * This is based on ixgbe_fdir_write_perfect_filter_82599() in 849 * base/ixgbe_82599.c, with the ability to set extra flags in FDIRCMD register 850 * added, and IPv6 support also added. The hash value is also pre-calculated 851 * as the pballoc value is needed to do it. 852 */ 853 static int 854 fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, 855 union ixgbe_atr_input *input, uint8_t queue, 856 uint32_t fdircmd, uint32_t fdirhash, 857 enum rte_fdir_mode mode) 858 { 859 uint32_t fdirport, fdirvlan; 860 u32 addr_low, addr_high; 861 u32 tunnel_type = 0; 862 int err = 0; 863 volatile uint32_t *reg; 864 865 if (mode == RTE_FDIR_MODE_PERFECT) { 866 /* record the IPv4 address (big-endian) 867 * can not use IXGBE_WRITE_REG. 868 */ 869 reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRIPSA); 870 *reg = input->formatted.src_ip[0]; 871 reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRIPDA); 872 *reg = input->formatted.dst_ip[0]; 873 874 /* record source and destination port (little-endian)*/ 875 fdirport = IXGBE_NTOHS(input->formatted.dst_port); 876 fdirport <<= IXGBE_FDIRPORT_DESTINATION_SHIFT; 877 fdirport |= IXGBE_NTOHS(input->formatted.src_port); 878 IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, fdirport); 879 } else if (mode >= RTE_FDIR_MODE_PERFECT_MAC_VLAN && 880 mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) { 881 /* for mac vlan and tunnel modes */ 882 addr_low = ((u32)input->formatted.inner_mac[0] | 883 ((u32)input->formatted.inner_mac[1] << 8) | 884 ((u32)input->formatted.inner_mac[2] << 16) | 885 ((u32)input->formatted.inner_mac[3] << 24)); 886 addr_high = ((u32)input->formatted.inner_mac[4] | 887 ((u32)input->formatted.inner_mac[5] << 8)); 888 889 if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { 890 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(0), addr_low); 891 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(1), addr_high); 892 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), 0); 893 } else { 894 /* tunnel mode */ 895 if (input->formatted.tunnel_type) 896 tunnel_type = 0x80000000; 897 tunnel_type |= addr_high; 898 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(0), addr_low); 899 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(1), tunnel_type); 900 IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), 901 input->formatted.tni_vni); 902 } 903 IXGBE_WRITE_REG(hw, IXGBE_FDIRIPSA, 0); 904 IXGBE_WRITE_REG(hw, IXGBE_FDIRIPDA, 0); 905 IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, 0); 906 } 907 908 /* record vlan (little-endian) and flex_bytes(big-endian) */ 909 fdirvlan = input->formatted.flex_bytes; 910 fdirvlan <<= IXGBE_FDIRVLAN_FLEX_SHIFT; 911 fdirvlan |= IXGBE_NTOHS(input->formatted.vlan_id); 912 IXGBE_WRITE_REG(hw, IXGBE_FDIRVLAN, fdirvlan); 913 914 /* configure FDIRHASH register */ 915 IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash); 916 917 /* 918 * flush all previous writes to make certain registers are 919 * programmed prior to issuing the command 920 */ 921 IXGBE_WRITE_FLUSH(hw); 922 923 /* configure FDIRCMD register */ 924 fdircmd |= IXGBE_FDIRCMD_CMD_ADD_FLOW | 925 IXGBE_FDIRCMD_LAST | IXGBE_FDIRCMD_QUEUE_EN; 926 fdircmd |= input->formatted.flow_type << IXGBE_FDIRCMD_FLOW_TYPE_SHIFT; 927 fdircmd |= (uint32_t)queue << IXGBE_FDIRCMD_RX_QUEUE_SHIFT; 928 fdircmd |= (uint32_t)input->formatted.vm_pool << IXGBE_FDIRCMD_VT_POOL_SHIFT; 929 930 IXGBE_WRITE_REG(hw, IXGBE_FDIRCMD, fdircmd); 931 932 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 933 934 err = ixgbe_fdir_check_cmd_complete(hw, &fdircmd); 935 if (err < 0) 936 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 937 938 return err; 939 } 940 941 /** 942 * This function is based on ixgbe_atr_add_signature_filter_82599() in 943 * base/ixgbe_82599.c, but uses a pre-calculated hash value. It also supports 944 * setting extra fields in the FDIRCMD register, and removes the code that was 945 * verifying the flow_type field. According to the documentation, a flow type of 946 * 00 (i.e. not TCP, UDP, or SCTP) is not supported, however it appears to 947 * work ok... 948 * 949 * Adds a signature hash filter 950 * @hw: pointer to hardware structure 951 * @input: unique input dword 952 * @queue: queue index to direct traffic to 953 * @fdircmd: any extra flags to set in fdircmd register 954 * @fdirhash: pre-calculated hash value for the filter 955 **/ 956 static int 957 fdir_add_signature_filter_82599(struct ixgbe_hw *hw, 958 union ixgbe_atr_input *input, u8 queue, uint32_t fdircmd, 959 uint32_t fdirhash) 960 { 961 int err = 0; 962 963 PMD_INIT_FUNC_TRACE(); 964 965 /* configure FDIRCMD register */ 966 fdircmd |= IXGBE_FDIRCMD_CMD_ADD_FLOW | 967 IXGBE_FDIRCMD_LAST | IXGBE_FDIRCMD_QUEUE_EN; 968 fdircmd |= input->formatted.flow_type << IXGBE_FDIRCMD_FLOW_TYPE_SHIFT; 969 fdircmd |= (uint32_t)queue << IXGBE_FDIRCMD_RX_QUEUE_SHIFT; 970 971 IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash); 972 IXGBE_WRITE_REG(hw, IXGBE_FDIRCMD, fdircmd); 973 974 PMD_DRV_LOG(DEBUG, "Rx Queue=%x hash=%x", queue, fdirhash); 975 976 err = ixgbe_fdir_check_cmd_complete(hw, &fdircmd); 977 if (err < 0) 978 PMD_DRV_LOG(ERR, "Timeout writing flow director filter."); 979 980 return err; 981 } 982 983 /* 984 * This is based on ixgbe_fdir_erase_perfect_filter_82599() in 985 * base/ixgbe_82599.c. It is modified to take in the hash as a parameter so 986 * that it can be used for removing signature and perfect filters. 987 */ 988 static int 989 fdir_erase_filter_82599(struct ixgbe_hw *hw, uint32_t fdirhash) 990 { 991 uint32_t fdircmd = 0; 992 int err = 0; 993 994 IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash); 995 996 /* flush hash to HW */ 997 IXGBE_WRITE_FLUSH(hw); 998 999 /* Query if filter is present */ 1000 IXGBE_WRITE_REG(hw, IXGBE_FDIRCMD, IXGBE_FDIRCMD_CMD_QUERY_REM_FILT); 1001 1002 err = ixgbe_fdir_check_cmd_complete(hw, &fdircmd); 1003 if (err < 0) { 1004 PMD_INIT_LOG(ERR, "Timeout querying for flow director filter."); 1005 return err; 1006 } 1007 1008 /* if filter exists in hardware then remove it */ 1009 if (fdircmd & IXGBE_FDIRCMD_FILTER_VALID) { 1010 IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash); 1011 IXGBE_WRITE_FLUSH(hw); 1012 IXGBE_WRITE_REG(hw, IXGBE_FDIRCMD, 1013 IXGBE_FDIRCMD_CMD_REMOVE_FLOW); 1014 } 1015 err = ixgbe_fdir_check_cmd_complete(hw, &fdircmd); 1016 if (err < 0) 1017 PMD_INIT_LOG(ERR, "Timeout erasing flow director filter."); 1018 return err; 1019 1020 } 1021 1022 static inline struct ixgbe_fdir_filter * 1023 ixgbe_fdir_filter_lookup(struct ixgbe_hw_fdir_info *fdir_info, 1024 union ixgbe_atr_input *key) 1025 { 1026 int ret; 1027 1028 ret = rte_hash_lookup(fdir_info->hash_handle, (const void *)key); 1029 if (ret < 0) 1030 return NULL; 1031 1032 return fdir_info->hash_map[ret]; 1033 } 1034 1035 static inline int 1036 ixgbe_insert_fdir_filter(struct ixgbe_hw_fdir_info *fdir_info, 1037 struct ixgbe_fdir_filter *fdir_filter) 1038 { 1039 int ret; 1040 1041 ret = rte_hash_add_key(fdir_info->hash_handle, 1042 &fdir_filter->ixgbe_fdir); 1043 1044 if (ret < 0) { 1045 PMD_DRV_LOG(ERR, 1046 "Failed to insert fdir filter to hash table %d!", 1047 ret); 1048 return ret; 1049 } 1050 1051 fdir_info->hash_map[ret] = fdir_filter; 1052 1053 TAILQ_INSERT_TAIL(&fdir_info->fdir_list, fdir_filter, entries); 1054 1055 return 0; 1056 } 1057 1058 static inline int 1059 ixgbe_remove_fdir_filter(struct ixgbe_hw_fdir_info *fdir_info, 1060 union ixgbe_atr_input *key) 1061 { 1062 int ret; 1063 struct ixgbe_fdir_filter *fdir_filter; 1064 1065 ret = rte_hash_del_key(fdir_info->hash_handle, key); 1066 1067 if (ret < 0) { 1068 PMD_DRV_LOG(ERR, "No such fdir filter to delete %d!", ret); 1069 return ret; 1070 } 1071 1072 fdir_filter = fdir_info->hash_map[ret]; 1073 fdir_info->hash_map[ret] = NULL; 1074 1075 TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries); 1076 rte_free(fdir_filter); 1077 1078 return 0; 1079 } 1080 1081 int 1082 ixgbe_fdir_filter_program(struct rte_eth_dev *dev, 1083 struct ixgbe_fdir_rule *rule, 1084 bool del, 1085 bool update) 1086 { 1087 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1088 uint32_t fdircmd_flags; 1089 uint32_t fdirhash; 1090 uint8_t queue; 1091 bool is_perfect = FALSE; 1092 int err; 1093 struct ixgbe_hw_fdir_info *info = 1094 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1095 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 1096 struct ixgbe_fdir_filter *node; 1097 bool add_node = FALSE; 1098 1099 if (fdir_mode == RTE_FDIR_MODE_NONE || 1100 fdir_mode != rule->mode) 1101 return -ENOTSUP; 1102 1103 /* 1104 * Sanity check for x550. 1105 * When adding a new filter with flow type set to IPv4, 1106 * the flow director mask should be configed before, 1107 * and the L4 protocol and ports are masked. 1108 */ 1109 if ((!del) && 1110 (hw->mac.type == ixgbe_mac_X550 || 1111 hw->mac.type == ixgbe_mac_X550EM_x || 1112 hw->mac.type == ixgbe_mac_X550EM_a) && 1113 (rule->ixgbe_fdir.formatted.flow_type == 1114 IXGBE_ATR_FLOW_TYPE_IPV4 || 1115 rule->ixgbe_fdir.formatted.flow_type == 1116 IXGBE_ATR_FLOW_TYPE_IPV6) && 1117 (info->mask.src_port_mask != 0 || 1118 info->mask.dst_port_mask != 0) && 1119 (rule->mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN && 1120 rule->mode != RTE_FDIR_MODE_PERFECT_TUNNEL)) { 1121 PMD_DRV_LOG(ERR, "By this device," 1122 " IPv4 is not supported without" 1123 " L4 protocol and ports masked!"); 1124 return -ENOTSUP; 1125 } 1126 1127 if (fdir_mode >= RTE_FDIR_MODE_PERFECT && 1128 fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 1129 is_perfect = TRUE; 1130 1131 if (is_perfect) { 1132 if (rule->ixgbe_fdir.formatted.flow_type & 1133 IXGBE_ATR_L4TYPE_IPV6_MASK) { 1134 PMD_DRV_LOG(ERR, "IPv6 is not supported in" 1135 " perfect mode!"); 1136 return -ENOTSUP; 1137 } 1138 fdirhash = atr_compute_perfect_hash_82599(&rule->ixgbe_fdir, 1139 dev->data->dev_conf.fdir_conf.pballoc); 1140 fdirhash |= rule->soft_id << 1141 IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT; 1142 } else 1143 fdirhash = atr_compute_sig_hash_82599(&rule->ixgbe_fdir, 1144 dev->data->dev_conf.fdir_conf.pballoc); 1145 1146 if (del) { 1147 err = ixgbe_remove_fdir_filter(info, &rule->ixgbe_fdir); 1148 if (err < 0) 1149 return err; 1150 1151 err = fdir_erase_filter_82599(hw, fdirhash); 1152 if (err < 0) 1153 PMD_DRV_LOG(ERR, "Fail to delete FDIR filter!"); 1154 else 1155 PMD_DRV_LOG(DEBUG, "Success to delete FDIR filter!"); 1156 return err; 1157 } 1158 /* add or update an fdir filter*/ 1159 fdircmd_flags = (update) ? IXGBE_FDIRCMD_FILTER_UPDATE : 0; 1160 if (rule->fdirflags & IXGBE_FDIRCMD_DROP) { 1161 if (is_perfect) { 1162 queue = dev->data->dev_conf.fdir_conf.drop_queue; 1163 fdircmd_flags |= IXGBE_FDIRCMD_DROP; 1164 } else { 1165 PMD_DRV_LOG(ERR, "Drop option is not supported in" 1166 " signature mode."); 1167 return -EINVAL; 1168 } 1169 } else if (rule->queue < IXGBE_MAX_RX_QUEUE_NUM) 1170 queue = (uint8_t)rule->queue; 1171 else 1172 return -EINVAL; 1173 1174 node = ixgbe_fdir_filter_lookup(info, &rule->ixgbe_fdir); 1175 if (node) { 1176 if (update) { 1177 node->fdirflags = fdircmd_flags; 1178 node->fdirhash = fdirhash; 1179 node->queue = queue; 1180 } else { 1181 PMD_DRV_LOG(ERR, "Conflict with existing fdir filter!"); 1182 return -EINVAL; 1183 } 1184 } else { 1185 add_node = TRUE; 1186 node = rte_zmalloc("ixgbe_fdir", 1187 sizeof(struct ixgbe_fdir_filter), 1188 0); 1189 if (!node) 1190 return -ENOMEM; 1191 rte_memcpy(&node->ixgbe_fdir, 1192 &rule->ixgbe_fdir, 1193 sizeof(union ixgbe_atr_input)); 1194 node->fdirflags = fdircmd_flags; 1195 node->fdirhash = fdirhash; 1196 node->queue = queue; 1197 1198 err = ixgbe_insert_fdir_filter(info, node); 1199 if (err < 0) { 1200 rte_free(node); 1201 return err; 1202 } 1203 } 1204 1205 if (is_perfect) { 1206 err = fdir_write_perfect_filter_82599(hw, &rule->ixgbe_fdir, 1207 queue, fdircmd_flags, 1208 fdirhash, fdir_mode); 1209 } else { 1210 err = fdir_add_signature_filter_82599(hw, &rule->ixgbe_fdir, 1211 queue, fdircmd_flags, 1212 fdirhash); 1213 } 1214 if (err < 0) { 1215 PMD_DRV_LOG(ERR, "Fail to add FDIR filter!"); 1216 1217 if (add_node) 1218 (void)ixgbe_remove_fdir_filter(info, &rule->ixgbe_fdir); 1219 } else { 1220 PMD_DRV_LOG(DEBUG, "Success to add FDIR filter"); 1221 } 1222 1223 return err; 1224 } 1225 1226 static int 1227 ixgbe_fdir_flush(struct rte_eth_dev *dev) 1228 { 1229 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1230 struct ixgbe_hw_fdir_info *info = 1231 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1232 int ret; 1233 1234 ret = ixgbe_reinit_fdir_tables_82599(hw); 1235 if (ret < 0) { 1236 PMD_INIT_LOG(ERR, "Failed to re-initialize FD table."); 1237 return ret; 1238 } 1239 1240 info->f_add = 0; 1241 info->f_remove = 0; 1242 info->add = 0; 1243 info->remove = 0; 1244 1245 return ret; 1246 } 1247 1248 #define FDIRENTRIES_NUM_SHIFT 10 1249 void 1250 ixgbe_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) 1251 { 1252 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1253 struct ixgbe_hw_fdir_info *info = 1254 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1255 uint32_t fdirctrl, max_num, i; 1256 uint8_t offset; 1257 1258 fdirctrl = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL); 1259 offset = ((fdirctrl & IXGBE_FDIRCTRL_FLEX_MASK) >> 1260 IXGBE_FDIRCTRL_FLEX_SHIFT) * sizeof(uint16_t); 1261 1262 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; 1263 max_num = (1 << (FDIRENTRIES_NUM_SHIFT + 1264 (fdirctrl & FDIRCTRL_PBALLOC_MASK))); 1265 if (fdir_info->mode >= RTE_FDIR_MODE_PERFECT && 1266 fdir_info->mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 1267 fdir_info->guarant_spc = max_num; 1268 else if (fdir_info->mode == RTE_FDIR_MODE_SIGNATURE) 1269 fdir_info->guarant_spc = max_num * 4; 1270 1271 fdir_info->mask.vlan_tci_mask = info->mask.vlan_tci_mask; 1272 fdir_info->mask.ipv4_mask.src_ip = info->mask.src_ipv4_mask; 1273 fdir_info->mask.ipv4_mask.dst_ip = info->mask.dst_ipv4_mask; 1274 IPV6_MASK_TO_ADDR(info->mask.src_ipv6_mask, 1275 fdir_info->mask.ipv6_mask.src_ip); 1276 IPV6_MASK_TO_ADDR(info->mask.dst_ipv6_mask, 1277 fdir_info->mask.ipv6_mask.dst_ip); 1278 fdir_info->mask.src_port_mask = info->mask.src_port_mask; 1279 fdir_info->mask.dst_port_mask = info->mask.dst_port_mask; 1280 fdir_info->mask.mac_addr_byte_mask = info->mask.mac_addr_byte_mask; 1281 fdir_info->mask.tunnel_id_mask = info->mask.tunnel_id_mask; 1282 fdir_info->mask.tunnel_type_mask = info->mask.tunnel_type_mask; 1283 fdir_info->max_flexpayload = IXGBE_FDIR_MAX_FLEX_LEN; 1284 1285 if (fdir_info->mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN || 1286 fdir_info->mode == RTE_FDIR_MODE_PERFECT_TUNNEL) 1287 fdir_info->flow_types_mask[0] = 0ULL; 1288 else 1289 fdir_info->flow_types_mask[0] = IXGBE_FDIR_FLOW_TYPES; 1290 for (i = 1; i < RTE_FLOW_MASK_ARRAY_SIZE; i++) 1291 fdir_info->flow_types_mask[i] = 0ULL; 1292 1293 fdir_info->flex_payload_unit = sizeof(uint16_t); 1294 fdir_info->max_flex_payload_segment_num = 1; 1295 fdir_info->flex_payload_limit = IXGBE_MAX_FLX_SOURCE_OFF; 1296 fdir_info->flex_conf.nb_payloads = 1; 1297 fdir_info->flex_conf.flex_set[0].type = RTE_ETH_RAW_PAYLOAD; 1298 fdir_info->flex_conf.flex_set[0].src_offset[0] = offset; 1299 fdir_info->flex_conf.flex_set[0].src_offset[1] = offset + 1; 1300 fdir_info->flex_conf.nb_flexmasks = 1; 1301 fdir_info->flex_conf.flex_mask[0].flow_type = RTE_ETH_FLOW_UNKNOWN; 1302 fdir_info->flex_conf.flex_mask[0].mask[0] = 1303 (uint8_t)(info->mask.flex_bytes_mask & 0x00FF); 1304 fdir_info->flex_conf.flex_mask[0].mask[1] = 1305 (uint8_t)((info->mask.flex_bytes_mask & 0xFF00) >> 8); 1306 } 1307 1308 void 1309 ixgbe_fdir_stats_get(struct rte_eth_dev *dev, struct rte_eth_fdir_stats *fdir_stats) 1310 { 1311 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1312 struct ixgbe_hw_fdir_info *info = 1313 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1314 uint32_t reg, max_num; 1315 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 1316 1317 /* Get the information from registers */ 1318 reg = IXGBE_READ_REG(hw, IXGBE_FDIRFREE); 1319 info->collision = (uint16_t)((reg & IXGBE_FDIRFREE_COLL_MASK) >> 1320 IXGBE_FDIRFREE_COLL_SHIFT); 1321 info->free = (uint16_t)((reg & IXGBE_FDIRFREE_FREE_MASK) >> 1322 IXGBE_FDIRFREE_FREE_SHIFT); 1323 1324 reg = IXGBE_READ_REG(hw, IXGBE_FDIRLEN); 1325 info->maxhash = (uint16_t)((reg & IXGBE_FDIRLEN_MAXHASH_MASK) >> 1326 IXGBE_FDIRLEN_MAXHASH_SHIFT); 1327 info->maxlen = (uint8_t)((reg & IXGBE_FDIRLEN_MAXLEN_MASK) >> 1328 IXGBE_FDIRLEN_MAXLEN_SHIFT); 1329 1330 reg = IXGBE_READ_REG(hw, IXGBE_FDIRUSTAT); 1331 info->remove += (reg & IXGBE_FDIRUSTAT_REMOVE_MASK) >> 1332 IXGBE_FDIRUSTAT_REMOVE_SHIFT; 1333 info->add += (reg & IXGBE_FDIRUSTAT_ADD_MASK) >> 1334 IXGBE_FDIRUSTAT_ADD_SHIFT; 1335 1336 reg = IXGBE_READ_REG(hw, IXGBE_FDIRFSTAT) & 0xFFFF; 1337 info->f_remove += (reg & IXGBE_FDIRFSTAT_FREMOVE_MASK) >> 1338 IXGBE_FDIRFSTAT_FREMOVE_SHIFT; 1339 info->f_add += (reg & IXGBE_FDIRFSTAT_FADD_MASK) >> 1340 IXGBE_FDIRFSTAT_FADD_SHIFT; 1341 1342 /* Copy the new information in the fdir parameter */ 1343 fdir_stats->collision = info->collision; 1344 fdir_stats->free = info->free; 1345 fdir_stats->maxhash = info->maxhash; 1346 fdir_stats->maxlen = info->maxlen; 1347 fdir_stats->remove = info->remove; 1348 fdir_stats->add = info->add; 1349 fdir_stats->f_remove = info->f_remove; 1350 fdir_stats->f_add = info->f_add; 1351 1352 reg = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL); 1353 max_num = (1 << (FDIRENTRIES_NUM_SHIFT + 1354 (reg & FDIRCTRL_PBALLOC_MASK))); 1355 if (fdir_mode >= RTE_FDIR_MODE_PERFECT && 1356 fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 1357 fdir_stats->guarant_cnt = max_num - fdir_stats->free; 1358 else if (fdir_mode == RTE_FDIR_MODE_SIGNATURE) 1359 fdir_stats->guarant_cnt = max_num * 4 - fdir_stats->free; 1360 1361 } 1362 1363 /* restore flow director filter */ 1364 void 1365 ixgbe_fdir_filter_restore(struct rte_eth_dev *dev) 1366 { 1367 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1368 struct ixgbe_hw_fdir_info *fdir_info = 1369 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1370 struct ixgbe_fdir_filter *node; 1371 bool is_perfect = FALSE; 1372 enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; 1373 1374 if (fdir_mode >= RTE_FDIR_MODE_PERFECT && 1375 fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) 1376 is_perfect = TRUE; 1377 1378 if (is_perfect) { 1379 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 1380 (void)fdir_write_perfect_filter_82599(hw, 1381 &node->ixgbe_fdir, 1382 node->queue, 1383 node->fdirflags, 1384 node->fdirhash, 1385 fdir_mode); 1386 } 1387 } else { 1388 TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { 1389 (void)fdir_add_signature_filter_82599(hw, 1390 &node->ixgbe_fdir, 1391 node->queue, 1392 node->fdirflags, 1393 node->fdirhash); 1394 } 1395 } 1396 } 1397 1398 /* remove all the flow director filters */ 1399 int 1400 ixgbe_clear_all_fdir_filter(struct rte_eth_dev *dev) 1401 { 1402 struct ixgbe_hw_fdir_info *fdir_info = 1403 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); 1404 struct ixgbe_fdir_filter *fdir_filter; 1405 struct ixgbe_fdir_filter *filter_flag; 1406 int ret = 0; 1407 1408 /* flush flow director */ 1409 rte_hash_reset(fdir_info->hash_handle); 1410 memset(fdir_info->hash_map, 0, 1411 sizeof(struct ixgbe_fdir_filter *) * IXGBE_MAX_FDIR_FILTER_NUM); 1412 filter_flag = TAILQ_FIRST(&fdir_info->fdir_list); 1413 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 1414 TAILQ_REMOVE(&fdir_info->fdir_list, 1415 fdir_filter, 1416 entries); 1417 rte_free(fdir_filter); 1418 } 1419 1420 if (filter_flag != NULL) 1421 ret = ixgbe_fdir_flush(dev); 1422 1423 return ret; 1424 } 1425