1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <netinet/in.h> 14 #include <rte_string_fns.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <rte_bus_pci.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_kvargs.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <ethdev_driver.h> 31 #include <ethdev_pci.h> 32 #include <rte_malloc.h> 33 #include <rte_random.h> 34 #include <rte_dev.h> 35 #include <rte_hash_crc.h> 36 #ifdef RTE_LIB_SECURITY 37 #include <rte_security_driver.h> 38 #endif 39 40 #include "ixgbe_logs.h" 41 #include "base/ixgbe_api.h" 42 #include "base/ixgbe_vf.h" 43 #include "base/ixgbe_common.h" 44 #include "ixgbe_ethdev.h" 45 #include "ixgbe_bypass.h" 46 #include "ixgbe_rxtx.h" 47 #include "base/ixgbe_type.h" 48 #include "base/ixgbe_phy.h" 49 #include "ixgbe_regs.h" 50 51 /* 52 * High threshold controlling when to start sending XOFF frames. Must be at 53 * least 8 bytes less than receive packet buffer size. This value is in units 54 * of 1024 bytes. 55 */ 56 #define IXGBE_FC_HI 0x80 57 58 /* 59 * Low threshold controlling when to start sending XON frames. This value is 60 * in units of 1024 bytes. 61 */ 62 #define IXGBE_FC_LO 0x40 63 64 /* Timer value included in XOFF frames. */ 65 #define IXGBE_FC_PAUSE 0x680 66 67 /*Default value of Max Rx Queue*/ 68 #define IXGBE_MAX_RX_QUEUE_NUM 128 69 70 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */ 71 #define IXGBE_LINK_UP_CHECK_TIMEOUT 1000 /* ms */ 72 #define IXGBE_VMDQ_NUM_UC_MAC 4096 /* Maximum nb. of UC MAC addr. */ 73 74 #define IXGBE_MMW_SIZE_DEFAULT 0x4 75 #define IXGBE_MMW_SIZE_JUMBO_FRAME 0x14 76 #define IXGBE_MAX_RING_DESC 4096 /* replicate define from rxtx */ 77 78 /* 79 * Default values for RX/TX configuration 80 */ 81 #define IXGBE_DEFAULT_RX_FREE_THRESH 32 82 #define IXGBE_DEFAULT_RX_PTHRESH 8 83 #define IXGBE_DEFAULT_RX_HTHRESH 8 84 #define IXGBE_DEFAULT_RX_WTHRESH 0 85 86 #define IXGBE_DEFAULT_TX_FREE_THRESH 32 87 #define IXGBE_DEFAULT_TX_PTHRESH 32 88 #define IXGBE_DEFAULT_TX_HTHRESH 0 89 #define IXGBE_DEFAULT_TX_WTHRESH 0 90 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32 91 92 /* Bit shift and mask */ 93 #define IXGBE_4_BIT_WIDTH (CHAR_BIT / 2) 94 #define IXGBE_4_BIT_MASK RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t) 95 #define IXGBE_8_BIT_WIDTH CHAR_BIT 96 #define IXGBE_8_BIT_MASK UINT8_MAX 97 98 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */ 99 100 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0])) 101 102 /* Additional timesync values. */ 103 #define NSEC_PER_SEC 1000000000L 104 #define IXGBE_INCVAL_10GB 0x66666666 105 #define IXGBE_INCVAL_1GB 0x40000000 106 #define IXGBE_INCVAL_100 0x50000000 107 #define IXGBE_INCVAL_SHIFT_10GB 28 108 #define IXGBE_INCVAL_SHIFT_1GB 24 109 #define IXGBE_INCVAL_SHIFT_100 21 110 #define IXGBE_INCVAL_SHIFT_82599 7 111 #define IXGBE_INCPER_SHIFT_82599 24 112 113 #define IXGBE_CYCLECOUNTER_MASK 0xffffffffffffffffULL 114 115 #define IXGBE_VT_CTL_POOLING_MODE_MASK 0x00030000 116 #define IXGBE_VT_CTL_POOLING_MODE_ETAG 0x00010000 117 #define IXGBE_ETAG_ETYPE 0x00005084 118 #define IXGBE_ETAG_ETYPE_MASK 0x0000ffff 119 #define IXGBE_ETAG_ETYPE_VALID 0x80000000 120 #define IXGBE_RAH_ADTYPE 0x40000000 121 #define IXGBE_RAL_ETAG_FILTER_MASK 0x00003fff 122 #define IXGBE_VMVIR_TAGA_MASK 0x18000000 123 #define IXGBE_VMVIR_TAGA_ETAG_INSERT 0x08000000 124 #define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */ 125 #define IXGBE_QDE_STRIP_TAG 0x00000004 126 #define IXGBE_VTEICR_MASK 0x07 127 128 #define IXGBE_EXVET_VET_EXT_SHIFT 16 129 #define IXGBE_DMATXCTL_VT_MASK 0xFFFF0000 130 131 #define IXGBEVF_DEVARG_PFLINK_FULLCHK "pflink_fullchk" 132 133 static const char * const ixgbevf_valid_arguments[] = { 134 IXGBEVF_DEVARG_PFLINK_FULLCHK, 135 NULL 136 }; 137 138 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params); 139 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev); 140 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev); 141 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev); 142 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev); 143 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev); 144 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev); 145 static int ixgbe_dev_configure(struct rte_eth_dev *dev); 146 static int ixgbe_dev_start(struct rte_eth_dev *dev); 147 static int ixgbe_dev_stop(struct rte_eth_dev *dev); 148 static int ixgbe_dev_set_link_up(struct rte_eth_dev *dev); 149 static int ixgbe_dev_set_link_down(struct rte_eth_dev *dev); 150 static int ixgbe_dev_close(struct rte_eth_dev *dev); 151 static int ixgbe_dev_reset(struct rte_eth_dev *dev); 152 static int ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev); 153 static int ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev); 154 static int ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev); 155 static int ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev); 156 static int ixgbe_dev_link_update(struct rte_eth_dev *dev, 157 int wait_to_complete); 158 static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, 159 struct rte_eth_stats *stats); 160 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev, 161 struct rte_eth_xstat *xstats, unsigned n); 162 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, 163 struct rte_eth_xstat *xstats, unsigned n); 164 static int 165 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 166 uint64_t *values, unsigned int n); 167 static int ixgbe_dev_stats_reset(struct rte_eth_dev *dev); 168 static int ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); 169 static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, 170 struct rte_eth_xstat_name *xstats_names, 171 unsigned int size); 172 static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, 173 struct rte_eth_xstat_name *xstats_names, unsigned limit); 174 static int ixgbe_dev_xstats_get_names_by_id( 175 struct rte_eth_dev *dev, 176 struct rte_eth_xstat_name *xstats_names, 177 const uint64_t *ids, 178 unsigned int limit); 179 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 180 uint16_t queue_id, 181 uint8_t stat_idx, 182 uint8_t is_rx); 183 static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 184 size_t fw_size); 185 static int ixgbe_dev_info_get(struct rte_eth_dev *dev, 186 struct rte_eth_dev_info *dev_info); 187 static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev); 188 static int ixgbevf_dev_info_get(struct rte_eth_dev *dev, 189 struct rte_eth_dev_info *dev_info); 190 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 191 192 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev, 193 uint16_t vlan_id, int on); 194 static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 195 enum rte_vlan_type vlan_type, 196 uint16_t tpid_id); 197 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, 198 uint16_t queue, bool on); 199 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, 200 int on); 201 static void ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, 202 int mask); 203 static int ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask); 204 static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); 205 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue); 206 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue); 207 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev); 208 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev); 209 210 static int ixgbe_dev_led_on(struct rte_eth_dev *dev); 211 static int ixgbe_dev_led_off(struct rte_eth_dev *dev); 212 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, 213 struct rte_eth_fc_conf *fc_conf); 214 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, 215 struct rte_eth_fc_conf *fc_conf); 216 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, 217 struct rte_eth_pfc_conf *pfc_conf); 218 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 219 struct rte_eth_rss_reta_entry64 *reta_conf, 220 uint16_t reta_size); 221 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 222 struct rte_eth_rss_reta_entry64 *reta_conf, 223 uint16_t reta_size); 224 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); 225 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 226 static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); 227 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); 228 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); 229 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev); 230 static void ixgbe_dev_interrupt_handler(void *param); 231 static void ixgbe_dev_interrupt_delayed_handler(void *param); 232 static void *ixgbe_dev_setup_link_thread_handler(void *param); 233 static int ixgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, 234 uint32_t timeout_ms); 235 236 static int ixgbe_add_rar(struct rte_eth_dev *dev, 237 struct rte_ether_addr *mac_addr, 238 uint32_t index, uint32_t pool); 239 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index); 240 static int ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, 241 struct rte_ether_addr *mac_addr); 242 static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config); 243 static bool is_device_supported(struct rte_eth_dev *dev, 244 struct rte_pci_driver *drv); 245 246 /* For Virtual Function support */ 247 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); 248 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); 249 static int ixgbevf_dev_configure(struct rte_eth_dev *dev); 250 static int ixgbevf_dev_start(struct rte_eth_dev *dev); 251 static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, 252 int wait_to_complete); 253 static int ixgbevf_dev_stop(struct rte_eth_dev *dev); 254 static int ixgbevf_dev_close(struct rte_eth_dev *dev); 255 static int ixgbevf_dev_reset(struct rte_eth_dev *dev); 256 static void ixgbevf_intr_disable(struct rte_eth_dev *dev); 257 static void ixgbevf_intr_enable(struct rte_eth_dev *dev); 258 static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, 259 struct rte_eth_stats *stats); 260 static int ixgbevf_dev_stats_reset(struct rte_eth_dev *dev); 261 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, 262 uint16_t vlan_id, int on); 263 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, 264 uint16_t queue, int on); 265 static int ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask); 266 static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); 267 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on); 268 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 269 uint16_t queue_id); 270 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 271 uint16_t queue_id); 272 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 273 uint8_t queue, uint8_t msix_vector); 274 static void ixgbevf_configure_msix(struct rte_eth_dev *dev); 275 static int ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev); 276 static int ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev); 277 static int ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev); 278 static int ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev); 279 280 /* For Eth VMDQ APIs support */ 281 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct 282 rte_ether_addr * mac_addr, uint8_t on); 283 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on); 284 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 285 struct rte_eth_mirror_conf *mirror_conf, 286 uint8_t rule_id, uint8_t on); 287 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, 288 uint8_t rule_id); 289 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 290 uint16_t queue_id); 291 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 292 uint16_t queue_id); 293 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 294 uint8_t queue, uint8_t msix_vector); 295 static void ixgbe_configure_msix(struct rte_eth_dev *dev); 296 297 static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, 298 struct rte_ether_addr *mac_addr, 299 uint32_t index, uint32_t pool); 300 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index); 301 static int ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 302 struct rte_ether_addr *mac_addr); 303 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 304 struct ixgbe_5tuple_filter *filter); 305 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 306 struct ixgbe_5tuple_filter *filter); 307 static int ixgbe_dev_flow_ops_get(struct rte_eth_dev *dev, 308 const struct rte_flow_ops **ops); 309 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); 310 311 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 312 struct rte_ether_addr *mc_addr_set, 313 uint32_t nb_mc_addr); 314 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 315 struct rte_eth_dcb_info *dcb_info); 316 317 static int ixgbe_get_reg_length(struct rte_eth_dev *dev); 318 static int ixgbe_get_regs(struct rte_eth_dev *dev, 319 struct rte_dev_reg_info *regs); 320 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev); 321 static int ixgbe_get_eeprom(struct rte_eth_dev *dev, 322 struct rte_dev_eeprom_info *eeprom); 323 static int ixgbe_set_eeprom(struct rte_eth_dev *dev, 324 struct rte_dev_eeprom_info *eeprom); 325 326 static int ixgbe_get_module_info(struct rte_eth_dev *dev, 327 struct rte_eth_dev_module_info *modinfo); 328 static int ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 329 struct rte_dev_eeprom_info *info); 330 331 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev); 332 static int ixgbevf_get_regs(struct rte_eth_dev *dev, 333 struct rte_dev_reg_info *regs); 334 335 static int ixgbe_timesync_enable(struct rte_eth_dev *dev); 336 static int ixgbe_timesync_disable(struct rte_eth_dev *dev); 337 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 338 struct timespec *timestamp, 339 uint32_t flags); 340 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 341 struct timespec *timestamp); 342 static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 343 static int ixgbe_timesync_read_time(struct rte_eth_dev *dev, 344 struct timespec *timestamp); 345 static int ixgbe_timesync_write_time(struct rte_eth_dev *dev, 346 const struct timespec *timestamp); 347 static void ixgbevf_dev_interrupt_handler(void *param); 348 349 static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 350 struct rte_eth_udp_tunnel *udp_tunnel); 351 static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 352 struct rte_eth_udp_tunnel *udp_tunnel); 353 static int ixgbe_filter_restore(struct rte_eth_dev *dev); 354 static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev); 355 static int ixgbe_wait_for_link_up(struct ixgbe_hw *hw); 356 357 /* 358 * Define VF Stats MACRO for Non "cleared on read" register 359 */ 360 #define UPDATE_VF_STAT(reg, last, cur) \ 361 { \ 362 uint32_t latest = IXGBE_READ_REG(hw, reg); \ 363 cur += (latest - last) & UINT_MAX; \ 364 last = latest; \ 365 } 366 367 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur) \ 368 { \ 369 u64 new_lsb = IXGBE_READ_REG(hw, lsb); \ 370 u64 new_msb = IXGBE_READ_REG(hw, msb); \ 371 u64 latest = ((new_msb << 32) | new_lsb); \ 372 cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \ 373 last = latest; \ 374 } 375 376 #define IXGBE_SET_HWSTRIP(h, q) do {\ 377 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 378 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 379 (h)->bitmap[idx] |= 1 << bit;\ 380 } while (0) 381 382 #define IXGBE_CLEAR_HWSTRIP(h, q) do {\ 383 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 384 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 385 (h)->bitmap[idx] &= ~(1 << bit);\ 386 } while (0) 387 388 #define IXGBE_GET_HWSTRIP(h, q, r) do {\ 389 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 390 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 391 (r) = (h)->bitmap[idx] >> bit & 1;\ 392 } while (0) 393 394 /* 395 * The set of PCI devices this driver supports 396 */ 397 static const struct rte_pci_id pci_id_ixgbe_map[] = { 398 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) }, 399 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) }, 400 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) }, 401 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) }, 402 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) }, 403 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) }, 404 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) }, 405 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) }, 406 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) }, 407 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) }, 408 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) }, 409 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) }, 410 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) }, 411 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, 412 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, 413 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, 414 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, 415 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, 416 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, 417 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, 418 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, 419 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) }, 420 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) }, 421 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) }, 422 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) }, 423 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) }, 424 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) }, 425 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) }, 426 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) }, 427 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) }, 428 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) }, 429 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) }, 430 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) }, 431 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) }, 432 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) }, 433 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) }, 434 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) }, 435 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) }, 436 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) }, 437 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) }, 438 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) }, 439 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) }, 440 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) }, 441 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) }, 442 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, 443 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, 444 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, 445 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI) }, 446 #ifdef RTE_LIBRTE_IXGBE_BYPASS 447 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, 448 #endif 449 { .vendor_id = 0, /* sentinel */ }, 450 }; 451 452 /* 453 * The set of PCI devices this driver supports (for 82599 VF) 454 */ 455 static const struct rte_pci_id pci_id_ixgbevf_map[] = { 456 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) }, 457 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) }, 458 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) }, 459 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) }, 460 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) }, 461 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) }, 462 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) }, 463 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) }, 464 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) }, 465 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) }, 466 { .vendor_id = 0, /* sentinel */ }, 467 }; 468 469 static const struct rte_eth_desc_lim rx_desc_lim = { 470 .nb_max = IXGBE_MAX_RING_DESC, 471 .nb_min = IXGBE_MIN_RING_DESC, 472 .nb_align = IXGBE_RXD_ALIGN, 473 }; 474 475 static const struct rte_eth_desc_lim tx_desc_lim = { 476 .nb_max = IXGBE_MAX_RING_DESC, 477 .nb_min = IXGBE_MIN_RING_DESC, 478 .nb_align = IXGBE_TXD_ALIGN, 479 .nb_seg_max = IXGBE_TX_MAX_SEG, 480 .nb_mtu_seg_max = IXGBE_TX_MAX_SEG, 481 }; 482 483 static const struct eth_dev_ops ixgbe_eth_dev_ops = { 484 .dev_configure = ixgbe_dev_configure, 485 .dev_start = ixgbe_dev_start, 486 .dev_stop = ixgbe_dev_stop, 487 .dev_set_link_up = ixgbe_dev_set_link_up, 488 .dev_set_link_down = ixgbe_dev_set_link_down, 489 .dev_close = ixgbe_dev_close, 490 .dev_reset = ixgbe_dev_reset, 491 .promiscuous_enable = ixgbe_dev_promiscuous_enable, 492 .promiscuous_disable = ixgbe_dev_promiscuous_disable, 493 .allmulticast_enable = ixgbe_dev_allmulticast_enable, 494 .allmulticast_disable = ixgbe_dev_allmulticast_disable, 495 .link_update = ixgbe_dev_link_update, 496 .stats_get = ixgbe_dev_stats_get, 497 .xstats_get = ixgbe_dev_xstats_get, 498 .xstats_get_by_id = ixgbe_dev_xstats_get_by_id, 499 .stats_reset = ixgbe_dev_stats_reset, 500 .xstats_reset = ixgbe_dev_xstats_reset, 501 .xstats_get_names = ixgbe_dev_xstats_get_names, 502 .xstats_get_names_by_id = ixgbe_dev_xstats_get_names_by_id, 503 .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set, 504 .fw_version_get = ixgbe_fw_version_get, 505 .dev_infos_get = ixgbe_dev_info_get, 506 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 507 .mtu_set = ixgbe_dev_mtu_set, 508 .vlan_filter_set = ixgbe_vlan_filter_set, 509 .vlan_tpid_set = ixgbe_vlan_tpid_set, 510 .vlan_offload_set = ixgbe_vlan_offload_set, 511 .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set, 512 .rx_queue_start = ixgbe_dev_rx_queue_start, 513 .rx_queue_stop = ixgbe_dev_rx_queue_stop, 514 .tx_queue_start = ixgbe_dev_tx_queue_start, 515 .tx_queue_stop = ixgbe_dev_tx_queue_stop, 516 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 517 .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable, 518 .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable, 519 .rx_queue_release = ixgbe_dev_rx_queue_release, 520 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 521 .tx_queue_release = ixgbe_dev_tx_queue_release, 522 .dev_led_on = ixgbe_dev_led_on, 523 .dev_led_off = ixgbe_dev_led_off, 524 .flow_ctrl_get = ixgbe_flow_ctrl_get, 525 .flow_ctrl_set = ixgbe_flow_ctrl_set, 526 .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set, 527 .mac_addr_add = ixgbe_add_rar, 528 .mac_addr_remove = ixgbe_remove_rar, 529 .mac_addr_set = ixgbe_set_default_mac_addr, 530 .uc_hash_table_set = ixgbe_uc_hash_table_set, 531 .uc_all_hash_table_set = ixgbe_uc_all_hash_table_set, 532 .mirror_rule_set = ixgbe_mirror_rule_set, 533 .mirror_rule_reset = ixgbe_mirror_rule_reset, 534 .set_queue_rate_limit = ixgbe_set_queue_rate_limit, 535 .reta_update = ixgbe_dev_rss_reta_update, 536 .reta_query = ixgbe_dev_rss_reta_query, 537 .rss_hash_update = ixgbe_dev_rss_hash_update, 538 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 539 .flow_ops_get = ixgbe_dev_flow_ops_get, 540 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 541 .rxq_info_get = ixgbe_rxq_info_get, 542 .txq_info_get = ixgbe_txq_info_get, 543 .timesync_enable = ixgbe_timesync_enable, 544 .timesync_disable = ixgbe_timesync_disable, 545 .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, 546 .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp, 547 .get_reg = ixgbe_get_regs, 548 .get_eeprom_length = ixgbe_get_eeprom_length, 549 .get_eeprom = ixgbe_get_eeprom, 550 .set_eeprom = ixgbe_set_eeprom, 551 .get_module_info = ixgbe_get_module_info, 552 .get_module_eeprom = ixgbe_get_module_eeprom, 553 .get_dcb_info = ixgbe_dev_get_dcb_info, 554 .timesync_adjust_time = ixgbe_timesync_adjust_time, 555 .timesync_read_time = ixgbe_timesync_read_time, 556 .timesync_write_time = ixgbe_timesync_write_time, 557 .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, 558 .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, 559 .tm_ops_get = ixgbe_tm_ops_get, 560 .tx_done_cleanup = ixgbe_dev_tx_done_cleanup, 561 .get_monitor_addr = ixgbe_get_monitor_addr, 562 }; 563 564 /* 565 * dev_ops for virtual function, bare necessities for basic vf 566 * operation have been implemented 567 */ 568 static const struct eth_dev_ops ixgbevf_eth_dev_ops = { 569 .dev_configure = ixgbevf_dev_configure, 570 .dev_start = ixgbevf_dev_start, 571 .dev_stop = ixgbevf_dev_stop, 572 .link_update = ixgbevf_dev_link_update, 573 .stats_get = ixgbevf_dev_stats_get, 574 .xstats_get = ixgbevf_dev_xstats_get, 575 .stats_reset = ixgbevf_dev_stats_reset, 576 .xstats_reset = ixgbevf_dev_stats_reset, 577 .xstats_get_names = ixgbevf_dev_xstats_get_names, 578 .dev_close = ixgbevf_dev_close, 579 .dev_reset = ixgbevf_dev_reset, 580 .promiscuous_enable = ixgbevf_dev_promiscuous_enable, 581 .promiscuous_disable = ixgbevf_dev_promiscuous_disable, 582 .allmulticast_enable = ixgbevf_dev_allmulticast_enable, 583 .allmulticast_disable = ixgbevf_dev_allmulticast_disable, 584 .dev_infos_get = ixgbevf_dev_info_get, 585 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 586 .mtu_set = ixgbevf_dev_set_mtu, 587 .vlan_filter_set = ixgbevf_vlan_filter_set, 588 .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set, 589 .vlan_offload_set = ixgbevf_vlan_offload_set, 590 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 591 .rx_queue_release = ixgbe_dev_rx_queue_release, 592 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 593 .tx_queue_release = ixgbe_dev_tx_queue_release, 594 .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable, 595 .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable, 596 .mac_addr_add = ixgbevf_add_mac_addr, 597 .mac_addr_remove = ixgbevf_remove_mac_addr, 598 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 599 .rxq_info_get = ixgbe_rxq_info_get, 600 .txq_info_get = ixgbe_txq_info_get, 601 .mac_addr_set = ixgbevf_set_default_mac_addr, 602 .get_reg = ixgbevf_get_regs, 603 .reta_update = ixgbe_dev_rss_reta_update, 604 .reta_query = ixgbe_dev_rss_reta_query, 605 .rss_hash_update = ixgbe_dev_rss_hash_update, 606 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 607 .tx_done_cleanup = ixgbe_dev_tx_done_cleanup, 608 }; 609 610 /* store statistics names and its offset in stats structure */ 611 struct rte_ixgbe_xstats_name_off { 612 char name[RTE_ETH_XSTATS_NAME_SIZE]; 613 unsigned offset; 614 }; 615 616 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = { 617 {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)}, 618 {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)}, 619 {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)}, 620 {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)}, 621 {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)}, 622 {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)}, 623 {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)}, 624 {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)}, 625 {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)}, 626 {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)}, 627 {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)}, 628 {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)}, 629 {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)}, 630 {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)}, 631 {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 632 prc1023)}, 633 {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 634 prc1522)}, 635 {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)}, 636 {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)}, 637 {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)}, 638 {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)}, 639 {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)}, 640 {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)}, 641 {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)}, 642 {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)}, 643 {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)}, 644 {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)}, 645 {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)}, 646 {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)}, 647 {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)}, 648 {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)}, 649 {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)}, 650 {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)}, 651 {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 652 ptc1023)}, 653 {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 654 ptc1522)}, 655 {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)}, 656 {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)}, 657 {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)}, 658 {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)}, 659 660 {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats, 661 fdirustat_add)}, 662 {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats, 663 fdirustat_remove)}, 664 {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats, 665 fdirfstat_fadd)}, 666 {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats, 667 fdirfstat_fremove)}, 668 {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats, 669 fdirmatch)}, 670 {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats, 671 fdirmiss)}, 672 673 {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)}, 674 {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)}, 675 {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, 676 fclast)}, 677 {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)}, 678 {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)}, 679 {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)}, 680 {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)}, 681 {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats, 682 fcoe_noddp)}, 683 {"rx_fcoe_no_direct_data_placement_ext_buff", 684 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)}, 685 686 {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 687 lxontxc)}, 688 {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 689 lxonrxc)}, 690 {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 691 lxofftxc)}, 692 {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 693 lxoffrxc)}, 694 {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)}, 695 }; 696 697 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \ 698 sizeof(rte_ixgbe_stats_strings[0])) 699 700 /* MACsec statistics */ 701 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = { 702 {"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 703 out_pkts_untagged)}, 704 {"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats, 705 out_pkts_encrypted)}, 706 {"out_pkts_protected", offsetof(struct ixgbe_macsec_stats, 707 out_pkts_protected)}, 708 {"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats, 709 out_octets_encrypted)}, 710 {"out_octets_protected", offsetof(struct ixgbe_macsec_stats, 711 out_octets_protected)}, 712 {"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 713 in_pkts_untagged)}, 714 {"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats, 715 in_pkts_badtag)}, 716 {"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats, 717 in_pkts_nosci)}, 718 {"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats, 719 in_pkts_unknownsci)}, 720 {"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats, 721 in_octets_decrypted)}, 722 {"in_octets_validated", offsetof(struct ixgbe_macsec_stats, 723 in_octets_validated)}, 724 {"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats, 725 in_pkts_unchecked)}, 726 {"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats, 727 in_pkts_delayed)}, 728 {"in_pkts_late", offsetof(struct ixgbe_macsec_stats, 729 in_pkts_late)}, 730 {"in_pkts_ok", offsetof(struct ixgbe_macsec_stats, 731 in_pkts_ok)}, 732 {"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats, 733 in_pkts_invalid)}, 734 {"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats, 735 in_pkts_notvalid)}, 736 {"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats, 737 in_pkts_unusedsa)}, 738 {"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats, 739 in_pkts_notusingsa)}, 740 }; 741 742 #define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \ 743 sizeof(rte_ixgbe_macsec_strings[0])) 744 745 /* Per-queue statistics */ 746 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = { 747 {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)}, 748 {"dropped", offsetof(struct ixgbe_hw_stats, mpc)}, 749 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)}, 750 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)}, 751 }; 752 753 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \ 754 sizeof(rte_ixgbe_rxq_strings[0])) 755 #define IXGBE_NB_RXQ_PRIO_VALUES 8 756 757 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = { 758 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)}, 759 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)}, 760 {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats, 761 pxon2offc)}, 762 }; 763 764 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \ 765 sizeof(rte_ixgbe_txq_strings[0])) 766 #define IXGBE_NB_TXQ_PRIO_VALUES 8 767 768 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = { 769 {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)}, 770 }; 771 772 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) / \ 773 sizeof(rte_ixgbevf_stats_strings[0])) 774 775 /* 776 * This function is the same as ixgbe_is_sfp() in base/ixgbe.h. 777 */ 778 static inline int 779 ixgbe_is_sfp(struct ixgbe_hw *hw) 780 { 781 switch (hw->phy.type) { 782 case ixgbe_phy_sfp_avago: 783 case ixgbe_phy_sfp_ftl: 784 case ixgbe_phy_sfp_intel: 785 case ixgbe_phy_sfp_unknown: 786 case ixgbe_phy_sfp_passive_tyco: 787 case ixgbe_phy_sfp_passive_unknown: 788 return 1; 789 default: 790 return 0; 791 } 792 } 793 794 static inline int32_t 795 ixgbe_pf_reset_hw(struct ixgbe_hw *hw) 796 { 797 uint32_t ctrl_ext; 798 int32_t status; 799 800 status = ixgbe_reset_hw(hw); 801 802 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 803 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 804 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 805 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 806 IXGBE_WRITE_FLUSH(hw); 807 808 if (status == IXGBE_ERR_SFP_NOT_PRESENT) 809 status = IXGBE_SUCCESS; 810 return status; 811 } 812 813 static inline void 814 ixgbe_enable_intr(struct rte_eth_dev *dev) 815 { 816 struct ixgbe_interrupt *intr = 817 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 818 struct ixgbe_hw *hw = 819 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 820 821 IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask); 822 IXGBE_WRITE_FLUSH(hw); 823 } 824 825 /* 826 * This function is based on ixgbe_disable_intr() in base/ixgbe.h. 827 */ 828 static void 829 ixgbe_disable_intr(struct ixgbe_hw *hw) 830 { 831 PMD_INIT_FUNC_TRACE(); 832 833 if (hw->mac.type == ixgbe_mac_82598EB) { 834 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0); 835 } else { 836 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000); 837 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0); 838 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0); 839 } 840 IXGBE_WRITE_FLUSH(hw); 841 } 842 843 /* 844 * This function resets queue statistics mapping registers. 845 * From Niantic datasheet, Initialization of Statistics section: 846 * "...if software requires the queue counters, the RQSMR and TQSM registers 847 * must be re-programmed following a device reset. 848 */ 849 static void 850 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw) 851 { 852 uint32_t i; 853 854 for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) { 855 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0); 856 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0); 857 } 858 } 859 860 861 static int 862 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 863 uint16_t queue_id, 864 uint8_t stat_idx, 865 uint8_t is_rx) 866 { 867 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8 868 #define NB_QMAP_FIELDS_PER_QSM_REG 4 869 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f 870 871 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 872 struct ixgbe_stat_mapping_registers *stat_mappings = 873 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private); 874 uint32_t qsmr_mask = 0; 875 uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK; 876 uint32_t q_map; 877 uint8_t n, offset; 878 879 if ((hw->mac.type != ixgbe_mac_82599EB) && 880 (hw->mac.type != ixgbe_mac_X540) && 881 (hw->mac.type != ixgbe_mac_X550) && 882 (hw->mac.type != ixgbe_mac_X550EM_x) && 883 (hw->mac.type != ixgbe_mac_X550EM_a)) 884 return -ENOSYS; 885 886 PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d", 887 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 888 queue_id, stat_idx); 889 890 n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG); 891 if (n >= IXGBE_NB_STAT_MAPPING_REGS) { 892 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded"); 893 return -EIO; 894 } 895 offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG); 896 897 /* Now clear any previous stat_idx set */ 898 clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 899 if (!is_rx) 900 stat_mappings->tqsm[n] &= ~clearing_mask; 901 else 902 stat_mappings->rqsmr[n] &= ~clearing_mask; 903 904 q_map = (uint32_t)stat_idx; 905 q_map &= QMAP_FIELD_RESERVED_BITS_MASK; 906 qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 907 if (!is_rx) 908 stat_mappings->tqsm[n] |= qsmr_mask; 909 else 910 stat_mappings->rqsmr[n] |= qsmr_mask; 911 912 PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d", 913 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 914 queue_id, stat_idx); 915 PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n, 916 is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]); 917 918 /* Now write the mapping in the appropriate register */ 919 if (is_rx) { 920 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d", 921 stat_mappings->rqsmr[n], n); 922 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]); 923 } else { 924 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d", 925 stat_mappings->tqsm[n], n); 926 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]); 927 } 928 return 0; 929 } 930 931 static void 932 ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev) 933 { 934 struct ixgbe_stat_mapping_registers *stat_mappings = 935 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private); 936 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 937 int i; 938 939 /* write whatever was in stat mapping table to the NIC */ 940 for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) { 941 /* rx */ 942 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]); 943 944 /* tx */ 945 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]); 946 } 947 } 948 949 static void 950 ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config) 951 { 952 uint8_t i; 953 struct ixgbe_dcb_tc_config *tc; 954 uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS; 955 956 dcb_config->num_tcs.pg_tcs = dcb_max_tc; 957 dcb_config->num_tcs.pfc_tcs = dcb_max_tc; 958 for (i = 0; i < dcb_max_tc; i++) { 959 tc = &dcb_config->tc_config[i]; 960 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i; 961 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 962 (uint8_t)(100/dcb_max_tc + (i & 1)); 963 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i; 964 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 965 (uint8_t)(100/dcb_max_tc + (i & 1)); 966 tc->pfc = ixgbe_dcb_pfc_disabled; 967 } 968 969 /* Initialize default user to priority mapping, UPx->TC0 */ 970 tc = &dcb_config->tc_config[0]; 971 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 972 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 973 for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) { 974 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100; 975 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100; 976 } 977 dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal; 978 dcb_config->pfc_mode_enable = false; 979 dcb_config->vt_mode = true; 980 dcb_config->round_robin_enable = false; 981 /* support all DCB capabilities in 82599 */ 982 dcb_config->support.capabilities = 0xFF; 983 984 /*we only support 4 Tcs for X540, X550 */ 985 if (hw->mac.type == ixgbe_mac_X540 || 986 hw->mac.type == ixgbe_mac_X550 || 987 hw->mac.type == ixgbe_mac_X550EM_x || 988 hw->mac.type == ixgbe_mac_X550EM_a) { 989 dcb_config->num_tcs.pg_tcs = 4; 990 dcb_config->num_tcs.pfc_tcs = 4; 991 } 992 } 993 994 /* 995 * Ensure that all locks are released before first NVM or PHY access 996 */ 997 static void 998 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) 999 { 1000 uint16_t mask; 1001 1002 /* 1003 * Phy lock should not fail in this early stage. If this is the case, 1004 * it is due to an improper exit of the application. 1005 * So force the release of the faulty lock. Release of common lock 1006 * is done automatically by swfw_sync function. 1007 */ 1008 mask = IXGBE_GSSR_PHY0_SM << hw->bus.func; 1009 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1010 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func); 1011 } 1012 ixgbe_release_swfw_semaphore(hw, mask); 1013 1014 /* 1015 * These ones are more tricky since they are common to all ports; but 1016 * swfw_sync retries last long enough (1s) to be almost sure that if 1017 * lock can not be taken it is due to an improper lock of the 1018 * semaphore. 1019 */ 1020 mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM; 1021 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1022 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 1023 } 1024 ixgbe_release_swfw_semaphore(hw, mask); 1025 } 1026 1027 /* 1028 * This function is based on code in ixgbe_attach() in base/ixgbe.c. 1029 * It returns 0 on success. 1030 */ 1031 static int 1032 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) 1033 { 1034 struct ixgbe_adapter *ad = eth_dev->data->dev_private; 1035 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1036 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1037 struct ixgbe_hw *hw = 1038 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1039 struct ixgbe_vfta *shadow_vfta = 1040 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1041 struct ixgbe_hwstrip *hwstrip = 1042 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1043 struct ixgbe_dcb_config *dcb_config = 1044 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private); 1045 struct ixgbe_filter_info *filter_info = 1046 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1047 struct ixgbe_bw_conf *bw_conf = 1048 IXGBE_DEV_PRIVATE_TO_BW_CONF(eth_dev->data->dev_private); 1049 uint32_t ctrl_ext; 1050 uint16_t csum; 1051 int diag, i, ret; 1052 1053 PMD_INIT_FUNC_TRACE(); 1054 1055 ixgbe_dev_macsec_setting_reset(eth_dev); 1056 1057 eth_dev->dev_ops = &ixgbe_eth_dev_ops; 1058 eth_dev->rx_queue_count = ixgbe_dev_rx_queue_count; 1059 eth_dev->rx_descriptor_done = ixgbe_dev_rx_descriptor_done; 1060 eth_dev->rx_descriptor_status = ixgbe_dev_rx_descriptor_status; 1061 eth_dev->tx_descriptor_status = ixgbe_dev_tx_descriptor_status; 1062 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1063 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1064 eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts; 1065 1066 /* 1067 * For secondary processes, we don't initialise any further as primary 1068 * has already done this work. Only check we don't need a different 1069 * RX and TX function. 1070 */ 1071 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1072 struct ixgbe_tx_queue *txq; 1073 /* TX queue function in primary, set by last queue initialized 1074 * Tx queue may not initialized by primary process 1075 */ 1076 if (eth_dev->data->tx_queues) { 1077 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1]; 1078 ixgbe_set_tx_function(eth_dev, txq); 1079 } else { 1080 /* Use default TX function if we get here */ 1081 PMD_INIT_LOG(NOTICE, "No TX queues configured yet. " 1082 "Using default TX function."); 1083 } 1084 1085 ixgbe_set_rx_function(eth_dev); 1086 1087 return 0; 1088 } 1089 1090 rte_atomic32_clear(&ad->link_thread_running); 1091 rte_eth_copy_pci_info(eth_dev, pci_dev); 1092 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1093 1094 /* Vendor and Device ID need to be set before init of shared code */ 1095 hw->device_id = pci_dev->id.device_id; 1096 hw->vendor_id = pci_dev->id.vendor_id; 1097 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1098 hw->allow_unsupported_sfp = 1; 1099 1100 /* Initialize the shared code (base driver) */ 1101 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1102 diag = ixgbe_bypass_init_shared_code(hw); 1103 #else 1104 diag = ixgbe_init_shared_code(hw); 1105 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1106 1107 if (diag != IXGBE_SUCCESS) { 1108 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); 1109 return -EIO; 1110 } 1111 1112 if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) { 1113 PMD_INIT_LOG(ERR, "\nERROR: " 1114 "Firmware recovery mode detected. Limiting functionality.\n" 1115 "Refer to the Intel(R) Ethernet Adapters and Devices " 1116 "User Guide for details on firmware recovery mode."); 1117 return -EIO; 1118 } 1119 1120 /* pick up the PCI bus settings for reporting later */ 1121 ixgbe_get_bus_info(hw); 1122 1123 /* Unlock any pending hardware semaphore */ 1124 ixgbe_swfw_lock_reset(hw); 1125 1126 #ifdef RTE_LIB_SECURITY 1127 /* Initialize security_ctx only for primary process*/ 1128 if (ixgbe_ipsec_ctx_create(eth_dev)) 1129 return -ENOMEM; 1130 #endif 1131 1132 /* Initialize DCB configuration*/ 1133 memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config)); 1134 ixgbe_dcb_init(hw, dcb_config); 1135 /* Get Hardware Flow Control setting */ 1136 hw->fc.requested_mode = ixgbe_fc_none; 1137 hw->fc.current_mode = ixgbe_fc_none; 1138 hw->fc.pause_time = IXGBE_FC_PAUSE; 1139 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 1140 hw->fc.low_water[i] = IXGBE_FC_LO; 1141 hw->fc.high_water[i] = IXGBE_FC_HI; 1142 } 1143 hw->fc.send_xon = 1; 1144 1145 /* Make sure we have a good EEPROM before we read from it */ 1146 diag = ixgbe_validate_eeprom_checksum(hw, &csum); 1147 if (diag != IXGBE_SUCCESS) { 1148 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag); 1149 return -EIO; 1150 } 1151 1152 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1153 diag = ixgbe_bypass_init_hw(hw); 1154 #else 1155 diag = ixgbe_init_hw(hw); 1156 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1157 1158 /* 1159 * Devices with copper phys will fail to initialise if ixgbe_init_hw() 1160 * is called too soon after the kernel driver unbinding/binding occurs. 1161 * The failure occurs in ixgbe_identify_phy_generic() for all devices, 1162 * but for non-copper devies, ixgbe_identify_sfp_module_generic() is 1163 * also called. See ixgbe_identify_phy_82599(). The reason for the 1164 * failure is not known, and only occuts when virtualisation features 1165 * are disabled in the bios. A delay of 100ms was found to be enough by 1166 * trial-and-error, and is doubled to be safe. 1167 */ 1168 if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) { 1169 rte_delay_ms(200); 1170 diag = ixgbe_init_hw(hw); 1171 } 1172 1173 if (diag == IXGBE_ERR_SFP_NOT_PRESENT) 1174 diag = IXGBE_SUCCESS; 1175 1176 if (diag == IXGBE_ERR_EEPROM_VERSION) { 1177 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/" 1178 "LOM. Please be aware there may be issues associated " 1179 "with your hardware."); 1180 PMD_INIT_LOG(ERR, "If you are experiencing problems " 1181 "please contact your Intel or hardware representative " 1182 "who provided you with this hardware."); 1183 } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED) 1184 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module"); 1185 if (diag) { 1186 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag); 1187 return -EIO; 1188 } 1189 1190 /* Reset the hw statistics */ 1191 ixgbe_dev_stats_reset(eth_dev); 1192 1193 /* disable interrupt */ 1194 ixgbe_disable_intr(hw); 1195 1196 /* reset mappings for queue statistics hw counters*/ 1197 ixgbe_reset_qstat_mappings(hw); 1198 1199 /* Allocate memory for storing MAC addresses */ 1200 eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", RTE_ETHER_ADDR_LEN * 1201 hw->mac.num_rar_entries, 0); 1202 if (eth_dev->data->mac_addrs == NULL) { 1203 PMD_INIT_LOG(ERR, 1204 "Failed to allocate %u bytes needed to store " 1205 "MAC addresses", 1206 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1207 return -ENOMEM; 1208 } 1209 /* Copy the permanent MAC address */ 1210 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1211 ð_dev->data->mac_addrs[0]); 1212 1213 /* Allocate memory for storing hash filter MAC addresses */ 1214 eth_dev->data->hash_mac_addrs = rte_zmalloc( 1215 "ixgbe", RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC, 0); 1216 if (eth_dev->data->hash_mac_addrs == NULL) { 1217 PMD_INIT_LOG(ERR, 1218 "Failed to allocate %d bytes needed to store MAC addresses", 1219 RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC); 1220 return -ENOMEM; 1221 } 1222 1223 /* initialize the vfta */ 1224 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1225 1226 /* initialize the hw strip bitmap*/ 1227 memset(hwstrip, 0, sizeof(*hwstrip)); 1228 1229 /* initialize PF if max_vfs not zero */ 1230 ret = ixgbe_pf_host_init(eth_dev); 1231 if (ret) { 1232 rte_free(eth_dev->data->mac_addrs); 1233 eth_dev->data->mac_addrs = NULL; 1234 rte_free(eth_dev->data->hash_mac_addrs); 1235 eth_dev->data->hash_mac_addrs = NULL; 1236 return ret; 1237 } 1238 1239 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 1240 /* let hardware know driver is loaded */ 1241 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 1242 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1243 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 1244 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 1245 IXGBE_WRITE_FLUSH(hw); 1246 1247 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 1248 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d", 1249 (int) hw->mac.type, (int) hw->phy.type, 1250 (int) hw->phy.sfp_type); 1251 else 1252 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d", 1253 (int) hw->mac.type, (int) hw->phy.type); 1254 1255 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1256 eth_dev->data->port_id, pci_dev->id.vendor_id, 1257 pci_dev->id.device_id); 1258 1259 rte_intr_callback_register(intr_handle, 1260 ixgbe_dev_interrupt_handler, eth_dev); 1261 1262 /* enable uio/vfio intr/eventfd mapping */ 1263 rte_intr_enable(intr_handle); 1264 1265 /* enable support intr */ 1266 ixgbe_enable_intr(eth_dev); 1267 1268 /* initialize filter info */ 1269 memset(filter_info, 0, 1270 sizeof(struct ixgbe_filter_info)); 1271 1272 /* initialize 5tuple filter list */ 1273 TAILQ_INIT(&filter_info->fivetuple_list); 1274 1275 /* initialize flow director filter list & hash */ 1276 ixgbe_fdir_filter_init(eth_dev); 1277 1278 /* initialize l2 tunnel filter list & hash */ 1279 ixgbe_l2_tn_filter_init(eth_dev); 1280 1281 /* initialize flow filter lists */ 1282 ixgbe_filterlist_init(); 1283 1284 /* initialize bandwidth configuration info */ 1285 memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); 1286 1287 /* initialize Traffic Manager configuration */ 1288 ixgbe_tm_conf_init(eth_dev); 1289 1290 return 0; 1291 } 1292 1293 static int 1294 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) 1295 { 1296 PMD_INIT_FUNC_TRACE(); 1297 1298 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1299 return 0; 1300 1301 ixgbe_dev_close(eth_dev); 1302 1303 return 0; 1304 } 1305 1306 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 1307 { 1308 struct ixgbe_filter_info *filter_info = 1309 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1310 struct ixgbe_5tuple_filter *p_5tuple; 1311 1312 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 1313 TAILQ_REMOVE(&filter_info->fivetuple_list, 1314 p_5tuple, 1315 entries); 1316 rte_free(p_5tuple); 1317 } 1318 memset(filter_info->fivetuple_mask, 0, 1319 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); 1320 1321 return 0; 1322 } 1323 1324 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev) 1325 { 1326 struct ixgbe_hw_fdir_info *fdir_info = 1327 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1328 struct ixgbe_fdir_filter *fdir_filter; 1329 1330 if (fdir_info->hash_map) 1331 rte_free(fdir_info->hash_map); 1332 if (fdir_info->hash_handle) 1333 rte_hash_free(fdir_info->hash_handle); 1334 1335 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 1336 TAILQ_REMOVE(&fdir_info->fdir_list, 1337 fdir_filter, 1338 entries); 1339 rte_free(fdir_filter); 1340 } 1341 1342 return 0; 1343 } 1344 1345 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev) 1346 { 1347 struct ixgbe_l2_tn_info *l2_tn_info = 1348 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1349 struct ixgbe_l2_tn_filter *l2_tn_filter; 1350 1351 if (l2_tn_info->hash_map) 1352 rte_free(l2_tn_info->hash_map); 1353 if (l2_tn_info->hash_handle) 1354 rte_hash_free(l2_tn_info->hash_handle); 1355 1356 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 1357 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, 1358 l2_tn_filter, 1359 entries); 1360 rte_free(l2_tn_filter); 1361 } 1362 1363 return 0; 1364 } 1365 1366 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) 1367 { 1368 struct ixgbe_hw_fdir_info *fdir_info = 1369 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1370 char fdir_hash_name[RTE_HASH_NAMESIZE]; 1371 struct rte_hash_parameters fdir_hash_params = { 1372 .name = fdir_hash_name, 1373 .entries = IXGBE_MAX_FDIR_FILTER_NUM, 1374 .key_len = sizeof(union ixgbe_atr_input), 1375 .hash_func = rte_hash_crc, 1376 .hash_func_init_val = 0, 1377 .socket_id = rte_socket_id(), 1378 }; 1379 1380 TAILQ_INIT(&fdir_info->fdir_list); 1381 snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, 1382 "fdir_%s", eth_dev->device->name); 1383 fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); 1384 if (!fdir_info->hash_handle) { 1385 PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); 1386 return -EINVAL; 1387 } 1388 fdir_info->hash_map = rte_zmalloc("ixgbe", 1389 sizeof(struct ixgbe_fdir_filter *) * 1390 IXGBE_MAX_FDIR_FILTER_NUM, 1391 0); 1392 if (!fdir_info->hash_map) { 1393 PMD_INIT_LOG(ERR, 1394 "Failed to allocate memory for fdir hash map!"); 1395 return -ENOMEM; 1396 } 1397 fdir_info->mask_added = FALSE; 1398 1399 return 0; 1400 } 1401 1402 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) 1403 { 1404 struct ixgbe_l2_tn_info *l2_tn_info = 1405 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1406 char l2_tn_hash_name[RTE_HASH_NAMESIZE]; 1407 struct rte_hash_parameters l2_tn_hash_params = { 1408 .name = l2_tn_hash_name, 1409 .entries = IXGBE_MAX_L2_TN_FILTER_NUM, 1410 .key_len = sizeof(struct ixgbe_l2_tn_key), 1411 .hash_func = rte_hash_crc, 1412 .hash_func_init_val = 0, 1413 .socket_id = rte_socket_id(), 1414 }; 1415 1416 TAILQ_INIT(&l2_tn_info->l2_tn_list); 1417 snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, 1418 "l2_tn_%s", eth_dev->device->name); 1419 l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); 1420 if (!l2_tn_info->hash_handle) { 1421 PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); 1422 return -EINVAL; 1423 } 1424 l2_tn_info->hash_map = rte_zmalloc("ixgbe", 1425 sizeof(struct ixgbe_l2_tn_filter *) * 1426 IXGBE_MAX_L2_TN_FILTER_NUM, 1427 0); 1428 if (!l2_tn_info->hash_map) { 1429 PMD_INIT_LOG(ERR, 1430 "Failed to allocate memory for L2 TN hash map!"); 1431 return -ENOMEM; 1432 } 1433 l2_tn_info->e_tag_en = FALSE; 1434 l2_tn_info->e_tag_fwd_en = FALSE; 1435 l2_tn_info->e_tag_ether_type = RTE_ETHER_TYPE_ETAG; 1436 1437 return 0; 1438 } 1439 /* 1440 * Negotiate mailbox API version with the PF. 1441 * After reset API version is always set to the basic one (ixgbe_mbox_api_10). 1442 * Then we try to negotiate starting with the most recent one. 1443 * If all negotiation attempts fail, then we will proceed with 1444 * the default one (ixgbe_mbox_api_10). 1445 */ 1446 static void 1447 ixgbevf_negotiate_api(struct ixgbe_hw *hw) 1448 { 1449 int32_t i; 1450 1451 /* start with highest supported, proceed down */ 1452 static const enum ixgbe_pfvf_api_rev sup_ver[] = { 1453 ixgbe_mbox_api_13, 1454 ixgbe_mbox_api_12, 1455 ixgbe_mbox_api_11, 1456 ixgbe_mbox_api_10, 1457 }; 1458 1459 for (i = 0; 1460 i != RTE_DIM(sup_ver) && 1461 ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0; 1462 i++) 1463 ; 1464 } 1465 1466 static void 1467 generate_random_mac_addr(struct rte_ether_addr *mac_addr) 1468 { 1469 uint64_t random; 1470 1471 /* Set Organizationally Unique Identifier (OUI) prefix. */ 1472 mac_addr->addr_bytes[0] = 0x00; 1473 mac_addr->addr_bytes[1] = 0x09; 1474 mac_addr->addr_bytes[2] = 0xC0; 1475 /* Force indication of locally assigned MAC address. */ 1476 mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR; 1477 /* Generate the last 3 bytes of the MAC address with a random number. */ 1478 random = rte_rand(); 1479 memcpy(&mac_addr->addr_bytes[3], &random, 3); 1480 } 1481 1482 static int 1483 devarg_handle_int(__rte_unused const char *key, const char *value, 1484 void *extra_args) 1485 { 1486 uint16_t *n = extra_args; 1487 1488 if (value == NULL || extra_args == NULL) 1489 return -EINVAL; 1490 1491 *n = (uint16_t)strtoul(value, NULL, 0); 1492 if (*n == USHRT_MAX && errno == ERANGE) 1493 return -1; 1494 1495 return 0; 1496 } 1497 1498 static void 1499 ixgbevf_parse_devargs(struct ixgbe_adapter *adapter, 1500 struct rte_devargs *devargs) 1501 { 1502 struct rte_kvargs *kvlist; 1503 uint16_t pflink_fullchk; 1504 1505 if (devargs == NULL) 1506 return; 1507 1508 kvlist = rte_kvargs_parse(devargs->args, ixgbevf_valid_arguments); 1509 if (kvlist == NULL) 1510 return; 1511 1512 if (rte_kvargs_count(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK) == 1 && 1513 rte_kvargs_process(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK, 1514 devarg_handle_int, &pflink_fullchk) == 0 && 1515 pflink_fullchk == 1) 1516 adapter->pflink_fullchk = 1; 1517 1518 rte_kvargs_free(kvlist); 1519 } 1520 1521 /* 1522 * Virtual Function device init 1523 */ 1524 static int 1525 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) 1526 { 1527 int diag; 1528 uint32_t tc, tcs; 1529 struct ixgbe_adapter *ad = eth_dev->data->dev_private; 1530 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1531 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1532 struct ixgbe_hw *hw = 1533 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1534 struct ixgbe_vfta *shadow_vfta = 1535 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1536 struct ixgbe_hwstrip *hwstrip = 1537 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1538 struct rte_ether_addr *perm_addr = 1539 (struct rte_ether_addr *)hw->mac.perm_addr; 1540 1541 PMD_INIT_FUNC_TRACE(); 1542 1543 eth_dev->dev_ops = &ixgbevf_eth_dev_ops; 1544 eth_dev->rx_descriptor_done = ixgbe_dev_rx_descriptor_done; 1545 eth_dev->rx_descriptor_status = ixgbe_dev_rx_descriptor_status; 1546 eth_dev->tx_descriptor_status = ixgbe_dev_tx_descriptor_status; 1547 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1548 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1549 1550 /* for secondary processes, we don't initialise any further as primary 1551 * has already done this work. Only check we don't need a different 1552 * RX function 1553 */ 1554 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1555 struct ixgbe_tx_queue *txq; 1556 /* TX queue function in primary, set by last queue initialized 1557 * Tx queue may not initialized by primary process 1558 */ 1559 if (eth_dev->data->tx_queues) { 1560 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1]; 1561 ixgbe_set_tx_function(eth_dev, txq); 1562 } else { 1563 /* Use default TX function if we get here */ 1564 PMD_INIT_LOG(NOTICE, 1565 "No TX queues configured yet. Using default TX function."); 1566 } 1567 1568 ixgbe_set_rx_function(eth_dev); 1569 1570 return 0; 1571 } 1572 1573 rte_atomic32_clear(&ad->link_thread_running); 1574 ixgbevf_parse_devargs(eth_dev->data->dev_private, 1575 pci_dev->device.devargs); 1576 1577 rte_eth_copy_pci_info(eth_dev, pci_dev); 1578 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1579 1580 hw->device_id = pci_dev->id.device_id; 1581 hw->vendor_id = pci_dev->id.vendor_id; 1582 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1583 1584 /* initialize the vfta */ 1585 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1586 1587 /* initialize the hw strip bitmap*/ 1588 memset(hwstrip, 0, sizeof(*hwstrip)); 1589 1590 /* Initialize the shared code (base driver) */ 1591 diag = ixgbe_init_shared_code(hw); 1592 if (diag != IXGBE_SUCCESS) { 1593 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag); 1594 return -EIO; 1595 } 1596 1597 /* init_mailbox_params */ 1598 hw->mbx.ops.init_params(hw); 1599 1600 /* Reset the hw statistics */ 1601 ixgbevf_dev_stats_reset(eth_dev); 1602 1603 /* Disable the interrupts for VF */ 1604 ixgbevf_intr_disable(eth_dev); 1605 1606 hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */ 1607 diag = hw->mac.ops.reset_hw(hw); 1608 1609 /* 1610 * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when 1611 * the underlying PF driver has not assigned a MAC address to the VF. 1612 * In this case, assign a random MAC address. 1613 */ 1614 if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) { 1615 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1616 /* 1617 * This error code will be propagated to the app by 1618 * rte_eth_dev_reset, so use a public error code rather than 1619 * the internal-only IXGBE_ERR_RESET_FAILED 1620 */ 1621 return -EAGAIN; 1622 } 1623 1624 /* negotiate mailbox API version to use with the PF. */ 1625 ixgbevf_negotiate_api(hw); 1626 1627 /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */ 1628 ixgbevf_get_queues(hw, &tcs, &tc); 1629 1630 /* Allocate memory for storing MAC addresses */ 1631 eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", RTE_ETHER_ADDR_LEN * 1632 hw->mac.num_rar_entries, 0); 1633 if (eth_dev->data->mac_addrs == NULL) { 1634 PMD_INIT_LOG(ERR, 1635 "Failed to allocate %u bytes needed to store " 1636 "MAC addresses", 1637 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1638 return -ENOMEM; 1639 } 1640 1641 /* Generate a random MAC address, if none was assigned by PF. */ 1642 if (rte_is_zero_ether_addr(perm_addr)) { 1643 generate_random_mac_addr(perm_addr); 1644 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1); 1645 if (diag) { 1646 rte_free(eth_dev->data->mac_addrs); 1647 eth_dev->data->mac_addrs = NULL; 1648 return diag; 1649 } 1650 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1651 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1652 "%02x:%02x:%02x:%02x:%02x:%02x", 1653 perm_addr->addr_bytes[0], 1654 perm_addr->addr_bytes[1], 1655 perm_addr->addr_bytes[2], 1656 perm_addr->addr_bytes[3], 1657 perm_addr->addr_bytes[4], 1658 perm_addr->addr_bytes[5]); 1659 } 1660 1661 /* Copy the permanent MAC address */ 1662 rte_ether_addr_copy(perm_addr, ð_dev->data->mac_addrs[0]); 1663 1664 /* reset the hardware with the new settings */ 1665 diag = hw->mac.ops.start_hw(hw); 1666 switch (diag) { 1667 case 0: 1668 break; 1669 1670 default: 1671 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1672 return -EIO; 1673 } 1674 1675 rte_intr_callback_register(intr_handle, 1676 ixgbevf_dev_interrupt_handler, eth_dev); 1677 rte_intr_enable(intr_handle); 1678 ixgbevf_intr_enable(eth_dev); 1679 1680 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s", 1681 eth_dev->data->port_id, pci_dev->id.vendor_id, 1682 pci_dev->id.device_id, "ixgbe_mac_82599_vf"); 1683 1684 return 0; 1685 } 1686 1687 /* Virtual Function device uninit */ 1688 1689 static int 1690 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) 1691 { 1692 PMD_INIT_FUNC_TRACE(); 1693 1694 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1695 return 0; 1696 1697 ixgbevf_dev_close(eth_dev); 1698 1699 return 0; 1700 } 1701 1702 static int 1703 eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1704 struct rte_pci_device *pci_dev) 1705 { 1706 char name[RTE_ETH_NAME_MAX_LEN]; 1707 struct rte_eth_dev *pf_ethdev; 1708 struct rte_eth_devargs eth_da; 1709 int i, retval; 1710 1711 if (pci_dev->device.devargs) { 1712 retval = rte_eth_devargs_parse(pci_dev->device.devargs->args, 1713 ð_da); 1714 if (retval) 1715 return retval; 1716 } else 1717 memset(ð_da, 0, sizeof(eth_da)); 1718 1719 if (eth_da.nb_representor_ports > 0 && 1720 eth_da.type != RTE_ETH_REPRESENTOR_VF) { 1721 PMD_DRV_LOG(ERR, "unsupported representor type: %s\n", 1722 pci_dev->device.devargs->args); 1723 return -ENOTSUP; 1724 } 1725 1726 retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name, 1727 sizeof(struct ixgbe_adapter), 1728 eth_dev_pci_specific_init, pci_dev, 1729 eth_ixgbe_dev_init, NULL); 1730 1731 if (retval || eth_da.nb_representor_ports < 1) 1732 return retval; 1733 1734 pf_ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1735 if (pf_ethdev == NULL) 1736 return -ENODEV; 1737 1738 /* probe VF representor ports */ 1739 for (i = 0; i < eth_da.nb_representor_ports; i++) { 1740 struct ixgbe_vf_info *vfinfo; 1741 struct ixgbe_vf_representor representor; 1742 1743 vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA( 1744 pf_ethdev->data->dev_private); 1745 if (vfinfo == NULL) { 1746 PMD_DRV_LOG(ERR, 1747 "no virtual functions supported by PF"); 1748 break; 1749 } 1750 1751 representor.vf_id = eth_da.representor_ports[i]; 1752 representor.switch_domain_id = vfinfo->switch_domain_id; 1753 representor.pf_ethdev = pf_ethdev; 1754 1755 /* representor port net_bdf_port */ 1756 snprintf(name, sizeof(name), "net_%s_representor_%d", 1757 pci_dev->device.name, 1758 eth_da.representor_ports[i]); 1759 1760 retval = rte_eth_dev_create(&pci_dev->device, name, 1761 sizeof(struct ixgbe_vf_representor), NULL, NULL, 1762 ixgbe_vf_representor_init, &representor); 1763 1764 if (retval) 1765 PMD_DRV_LOG(ERR, "failed to create ixgbe vf " 1766 "representor %s.", name); 1767 } 1768 1769 return 0; 1770 } 1771 1772 static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev) 1773 { 1774 struct rte_eth_dev *ethdev; 1775 1776 ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1777 if (!ethdev) 1778 return 0; 1779 1780 if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) 1781 return rte_eth_dev_pci_generic_remove(pci_dev, 1782 ixgbe_vf_representor_uninit); 1783 else 1784 return rte_eth_dev_pci_generic_remove(pci_dev, 1785 eth_ixgbe_dev_uninit); 1786 } 1787 1788 static struct rte_pci_driver rte_ixgbe_pmd = { 1789 .id_table = pci_id_ixgbe_map, 1790 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1791 .probe = eth_ixgbe_pci_probe, 1792 .remove = eth_ixgbe_pci_remove, 1793 }; 1794 1795 static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1796 struct rte_pci_device *pci_dev) 1797 { 1798 return rte_eth_dev_pci_generic_probe(pci_dev, 1799 sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init); 1800 } 1801 1802 static int eth_ixgbevf_pci_remove(struct rte_pci_device *pci_dev) 1803 { 1804 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbevf_dev_uninit); 1805 } 1806 1807 /* 1808 * virtual function driver struct 1809 */ 1810 static struct rte_pci_driver rte_ixgbevf_pmd = { 1811 .id_table = pci_id_ixgbevf_map, 1812 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1813 .probe = eth_ixgbevf_pci_probe, 1814 .remove = eth_ixgbevf_pci_remove, 1815 }; 1816 1817 static int 1818 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1819 { 1820 struct ixgbe_hw *hw = 1821 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1822 struct ixgbe_vfta *shadow_vfta = 1823 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1824 uint32_t vfta; 1825 uint32_t vid_idx; 1826 uint32_t vid_bit; 1827 1828 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 1829 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 1830 vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx)); 1831 if (on) 1832 vfta |= vid_bit; 1833 else 1834 vfta &= ~vid_bit; 1835 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta); 1836 1837 /* update local VFTA copy */ 1838 shadow_vfta->vfta[vid_idx] = vfta; 1839 1840 return 0; 1841 } 1842 1843 static void 1844 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 1845 { 1846 if (on) 1847 ixgbe_vlan_hw_strip_enable(dev, queue); 1848 else 1849 ixgbe_vlan_hw_strip_disable(dev, queue); 1850 } 1851 1852 static int 1853 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 1854 enum rte_vlan_type vlan_type, 1855 uint16_t tpid) 1856 { 1857 struct ixgbe_hw *hw = 1858 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1859 int ret = 0; 1860 uint32_t reg; 1861 uint32_t qinq; 1862 1863 qinq = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1864 qinq &= IXGBE_DMATXCTL_GDV; 1865 1866 switch (vlan_type) { 1867 case ETH_VLAN_TYPE_INNER: 1868 if (qinq) { 1869 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1870 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1871 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1872 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1873 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1874 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1875 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1876 } else { 1877 ret = -ENOTSUP; 1878 PMD_DRV_LOG(ERR, "Inner type is not supported" 1879 " by single VLAN"); 1880 } 1881 break; 1882 case ETH_VLAN_TYPE_OUTER: 1883 if (qinq) { 1884 /* Only the high 16-bits is valid */ 1885 IXGBE_WRITE_REG(hw, IXGBE_EXVET, (uint32_t)tpid << 1886 IXGBE_EXVET_VET_EXT_SHIFT); 1887 } else { 1888 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1889 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1890 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1891 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1892 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1893 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1894 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1895 } 1896 1897 break; 1898 default: 1899 ret = -EINVAL; 1900 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type); 1901 break; 1902 } 1903 1904 return ret; 1905 } 1906 1907 void 1908 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev) 1909 { 1910 struct ixgbe_hw *hw = 1911 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1912 uint32_t vlnctrl; 1913 1914 PMD_INIT_FUNC_TRACE(); 1915 1916 /* Filter Table Disable */ 1917 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1918 vlnctrl &= ~IXGBE_VLNCTRL_VFE; 1919 1920 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1921 } 1922 1923 void 1924 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1925 { 1926 struct ixgbe_hw *hw = 1927 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1928 struct ixgbe_vfta *shadow_vfta = 1929 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1930 uint32_t vlnctrl; 1931 uint16_t i; 1932 1933 PMD_INIT_FUNC_TRACE(); 1934 1935 /* Filter Table Enable */ 1936 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1937 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 1938 vlnctrl |= IXGBE_VLNCTRL_VFE; 1939 1940 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1941 1942 /* write whatever is in local vfta copy */ 1943 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 1944 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]); 1945 } 1946 1947 static void 1948 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on) 1949 { 1950 struct ixgbe_hwstrip *hwstrip = 1951 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private); 1952 struct ixgbe_rx_queue *rxq; 1953 1954 if (queue >= IXGBE_MAX_RX_QUEUE_NUM) 1955 return; 1956 1957 if (on) 1958 IXGBE_SET_HWSTRIP(hwstrip, queue); 1959 else 1960 IXGBE_CLEAR_HWSTRIP(hwstrip, queue); 1961 1962 if (queue >= dev->data->nb_rx_queues) 1963 return; 1964 1965 rxq = dev->data->rx_queues[queue]; 1966 1967 if (on) { 1968 rxq->vlan_flags = PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1969 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 1970 } else { 1971 rxq->vlan_flags = PKT_RX_VLAN; 1972 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 1973 } 1974 } 1975 1976 static void 1977 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue) 1978 { 1979 struct ixgbe_hw *hw = 1980 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1981 uint32_t ctrl; 1982 1983 PMD_INIT_FUNC_TRACE(); 1984 1985 if (hw->mac.type == ixgbe_mac_82598EB) { 1986 /* No queue level support */ 1987 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 1988 return; 1989 } 1990 1991 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 1992 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 1993 ctrl &= ~IXGBE_RXDCTL_VME; 1994 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 1995 1996 /* record those setting for HW strip per queue */ 1997 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0); 1998 } 1999 2000 static void 2001 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue) 2002 { 2003 struct ixgbe_hw *hw = 2004 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2005 uint32_t ctrl; 2006 2007 PMD_INIT_FUNC_TRACE(); 2008 2009 if (hw->mac.type == ixgbe_mac_82598EB) { 2010 /* No queue level supported */ 2011 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2012 return; 2013 } 2014 2015 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2016 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2017 ctrl |= IXGBE_RXDCTL_VME; 2018 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2019 2020 /* record those setting for HW strip per queue */ 2021 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1); 2022 } 2023 2024 static void 2025 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2026 { 2027 struct ixgbe_hw *hw = 2028 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2029 uint32_t ctrl; 2030 2031 PMD_INIT_FUNC_TRACE(); 2032 2033 /* DMATXCTRL: Geric Double VLAN Disable */ 2034 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2035 ctrl &= ~IXGBE_DMATXCTL_GDV; 2036 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2037 2038 /* CTRL_EXT: Global Double VLAN Disable */ 2039 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2040 ctrl &= ~IXGBE_EXTENDED_VLAN; 2041 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2042 2043 } 2044 2045 static void 2046 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2047 { 2048 struct ixgbe_hw *hw = 2049 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2050 uint32_t ctrl; 2051 2052 PMD_INIT_FUNC_TRACE(); 2053 2054 /* DMATXCTRL: Geric Double VLAN Enable */ 2055 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2056 ctrl |= IXGBE_DMATXCTL_GDV; 2057 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2058 2059 /* CTRL_EXT: Global Double VLAN Enable */ 2060 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2061 ctrl |= IXGBE_EXTENDED_VLAN; 2062 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2063 2064 /* Clear pooling mode of PFVTCTL. It's required by X550. */ 2065 if (hw->mac.type == ixgbe_mac_X550 || 2066 hw->mac.type == ixgbe_mac_X550EM_x || 2067 hw->mac.type == ixgbe_mac_X550EM_a) { 2068 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 2069 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 2070 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 2071 } 2072 2073 /* 2074 * VET EXT field in the EXVET register = 0x8100 by default 2075 * So no need to change. Same to VT field of DMATXCTL register 2076 */ 2077 } 2078 2079 void 2080 ixgbe_vlan_hw_strip_config(struct rte_eth_dev *dev) 2081 { 2082 struct ixgbe_hw *hw = 2083 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2084 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2085 uint32_t ctrl; 2086 uint16_t i; 2087 struct ixgbe_rx_queue *rxq; 2088 bool on; 2089 2090 PMD_INIT_FUNC_TRACE(); 2091 2092 if (hw->mac.type == ixgbe_mac_82598EB) { 2093 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2094 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2095 ctrl |= IXGBE_VLNCTRL_VME; 2096 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2097 } else { 2098 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2099 ctrl &= ~IXGBE_VLNCTRL_VME; 2100 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2101 } 2102 } else { 2103 /* 2104 * Other 10G NIC, the VLAN strip can be setup 2105 * per queue in RXDCTL 2106 */ 2107 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2108 rxq = dev->data->rx_queues[i]; 2109 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 2110 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2111 ctrl |= IXGBE_RXDCTL_VME; 2112 on = TRUE; 2113 } else { 2114 ctrl &= ~IXGBE_RXDCTL_VME; 2115 on = FALSE; 2116 } 2117 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); 2118 2119 /* record those setting for HW strip per queue */ 2120 ixgbe_vlan_hw_strip_bitmap_set(dev, i, on); 2121 } 2122 } 2123 } 2124 2125 static void 2126 ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask) 2127 { 2128 uint16_t i; 2129 struct rte_eth_rxmode *rxmode; 2130 struct ixgbe_rx_queue *rxq; 2131 2132 if (mask & ETH_VLAN_STRIP_MASK) { 2133 rxmode = &dev->data->dev_conf.rxmode; 2134 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2135 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2136 rxq = dev->data->rx_queues[i]; 2137 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 2138 } 2139 else 2140 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2141 rxq = dev->data->rx_queues[i]; 2142 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 2143 } 2144 } 2145 } 2146 2147 static int 2148 ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask) 2149 { 2150 struct rte_eth_rxmode *rxmode; 2151 rxmode = &dev->data->dev_conf.rxmode; 2152 2153 if (mask & ETH_VLAN_STRIP_MASK) { 2154 ixgbe_vlan_hw_strip_config(dev); 2155 } 2156 2157 if (mask & ETH_VLAN_FILTER_MASK) { 2158 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2159 ixgbe_vlan_hw_filter_enable(dev); 2160 else 2161 ixgbe_vlan_hw_filter_disable(dev); 2162 } 2163 2164 if (mask & ETH_VLAN_EXTEND_MASK) { 2165 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2166 ixgbe_vlan_hw_extend_enable(dev); 2167 else 2168 ixgbe_vlan_hw_extend_disable(dev); 2169 } 2170 2171 return 0; 2172 } 2173 2174 static int 2175 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2176 { 2177 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 2178 2179 ixgbe_vlan_offload_config(dev, mask); 2180 2181 return 0; 2182 } 2183 2184 static void 2185 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2186 { 2187 struct ixgbe_hw *hw = 2188 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2189 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 2190 uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2191 2192 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 2193 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 2194 } 2195 2196 static int 2197 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) 2198 { 2199 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2200 2201 switch (nb_rx_q) { 2202 case 1: 2203 case 2: 2204 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS; 2205 break; 2206 case 4: 2207 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS; 2208 break; 2209 default: 2210 return -EINVAL; 2211 } 2212 2213 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 2214 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2215 RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = 2216 pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2217 return 0; 2218 } 2219 2220 static int 2221 ixgbe_check_mq_mode(struct rte_eth_dev *dev) 2222 { 2223 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 2224 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2225 uint16_t nb_rx_q = dev->data->nb_rx_queues; 2226 uint16_t nb_tx_q = dev->data->nb_tx_queues; 2227 2228 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 2229 /* check multi-queue mode */ 2230 switch (dev_conf->rxmode.mq_mode) { 2231 case ETH_MQ_RX_VMDQ_DCB: 2232 PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV"); 2233 break; 2234 case ETH_MQ_RX_VMDQ_DCB_RSS: 2235 /* DCB/RSS VMDQ in SRIOV mode, not implement yet */ 2236 PMD_INIT_LOG(ERR, "SRIOV active," 2237 " unsupported mq_mode rx %d.", 2238 dev_conf->rxmode.mq_mode); 2239 return -EINVAL; 2240 case ETH_MQ_RX_RSS: 2241 case ETH_MQ_RX_VMDQ_RSS: 2242 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS; 2243 if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) 2244 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) { 2245 PMD_INIT_LOG(ERR, "SRIOV is active," 2246 " invalid queue number" 2247 " for VMDQ RSS, allowed" 2248 " value are 1, 2 or 4."); 2249 return -EINVAL; 2250 } 2251 break; 2252 case ETH_MQ_RX_VMDQ_ONLY: 2253 case ETH_MQ_RX_NONE: 2254 /* if nothing mq mode configure, use default scheme */ 2255 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 2256 break; 2257 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/ 2258 /* SRIOV only works in VMDq enable mode */ 2259 PMD_INIT_LOG(ERR, "SRIOV is active," 2260 " wrong mq_mode rx %d.", 2261 dev_conf->rxmode.mq_mode); 2262 return -EINVAL; 2263 } 2264 2265 switch (dev_conf->txmode.mq_mode) { 2266 case ETH_MQ_TX_VMDQ_DCB: 2267 PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV"); 2268 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2269 break; 2270 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */ 2271 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY; 2272 break; 2273 } 2274 2275 /* check valid queue number */ 2276 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) || 2277 (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) { 2278 PMD_INIT_LOG(ERR, "SRIOV is active," 2279 " nb_rx_q=%d nb_tx_q=%d queue number" 2280 " must be less than or equal to %d.", 2281 nb_rx_q, nb_tx_q, 2282 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool); 2283 return -EINVAL; 2284 } 2285 } else { 2286 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) { 2287 PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is" 2288 " not supported."); 2289 return -EINVAL; 2290 } 2291 /* check configuration for vmdb+dcb mode */ 2292 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) { 2293 const struct rte_eth_vmdq_dcb_conf *conf; 2294 2295 if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2296 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.", 2297 IXGBE_VMDQ_DCB_NB_QUEUES); 2298 return -EINVAL; 2299 } 2300 conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf; 2301 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2302 conf->nb_queue_pools == ETH_32_POOLS)) { 2303 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2304 " nb_queue_pools must be %d or %d.", 2305 ETH_16_POOLS, ETH_32_POOLS); 2306 return -EINVAL; 2307 } 2308 } 2309 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { 2310 const struct rte_eth_vmdq_dcb_tx_conf *conf; 2311 2312 if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2313 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d", 2314 IXGBE_VMDQ_DCB_NB_QUEUES); 2315 return -EINVAL; 2316 } 2317 conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2318 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2319 conf->nb_queue_pools == ETH_32_POOLS)) { 2320 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2321 " nb_queue_pools != %d and" 2322 " nb_queue_pools != %d.", 2323 ETH_16_POOLS, ETH_32_POOLS); 2324 return -EINVAL; 2325 } 2326 } 2327 2328 /* For DCB mode check our configuration before we go further */ 2329 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) { 2330 const struct rte_eth_dcb_rx_conf *conf; 2331 2332 conf = &dev_conf->rx_adv_conf.dcb_rx_conf; 2333 if (!(conf->nb_tcs == ETH_4_TCS || 2334 conf->nb_tcs == ETH_8_TCS)) { 2335 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2336 " and nb_tcs != %d.", 2337 ETH_4_TCS, ETH_8_TCS); 2338 return -EINVAL; 2339 } 2340 } 2341 2342 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { 2343 const struct rte_eth_dcb_tx_conf *conf; 2344 2345 conf = &dev_conf->tx_adv_conf.dcb_tx_conf; 2346 if (!(conf->nb_tcs == ETH_4_TCS || 2347 conf->nb_tcs == ETH_8_TCS)) { 2348 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2349 " and nb_tcs != %d.", 2350 ETH_4_TCS, ETH_8_TCS); 2351 return -EINVAL; 2352 } 2353 } 2354 2355 /* 2356 * When DCB/VT is off, maximum number of queues changes, 2357 * except for 82598EB, which remains constant. 2358 */ 2359 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 2360 hw->mac.type != ixgbe_mac_82598EB) { 2361 if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) { 2362 PMD_INIT_LOG(ERR, 2363 "Neither VT nor DCB are enabled, " 2364 "nb_tx_q > %d.", 2365 IXGBE_NONE_MODE_TX_NB_QUEUES); 2366 return -EINVAL; 2367 } 2368 } 2369 } 2370 return 0; 2371 } 2372 2373 static int 2374 ixgbe_dev_configure(struct rte_eth_dev *dev) 2375 { 2376 struct ixgbe_interrupt *intr = 2377 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2378 struct ixgbe_adapter *adapter = dev->data->dev_private; 2379 int ret; 2380 2381 PMD_INIT_FUNC_TRACE(); 2382 2383 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 2384 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 2385 2386 /* multipe queue mode checking */ 2387 ret = ixgbe_check_mq_mode(dev); 2388 if (ret != 0) { 2389 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.", 2390 ret); 2391 return ret; 2392 } 2393 2394 /* set flag to update link status after init */ 2395 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2396 2397 /* 2398 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 2399 * allocation or vector Rx preconditions we will reset it. 2400 */ 2401 adapter->rx_bulk_alloc_allowed = true; 2402 adapter->rx_vec_allowed = true; 2403 2404 return 0; 2405 } 2406 2407 static void 2408 ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) 2409 { 2410 struct ixgbe_hw *hw = 2411 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2412 struct ixgbe_interrupt *intr = 2413 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2414 uint32_t gpie; 2415 2416 /* only set up it on X550EM_X */ 2417 if (hw->mac.type == ixgbe_mac_X550EM_x) { 2418 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 2419 gpie |= IXGBE_SDP0_GPIEN_X550EM_x; 2420 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 2421 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 2422 intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x; 2423 } 2424 } 2425 2426 int 2427 ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, 2428 uint16_t tx_rate, uint64_t q_msk) 2429 { 2430 struct ixgbe_hw *hw; 2431 struct ixgbe_vf_info *vfinfo; 2432 struct rte_eth_link link; 2433 uint8_t nb_q_per_pool; 2434 uint32_t queue_stride; 2435 uint32_t queue_idx, idx = 0, vf_idx; 2436 uint32_t queue_end; 2437 uint16_t total_rate = 0; 2438 struct rte_pci_device *pci_dev; 2439 int ret; 2440 2441 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2442 ret = rte_eth_link_get_nowait(dev->data->port_id, &link); 2443 if (ret < 0) 2444 return ret; 2445 2446 if (vf >= pci_dev->max_vfs) 2447 return -EINVAL; 2448 2449 if (tx_rate > link.link_speed) 2450 return -EINVAL; 2451 2452 if (q_msk == 0) 2453 return 0; 2454 2455 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2456 vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); 2457 nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2458 queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2459 queue_idx = vf * queue_stride; 2460 queue_end = queue_idx + nb_q_per_pool - 1; 2461 if (queue_end >= hw->mac.max_tx_queues) 2462 return -EINVAL; 2463 2464 if (vfinfo) { 2465 for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) { 2466 if (vf_idx == vf) 2467 continue; 2468 for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); 2469 idx++) 2470 total_rate += vfinfo[vf_idx].tx_rate[idx]; 2471 } 2472 } else { 2473 return -EINVAL; 2474 } 2475 2476 /* Store tx_rate for this vf. */ 2477 for (idx = 0; idx < nb_q_per_pool; idx++) { 2478 if (((uint64_t)0x1 << idx) & q_msk) { 2479 if (vfinfo[vf].tx_rate[idx] != tx_rate) 2480 vfinfo[vf].tx_rate[idx] = tx_rate; 2481 total_rate += tx_rate; 2482 } 2483 } 2484 2485 if (total_rate > dev->data->dev_link.link_speed) { 2486 /* Reset stored TX rate of the VF if it causes exceed 2487 * link speed. 2488 */ 2489 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); 2490 return -EINVAL; 2491 } 2492 2493 /* Set RTTBCNRC of each queue/pool for vf X */ 2494 for (; queue_idx <= queue_end; queue_idx++) { 2495 if (0x1 & q_msk) 2496 ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); 2497 q_msk = q_msk >> 1; 2498 } 2499 2500 return 0; 2501 } 2502 2503 static int 2504 ixgbe_flow_ctrl_enable(struct rte_eth_dev *dev, struct ixgbe_hw *hw) 2505 { 2506 struct ixgbe_adapter *adapter = dev->data->dev_private; 2507 int err; 2508 uint32_t mflcn; 2509 2510 ixgbe_setup_fc(hw); 2511 2512 err = ixgbe_fc_enable(hw); 2513 2514 /* Not negotiated is not an error case */ 2515 if (err == IXGBE_SUCCESS || err == IXGBE_ERR_FC_NOT_NEGOTIATED) { 2516 /* 2517 *check if we want to forward MAC frames - driver doesn't 2518 *have native capability to do that, 2519 *so we'll write the registers ourselves 2520 */ 2521 2522 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 2523 2524 /* set or clear MFLCN.PMCF bit depending on configuration */ 2525 if (adapter->mac_ctrl_frame_fwd != 0) 2526 mflcn |= IXGBE_MFLCN_PMCF; 2527 else 2528 mflcn &= ~IXGBE_MFLCN_PMCF; 2529 2530 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn); 2531 IXGBE_WRITE_FLUSH(hw); 2532 2533 return 0; 2534 } 2535 return err; 2536 } 2537 2538 /* 2539 * Configure device link speed and setup link. 2540 * It returns 0 on success. 2541 */ 2542 static int 2543 ixgbe_dev_start(struct rte_eth_dev *dev) 2544 { 2545 struct ixgbe_hw *hw = 2546 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2547 struct ixgbe_vf_info *vfinfo = 2548 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2549 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2550 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2551 uint32_t intr_vector = 0; 2552 int err; 2553 bool link_up = false, negotiate = 0; 2554 uint32_t speed = 0; 2555 uint32_t allowed_speeds = 0; 2556 int mask = 0; 2557 int status; 2558 uint16_t vf, idx; 2559 uint32_t *link_speeds; 2560 struct ixgbe_tm_conf *tm_conf = 2561 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2562 struct ixgbe_macsec_setting *macsec_setting = 2563 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 2564 2565 PMD_INIT_FUNC_TRACE(); 2566 2567 /* Stop the link setup handler before resetting the HW. */ 2568 ixgbe_dev_wait_setup_link_complete(dev, 0); 2569 2570 /* disable uio/vfio intr/eventfd mapping */ 2571 rte_intr_disable(intr_handle); 2572 2573 /* stop adapter */ 2574 hw->adapter_stopped = 0; 2575 ixgbe_stop_adapter(hw); 2576 2577 /* reinitialize adapter 2578 * this calls reset and start 2579 */ 2580 status = ixgbe_pf_reset_hw(hw); 2581 if (status != 0) 2582 return -1; 2583 hw->mac.ops.start_hw(hw); 2584 hw->mac.get_link_status = true; 2585 2586 /* configure PF module if SRIOV enabled */ 2587 ixgbe_pf_host_configure(dev); 2588 2589 ixgbe_dev_phy_intr_setup(dev); 2590 2591 /* check and configure queue intr-vector mapping */ 2592 if ((rte_intr_cap_multiple(intr_handle) || 2593 !RTE_ETH_DEV_SRIOV(dev).active) && 2594 dev->data->dev_conf.intr_conf.rxq != 0) { 2595 intr_vector = dev->data->nb_rx_queues; 2596 if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) { 2597 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 2598 IXGBE_MAX_INTR_QUEUE_NUM); 2599 return -ENOTSUP; 2600 } 2601 if (rte_intr_efd_enable(intr_handle, intr_vector)) 2602 return -1; 2603 } 2604 2605 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 2606 intr_handle->intr_vec = 2607 rte_zmalloc("intr_vec", 2608 dev->data->nb_rx_queues * sizeof(int), 0); 2609 if (intr_handle->intr_vec == NULL) { 2610 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 2611 " intr_vec", dev->data->nb_rx_queues); 2612 return -ENOMEM; 2613 } 2614 } 2615 2616 /* confiugre msix for sleep until rx interrupt */ 2617 ixgbe_configure_msix(dev); 2618 2619 /* initialize transmission unit */ 2620 ixgbe_dev_tx_init(dev); 2621 2622 /* This can fail when allocating mbufs for descriptor rings */ 2623 err = ixgbe_dev_rx_init(dev); 2624 if (err) { 2625 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 2626 goto error; 2627 } 2628 2629 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 2630 ETH_VLAN_EXTEND_MASK; 2631 err = ixgbe_vlan_offload_config(dev, mask); 2632 if (err) { 2633 PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); 2634 goto error; 2635 } 2636 2637 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 2638 /* Enable vlan filtering for VMDq */ 2639 ixgbe_vmdq_vlan_hw_filter_enable(dev); 2640 } 2641 2642 /* Configure DCB hw */ 2643 ixgbe_configure_dcb(dev); 2644 2645 if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { 2646 err = ixgbe_fdir_configure(dev); 2647 if (err) 2648 goto error; 2649 } 2650 2651 /* Restore vf rate limit */ 2652 if (vfinfo != NULL) { 2653 for (vf = 0; vf < pci_dev->max_vfs; vf++) 2654 for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) 2655 if (vfinfo[vf].tx_rate[idx] != 0) 2656 ixgbe_set_vf_rate_limit( 2657 dev, vf, 2658 vfinfo[vf].tx_rate[idx], 2659 1 << idx); 2660 } 2661 2662 ixgbe_restore_statistics_mapping(dev); 2663 2664 err = ixgbe_flow_ctrl_enable(dev, hw); 2665 if (err < 0) { 2666 PMD_INIT_LOG(ERR, "enable flow ctrl err"); 2667 goto error; 2668 } 2669 2670 err = ixgbe_dev_rxtx_start(dev); 2671 if (err < 0) { 2672 PMD_INIT_LOG(ERR, "Unable to start rxtx queues"); 2673 goto error; 2674 } 2675 2676 /* Skip link setup if loopback mode is enabled. */ 2677 if (dev->data->dev_conf.lpbk_mode != 0) { 2678 err = ixgbe_check_supported_loopback_mode(dev); 2679 if (err < 0) { 2680 PMD_INIT_LOG(ERR, "Unsupported loopback mode"); 2681 goto error; 2682 } else { 2683 goto skip_link_setup; 2684 } 2685 } 2686 2687 if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) { 2688 err = hw->mac.ops.setup_sfp(hw); 2689 if (err) 2690 goto error; 2691 } 2692 2693 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2694 /* Turn on the copper */ 2695 ixgbe_set_phy_power(hw, true); 2696 } else { 2697 /* Turn on the laser */ 2698 ixgbe_enable_tx_laser(hw); 2699 } 2700 2701 err = ixgbe_check_link(hw, &speed, &link_up, 0); 2702 if (err) 2703 goto error; 2704 dev->data->dev_link.link_status = link_up; 2705 2706 err = ixgbe_get_link_capabilities(hw, &speed, &negotiate); 2707 if (err) 2708 goto error; 2709 2710 switch (hw->mac.type) { 2711 case ixgbe_mac_X550: 2712 case ixgbe_mac_X550EM_x: 2713 case ixgbe_mac_X550EM_a: 2714 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2715 ETH_LINK_SPEED_2_5G | ETH_LINK_SPEED_5G | 2716 ETH_LINK_SPEED_10G; 2717 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 2718 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 2719 allowed_speeds = ETH_LINK_SPEED_10M | 2720 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 2721 break; 2722 default: 2723 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2724 ETH_LINK_SPEED_10G; 2725 } 2726 2727 link_speeds = &dev->data->dev_conf.link_speeds; 2728 2729 /* Ignore autoneg flag bit and check the validity of 2730 * link_speed 2731 */ 2732 if (((*link_speeds) >> 1) & ~(allowed_speeds >> 1)) { 2733 PMD_INIT_LOG(ERR, "Invalid link setting"); 2734 goto error; 2735 } 2736 2737 speed = 0x0; 2738 if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { 2739 switch (hw->mac.type) { 2740 case ixgbe_mac_82598EB: 2741 speed = IXGBE_LINK_SPEED_82598_AUTONEG; 2742 break; 2743 case ixgbe_mac_82599EB: 2744 case ixgbe_mac_X540: 2745 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2746 break; 2747 case ixgbe_mac_X550: 2748 case ixgbe_mac_X550EM_x: 2749 case ixgbe_mac_X550EM_a: 2750 speed = IXGBE_LINK_SPEED_X550_AUTONEG; 2751 break; 2752 default: 2753 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2754 } 2755 } else { 2756 if (*link_speeds & ETH_LINK_SPEED_10G) 2757 speed |= IXGBE_LINK_SPEED_10GB_FULL; 2758 if (*link_speeds & ETH_LINK_SPEED_5G) 2759 speed |= IXGBE_LINK_SPEED_5GB_FULL; 2760 if (*link_speeds & ETH_LINK_SPEED_2_5G) 2761 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 2762 if (*link_speeds & ETH_LINK_SPEED_1G) 2763 speed |= IXGBE_LINK_SPEED_1GB_FULL; 2764 if (*link_speeds & ETH_LINK_SPEED_100M) 2765 speed |= IXGBE_LINK_SPEED_100_FULL; 2766 if (*link_speeds & ETH_LINK_SPEED_10M) 2767 speed |= IXGBE_LINK_SPEED_10_FULL; 2768 } 2769 2770 err = ixgbe_setup_link(hw, speed, link_up); 2771 if (err) 2772 goto error; 2773 2774 skip_link_setup: 2775 2776 if (rte_intr_allow_others(intr_handle)) { 2777 /* check if lsc interrupt is enabled */ 2778 if (dev->data->dev_conf.intr_conf.lsc != 0) 2779 ixgbe_dev_lsc_interrupt_setup(dev, TRUE); 2780 else 2781 ixgbe_dev_lsc_interrupt_setup(dev, FALSE); 2782 ixgbe_dev_macsec_interrupt_setup(dev); 2783 } else { 2784 rte_intr_callback_unregister(intr_handle, 2785 ixgbe_dev_interrupt_handler, dev); 2786 if (dev->data->dev_conf.intr_conf.lsc != 0) 2787 PMD_INIT_LOG(INFO, "lsc won't enable because of" 2788 " no intr multiplex"); 2789 } 2790 2791 /* check if rxq interrupt is enabled */ 2792 if (dev->data->dev_conf.intr_conf.rxq != 0 && 2793 rte_intr_dp_is_en(intr_handle)) 2794 ixgbe_dev_rxq_interrupt_setup(dev); 2795 2796 /* enable uio/vfio intr/eventfd mapping */ 2797 rte_intr_enable(intr_handle); 2798 2799 /* resume enabled intr since hw reset */ 2800 ixgbe_enable_intr(dev); 2801 ixgbe_l2_tunnel_conf(dev); 2802 ixgbe_filter_restore(dev); 2803 2804 if (tm_conf->root && !tm_conf->committed) 2805 PMD_DRV_LOG(WARNING, 2806 "please call hierarchy_commit() " 2807 "before starting the port"); 2808 2809 /* wait for the controller to acquire link */ 2810 err = ixgbe_wait_for_link_up(hw); 2811 if (err) 2812 goto error; 2813 2814 /* 2815 * Update link status right before return, because it may 2816 * start link configuration process in a separate thread. 2817 */ 2818 ixgbe_dev_link_update(dev, 0); 2819 2820 /* setup the macsec setting register */ 2821 if (macsec_setting->offload_en) 2822 ixgbe_dev_macsec_register_enable(dev, macsec_setting); 2823 2824 return 0; 2825 2826 error: 2827 PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err); 2828 ixgbe_dev_clear_queues(dev); 2829 return -EIO; 2830 } 2831 2832 /* 2833 * Stop device: disable rx and tx functions to allow for reconfiguring. 2834 */ 2835 static int 2836 ixgbe_dev_stop(struct rte_eth_dev *dev) 2837 { 2838 struct rte_eth_link link; 2839 struct ixgbe_adapter *adapter = dev->data->dev_private; 2840 struct ixgbe_hw *hw = 2841 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2842 struct ixgbe_vf_info *vfinfo = 2843 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2844 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2845 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2846 int vf; 2847 struct ixgbe_tm_conf *tm_conf = 2848 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2849 2850 if (hw->adapter_stopped) 2851 return 0; 2852 2853 PMD_INIT_FUNC_TRACE(); 2854 2855 ixgbe_dev_wait_setup_link_complete(dev, 0); 2856 2857 /* disable interrupts */ 2858 ixgbe_disable_intr(hw); 2859 2860 /* reset the NIC */ 2861 ixgbe_pf_reset_hw(hw); 2862 hw->adapter_stopped = 0; 2863 2864 /* stop adapter */ 2865 ixgbe_stop_adapter(hw); 2866 2867 for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) 2868 vfinfo[vf].clear_to_send = false; 2869 2870 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2871 /* Turn off the copper */ 2872 ixgbe_set_phy_power(hw, false); 2873 } else { 2874 /* Turn off the laser */ 2875 ixgbe_disable_tx_laser(hw); 2876 } 2877 2878 ixgbe_dev_clear_queues(dev); 2879 2880 /* Clear stored conf */ 2881 dev->data->scattered_rx = 0; 2882 dev->data->lro = 0; 2883 2884 /* Clear recorded link status */ 2885 memset(&link, 0, sizeof(link)); 2886 rte_eth_linkstatus_set(dev, &link); 2887 2888 if (!rte_intr_allow_others(intr_handle)) 2889 /* resume to the default handler */ 2890 rte_intr_callback_register(intr_handle, 2891 ixgbe_dev_interrupt_handler, 2892 (void *)dev); 2893 2894 /* Clean datapath event and queue/vec mapping */ 2895 rte_intr_efd_disable(intr_handle); 2896 if (intr_handle->intr_vec != NULL) { 2897 rte_free(intr_handle->intr_vec); 2898 intr_handle->intr_vec = NULL; 2899 } 2900 2901 /* reset hierarchy commit */ 2902 tm_conf->committed = false; 2903 2904 adapter->rss_reta_updated = 0; 2905 2906 hw->adapter_stopped = true; 2907 dev->data->dev_started = 0; 2908 2909 return 0; 2910 } 2911 2912 /* 2913 * Set device link up: enable tx. 2914 */ 2915 static int 2916 ixgbe_dev_set_link_up(struct rte_eth_dev *dev) 2917 { 2918 struct ixgbe_hw *hw = 2919 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2920 if (hw->mac.type == ixgbe_mac_82599EB) { 2921 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2922 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2923 /* Not suported in bypass mode */ 2924 PMD_INIT_LOG(ERR, "Set link up is not supported " 2925 "by device id 0x%x", hw->device_id); 2926 return -ENOTSUP; 2927 } 2928 #endif 2929 } 2930 2931 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2932 /* Turn on the copper */ 2933 ixgbe_set_phy_power(hw, true); 2934 } else { 2935 /* Turn on the laser */ 2936 ixgbe_enable_tx_laser(hw); 2937 ixgbe_dev_link_update(dev, 0); 2938 } 2939 2940 return 0; 2941 } 2942 2943 /* 2944 * Set device link down: disable tx. 2945 */ 2946 static int 2947 ixgbe_dev_set_link_down(struct rte_eth_dev *dev) 2948 { 2949 struct ixgbe_hw *hw = 2950 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2951 if (hw->mac.type == ixgbe_mac_82599EB) { 2952 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2953 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2954 /* Not suported in bypass mode */ 2955 PMD_INIT_LOG(ERR, "Set link down is not supported " 2956 "by device id 0x%x", hw->device_id); 2957 return -ENOTSUP; 2958 } 2959 #endif 2960 } 2961 2962 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2963 /* Turn off the copper */ 2964 ixgbe_set_phy_power(hw, false); 2965 } else { 2966 /* Turn off the laser */ 2967 ixgbe_disable_tx_laser(hw); 2968 ixgbe_dev_link_update(dev, 0); 2969 } 2970 2971 return 0; 2972 } 2973 2974 /* 2975 * Reset and stop device. 2976 */ 2977 static int 2978 ixgbe_dev_close(struct rte_eth_dev *dev) 2979 { 2980 struct ixgbe_hw *hw = 2981 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2982 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2983 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2984 int retries = 0; 2985 int ret; 2986 2987 PMD_INIT_FUNC_TRACE(); 2988 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2989 return 0; 2990 2991 ixgbe_pf_reset_hw(hw); 2992 2993 ret = ixgbe_dev_stop(dev); 2994 2995 ixgbe_dev_free_queues(dev); 2996 2997 ixgbe_disable_pcie_master(hw); 2998 2999 /* reprogram the RAR[0] in case user changed it. */ 3000 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 3001 3002 /* Unlock any pending hardware semaphore */ 3003 ixgbe_swfw_lock_reset(hw); 3004 3005 /* disable uio intr before callback unregister */ 3006 rte_intr_disable(intr_handle); 3007 3008 do { 3009 ret = rte_intr_callback_unregister(intr_handle, 3010 ixgbe_dev_interrupt_handler, dev); 3011 if (ret >= 0 || ret == -ENOENT) { 3012 break; 3013 } else if (ret != -EAGAIN) { 3014 PMD_INIT_LOG(ERR, 3015 "intr callback unregister failed: %d", 3016 ret); 3017 } 3018 rte_delay_ms(100); 3019 } while (retries++ < (10 + IXGBE_LINK_UP_TIME)); 3020 3021 /* cancel the delay handler before remove dev */ 3022 rte_eal_alarm_cancel(ixgbe_dev_interrupt_delayed_handler, dev); 3023 3024 /* uninitialize PF if max_vfs not zero */ 3025 ixgbe_pf_host_uninit(dev); 3026 3027 /* remove all the fdir filters & hash */ 3028 ixgbe_fdir_filter_uninit(dev); 3029 3030 /* remove all the L2 tunnel filters & hash */ 3031 ixgbe_l2_tn_filter_uninit(dev); 3032 3033 /* Remove all ntuple filters of the device */ 3034 ixgbe_ntuple_filter_uninit(dev); 3035 3036 /* clear all the filters list */ 3037 ixgbe_filterlist_flush(); 3038 3039 /* Remove all Traffic Manager configuration */ 3040 ixgbe_tm_conf_uninit(dev); 3041 3042 #ifdef RTE_LIB_SECURITY 3043 rte_free(dev->security_ctx); 3044 #endif 3045 3046 return ret; 3047 } 3048 3049 /* 3050 * Reset PF device. 3051 */ 3052 static int 3053 ixgbe_dev_reset(struct rte_eth_dev *dev) 3054 { 3055 int ret; 3056 3057 /* When a DPDK PMD PF begin to reset PF port, it should notify all 3058 * its VF to make them align with it. The detailed notification 3059 * mechanism is PMD specific. As to ixgbe PF, it is rather complex. 3060 * To avoid unexpected behavior in VF, currently reset of PF with 3061 * SR-IOV activation is not supported. It might be supported later. 3062 */ 3063 if (dev->data->sriov.active) 3064 return -ENOTSUP; 3065 3066 ret = eth_ixgbe_dev_uninit(dev); 3067 if (ret) 3068 return ret; 3069 3070 ret = eth_ixgbe_dev_init(dev, NULL); 3071 3072 return ret; 3073 } 3074 3075 static void 3076 ixgbe_read_stats_registers(struct ixgbe_hw *hw, 3077 struct ixgbe_hw_stats *hw_stats, 3078 struct ixgbe_macsec_stats *macsec_stats, 3079 uint64_t *total_missed_rx, uint64_t *total_qbrc, 3080 uint64_t *total_qprc, uint64_t *total_qprdc) 3081 { 3082 uint32_t bprc, lxon, lxoff, total; 3083 uint32_t delta_gprc = 0; 3084 unsigned i; 3085 /* Workaround for RX byte count not including CRC bytes when CRC 3086 * strip is enabled. CRC bytes are removed from counters when crc_strip 3087 * is disabled. 3088 */ 3089 int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) & 3090 IXGBE_HLREG0_RXCRCSTRP); 3091 3092 hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3093 hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3094 hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3095 hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3096 3097 for (i = 0; i < 8; i++) { 3098 uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3099 3100 /* global total per queue */ 3101 hw_stats->mpc[i] += mp; 3102 /* Running comprehensive total for stats display */ 3103 *total_missed_rx += hw_stats->mpc[i]; 3104 if (hw->mac.type == ixgbe_mac_82598EB) { 3105 hw_stats->rnbc[i] += 3106 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3107 hw_stats->pxonrxc[i] += 3108 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 3109 hw_stats->pxoffrxc[i] += 3110 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 3111 } else { 3112 hw_stats->pxonrxc[i] += 3113 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 3114 hw_stats->pxoffrxc[i] += 3115 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 3116 hw_stats->pxon2offc[i] += 3117 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 3118 } 3119 hw_stats->pxontxc[i] += 3120 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 3121 hw_stats->pxofftxc[i] += 3122 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 3123 } 3124 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3125 uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3126 uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3127 uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3128 3129 delta_gprc += delta_qprc; 3130 3131 hw_stats->qprc[i] += delta_qprc; 3132 hw_stats->qptc[i] += delta_qptc; 3133 3134 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 3135 hw_stats->qbrc[i] += 3136 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32); 3137 if (crc_strip == 0) 3138 hw_stats->qbrc[i] -= delta_qprc * RTE_ETHER_CRC_LEN; 3139 3140 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 3141 hw_stats->qbtc[i] += 3142 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32); 3143 3144 hw_stats->qprdc[i] += delta_qprdc; 3145 *total_qprdc += hw_stats->qprdc[i]; 3146 3147 *total_qprc += hw_stats->qprc[i]; 3148 *total_qbrc += hw_stats->qbrc[i]; 3149 } 3150 hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3151 hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3152 hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3153 3154 /* 3155 * An errata states that gprc actually counts good + missed packets: 3156 * Workaround to set gprc to summated queue packet receives 3157 */ 3158 hw_stats->gprc = *total_qprc; 3159 3160 if (hw->mac.type != ixgbe_mac_82598EB) { 3161 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 3162 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3163 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 3164 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3165 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 3166 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3167 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3168 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3169 } else { 3170 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3171 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3172 /* 82598 only has a counter in the high register */ 3173 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3174 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3175 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3176 } 3177 uint64_t old_tpr = hw_stats->tpr; 3178 3179 hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3180 hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3181 3182 if (crc_strip == 0) 3183 hw_stats->gorc -= delta_gprc * RTE_ETHER_CRC_LEN; 3184 3185 uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC); 3186 hw_stats->gptc += delta_gptc; 3187 hw_stats->gotc -= delta_gptc * RTE_ETHER_CRC_LEN; 3188 hw_stats->tor -= (hw_stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 3189 3190 /* 3191 * Workaround: mprc hardware is incorrectly counting 3192 * broadcasts, so for now we subtract those. 3193 */ 3194 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3195 hw_stats->bprc += bprc; 3196 hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3197 if (hw->mac.type == ixgbe_mac_82598EB) 3198 hw_stats->mprc -= bprc; 3199 3200 hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3201 hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3202 hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3203 hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3204 hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3205 hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3206 3207 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3208 hw_stats->lxontxc += lxon; 3209 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3210 hw_stats->lxofftxc += lxoff; 3211 total = lxon + lxoff; 3212 3213 hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3214 hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3215 hw_stats->gptc -= total; 3216 hw_stats->mptc -= total; 3217 hw_stats->ptc64 -= total; 3218 hw_stats->gotc -= total * RTE_ETHER_MIN_LEN; 3219 3220 hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3221 hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3222 hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3223 hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3224 hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3225 hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3226 hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3227 hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3228 hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3229 hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3230 hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3231 hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3232 hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3233 hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3234 hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3235 hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3236 /* Only read FCOE on 82599 */ 3237 if (hw->mac.type != ixgbe_mac_82598EB) { 3238 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3239 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3240 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3241 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3242 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3243 } 3244 3245 /* Flow Director Stats registers */ 3246 if (hw->mac.type != ixgbe_mac_82598EB) { 3247 hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 3248 hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 3249 hw_stats->fdirustat_add += IXGBE_READ_REG(hw, 3250 IXGBE_FDIRUSTAT) & 0xFFFF; 3251 hw_stats->fdirustat_remove += (IXGBE_READ_REG(hw, 3252 IXGBE_FDIRUSTAT) >> 16) & 0xFFFF; 3253 hw_stats->fdirfstat_fadd += IXGBE_READ_REG(hw, 3254 IXGBE_FDIRFSTAT) & 0xFFFF; 3255 hw_stats->fdirfstat_fremove += (IXGBE_READ_REG(hw, 3256 IXGBE_FDIRFSTAT) >> 16) & 0xFFFF; 3257 } 3258 /* MACsec Stats registers */ 3259 macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT); 3260 macsec_stats->out_pkts_encrypted += 3261 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE); 3262 macsec_stats->out_pkts_protected += 3263 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP); 3264 macsec_stats->out_octets_encrypted += 3265 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE); 3266 macsec_stats->out_octets_protected += 3267 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP); 3268 macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT); 3269 macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD); 3270 macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI); 3271 macsec_stats->in_pkts_unknownsci += 3272 IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI); 3273 macsec_stats->in_octets_decrypted += 3274 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD); 3275 macsec_stats->in_octets_validated += 3276 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV); 3277 macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH); 3278 macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY); 3279 macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE); 3280 for (i = 0; i < 2; i++) { 3281 macsec_stats->in_pkts_ok += 3282 IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i)); 3283 macsec_stats->in_pkts_invalid += 3284 IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i)); 3285 macsec_stats->in_pkts_notvalid += 3286 IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i)); 3287 } 3288 macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA); 3289 macsec_stats->in_pkts_notusingsa += 3290 IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA); 3291 } 3292 3293 /* 3294 * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c 3295 */ 3296 static int 3297 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3298 { 3299 struct ixgbe_hw *hw = 3300 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3301 struct ixgbe_hw_stats *hw_stats = 3302 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3303 struct ixgbe_macsec_stats *macsec_stats = 3304 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3305 dev->data->dev_private); 3306 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3307 unsigned i; 3308 3309 total_missed_rx = 0; 3310 total_qbrc = 0; 3311 total_qprc = 0; 3312 total_qprdc = 0; 3313 3314 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3315 &total_qbrc, &total_qprc, &total_qprdc); 3316 3317 if (stats == NULL) 3318 return -EINVAL; 3319 3320 /* Fill out the rte_eth_stats statistics structure */ 3321 stats->ipackets = total_qprc; 3322 stats->ibytes = total_qbrc; 3323 stats->opackets = hw_stats->gptc; 3324 stats->obytes = hw_stats->gotc; 3325 3326 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3327 stats->q_ipackets[i] = hw_stats->qprc[i]; 3328 stats->q_opackets[i] = hw_stats->qptc[i]; 3329 stats->q_ibytes[i] = hw_stats->qbrc[i]; 3330 stats->q_obytes[i] = hw_stats->qbtc[i]; 3331 stats->q_errors[i] = hw_stats->qprdc[i]; 3332 } 3333 3334 /* Rx Errors */ 3335 stats->imissed = total_missed_rx; 3336 stats->ierrors = hw_stats->crcerrs + 3337 hw_stats->mspdc + 3338 hw_stats->rlec + 3339 hw_stats->ruc + 3340 hw_stats->roc + 3341 hw_stats->illerrc + 3342 hw_stats->errbc + 3343 hw_stats->rfc + 3344 hw_stats->fccrc + 3345 hw_stats->fclast; 3346 3347 /* 3348 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum 3349 * errors. 3350 */ 3351 if (hw->mac.type != ixgbe_mac_82599EB) 3352 stats->ierrors += hw_stats->xec; 3353 3354 /* Tx Errors */ 3355 stats->oerrors = 0; 3356 return 0; 3357 } 3358 3359 static int 3360 ixgbe_dev_stats_reset(struct rte_eth_dev *dev) 3361 { 3362 struct ixgbe_hw_stats *stats = 3363 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3364 3365 /* HW registers are cleared on read */ 3366 ixgbe_dev_stats_get(dev, NULL); 3367 3368 /* Reset software totals */ 3369 memset(stats, 0, sizeof(*stats)); 3370 3371 return 0; 3372 } 3373 3374 /* This function calculates the number of xstats based on the current config */ 3375 static unsigned 3376 ixgbe_xstats_calc_num(void) { 3377 return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS + 3378 (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) + 3379 (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES); 3380 } 3381 3382 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3383 struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int size) 3384 { 3385 const unsigned cnt_stats = ixgbe_xstats_calc_num(); 3386 unsigned stat, i, count; 3387 3388 if (xstats_names != NULL) { 3389 count = 0; 3390 3391 /* Note: limit >= cnt_stats checked upstream 3392 * in rte_eth_xstats_names() 3393 */ 3394 3395 /* Extended stats from ixgbe_hw_stats */ 3396 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3397 strlcpy(xstats_names[count].name, 3398 rte_ixgbe_stats_strings[i].name, 3399 sizeof(xstats_names[count].name)); 3400 count++; 3401 } 3402 3403 /* MACsec Stats */ 3404 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3405 strlcpy(xstats_names[count].name, 3406 rte_ixgbe_macsec_strings[i].name, 3407 sizeof(xstats_names[count].name)); 3408 count++; 3409 } 3410 3411 /* RX Priority Stats */ 3412 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3413 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3414 snprintf(xstats_names[count].name, 3415 sizeof(xstats_names[count].name), 3416 "rx_priority%u_%s", i, 3417 rte_ixgbe_rxq_strings[stat].name); 3418 count++; 3419 } 3420 } 3421 3422 /* TX Priority Stats */ 3423 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3424 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3425 snprintf(xstats_names[count].name, 3426 sizeof(xstats_names[count].name), 3427 "tx_priority%u_%s", i, 3428 rte_ixgbe_txq_strings[stat].name); 3429 count++; 3430 } 3431 } 3432 } 3433 return cnt_stats; 3434 } 3435 3436 static int ixgbe_dev_xstats_get_names_by_id( 3437 struct rte_eth_dev *dev, 3438 struct rte_eth_xstat_name *xstats_names, 3439 const uint64_t *ids, 3440 unsigned int limit) 3441 { 3442 if (!ids) { 3443 const unsigned int cnt_stats = ixgbe_xstats_calc_num(); 3444 unsigned int stat, i, count; 3445 3446 if (xstats_names != NULL) { 3447 count = 0; 3448 3449 /* Note: limit >= cnt_stats checked upstream 3450 * in rte_eth_xstats_names() 3451 */ 3452 3453 /* Extended stats from ixgbe_hw_stats */ 3454 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3455 strlcpy(xstats_names[count].name, 3456 rte_ixgbe_stats_strings[i].name, 3457 sizeof(xstats_names[count].name)); 3458 count++; 3459 } 3460 3461 /* MACsec Stats */ 3462 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3463 strlcpy(xstats_names[count].name, 3464 rte_ixgbe_macsec_strings[i].name, 3465 sizeof(xstats_names[count].name)); 3466 count++; 3467 } 3468 3469 /* RX Priority Stats */ 3470 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3471 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3472 snprintf(xstats_names[count].name, 3473 sizeof(xstats_names[count].name), 3474 "rx_priority%u_%s", i, 3475 rte_ixgbe_rxq_strings[stat].name); 3476 count++; 3477 } 3478 } 3479 3480 /* TX Priority Stats */ 3481 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3482 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3483 snprintf(xstats_names[count].name, 3484 sizeof(xstats_names[count].name), 3485 "tx_priority%u_%s", i, 3486 rte_ixgbe_txq_strings[stat].name); 3487 count++; 3488 } 3489 } 3490 } 3491 return cnt_stats; 3492 } 3493 3494 uint16_t i; 3495 uint16_t size = ixgbe_xstats_calc_num(); 3496 struct rte_eth_xstat_name xstats_names_copy[size]; 3497 3498 ixgbe_dev_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 3499 size); 3500 3501 for (i = 0; i < limit; i++) { 3502 if (ids[i] >= size) { 3503 PMD_INIT_LOG(ERR, "id value isn't valid"); 3504 return -1; 3505 } 3506 strcpy(xstats_names[i].name, 3507 xstats_names_copy[ids[i]].name); 3508 } 3509 return limit; 3510 } 3511 3512 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3513 struct rte_eth_xstat_name *xstats_names, unsigned limit) 3514 { 3515 unsigned i; 3516 3517 if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL) 3518 return -ENOMEM; 3519 3520 if (xstats_names != NULL) 3521 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) 3522 strlcpy(xstats_names[i].name, 3523 rte_ixgbevf_stats_strings[i].name, 3524 sizeof(xstats_names[i].name)); 3525 return IXGBEVF_NB_XSTATS; 3526 } 3527 3528 static int 3529 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3530 unsigned n) 3531 { 3532 struct ixgbe_hw *hw = 3533 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3534 struct ixgbe_hw_stats *hw_stats = 3535 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3536 struct ixgbe_macsec_stats *macsec_stats = 3537 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3538 dev->data->dev_private); 3539 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3540 unsigned i, stat, count = 0; 3541 3542 count = ixgbe_xstats_calc_num(); 3543 3544 if (n < count) 3545 return count; 3546 3547 total_missed_rx = 0; 3548 total_qbrc = 0; 3549 total_qprc = 0; 3550 total_qprdc = 0; 3551 3552 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3553 &total_qbrc, &total_qprc, &total_qprdc); 3554 3555 /* If this is a reset xstats is NULL, and we have cleared the 3556 * registers by reading them. 3557 */ 3558 if (!xstats) 3559 return 0; 3560 3561 /* Extended stats from ixgbe_hw_stats */ 3562 count = 0; 3563 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3564 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3565 rte_ixgbe_stats_strings[i].offset); 3566 xstats[count].id = count; 3567 count++; 3568 } 3569 3570 /* MACsec Stats */ 3571 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3572 xstats[count].value = *(uint64_t *)(((char *)macsec_stats) + 3573 rte_ixgbe_macsec_strings[i].offset); 3574 xstats[count].id = count; 3575 count++; 3576 } 3577 3578 /* RX Priority Stats */ 3579 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3580 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3581 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3582 rte_ixgbe_rxq_strings[stat].offset + 3583 (sizeof(uint64_t) * i)); 3584 xstats[count].id = count; 3585 count++; 3586 } 3587 } 3588 3589 /* TX Priority Stats */ 3590 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3591 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3592 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3593 rte_ixgbe_txq_strings[stat].offset + 3594 (sizeof(uint64_t) * i)); 3595 xstats[count].id = count; 3596 count++; 3597 } 3598 } 3599 return count; 3600 } 3601 3602 static int 3603 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 3604 uint64_t *values, unsigned int n) 3605 { 3606 if (!ids) { 3607 struct ixgbe_hw *hw = 3608 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3609 struct ixgbe_hw_stats *hw_stats = 3610 IXGBE_DEV_PRIVATE_TO_STATS( 3611 dev->data->dev_private); 3612 struct ixgbe_macsec_stats *macsec_stats = 3613 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3614 dev->data->dev_private); 3615 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3616 unsigned int i, stat, count = 0; 3617 3618 count = ixgbe_xstats_calc_num(); 3619 3620 if (!ids && n < count) 3621 return count; 3622 3623 total_missed_rx = 0; 3624 total_qbrc = 0; 3625 total_qprc = 0; 3626 total_qprdc = 0; 3627 3628 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, 3629 &total_missed_rx, &total_qbrc, &total_qprc, 3630 &total_qprdc); 3631 3632 /* If this is a reset xstats is NULL, and we have cleared the 3633 * registers by reading them. 3634 */ 3635 if (!ids && !values) 3636 return 0; 3637 3638 /* Extended stats from ixgbe_hw_stats */ 3639 count = 0; 3640 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3641 values[count] = *(uint64_t *)(((char *)hw_stats) + 3642 rte_ixgbe_stats_strings[i].offset); 3643 count++; 3644 } 3645 3646 /* MACsec Stats */ 3647 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3648 values[count] = *(uint64_t *)(((char *)macsec_stats) + 3649 rte_ixgbe_macsec_strings[i].offset); 3650 count++; 3651 } 3652 3653 /* RX Priority Stats */ 3654 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3655 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3656 values[count] = 3657 *(uint64_t *)(((char *)hw_stats) + 3658 rte_ixgbe_rxq_strings[stat].offset + 3659 (sizeof(uint64_t) * i)); 3660 count++; 3661 } 3662 } 3663 3664 /* TX Priority Stats */ 3665 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3666 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3667 values[count] = 3668 *(uint64_t *)(((char *)hw_stats) + 3669 rte_ixgbe_txq_strings[stat].offset + 3670 (sizeof(uint64_t) * i)); 3671 count++; 3672 } 3673 } 3674 return count; 3675 } 3676 3677 uint16_t i; 3678 uint16_t size = ixgbe_xstats_calc_num(); 3679 uint64_t values_copy[size]; 3680 3681 ixgbe_dev_xstats_get_by_id(dev, NULL, values_copy, size); 3682 3683 for (i = 0; i < n; i++) { 3684 if (ids[i] >= size) { 3685 PMD_INIT_LOG(ERR, "id value isn't valid"); 3686 return -1; 3687 } 3688 values[i] = values_copy[ids[i]]; 3689 } 3690 return n; 3691 } 3692 3693 static int 3694 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) 3695 { 3696 struct ixgbe_hw_stats *stats = 3697 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3698 struct ixgbe_macsec_stats *macsec_stats = 3699 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3700 dev->data->dev_private); 3701 3702 unsigned count = ixgbe_xstats_calc_num(); 3703 3704 /* HW registers are cleared on read */ 3705 ixgbe_dev_xstats_get(dev, NULL, count); 3706 3707 /* Reset software totals */ 3708 memset(stats, 0, sizeof(*stats)); 3709 memset(macsec_stats, 0, sizeof(*macsec_stats)); 3710 3711 return 0; 3712 } 3713 3714 static void 3715 ixgbevf_update_stats(struct rte_eth_dev *dev) 3716 { 3717 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3718 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3719 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3720 3721 /* Good Rx packet, include VF loopback */ 3722 UPDATE_VF_STAT(IXGBE_VFGPRC, 3723 hw_stats->last_vfgprc, hw_stats->vfgprc); 3724 3725 /* Good Rx octets, include VF loopback */ 3726 UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3727 hw_stats->last_vfgorc, hw_stats->vfgorc); 3728 3729 /* Good Tx packet, include VF loopback */ 3730 UPDATE_VF_STAT(IXGBE_VFGPTC, 3731 hw_stats->last_vfgptc, hw_stats->vfgptc); 3732 3733 /* Good Tx octets, include VF loopback */ 3734 UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3735 hw_stats->last_vfgotc, hw_stats->vfgotc); 3736 3737 /* Rx Multicst Packet */ 3738 UPDATE_VF_STAT(IXGBE_VFMPRC, 3739 hw_stats->last_vfmprc, hw_stats->vfmprc); 3740 } 3741 3742 static int 3743 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3744 unsigned n) 3745 { 3746 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3747 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3748 unsigned i; 3749 3750 if (n < IXGBEVF_NB_XSTATS) 3751 return IXGBEVF_NB_XSTATS; 3752 3753 ixgbevf_update_stats(dev); 3754 3755 if (!xstats) 3756 return 0; 3757 3758 /* Extended stats */ 3759 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) { 3760 xstats[i].id = i; 3761 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 3762 rte_ixgbevf_stats_strings[i].offset); 3763 } 3764 3765 return IXGBEVF_NB_XSTATS; 3766 } 3767 3768 static int 3769 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3770 { 3771 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3772 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3773 3774 ixgbevf_update_stats(dev); 3775 3776 if (stats == NULL) 3777 return -EINVAL; 3778 3779 stats->ipackets = hw_stats->vfgprc; 3780 stats->ibytes = hw_stats->vfgorc; 3781 stats->opackets = hw_stats->vfgptc; 3782 stats->obytes = hw_stats->vfgotc; 3783 return 0; 3784 } 3785 3786 static int 3787 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev) 3788 { 3789 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3790 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3791 3792 /* Sync HW register to the last stats */ 3793 ixgbevf_dev_stats_get(dev, NULL); 3794 3795 /* reset HW current stats*/ 3796 hw_stats->vfgprc = 0; 3797 hw_stats->vfgorc = 0; 3798 hw_stats->vfgptc = 0; 3799 hw_stats->vfgotc = 0; 3800 3801 return 0; 3802 } 3803 3804 static int 3805 ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 3806 { 3807 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3808 u16 eeprom_verh, eeprom_verl; 3809 u32 etrack_id; 3810 int ret; 3811 3812 ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); 3813 ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); 3814 3815 etrack_id = (eeprom_verh << 16) | eeprom_verl; 3816 ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); 3817 3818 ret += 1; /* add the size of '\0' */ 3819 if (fw_size < (u32)ret) 3820 return ret; 3821 else 3822 return 0; 3823 } 3824 3825 static int 3826 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 3827 { 3828 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3829 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3830 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3831 3832 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3833 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3834 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3835 /* 3836 * When DCB/VT is off, maximum number of queues changes, 3837 * except for 82598EB, which remains constant. 3838 */ 3839 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 3840 hw->mac.type != ixgbe_mac_82598EB) 3841 dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES; 3842 } 3843 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */ 3844 dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */ 3845 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3846 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3847 dev_info->max_vfs = pci_dev->max_vfs; 3848 if (hw->mac.type == ixgbe_mac_82598EB) 3849 dev_info->max_vmdq_pools = ETH_16_POOLS; 3850 else 3851 dev_info->max_vmdq_pools = ETH_64_POOLS; 3852 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3853 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 3854 dev_info->vmdq_queue_num = dev_info->max_rx_queues; 3855 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3856 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3857 dev_info->rx_queue_offload_capa); 3858 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3859 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3860 3861 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3862 .rx_thresh = { 3863 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3864 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3865 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3866 }, 3867 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3868 .rx_drop_en = 0, 3869 .offloads = 0, 3870 }; 3871 3872 dev_info->default_txconf = (struct rte_eth_txconf) { 3873 .tx_thresh = { 3874 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3875 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3876 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3877 }, 3878 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3879 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3880 .offloads = 0, 3881 }; 3882 3883 dev_info->rx_desc_lim = rx_desc_lim; 3884 dev_info->tx_desc_lim = tx_desc_lim; 3885 3886 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3887 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3888 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3889 3890 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; 3891 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 3892 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 3893 dev_info->speed_capa = ETH_LINK_SPEED_10M | 3894 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 3895 3896 if (hw->mac.type == ixgbe_mac_X540 || 3897 hw->mac.type == ixgbe_mac_X540_vf || 3898 hw->mac.type == ixgbe_mac_X550 || 3899 hw->mac.type == ixgbe_mac_X550_vf) { 3900 dev_info->speed_capa |= ETH_LINK_SPEED_100M; 3901 } 3902 if (hw->mac.type == ixgbe_mac_X550) { 3903 dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; 3904 dev_info->speed_capa |= ETH_LINK_SPEED_5G; 3905 } 3906 3907 /* Driver-preferred Rx/Tx parameters */ 3908 dev_info->default_rxportconf.burst_size = 32; 3909 dev_info->default_txportconf.burst_size = 32; 3910 dev_info->default_rxportconf.nb_queues = 1; 3911 dev_info->default_txportconf.nb_queues = 1; 3912 dev_info->default_rxportconf.ring_size = 256; 3913 dev_info->default_txportconf.ring_size = 256; 3914 3915 return 0; 3916 } 3917 3918 static const uint32_t * 3919 ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) 3920 { 3921 static const uint32_t ptypes[] = { 3922 /* For non-vec functions, 3923 * refers to ixgbe_rxd_pkt_info_to_pkt_type(); 3924 * for vec functions, 3925 * refers to _recv_raw_pkts_vec(). 3926 */ 3927 RTE_PTYPE_L2_ETHER, 3928 RTE_PTYPE_L3_IPV4, 3929 RTE_PTYPE_L3_IPV4_EXT, 3930 RTE_PTYPE_L3_IPV6, 3931 RTE_PTYPE_L3_IPV6_EXT, 3932 RTE_PTYPE_L4_SCTP, 3933 RTE_PTYPE_L4_TCP, 3934 RTE_PTYPE_L4_UDP, 3935 RTE_PTYPE_TUNNEL_IP, 3936 RTE_PTYPE_INNER_L3_IPV6, 3937 RTE_PTYPE_INNER_L3_IPV6_EXT, 3938 RTE_PTYPE_INNER_L4_TCP, 3939 RTE_PTYPE_INNER_L4_UDP, 3940 RTE_PTYPE_UNKNOWN 3941 }; 3942 3943 if (dev->rx_pkt_burst == ixgbe_recv_pkts || 3944 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc || 3945 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || 3946 dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) 3947 return ptypes; 3948 3949 #if defined(RTE_ARCH_X86) || defined(__ARM_NEON) 3950 if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || 3951 dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) 3952 return ptypes; 3953 #endif 3954 return NULL; 3955 } 3956 3957 static int 3958 ixgbevf_dev_info_get(struct rte_eth_dev *dev, 3959 struct rte_eth_dev_info *dev_info) 3960 { 3961 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3962 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3963 3964 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3965 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3966 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */ 3967 dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */ 3968 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3969 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3970 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3971 dev_info->max_vfs = pci_dev->max_vfs; 3972 if (hw->mac.type == ixgbe_mac_82598EB) 3973 dev_info->max_vmdq_pools = ETH_16_POOLS; 3974 else 3975 dev_info->max_vmdq_pools = ETH_64_POOLS; 3976 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3977 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3978 dev_info->rx_queue_offload_capa); 3979 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3980 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3981 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3982 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3983 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3984 3985 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3986 .rx_thresh = { 3987 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3988 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3989 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3990 }, 3991 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3992 .rx_drop_en = 0, 3993 .offloads = 0, 3994 }; 3995 3996 dev_info->default_txconf = (struct rte_eth_txconf) { 3997 .tx_thresh = { 3998 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3999 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 4000 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 4001 }, 4002 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 4003 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 4004 .offloads = 0, 4005 }; 4006 4007 dev_info->rx_desc_lim = rx_desc_lim; 4008 dev_info->tx_desc_lim = tx_desc_lim; 4009 4010 return 0; 4011 } 4012 4013 static int 4014 ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, 4015 bool *link_up, int wait_to_complete) 4016 { 4017 struct ixgbe_adapter *adapter = container_of(hw, 4018 struct ixgbe_adapter, hw); 4019 struct ixgbe_mbx_info *mbx = &hw->mbx; 4020 struct ixgbe_mac_info *mac = &hw->mac; 4021 uint32_t links_reg, in_msg; 4022 int ret_val = 0; 4023 4024 /* If we were hit with a reset drop the link */ 4025 if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) 4026 mac->get_link_status = true; 4027 4028 if (!mac->get_link_status) 4029 goto out; 4030 4031 /* if link status is down no point in checking to see if pf is up */ 4032 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 4033 if (!(links_reg & IXGBE_LINKS_UP)) 4034 goto out; 4035 4036 /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs 4037 * before the link status is correct 4038 */ 4039 if (mac->type == ixgbe_mac_82599_vf && wait_to_complete) { 4040 int i; 4041 4042 for (i = 0; i < 5; i++) { 4043 rte_delay_us(100); 4044 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 4045 4046 if (!(links_reg & IXGBE_LINKS_UP)) 4047 goto out; 4048 } 4049 } 4050 4051 switch (links_reg & IXGBE_LINKS_SPEED_82599) { 4052 case IXGBE_LINKS_SPEED_10G_82599: 4053 *speed = IXGBE_LINK_SPEED_10GB_FULL; 4054 if (hw->mac.type >= ixgbe_mac_X550) { 4055 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4056 *speed = IXGBE_LINK_SPEED_2_5GB_FULL; 4057 } 4058 break; 4059 case IXGBE_LINKS_SPEED_1G_82599: 4060 *speed = IXGBE_LINK_SPEED_1GB_FULL; 4061 break; 4062 case IXGBE_LINKS_SPEED_100_82599: 4063 *speed = IXGBE_LINK_SPEED_100_FULL; 4064 if (hw->mac.type == ixgbe_mac_X550) { 4065 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4066 *speed = IXGBE_LINK_SPEED_5GB_FULL; 4067 } 4068 break; 4069 case IXGBE_LINKS_SPEED_10_X550EM_A: 4070 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4071 /* Since Reserved in older MAC's */ 4072 if (hw->mac.type >= ixgbe_mac_X550) 4073 *speed = IXGBE_LINK_SPEED_10_FULL; 4074 break; 4075 default: 4076 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4077 } 4078 4079 if (wait_to_complete == 0 && adapter->pflink_fullchk == 0) { 4080 if (*speed == IXGBE_LINK_SPEED_UNKNOWN) 4081 mac->get_link_status = true; 4082 else 4083 mac->get_link_status = false; 4084 4085 goto out; 4086 } 4087 4088 /* if the read failed it could just be a mailbox collision, best wait 4089 * until we are called again and don't report an error 4090 */ 4091 if (mbx->ops.read(hw, &in_msg, 1, 0)) 4092 goto out; 4093 4094 if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { 4095 /* msg is not CTS and is NACK we must have lost CTS status */ 4096 if (in_msg & IXGBE_VT_MSGTYPE_NACK) 4097 mac->get_link_status = false; 4098 goto out; 4099 } 4100 4101 /* the pf is talking, if we timed out in the past we reinit */ 4102 if (!mbx->timeout) { 4103 ret_val = -1; 4104 goto out; 4105 } 4106 4107 /* if we passed all the tests above then the link is up and we no 4108 * longer need to check for link 4109 */ 4110 mac->get_link_status = false; 4111 4112 out: 4113 *link_up = !mac->get_link_status; 4114 return ret_val; 4115 } 4116 4117 /* 4118 * If @timeout_ms was 0, it means that it will not return until link complete. 4119 * It returns 1 on complete, return 0 on timeout. 4120 */ 4121 static int 4122 ixgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, uint32_t timeout_ms) 4123 { 4124 #define WARNING_TIMEOUT 9000 /* 9s in total */ 4125 struct ixgbe_adapter *ad = dev->data->dev_private; 4126 uint32_t timeout = timeout_ms ? timeout_ms : WARNING_TIMEOUT; 4127 4128 while (rte_atomic32_read(&ad->link_thread_running)) { 4129 msec_delay(1); 4130 timeout--; 4131 4132 if (timeout_ms) { 4133 if (!timeout) 4134 return 0; 4135 } else if (!timeout) { 4136 /* It will not return until link complete */ 4137 timeout = WARNING_TIMEOUT; 4138 PMD_DRV_LOG(ERR, "IXGBE link thread not complete too long time!"); 4139 } 4140 } 4141 4142 return 1; 4143 } 4144 4145 static void * 4146 ixgbe_dev_setup_link_thread_handler(void *param) 4147 { 4148 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4149 struct ixgbe_adapter *ad = dev->data->dev_private; 4150 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4151 struct ixgbe_interrupt *intr = 4152 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4153 u32 speed; 4154 bool autoneg = false; 4155 4156 pthread_detach(pthread_self()); 4157 speed = hw->phy.autoneg_advertised; 4158 if (!speed) 4159 ixgbe_get_link_capabilities(hw, &speed, &autoneg); 4160 4161 ixgbe_setup_link(hw, speed, true); 4162 4163 intr->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4164 rte_atomic32_clear(&ad->link_thread_running); 4165 return NULL; 4166 } 4167 4168 /* 4169 * In freebsd environment, nic_uio drivers do not support interrupts, 4170 * rte_intr_callback_register() will fail to register interrupts. 4171 * We can not make link status to change from down to up by interrupt 4172 * callback. So we need to wait for the controller to acquire link 4173 * when ports start. 4174 * It returns 0 on link up. 4175 */ 4176 static int 4177 ixgbe_wait_for_link_up(struct ixgbe_hw *hw) 4178 { 4179 #ifdef RTE_EXEC_ENV_FREEBSD 4180 int err, i; 4181 bool link_up = false; 4182 uint32_t speed = 0; 4183 const int nb_iter = 25; 4184 4185 for (i = 0; i < nb_iter; i++) { 4186 err = ixgbe_check_link(hw, &speed, &link_up, 0); 4187 if (err) 4188 return err; 4189 if (link_up) 4190 return 0; 4191 msec_delay(200); 4192 } 4193 4194 return 0; 4195 #else 4196 RTE_SET_USED(hw); 4197 return 0; 4198 #endif 4199 } 4200 4201 /* return 0 means link status changed, -1 means not changed */ 4202 int 4203 ixgbe_dev_link_update_share(struct rte_eth_dev *dev, 4204 int wait_to_complete, int vf) 4205 { 4206 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4207 struct ixgbe_adapter *ad = dev->data->dev_private; 4208 struct rte_eth_link link; 4209 ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN; 4210 struct ixgbe_interrupt *intr = 4211 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4212 bool link_up; 4213 int diag; 4214 int wait = 1; 4215 u32 esdp_reg; 4216 4217 memset(&link, 0, sizeof(link)); 4218 link.link_status = ETH_LINK_DOWN; 4219 link.link_speed = ETH_SPEED_NUM_NONE; 4220 link.link_duplex = ETH_LINK_HALF_DUPLEX; 4221 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 4222 ETH_LINK_SPEED_FIXED); 4223 4224 hw->mac.get_link_status = true; 4225 4226 if (intr->flags & IXGBE_FLAG_NEED_LINK_CONFIG) 4227 return rte_eth_linkstatus_set(dev, &link); 4228 4229 /* check if it needs to wait to complete, if lsc interrupt is enabled */ 4230 if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) 4231 wait = 0; 4232 4233 /* BSD has no interrupt mechanism, so force NIC status synchronization. */ 4234 #ifdef RTE_EXEC_ENV_FREEBSD 4235 wait = 1; 4236 #endif 4237 4238 if (vf) 4239 diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); 4240 else 4241 diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); 4242 4243 if (diag != 0) { 4244 link.link_speed = ETH_SPEED_NUM_100M; 4245 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4246 return rte_eth_linkstatus_set(dev, &link); 4247 } 4248 4249 if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 4250 esdp_reg = IXGBE_READ_REG(hw, IXGBE_ESDP); 4251 if ((esdp_reg & IXGBE_ESDP_SDP3)) 4252 link_up = 0; 4253 } 4254 4255 if (link_up == 0) { 4256 if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 4257 ixgbe_dev_wait_setup_link_complete(dev, 0); 4258 if (rte_atomic32_test_and_set(&ad->link_thread_running)) { 4259 /* To avoid race condition between threads, set 4260 * the IXGBE_FLAG_NEED_LINK_CONFIG flag only 4261 * when there is no link thread running. 4262 */ 4263 intr->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 4264 if (rte_ctrl_thread_create(&ad->link_thread_tid, 4265 "ixgbe-link-handler", 4266 NULL, 4267 ixgbe_dev_setup_link_thread_handler, 4268 dev) < 0) { 4269 PMD_DRV_LOG(ERR, 4270 "Create link thread failed!"); 4271 rte_atomic32_clear(&ad->link_thread_running); 4272 } 4273 } else { 4274 PMD_DRV_LOG(ERR, 4275 "Other link thread is running now!"); 4276 } 4277 } 4278 return rte_eth_linkstatus_set(dev, &link); 4279 } 4280 4281 link.link_status = ETH_LINK_UP; 4282 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4283 4284 switch (link_speed) { 4285 default: 4286 case IXGBE_LINK_SPEED_UNKNOWN: 4287 link.link_speed = ETH_SPEED_NUM_UNKNOWN; 4288 break; 4289 4290 case IXGBE_LINK_SPEED_10_FULL: 4291 link.link_speed = ETH_SPEED_NUM_10M; 4292 break; 4293 4294 case IXGBE_LINK_SPEED_100_FULL: 4295 link.link_speed = ETH_SPEED_NUM_100M; 4296 break; 4297 4298 case IXGBE_LINK_SPEED_1GB_FULL: 4299 link.link_speed = ETH_SPEED_NUM_1G; 4300 break; 4301 4302 case IXGBE_LINK_SPEED_2_5GB_FULL: 4303 link.link_speed = ETH_SPEED_NUM_2_5G; 4304 break; 4305 4306 case IXGBE_LINK_SPEED_5GB_FULL: 4307 link.link_speed = ETH_SPEED_NUM_5G; 4308 break; 4309 4310 case IXGBE_LINK_SPEED_10GB_FULL: 4311 link.link_speed = ETH_SPEED_NUM_10G; 4312 break; 4313 } 4314 4315 return rte_eth_linkstatus_set(dev, &link); 4316 } 4317 4318 static int 4319 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4320 { 4321 return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); 4322 } 4323 4324 static int 4325 ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4326 { 4327 return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); 4328 } 4329 4330 static int 4331 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) 4332 { 4333 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4334 uint32_t fctrl; 4335 4336 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4337 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4338 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4339 4340 return 0; 4341 } 4342 4343 static int 4344 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev) 4345 { 4346 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4347 uint32_t fctrl; 4348 4349 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4350 fctrl &= (~IXGBE_FCTRL_UPE); 4351 if (dev->data->all_multicast == 1) 4352 fctrl |= IXGBE_FCTRL_MPE; 4353 else 4354 fctrl &= (~IXGBE_FCTRL_MPE); 4355 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4356 4357 return 0; 4358 } 4359 4360 static int 4361 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev) 4362 { 4363 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4364 uint32_t fctrl; 4365 4366 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4367 fctrl |= IXGBE_FCTRL_MPE; 4368 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4369 4370 return 0; 4371 } 4372 4373 static int 4374 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) 4375 { 4376 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4377 uint32_t fctrl; 4378 4379 if (dev->data->promiscuous == 1) 4380 return 0; /* must remain in all_multicast mode */ 4381 4382 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4383 fctrl &= (~IXGBE_FCTRL_MPE); 4384 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4385 4386 return 0; 4387 } 4388 4389 /** 4390 * It clears the interrupt causes and enables the interrupt. 4391 * It will be called once only during nic initialized. 4392 * 4393 * @param dev 4394 * Pointer to struct rte_eth_dev. 4395 * @param on 4396 * Enable or Disable. 4397 * 4398 * @return 4399 * - On success, zero. 4400 * - On failure, a negative value. 4401 */ 4402 static int 4403 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 4404 { 4405 struct ixgbe_interrupt *intr = 4406 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4407 4408 ixgbe_dev_link_status_print(dev); 4409 if (on) 4410 intr->mask |= IXGBE_EICR_LSC; 4411 else 4412 intr->mask &= ~IXGBE_EICR_LSC; 4413 4414 return 0; 4415 } 4416 4417 /** 4418 * It clears the interrupt causes and enables the interrupt. 4419 * It will be called once only during nic initialized. 4420 * 4421 * @param dev 4422 * Pointer to struct rte_eth_dev. 4423 * 4424 * @return 4425 * - On success, zero. 4426 * - On failure, a negative value. 4427 */ 4428 static int 4429 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) 4430 { 4431 struct ixgbe_interrupt *intr = 4432 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4433 4434 intr->mask |= IXGBE_EICR_RTX_QUEUE; 4435 4436 return 0; 4437 } 4438 4439 /** 4440 * It clears the interrupt causes and enables the interrupt. 4441 * It will be called once only during nic initialized. 4442 * 4443 * @param dev 4444 * Pointer to struct rte_eth_dev. 4445 * 4446 * @return 4447 * - On success, zero. 4448 * - On failure, a negative value. 4449 */ 4450 static int 4451 ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev) 4452 { 4453 struct ixgbe_interrupt *intr = 4454 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4455 4456 intr->mask |= IXGBE_EICR_LINKSEC; 4457 4458 return 0; 4459 } 4460 4461 /* 4462 * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update. 4463 * 4464 * @param dev 4465 * Pointer to struct rte_eth_dev. 4466 * 4467 * @return 4468 * - On success, zero. 4469 * - On failure, a negative value. 4470 */ 4471 static int 4472 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) 4473 { 4474 uint32_t eicr; 4475 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4476 struct ixgbe_interrupt *intr = 4477 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4478 4479 /* clear all cause mask */ 4480 ixgbe_disable_intr(hw); 4481 4482 /* read-on-clear nic registers here */ 4483 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4484 PMD_DRV_LOG(DEBUG, "eicr %x", eicr); 4485 4486 intr->flags = 0; 4487 4488 /* set flag for async link update */ 4489 if (eicr & IXGBE_EICR_LSC) 4490 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4491 4492 if (eicr & IXGBE_EICR_MAILBOX) 4493 intr->flags |= IXGBE_FLAG_MAILBOX; 4494 4495 if (eicr & IXGBE_EICR_LINKSEC) 4496 intr->flags |= IXGBE_FLAG_MACSEC; 4497 4498 if (hw->mac.type == ixgbe_mac_X550EM_x && 4499 hw->phy.type == ixgbe_phy_x550em_ext_t && 4500 (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x)) 4501 intr->flags |= IXGBE_FLAG_PHY_INTERRUPT; 4502 4503 return 0; 4504 } 4505 4506 /** 4507 * It gets and then prints the link status. 4508 * 4509 * @param dev 4510 * Pointer to struct rte_eth_dev. 4511 * 4512 * @return 4513 * - On success, zero. 4514 * - On failure, a negative value. 4515 */ 4516 static void 4517 ixgbe_dev_link_status_print(struct rte_eth_dev *dev) 4518 { 4519 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4520 struct rte_eth_link link; 4521 4522 rte_eth_linkstatus_get(dev, &link); 4523 4524 if (link.link_status) { 4525 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 4526 (int)(dev->data->port_id), 4527 (unsigned)link.link_speed, 4528 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 4529 "full-duplex" : "half-duplex"); 4530 } else { 4531 PMD_INIT_LOG(INFO, " Port %d: Link Down", 4532 (int)(dev->data->port_id)); 4533 } 4534 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 4535 pci_dev->addr.domain, 4536 pci_dev->addr.bus, 4537 pci_dev->addr.devid, 4538 pci_dev->addr.function); 4539 } 4540 4541 /* 4542 * It executes link_update after knowing an interrupt occurred. 4543 * 4544 * @param dev 4545 * Pointer to struct rte_eth_dev. 4546 * 4547 * @return 4548 * - On success, zero. 4549 * - On failure, a negative value. 4550 */ 4551 static int 4552 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) 4553 { 4554 struct ixgbe_interrupt *intr = 4555 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4556 int64_t timeout; 4557 struct ixgbe_hw *hw = 4558 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4559 4560 PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags); 4561 4562 if (intr->flags & IXGBE_FLAG_MAILBOX) { 4563 ixgbe_pf_mbx_process(dev); 4564 intr->flags &= ~IXGBE_FLAG_MAILBOX; 4565 } 4566 4567 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4568 ixgbe_handle_lasi(hw); 4569 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4570 } 4571 4572 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4573 struct rte_eth_link link; 4574 4575 /* get the link status before link update, for predicting later */ 4576 rte_eth_linkstatus_get(dev, &link); 4577 4578 ixgbe_dev_link_update(dev, 0); 4579 4580 /* likely to up */ 4581 if (!link.link_status) 4582 /* handle it 1 sec later, wait it being stable */ 4583 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT; 4584 /* likely to down */ 4585 else 4586 /* handle it 4 sec later, wait it being stable */ 4587 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; 4588 4589 ixgbe_dev_link_status_print(dev); 4590 if (rte_eal_alarm_set(timeout * 1000, 4591 ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) 4592 PMD_DRV_LOG(ERR, "Error setting alarm"); 4593 else { 4594 /* remember original mask */ 4595 intr->mask_original = intr->mask; 4596 /* only disable lsc interrupt */ 4597 intr->mask &= ~IXGBE_EIMS_LSC; 4598 } 4599 } 4600 4601 PMD_DRV_LOG(DEBUG, "enable intr immediately"); 4602 ixgbe_enable_intr(dev); 4603 4604 return 0; 4605 } 4606 4607 /** 4608 * Interrupt handler which shall be registered for alarm callback for delayed 4609 * handling specific interrupt to wait for the stable nic state. As the 4610 * NIC interrupt state is not stable for ixgbe after link is just down, 4611 * it needs to wait 4 seconds to get the stable status. 4612 * 4613 * @param handle 4614 * Pointer to interrupt handle. 4615 * @param param 4616 * The address of parameter (struct rte_eth_dev *) regsitered before. 4617 * 4618 * @return 4619 * void 4620 */ 4621 static void 4622 ixgbe_dev_interrupt_delayed_handler(void *param) 4623 { 4624 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4625 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4626 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 4627 struct ixgbe_interrupt *intr = 4628 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4629 struct ixgbe_hw *hw = 4630 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4631 uint32_t eicr; 4632 4633 ixgbe_disable_intr(hw); 4634 4635 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4636 if (eicr & IXGBE_EICR_MAILBOX) 4637 ixgbe_pf_mbx_process(dev); 4638 4639 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4640 ixgbe_handle_lasi(hw); 4641 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4642 } 4643 4644 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4645 ixgbe_dev_link_update(dev, 0); 4646 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4647 ixgbe_dev_link_status_print(dev); 4648 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 4649 } 4650 4651 if (intr->flags & IXGBE_FLAG_MACSEC) { 4652 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, NULL); 4653 intr->flags &= ~IXGBE_FLAG_MACSEC; 4654 } 4655 4656 /* restore original mask */ 4657 intr->mask = intr->mask_original; 4658 intr->mask_original = 0; 4659 4660 PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr); 4661 ixgbe_enable_intr(dev); 4662 rte_intr_ack(intr_handle); 4663 } 4664 4665 /** 4666 * Interrupt handler triggered by NIC for handling 4667 * specific interrupt. 4668 * 4669 * @param handle 4670 * Pointer to interrupt handle. 4671 * @param param 4672 * The address of parameter (struct rte_eth_dev *) regsitered before. 4673 * 4674 * @return 4675 * void 4676 */ 4677 static void 4678 ixgbe_dev_interrupt_handler(void *param) 4679 { 4680 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4681 4682 ixgbe_dev_interrupt_get_status(dev); 4683 ixgbe_dev_interrupt_action(dev); 4684 } 4685 4686 static int 4687 ixgbe_dev_led_on(struct rte_eth_dev *dev) 4688 { 4689 struct ixgbe_hw *hw; 4690 4691 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4692 return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4693 } 4694 4695 static int 4696 ixgbe_dev_led_off(struct rte_eth_dev *dev) 4697 { 4698 struct ixgbe_hw *hw; 4699 4700 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4701 return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4702 } 4703 4704 static int 4705 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4706 { 4707 struct ixgbe_hw *hw; 4708 uint32_t mflcn_reg; 4709 uint32_t fccfg_reg; 4710 int rx_pause; 4711 int tx_pause; 4712 4713 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4714 4715 fc_conf->pause_time = hw->fc.pause_time; 4716 fc_conf->high_water = hw->fc.high_water[0]; 4717 fc_conf->low_water = hw->fc.low_water[0]; 4718 fc_conf->send_xon = hw->fc.send_xon; 4719 fc_conf->autoneg = !hw->fc.disable_fc_autoneg; 4720 4721 /* 4722 * Return rx_pause status according to actual setting of 4723 * MFLCN register. 4724 */ 4725 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4726 if (mflcn_reg & IXGBE_MFLCN_PMCF) 4727 fc_conf->mac_ctrl_frame_fwd = 1; 4728 else 4729 fc_conf->mac_ctrl_frame_fwd = 0; 4730 4731 if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE)) 4732 rx_pause = 1; 4733 else 4734 rx_pause = 0; 4735 4736 /* 4737 * Return tx_pause status according to actual setting of 4738 * FCCFG register. 4739 */ 4740 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4741 if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY)) 4742 tx_pause = 1; 4743 else 4744 tx_pause = 0; 4745 4746 if (rx_pause && tx_pause) 4747 fc_conf->mode = RTE_FC_FULL; 4748 else if (rx_pause) 4749 fc_conf->mode = RTE_FC_RX_PAUSE; 4750 else if (tx_pause) 4751 fc_conf->mode = RTE_FC_TX_PAUSE; 4752 else 4753 fc_conf->mode = RTE_FC_NONE; 4754 4755 return 0; 4756 } 4757 4758 static int 4759 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4760 { 4761 struct ixgbe_hw *hw; 4762 struct ixgbe_adapter *adapter = dev->data->dev_private; 4763 int err; 4764 uint32_t rx_buf_size; 4765 uint32_t max_high_water; 4766 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4767 ixgbe_fc_none, 4768 ixgbe_fc_rx_pause, 4769 ixgbe_fc_tx_pause, 4770 ixgbe_fc_full 4771 }; 4772 4773 PMD_INIT_FUNC_TRACE(); 4774 4775 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4776 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)); 4777 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4778 4779 /* 4780 * At least reserve one Ethernet frame for watermark 4781 * high_water/low_water in kilo bytes for ixgbe 4782 */ 4783 max_high_water = (rx_buf_size - 4784 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4785 if ((fc_conf->high_water > max_high_water) || 4786 (fc_conf->high_water < fc_conf->low_water)) { 4787 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4788 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4789 return -EINVAL; 4790 } 4791 4792 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode]; 4793 hw->fc.pause_time = fc_conf->pause_time; 4794 hw->fc.high_water[0] = fc_conf->high_water; 4795 hw->fc.low_water[0] = fc_conf->low_water; 4796 hw->fc.send_xon = fc_conf->send_xon; 4797 hw->fc.disable_fc_autoneg = !fc_conf->autoneg; 4798 adapter->mac_ctrl_frame_fwd = fc_conf->mac_ctrl_frame_fwd; 4799 4800 err = ixgbe_flow_ctrl_enable(dev, hw); 4801 if (err < 0) { 4802 PMD_INIT_LOG(ERR, "ixgbe_flow_ctrl_enable = 0x%x", err); 4803 return -EIO; 4804 } 4805 return err; 4806 } 4807 4808 /** 4809 * ixgbe_pfc_enable_generic - Enable flow control 4810 * @hw: pointer to hardware structure 4811 * @tc_num: traffic class number 4812 * Enable flow control according to the current settings. 4813 */ 4814 static int 4815 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num) 4816 { 4817 int ret_val = 0; 4818 uint32_t mflcn_reg, fccfg_reg; 4819 uint32_t reg; 4820 uint32_t fcrtl, fcrth; 4821 uint8_t i; 4822 uint8_t nb_rx_en; 4823 4824 /* Validate the water mark configuration */ 4825 if (!hw->fc.pause_time) { 4826 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4827 goto out; 4828 } 4829 4830 /* Low water mark of zero causes XOFF floods */ 4831 if (hw->fc.current_mode & ixgbe_fc_tx_pause) { 4832 /* High/Low water can not be 0 */ 4833 if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) { 4834 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4835 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4836 goto out; 4837 } 4838 4839 if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) { 4840 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4841 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4842 goto out; 4843 } 4844 } 4845 /* Negotiate the fc mode to use */ 4846 ixgbe_fc_autoneg(hw); 4847 4848 /* Disable any previous flow control settings */ 4849 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4850 mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE); 4851 4852 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4853 fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY); 4854 4855 switch (hw->fc.current_mode) { 4856 case ixgbe_fc_none: 4857 /* 4858 * If the count of enabled RX Priority Flow control >1, 4859 * and the TX pause can not be disabled 4860 */ 4861 nb_rx_en = 0; 4862 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4863 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4864 if (reg & IXGBE_FCRTH_FCEN) 4865 nb_rx_en++; 4866 } 4867 if (nb_rx_en > 1) 4868 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4869 break; 4870 case ixgbe_fc_rx_pause: 4871 /* 4872 * Rx Flow control is enabled and Tx Flow control is 4873 * disabled by software override. Since there really 4874 * isn't a way to advertise that we are capable of RX 4875 * Pause ONLY, we will advertise that we support both 4876 * symmetric and asymmetric Rx PAUSE. Later, we will 4877 * disable the adapter's ability to send PAUSE frames. 4878 */ 4879 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4880 /* 4881 * If the count of enabled RX Priority Flow control >1, 4882 * and the TX pause can not be disabled 4883 */ 4884 nb_rx_en = 0; 4885 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4886 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4887 if (reg & IXGBE_FCRTH_FCEN) 4888 nb_rx_en++; 4889 } 4890 if (nb_rx_en > 1) 4891 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4892 break; 4893 case ixgbe_fc_tx_pause: 4894 /* 4895 * Tx Flow control is enabled, and Rx Flow control is 4896 * disabled by software override. 4897 */ 4898 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4899 break; 4900 case ixgbe_fc_full: 4901 /* Flow control (both Rx and Tx) is enabled by SW override. */ 4902 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4903 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4904 break; 4905 default: 4906 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly"); 4907 ret_val = IXGBE_ERR_CONFIG; 4908 goto out; 4909 } 4910 4911 /* Set 802.3x based flow control settings. */ 4912 mflcn_reg |= IXGBE_MFLCN_DPF; 4913 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg); 4914 IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg); 4915 4916 /* Set up and enable Rx high/low water mark thresholds, enable XON. */ 4917 if ((hw->fc.current_mode & ixgbe_fc_tx_pause) && 4918 hw->fc.high_water[tc_num]) { 4919 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE; 4920 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl); 4921 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN; 4922 } else { 4923 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0); 4924 /* 4925 * In order to prevent Tx hangs when the internal Tx 4926 * switch is enabled we must set the high water mark 4927 * to the maximum FCRTH value. This allows the Tx 4928 * switch to function even under heavy Rx workloads. 4929 */ 4930 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32; 4931 } 4932 IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth); 4933 4934 /* Configure pause time (2 TCs per register) */ 4935 reg = hw->fc.pause_time * 0x00010001; 4936 for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++) 4937 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg); 4938 4939 /* Configure flow control refresh threshold value */ 4940 IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2); 4941 4942 out: 4943 return ret_val; 4944 } 4945 4946 static int 4947 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num) 4948 { 4949 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4950 int32_t ret_val = IXGBE_NOT_IMPLEMENTED; 4951 4952 if (hw->mac.type != ixgbe_mac_82598EB) { 4953 ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num); 4954 } 4955 return ret_val; 4956 } 4957 4958 static int 4959 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf) 4960 { 4961 int err; 4962 uint32_t rx_buf_size; 4963 uint32_t max_high_water; 4964 uint8_t tc_num; 4965 uint8_t map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 }; 4966 struct ixgbe_hw *hw = 4967 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4968 struct ixgbe_dcb_config *dcb_config = 4969 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4970 4971 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4972 ixgbe_fc_none, 4973 ixgbe_fc_rx_pause, 4974 ixgbe_fc_tx_pause, 4975 ixgbe_fc_full 4976 }; 4977 4978 PMD_INIT_FUNC_TRACE(); 4979 4980 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4981 tc_num = map[pfc_conf->priority]; 4982 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)); 4983 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4984 /* 4985 * At least reserve one Ethernet frame for watermark 4986 * high_water/low_water in kilo bytes for ixgbe 4987 */ 4988 max_high_water = (rx_buf_size - 4989 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4990 if ((pfc_conf->fc.high_water > max_high_water) || 4991 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) { 4992 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4993 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4994 return -EINVAL; 4995 } 4996 4997 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode]; 4998 hw->fc.pause_time = pfc_conf->fc.pause_time; 4999 hw->fc.send_xon = pfc_conf->fc.send_xon; 5000 hw->fc.low_water[tc_num] = pfc_conf->fc.low_water; 5001 hw->fc.high_water[tc_num] = pfc_conf->fc.high_water; 5002 5003 err = ixgbe_dcb_pfc_enable(dev, tc_num); 5004 5005 /* Not negotiated is not an error case */ 5006 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) 5007 return 0; 5008 5009 PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err); 5010 return -EIO; 5011 } 5012 5013 static int 5014 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 5015 struct rte_eth_rss_reta_entry64 *reta_conf, 5016 uint16_t reta_size) 5017 { 5018 uint16_t i, sp_reta_size; 5019 uint8_t j, mask; 5020 uint32_t reta, r; 5021 uint16_t idx, shift; 5022 struct ixgbe_adapter *adapter = dev->data->dev_private; 5023 struct rte_eth_dev_data *dev_data = dev->data; 5024 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5025 uint32_t reta_reg; 5026 5027 PMD_INIT_FUNC_TRACE(); 5028 5029 if (!dev_data->dev_started) { 5030 PMD_DRV_LOG(ERR, 5031 "port %d must be started before rss reta update", 5032 dev_data->port_id); 5033 return -EIO; 5034 } 5035 5036 if (!ixgbe_rss_update_sp(hw->mac.type)) { 5037 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this " 5038 "NIC."); 5039 return -ENOTSUP; 5040 } 5041 5042 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 5043 if (reta_size != sp_reta_size) { 5044 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 5045 "(%d) doesn't match the number hardware can supported " 5046 "(%d)", reta_size, sp_reta_size); 5047 return -EINVAL; 5048 } 5049 5050 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 5051 idx = i / RTE_RETA_GROUP_SIZE; 5052 shift = i % RTE_RETA_GROUP_SIZE; 5053 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 5054 IXGBE_4_BIT_MASK); 5055 if (!mask) 5056 continue; 5057 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 5058 if (mask == IXGBE_4_BIT_MASK) 5059 r = 0; 5060 else 5061 r = IXGBE_READ_REG(hw, reta_reg); 5062 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) { 5063 if (mask & (0x1 << j)) 5064 reta |= reta_conf[idx].reta[shift + j] << 5065 (CHAR_BIT * j); 5066 else 5067 reta |= r & (IXGBE_8_BIT_MASK << 5068 (CHAR_BIT * j)); 5069 } 5070 IXGBE_WRITE_REG(hw, reta_reg, reta); 5071 } 5072 adapter->rss_reta_updated = 1; 5073 5074 return 0; 5075 } 5076 5077 static int 5078 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 5079 struct rte_eth_rss_reta_entry64 *reta_conf, 5080 uint16_t reta_size) 5081 { 5082 uint16_t i, sp_reta_size; 5083 uint8_t j, mask; 5084 uint32_t reta; 5085 uint16_t idx, shift; 5086 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5087 uint32_t reta_reg; 5088 5089 PMD_INIT_FUNC_TRACE(); 5090 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 5091 if (reta_size != sp_reta_size) { 5092 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 5093 "(%d) doesn't match the number hardware can supported " 5094 "(%d)", reta_size, sp_reta_size); 5095 return -EINVAL; 5096 } 5097 5098 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 5099 idx = i / RTE_RETA_GROUP_SIZE; 5100 shift = i % RTE_RETA_GROUP_SIZE; 5101 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 5102 IXGBE_4_BIT_MASK); 5103 if (!mask) 5104 continue; 5105 5106 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 5107 reta = IXGBE_READ_REG(hw, reta_reg); 5108 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) { 5109 if (mask & (0x1 << j)) 5110 reta_conf[idx].reta[shift + j] = 5111 ((reta >> (CHAR_BIT * j)) & 5112 IXGBE_8_BIT_MASK); 5113 } 5114 } 5115 5116 return 0; 5117 } 5118 5119 static int 5120 ixgbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 5121 uint32_t index, uint32_t pool) 5122 { 5123 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5124 uint32_t enable_addr = 1; 5125 5126 return ixgbe_set_rar(hw, index, mac_addr->addr_bytes, 5127 pool, enable_addr); 5128 } 5129 5130 static void 5131 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) 5132 { 5133 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5134 5135 ixgbe_clear_rar(hw, index); 5136 } 5137 5138 static int 5139 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 5140 { 5141 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5142 5143 ixgbe_remove_rar(dev, 0); 5144 ixgbe_add_rar(dev, addr, 0, pci_dev->max_vfs); 5145 5146 return 0; 5147 } 5148 5149 static bool 5150 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) 5151 { 5152 if (strcmp(dev->device->driver->name, drv->driver.name)) 5153 return false; 5154 5155 return true; 5156 } 5157 5158 bool 5159 is_ixgbe_supported(struct rte_eth_dev *dev) 5160 { 5161 return is_device_supported(dev, &rte_ixgbe_pmd); 5162 } 5163 5164 static int 5165 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 5166 { 5167 uint32_t hlreg0; 5168 uint32_t maxfrs; 5169 struct ixgbe_hw *hw; 5170 struct rte_eth_dev_info dev_info; 5171 uint32_t frame_size = mtu + IXGBE_ETH_OVERHEAD; 5172 struct rte_eth_dev_data *dev_data = dev->data; 5173 int ret; 5174 5175 ret = ixgbe_dev_info_get(dev, &dev_info); 5176 if (ret != 0) 5177 return ret; 5178 5179 /* check that mtu is within the allowed range */ 5180 if (mtu < RTE_ETHER_MIN_MTU || frame_size > dev_info.max_rx_pktlen) 5181 return -EINVAL; 5182 5183 /* If device is started, refuse mtu that requires the support of 5184 * scattered packets when this feature has not been enabled before. 5185 */ 5186 if (dev_data->dev_started && !dev_data->scattered_rx && 5187 (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > 5188 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 5189 PMD_INIT_LOG(ERR, "Stop port first."); 5190 return -EINVAL; 5191 } 5192 5193 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5194 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5195 5196 /* switch to jumbo mode if needed */ 5197 if (frame_size > IXGBE_ETH_MAX_LEN) { 5198 dev->data->dev_conf.rxmode.offloads |= 5199 DEV_RX_OFFLOAD_JUMBO_FRAME; 5200 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 5201 } else { 5202 dev->data->dev_conf.rxmode.offloads &= 5203 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 5204 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 5205 } 5206 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5207 5208 /* update max frame size */ 5209 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 5210 5211 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 5212 maxfrs &= 0x0000FFFF; 5213 maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16); 5214 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 5215 5216 return 0; 5217 } 5218 5219 /* 5220 * Virtual Function operations 5221 */ 5222 static void 5223 ixgbevf_intr_disable(struct rte_eth_dev *dev) 5224 { 5225 struct ixgbe_interrupt *intr = 5226 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5227 struct ixgbe_hw *hw = 5228 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5229 5230 PMD_INIT_FUNC_TRACE(); 5231 5232 /* Clear interrupt mask to stop from interrupts being generated */ 5233 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK); 5234 5235 IXGBE_WRITE_FLUSH(hw); 5236 5237 /* Clear mask value. */ 5238 intr->mask = 0; 5239 } 5240 5241 static void 5242 ixgbevf_intr_enable(struct rte_eth_dev *dev) 5243 { 5244 struct ixgbe_interrupt *intr = 5245 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5246 struct ixgbe_hw *hw = 5247 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5248 5249 PMD_INIT_FUNC_TRACE(); 5250 5251 /* VF enable interrupt autoclean */ 5252 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK); 5253 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK); 5254 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK); 5255 5256 IXGBE_WRITE_FLUSH(hw); 5257 5258 /* Save IXGBE_VTEIMS value to mask. */ 5259 intr->mask = IXGBE_VF_IRQ_ENABLE_MASK; 5260 } 5261 5262 static int 5263 ixgbevf_dev_configure(struct rte_eth_dev *dev) 5264 { 5265 struct rte_eth_conf *conf = &dev->data->dev_conf; 5266 struct ixgbe_adapter *adapter = dev->data->dev_private; 5267 5268 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 5269 dev->data->port_id); 5270 5271 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 5272 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 5273 5274 /* 5275 * VF has no ability to enable/disable HW CRC 5276 * Keep the persistent behavior the same as Host PF 5277 */ 5278 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC 5279 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 5280 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 5281 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 5282 } 5283 #else 5284 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 5285 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 5286 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 5287 } 5288 #endif 5289 5290 /* 5291 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 5292 * allocation or vector Rx preconditions we will reset it. 5293 */ 5294 adapter->rx_bulk_alloc_allowed = true; 5295 adapter->rx_vec_allowed = true; 5296 5297 return 0; 5298 } 5299 5300 static int 5301 ixgbevf_dev_start(struct rte_eth_dev *dev) 5302 { 5303 struct ixgbe_hw *hw = 5304 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5305 uint32_t intr_vector = 0; 5306 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5307 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5308 5309 int err, mask = 0; 5310 5311 PMD_INIT_FUNC_TRACE(); 5312 5313 /* Stop the link setup handler before resetting the HW. */ 5314 ixgbe_dev_wait_setup_link_complete(dev, 0); 5315 5316 err = hw->mac.ops.reset_hw(hw); 5317 5318 /** 5319 * In this case, reuses the MAC address assigned by VF 5320 * initialization. 5321 */ 5322 if (err != IXGBE_SUCCESS && err != IXGBE_ERR_INVALID_MAC_ADDR) { 5323 PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); 5324 return err; 5325 } 5326 5327 hw->mac.get_link_status = true; 5328 5329 /* negotiate mailbox API version to use with the PF. */ 5330 ixgbevf_negotiate_api(hw); 5331 5332 ixgbevf_dev_tx_init(dev); 5333 5334 /* This can fail when allocating mbufs for descriptor rings */ 5335 err = ixgbevf_dev_rx_init(dev); 5336 if (err) { 5337 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err); 5338 ixgbe_dev_clear_queues(dev); 5339 return err; 5340 } 5341 5342 /* Set vfta */ 5343 ixgbevf_set_vfta_all(dev, 1); 5344 5345 /* Set HW strip */ 5346 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 5347 ETH_VLAN_EXTEND_MASK; 5348 err = ixgbevf_vlan_offload_config(dev, mask); 5349 if (err) { 5350 PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); 5351 ixgbe_dev_clear_queues(dev); 5352 return err; 5353 } 5354 5355 ixgbevf_dev_rxtx_start(dev); 5356 5357 /* check and configure queue intr-vector mapping */ 5358 if (rte_intr_cap_multiple(intr_handle) && 5359 dev->data->dev_conf.intr_conf.rxq) { 5360 /* According to datasheet, only vector 0/1/2 can be used, 5361 * now only one vector is used for Rx queue 5362 */ 5363 intr_vector = 1; 5364 if (rte_intr_efd_enable(intr_handle, intr_vector)) 5365 return -1; 5366 } 5367 5368 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 5369 intr_handle->intr_vec = 5370 rte_zmalloc("intr_vec", 5371 dev->data->nb_rx_queues * sizeof(int), 0); 5372 if (intr_handle->intr_vec == NULL) { 5373 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 5374 " intr_vec", dev->data->nb_rx_queues); 5375 return -ENOMEM; 5376 } 5377 } 5378 ixgbevf_configure_msix(dev); 5379 5380 /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt 5381 * is mapped to VFIO vector 0 in eth_ixgbevf_dev_init( ). 5382 * If previous VFIO interrupt mapping setting in eth_ixgbevf_dev_init( ) 5383 * is not cleared, it will fail when following rte_intr_enable( ) tries 5384 * to map Rx queue interrupt to other VFIO vectors. 5385 * So clear uio/vfio intr/evevnfd first to avoid failure. 5386 */ 5387 rte_intr_disable(intr_handle); 5388 5389 rte_intr_enable(intr_handle); 5390 5391 /* Re-enable interrupt for VF */ 5392 ixgbevf_intr_enable(dev); 5393 5394 /* 5395 * Update link status right before return, because it may 5396 * start link configuration process in a separate thread. 5397 */ 5398 ixgbevf_dev_link_update(dev, 0); 5399 5400 hw->adapter_stopped = false; 5401 5402 return 0; 5403 } 5404 5405 static int 5406 ixgbevf_dev_stop(struct rte_eth_dev *dev) 5407 { 5408 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5409 struct ixgbe_adapter *adapter = dev->data->dev_private; 5410 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5411 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5412 5413 if (hw->adapter_stopped) 5414 return 0; 5415 5416 PMD_INIT_FUNC_TRACE(); 5417 5418 ixgbe_dev_wait_setup_link_complete(dev, 0); 5419 5420 ixgbevf_intr_disable(dev); 5421 5422 dev->data->dev_started = 0; 5423 hw->adapter_stopped = 1; 5424 ixgbe_stop_adapter(hw); 5425 5426 /* 5427 * Clear what we set, but we still keep shadow_vfta to 5428 * restore after device starts 5429 */ 5430 ixgbevf_set_vfta_all(dev, 0); 5431 5432 /* Clear stored conf */ 5433 dev->data->scattered_rx = 0; 5434 5435 ixgbe_dev_clear_queues(dev); 5436 5437 /* Clean datapath event and queue/vec mapping */ 5438 rte_intr_efd_disable(intr_handle); 5439 if (intr_handle->intr_vec != NULL) { 5440 rte_free(intr_handle->intr_vec); 5441 intr_handle->intr_vec = NULL; 5442 } 5443 5444 adapter->rss_reta_updated = 0; 5445 5446 return 0; 5447 } 5448 5449 static int 5450 ixgbevf_dev_close(struct rte_eth_dev *dev) 5451 { 5452 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5453 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5454 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5455 int ret; 5456 5457 PMD_INIT_FUNC_TRACE(); 5458 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 5459 return 0; 5460 5461 ixgbe_reset_hw(hw); 5462 5463 ret = ixgbevf_dev_stop(dev); 5464 5465 ixgbe_dev_free_queues(dev); 5466 5467 /** 5468 * Remove the VF MAC address ro ensure 5469 * that the VF traffic goes to the PF 5470 * after stop, close and detach of the VF 5471 **/ 5472 ixgbevf_remove_mac_addr(dev, 0); 5473 5474 rte_intr_disable(intr_handle); 5475 rte_intr_callback_unregister(intr_handle, 5476 ixgbevf_dev_interrupt_handler, dev); 5477 5478 return ret; 5479 } 5480 5481 /* 5482 * Reset VF device 5483 */ 5484 static int 5485 ixgbevf_dev_reset(struct rte_eth_dev *dev) 5486 { 5487 int ret; 5488 5489 ret = eth_ixgbevf_dev_uninit(dev); 5490 if (ret) 5491 return ret; 5492 5493 ret = eth_ixgbevf_dev_init(dev); 5494 5495 return ret; 5496 } 5497 5498 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) 5499 { 5500 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5501 struct ixgbe_vfta *shadow_vfta = 5502 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5503 int i = 0, j = 0, vfta = 0, mask = 1; 5504 5505 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 5506 vfta = shadow_vfta->vfta[i]; 5507 if (vfta) { 5508 mask = 1; 5509 for (j = 0; j < 32; j++) { 5510 if (vfta & mask) 5511 ixgbe_set_vfta(hw, (i<<5)+j, 0, 5512 on, false); 5513 mask <<= 1; 5514 } 5515 } 5516 } 5517 5518 } 5519 5520 static int 5521 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 5522 { 5523 struct ixgbe_hw *hw = 5524 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5525 struct ixgbe_vfta *shadow_vfta = 5526 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5527 uint32_t vid_idx = 0; 5528 uint32_t vid_bit = 0; 5529 int ret = 0; 5530 5531 PMD_INIT_FUNC_TRACE(); 5532 5533 /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */ 5534 ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on, false); 5535 if (ret) { 5536 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 5537 return ret; 5538 } 5539 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 5540 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 5541 5542 /* Save what we set and retore it after device reset */ 5543 if (on) 5544 shadow_vfta->vfta[vid_idx] |= vid_bit; 5545 else 5546 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 5547 5548 return 0; 5549 } 5550 5551 static void 5552 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 5553 { 5554 struct ixgbe_hw *hw = 5555 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5556 uint32_t ctrl; 5557 5558 PMD_INIT_FUNC_TRACE(); 5559 5560 if (queue >= hw->mac.max_rx_queues) 5561 return; 5562 5563 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 5564 if (on) 5565 ctrl |= IXGBE_RXDCTL_VME; 5566 else 5567 ctrl &= ~IXGBE_RXDCTL_VME; 5568 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 5569 5570 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on); 5571 } 5572 5573 static int 5574 ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask) 5575 { 5576 struct ixgbe_rx_queue *rxq; 5577 uint16_t i; 5578 int on = 0; 5579 5580 /* VF function only support hw strip feature, others are not support */ 5581 if (mask & ETH_VLAN_STRIP_MASK) { 5582 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5583 rxq = dev->data->rx_queues[i]; 5584 on = !!(rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 5585 ixgbevf_vlan_strip_queue_set(dev, i, on); 5586 } 5587 } 5588 5589 return 0; 5590 } 5591 5592 static int 5593 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) 5594 { 5595 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 5596 5597 ixgbevf_vlan_offload_config(dev, mask); 5598 5599 return 0; 5600 } 5601 5602 int 5603 ixgbe_vt_check(struct ixgbe_hw *hw) 5604 { 5605 uint32_t reg_val; 5606 5607 /* if Virtualization Technology is enabled */ 5608 reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 5609 if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) { 5610 PMD_INIT_LOG(ERR, "VT must be enabled for this setting"); 5611 return -1; 5612 } 5613 5614 return 0; 5615 } 5616 5617 static uint32_t 5618 ixgbe_uta_vector(struct ixgbe_hw *hw, struct rte_ether_addr *uc_addr) 5619 { 5620 uint32_t vector = 0; 5621 5622 switch (hw->mac.mc_filter_type) { 5623 case 0: /* use bits [47:36] of the address */ 5624 vector = ((uc_addr->addr_bytes[4] >> 4) | 5625 (((uint16_t)uc_addr->addr_bytes[5]) << 4)); 5626 break; 5627 case 1: /* use bits [46:35] of the address */ 5628 vector = ((uc_addr->addr_bytes[4] >> 3) | 5629 (((uint16_t)uc_addr->addr_bytes[5]) << 5)); 5630 break; 5631 case 2: /* use bits [45:34] of the address */ 5632 vector = ((uc_addr->addr_bytes[4] >> 2) | 5633 (((uint16_t)uc_addr->addr_bytes[5]) << 6)); 5634 break; 5635 case 3: /* use bits [43:32] of the address */ 5636 vector = ((uc_addr->addr_bytes[4]) | 5637 (((uint16_t)uc_addr->addr_bytes[5]) << 8)); 5638 break; 5639 default: /* Invalid mc_filter_type */ 5640 break; 5641 } 5642 5643 /* vector can only be 12-bits or boundary will be exceeded */ 5644 vector &= 0xFFF; 5645 return vector; 5646 } 5647 5648 static int 5649 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, 5650 struct rte_ether_addr *mac_addr, uint8_t on) 5651 { 5652 uint32_t vector; 5653 uint32_t uta_idx; 5654 uint32_t reg_val; 5655 uint32_t uta_shift; 5656 uint32_t rc; 5657 const uint32_t ixgbe_uta_idx_mask = 0x7F; 5658 const uint32_t ixgbe_uta_bit_shift = 5; 5659 const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1; 5660 const uint32_t bit1 = 0x1; 5661 5662 struct ixgbe_hw *hw = 5663 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5664 struct ixgbe_uta_info *uta_info = 5665 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5666 5667 /* The UTA table only exists on 82599 hardware and newer */ 5668 if (hw->mac.type < ixgbe_mac_82599EB) 5669 return -ENOTSUP; 5670 5671 vector = ixgbe_uta_vector(hw, mac_addr); 5672 uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask; 5673 uta_shift = vector & ixgbe_uta_bit_mask; 5674 5675 rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0); 5676 if (rc == on) 5677 return 0; 5678 5679 reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx)); 5680 if (on) { 5681 uta_info->uta_in_use++; 5682 reg_val |= (bit1 << uta_shift); 5683 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift); 5684 } else { 5685 uta_info->uta_in_use--; 5686 reg_val &= ~(bit1 << uta_shift); 5687 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift); 5688 } 5689 5690 IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val); 5691 5692 if (uta_info->uta_in_use > 0) 5693 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, 5694 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type); 5695 else 5696 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type); 5697 5698 return 0; 5699 } 5700 5701 static int 5702 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on) 5703 { 5704 int i; 5705 struct ixgbe_hw *hw = 5706 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5707 struct ixgbe_uta_info *uta_info = 5708 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5709 5710 /* The UTA table only exists on 82599 hardware and newer */ 5711 if (hw->mac.type < ixgbe_mac_82599EB) 5712 return -ENOTSUP; 5713 5714 if (on) { 5715 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5716 uta_info->uta_shadow[i] = ~0; 5717 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0); 5718 } 5719 } else { 5720 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5721 uta_info->uta_shadow[i] = 0; 5722 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0); 5723 } 5724 } 5725 return 0; 5726 5727 } 5728 5729 uint32_t 5730 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val) 5731 { 5732 uint32_t new_val = orig_val; 5733 5734 if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG) 5735 new_val |= IXGBE_VMOLR_AUPE; 5736 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC) 5737 new_val |= IXGBE_VMOLR_ROMPE; 5738 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC) 5739 new_val |= IXGBE_VMOLR_ROPE; 5740 if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) 5741 new_val |= IXGBE_VMOLR_BAM; 5742 if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) 5743 new_val |= IXGBE_VMOLR_MPE; 5744 5745 return new_val; 5746 } 5747 5748 #define IXGBE_MRCTL_VPME 0x01 /* Virtual Pool Mirroring. */ 5749 #define IXGBE_MRCTL_UPME 0x02 /* Uplink Port Mirroring. */ 5750 #define IXGBE_MRCTL_DPME 0x04 /* Downlink Port Mirroring. */ 5751 #define IXGBE_MRCTL_VLME 0x08 /* VLAN Mirroring. */ 5752 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \ 5753 ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \ 5754 ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN)) 5755 5756 static int 5757 ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 5758 struct rte_eth_mirror_conf *mirror_conf, 5759 uint8_t rule_id, uint8_t on) 5760 { 5761 uint32_t mr_ctl, vlvf; 5762 uint32_t mp_lsb = 0; 5763 uint32_t mv_msb = 0; 5764 uint32_t mv_lsb = 0; 5765 uint32_t mp_msb = 0; 5766 uint8_t i = 0; 5767 int reg_index = 0; 5768 uint64_t vlan_mask = 0; 5769 5770 const uint8_t pool_mask_offset = 32; 5771 const uint8_t vlan_mask_offset = 32; 5772 const uint8_t dst_pool_offset = 8; 5773 const uint8_t rule_mr_offset = 4; 5774 const uint8_t mirror_rule_mask = 0x0F; 5775 5776 struct ixgbe_mirror_info *mr_info = 5777 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5778 struct ixgbe_hw *hw = 5779 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5780 uint8_t mirror_type = 0; 5781 5782 if (ixgbe_vt_check(hw) < 0) 5783 return -ENOTSUP; 5784 5785 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5786 return -EINVAL; 5787 5788 if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) { 5789 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.", 5790 mirror_conf->rule_type); 5791 return -EINVAL; 5792 } 5793 5794 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5795 mirror_type |= IXGBE_MRCTL_VLME; 5796 /* Check if vlan id is valid and find conresponding VLAN ID 5797 * index in VLVF 5798 */ 5799 for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) { 5800 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) { 5801 /* search vlan id related pool vlan filter 5802 * index 5803 */ 5804 reg_index = ixgbe_find_vlvf_slot( 5805 hw, 5806 mirror_conf->vlan.vlan_id[i], 5807 false); 5808 if (reg_index < 0) 5809 return -EINVAL; 5810 vlvf = IXGBE_READ_REG(hw, 5811 IXGBE_VLVF(reg_index)); 5812 if ((vlvf & IXGBE_VLVF_VIEN) && 5813 ((vlvf & IXGBE_VLVF_VLANID_MASK) == 5814 mirror_conf->vlan.vlan_id[i])) 5815 vlan_mask |= (1ULL << reg_index); 5816 else 5817 return -EINVAL; 5818 } 5819 } 5820 5821 if (on) { 5822 mv_lsb = vlan_mask & 0xFFFFFFFF; 5823 mv_msb = vlan_mask >> vlan_mask_offset; 5824 5825 mr_info->mr_conf[rule_id].vlan.vlan_mask = 5826 mirror_conf->vlan.vlan_mask; 5827 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) { 5828 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) 5829 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 5830 mirror_conf->vlan.vlan_id[i]; 5831 } 5832 } else { 5833 mv_lsb = 0; 5834 mv_msb = 0; 5835 mr_info->mr_conf[rule_id].vlan.vlan_mask = 0; 5836 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) 5837 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0; 5838 } 5839 } 5840 5841 /** 5842 * if enable pool mirror, write related pool mask register,if disable 5843 * pool mirror, clear PFMRVM register 5844 */ 5845 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5846 mirror_type |= IXGBE_MRCTL_VPME; 5847 if (on) { 5848 mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF; 5849 mp_msb = mirror_conf->pool_mask >> pool_mask_offset; 5850 mr_info->mr_conf[rule_id].pool_mask = 5851 mirror_conf->pool_mask; 5852 5853 } else { 5854 mp_lsb = 0; 5855 mp_msb = 0; 5856 mr_info->mr_conf[rule_id].pool_mask = 0; 5857 } 5858 } 5859 if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT) 5860 mirror_type |= IXGBE_MRCTL_UPME; 5861 if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT) 5862 mirror_type |= IXGBE_MRCTL_DPME; 5863 5864 /* read mirror control register and recalculate it */ 5865 mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id)); 5866 5867 if (on) { 5868 mr_ctl |= mirror_type; 5869 mr_ctl &= mirror_rule_mask; 5870 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset; 5871 } else { 5872 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask); 5873 } 5874 5875 mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type; 5876 mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool; 5877 5878 /* write mirrror control register */ 5879 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5880 5881 /* write pool mirrror control register */ 5882 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5883 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb); 5884 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), 5885 mp_msb); 5886 } 5887 /* write VLAN mirrror control register */ 5888 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5889 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb); 5890 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), 5891 mv_msb); 5892 } 5893 5894 return 0; 5895 } 5896 5897 static int 5898 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) 5899 { 5900 int mr_ctl = 0; 5901 uint32_t lsb_val = 0; 5902 uint32_t msb_val = 0; 5903 const uint8_t rule_mr_offset = 4; 5904 5905 struct ixgbe_hw *hw = 5906 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5907 struct ixgbe_mirror_info *mr_info = 5908 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5909 5910 if (ixgbe_vt_check(hw) < 0) 5911 return -ENOTSUP; 5912 5913 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5914 return -EINVAL; 5915 5916 memset(&mr_info->mr_conf[rule_id], 0, 5917 sizeof(struct rte_eth_mirror_conf)); 5918 5919 /* clear PFVMCTL register */ 5920 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5921 5922 /* clear pool mask register */ 5923 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val); 5924 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val); 5925 5926 /* clear vlan mask register */ 5927 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val); 5928 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val); 5929 5930 return 0; 5931 } 5932 5933 static int 5934 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5935 { 5936 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5937 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5938 struct ixgbe_interrupt *intr = 5939 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5940 struct ixgbe_hw *hw = 5941 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5942 uint32_t vec = IXGBE_MISC_VEC_ID; 5943 5944 if (rte_intr_allow_others(intr_handle)) 5945 vec = IXGBE_RX_VEC_START; 5946 intr->mask |= (1 << vec); 5947 RTE_SET_USED(queue_id); 5948 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5949 5950 rte_intr_ack(intr_handle); 5951 5952 return 0; 5953 } 5954 5955 static int 5956 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5957 { 5958 struct ixgbe_interrupt *intr = 5959 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5960 struct ixgbe_hw *hw = 5961 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5962 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5963 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5964 uint32_t vec = IXGBE_MISC_VEC_ID; 5965 5966 if (rte_intr_allow_others(intr_handle)) 5967 vec = IXGBE_RX_VEC_START; 5968 intr->mask &= ~(1 << vec); 5969 RTE_SET_USED(queue_id); 5970 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5971 5972 return 0; 5973 } 5974 5975 static int 5976 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5977 { 5978 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5979 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5980 uint32_t mask; 5981 struct ixgbe_hw *hw = 5982 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5983 struct ixgbe_interrupt *intr = 5984 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5985 5986 if (queue_id < 16) { 5987 ixgbe_disable_intr(hw); 5988 intr->mask |= (1 << queue_id); 5989 ixgbe_enable_intr(dev); 5990 } else if (queue_id < 32) { 5991 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5992 mask &= (1 << queue_id); 5993 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5994 } else if (queue_id < 64) { 5995 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5996 mask &= (1 << (queue_id - 32)); 5997 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5998 } 5999 rte_intr_ack(intr_handle); 6000 6001 return 0; 6002 } 6003 6004 static int 6005 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 6006 { 6007 uint32_t mask; 6008 struct ixgbe_hw *hw = 6009 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6010 struct ixgbe_interrupt *intr = 6011 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 6012 6013 if (queue_id < 16) { 6014 ixgbe_disable_intr(hw); 6015 intr->mask &= ~(1 << queue_id); 6016 ixgbe_enable_intr(dev); 6017 } else if (queue_id < 32) { 6018 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 6019 mask &= ~(1 << queue_id); 6020 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 6021 } else if (queue_id < 64) { 6022 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 6023 mask &= ~(1 << (queue_id - 32)); 6024 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 6025 } 6026 6027 return 0; 6028 } 6029 6030 static void 6031 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 6032 uint8_t queue, uint8_t msix_vector) 6033 { 6034 uint32_t tmp, idx; 6035 6036 if (direction == -1) { 6037 /* other causes */ 6038 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6039 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 6040 tmp &= ~0xFF; 6041 tmp |= msix_vector; 6042 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp); 6043 } else { 6044 /* rx or tx cause */ 6045 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6046 idx = ((16 * (queue & 1)) + (8 * direction)); 6047 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 6048 tmp &= ~(0xFF << idx); 6049 tmp |= (msix_vector << idx); 6050 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp); 6051 } 6052 } 6053 6054 /** 6055 * set the IVAR registers, mapping interrupt causes to vectors 6056 * @param hw 6057 * pointer to ixgbe_hw struct 6058 * @direction 6059 * 0 for Rx, 1 for Tx, -1 for other causes 6060 * @queue 6061 * queue to map the corresponding interrupt to 6062 * @msix_vector 6063 * the vector to map to the corresponding queue 6064 */ 6065 static void 6066 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 6067 uint8_t queue, uint8_t msix_vector) 6068 { 6069 uint32_t tmp, idx; 6070 6071 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6072 if (hw->mac.type == ixgbe_mac_82598EB) { 6073 if (direction == -1) 6074 direction = 0; 6075 idx = (((direction * 64) + queue) >> 2) & 0x1F; 6076 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx)); 6077 tmp &= ~(0xFF << (8 * (queue & 0x3))); 6078 tmp |= (msix_vector << (8 * (queue & 0x3))); 6079 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); 6080 } else if ((hw->mac.type == ixgbe_mac_82599EB) || 6081 (hw->mac.type == ixgbe_mac_X540) || 6082 (hw->mac.type == ixgbe_mac_X550) || 6083 (hw->mac.type == ixgbe_mac_X550EM_x)) { 6084 if (direction == -1) { 6085 /* other causes */ 6086 idx = ((queue & 1) * 8); 6087 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 6088 tmp &= ~(0xFF << idx); 6089 tmp |= (msix_vector << idx); 6090 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp); 6091 } else { 6092 /* rx or tx causes */ 6093 idx = ((16 * (queue & 1)) + (8 * direction)); 6094 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 6095 tmp &= ~(0xFF << idx); 6096 tmp |= (msix_vector << idx); 6097 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp); 6098 } 6099 } 6100 } 6101 6102 static void 6103 ixgbevf_configure_msix(struct rte_eth_dev *dev) 6104 { 6105 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 6106 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 6107 struct ixgbe_hw *hw = 6108 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6109 uint32_t q_idx; 6110 uint32_t vector_idx = IXGBE_MISC_VEC_ID; 6111 uint32_t base = IXGBE_MISC_VEC_ID; 6112 6113 /* Configure VF other cause ivar */ 6114 ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); 6115 6116 /* won't configure msix register if no mapping is done 6117 * between intr vector and event fd. 6118 */ 6119 if (!rte_intr_dp_is_en(intr_handle)) 6120 return; 6121 6122 if (rte_intr_allow_others(intr_handle)) { 6123 base = IXGBE_RX_VEC_START; 6124 vector_idx = IXGBE_RX_VEC_START; 6125 } 6126 6127 /* Configure all RX queues of VF */ 6128 for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) { 6129 /* Force all queue use vector 0, 6130 * as IXGBE_VF_MAXMSIVECOTR = 1 6131 */ 6132 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx); 6133 intr_handle->intr_vec[q_idx] = vector_idx; 6134 if (vector_idx < base + intr_handle->nb_efd - 1) 6135 vector_idx++; 6136 } 6137 6138 /* As RX queue setting above show, all queues use the vector 0. 6139 * Set only the ITR value of IXGBE_MISC_VEC_ID. 6140 */ 6141 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(IXGBE_MISC_VEC_ID), 6142 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6143 | IXGBE_EITR_CNT_WDIS); 6144 } 6145 6146 /** 6147 * Sets up the hardware to properly generate MSI-X interrupts 6148 * @hw 6149 * board private structure 6150 */ 6151 static void 6152 ixgbe_configure_msix(struct rte_eth_dev *dev) 6153 { 6154 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 6155 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 6156 struct ixgbe_hw *hw = 6157 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6158 uint32_t queue_id, base = IXGBE_MISC_VEC_ID; 6159 uint32_t vec = IXGBE_MISC_VEC_ID; 6160 uint32_t mask; 6161 uint32_t gpie; 6162 6163 /* won't configure msix register if no mapping is done 6164 * between intr vector and event fd 6165 * but if misx has been enabled already, need to configure 6166 * auto clean, auto mask and throttling. 6167 */ 6168 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6169 if (!rte_intr_dp_is_en(intr_handle) && 6170 !(gpie & (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT))) 6171 return; 6172 6173 if (rte_intr_allow_others(intr_handle)) 6174 vec = base = IXGBE_RX_VEC_START; 6175 6176 /* setup GPIE for MSI-x mode */ 6177 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6178 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 6179 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME; 6180 /* auto clearing and auto setting corresponding bits in EIMS 6181 * when MSI-X interrupt is triggered 6182 */ 6183 if (hw->mac.type == ixgbe_mac_82598EB) { 6184 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 6185 } else { 6186 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 6187 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 6188 } 6189 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 6190 6191 /* Populate the IVAR table and set the ITR values to the 6192 * corresponding register. 6193 */ 6194 if (rte_intr_dp_is_en(intr_handle)) { 6195 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; 6196 queue_id++) { 6197 /* by default, 1:1 mapping */ 6198 ixgbe_set_ivar_map(hw, 0, queue_id, vec); 6199 intr_handle->intr_vec[queue_id] = vec; 6200 if (vec < base + intr_handle->nb_efd - 1) 6201 vec++; 6202 } 6203 6204 switch (hw->mac.type) { 6205 case ixgbe_mac_82598EB: 6206 ixgbe_set_ivar_map(hw, -1, 6207 IXGBE_IVAR_OTHER_CAUSES_INDEX, 6208 IXGBE_MISC_VEC_ID); 6209 break; 6210 case ixgbe_mac_82599EB: 6211 case ixgbe_mac_X540: 6212 case ixgbe_mac_X550: 6213 case ixgbe_mac_X550EM_x: 6214 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); 6215 break; 6216 default: 6217 break; 6218 } 6219 } 6220 IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID), 6221 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6222 | IXGBE_EITR_CNT_WDIS); 6223 6224 /* set up to autoclear timer, and the vectors */ 6225 mask = IXGBE_EIMS_ENABLE_MASK; 6226 mask &= ~(IXGBE_EIMS_OTHER | 6227 IXGBE_EIMS_MAILBOX | 6228 IXGBE_EIMS_LSC); 6229 6230 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 6231 } 6232 6233 int 6234 ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, 6235 uint16_t queue_idx, uint16_t tx_rate) 6236 { 6237 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6238 struct rte_eth_rxmode *rxmode; 6239 uint32_t rf_dec, rf_int; 6240 uint32_t bcnrc_val; 6241 uint16_t link_speed = dev->data->dev_link.link_speed; 6242 6243 if (queue_idx >= hw->mac.max_tx_queues) 6244 return -EINVAL; 6245 6246 if (tx_rate != 0) { 6247 /* Calculate the rate factor values to set */ 6248 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate; 6249 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate; 6250 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate; 6251 6252 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA; 6253 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) & 6254 IXGBE_RTTBCNRC_RF_INT_MASK_M); 6255 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK); 6256 } else { 6257 bcnrc_val = 0; 6258 } 6259 6260 rxmode = &dev->data->dev_conf.rxmode; 6261 /* 6262 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 6263 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise 6264 * set as 0x4. 6265 */ 6266 if ((rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) && 6267 (rxmode->max_rx_pkt_len >= IXGBE_MAX_JUMBO_FRAME_SIZE)) 6268 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6269 IXGBE_MMW_SIZE_JUMBO_FRAME); 6270 else 6271 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6272 IXGBE_MMW_SIZE_DEFAULT); 6273 6274 /* Set RTTBCNRC of queue X */ 6275 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx); 6276 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val); 6277 IXGBE_WRITE_FLUSH(hw); 6278 6279 return 0; 6280 } 6281 6282 static int 6283 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 6284 __rte_unused uint32_t index, 6285 __rte_unused uint32_t pool) 6286 { 6287 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6288 int diag; 6289 6290 /* 6291 * On a 82599 VF, adding again the same MAC addr is not an idempotent 6292 * operation. Trap this case to avoid exhausting the [very limited] 6293 * set of PF resources used to store VF MAC addresses. 6294 */ 6295 if (memcmp(hw->mac.perm_addr, mac_addr, 6296 sizeof(struct rte_ether_addr)) == 0) 6297 return -1; 6298 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6299 if (diag != 0) 6300 PMD_DRV_LOG(ERR, "Unable to add MAC address " 6301 "%02x:%02x:%02x:%02x:%02x:%02x - diag=%d", 6302 mac_addr->addr_bytes[0], 6303 mac_addr->addr_bytes[1], 6304 mac_addr->addr_bytes[2], 6305 mac_addr->addr_bytes[3], 6306 mac_addr->addr_bytes[4], 6307 mac_addr->addr_bytes[5], 6308 diag); 6309 return diag; 6310 } 6311 6312 static void 6313 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index) 6314 { 6315 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6316 struct rte_ether_addr *perm_addr = 6317 (struct rte_ether_addr *)hw->mac.perm_addr; 6318 struct rte_ether_addr *mac_addr; 6319 uint32_t i; 6320 int diag; 6321 6322 /* 6323 * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does 6324 * not support the deletion of a given MAC address. 6325 * Instead, it imposes to delete all MAC addresses, then to add again 6326 * all MAC addresses with the exception of the one to be deleted. 6327 */ 6328 (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL); 6329 6330 /* 6331 * Add again all MAC addresses, with the exception of the deleted one 6332 * and of the permanent MAC address. 6333 */ 6334 for (i = 0, mac_addr = dev->data->mac_addrs; 6335 i < hw->mac.num_rar_entries; i++, mac_addr++) { 6336 /* Skip the deleted MAC address */ 6337 if (i == index) 6338 continue; 6339 /* Skip NULL MAC addresses */ 6340 if (rte_is_zero_ether_addr(mac_addr)) 6341 continue; 6342 /* Skip the permanent MAC address */ 6343 if (memcmp(perm_addr, mac_addr, 6344 sizeof(struct rte_ether_addr)) == 0) 6345 continue; 6346 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6347 if (diag != 0) 6348 PMD_DRV_LOG(ERR, 6349 "Adding again MAC address " 6350 "%02x:%02x:%02x:%02x:%02x:%02x failed " 6351 "diag=%d", 6352 mac_addr->addr_bytes[0], 6353 mac_addr->addr_bytes[1], 6354 mac_addr->addr_bytes[2], 6355 mac_addr->addr_bytes[3], 6356 mac_addr->addr_bytes[4], 6357 mac_addr->addr_bytes[5], 6358 diag); 6359 } 6360 } 6361 6362 static int 6363 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 6364 struct rte_ether_addr *addr) 6365 { 6366 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6367 6368 hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0); 6369 6370 return 0; 6371 } 6372 6373 int 6374 ixgbe_syn_filter_set(struct rte_eth_dev *dev, 6375 struct rte_eth_syn_filter *filter, 6376 bool add) 6377 { 6378 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6379 struct ixgbe_filter_info *filter_info = 6380 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6381 uint32_t syn_info; 6382 uint32_t synqf; 6383 6384 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6385 return -EINVAL; 6386 6387 syn_info = filter_info->syn_info; 6388 6389 if (add) { 6390 if (syn_info & IXGBE_SYN_FILTER_ENABLE) 6391 return -EINVAL; 6392 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) & 6393 IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE); 6394 6395 if (filter->hig_pri) 6396 synqf |= IXGBE_SYN_FILTER_SYNQFP; 6397 else 6398 synqf &= ~IXGBE_SYN_FILTER_SYNQFP; 6399 } else { 6400 synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6401 if (!(syn_info & IXGBE_SYN_FILTER_ENABLE)) 6402 return -ENOENT; 6403 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE); 6404 } 6405 6406 filter_info->syn_info = synqf; 6407 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 6408 IXGBE_WRITE_FLUSH(hw); 6409 return 0; 6410 } 6411 6412 6413 static inline enum ixgbe_5tuple_protocol 6414 convert_protocol_type(uint8_t protocol_value) 6415 { 6416 if (protocol_value == IPPROTO_TCP) 6417 return IXGBE_FILTER_PROTOCOL_TCP; 6418 else if (protocol_value == IPPROTO_UDP) 6419 return IXGBE_FILTER_PROTOCOL_UDP; 6420 else if (protocol_value == IPPROTO_SCTP) 6421 return IXGBE_FILTER_PROTOCOL_SCTP; 6422 else 6423 return IXGBE_FILTER_PROTOCOL_NONE; 6424 } 6425 6426 /* inject a 5-tuple filter to HW */ 6427 static inline void 6428 ixgbe_inject_5tuple_filter(struct rte_eth_dev *dev, 6429 struct ixgbe_5tuple_filter *filter) 6430 { 6431 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6432 int i; 6433 uint32_t ftqf, sdpqf; 6434 uint32_t l34timir = 0; 6435 uint8_t mask = 0xff; 6436 6437 i = filter->index; 6438 6439 sdpqf = (uint32_t)(filter->filter_info.dst_port << 6440 IXGBE_SDPQF_DSTPORT_SHIFT); 6441 sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT); 6442 6443 ftqf = (uint32_t)(filter->filter_info.proto & 6444 IXGBE_FTQF_PROTOCOL_MASK); 6445 ftqf |= (uint32_t)((filter->filter_info.priority & 6446 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT); 6447 if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */ 6448 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK; 6449 if (filter->filter_info.dst_ip_mask == 0) 6450 mask &= IXGBE_FTQF_DEST_ADDR_MASK; 6451 if (filter->filter_info.src_port_mask == 0) 6452 mask &= IXGBE_FTQF_SOURCE_PORT_MASK; 6453 if (filter->filter_info.dst_port_mask == 0) 6454 mask &= IXGBE_FTQF_DEST_PORT_MASK; 6455 if (filter->filter_info.proto_mask == 0) 6456 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK; 6457 ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT; 6458 ftqf |= IXGBE_FTQF_POOL_MASK_EN; 6459 ftqf |= IXGBE_FTQF_QUEUE_ENABLE; 6460 6461 IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip); 6462 IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip); 6463 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf); 6464 IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf); 6465 6466 l34timir |= IXGBE_L34T_IMIR_RESERVE; 6467 l34timir |= (uint32_t)(filter->queue << 6468 IXGBE_L34T_IMIR_QUEUE_SHIFT); 6469 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir); 6470 } 6471 6472 /* 6473 * add a 5tuple filter 6474 * 6475 * @param 6476 * dev: Pointer to struct rte_eth_dev. 6477 * index: the index the filter allocates. 6478 * filter: ponter to the filter that will be added. 6479 * rx_queue: the queue id the filter assigned to. 6480 * 6481 * @return 6482 * - On success, zero. 6483 * - On failure, a negative value. 6484 */ 6485 static int 6486 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 6487 struct ixgbe_5tuple_filter *filter) 6488 { 6489 struct ixgbe_filter_info *filter_info = 6490 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6491 int i, idx, shift; 6492 6493 /* 6494 * look for an unused 5tuple filter index, 6495 * and insert the filter to list. 6496 */ 6497 for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { 6498 idx = i / (sizeof(uint32_t) * NBBY); 6499 shift = i % (sizeof(uint32_t) * NBBY); 6500 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { 6501 filter_info->fivetuple_mask[idx] |= 1 << shift; 6502 filter->index = i; 6503 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 6504 filter, 6505 entries); 6506 break; 6507 } 6508 } 6509 if (i >= IXGBE_MAX_FTQF_FILTERS) { 6510 PMD_DRV_LOG(ERR, "5tuple filters are full."); 6511 return -ENOSYS; 6512 } 6513 6514 ixgbe_inject_5tuple_filter(dev, filter); 6515 6516 return 0; 6517 } 6518 6519 /* 6520 * remove a 5tuple filter 6521 * 6522 * @param 6523 * dev: Pointer to struct rte_eth_dev. 6524 * filter: the pointer of the filter will be removed. 6525 */ 6526 static void 6527 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 6528 struct ixgbe_5tuple_filter *filter) 6529 { 6530 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6531 struct ixgbe_filter_info *filter_info = 6532 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6533 uint16_t index = filter->index; 6534 6535 filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &= 6536 ~(1 << (index % (sizeof(uint32_t) * NBBY))); 6537 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 6538 rte_free(filter); 6539 6540 IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0); 6541 IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0); 6542 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0); 6543 IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0); 6544 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0); 6545 } 6546 6547 static int 6548 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 6549 { 6550 struct ixgbe_hw *hw; 6551 uint32_t max_frame = mtu + IXGBE_ETH_OVERHEAD; 6552 struct rte_eth_dev_data *dev_data = dev->data; 6553 6554 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6555 6556 if (mtu < RTE_ETHER_MIN_MTU || 6557 max_frame > RTE_ETHER_MAX_JUMBO_FRAME_LEN) 6558 return -EINVAL; 6559 6560 /* If device is started, refuse mtu that requires the support of 6561 * scattered packets when this feature has not been enabled before. 6562 */ 6563 if (dev_data->dev_started && !dev_data->scattered_rx && 6564 (max_frame + 2 * IXGBE_VLAN_TAG_SIZE > 6565 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 6566 PMD_INIT_LOG(ERR, "Stop port first."); 6567 return -EINVAL; 6568 } 6569 6570 /* 6571 * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU 6572 * request of the version 2.0 of the mailbox API. 6573 * For now, use the IXGBE_VF_SET_LPE request of the version 1.0 6574 * of the mailbox API. 6575 * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers 6576 * prior to 3.11.33 which contains the following change: 6577 * "ixgbe: Enable jumbo frames support w/ SR-IOV" 6578 */ 6579 if (ixgbevf_rlpml_set_vf(hw, max_frame)) 6580 return -EINVAL; 6581 6582 /* update max frame size */ 6583 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame; 6584 return 0; 6585 } 6586 6587 static inline struct ixgbe_5tuple_filter * 6588 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list, 6589 struct ixgbe_5tuple_filter_info *key) 6590 { 6591 struct ixgbe_5tuple_filter *it; 6592 6593 TAILQ_FOREACH(it, filter_list, entries) { 6594 if (memcmp(key, &it->filter_info, 6595 sizeof(struct ixgbe_5tuple_filter_info)) == 0) { 6596 return it; 6597 } 6598 } 6599 return NULL; 6600 } 6601 6602 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/ 6603 static inline int 6604 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter, 6605 struct ixgbe_5tuple_filter_info *filter_info) 6606 { 6607 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM || 6608 filter->priority > IXGBE_5TUPLE_MAX_PRI || 6609 filter->priority < IXGBE_5TUPLE_MIN_PRI) 6610 return -EINVAL; 6611 6612 switch (filter->dst_ip_mask) { 6613 case UINT32_MAX: 6614 filter_info->dst_ip_mask = 0; 6615 filter_info->dst_ip = filter->dst_ip; 6616 break; 6617 case 0: 6618 filter_info->dst_ip_mask = 1; 6619 break; 6620 default: 6621 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 6622 return -EINVAL; 6623 } 6624 6625 switch (filter->src_ip_mask) { 6626 case UINT32_MAX: 6627 filter_info->src_ip_mask = 0; 6628 filter_info->src_ip = filter->src_ip; 6629 break; 6630 case 0: 6631 filter_info->src_ip_mask = 1; 6632 break; 6633 default: 6634 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 6635 return -EINVAL; 6636 } 6637 6638 switch (filter->dst_port_mask) { 6639 case UINT16_MAX: 6640 filter_info->dst_port_mask = 0; 6641 filter_info->dst_port = filter->dst_port; 6642 break; 6643 case 0: 6644 filter_info->dst_port_mask = 1; 6645 break; 6646 default: 6647 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 6648 return -EINVAL; 6649 } 6650 6651 switch (filter->src_port_mask) { 6652 case UINT16_MAX: 6653 filter_info->src_port_mask = 0; 6654 filter_info->src_port = filter->src_port; 6655 break; 6656 case 0: 6657 filter_info->src_port_mask = 1; 6658 break; 6659 default: 6660 PMD_DRV_LOG(ERR, "invalid src_port mask."); 6661 return -EINVAL; 6662 } 6663 6664 switch (filter->proto_mask) { 6665 case UINT8_MAX: 6666 filter_info->proto_mask = 0; 6667 filter_info->proto = 6668 convert_protocol_type(filter->proto); 6669 break; 6670 case 0: 6671 filter_info->proto_mask = 1; 6672 break; 6673 default: 6674 PMD_DRV_LOG(ERR, "invalid protocol mask."); 6675 return -EINVAL; 6676 } 6677 6678 filter_info->priority = (uint8_t)filter->priority; 6679 return 0; 6680 } 6681 6682 /* 6683 * add or delete a ntuple filter 6684 * 6685 * @param 6686 * dev: Pointer to struct rte_eth_dev. 6687 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6688 * add: if true, add filter, if false, remove filter 6689 * 6690 * @return 6691 * - On success, zero. 6692 * - On failure, a negative value. 6693 */ 6694 int 6695 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, 6696 struct rte_eth_ntuple_filter *ntuple_filter, 6697 bool add) 6698 { 6699 struct ixgbe_filter_info *filter_info = 6700 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6701 struct ixgbe_5tuple_filter_info filter_5tuple; 6702 struct ixgbe_5tuple_filter *filter; 6703 int ret; 6704 6705 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6706 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6707 return -EINVAL; 6708 } 6709 6710 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6711 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6712 if (ret < 0) 6713 return ret; 6714 6715 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6716 &filter_5tuple); 6717 if (filter != NULL && add) { 6718 PMD_DRV_LOG(ERR, "filter exists."); 6719 return -EEXIST; 6720 } 6721 if (filter == NULL && !add) { 6722 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6723 return -ENOENT; 6724 } 6725 6726 if (add) { 6727 filter = rte_zmalloc("ixgbe_5tuple_filter", 6728 sizeof(struct ixgbe_5tuple_filter), 0); 6729 if (filter == NULL) 6730 return -ENOMEM; 6731 rte_memcpy(&filter->filter_info, 6732 &filter_5tuple, 6733 sizeof(struct ixgbe_5tuple_filter_info)); 6734 filter->queue = ntuple_filter->queue; 6735 ret = ixgbe_add_5tuple_filter(dev, filter); 6736 if (ret < 0) { 6737 rte_free(filter); 6738 return ret; 6739 } 6740 } else 6741 ixgbe_remove_5tuple_filter(dev, filter); 6742 6743 return 0; 6744 } 6745 6746 int 6747 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, 6748 struct rte_eth_ethertype_filter *filter, 6749 bool add) 6750 { 6751 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6752 struct ixgbe_filter_info *filter_info = 6753 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6754 uint32_t etqf = 0; 6755 uint32_t etqs = 0; 6756 int ret; 6757 struct ixgbe_ethertype_filter ethertype_filter; 6758 6759 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6760 return -EINVAL; 6761 6762 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 6763 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 6764 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 6765 " ethertype filter.", filter->ether_type); 6766 return -EINVAL; 6767 } 6768 6769 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 6770 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 6771 return -EINVAL; 6772 } 6773 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 6774 PMD_DRV_LOG(ERR, "drop option is unsupported."); 6775 return -EINVAL; 6776 } 6777 6778 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6779 if (ret >= 0 && add) { 6780 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 6781 filter->ether_type); 6782 return -EEXIST; 6783 } 6784 if (ret < 0 && !add) { 6785 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6786 filter->ether_type); 6787 return -ENOENT; 6788 } 6789 6790 if (add) { 6791 etqf = IXGBE_ETQF_FILTER_EN; 6792 etqf |= (uint32_t)filter->ether_type; 6793 etqs |= (uint32_t)((filter->queue << 6794 IXGBE_ETQS_RX_QUEUE_SHIFT) & 6795 IXGBE_ETQS_RX_QUEUE); 6796 etqs |= IXGBE_ETQS_QUEUE_EN; 6797 6798 ethertype_filter.ethertype = filter->ether_type; 6799 ethertype_filter.etqf = etqf; 6800 ethertype_filter.etqs = etqs; 6801 ethertype_filter.conf = FALSE; 6802 ret = ixgbe_ethertype_filter_insert(filter_info, 6803 ðertype_filter); 6804 if (ret < 0) { 6805 PMD_DRV_LOG(ERR, "ethertype filters are full."); 6806 return -ENOSPC; 6807 } 6808 } else { 6809 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret); 6810 if (ret < 0) 6811 return -ENOSYS; 6812 } 6813 IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf); 6814 IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs); 6815 IXGBE_WRITE_FLUSH(hw); 6816 6817 return 0; 6818 } 6819 6820 static int 6821 ixgbe_dev_flow_ops_get(__rte_unused struct rte_eth_dev *dev, 6822 const struct rte_flow_ops **ops) 6823 { 6824 *ops = &ixgbe_flow_ops; 6825 return 0; 6826 } 6827 6828 static u8 * 6829 ixgbe_dev_addr_list_itr(__rte_unused struct ixgbe_hw *hw, 6830 u8 **mc_addr_ptr, u32 *vmdq) 6831 { 6832 u8 *mc_addr; 6833 6834 *vmdq = 0; 6835 mc_addr = *mc_addr_ptr; 6836 *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr)); 6837 return mc_addr; 6838 } 6839 6840 static int 6841 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 6842 struct rte_ether_addr *mc_addr_set, 6843 uint32_t nb_mc_addr) 6844 { 6845 struct ixgbe_hw *hw; 6846 u8 *mc_addr_list; 6847 6848 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6849 mc_addr_list = (u8 *)mc_addr_set; 6850 return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr, 6851 ixgbe_dev_addr_list_itr, TRUE); 6852 } 6853 6854 static uint64_t 6855 ixgbe_read_systime_cyclecounter(struct rte_eth_dev *dev) 6856 { 6857 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6858 uint64_t systime_cycles; 6859 6860 switch (hw->mac.type) { 6861 case ixgbe_mac_X550: 6862 case ixgbe_mac_X550EM_x: 6863 case ixgbe_mac_X550EM_a: 6864 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 6865 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6866 systime_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6867 * NSEC_PER_SEC; 6868 break; 6869 default: 6870 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6871 systime_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6872 << 32; 6873 } 6874 6875 return systime_cycles; 6876 } 6877 6878 static uint64_t 6879 ixgbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6880 { 6881 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6882 uint64_t rx_tstamp_cycles; 6883 6884 switch (hw->mac.type) { 6885 case ixgbe_mac_X550: 6886 case ixgbe_mac_X550EM_x: 6887 case ixgbe_mac_X550EM_a: 6888 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6889 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6890 rx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6891 * NSEC_PER_SEC; 6892 break; 6893 default: 6894 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6895 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6896 rx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6897 << 32; 6898 } 6899 6900 return rx_tstamp_cycles; 6901 } 6902 6903 static uint64_t 6904 ixgbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6905 { 6906 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6907 uint64_t tx_tstamp_cycles; 6908 6909 switch (hw->mac.type) { 6910 case ixgbe_mac_X550: 6911 case ixgbe_mac_X550EM_x: 6912 case ixgbe_mac_X550EM_a: 6913 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6914 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6915 tx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6916 * NSEC_PER_SEC; 6917 break; 6918 default: 6919 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6920 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6921 tx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6922 << 32; 6923 } 6924 6925 return tx_tstamp_cycles; 6926 } 6927 6928 static void 6929 ixgbe_start_timecounters(struct rte_eth_dev *dev) 6930 { 6931 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6932 struct ixgbe_adapter *adapter = dev->data->dev_private; 6933 struct rte_eth_link link; 6934 uint32_t incval = 0; 6935 uint32_t shift = 0; 6936 6937 /* Get current link speed. */ 6938 ixgbe_dev_link_update(dev, 1); 6939 rte_eth_linkstatus_get(dev, &link); 6940 6941 switch (link.link_speed) { 6942 case ETH_SPEED_NUM_100M: 6943 incval = IXGBE_INCVAL_100; 6944 shift = IXGBE_INCVAL_SHIFT_100; 6945 break; 6946 case ETH_SPEED_NUM_1G: 6947 incval = IXGBE_INCVAL_1GB; 6948 shift = IXGBE_INCVAL_SHIFT_1GB; 6949 break; 6950 case ETH_SPEED_NUM_10G: 6951 default: 6952 incval = IXGBE_INCVAL_10GB; 6953 shift = IXGBE_INCVAL_SHIFT_10GB; 6954 break; 6955 } 6956 6957 switch (hw->mac.type) { 6958 case ixgbe_mac_X550: 6959 case ixgbe_mac_X550EM_x: 6960 case ixgbe_mac_X550EM_a: 6961 /* Independent of link speed. */ 6962 incval = 1; 6963 /* Cycles read will be interpreted as ns. */ 6964 shift = 0; 6965 /* Fall-through */ 6966 case ixgbe_mac_X540: 6967 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval); 6968 break; 6969 case ixgbe_mac_82599EB: 6970 incval >>= IXGBE_INCVAL_SHIFT_82599; 6971 shift -= IXGBE_INCVAL_SHIFT_82599; 6972 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 6973 (1 << IXGBE_INCPER_SHIFT_82599) | incval); 6974 break; 6975 default: 6976 /* Not supported. */ 6977 return; 6978 } 6979 6980 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 6981 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6982 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6983 6984 adapter->systime_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6985 adapter->systime_tc.cc_shift = shift; 6986 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 6987 6988 adapter->rx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6989 adapter->rx_tstamp_tc.cc_shift = shift; 6990 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6991 6992 adapter->tx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6993 adapter->tx_tstamp_tc.cc_shift = shift; 6994 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6995 } 6996 6997 static int 6998 ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 6999 { 7000 struct ixgbe_adapter *adapter = dev->data->dev_private; 7001 7002 adapter->systime_tc.nsec += delta; 7003 adapter->rx_tstamp_tc.nsec += delta; 7004 adapter->tx_tstamp_tc.nsec += delta; 7005 7006 return 0; 7007 } 7008 7009 static int 7010 ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 7011 { 7012 uint64_t ns; 7013 struct ixgbe_adapter *adapter = dev->data->dev_private; 7014 7015 ns = rte_timespec_to_ns(ts); 7016 /* Set the timecounters to a new value. */ 7017 adapter->systime_tc.nsec = ns; 7018 adapter->rx_tstamp_tc.nsec = ns; 7019 adapter->tx_tstamp_tc.nsec = ns; 7020 7021 return 0; 7022 } 7023 7024 static int 7025 ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 7026 { 7027 uint64_t ns, systime_cycles; 7028 struct ixgbe_adapter *adapter = dev->data->dev_private; 7029 7030 systime_cycles = ixgbe_read_systime_cyclecounter(dev); 7031 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 7032 *ts = rte_ns_to_timespec(ns); 7033 7034 return 0; 7035 } 7036 7037 static int 7038 ixgbe_timesync_enable(struct rte_eth_dev *dev) 7039 { 7040 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7041 uint32_t tsync_ctl; 7042 uint32_t tsauxc; 7043 7044 /* Stop the timesync system time. */ 7045 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0x0); 7046 /* Reset the timesync system time value. */ 7047 IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x0); 7048 IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x0); 7049 7050 /* Enable system time for platforms where it isn't on by default. */ 7051 tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); 7052 tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME; 7053 IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc); 7054 7055 ixgbe_start_timecounters(dev); 7056 7057 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7058 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 7059 (RTE_ETHER_TYPE_1588 | 7060 IXGBE_ETQF_FILTER_EN | 7061 IXGBE_ETQF_1588)); 7062 7063 /* Enable timestamping of received PTP packets. */ 7064 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7065 tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED; 7066 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7067 7068 /* Enable timestamping of transmitted PTP packets. */ 7069 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7070 tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED; 7071 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7072 7073 IXGBE_WRITE_FLUSH(hw); 7074 7075 return 0; 7076 } 7077 7078 static int 7079 ixgbe_timesync_disable(struct rte_eth_dev *dev) 7080 { 7081 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7082 uint32_t tsync_ctl; 7083 7084 /* Disable timestamping of transmitted PTP packets. */ 7085 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7086 tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED; 7087 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7088 7089 /* Disable timestamping of received PTP packets. */ 7090 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7091 tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED; 7092 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7093 7094 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7095 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0); 7096 7097 /* Stop incrementating the System Time registers. */ 7098 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0); 7099 7100 return 0; 7101 } 7102 7103 static int 7104 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 7105 struct timespec *timestamp, 7106 uint32_t flags __rte_unused) 7107 { 7108 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7109 struct ixgbe_adapter *adapter = dev->data->dev_private; 7110 uint32_t tsync_rxctl; 7111 uint64_t rx_tstamp_cycles; 7112 uint64_t ns; 7113 7114 tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7115 if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0) 7116 return -EINVAL; 7117 7118 rx_tstamp_cycles = ixgbe_read_rx_tstamp_cyclecounter(dev); 7119 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 7120 *timestamp = rte_ns_to_timespec(ns); 7121 7122 return 0; 7123 } 7124 7125 static int 7126 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 7127 struct timespec *timestamp) 7128 { 7129 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7130 struct ixgbe_adapter *adapter = dev->data->dev_private; 7131 uint32_t tsync_txctl; 7132 uint64_t tx_tstamp_cycles; 7133 uint64_t ns; 7134 7135 tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7136 if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0) 7137 return -EINVAL; 7138 7139 tx_tstamp_cycles = ixgbe_read_tx_tstamp_cyclecounter(dev); 7140 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 7141 *timestamp = rte_ns_to_timespec(ns); 7142 7143 return 0; 7144 } 7145 7146 static int 7147 ixgbe_get_reg_length(struct rte_eth_dev *dev) 7148 { 7149 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7150 int count = 0; 7151 int g_ind = 0; 7152 const struct reg_info *reg_group; 7153 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7154 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7155 7156 while ((reg_group = reg_set[g_ind++])) 7157 count += ixgbe_regs_group_count(reg_group); 7158 7159 return count; 7160 } 7161 7162 static int 7163 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 7164 { 7165 int count = 0; 7166 int g_ind = 0; 7167 const struct reg_info *reg_group; 7168 7169 while ((reg_group = ixgbevf_regs[g_ind++])) 7170 count += ixgbe_regs_group_count(reg_group); 7171 7172 return count; 7173 } 7174 7175 static int 7176 ixgbe_get_regs(struct rte_eth_dev *dev, 7177 struct rte_dev_reg_info *regs) 7178 { 7179 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7180 uint32_t *data = regs->data; 7181 int g_ind = 0; 7182 int count = 0; 7183 const struct reg_info *reg_group; 7184 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7185 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7186 7187 if (data == NULL) { 7188 regs->length = ixgbe_get_reg_length(dev); 7189 regs->width = sizeof(uint32_t); 7190 return 0; 7191 } 7192 7193 /* Support only full register dump */ 7194 if ((regs->length == 0) || 7195 (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) { 7196 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7197 hw->device_id; 7198 while ((reg_group = reg_set[g_ind++])) 7199 count += ixgbe_read_regs_group(dev, &data[count], 7200 reg_group); 7201 return 0; 7202 } 7203 7204 return -ENOTSUP; 7205 } 7206 7207 static int 7208 ixgbevf_get_regs(struct rte_eth_dev *dev, 7209 struct rte_dev_reg_info *regs) 7210 { 7211 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7212 uint32_t *data = regs->data; 7213 int g_ind = 0; 7214 int count = 0; 7215 const struct reg_info *reg_group; 7216 7217 if (data == NULL) { 7218 regs->length = ixgbevf_get_reg_length(dev); 7219 regs->width = sizeof(uint32_t); 7220 return 0; 7221 } 7222 7223 /* Support only full register dump */ 7224 if ((regs->length == 0) || 7225 (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) { 7226 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7227 hw->device_id; 7228 while ((reg_group = ixgbevf_regs[g_ind++])) 7229 count += ixgbe_read_regs_group(dev, &data[count], 7230 reg_group); 7231 return 0; 7232 } 7233 7234 return -ENOTSUP; 7235 } 7236 7237 static int 7238 ixgbe_get_eeprom_length(struct rte_eth_dev *dev) 7239 { 7240 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7241 7242 /* Return unit is byte count */ 7243 return hw->eeprom.word_size * 2; 7244 } 7245 7246 static int 7247 ixgbe_get_eeprom(struct rte_eth_dev *dev, 7248 struct rte_dev_eeprom_info *in_eeprom) 7249 { 7250 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7251 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7252 uint16_t *data = in_eeprom->data; 7253 int first, length; 7254 7255 first = in_eeprom->offset >> 1; 7256 length = in_eeprom->length >> 1; 7257 if ((first > hw->eeprom.word_size) || 7258 ((first + length) > hw->eeprom.word_size)) 7259 return -EINVAL; 7260 7261 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7262 7263 return eeprom->ops.read_buffer(hw, first, length, data); 7264 } 7265 7266 static int 7267 ixgbe_set_eeprom(struct rte_eth_dev *dev, 7268 struct rte_dev_eeprom_info *in_eeprom) 7269 { 7270 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7271 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7272 uint16_t *data = in_eeprom->data; 7273 int first, length; 7274 7275 first = in_eeprom->offset >> 1; 7276 length = in_eeprom->length >> 1; 7277 if ((first > hw->eeprom.word_size) || 7278 ((first + length) > hw->eeprom.word_size)) 7279 return -EINVAL; 7280 7281 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7282 7283 return eeprom->ops.write_buffer(hw, first, length, data); 7284 } 7285 7286 static int 7287 ixgbe_get_module_info(struct rte_eth_dev *dev, 7288 struct rte_eth_dev_module_info *modinfo) 7289 { 7290 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7291 uint32_t status; 7292 uint8_t sff8472_rev, addr_mode; 7293 bool page_swap = false; 7294 7295 /* Check whether we support SFF-8472 or not */ 7296 status = hw->phy.ops.read_i2c_eeprom(hw, 7297 IXGBE_SFF_SFF_8472_COMP, 7298 &sff8472_rev); 7299 if (status != 0) 7300 return -EIO; 7301 7302 /* addressing mode is not supported */ 7303 status = hw->phy.ops.read_i2c_eeprom(hw, 7304 IXGBE_SFF_SFF_8472_SWAP, 7305 &addr_mode); 7306 if (status != 0) 7307 return -EIO; 7308 7309 if (addr_mode & IXGBE_SFF_ADDRESSING_MODE) { 7310 PMD_DRV_LOG(ERR, 7311 "Address change required to access page 0xA2, " 7312 "but not supported. Please report the module " 7313 "type to the driver maintainers."); 7314 page_swap = true; 7315 } 7316 7317 if (sff8472_rev == IXGBE_SFF_SFF_8472_UNSUP || page_swap) { 7318 /* We have a SFP, but it does not support SFF-8472 */ 7319 modinfo->type = RTE_ETH_MODULE_SFF_8079; 7320 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 7321 } else { 7322 /* We have a SFP which supports a revision of SFF-8472. */ 7323 modinfo->type = RTE_ETH_MODULE_SFF_8472; 7324 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 7325 } 7326 7327 return 0; 7328 } 7329 7330 static int 7331 ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 7332 struct rte_dev_eeprom_info *info) 7333 { 7334 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7335 uint32_t status = IXGBE_ERR_PHY_ADDR_INVALID; 7336 uint8_t databyte = 0xFF; 7337 uint8_t *data = info->data; 7338 uint32_t i = 0; 7339 7340 for (i = info->offset; i < info->offset + info->length; i++) { 7341 if (i < RTE_ETH_MODULE_SFF_8079_LEN) 7342 status = hw->phy.ops.read_i2c_eeprom(hw, i, &databyte); 7343 else 7344 status = hw->phy.ops.read_i2c_sff8472(hw, i, &databyte); 7345 7346 if (status != 0) 7347 return -EIO; 7348 7349 data[i - info->offset] = databyte; 7350 } 7351 7352 return 0; 7353 } 7354 7355 uint16_t 7356 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) { 7357 switch (mac_type) { 7358 case ixgbe_mac_X550: 7359 case ixgbe_mac_X550EM_x: 7360 case ixgbe_mac_X550EM_a: 7361 return ETH_RSS_RETA_SIZE_512; 7362 case ixgbe_mac_X550_vf: 7363 case ixgbe_mac_X550EM_x_vf: 7364 case ixgbe_mac_X550EM_a_vf: 7365 return ETH_RSS_RETA_SIZE_64; 7366 case ixgbe_mac_X540_vf: 7367 case ixgbe_mac_82599_vf: 7368 return 0; 7369 default: 7370 return ETH_RSS_RETA_SIZE_128; 7371 } 7372 } 7373 7374 uint32_t 7375 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) { 7376 switch (mac_type) { 7377 case ixgbe_mac_X550: 7378 case ixgbe_mac_X550EM_x: 7379 case ixgbe_mac_X550EM_a: 7380 if (reta_idx < ETH_RSS_RETA_SIZE_128) 7381 return IXGBE_RETA(reta_idx >> 2); 7382 else 7383 return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2); 7384 case ixgbe_mac_X550_vf: 7385 case ixgbe_mac_X550EM_x_vf: 7386 case ixgbe_mac_X550EM_a_vf: 7387 return IXGBE_VFRETA(reta_idx >> 2); 7388 default: 7389 return IXGBE_RETA(reta_idx >> 2); 7390 } 7391 } 7392 7393 uint32_t 7394 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) { 7395 switch (mac_type) { 7396 case ixgbe_mac_X550_vf: 7397 case ixgbe_mac_X550EM_x_vf: 7398 case ixgbe_mac_X550EM_a_vf: 7399 return IXGBE_VFMRQC; 7400 default: 7401 return IXGBE_MRQC; 7402 } 7403 } 7404 7405 uint32_t 7406 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) { 7407 switch (mac_type) { 7408 case ixgbe_mac_X550_vf: 7409 case ixgbe_mac_X550EM_x_vf: 7410 case ixgbe_mac_X550EM_a_vf: 7411 return IXGBE_VFRSSRK(i); 7412 default: 7413 return IXGBE_RSSRK(i); 7414 } 7415 } 7416 7417 bool 7418 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) { 7419 switch (mac_type) { 7420 case ixgbe_mac_82599_vf: 7421 case ixgbe_mac_X540_vf: 7422 return 0; 7423 default: 7424 return 1; 7425 } 7426 } 7427 7428 static int 7429 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 7430 struct rte_eth_dcb_info *dcb_info) 7431 { 7432 struct ixgbe_dcb_config *dcb_config = 7433 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 7434 struct ixgbe_dcb_tc_config *tc; 7435 struct rte_eth_dcb_tc_queue_mapping *tc_queue; 7436 uint8_t nb_tcs; 7437 uint8_t i, j; 7438 7439 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) 7440 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs; 7441 else 7442 dcb_info->nb_tcs = 1; 7443 7444 tc_queue = &dcb_info->tc_queue; 7445 nb_tcs = dcb_info->nb_tcs; 7446 7447 if (dcb_config->vt_mode) { /* vt is enabled*/ 7448 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 7449 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 7450 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7451 dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i]; 7452 if (RTE_ETH_DEV_SRIOV(dev).active > 0) { 7453 for (j = 0; j < nb_tcs; j++) { 7454 tc_queue->tc_rxq[0][j].base = j; 7455 tc_queue->tc_rxq[0][j].nb_queue = 1; 7456 tc_queue->tc_txq[0][j].base = j; 7457 tc_queue->tc_txq[0][j].nb_queue = 1; 7458 } 7459 } else { 7460 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { 7461 for (j = 0; j < nb_tcs; j++) { 7462 tc_queue->tc_rxq[i][j].base = 7463 i * nb_tcs + j; 7464 tc_queue->tc_rxq[i][j].nb_queue = 1; 7465 tc_queue->tc_txq[i][j].base = 7466 i * nb_tcs + j; 7467 tc_queue->tc_txq[i][j].nb_queue = 1; 7468 } 7469 } 7470 } 7471 } else { /* vt is disabled*/ 7472 struct rte_eth_dcb_rx_conf *rx_conf = 7473 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 7474 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7475 dcb_info->prio_tc[i] = rx_conf->dcb_tc[i]; 7476 if (dcb_info->nb_tcs == ETH_4_TCS) { 7477 for (i = 0; i < dcb_info->nb_tcs; i++) { 7478 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32; 7479 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7480 } 7481 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7482 dcb_info->tc_queue.tc_txq[0][1].base = 64; 7483 dcb_info->tc_queue.tc_txq[0][2].base = 96; 7484 dcb_info->tc_queue.tc_txq[0][3].base = 112; 7485 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64; 7486 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7487 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7488 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7489 } else if (dcb_info->nb_tcs == ETH_8_TCS) { 7490 for (i = 0; i < dcb_info->nb_tcs; i++) { 7491 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16; 7492 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7493 } 7494 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7495 dcb_info->tc_queue.tc_txq[0][1].base = 32; 7496 dcb_info->tc_queue.tc_txq[0][2].base = 64; 7497 dcb_info->tc_queue.tc_txq[0][3].base = 80; 7498 dcb_info->tc_queue.tc_txq[0][4].base = 96; 7499 dcb_info->tc_queue.tc_txq[0][5].base = 104; 7500 dcb_info->tc_queue.tc_txq[0][6].base = 112; 7501 dcb_info->tc_queue.tc_txq[0][7].base = 120; 7502 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32; 7503 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7504 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7505 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7506 dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8; 7507 dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8; 7508 dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8; 7509 dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8; 7510 } 7511 } 7512 for (i = 0; i < dcb_info->nb_tcs; i++) { 7513 tc = &dcb_config->tc_config[i]; 7514 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent; 7515 } 7516 return 0; 7517 } 7518 7519 /* Update e-tag ether type */ 7520 static int 7521 ixgbe_update_e_tag_eth_type(struct ixgbe_hw *hw, 7522 uint16_t ether_type) 7523 { 7524 uint32_t etag_etype; 7525 7526 if (hw->mac.type != ixgbe_mac_X550 && 7527 hw->mac.type != ixgbe_mac_X550EM_x && 7528 hw->mac.type != ixgbe_mac_X550EM_a) { 7529 return -ENOTSUP; 7530 } 7531 7532 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7533 etag_etype &= ~IXGBE_ETAG_ETYPE_MASK; 7534 etag_etype |= ether_type; 7535 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7536 IXGBE_WRITE_FLUSH(hw); 7537 7538 return 0; 7539 } 7540 7541 /* Enable e-tag tunnel */ 7542 static int 7543 ixgbe_e_tag_enable(struct ixgbe_hw *hw) 7544 { 7545 uint32_t etag_etype; 7546 7547 if (hw->mac.type != ixgbe_mac_X550 && 7548 hw->mac.type != ixgbe_mac_X550EM_x && 7549 hw->mac.type != ixgbe_mac_X550EM_a) { 7550 return -ENOTSUP; 7551 } 7552 7553 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7554 etag_etype |= IXGBE_ETAG_ETYPE_VALID; 7555 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7556 IXGBE_WRITE_FLUSH(hw); 7557 7558 return 0; 7559 } 7560 7561 static int 7562 ixgbe_e_tag_filter_del(struct rte_eth_dev *dev, 7563 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7564 { 7565 int ret = 0; 7566 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7567 uint32_t i, rar_entries; 7568 uint32_t rar_low, rar_high; 7569 7570 if (hw->mac.type != ixgbe_mac_X550 && 7571 hw->mac.type != ixgbe_mac_X550EM_x && 7572 hw->mac.type != ixgbe_mac_X550EM_a) { 7573 return -ENOTSUP; 7574 } 7575 7576 rar_entries = ixgbe_get_num_rx_addrs(hw); 7577 7578 for (i = 1; i < rar_entries; i++) { 7579 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7580 rar_low = IXGBE_READ_REG(hw, IXGBE_RAL(i)); 7581 if ((rar_high & IXGBE_RAH_AV) && 7582 (rar_high & IXGBE_RAH_ADTYPE) && 7583 ((rar_low & IXGBE_RAL_ETAG_FILTER_MASK) == 7584 l2_tunnel->tunnel_id)) { 7585 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0); 7586 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0); 7587 7588 ixgbe_clear_vmdq(hw, i, IXGBE_CLEAR_VMDQ_ALL); 7589 7590 return ret; 7591 } 7592 } 7593 7594 return ret; 7595 } 7596 7597 static int 7598 ixgbe_e_tag_filter_add(struct rte_eth_dev *dev, 7599 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7600 { 7601 int ret = 0; 7602 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7603 uint32_t i, rar_entries; 7604 uint32_t rar_low, rar_high; 7605 7606 if (hw->mac.type != ixgbe_mac_X550 && 7607 hw->mac.type != ixgbe_mac_X550EM_x && 7608 hw->mac.type != ixgbe_mac_X550EM_a) { 7609 return -ENOTSUP; 7610 } 7611 7612 /* One entry for one tunnel. Try to remove potential existing entry. */ 7613 ixgbe_e_tag_filter_del(dev, l2_tunnel); 7614 7615 rar_entries = ixgbe_get_num_rx_addrs(hw); 7616 7617 for (i = 1; i < rar_entries; i++) { 7618 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7619 if (rar_high & IXGBE_RAH_AV) { 7620 continue; 7621 } else { 7622 ixgbe_set_vmdq(hw, i, l2_tunnel->pool); 7623 rar_high = IXGBE_RAH_AV | IXGBE_RAH_ADTYPE; 7624 rar_low = l2_tunnel->tunnel_id; 7625 7626 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), rar_low); 7627 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), rar_high); 7628 7629 return ret; 7630 } 7631 } 7632 7633 PMD_INIT_LOG(NOTICE, "The table of E-tag forwarding rule is full." 7634 " Please remove a rule before adding a new one."); 7635 return -EINVAL; 7636 } 7637 7638 static inline struct ixgbe_l2_tn_filter * 7639 ixgbe_l2_tn_filter_lookup(struct ixgbe_l2_tn_info *l2_tn_info, 7640 struct ixgbe_l2_tn_key *key) 7641 { 7642 int ret; 7643 7644 ret = rte_hash_lookup(l2_tn_info->hash_handle, (const void *)key); 7645 if (ret < 0) 7646 return NULL; 7647 7648 return l2_tn_info->hash_map[ret]; 7649 } 7650 7651 static inline int 7652 ixgbe_insert_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7653 struct ixgbe_l2_tn_filter *l2_tn_filter) 7654 { 7655 int ret; 7656 7657 ret = rte_hash_add_key(l2_tn_info->hash_handle, 7658 &l2_tn_filter->key); 7659 7660 if (ret < 0) { 7661 PMD_DRV_LOG(ERR, 7662 "Failed to insert L2 tunnel filter" 7663 " to hash table %d!", 7664 ret); 7665 return ret; 7666 } 7667 7668 l2_tn_info->hash_map[ret] = l2_tn_filter; 7669 7670 TAILQ_INSERT_TAIL(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7671 7672 return 0; 7673 } 7674 7675 static inline int 7676 ixgbe_remove_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7677 struct ixgbe_l2_tn_key *key) 7678 { 7679 int ret; 7680 struct ixgbe_l2_tn_filter *l2_tn_filter; 7681 7682 ret = rte_hash_del_key(l2_tn_info->hash_handle, key); 7683 7684 if (ret < 0) { 7685 PMD_DRV_LOG(ERR, 7686 "No such L2 tunnel filter to delete %d!", 7687 ret); 7688 return ret; 7689 } 7690 7691 l2_tn_filter = l2_tn_info->hash_map[ret]; 7692 l2_tn_info->hash_map[ret] = NULL; 7693 7694 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7695 rte_free(l2_tn_filter); 7696 7697 return 0; 7698 } 7699 7700 /* Add l2 tunnel filter */ 7701 int 7702 ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, 7703 struct ixgbe_l2_tunnel_conf *l2_tunnel, 7704 bool restore) 7705 { 7706 int ret; 7707 struct ixgbe_l2_tn_info *l2_tn_info = 7708 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7709 struct ixgbe_l2_tn_key key; 7710 struct ixgbe_l2_tn_filter *node; 7711 7712 if (!restore) { 7713 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7714 key.tn_id = l2_tunnel->tunnel_id; 7715 7716 node = ixgbe_l2_tn_filter_lookup(l2_tn_info, &key); 7717 7718 if (node) { 7719 PMD_DRV_LOG(ERR, 7720 "The L2 tunnel filter already exists!"); 7721 return -EINVAL; 7722 } 7723 7724 node = rte_zmalloc("ixgbe_l2_tn", 7725 sizeof(struct ixgbe_l2_tn_filter), 7726 0); 7727 if (!node) 7728 return -ENOMEM; 7729 7730 rte_memcpy(&node->key, 7731 &key, 7732 sizeof(struct ixgbe_l2_tn_key)); 7733 node->pool = l2_tunnel->pool; 7734 ret = ixgbe_insert_l2_tn_filter(l2_tn_info, node); 7735 if (ret < 0) { 7736 rte_free(node); 7737 return ret; 7738 } 7739 } 7740 7741 switch (l2_tunnel->l2_tunnel_type) { 7742 case RTE_L2_TUNNEL_TYPE_E_TAG: 7743 ret = ixgbe_e_tag_filter_add(dev, l2_tunnel); 7744 break; 7745 default: 7746 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7747 ret = -EINVAL; 7748 break; 7749 } 7750 7751 if ((!restore) && (ret < 0)) 7752 (void)ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7753 7754 return ret; 7755 } 7756 7757 /* Delete l2 tunnel filter */ 7758 int 7759 ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, 7760 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7761 { 7762 int ret; 7763 struct ixgbe_l2_tn_info *l2_tn_info = 7764 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7765 struct ixgbe_l2_tn_key key; 7766 7767 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7768 key.tn_id = l2_tunnel->tunnel_id; 7769 ret = ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7770 if (ret < 0) 7771 return ret; 7772 7773 switch (l2_tunnel->l2_tunnel_type) { 7774 case RTE_L2_TUNNEL_TYPE_E_TAG: 7775 ret = ixgbe_e_tag_filter_del(dev, l2_tunnel); 7776 break; 7777 default: 7778 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7779 ret = -EINVAL; 7780 break; 7781 } 7782 7783 return ret; 7784 } 7785 7786 static int 7787 ixgbe_e_tag_forwarding_en_dis(struct rte_eth_dev *dev, bool en) 7788 { 7789 int ret = 0; 7790 uint32_t ctrl; 7791 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7792 7793 if (hw->mac.type != ixgbe_mac_X550 && 7794 hw->mac.type != ixgbe_mac_X550EM_x && 7795 hw->mac.type != ixgbe_mac_X550EM_a) { 7796 return -ENOTSUP; 7797 } 7798 7799 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 7800 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 7801 if (en) 7802 ctrl |= IXGBE_VT_CTL_POOLING_MODE_ETAG; 7803 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 7804 7805 return ret; 7806 } 7807 7808 static int 7809 ixgbe_update_vxlan_port(struct ixgbe_hw *hw, 7810 uint16_t port) 7811 { 7812 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, port); 7813 IXGBE_WRITE_FLUSH(hw); 7814 7815 return 0; 7816 } 7817 7818 /* There's only one register for VxLAN UDP port. 7819 * So, we cannot add several ports. Will update it. 7820 */ 7821 static int 7822 ixgbe_add_vxlan_port(struct ixgbe_hw *hw, 7823 uint16_t port) 7824 { 7825 if (port == 0) { 7826 PMD_DRV_LOG(ERR, "Add VxLAN port 0 is not allowed."); 7827 return -EINVAL; 7828 } 7829 7830 return ixgbe_update_vxlan_port(hw, port); 7831 } 7832 7833 /* We cannot delete the VxLAN port. For there's a register for VxLAN 7834 * UDP port, it must have a value. 7835 * So, will reset it to the original value 0. 7836 */ 7837 static int 7838 ixgbe_del_vxlan_port(struct ixgbe_hw *hw, 7839 uint16_t port) 7840 { 7841 uint16_t cur_port; 7842 7843 cur_port = (uint16_t)IXGBE_READ_REG(hw, IXGBE_VXLANCTRL); 7844 7845 if (cur_port != port) { 7846 PMD_DRV_LOG(ERR, "Port %u does not exist.", port); 7847 return -EINVAL; 7848 } 7849 7850 return ixgbe_update_vxlan_port(hw, 0); 7851 } 7852 7853 /* Add UDP tunneling port */ 7854 static int 7855 ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 7856 struct rte_eth_udp_tunnel *udp_tunnel) 7857 { 7858 int ret = 0; 7859 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7860 7861 if (hw->mac.type != ixgbe_mac_X550 && 7862 hw->mac.type != ixgbe_mac_X550EM_x && 7863 hw->mac.type != ixgbe_mac_X550EM_a) { 7864 return -ENOTSUP; 7865 } 7866 7867 if (udp_tunnel == NULL) 7868 return -EINVAL; 7869 7870 switch (udp_tunnel->prot_type) { 7871 case RTE_TUNNEL_TYPE_VXLAN: 7872 ret = ixgbe_add_vxlan_port(hw, udp_tunnel->udp_port); 7873 break; 7874 7875 case RTE_TUNNEL_TYPE_GENEVE: 7876 case RTE_TUNNEL_TYPE_TEREDO: 7877 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 7878 ret = -EINVAL; 7879 break; 7880 7881 default: 7882 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7883 ret = -EINVAL; 7884 break; 7885 } 7886 7887 return ret; 7888 } 7889 7890 /* Remove UDP tunneling port */ 7891 static int 7892 ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 7893 struct rte_eth_udp_tunnel *udp_tunnel) 7894 { 7895 int ret = 0; 7896 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7897 7898 if (hw->mac.type != ixgbe_mac_X550 && 7899 hw->mac.type != ixgbe_mac_X550EM_x && 7900 hw->mac.type != ixgbe_mac_X550EM_a) { 7901 return -ENOTSUP; 7902 } 7903 7904 if (udp_tunnel == NULL) 7905 return -EINVAL; 7906 7907 switch (udp_tunnel->prot_type) { 7908 case RTE_TUNNEL_TYPE_VXLAN: 7909 ret = ixgbe_del_vxlan_port(hw, udp_tunnel->udp_port); 7910 break; 7911 case RTE_TUNNEL_TYPE_GENEVE: 7912 case RTE_TUNNEL_TYPE_TEREDO: 7913 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 7914 ret = -EINVAL; 7915 break; 7916 default: 7917 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7918 ret = -EINVAL; 7919 break; 7920 } 7921 7922 return ret; 7923 } 7924 7925 static int 7926 ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev) 7927 { 7928 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7929 int ret; 7930 7931 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_PROMISC)) { 7932 case IXGBE_SUCCESS: 7933 ret = 0; 7934 break; 7935 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7936 ret = -ENOTSUP; 7937 break; 7938 default: 7939 ret = -EAGAIN; 7940 break; 7941 } 7942 7943 return ret; 7944 } 7945 7946 static int 7947 ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev) 7948 { 7949 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7950 int ret; 7951 7952 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_NONE)) { 7953 case IXGBE_SUCCESS: 7954 ret = 0; 7955 break; 7956 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7957 ret = -ENOTSUP; 7958 break; 7959 default: 7960 ret = -EAGAIN; 7961 break; 7962 } 7963 7964 return ret; 7965 } 7966 7967 static int 7968 ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) 7969 { 7970 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7971 int ret; 7972 int mode = IXGBEVF_XCAST_MODE_ALLMULTI; 7973 7974 switch (hw->mac.ops.update_xcast_mode(hw, mode)) { 7975 case IXGBE_SUCCESS: 7976 ret = 0; 7977 break; 7978 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7979 ret = -ENOTSUP; 7980 break; 7981 default: 7982 ret = -EAGAIN; 7983 break; 7984 } 7985 7986 return ret; 7987 } 7988 7989 static int 7990 ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) 7991 { 7992 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7993 int ret; 7994 7995 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI)) { 7996 case IXGBE_SUCCESS: 7997 ret = 0; 7998 break; 7999 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 8000 ret = -ENOTSUP; 8001 break; 8002 default: 8003 ret = -EAGAIN; 8004 break; 8005 } 8006 8007 return ret; 8008 } 8009 8010 static void ixgbevf_mbx_process(struct rte_eth_dev *dev) 8011 { 8012 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8013 u32 in_msg = 0; 8014 8015 /* peek the message first */ 8016 in_msg = IXGBE_READ_REG(hw, IXGBE_VFMBMEM); 8017 8018 /* PF reset VF event */ 8019 if (in_msg == IXGBE_PF_CONTROL_MSG) { 8020 /* dummy mbx read to ack pf */ 8021 if (ixgbe_read_mbx(hw, &in_msg, 1, 0)) 8022 return; 8023 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 8024 NULL); 8025 } 8026 } 8027 8028 static int 8029 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev) 8030 { 8031 uint32_t eicr; 8032 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8033 struct ixgbe_interrupt *intr = 8034 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8035 ixgbevf_intr_disable(dev); 8036 8037 /* read-on-clear nic registers here */ 8038 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR); 8039 intr->flags = 0; 8040 8041 /* only one misc vector supported - mailbox */ 8042 eicr &= IXGBE_VTEICR_MASK; 8043 if (eicr == IXGBE_MISC_VEC_ID) 8044 intr->flags |= IXGBE_FLAG_MAILBOX; 8045 8046 return 0; 8047 } 8048 8049 static int 8050 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev) 8051 { 8052 struct ixgbe_interrupt *intr = 8053 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8054 8055 if (intr->flags & IXGBE_FLAG_MAILBOX) { 8056 ixgbevf_mbx_process(dev); 8057 intr->flags &= ~IXGBE_FLAG_MAILBOX; 8058 } 8059 8060 ixgbevf_intr_enable(dev); 8061 8062 return 0; 8063 } 8064 8065 static void 8066 ixgbevf_dev_interrupt_handler(void *param) 8067 { 8068 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 8069 8070 ixgbevf_dev_interrupt_get_status(dev); 8071 ixgbevf_dev_interrupt_action(dev); 8072 } 8073 8074 /** 8075 * ixgbe_disable_sec_tx_path_generic - Stops the transmit data path 8076 * @hw: pointer to hardware structure 8077 * 8078 * Stops the transmit data path and waits for the HW to internally empty 8079 * the Tx security block 8080 **/ 8081 int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw) 8082 { 8083 #define IXGBE_MAX_SECTX_POLL 40 8084 8085 int i; 8086 int sectxreg; 8087 8088 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8089 sectxreg |= IXGBE_SECTXCTRL_TX_DIS; 8090 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8091 for (i = 0; i < IXGBE_MAX_SECTX_POLL; i++) { 8092 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT); 8093 if (sectxreg & IXGBE_SECTXSTAT_SECTX_RDY) 8094 break; 8095 /* Use interrupt-safe sleep just in case */ 8096 usec_delay(1000); 8097 } 8098 8099 /* For informational purposes only */ 8100 if (i >= IXGBE_MAX_SECTX_POLL) 8101 PMD_DRV_LOG(DEBUG, "Tx unit being enabled before security " 8102 "path fully disabled. Continuing with init."); 8103 8104 return IXGBE_SUCCESS; 8105 } 8106 8107 /** 8108 * ixgbe_enable_sec_tx_path_generic - Enables the transmit data path 8109 * @hw: pointer to hardware structure 8110 * 8111 * Enables the transmit data path. 8112 **/ 8113 int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw) 8114 { 8115 uint32_t sectxreg; 8116 8117 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8118 sectxreg &= ~IXGBE_SECTXCTRL_TX_DIS; 8119 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8120 IXGBE_WRITE_FLUSH(hw); 8121 8122 return IXGBE_SUCCESS; 8123 } 8124 8125 /* restore n-tuple filter */ 8126 static inline void 8127 ixgbe_ntuple_filter_restore(struct rte_eth_dev *dev) 8128 { 8129 struct ixgbe_filter_info *filter_info = 8130 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8131 struct ixgbe_5tuple_filter *node; 8132 8133 TAILQ_FOREACH(node, &filter_info->fivetuple_list, entries) { 8134 ixgbe_inject_5tuple_filter(dev, node); 8135 } 8136 } 8137 8138 /* restore ethernet type filter */ 8139 static inline void 8140 ixgbe_ethertype_filter_restore(struct rte_eth_dev *dev) 8141 { 8142 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8143 struct ixgbe_filter_info *filter_info = 8144 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8145 int i; 8146 8147 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8148 if (filter_info->ethertype_mask & (1 << i)) { 8149 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 8150 filter_info->ethertype_filters[i].etqf); 8151 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 8152 filter_info->ethertype_filters[i].etqs); 8153 IXGBE_WRITE_FLUSH(hw); 8154 } 8155 } 8156 } 8157 8158 /* restore SYN filter */ 8159 static inline void 8160 ixgbe_syn_filter_restore(struct rte_eth_dev *dev) 8161 { 8162 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8163 struct ixgbe_filter_info *filter_info = 8164 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8165 uint32_t synqf; 8166 8167 synqf = filter_info->syn_info; 8168 8169 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 8170 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 8171 IXGBE_WRITE_FLUSH(hw); 8172 } 8173 } 8174 8175 /* restore L2 tunnel filter */ 8176 static inline void 8177 ixgbe_l2_tn_filter_restore(struct rte_eth_dev *dev) 8178 { 8179 struct ixgbe_l2_tn_info *l2_tn_info = 8180 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8181 struct ixgbe_l2_tn_filter *node; 8182 struct ixgbe_l2_tunnel_conf l2_tn_conf; 8183 8184 TAILQ_FOREACH(node, &l2_tn_info->l2_tn_list, entries) { 8185 l2_tn_conf.l2_tunnel_type = node->key.l2_tn_type; 8186 l2_tn_conf.tunnel_id = node->key.tn_id; 8187 l2_tn_conf.pool = node->pool; 8188 (void)ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_conf, TRUE); 8189 } 8190 } 8191 8192 /* restore rss filter */ 8193 static inline void 8194 ixgbe_rss_filter_restore(struct rte_eth_dev *dev) 8195 { 8196 struct ixgbe_filter_info *filter_info = 8197 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8198 8199 if (filter_info->rss_info.conf.queue_num) 8200 ixgbe_config_rss_filter(dev, 8201 &filter_info->rss_info, TRUE); 8202 } 8203 8204 static int 8205 ixgbe_filter_restore(struct rte_eth_dev *dev) 8206 { 8207 ixgbe_ntuple_filter_restore(dev); 8208 ixgbe_ethertype_filter_restore(dev); 8209 ixgbe_syn_filter_restore(dev); 8210 ixgbe_fdir_filter_restore(dev); 8211 ixgbe_l2_tn_filter_restore(dev); 8212 ixgbe_rss_filter_restore(dev); 8213 8214 return 0; 8215 } 8216 8217 static void 8218 ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev) 8219 { 8220 struct ixgbe_l2_tn_info *l2_tn_info = 8221 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8222 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8223 8224 if (l2_tn_info->e_tag_en) 8225 (void)ixgbe_e_tag_enable(hw); 8226 8227 if (l2_tn_info->e_tag_fwd_en) 8228 (void)ixgbe_e_tag_forwarding_en_dis(dev, 1); 8229 8230 (void)ixgbe_update_e_tag_eth_type(hw, l2_tn_info->e_tag_ether_type); 8231 } 8232 8233 /* remove all the n-tuple filters */ 8234 void 8235 ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev) 8236 { 8237 struct ixgbe_filter_info *filter_info = 8238 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8239 struct ixgbe_5tuple_filter *p_5tuple; 8240 8241 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) 8242 ixgbe_remove_5tuple_filter(dev, p_5tuple); 8243 } 8244 8245 /* remove all the ether type filters */ 8246 void 8247 ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev) 8248 { 8249 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8250 struct ixgbe_filter_info *filter_info = 8251 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8252 int i; 8253 8254 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8255 if (filter_info->ethertype_mask & (1 << i) && 8256 !filter_info->ethertype_filters[i].conf) { 8257 (void)ixgbe_ethertype_filter_remove(filter_info, 8258 (uint8_t)i); 8259 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 0); 8260 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 0); 8261 IXGBE_WRITE_FLUSH(hw); 8262 } 8263 } 8264 } 8265 8266 /* remove the SYN filter */ 8267 void 8268 ixgbe_clear_syn_filter(struct rte_eth_dev *dev) 8269 { 8270 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8271 struct ixgbe_filter_info *filter_info = 8272 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8273 8274 if (filter_info->syn_info & IXGBE_SYN_FILTER_ENABLE) { 8275 filter_info->syn_info = 0; 8276 8277 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, 0); 8278 IXGBE_WRITE_FLUSH(hw); 8279 } 8280 } 8281 8282 /* remove all the L2 tunnel filters */ 8283 int 8284 ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev) 8285 { 8286 struct ixgbe_l2_tn_info *l2_tn_info = 8287 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8288 struct ixgbe_l2_tn_filter *l2_tn_filter; 8289 struct ixgbe_l2_tunnel_conf l2_tn_conf; 8290 int ret = 0; 8291 8292 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 8293 l2_tn_conf.l2_tunnel_type = l2_tn_filter->key.l2_tn_type; 8294 l2_tn_conf.tunnel_id = l2_tn_filter->key.tn_id; 8295 l2_tn_conf.pool = l2_tn_filter->pool; 8296 ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_conf); 8297 if (ret < 0) 8298 return ret; 8299 } 8300 8301 return 0; 8302 } 8303 8304 void 8305 ixgbe_dev_macsec_setting_save(struct rte_eth_dev *dev, 8306 struct ixgbe_macsec_setting *macsec_setting) 8307 { 8308 struct ixgbe_macsec_setting *macsec = 8309 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 8310 8311 macsec->offload_en = macsec_setting->offload_en; 8312 macsec->encrypt_en = macsec_setting->encrypt_en; 8313 macsec->replayprotect_en = macsec_setting->replayprotect_en; 8314 } 8315 8316 void 8317 ixgbe_dev_macsec_setting_reset(struct rte_eth_dev *dev) 8318 { 8319 struct ixgbe_macsec_setting *macsec = 8320 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 8321 8322 macsec->offload_en = 0; 8323 macsec->encrypt_en = 0; 8324 macsec->replayprotect_en = 0; 8325 } 8326 8327 void 8328 ixgbe_dev_macsec_register_enable(struct rte_eth_dev *dev, 8329 struct ixgbe_macsec_setting *macsec_setting) 8330 { 8331 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8332 uint32_t ctrl; 8333 uint8_t en = macsec_setting->encrypt_en; 8334 uint8_t rp = macsec_setting->replayprotect_en; 8335 8336 /** 8337 * Workaround: 8338 * As no ixgbe_disable_sec_rx_path equivalent is 8339 * implemented for tx in the base code, and we are 8340 * not allowed to modify the base code in DPDK, so 8341 * just call the hand-written one directly for now. 8342 * The hardware support has been checked by 8343 * ixgbe_disable_sec_rx_path(). 8344 */ 8345 ixgbe_disable_sec_tx_path_generic(hw); 8346 8347 /* Enable Ethernet CRC (required by MACsec offload) */ 8348 ctrl = IXGBE_READ_REG(hw, IXGBE_HLREG0); 8349 ctrl |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP; 8350 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, ctrl); 8351 8352 /* Enable the TX and RX crypto engines */ 8353 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8354 ctrl &= ~IXGBE_SECTXCTRL_SECTX_DIS; 8355 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); 8356 8357 ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); 8358 ctrl &= ~IXGBE_SECRXCTRL_SECRX_DIS; 8359 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); 8360 8361 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 8362 ctrl &= ~IXGBE_SECTX_MINSECIFG_MASK; 8363 ctrl |= 0x3; 8364 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, ctrl); 8365 8366 /* Enable SA lookup */ 8367 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); 8368 ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; 8369 ctrl |= en ? IXGBE_LSECTXCTRL_AUTH_ENCRYPT : 8370 IXGBE_LSECTXCTRL_AUTH; 8371 ctrl |= IXGBE_LSECTXCTRL_AISCI; 8372 ctrl &= ~IXGBE_LSECTXCTRL_PNTHRSH_MASK; 8373 ctrl |= IXGBE_MACSEC_PNTHRSH & IXGBE_LSECTXCTRL_PNTHRSH_MASK; 8374 IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); 8375 8376 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); 8377 ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; 8378 ctrl |= IXGBE_LSECRXCTRL_STRICT << IXGBE_LSECRXCTRL_EN_SHIFT; 8379 ctrl &= ~IXGBE_LSECRXCTRL_PLSH; 8380 if (rp) 8381 ctrl |= IXGBE_LSECRXCTRL_RP; 8382 else 8383 ctrl &= ~IXGBE_LSECRXCTRL_RP; 8384 IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); 8385 8386 /* Start the data paths */ 8387 ixgbe_enable_sec_rx_path(hw); 8388 /** 8389 * Workaround: 8390 * As no ixgbe_enable_sec_rx_path equivalent is 8391 * implemented for tx in the base code, and we are 8392 * not allowed to modify the base code in DPDK, so 8393 * just call the hand-written one directly for now. 8394 */ 8395 ixgbe_enable_sec_tx_path_generic(hw); 8396 } 8397 8398 void 8399 ixgbe_dev_macsec_register_disable(struct rte_eth_dev *dev) 8400 { 8401 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8402 uint32_t ctrl; 8403 8404 /** 8405 * Workaround: 8406 * As no ixgbe_disable_sec_rx_path equivalent is 8407 * implemented for tx in the base code, and we are 8408 * not allowed to modify the base code in DPDK, so 8409 * just call the hand-written one directly for now. 8410 * The hardware support has been checked by 8411 * ixgbe_disable_sec_rx_path(). 8412 */ 8413 ixgbe_disable_sec_tx_path_generic(hw); 8414 8415 /* Disable the TX and RX crypto engines */ 8416 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8417 ctrl |= IXGBE_SECTXCTRL_SECTX_DIS; 8418 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); 8419 8420 ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); 8421 ctrl |= IXGBE_SECRXCTRL_SECRX_DIS; 8422 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); 8423 8424 /* Disable SA lookup */ 8425 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); 8426 ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; 8427 ctrl |= IXGBE_LSECTXCTRL_DISABLE; 8428 IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); 8429 8430 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); 8431 ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; 8432 ctrl |= IXGBE_LSECRXCTRL_DISABLE << IXGBE_LSECRXCTRL_EN_SHIFT; 8433 IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); 8434 8435 /* Start the data paths */ 8436 ixgbe_enable_sec_rx_path(hw); 8437 /** 8438 * Workaround: 8439 * As no ixgbe_enable_sec_rx_path equivalent is 8440 * implemented for tx in the base code, and we are 8441 * not allowed to modify the base code in DPDK, so 8442 * just call the hand-written one directly for now. 8443 */ 8444 ixgbe_enable_sec_tx_path_generic(hw); 8445 } 8446 8447 RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 8448 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe, pci_id_ixgbe_map); 8449 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe, "* igb_uio | uio_pci_generic | vfio-pci"); 8450 RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd); 8451 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map); 8452 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci"); 8453 RTE_PMD_REGISTER_PARAM_STRING(net_ixgbe_vf, 8454 IXGBEVF_DEVARG_PFLINK_FULLCHK "=<0|1>"); 8455 8456 RTE_LOG_REGISTER(ixgbe_logtype_init, pmd.net.ixgbe.init, NOTICE); 8457 RTE_LOG_REGISTER(ixgbe_logtype_driver, pmd.net.ixgbe.driver, NOTICE); 8458 8459 #ifdef RTE_ETHDEV_DEBUG_RX 8460 RTE_LOG_REGISTER(ixgbe_logtype_rx, pmd.net.ixgbe.rx, DEBUG); 8461 #endif 8462 #ifdef RTE_ETHDEV_DEBUG_TX 8463 RTE_LOG_REGISTER(ixgbe_logtype_tx, pmd.net.ixgbe.tx, DEBUG); 8464 #endif 8465