1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <netinet/in.h> 14 #include <rte_string_fns.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <rte_bus_pci.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_kvargs.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <rte_ethdev_driver.h> 31 #include <rte_ethdev_pci.h> 32 #include <rte_malloc.h> 33 #include <rte_random.h> 34 #include <rte_dev.h> 35 #include <rte_hash_crc.h> 36 #ifdef RTE_LIB_SECURITY 37 #include <rte_security_driver.h> 38 #endif 39 40 #include "ixgbe_logs.h" 41 #include "base/ixgbe_api.h" 42 #include "base/ixgbe_vf.h" 43 #include "base/ixgbe_common.h" 44 #include "ixgbe_ethdev.h" 45 #include "ixgbe_bypass.h" 46 #include "ixgbe_rxtx.h" 47 #include "base/ixgbe_type.h" 48 #include "base/ixgbe_phy.h" 49 #include "ixgbe_regs.h" 50 51 /* 52 * High threshold controlling when to start sending XOFF frames. Must be at 53 * least 8 bytes less than receive packet buffer size. This value is in units 54 * of 1024 bytes. 55 */ 56 #define IXGBE_FC_HI 0x80 57 58 /* 59 * Low threshold controlling when to start sending XON frames. This value is 60 * in units of 1024 bytes. 61 */ 62 #define IXGBE_FC_LO 0x40 63 64 /* Timer value included in XOFF frames. */ 65 #define IXGBE_FC_PAUSE 0x680 66 67 /*Default value of Max Rx Queue*/ 68 #define IXGBE_MAX_RX_QUEUE_NUM 128 69 70 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */ 71 #define IXGBE_LINK_UP_CHECK_TIMEOUT 1000 /* ms */ 72 #define IXGBE_VMDQ_NUM_UC_MAC 4096 /* Maximum nb. of UC MAC addr. */ 73 74 #define IXGBE_MMW_SIZE_DEFAULT 0x4 75 #define IXGBE_MMW_SIZE_JUMBO_FRAME 0x14 76 #define IXGBE_MAX_RING_DESC 4096 /* replicate define from rxtx */ 77 78 /* 79 * Default values for RX/TX configuration 80 */ 81 #define IXGBE_DEFAULT_RX_FREE_THRESH 32 82 #define IXGBE_DEFAULT_RX_PTHRESH 8 83 #define IXGBE_DEFAULT_RX_HTHRESH 8 84 #define IXGBE_DEFAULT_RX_WTHRESH 0 85 86 #define IXGBE_DEFAULT_TX_FREE_THRESH 32 87 #define IXGBE_DEFAULT_TX_PTHRESH 32 88 #define IXGBE_DEFAULT_TX_HTHRESH 0 89 #define IXGBE_DEFAULT_TX_WTHRESH 0 90 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32 91 92 /* Bit shift and mask */ 93 #define IXGBE_4_BIT_WIDTH (CHAR_BIT / 2) 94 #define IXGBE_4_BIT_MASK RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t) 95 #define IXGBE_8_BIT_WIDTH CHAR_BIT 96 #define IXGBE_8_BIT_MASK UINT8_MAX 97 98 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */ 99 100 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0])) 101 102 /* Additional timesync values. */ 103 #define NSEC_PER_SEC 1000000000L 104 #define IXGBE_INCVAL_10GB 0x66666666 105 #define IXGBE_INCVAL_1GB 0x40000000 106 #define IXGBE_INCVAL_100 0x50000000 107 #define IXGBE_INCVAL_SHIFT_10GB 28 108 #define IXGBE_INCVAL_SHIFT_1GB 24 109 #define IXGBE_INCVAL_SHIFT_100 21 110 #define IXGBE_INCVAL_SHIFT_82599 7 111 #define IXGBE_INCPER_SHIFT_82599 24 112 113 #define IXGBE_CYCLECOUNTER_MASK 0xffffffffffffffffULL 114 115 #define IXGBE_VT_CTL_POOLING_MODE_MASK 0x00030000 116 #define IXGBE_VT_CTL_POOLING_MODE_ETAG 0x00010000 117 #define IXGBE_ETAG_ETYPE 0x00005084 118 #define IXGBE_ETAG_ETYPE_MASK 0x0000ffff 119 #define IXGBE_ETAG_ETYPE_VALID 0x80000000 120 #define IXGBE_RAH_ADTYPE 0x40000000 121 #define IXGBE_RAL_ETAG_FILTER_MASK 0x00003fff 122 #define IXGBE_VMVIR_TAGA_MASK 0x18000000 123 #define IXGBE_VMVIR_TAGA_ETAG_INSERT 0x08000000 124 #define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */ 125 #define IXGBE_QDE_STRIP_TAG 0x00000004 126 #define IXGBE_VTEICR_MASK 0x07 127 128 #define IXGBE_EXVET_VET_EXT_SHIFT 16 129 #define IXGBE_DMATXCTL_VT_MASK 0xFFFF0000 130 131 #define IXGBEVF_DEVARG_PFLINK_FULLCHK "pflink_fullchk" 132 133 static const char * const ixgbevf_valid_arguments[] = { 134 IXGBEVF_DEVARG_PFLINK_FULLCHK, 135 NULL 136 }; 137 138 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params); 139 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev); 140 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev); 141 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev); 142 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev); 143 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev); 144 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev); 145 static int ixgbe_dev_configure(struct rte_eth_dev *dev); 146 static int ixgbe_dev_start(struct rte_eth_dev *dev); 147 static int ixgbe_dev_stop(struct rte_eth_dev *dev); 148 static int ixgbe_dev_set_link_up(struct rte_eth_dev *dev); 149 static int ixgbe_dev_set_link_down(struct rte_eth_dev *dev); 150 static int ixgbe_dev_close(struct rte_eth_dev *dev); 151 static int ixgbe_dev_reset(struct rte_eth_dev *dev); 152 static int ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev); 153 static int ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev); 154 static int ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev); 155 static int ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev); 156 static int ixgbe_dev_link_update(struct rte_eth_dev *dev, 157 int wait_to_complete); 158 static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, 159 struct rte_eth_stats *stats); 160 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev, 161 struct rte_eth_xstat *xstats, unsigned n); 162 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, 163 struct rte_eth_xstat *xstats, unsigned n); 164 static int 165 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 166 uint64_t *values, unsigned int n); 167 static int ixgbe_dev_stats_reset(struct rte_eth_dev *dev); 168 static int ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); 169 static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, 170 struct rte_eth_xstat_name *xstats_names, 171 unsigned int size); 172 static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, 173 struct rte_eth_xstat_name *xstats_names, unsigned limit); 174 static int ixgbe_dev_xstats_get_names_by_id( 175 struct rte_eth_dev *dev, 176 struct rte_eth_xstat_name *xstats_names, 177 const uint64_t *ids, 178 unsigned int limit); 179 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 180 uint16_t queue_id, 181 uint8_t stat_idx, 182 uint8_t is_rx); 183 static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 184 size_t fw_size); 185 static int ixgbe_dev_info_get(struct rte_eth_dev *dev, 186 struct rte_eth_dev_info *dev_info); 187 static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev); 188 static int ixgbevf_dev_info_get(struct rte_eth_dev *dev, 189 struct rte_eth_dev_info *dev_info); 190 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 191 192 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev, 193 uint16_t vlan_id, int on); 194 static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 195 enum rte_vlan_type vlan_type, 196 uint16_t tpid_id); 197 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, 198 uint16_t queue, bool on); 199 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, 200 int on); 201 static void ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, 202 int mask); 203 static int ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask); 204 static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); 205 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue); 206 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue); 207 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev); 208 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev); 209 210 static int ixgbe_dev_led_on(struct rte_eth_dev *dev); 211 static int ixgbe_dev_led_off(struct rte_eth_dev *dev); 212 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, 213 struct rte_eth_fc_conf *fc_conf); 214 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, 215 struct rte_eth_fc_conf *fc_conf); 216 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, 217 struct rte_eth_pfc_conf *pfc_conf); 218 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 219 struct rte_eth_rss_reta_entry64 *reta_conf, 220 uint16_t reta_size); 221 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 222 struct rte_eth_rss_reta_entry64 *reta_conf, 223 uint16_t reta_size); 224 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); 225 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 226 static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); 227 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); 228 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); 229 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev); 230 static void ixgbe_dev_interrupt_handler(void *param); 231 static void ixgbe_dev_interrupt_delayed_handler(void *param); 232 static void *ixgbe_dev_setup_link_thread_handler(void *param); 233 static int ixgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, 234 uint32_t timeout_ms); 235 236 static int ixgbe_add_rar(struct rte_eth_dev *dev, 237 struct rte_ether_addr *mac_addr, 238 uint32_t index, uint32_t pool); 239 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index); 240 static int ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, 241 struct rte_ether_addr *mac_addr); 242 static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config); 243 static bool is_device_supported(struct rte_eth_dev *dev, 244 struct rte_pci_driver *drv); 245 246 /* For Virtual Function support */ 247 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); 248 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); 249 static int ixgbevf_dev_configure(struct rte_eth_dev *dev); 250 static int ixgbevf_dev_start(struct rte_eth_dev *dev); 251 static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, 252 int wait_to_complete); 253 static int ixgbevf_dev_stop(struct rte_eth_dev *dev); 254 static int ixgbevf_dev_close(struct rte_eth_dev *dev); 255 static int ixgbevf_dev_reset(struct rte_eth_dev *dev); 256 static void ixgbevf_intr_disable(struct rte_eth_dev *dev); 257 static void ixgbevf_intr_enable(struct rte_eth_dev *dev); 258 static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, 259 struct rte_eth_stats *stats); 260 static int ixgbevf_dev_stats_reset(struct rte_eth_dev *dev); 261 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, 262 uint16_t vlan_id, int on); 263 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, 264 uint16_t queue, int on); 265 static int ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask); 266 static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); 267 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on); 268 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 269 uint16_t queue_id); 270 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 271 uint16_t queue_id); 272 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 273 uint8_t queue, uint8_t msix_vector); 274 static void ixgbevf_configure_msix(struct rte_eth_dev *dev); 275 static int ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev); 276 static int ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev); 277 static int ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev); 278 static int ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev); 279 280 /* For Eth VMDQ APIs support */ 281 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct 282 rte_ether_addr * mac_addr, uint8_t on); 283 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on); 284 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 285 struct rte_eth_mirror_conf *mirror_conf, 286 uint8_t rule_id, uint8_t on); 287 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, 288 uint8_t rule_id); 289 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 290 uint16_t queue_id); 291 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 292 uint16_t queue_id); 293 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 294 uint8_t queue, uint8_t msix_vector); 295 static void ixgbe_configure_msix(struct rte_eth_dev *dev); 296 297 static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, 298 struct rte_ether_addr *mac_addr, 299 uint32_t index, uint32_t pool); 300 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index); 301 static int ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 302 struct rte_ether_addr *mac_addr); 303 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 304 struct ixgbe_5tuple_filter *filter); 305 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 306 struct ixgbe_5tuple_filter *filter); 307 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, 308 enum rte_filter_type filter_type, 309 enum rte_filter_op filter_op, 310 void *arg); 311 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); 312 313 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 314 struct rte_ether_addr *mc_addr_set, 315 uint32_t nb_mc_addr); 316 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 317 struct rte_eth_dcb_info *dcb_info); 318 319 static int ixgbe_get_reg_length(struct rte_eth_dev *dev); 320 static int ixgbe_get_regs(struct rte_eth_dev *dev, 321 struct rte_dev_reg_info *regs); 322 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev); 323 static int ixgbe_get_eeprom(struct rte_eth_dev *dev, 324 struct rte_dev_eeprom_info *eeprom); 325 static int ixgbe_set_eeprom(struct rte_eth_dev *dev, 326 struct rte_dev_eeprom_info *eeprom); 327 328 static int ixgbe_get_module_info(struct rte_eth_dev *dev, 329 struct rte_eth_dev_module_info *modinfo); 330 static int ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 331 struct rte_dev_eeprom_info *info); 332 333 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev); 334 static int ixgbevf_get_regs(struct rte_eth_dev *dev, 335 struct rte_dev_reg_info *regs); 336 337 static int ixgbe_timesync_enable(struct rte_eth_dev *dev); 338 static int ixgbe_timesync_disable(struct rte_eth_dev *dev); 339 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 340 struct timespec *timestamp, 341 uint32_t flags); 342 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 343 struct timespec *timestamp); 344 static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 345 static int ixgbe_timesync_read_time(struct rte_eth_dev *dev, 346 struct timespec *timestamp); 347 static int ixgbe_timesync_write_time(struct rte_eth_dev *dev, 348 const struct timespec *timestamp); 349 static void ixgbevf_dev_interrupt_handler(void *param); 350 351 static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 352 struct rte_eth_udp_tunnel *udp_tunnel); 353 static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 354 struct rte_eth_udp_tunnel *udp_tunnel); 355 static int ixgbe_filter_restore(struct rte_eth_dev *dev); 356 static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev); 357 static int ixgbe_wait_for_link_up(struct ixgbe_hw *hw); 358 359 /* 360 * Define VF Stats MACRO for Non "cleared on read" register 361 */ 362 #define UPDATE_VF_STAT(reg, last, cur) \ 363 { \ 364 uint32_t latest = IXGBE_READ_REG(hw, reg); \ 365 cur += (latest - last) & UINT_MAX; \ 366 last = latest; \ 367 } 368 369 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur) \ 370 { \ 371 u64 new_lsb = IXGBE_READ_REG(hw, lsb); \ 372 u64 new_msb = IXGBE_READ_REG(hw, msb); \ 373 u64 latest = ((new_msb << 32) | new_lsb); \ 374 cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \ 375 last = latest; \ 376 } 377 378 #define IXGBE_SET_HWSTRIP(h, q) do {\ 379 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 380 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 381 (h)->bitmap[idx] |= 1 << bit;\ 382 } while (0) 383 384 #define IXGBE_CLEAR_HWSTRIP(h, q) do {\ 385 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 386 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 387 (h)->bitmap[idx] &= ~(1 << bit);\ 388 } while (0) 389 390 #define IXGBE_GET_HWSTRIP(h, q, r) do {\ 391 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 392 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 393 (r) = (h)->bitmap[idx] >> bit & 1;\ 394 } while (0) 395 396 /* 397 * The set of PCI devices this driver supports 398 */ 399 static const struct rte_pci_id pci_id_ixgbe_map[] = { 400 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) }, 401 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) }, 402 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) }, 403 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) }, 404 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) }, 405 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) }, 406 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) }, 407 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) }, 408 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) }, 409 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) }, 410 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) }, 411 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) }, 412 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) }, 413 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, 414 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, 415 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, 416 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, 417 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, 418 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, 419 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, 420 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, 421 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) }, 422 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) }, 423 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) }, 424 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) }, 425 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) }, 426 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) }, 427 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) }, 428 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) }, 429 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) }, 430 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) }, 431 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) }, 432 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) }, 433 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) }, 434 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) }, 435 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) }, 436 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) }, 437 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) }, 438 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) }, 439 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) }, 440 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) }, 441 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) }, 442 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) }, 443 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) }, 444 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, 445 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, 446 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, 447 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI) }, 448 #ifdef RTE_LIBRTE_IXGBE_BYPASS 449 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, 450 #endif 451 { .vendor_id = 0, /* sentinel */ }, 452 }; 453 454 /* 455 * The set of PCI devices this driver supports (for 82599 VF) 456 */ 457 static const struct rte_pci_id pci_id_ixgbevf_map[] = { 458 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) }, 459 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) }, 460 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) }, 461 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) }, 462 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) }, 463 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) }, 464 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) }, 465 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) }, 466 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) }, 467 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) }, 468 { .vendor_id = 0, /* sentinel */ }, 469 }; 470 471 static const struct rte_eth_desc_lim rx_desc_lim = { 472 .nb_max = IXGBE_MAX_RING_DESC, 473 .nb_min = IXGBE_MIN_RING_DESC, 474 .nb_align = IXGBE_RXD_ALIGN, 475 }; 476 477 static const struct rte_eth_desc_lim tx_desc_lim = { 478 .nb_max = IXGBE_MAX_RING_DESC, 479 .nb_min = IXGBE_MIN_RING_DESC, 480 .nb_align = IXGBE_TXD_ALIGN, 481 .nb_seg_max = IXGBE_TX_MAX_SEG, 482 .nb_mtu_seg_max = IXGBE_TX_MAX_SEG, 483 }; 484 485 static const struct eth_dev_ops ixgbe_eth_dev_ops = { 486 .dev_configure = ixgbe_dev_configure, 487 .dev_start = ixgbe_dev_start, 488 .dev_stop = ixgbe_dev_stop, 489 .dev_set_link_up = ixgbe_dev_set_link_up, 490 .dev_set_link_down = ixgbe_dev_set_link_down, 491 .dev_close = ixgbe_dev_close, 492 .dev_reset = ixgbe_dev_reset, 493 .promiscuous_enable = ixgbe_dev_promiscuous_enable, 494 .promiscuous_disable = ixgbe_dev_promiscuous_disable, 495 .allmulticast_enable = ixgbe_dev_allmulticast_enable, 496 .allmulticast_disable = ixgbe_dev_allmulticast_disable, 497 .link_update = ixgbe_dev_link_update, 498 .stats_get = ixgbe_dev_stats_get, 499 .xstats_get = ixgbe_dev_xstats_get, 500 .xstats_get_by_id = ixgbe_dev_xstats_get_by_id, 501 .stats_reset = ixgbe_dev_stats_reset, 502 .xstats_reset = ixgbe_dev_xstats_reset, 503 .xstats_get_names = ixgbe_dev_xstats_get_names, 504 .xstats_get_names_by_id = ixgbe_dev_xstats_get_names_by_id, 505 .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set, 506 .fw_version_get = ixgbe_fw_version_get, 507 .dev_infos_get = ixgbe_dev_info_get, 508 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 509 .mtu_set = ixgbe_dev_mtu_set, 510 .vlan_filter_set = ixgbe_vlan_filter_set, 511 .vlan_tpid_set = ixgbe_vlan_tpid_set, 512 .vlan_offload_set = ixgbe_vlan_offload_set, 513 .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set, 514 .rx_queue_start = ixgbe_dev_rx_queue_start, 515 .rx_queue_stop = ixgbe_dev_rx_queue_stop, 516 .tx_queue_start = ixgbe_dev_tx_queue_start, 517 .tx_queue_stop = ixgbe_dev_tx_queue_stop, 518 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 519 .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable, 520 .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable, 521 .rx_queue_release = ixgbe_dev_rx_queue_release, 522 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 523 .tx_queue_release = ixgbe_dev_tx_queue_release, 524 .dev_led_on = ixgbe_dev_led_on, 525 .dev_led_off = ixgbe_dev_led_off, 526 .flow_ctrl_get = ixgbe_flow_ctrl_get, 527 .flow_ctrl_set = ixgbe_flow_ctrl_set, 528 .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set, 529 .mac_addr_add = ixgbe_add_rar, 530 .mac_addr_remove = ixgbe_remove_rar, 531 .mac_addr_set = ixgbe_set_default_mac_addr, 532 .uc_hash_table_set = ixgbe_uc_hash_table_set, 533 .uc_all_hash_table_set = ixgbe_uc_all_hash_table_set, 534 .mirror_rule_set = ixgbe_mirror_rule_set, 535 .mirror_rule_reset = ixgbe_mirror_rule_reset, 536 .set_queue_rate_limit = ixgbe_set_queue_rate_limit, 537 .reta_update = ixgbe_dev_rss_reta_update, 538 .reta_query = ixgbe_dev_rss_reta_query, 539 .rss_hash_update = ixgbe_dev_rss_hash_update, 540 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 541 .filter_ctrl = ixgbe_dev_filter_ctrl, 542 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 543 .rxq_info_get = ixgbe_rxq_info_get, 544 .txq_info_get = ixgbe_txq_info_get, 545 .timesync_enable = ixgbe_timesync_enable, 546 .timesync_disable = ixgbe_timesync_disable, 547 .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, 548 .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp, 549 .get_reg = ixgbe_get_regs, 550 .get_eeprom_length = ixgbe_get_eeprom_length, 551 .get_eeprom = ixgbe_get_eeprom, 552 .set_eeprom = ixgbe_set_eeprom, 553 .get_module_info = ixgbe_get_module_info, 554 .get_module_eeprom = ixgbe_get_module_eeprom, 555 .get_dcb_info = ixgbe_dev_get_dcb_info, 556 .timesync_adjust_time = ixgbe_timesync_adjust_time, 557 .timesync_read_time = ixgbe_timesync_read_time, 558 .timesync_write_time = ixgbe_timesync_write_time, 559 .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, 560 .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, 561 .tm_ops_get = ixgbe_tm_ops_get, 562 .tx_done_cleanup = ixgbe_dev_tx_done_cleanup, 563 }; 564 565 /* 566 * dev_ops for virtual function, bare necessities for basic vf 567 * operation have been implemented 568 */ 569 static const struct eth_dev_ops ixgbevf_eth_dev_ops = { 570 .dev_configure = ixgbevf_dev_configure, 571 .dev_start = ixgbevf_dev_start, 572 .dev_stop = ixgbevf_dev_stop, 573 .link_update = ixgbevf_dev_link_update, 574 .stats_get = ixgbevf_dev_stats_get, 575 .xstats_get = ixgbevf_dev_xstats_get, 576 .stats_reset = ixgbevf_dev_stats_reset, 577 .xstats_reset = ixgbevf_dev_stats_reset, 578 .xstats_get_names = ixgbevf_dev_xstats_get_names, 579 .dev_close = ixgbevf_dev_close, 580 .dev_reset = ixgbevf_dev_reset, 581 .promiscuous_enable = ixgbevf_dev_promiscuous_enable, 582 .promiscuous_disable = ixgbevf_dev_promiscuous_disable, 583 .allmulticast_enable = ixgbevf_dev_allmulticast_enable, 584 .allmulticast_disable = ixgbevf_dev_allmulticast_disable, 585 .dev_infos_get = ixgbevf_dev_info_get, 586 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 587 .mtu_set = ixgbevf_dev_set_mtu, 588 .vlan_filter_set = ixgbevf_vlan_filter_set, 589 .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set, 590 .vlan_offload_set = ixgbevf_vlan_offload_set, 591 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 592 .rx_queue_release = ixgbe_dev_rx_queue_release, 593 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 594 .tx_queue_release = ixgbe_dev_tx_queue_release, 595 .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable, 596 .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable, 597 .mac_addr_add = ixgbevf_add_mac_addr, 598 .mac_addr_remove = ixgbevf_remove_mac_addr, 599 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 600 .rxq_info_get = ixgbe_rxq_info_get, 601 .txq_info_get = ixgbe_txq_info_get, 602 .mac_addr_set = ixgbevf_set_default_mac_addr, 603 .get_reg = ixgbevf_get_regs, 604 .reta_update = ixgbe_dev_rss_reta_update, 605 .reta_query = ixgbe_dev_rss_reta_query, 606 .rss_hash_update = ixgbe_dev_rss_hash_update, 607 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 608 .tx_done_cleanup = ixgbe_dev_tx_done_cleanup, 609 }; 610 611 /* store statistics names and its offset in stats structure */ 612 struct rte_ixgbe_xstats_name_off { 613 char name[RTE_ETH_XSTATS_NAME_SIZE]; 614 unsigned offset; 615 }; 616 617 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = { 618 {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)}, 619 {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)}, 620 {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)}, 621 {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)}, 622 {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)}, 623 {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)}, 624 {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)}, 625 {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)}, 626 {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)}, 627 {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)}, 628 {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)}, 629 {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)}, 630 {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)}, 631 {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)}, 632 {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 633 prc1023)}, 634 {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 635 prc1522)}, 636 {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)}, 637 {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)}, 638 {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)}, 639 {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)}, 640 {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)}, 641 {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)}, 642 {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)}, 643 {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)}, 644 {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)}, 645 {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)}, 646 {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)}, 647 {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)}, 648 {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)}, 649 {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)}, 650 {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)}, 651 {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)}, 652 {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 653 ptc1023)}, 654 {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 655 ptc1522)}, 656 {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)}, 657 {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)}, 658 {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)}, 659 {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)}, 660 661 {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats, 662 fdirustat_add)}, 663 {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats, 664 fdirustat_remove)}, 665 {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats, 666 fdirfstat_fadd)}, 667 {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats, 668 fdirfstat_fremove)}, 669 {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats, 670 fdirmatch)}, 671 {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats, 672 fdirmiss)}, 673 674 {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)}, 675 {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)}, 676 {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, 677 fclast)}, 678 {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)}, 679 {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)}, 680 {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)}, 681 {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)}, 682 {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats, 683 fcoe_noddp)}, 684 {"rx_fcoe_no_direct_data_placement_ext_buff", 685 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)}, 686 687 {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 688 lxontxc)}, 689 {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 690 lxonrxc)}, 691 {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 692 lxofftxc)}, 693 {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 694 lxoffrxc)}, 695 {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)}, 696 }; 697 698 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \ 699 sizeof(rte_ixgbe_stats_strings[0])) 700 701 /* MACsec statistics */ 702 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = { 703 {"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 704 out_pkts_untagged)}, 705 {"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats, 706 out_pkts_encrypted)}, 707 {"out_pkts_protected", offsetof(struct ixgbe_macsec_stats, 708 out_pkts_protected)}, 709 {"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats, 710 out_octets_encrypted)}, 711 {"out_octets_protected", offsetof(struct ixgbe_macsec_stats, 712 out_octets_protected)}, 713 {"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 714 in_pkts_untagged)}, 715 {"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats, 716 in_pkts_badtag)}, 717 {"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats, 718 in_pkts_nosci)}, 719 {"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats, 720 in_pkts_unknownsci)}, 721 {"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats, 722 in_octets_decrypted)}, 723 {"in_octets_validated", offsetof(struct ixgbe_macsec_stats, 724 in_octets_validated)}, 725 {"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats, 726 in_pkts_unchecked)}, 727 {"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats, 728 in_pkts_delayed)}, 729 {"in_pkts_late", offsetof(struct ixgbe_macsec_stats, 730 in_pkts_late)}, 731 {"in_pkts_ok", offsetof(struct ixgbe_macsec_stats, 732 in_pkts_ok)}, 733 {"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats, 734 in_pkts_invalid)}, 735 {"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats, 736 in_pkts_notvalid)}, 737 {"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats, 738 in_pkts_unusedsa)}, 739 {"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats, 740 in_pkts_notusingsa)}, 741 }; 742 743 #define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \ 744 sizeof(rte_ixgbe_macsec_strings[0])) 745 746 /* Per-queue statistics */ 747 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = { 748 {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)}, 749 {"dropped", offsetof(struct ixgbe_hw_stats, mpc)}, 750 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)}, 751 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)}, 752 }; 753 754 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \ 755 sizeof(rte_ixgbe_rxq_strings[0])) 756 #define IXGBE_NB_RXQ_PRIO_VALUES 8 757 758 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = { 759 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)}, 760 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)}, 761 {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats, 762 pxon2offc)}, 763 }; 764 765 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \ 766 sizeof(rte_ixgbe_txq_strings[0])) 767 #define IXGBE_NB_TXQ_PRIO_VALUES 8 768 769 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = { 770 {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)}, 771 }; 772 773 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) / \ 774 sizeof(rte_ixgbevf_stats_strings[0])) 775 776 /* 777 * This function is the same as ixgbe_is_sfp() in base/ixgbe.h. 778 */ 779 static inline int 780 ixgbe_is_sfp(struct ixgbe_hw *hw) 781 { 782 switch (hw->phy.type) { 783 case ixgbe_phy_sfp_avago: 784 case ixgbe_phy_sfp_ftl: 785 case ixgbe_phy_sfp_intel: 786 case ixgbe_phy_sfp_unknown: 787 case ixgbe_phy_sfp_passive_tyco: 788 case ixgbe_phy_sfp_passive_unknown: 789 return 1; 790 default: 791 return 0; 792 } 793 } 794 795 static inline int32_t 796 ixgbe_pf_reset_hw(struct ixgbe_hw *hw) 797 { 798 uint32_t ctrl_ext; 799 int32_t status; 800 801 status = ixgbe_reset_hw(hw); 802 803 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 804 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 805 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 806 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 807 IXGBE_WRITE_FLUSH(hw); 808 809 if (status == IXGBE_ERR_SFP_NOT_PRESENT) 810 status = IXGBE_SUCCESS; 811 return status; 812 } 813 814 static inline void 815 ixgbe_enable_intr(struct rte_eth_dev *dev) 816 { 817 struct ixgbe_interrupt *intr = 818 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 819 struct ixgbe_hw *hw = 820 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 821 822 IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask); 823 IXGBE_WRITE_FLUSH(hw); 824 } 825 826 /* 827 * This function is based on ixgbe_disable_intr() in base/ixgbe.h. 828 */ 829 static void 830 ixgbe_disable_intr(struct ixgbe_hw *hw) 831 { 832 PMD_INIT_FUNC_TRACE(); 833 834 if (hw->mac.type == ixgbe_mac_82598EB) { 835 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0); 836 } else { 837 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000); 838 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0); 839 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0); 840 } 841 IXGBE_WRITE_FLUSH(hw); 842 } 843 844 /* 845 * This function resets queue statistics mapping registers. 846 * From Niantic datasheet, Initialization of Statistics section: 847 * "...if software requires the queue counters, the RQSMR and TQSM registers 848 * must be re-programmed following a device reset. 849 */ 850 static void 851 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw) 852 { 853 uint32_t i; 854 855 for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) { 856 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0); 857 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0); 858 } 859 } 860 861 862 static int 863 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 864 uint16_t queue_id, 865 uint8_t stat_idx, 866 uint8_t is_rx) 867 { 868 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8 869 #define NB_QMAP_FIELDS_PER_QSM_REG 4 870 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f 871 872 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 873 struct ixgbe_stat_mapping_registers *stat_mappings = 874 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private); 875 uint32_t qsmr_mask = 0; 876 uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK; 877 uint32_t q_map; 878 uint8_t n, offset; 879 880 if ((hw->mac.type != ixgbe_mac_82599EB) && 881 (hw->mac.type != ixgbe_mac_X540) && 882 (hw->mac.type != ixgbe_mac_X550) && 883 (hw->mac.type != ixgbe_mac_X550EM_x) && 884 (hw->mac.type != ixgbe_mac_X550EM_a)) 885 return -ENOSYS; 886 887 PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d", 888 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 889 queue_id, stat_idx); 890 891 n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG); 892 if (n >= IXGBE_NB_STAT_MAPPING_REGS) { 893 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded"); 894 return -EIO; 895 } 896 offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG); 897 898 /* Now clear any previous stat_idx set */ 899 clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 900 if (!is_rx) 901 stat_mappings->tqsm[n] &= ~clearing_mask; 902 else 903 stat_mappings->rqsmr[n] &= ~clearing_mask; 904 905 q_map = (uint32_t)stat_idx; 906 q_map &= QMAP_FIELD_RESERVED_BITS_MASK; 907 qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 908 if (!is_rx) 909 stat_mappings->tqsm[n] |= qsmr_mask; 910 else 911 stat_mappings->rqsmr[n] |= qsmr_mask; 912 913 PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d", 914 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 915 queue_id, stat_idx); 916 PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n, 917 is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]); 918 919 /* Now write the mapping in the appropriate register */ 920 if (is_rx) { 921 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d", 922 stat_mappings->rqsmr[n], n); 923 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]); 924 } else { 925 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d", 926 stat_mappings->tqsm[n], n); 927 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]); 928 } 929 return 0; 930 } 931 932 static void 933 ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev) 934 { 935 struct ixgbe_stat_mapping_registers *stat_mappings = 936 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private); 937 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 938 int i; 939 940 /* write whatever was in stat mapping table to the NIC */ 941 for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) { 942 /* rx */ 943 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]); 944 945 /* tx */ 946 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]); 947 } 948 } 949 950 static void 951 ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config) 952 { 953 uint8_t i; 954 struct ixgbe_dcb_tc_config *tc; 955 uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS; 956 957 dcb_config->num_tcs.pg_tcs = dcb_max_tc; 958 dcb_config->num_tcs.pfc_tcs = dcb_max_tc; 959 for (i = 0; i < dcb_max_tc; i++) { 960 tc = &dcb_config->tc_config[i]; 961 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i; 962 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 963 (uint8_t)(100/dcb_max_tc + (i & 1)); 964 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i; 965 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 966 (uint8_t)(100/dcb_max_tc + (i & 1)); 967 tc->pfc = ixgbe_dcb_pfc_disabled; 968 } 969 970 /* Initialize default user to priority mapping, UPx->TC0 */ 971 tc = &dcb_config->tc_config[0]; 972 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 973 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 974 for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) { 975 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100; 976 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100; 977 } 978 dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal; 979 dcb_config->pfc_mode_enable = false; 980 dcb_config->vt_mode = true; 981 dcb_config->round_robin_enable = false; 982 /* support all DCB capabilities in 82599 */ 983 dcb_config->support.capabilities = 0xFF; 984 985 /*we only support 4 Tcs for X540, X550 */ 986 if (hw->mac.type == ixgbe_mac_X540 || 987 hw->mac.type == ixgbe_mac_X550 || 988 hw->mac.type == ixgbe_mac_X550EM_x || 989 hw->mac.type == ixgbe_mac_X550EM_a) { 990 dcb_config->num_tcs.pg_tcs = 4; 991 dcb_config->num_tcs.pfc_tcs = 4; 992 } 993 } 994 995 /* 996 * Ensure that all locks are released before first NVM or PHY access 997 */ 998 static void 999 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) 1000 { 1001 uint16_t mask; 1002 1003 /* 1004 * Phy lock should not fail in this early stage. If this is the case, 1005 * it is due to an improper exit of the application. 1006 * So force the release of the faulty lock. Release of common lock 1007 * is done automatically by swfw_sync function. 1008 */ 1009 mask = IXGBE_GSSR_PHY0_SM << hw->bus.func; 1010 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1011 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func); 1012 } 1013 ixgbe_release_swfw_semaphore(hw, mask); 1014 1015 /* 1016 * These ones are more tricky since they are common to all ports; but 1017 * swfw_sync retries last long enough (1s) to be almost sure that if 1018 * lock can not be taken it is due to an improper lock of the 1019 * semaphore. 1020 */ 1021 mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM; 1022 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1023 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 1024 } 1025 ixgbe_release_swfw_semaphore(hw, mask); 1026 } 1027 1028 /* 1029 * This function is based on code in ixgbe_attach() in base/ixgbe.c. 1030 * It returns 0 on success. 1031 */ 1032 static int 1033 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) 1034 { 1035 struct ixgbe_adapter *ad = eth_dev->data->dev_private; 1036 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1037 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1038 struct ixgbe_hw *hw = 1039 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1040 struct ixgbe_vfta *shadow_vfta = 1041 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1042 struct ixgbe_hwstrip *hwstrip = 1043 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1044 struct ixgbe_dcb_config *dcb_config = 1045 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private); 1046 struct ixgbe_filter_info *filter_info = 1047 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1048 struct ixgbe_bw_conf *bw_conf = 1049 IXGBE_DEV_PRIVATE_TO_BW_CONF(eth_dev->data->dev_private); 1050 uint32_t ctrl_ext; 1051 uint16_t csum; 1052 int diag, i, ret; 1053 1054 PMD_INIT_FUNC_TRACE(); 1055 1056 ixgbe_dev_macsec_setting_reset(eth_dev); 1057 1058 eth_dev->dev_ops = &ixgbe_eth_dev_ops; 1059 eth_dev->rx_queue_count = ixgbe_dev_rx_queue_count; 1060 eth_dev->rx_descriptor_done = ixgbe_dev_rx_descriptor_done; 1061 eth_dev->rx_descriptor_status = ixgbe_dev_rx_descriptor_status; 1062 eth_dev->tx_descriptor_status = ixgbe_dev_tx_descriptor_status; 1063 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1064 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1065 eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts; 1066 1067 /* 1068 * For secondary processes, we don't initialise any further as primary 1069 * has already done this work. Only check we don't need a different 1070 * RX and TX function. 1071 */ 1072 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1073 struct ixgbe_tx_queue *txq; 1074 /* TX queue function in primary, set by last queue initialized 1075 * Tx queue may not initialized by primary process 1076 */ 1077 if (eth_dev->data->tx_queues) { 1078 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1]; 1079 ixgbe_set_tx_function(eth_dev, txq); 1080 } else { 1081 /* Use default TX function if we get here */ 1082 PMD_INIT_LOG(NOTICE, "No TX queues configured yet. " 1083 "Using default TX function."); 1084 } 1085 1086 ixgbe_set_rx_function(eth_dev); 1087 1088 return 0; 1089 } 1090 1091 rte_atomic32_clear(&ad->link_thread_running); 1092 rte_eth_copy_pci_info(eth_dev, pci_dev); 1093 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1094 1095 /* Vendor and Device ID need to be set before init of shared code */ 1096 hw->device_id = pci_dev->id.device_id; 1097 hw->vendor_id = pci_dev->id.vendor_id; 1098 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1099 hw->allow_unsupported_sfp = 1; 1100 1101 /* Initialize the shared code (base driver) */ 1102 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1103 diag = ixgbe_bypass_init_shared_code(hw); 1104 #else 1105 diag = ixgbe_init_shared_code(hw); 1106 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1107 1108 if (diag != IXGBE_SUCCESS) { 1109 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); 1110 return -EIO; 1111 } 1112 1113 if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) { 1114 PMD_INIT_LOG(ERR, "\nERROR: " 1115 "Firmware recovery mode detected. Limiting functionality.\n" 1116 "Refer to the Intel(R) Ethernet Adapters and Devices " 1117 "User Guide for details on firmware recovery mode."); 1118 return -EIO; 1119 } 1120 1121 /* pick up the PCI bus settings for reporting later */ 1122 ixgbe_get_bus_info(hw); 1123 1124 /* Unlock any pending hardware semaphore */ 1125 ixgbe_swfw_lock_reset(hw); 1126 1127 #ifdef RTE_LIB_SECURITY 1128 /* Initialize security_ctx only for primary process*/ 1129 if (ixgbe_ipsec_ctx_create(eth_dev)) 1130 return -ENOMEM; 1131 #endif 1132 1133 /* Initialize DCB configuration*/ 1134 memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config)); 1135 ixgbe_dcb_init(hw, dcb_config); 1136 /* Get Hardware Flow Control setting */ 1137 hw->fc.requested_mode = ixgbe_fc_none; 1138 hw->fc.current_mode = ixgbe_fc_none; 1139 hw->fc.pause_time = IXGBE_FC_PAUSE; 1140 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 1141 hw->fc.low_water[i] = IXGBE_FC_LO; 1142 hw->fc.high_water[i] = IXGBE_FC_HI; 1143 } 1144 hw->fc.send_xon = 1; 1145 1146 /* Make sure we have a good EEPROM before we read from it */ 1147 diag = ixgbe_validate_eeprom_checksum(hw, &csum); 1148 if (diag != IXGBE_SUCCESS) { 1149 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag); 1150 return -EIO; 1151 } 1152 1153 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1154 diag = ixgbe_bypass_init_hw(hw); 1155 #else 1156 diag = ixgbe_init_hw(hw); 1157 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1158 1159 /* 1160 * Devices with copper phys will fail to initialise if ixgbe_init_hw() 1161 * is called too soon after the kernel driver unbinding/binding occurs. 1162 * The failure occurs in ixgbe_identify_phy_generic() for all devices, 1163 * but for non-copper devies, ixgbe_identify_sfp_module_generic() is 1164 * also called. See ixgbe_identify_phy_82599(). The reason for the 1165 * failure is not known, and only occuts when virtualisation features 1166 * are disabled in the bios. A delay of 100ms was found to be enough by 1167 * trial-and-error, and is doubled to be safe. 1168 */ 1169 if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) { 1170 rte_delay_ms(200); 1171 diag = ixgbe_init_hw(hw); 1172 } 1173 1174 if (diag == IXGBE_ERR_SFP_NOT_PRESENT) 1175 diag = IXGBE_SUCCESS; 1176 1177 if (diag == IXGBE_ERR_EEPROM_VERSION) { 1178 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/" 1179 "LOM. Please be aware there may be issues associated " 1180 "with your hardware."); 1181 PMD_INIT_LOG(ERR, "If you are experiencing problems " 1182 "please contact your Intel or hardware representative " 1183 "who provided you with this hardware."); 1184 } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED) 1185 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module"); 1186 if (diag) { 1187 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag); 1188 return -EIO; 1189 } 1190 1191 /* Reset the hw statistics */ 1192 ixgbe_dev_stats_reset(eth_dev); 1193 1194 /* disable interrupt */ 1195 ixgbe_disable_intr(hw); 1196 1197 /* reset mappings for queue statistics hw counters*/ 1198 ixgbe_reset_qstat_mappings(hw); 1199 1200 /* Allocate memory for storing MAC addresses */ 1201 eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", RTE_ETHER_ADDR_LEN * 1202 hw->mac.num_rar_entries, 0); 1203 if (eth_dev->data->mac_addrs == NULL) { 1204 PMD_INIT_LOG(ERR, 1205 "Failed to allocate %u bytes needed to store " 1206 "MAC addresses", 1207 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1208 return -ENOMEM; 1209 } 1210 /* Copy the permanent MAC address */ 1211 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1212 ð_dev->data->mac_addrs[0]); 1213 1214 /* Allocate memory for storing hash filter MAC addresses */ 1215 eth_dev->data->hash_mac_addrs = rte_zmalloc( 1216 "ixgbe", RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC, 0); 1217 if (eth_dev->data->hash_mac_addrs == NULL) { 1218 PMD_INIT_LOG(ERR, 1219 "Failed to allocate %d bytes needed to store MAC addresses", 1220 RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC); 1221 return -ENOMEM; 1222 } 1223 1224 /* initialize the vfta */ 1225 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1226 1227 /* initialize the hw strip bitmap*/ 1228 memset(hwstrip, 0, sizeof(*hwstrip)); 1229 1230 /* initialize PF if max_vfs not zero */ 1231 ret = ixgbe_pf_host_init(eth_dev); 1232 if (ret) { 1233 rte_free(eth_dev->data->mac_addrs); 1234 eth_dev->data->mac_addrs = NULL; 1235 rte_free(eth_dev->data->hash_mac_addrs); 1236 eth_dev->data->hash_mac_addrs = NULL; 1237 return ret; 1238 } 1239 1240 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 1241 /* let hardware know driver is loaded */ 1242 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 1243 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1244 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 1245 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 1246 IXGBE_WRITE_FLUSH(hw); 1247 1248 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 1249 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d", 1250 (int) hw->mac.type, (int) hw->phy.type, 1251 (int) hw->phy.sfp_type); 1252 else 1253 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d", 1254 (int) hw->mac.type, (int) hw->phy.type); 1255 1256 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1257 eth_dev->data->port_id, pci_dev->id.vendor_id, 1258 pci_dev->id.device_id); 1259 1260 rte_intr_callback_register(intr_handle, 1261 ixgbe_dev_interrupt_handler, eth_dev); 1262 1263 /* enable uio/vfio intr/eventfd mapping */ 1264 rte_intr_enable(intr_handle); 1265 1266 /* enable support intr */ 1267 ixgbe_enable_intr(eth_dev); 1268 1269 /* initialize filter info */ 1270 memset(filter_info, 0, 1271 sizeof(struct ixgbe_filter_info)); 1272 1273 /* initialize 5tuple filter list */ 1274 TAILQ_INIT(&filter_info->fivetuple_list); 1275 1276 /* initialize flow director filter list & hash */ 1277 ixgbe_fdir_filter_init(eth_dev); 1278 1279 /* initialize l2 tunnel filter list & hash */ 1280 ixgbe_l2_tn_filter_init(eth_dev); 1281 1282 /* initialize flow filter lists */ 1283 ixgbe_filterlist_init(); 1284 1285 /* initialize bandwidth configuration info */ 1286 memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); 1287 1288 /* initialize Traffic Manager configuration */ 1289 ixgbe_tm_conf_init(eth_dev); 1290 1291 return 0; 1292 } 1293 1294 static int 1295 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) 1296 { 1297 PMD_INIT_FUNC_TRACE(); 1298 1299 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1300 return 0; 1301 1302 ixgbe_dev_close(eth_dev); 1303 1304 return 0; 1305 } 1306 1307 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 1308 { 1309 struct ixgbe_filter_info *filter_info = 1310 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1311 struct ixgbe_5tuple_filter *p_5tuple; 1312 1313 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 1314 TAILQ_REMOVE(&filter_info->fivetuple_list, 1315 p_5tuple, 1316 entries); 1317 rte_free(p_5tuple); 1318 } 1319 memset(filter_info->fivetuple_mask, 0, 1320 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); 1321 1322 return 0; 1323 } 1324 1325 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev) 1326 { 1327 struct ixgbe_hw_fdir_info *fdir_info = 1328 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1329 struct ixgbe_fdir_filter *fdir_filter; 1330 1331 if (fdir_info->hash_map) 1332 rte_free(fdir_info->hash_map); 1333 if (fdir_info->hash_handle) 1334 rte_hash_free(fdir_info->hash_handle); 1335 1336 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 1337 TAILQ_REMOVE(&fdir_info->fdir_list, 1338 fdir_filter, 1339 entries); 1340 rte_free(fdir_filter); 1341 } 1342 1343 return 0; 1344 } 1345 1346 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev) 1347 { 1348 struct ixgbe_l2_tn_info *l2_tn_info = 1349 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1350 struct ixgbe_l2_tn_filter *l2_tn_filter; 1351 1352 if (l2_tn_info->hash_map) 1353 rte_free(l2_tn_info->hash_map); 1354 if (l2_tn_info->hash_handle) 1355 rte_hash_free(l2_tn_info->hash_handle); 1356 1357 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 1358 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, 1359 l2_tn_filter, 1360 entries); 1361 rte_free(l2_tn_filter); 1362 } 1363 1364 return 0; 1365 } 1366 1367 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) 1368 { 1369 struct ixgbe_hw_fdir_info *fdir_info = 1370 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1371 char fdir_hash_name[RTE_HASH_NAMESIZE]; 1372 struct rte_hash_parameters fdir_hash_params = { 1373 .name = fdir_hash_name, 1374 .entries = IXGBE_MAX_FDIR_FILTER_NUM, 1375 .key_len = sizeof(union ixgbe_atr_input), 1376 .hash_func = rte_hash_crc, 1377 .hash_func_init_val = 0, 1378 .socket_id = rte_socket_id(), 1379 }; 1380 1381 TAILQ_INIT(&fdir_info->fdir_list); 1382 snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, 1383 "fdir_%s", eth_dev->device->name); 1384 fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); 1385 if (!fdir_info->hash_handle) { 1386 PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); 1387 return -EINVAL; 1388 } 1389 fdir_info->hash_map = rte_zmalloc("ixgbe", 1390 sizeof(struct ixgbe_fdir_filter *) * 1391 IXGBE_MAX_FDIR_FILTER_NUM, 1392 0); 1393 if (!fdir_info->hash_map) { 1394 PMD_INIT_LOG(ERR, 1395 "Failed to allocate memory for fdir hash map!"); 1396 return -ENOMEM; 1397 } 1398 fdir_info->mask_added = FALSE; 1399 1400 return 0; 1401 } 1402 1403 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) 1404 { 1405 struct ixgbe_l2_tn_info *l2_tn_info = 1406 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1407 char l2_tn_hash_name[RTE_HASH_NAMESIZE]; 1408 struct rte_hash_parameters l2_tn_hash_params = { 1409 .name = l2_tn_hash_name, 1410 .entries = IXGBE_MAX_L2_TN_FILTER_NUM, 1411 .key_len = sizeof(struct ixgbe_l2_tn_key), 1412 .hash_func = rte_hash_crc, 1413 .hash_func_init_val = 0, 1414 .socket_id = rte_socket_id(), 1415 }; 1416 1417 TAILQ_INIT(&l2_tn_info->l2_tn_list); 1418 snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, 1419 "l2_tn_%s", eth_dev->device->name); 1420 l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); 1421 if (!l2_tn_info->hash_handle) { 1422 PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); 1423 return -EINVAL; 1424 } 1425 l2_tn_info->hash_map = rte_zmalloc("ixgbe", 1426 sizeof(struct ixgbe_l2_tn_filter *) * 1427 IXGBE_MAX_L2_TN_FILTER_NUM, 1428 0); 1429 if (!l2_tn_info->hash_map) { 1430 PMD_INIT_LOG(ERR, 1431 "Failed to allocate memory for L2 TN hash map!"); 1432 return -ENOMEM; 1433 } 1434 l2_tn_info->e_tag_en = FALSE; 1435 l2_tn_info->e_tag_fwd_en = FALSE; 1436 l2_tn_info->e_tag_ether_type = RTE_ETHER_TYPE_ETAG; 1437 1438 return 0; 1439 } 1440 /* 1441 * Negotiate mailbox API version with the PF. 1442 * After reset API version is always set to the basic one (ixgbe_mbox_api_10). 1443 * Then we try to negotiate starting with the most recent one. 1444 * If all negotiation attempts fail, then we will proceed with 1445 * the default one (ixgbe_mbox_api_10). 1446 */ 1447 static void 1448 ixgbevf_negotiate_api(struct ixgbe_hw *hw) 1449 { 1450 int32_t i; 1451 1452 /* start with highest supported, proceed down */ 1453 static const enum ixgbe_pfvf_api_rev sup_ver[] = { 1454 ixgbe_mbox_api_13, 1455 ixgbe_mbox_api_12, 1456 ixgbe_mbox_api_11, 1457 ixgbe_mbox_api_10, 1458 }; 1459 1460 for (i = 0; 1461 i != RTE_DIM(sup_ver) && 1462 ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0; 1463 i++) 1464 ; 1465 } 1466 1467 static void 1468 generate_random_mac_addr(struct rte_ether_addr *mac_addr) 1469 { 1470 uint64_t random; 1471 1472 /* Set Organizationally Unique Identifier (OUI) prefix. */ 1473 mac_addr->addr_bytes[0] = 0x00; 1474 mac_addr->addr_bytes[1] = 0x09; 1475 mac_addr->addr_bytes[2] = 0xC0; 1476 /* Force indication of locally assigned MAC address. */ 1477 mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR; 1478 /* Generate the last 3 bytes of the MAC address with a random number. */ 1479 random = rte_rand(); 1480 memcpy(&mac_addr->addr_bytes[3], &random, 3); 1481 } 1482 1483 static int 1484 devarg_handle_int(__rte_unused const char *key, const char *value, 1485 void *extra_args) 1486 { 1487 uint16_t *n = extra_args; 1488 1489 if (value == NULL || extra_args == NULL) 1490 return -EINVAL; 1491 1492 *n = (uint16_t)strtoul(value, NULL, 0); 1493 if (*n == USHRT_MAX && errno == ERANGE) 1494 return -1; 1495 1496 return 0; 1497 } 1498 1499 static void 1500 ixgbevf_parse_devargs(struct ixgbe_adapter *adapter, 1501 struct rte_devargs *devargs) 1502 { 1503 struct rte_kvargs *kvlist; 1504 uint16_t pflink_fullchk; 1505 1506 if (devargs == NULL) 1507 return; 1508 1509 kvlist = rte_kvargs_parse(devargs->args, ixgbevf_valid_arguments); 1510 if (kvlist == NULL) 1511 return; 1512 1513 if (rte_kvargs_count(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK) == 1 && 1514 rte_kvargs_process(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK, 1515 devarg_handle_int, &pflink_fullchk) == 0 && 1516 pflink_fullchk == 1) 1517 adapter->pflink_fullchk = 1; 1518 1519 rte_kvargs_free(kvlist); 1520 } 1521 1522 /* 1523 * Virtual Function device init 1524 */ 1525 static int 1526 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) 1527 { 1528 int diag; 1529 uint32_t tc, tcs; 1530 struct ixgbe_adapter *ad = eth_dev->data->dev_private; 1531 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1532 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1533 struct ixgbe_hw *hw = 1534 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1535 struct ixgbe_vfta *shadow_vfta = 1536 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1537 struct ixgbe_hwstrip *hwstrip = 1538 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1539 struct rte_ether_addr *perm_addr = 1540 (struct rte_ether_addr *)hw->mac.perm_addr; 1541 1542 PMD_INIT_FUNC_TRACE(); 1543 1544 eth_dev->dev_ops = &ixgbevf_eth_dev_ops; 1545 eth_dev->rx_descriptor_done = ixgbe_dev_rx_descriptor_done; 1546 eth_dev->rx_descriptor_status = ixgbe_dev_rx_descriptor_status; 1547 eth_dev->tx_descriptor_status = ixgbe_dev_tx_descriptor_status; 1548 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1549 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1550 1551 /* for secondary processes, we don't initialise any further as primary 1552 * has already done this work. Only check we don't need a different 1553 * RX function 1554 */ 1555 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1556 struct ixgbe_tx_queue *txq; 1557 /* TX queue function in primary, set by last queue initialized 1558 * Tx queue may not initialized by primary process 1559 */ 1560 if (eth_dev->data->tx_queues) { 1561 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1]; 1562 ixgbe_set_tx_function(eth_dev, txq); 1563 } else { 1564 /* Use default TX function if we get here */ 1565 PMD_INIT_LOG(NOTICE, 1566 "No TX queues configured yet. Using default TX function."); 1567 } 1568 1569 ixgbe_set_rx_function(eth_dev); 1570 1571 return 0; 1572 } 1573 1574 rte_atomic32_clear(&ad->link_thread_running); 1575 ixgbevf_parse_devargs(eth_dev->data->dev_private, 1576 pci_dev->device.devargs); 1577 1578 rte_eth_copy_pci_info(eth_dev, pci_dev); 1579 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1580 1581 hw->device_id = pci_dev->id.device_id; 1582 hw->vendor_id = pci_dev->id.vendor_id; 1583 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1584 1585 /* initialize the vfta */ 1586 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1587 1588 /* initialize the hw strip bitmap*/ 1589 memset(hwstrip, 0, sizeof(*hwstrip)); 1590 1591 /* Initialize the shared code (base driver) */ 1592 diag = ixgbe_init_shared_code(hw); 1593 if (diag != IXGBE_SUCCESS) { 1594 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag); 1595 return -EIO; 1596 } 1597 1598 /* init_mailbox_params */ 1599 hw->mbx.ops.init_params(hw); 1600 1601 /* Reset the hw statistics */ 1602 ixgbevf_dev_stats_reset(eth_dev); 1603 1604 /* Disable the interrupts for VF */ 1605 ixgbevf_intr_disable(eth_dev); 1606 1607 hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */ 1608 diag = hw->mac.ops.reset_hw(hw); 1609 1610 /* 1611 * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when 1612 * the underlying PF driver has not assigned a MAC address to the VF. 1613 * In this case, assign a random MAC address. 1614 */ 1615 if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) { 1616 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1617 /* 1618 * This error code will be propagated to the app by 1619 * rte_eth_dev_reset, so use a public error code rather than 1620 * the internal-only IXGBE_ERR_RESET_FAILED 1621 */ 1622 return -EAGAIN; 1623 } 1624 1625 /* negotiate mailbox API version to use with the PF. */ 1626 ixgbevf_negotiate_api(hw); 1627 1628 /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */ 1629 ixgbevf_get_queues(hw, &tcs, &tc); 1630 1631 /* Allocate memory for storing MAC addresses */ 1632 eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", RTE_ETHER_ADDR_LEN * 1633 hw->mac.num_rar_entries, 0); 1634 if (eth_dev->data->mac_addrs == NULL) { 1635 PMD_INIT_LOG(ERR, 1636 "Failed to allocate %u bytes needed to store " 1637 "MAC addresses", 1638 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1639 return -ENOMEM; 1640 } 1641 1642 /* Generate a random MAC address, if none was assigned by PF. */ 1643 if (rte_is_zero_ether_addr(perm_addr)) { 1644 generate_random_mac_addr(perm_addr); 1645 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1); 1646 if (diag) { 1647 rte_free(eth_dev->data->mac_addrs); 1648 eth_dev->data->mac_addrs = NULL; 1649 return diag; 1650 } 1651 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1652 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1653 "%02x:%02x:%02x:%02x:%02x:%02x", 1654 perm_addr->addr_bytes[0], 1655 perm_addr->addr_bytes[1], 1656 perm_addr->addr_bytes[2], 1657 perm_addr->addr_bytes[3], 1658 perm_addr->addr_bytes[4], 1659 perm_addr->addr_bytes[5]); 1660 } 1661 1662 /* Copy the permanent MAC address */ 1663 rte_ether_addr_copy(perm_addr, ð_dev->data->mac_addrs[0]); 1664 1665 /* reset the hardware with the new settings */ 1666 diag = hw->mac.ops.start_hw(hw); 1667 switch (diag) { 1668 case 0: 1669 break; 1670 1671 default: 1672 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1673 return -EIO; 1674 } 1675 1676 rte_intr_callback_register(intr_handle, 1677 ixgbevf_dev_interrupt_handler, eth_dev); 1678 rte_intr_enable(intr_handle); 1679 ixgbevf_intr_enable(eth_dev); 1680 1681 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s", 1682 eth_dev->data->port_id, pci_dev->id.vendor_id, 1683 pci_dev->id.device_id, "ixgbe_mac_82599_vf"); 1684 1685 return 0; 1686 } 1687 1688 /* Virtual Function device uninit */ 1689 1690 static int 1691 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) 1692 { 1693 PMD_INIT_FUNC_TRACE(); 1694 1695 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1696 return 0; 1697 1698 ixgbevf_dev_close(eth_dev); 1699 1700 return 0; 1701 } 1702 1703 static int 1704 eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1705 struct rte_pci_device *pci_dev) 1706 { 1707 char name[RTE_ETH_NAME_MAX_LEN]; 1708 struct rte_eth_dev *pf_ethdev; 1709 struct rte_eth_devargs eth_da; 1710 int i, retval; 1711 1712 if (pci_dev->device.devargs) { 1713 retval = rte_eth_devargs_parse(pci_dev->device.devargs->args, 1714 ð_da); 1715 if (retval) 1716 return retval; 1717 } else 1718 memset(ð_da, 0, sizeof(eth_da)); 1719 1720 retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name, 1721 sizeof(struct ixgbe_adapter), 1722 eth_dev_pci_specific_init, pci_dev, 1723 eth_ixgbe_dev_init, NULL); 1724 1725 if (retval || eth_da.nb_representor_ports < 1) 1726 return retval; 1727 1728 pf_ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1729 if (pf_ethdev == NULL) 1730 return -ENODEV; 1731 1732 /* probe VF representor ports */ 1733 for (i = 0; i < eth_da.nb_representor_ports; i++) { 1734 struct ixgbe_vf_info *vfinfo; 1735 struct ixgbe_vf_representor representor; 1736 1737 vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA( 1738 pf_ethdev->data->dev_private); 1739 if (vfinfo == NULL) { 1740 PMD_DRV_LOG(ERR, 1741 "no virtual functions supported by PF"); 1742 break; 1743 } 1744 1745 representor.vf_id = eth_da.representor_ports[i]; 1746 representor.switch_domain_id = vfinfo->switch_domain_id; 1747 representor.pf_ethdev = pf_ethdev; 1748 1749 /* representor port net_bdf_port */ 1750 snprintf(name, sizeof(name), "net_%s_representor_%d", 1751 pci_dev->device.name, 1752 eth_da.representor_ports[i]); 1753 1754 retval = rte_eth_dev_create(&pci_dev->device, name, 1755 sizeof(struct ixgbe_vf_representor), NULL, NULL, 1756 ixgbe_vf_representor_init, &representor); 1757 1758 if (retval) 1759 PMD_DRV_LOG(ERR, "failed to create ixgbe vf " 1760 "representor %s.", name); 1761 } 1762 1763 return 0; 1764 } 1765 1766 static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev) 1767 { 1768 struct rte_eth_dev *ethdev; 1769 1770 ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1771 if (!ethdev) 1772 return 0; 1773 1774 if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) 1775 return rte_eth_dev_pci_generic_remove(pci_dev, 1776 ixgbe_vf_representor_uninit); 1777 else 1778 return rte_eth_dev_pci_generic_remove(pci_dev, 1779 eth_ixgbe_dev_uninit); 1780 } 1781 1782 static struct rte_pci_driver rte_ixgbe_pmd = { 1783 .id_table = pci_id_ixgbe_map, 1784 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1785 .probe = eth_ixgbe_pci_probe, 1786 .remove = eth_ixgbe_pci_remove, 1787 }; 1788 1789 static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1790 struct rte_pci_device *pci_dev) 1791 { 1792 return rte_eth_dev_pci_generic_probe(pci_dev, 1793 sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init); 1794 } 1795 1796 static int eth_ixgbevf_pci_remove(struct rte_pci_device *pci_dev) 1797 { 1798 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbevf_dev_uninit); 1799 } 1800 1801 /* 1802 * virtual function driver struct 1803 */ 1804 static struct rte_pci_driver rte_ixgbevf_pmd = { 1805 .id_table = pci_id_ixgbevf_map, 1806 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1807 .probe = eth_ixgbevf_pci_probe, 1808 .remove = eth_ixgbevf_pci_remove, 1809 }; 1810 1811 static int 1812 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1813 { 1814 struct ixgbe_hw *hw = 1815 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1816 struct ixgbe_vfta *shadow_vfta = 1817 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1818 uint32_t vfta; 1819 uint32_t vid_idx; 1820 uint32_t vid_bit; 1821 1822 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 1823 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 1824 vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx)); 1825 if (on) 1826 vfta |= vid_bit; 1827 else 1828 vfta &= ~vid_bit; 1829 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta); 1830 1831 /* update local VFTA copy */ 1832 shadow_vfta->vfta[vid_idx] = vfta; 1833 1834 return 0; 1835 } 1836 1837 static void 1838 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 1839 { 1840 if (on) 1841 ixgbe_vlan_hw_strip_enable(dev, queue); 1842 else 1843 ixgbe_vlan_hw_strip_disable(dev, queue); 1844 } 1845 1846 static int 1847 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 1848 enum rte_vlan_type vlan_type, 1849 uint16_t tpid) 1850 { 1851 struct ixgbe_hw *hw = 1852 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1853 int ret = 0; 1854 uint32_t reg; 1855 uint32_t qinq; 1856 1857 qinq = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1858 qinq &= IXGBE_DMATXCTL_GDV; 1859 1860 switch (vlan_type) { 1861 case ETH_VLAN_TYPE_INNER: 1862 if (qinq) { 1863 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1864 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1865 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1866 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1867 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1868 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1869 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1870 } else { 1871 ret = -ENOTSUP; 1872 PMD_DRV_LOG(ERR, "Inner type is not supported" 1873 " by single VLAN"); 1874 } 1875 break; 1876 case ETH_VLAN_TYPE_OUTER: 1877 if (qinq) { 1878 /* Only the high 16-bits is valid */ 1879 IXGBE_WRITE_REG(hw, IXGBE_EXVET, (uint32_t)tpid << 1880 IXGBE_EXVET_VET_EXT_SHIFT); 1881 } else { 1882 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1883 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1884 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1885 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1886 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1887 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1888 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1889 } 1890 1891 break; 1892 default: 1893 ret = -EINVAL; 1894 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type); 1895 break; 1896 } 1897 1898 return ret; 1899 } 1900 1901 void 1902 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev) 1903 { 1904 struct ixgbe_hw *hw = 1905 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1906 uint32_t vlnctrl; 1907 1908 PMD_INIT_FUNC_TRACE(); 1909 1910 /* Filter Table Disable */ 1911 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1912 vlnctrl &= ~IXGBE_VLNCTRL_VFE; 1913 1914 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1915 } 1916 1917 void 1918 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1919 { 1920 struct ixgbe_hw *hw = 1921 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1922 struct ixgbe_vfta *shadow_vfta = 1923 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1924 uint32_t vlnctrl; 1925 uint16_t i; 1926 1927 PMD_INIT_FUNC_TRACE(); 1928 1929 /* Filter Table Enable */ 1930 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1931 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 1932 vlnctrl |= IXGBE_VLNCTRL_VFE; 1933 1934 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1935 1936 /* write whatever is in local vfta copy */ 1937 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 1938 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]); 1939 } 1940 1941 static void 1942 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on) 1943 { 1944 struct ixgbe_hwstrip *hwstrip = 1945 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private); 1946 struct ixgbe_rx_queue *rxq; 1947 1948 if (queue >= IXGBE_MAX_RX_QUEUE_NUM) 1949 return; 1950 1951 if (on) 1952 IXGBE_SET_HWSTRIP(hwstrip, queue); 1953 else 1954 IXGBE_CLEAR_HWSTRIP(hwstrip, queue); 1955 1956 if (queue >= dev->data->nb_rx_queues) 1957 return; 1958 1959 rxq = dev->data->rx_queues[queue]; 1960 1961 if (on) { 1962 rxq->vlan_flags = PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1963 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 1964 } else { 1965 rxq->vlan_flags = PKT_RX_VLAN; 1966 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 1967 } 1968 } 1969 1970 static void 1971 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue) 1972 { 1973 struct ixgbe_hw *hw = 1974 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1975 uint32_t ctrl; 1976 1977 PMD_INIT_FUNC_TRACE(); 1978 1979 if (hw->mac.type == ixgbe_mac_82598EB) { 1980 /* No queue level support */ 1981 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 1982 return; 1983 } 1984 1985 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 1986 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 1987 ctrl &= ~IXGBE_RXDCTL_VME; 1988 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 1989 1990 /* record those setting for HW strip per queue */ 1991 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0); 1992 } 1993 1994 static void 1995 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue) 1996 { 1997 struct ixgbe_hw *hw = 1998 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1999 uint32_t ctrl; 2000 2001 PMD_INIT_FUNC_TRACE(); 2002 2003 if (hw->mac.type == ixgbe_mac_82598EB) { 2004 /* No queue level supported */ 2005 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2006 return; 2007 } 2008 2009 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2010 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2011 ctrl |= IXGBE_RXDCTL_VME; 2012 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2013 2014 /* record those setting for HW strip per queue */ 2015 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1); 2016 } 2017 2018 static void 2019 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2020 { 2021 struct ixgbe_hw *hw = 2022 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2023 uint32_t ctrl; 2024 2025 PMD_INIT_FUNC_TRACE(); 2026 2027 /* DMATXCTRL: Geric Double VLAN Disable */ 2028 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2029 ctrl &= ~IXGBE_DMATXCTL_GDV; 2030 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2031 2032 /* CTRL_EXT: Global Double VLAN Disable */ 2033 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2034 ctrl &= ~IXGBE_EXTENDED_VLAN; 2035 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2036 2037 } 2038 2039 static void 2040 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2041 { 2042 struct ixgbe_hw *hw = 2043 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2044 uint32_t ctrl; 2045 2046 PMD_INIT_FUNC_TRACE(); 2047 2048 /* DMATXCTRL: Geric Double VLAN Enable */ 2049 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2050 ctrl |= IXGBE_DMATXCTL_GDV; 2051 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2052 2053 /* CTRL_EXT: Global Double VLAN Enable */ 2054 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2055 ctrl |= IXGBE_EXTENDED_VLAN; 2056 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2057 2058 /* Clear pooling mode of PFVTCTL. It's required by X550. */ 2059 if (hw->mac.type == ixgbe_mac_X550 || 2060 hw->mac.type == ixgbe_mac_X550EM_x || 2061 hw->mac.type == ixgbe_mac_X550EM_a) { 2062 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 2063 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 2064 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 2065 } 2066 2067 /* 2068 * VET EXT field in the EXVET register = 0x8100 by default 2069 * So no need to change. Same to VT field of DMATXCTL register 2070 */ 2071 } 2072 2073 void 2074 ixgbe_vlan_hw_strip_config(struct rte_eth_dev *dev) 2075 { 2076 struct ixgbe_hw *hw = 2077 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2078 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2079 uint32_t ctrl; 2080 uint16_t i; 2081 struct ixgbe_rx_queue *rxq; 2082 bool on; 2083 2084 PMD_INIT_FUNC_TRACE(); 2085 2086 if (hw->mac.type == ixgbe_mac_82598EB) { 2087 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2088 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2089 ctrl |= IXGBE_VLNCTRL_VME; 2090 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2091 } else { 2092 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2093 ctrl &= ~IXGBE_VLNCTRL_VME; 2094 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2095 } 2096 } else { 2097 /* 2098 * Other 10G NIC, the VLAN strip can be setup 2099 * per queue in RXDCTL 2100 */ 2101 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2102 rxq = dev->data->rx_queues[i]; 2103 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 2104 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2105 ctrl |= IXGBE_RXDCTL_VME; 2106 on = TRUE; 2107 } else { 2108 ctrl &= ~IXGBE_RXDCTL_VME; 2109 on = FALSE; 2110 } 2111 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); 2112 2113 /* record those setting for HW strip per queue */ 2114 ixgbe_vlan_hw_strip_bitmap_set(dev, i, on); 2115 } 2116 } 2117 } 2118 2119 static void 2120 ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask) 2121 { 2122 uint16_t i; 2123 struct rte_eth_rxmode *rxmode; 2124 struct ixgbe_rx_queue *rxq; 2125 2126 if (mask & ETH_VLAN_STRIP_MASK) { 2127 rxmode = &dev->data->dev_conf.rxmode; 2128 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2129 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2130 rxq = dev->data->rx_queues[i]; 2131 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 2132 } 2133 else 2134 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2135 rxq = dev->data->rx_queues[i]; 2136 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 2137 } 2138 } 2139 } 2140 2141 static int 2142 ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask) 2143 { 2144 struct rte_eth_rxmode *rxmode; 2145 rxmode = &dev->data->dev_conf.rxmode; 2146 2147 if (mask & ETH_VLAN_STRIP_MASK) { 2148 ixgbe_vlan_hw_strip_config(dev); 2149 } 2150 2151 if (mask & ETH_VLAN_FILTER_MASK) { 2152 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2153 ixgbe_vlan_hw_filter_enable(dev); 2154 else 2155 ixgbe_vlan_hw_filter_disable(dev); 2156 } 2157 2158 if (mask & ETH_VLAN_EXTEND_MASK) { 2159 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2160 ixgbe_vlan_hw_extend_enable(dev); 2161 else 2162 ixgbe_vlan_hw_extend_disable(dev); 2163 } 2164 2165 return 0; 2166 } 2167 2168 static int 2169 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2170 { 2171 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 2172 2173 ixgbe_vlan_offload_config(dev, mask); 2174 2175 return 0; 2176 } 2177 2178 static void 2179 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2180 { 2181 struct ixgbe_hw *hw = 2182 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2183 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 2184 uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2185 2186 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 2187 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 2188 } 2189 2190 static int 2191 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) 2192 { 2193 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2194 2195 switch (nb_rx_q) { 2196 case 1: 2197 case 2: 2198 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS; 2199 break; 2200 case 4: 2201 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS; 2202 break; 2203 default: 2204 return -EINVAL; 2205 } 2206 2207 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 2208 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2209 RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = 2210 pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2211 return 0; 2212 } 2213 2214 static int 2215 ixgbe_check_mq_mode(struct rte_eth_dev *dev) 2216 { 2217 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 2218 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2219 uint16_t nb_rx_q = dev->data->nb_rx_queues; 2220 uint16_t nb_tx_q = dev->data->nb_tx_queues; 2221 2222 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 2223 /* check multi-queue mode */ 2224 switch (dev_conf->rxmode.mq_mode) { 2225 case ETH_MQ_RX_VMDQ_DCB: 2226 PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV"); 2227 break; 2228 case ETH_MQ_RX_VMDQ_DCB_RSS: 2229 /* DCB/RSS VMDQ in SRIOV mode, not implement yet */ 2230 PMD_INIT_LOG(ERR, "SRIOV active," 2231 " unsupported mq_mode rx %d.", 2232 dev_conf->rxmode.mq_mode); 2233 return -EINVAL; 2234 case ETH_MQ_RX_RSS: 2235 case ETH_MQ_RX_VMDQ_RSS: 2236 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS; 2237 if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) 2238 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) { 2239 PMD_INIT_LOG(ERR, "SRIOV is active," 2240 " invalid queue number" 2241 " for VMDQ RSS, allowed" 2242 " value are 1, 2 or 4."); 2243 return -EINVAL; 2244 } 2245 break; 2246 case ETH_MQ_RX_VMDQ_ONLY: 2247 case ETH_MQ_RX_NONE: 2248 /* if nothing mq mode configure, use default scheme */ 2249 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 2250 break; 2251 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/ 2252 /* SRIOV only works in VMDq enable mode */ 2253 PMD_INIT_LOG(ERR, "SRIOV is active," 2254 " wrong mq_mode rx %d.", 2255 dev_conf->rxmode.mq_mode); 2256 return -EINVAL; 2257 } 2258 2259 switch (dev_conf->txmode.mq_mode) { 2260 case ETH_MQ_TX_VMDQ_DCB: 2261 PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV"); 2262 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2263 break; 2264 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */ 2265 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY; 2266 break; 2267 } 2268 2269 /* check valid queue number */ 2270 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) || 2271 (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) { 2272 PMD_INIT_LOG(ERR, "SRIOV is active," 2273 " nb_rx_q=%d nb_tx_q=%d queue number" 2274 " must be less than or equal to %d.", 2275 nb_rx_q, nb_tx_q, 2276 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool); 2277 return -EINVAL; 2278 } 2279 } else { 2280 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) { 2281 PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is" 2282 " not supported."); 2283 return -EINVAL; 2284 } 2285 /* check configuration for vmdb+dcb mode */ 2286 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) { 2287 const struct rte_eth_vmdq_dcb_conf *conf; 2288 2289 if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2290 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.", 2291 IXGBE_VMDQ_DCB_NB_QUEUES); 2292 return -EINVAL; 2293 } 2294 conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf; 2295 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2296 conf->nb_queue_pools == ETH_32_POOLS)) { 2297 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2298 " nb_queue_pools must be %d or %d.", 2299 ETH_16_POOLS, ETH_32_POOLS); 2300 return -EINVAL; 2301 } 2302 } 2303 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { 2304 const struct rte_eth_vmdq_dcb_tx_conf *conf; 2305 2306 if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2307 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d", 2308 IXGBE_VMDQ_DCB_NB_QUEUES); 2309 return -EINVAL; 2310 } 2311 conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2312 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2313 conf->nb_queue_pools == ETH_32_POOLS)) { 2314 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2315 " nb_queue_pools != %d and" 2316 " nb_queue_pools != %d.", 2317 ETH_16_POOLS, ETH_32_POOLS); 2318 return -EINVAL; 2319 } 2320 } 2321 2322 /* For DCB mode check our configuration before we go further */ 2323 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) { 2324 const struct rte_eth_dcb_rx_conf *conf; 2325 2326 conf = &dev_conf->rx_adv_conf.dcb_rx_conf; 2327 if (!(conf->nb_tcs == ETH_4_TCS || 2328 conf->nb_tcs == ETH_8_TCS)) { 2329 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2330 " and nb_tcs != %d.", 2331 ETH_4_TCS, ETH_8_TCS); 2332 return -EINVAL; 2333 } 2334 } 2335 2336 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { 2337 const struct rte_eth_dcb_tx_conf *conf; 2338 2339 conf = &dev_conf->tx_adv_conf.dcb_tx_conf; 2340 if (!(conf->nb_tcs == ETH_4_TCS || 2341 conf->nb_tcs == ETH_8_TCS)) { 2342 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2343 " and nb_tcs != %d.", 2344 ETH_4_TCS, ETH_8_TCS); 2345 return -EINVAL; 2346 } 2347 } 2348 2349 /* 2350 * When DCB/VT is off, maximum number of queues changes, 2351 * except for 82598EB, which remains constant. 2352 */ 2353 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 2354 hw->mac.type != ixgbe_mac_82598EB) { 2355 if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) { 2356 PMD_INIT_LOG(ERR, 2357 "Neither VT nor DCB are enabled, " 2358 "nb_tx_q > %d.", 2359 IXGBE_NONE_MODE_TX_NB_QUEUES); 2360 return -EINVAL; 2361 } 2362 } 2363 } 2364 return 0; 2365 } 2366 2367 static int 2368 ixgbe_dev_configure(struct rte_eth_dev *dev) 2369 { 2370 struct ixgbe_interrupt *intr = 2371 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2372 struct ixgbe_adapter *adapter = dev->data->dev_private; 2373 int ret; 2374 2375 PMD_INIT_FUNC_TRACE(); 2376 2377 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 2378 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 2379 2380 /* multipe queue mode checking */ 2381 ret = ixgbe_check_mq_mode(dev); 2382 if (ret != 0) { 2383 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.", 2384 ret); 2385 return ret; 2386 } 2387 2388 /* set flag to update link status after init */ 2389 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2390 2391 /* 2392 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 2393 * allocation or vector Rx preconditions we will reset it. 2394 */ 2395 adapter->rx_bulk_alloc_allowed = true; 2396 adapter->rx_vec_allowed = true; 2397 2398 return 0; 2399 } 2400 2401 static void 2402 ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) 2403 { 2404 struct ixgbe_hw *hw = 2405 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2406 struct ixgbe_interrupt *intr = 2407 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2408 uint32_t gpie; 2409 2410 /* only set up it on X550EM_X */ 2411 if (hw->mac.type == ixgbe_mac_X550EM_x) { 2412 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 2413 gpie |= IXGBE_SDP0_GPIEN_X550EM_x; 2414 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 2415 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 2416 intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x; 2417 } 2418 } 2419 2420 int 2421 ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, 2422 uint16_t tx_rate, uint64_t q_msk) 2423 { 2424 struct ixgbe_hw *hw; 2425 struct ixgbe_vf_info *vfinfo; 2426 struct rte_eth_link link; 2427 uint8_t nb_q_per_pool; 2428 uint32_t queue_stride; 2429 uint32_t queue_idx, idx = 0, vf_idx; 2430 uint32_t queue_end; 2431 uint16_t total_rate = 0; 2432 struct rte_pci_device *pci_dev; 2433 int ret; 2434 2435 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2436 ret = rte_eth_link_get_nowait(dev->data->port_id, &link); 2437 if (ret < 0) 2438 return ret; 2439 2440 if (vf >= pci_dev->max_vfs) 2441 return -EINVAL; 2442 2443 if (tx_rate > link.link_speed) 2444 return -EINVAL; 2445 2446 if (q_msk == 0) 2447 return 0; 2448 2449 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2450 vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); 2451 nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2452 queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2453 queue_idx = vf * queue_stride; 2454 queue_end = queue_idx + nb_q_per_pool - 1; 2455 if (queue_end >= hw->mac.max_tx_queues) 2456 return -EINVAL; 2457 2458 if (vfinfo) { 2459 for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) { 2460 if (vf_idx == vf) 2461 continue; 2462 for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); 2463 idx++) 2464 total_rate += vfinfo[vf_idx].tx_rate[idx]; 2465 } 2466 } else { 2467 return -EINVAL; 2468 } 2469 2470 /* Store tx_rate for this vf. */ 2471 for (idx = 0; idx < nb_q_per_pool; idx++) { 2472 if (((uint64_t)0x1 << idx) & q_msk) { 2473 if (vfinfo[vf].tx_rate[idx] != tx_rate) 2474 vfinfo[vf].tx_rate[idx] = tx_rate; 2475 total_rate += tx_rate; 2476 } 2477 } 2478 2479 if (total_rate > dev->data->dev_link.link_speed) { 2480 /* Reset stored TX rate of the VF if it causes exceed 2481 * link speed. 2482 */ 2483 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); 2484 return -EINVAL; 2485 } 2486 2487 /* Set RTTBCNRC of each queue/pool for vf X */ 2488 for (; queue_idx <= queue_end; queue_idx++) { 2489 if (0x1 & q_msk) 2490 ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); 2491 q_msk = q_msk >> 1; 2492 } 2493 2494 return 0; 2495 } 2496 2497 static int 2498 ixgbe_flow_ctrl_enable(struct rte_eth_dev *dev, struct ixgbe_hw *hw) 2499 { 2500 struct ixgbe_adapter *adapter = dev->data->dev_private; 2501 int err; 2502 uint32_t mflcn; 2503 2504 ixgbe_setup_fc(hw); 2505 2506 err = ixgbe_fc_enable(hw); 2507 2508 /* Not negotiated is not an error case */ 2509 if (err == IXGBE_SUCCESS || err == IXGBE_ERR_FC_NOT_NEGOTIATED) { 2510 /* 2511 *check if we want to forward MAC frames - driver doesn't 2512 *have native capability to do that, 2513 *so we'll write the registers ourselves 2514 */ 2515 2516 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 2517 2518 /* set or clear MFLCN.PMCF bit depending on configuration */ 2519 if (adapter->mac_ctrl_frame_fwd != 0) 2520 mflcn |= IXGBE_MFLCN_PMCF; 2521 else 2522 mflcn &= ~IXGBE_MFLCN_PMCF; 2523 2524 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn); 2525 IXGBE_WRITE_FLUSH(hw); 2526 2527 return 0; 2528 } 2529 return err; 2530 } 2531 2532 /* 2533 * Configure device link speed and setup link. 2534 * It returns 0 on success. 2535 */ 2536 static int 2537 ixgbe_dev_start(struct rte_eth_dev *dev) 2538 { 2539 struct ixgbe_hw *hw = 2540 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2541 struct ixgbe_vf_info *vfinfo = 2542 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2543 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2544 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2545 uint32_t intr_vector = 0; 2546 int err; 2547 bool link_up = false, negotiate = 0; 2548 uint32_t speed = 0; 2549 uint32_t allowed_speeds = 0; 2550 int mask = 0; 2551 int status; 2552 uint16_t vf, idx; 2553 uint32_t *link_speeds; 2554 struct ixgbe_tm_conf *tm_conf = 2555 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2556 struct ixgbe_macsec_setting *macsec_setting = 2557 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 2558 2559 PMD_INIT_FUNC_TRACE(); 2560 2561 /* Stop the link setup handler before resetting the HW. */ 2562 ixgbe_dev_wait_setup_link_complete(dev, 0); 2563 2564 /* disable uio/vfio intr/eventfd mapping */ 2565 rte_intr_disable(intr_handle); 2566 2567 /* stop adapter */ 2568 hw->adapter_stopped = 0; 2569 ixgbe_stop_adapter(hw); 2570 2571 /* reinitialize adapter 2572 * this calls reset and start 2573 */ 2574 status = ixgbe_pf_reset_hw(hw); 2575 if (status != 0) 2576 return -1; 2577 hw->mac.ops.start_hw(hw); 2578 hw->mac.get_link_status = true; 2579 2580 /* configure PF module if SRIOV enabled */ 2581 ixgbe_pf_host_configure(dev); 2582 2583 ixgbe_dev_phy_intr_setup(dev); 2584 2585 /* check and configure queue intr-vector mapping */ 2586 if ((rte_intr_cap_multiple(intr_handle) || 2587 !RTE_ETH_DEV_SRIOV(dev).active) && 2588 dev->data->dev_conf.intr_conf.rxq != 0) { 2589 intr_vector = dev->data->nb_rx_queues; 2590 if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) { 2591 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 2592 IXGBE_MAX_INTR_QUEUE_NUM); 2593 return -ENOTSUP; 2594 } 2595 if (rte_intr_efd_enable(intr_handle, intr_vector)) 2596 return -1; 2597 } 2598 2599 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 2600 intr_handle->intr_vec = 2601 rte_zmalloc("intr_vec", 2602 dev->data->nb_rx_queues * sizeof(int), 0); 2603 if (intr_handle->intr_vec == NULL) { 2604 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 2605 " intr_vec", dev->data->nb_rx_queues); 2606 return -ENOMEM; 2607 } 2608 } 2609 2610 /* confiugre msix for sleep until rx interrupt */ 2611 ixgbe_configure_msix(dev); 2612 2613 /* initialize transmission unit */ 2614 ixgbe_dev_tx_init(dev); 2615 2616 /* This can fail when allocating mbufs for descriptor rings */ 2617 err = ixgbe_dev_rx_init(dev); 2618 if (err) { 2619 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 2620 goto error; 2621 } 2622 2623 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 2624 ETH_VLAN_EXTEND_MASK; 2625 err = ixgbe_vlan_offload_config(dev, mask); 2626 if (err) { 2627 PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); 2628 goto error; 2629 } 2630 2631 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 2632 /* Enable vlan filtering for VMDq */ 2633 ixgbe_vmdq_vlan_hw_filter_enable(dev); 2634 } 2635 2636 /* Configure DCB hw */ 2637 ixgbe_configure_dcb(dev); 2638 2639 if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { 2640 err = ixgbe_fdir_configure(dev); 2641 if (err) 2642 goto error; 2643 } 2644 2645 /* Restore vf rate limit */ 2646 if (vfinfo != NULL) { 2647 for (vf = 0; vf < pci_dev->max_vfs; vf++) 2648 for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) 2649 if (vfinfo[vf].tx_rate[idx] != 0) 2650 ixgbe_set_vf_rate_limit( 2651 dev, vf, 2652 vfinfo[vf].tx_rate[idx], 2653 1 << idx); 2654 } 2655 2656 ixgbe_restore_statistics_mapping(dev); 2657 2658 err = ixgbe_flow_ctrl_enable(dev, hw); 2659 if (err < 0) { 2660 PMD_INIT_LOG(ERR, "enable flow ctrl err"); 2661 goto error; 2662 } 2663 2664 err = ixgbe_dev_rxtx_start(dev); 2665 if (err < 0) { 2666 PMD_INIT_LOG(ERR, "Unable to start rxtx queues"); 2667 goto error; 2668 } 2669 2670 /* Skip link setup if loopback mode is enabled. */ 2671 if (dev->data->dev_conf.lpbk_mode != 0) { 2672 err = ixgbe_check_supported_loopback_mode(dev); 2673 if (err < 0) { 2674 PMD_INIT_LOG(ERR, "Unsupported loopback mode"); 2675 goto error; 2676 } else { 2677 goto skip_link_setup; 2678 } 2679 } 2680 2681 if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) { 2682 err = hw->mac.ops.setup_sfp(hw); 2683 if (err) 2684 goto error; 2685 } 2686 2687 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2688 /* Turn on the copper */ 2689 ixgbe_set_phy_power(hw, true); 2690 } else { 2691 /* Turn on the laser */ 2692 ixgbe_enable_tx_laser(hw); 2693 } 2694 2695 err = ixgbe_check_link(hw, &speed, &link_up, 0); 2696 if (err) 2697 goto error; 2698 dev->data->dev_link.link_status = link_up; 2699 2700 err = ixgbe_get_link_capabilities(hw, &speed, &negotiate); 2701 if (err) 2702 goto error; 2703 2704 switch (hw->mac.type) { 2705 case ixgbe_mac_X550: 2706 case ixgbe_mac_X550EM_x: 2707 case ixgbe_mac_X550EM_a: 2708 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2709 ETH_LINK_SPEED_2_5G | ETH_LINK_SPEED_5G | 2710 ETH_LINK_SPEED_10G; 2711 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 2712 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 2713 allowed_speeds = ETH_LINK_SPEED_10M | 2714 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 2715 break; 2716 default: 2717 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2718 ETH_LINK_SPEED_10G; 2719 } 2720 2721 link_speeds = &dev->data->dev_conf.link_speeds; 2722 2723 /* Ignore autoneg flag bit and check the validity of 2724 * link_speed 2725 */ 2726 if (((*link_speeds) >> 1) & ~(allowed_speeds >> 1)) { 2727 PMD_INIT_LOG(ERR, "Invalid link setting"); 2728 goto error; 2729 } 2730 2731 speed = 0x0; 2732 if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { 2733 switch (hw->mac.type) { 2734 case ixgbe_mac_82598EB: 2735 speed = IXGBE_LINK_SPEED_82598_AUTONEG; 2736 break; 2737 case ixgbe_mac_82599EB: 2738 case ixgbe_mac_X540: 2739 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2740 break; 2741 case ixgbe_mac_X550: 2742 case ixgbe_mac_X550EM_x: 2743 case ixgbe_mac_X550EM_a: 2744 speed = IXGBE_LINK_SPEED_X550_AUTONEG; 2745 break; 2746 default: 2747 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2748 } 2749 } else { 2750 if (*link_speeds & ETH_LINK_SPEED_10G) 2751 speed |= IXGBE_LINK_SPEED_10GB_FULL; 2752 if (*link_speeds & ETH_LINK_SPEED_5G) 2753 speed |= IXGBE_LINK_SPEED_5GB_FULL; 2754 if (*link_speeds & ETH_LINK_SPEED_2_5G) 2755 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 2756 if (*link_speeds & ETH_LINK_SPEED_1G) 2757 speed |= IXGBE_LINK_SPEED_1GB_FULL; 2758 if (*link_speeds & ETH_LINK_SPEED_100M) 2759 speed |= IXGBE_LINK_SPEED_100_FULL; 2760 if (*link_speeds & ETH_LINK_SPEED_10M) 2761 speed |= IXGBE_LINK_SPEED_10_FULL; 2762 } 2763 2764 err = ixgbe_setup_link(hw, speed, link_up); 2765 if (err) 2766 goto error; 2767 2768 skip_link_setup: 2769 2770 if (rte_intr_allow_others(intr_handle)) { 2771 /* check if lsc interrupt is enabled */ 2772 if (dev->data->dev_conf.intr_conf.lsc != 0) 2773 ixgbe_dev_lsc_interrupt_setup(dev, TRUE); 2774 else 2775 ixgbe_dev_lsc_interrupt_setup(dev, FALSE); 2776 ixgbe_dev_macsec_interrupt_setup(dev); 2777 } else { 2778 rte_intr_callback_unregister(intr_handle, 2779 ixgbe_dev_interrupt_handler, dev); 2780 if (dev->data->dev_conf.intr_conf.lsc != 0) 2781 PMD_INIT_LOG(INFO, "lsc won't enable because of" 2782 " no intr multiplex"); 2783 } 2784 2785 /* check if rxq interrupt is enabled */ 2786 if (dev->data->dev_conf.intr_conf.rxq != 0 && 2787 rte_intr_dp_is_en(intr_handle)) 2788 ixgbe_dev_rxq_interrupt_setup(dev); 2789 2790 /* enable uio/vfio intr/eventfd mapping */ 2791 rte_intr_enable(intr_handle); 2792 2793 /* resume enabled intr since hw reset */ 2794 ixgbe_enable_intr(dev); 2795 ixgbe_l2_tunnel_conf(dev); 2796 ixgbe_filter_restore(dev); 2797 2798 if (tm_conf->root && !tm_conf->committed) 2799 PMD_DRV_LOG(WARNING, 2800 "please call hierarchy_commit() " 2801 "before starting the port"); 2802 2803 /* wait for the controller to acquire link */ 2804 err = ixgbe_wait_for_link_up(hw); 2805 if (err) 2806 goto error; 2807 2808 /* 2809 * Update link status right before return, because it may 2810 * start link configuration process in a separate thread. 2811 */ 2812 ixgbe_dev_link_update(dev, 0); 2813 2814 /* setup the macsec setting register */ 2815 if (macsec_setting->offload_en) 2816 ixgbe_dev_macsec_register_enable(dev, macsec_setting); 2817 2818 return 0; 2819 2820 error: 2821 PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err); 2822 ixgbe_dev_clear_queues(dev); 2823 return -EIO; 2824 } 2825 2826 /* 2827 * Stop device: disable rx and tx functions to allow for reconfiguring. 2828 */ 2829 static int 2830 ixgbe_dev_stop(struct rte_eth_dev *dev) 2831 { 2832 struct rte_eth_link link; 2833 struct ixgbe_adapter *adapter = dev->data->dev_private; 2834 struct ixgbe_hw *hw = 2835 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2836 struct ixgbe_vf_info *vfinfo = 2837 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2838 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2839 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2840 int vf; 2841 struct ixgbe_tm_conf *tm_conf = 2842 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2843 2844 if (hw->adapter_stopped) 2845 return 0; 2846 2847 PMD_INIT_FUNC_TRACE(); 2848 2849 ixgbe_dev_wait_setup_link_complete(dev, 0); 2850 2851 /* disable interrupts */ 2852 ixgbe_disable_intr(hw); 2853 2854 /* reset the NIC */ 2855 ixgbe_pf_reset_hw(hw); 2856 hw->adapter_stopped = 0; 2857 2858 /* stop adapter */ 2859 ixgbe_stop_adapter(hw); 2860 2861 for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) 2862 vfinfo[vf].clear_to_send = false; 2863 2864 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2865 /* Turn off the copper */ 2866 ixgbe_set_phy_power(hw, false); 2867 } else { 2868 /* Turn off the laser */ 2869 ixgbe_disable_tx_laser(hw); 2870 } 2871 2872 ixgbe_dev_clear_queues(dev); 2873 2874 /* Clear stored conf */ 2875 dev->data->scattered_rx = 0; 2876 dev->data->lro = 0; 2877 2878 /* Clear recorded link status */ 2879 memset(&link, 0, sizeof(link)); 2880 rte_eth_linkstatus_set(dev, &link); 2881 2882 if (!rte_intr_allow_others(intr_handle)) 2883 /* resume to the default handler */ 2884 rte_intr_callback_register(intr_handle, 2885 ixgbe_dev_interrupt_handler, 2886 (void *)dev); 2887 2888 /* Clean datapath event and queue/vec mapping */ 2889 rte_intr_efd_disable(intr_handle); 2890 if (intr_handle->intr_vec != NULL) { 2891 rte_free(intr_handle->intr_vec); 2892 intr_handle->intr_vec = NULL; 2893 } 2894 2895 /* reset hierarchy commit */ 2896 tm_conf->committed = false; 2897 2898 adapter->rss_reta_updated = 0; 2899 2900 hw->adapter_stopped = true; 2901 dev->data->dev_started = 0; 2902 2903 return 0; 2904 } 2905 2906 /* 2907 * Set device link up: enable tx. 2908 */ 2909 static int 2910 ixgbe_dev_set_link_up(struct rte_eth_dev *dev) 2911 { 2912 struct ixgbe_hw *hw = 2913 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2914 if (hw->mac.type == ixgbe_mac_82599EB) { 2915 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2916 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2917 /* Not suported in bypass mode */ 2918 PMD_INIT_LOG(ERR, "Set link up is not supported " 2919 "by device id 0x%x", hw->device_id); 2920 return -ENOTSUP; 2921 } 2922 #endif 2923 } 2924 2925 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2926 /* Turn on the copper */ 2927 ixgbe_set_phy_power(hw, true); 2928 } else { 2929 /* Turn on the laser */ 2930 ixgbe_enable_tx_laser(hw); 2931 ixgbe_dev_link_update(dev, 0); 2932 } 2933 2934 return 0; 2935 } 2936 2937 /* 2938 * Set device link down: disable tx. 2939 */ 2940 static int 2941 ixgbe_dev_set_link_down(struct rte_eth_dev *dev) 2942 { 2943 struct ixgbe_hw *hw = 2944 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2945 if (hw->mac.type == ixgbe_mac_82599EB) { 2946 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2947 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2948 /* Not suported in bypass mode */ 2949 PMD_INIT_LOG(ERR, "Set link down is not supported " 2950 "by device id 0x%x", hw->device_id); 2951 return -ENOTSUP; 2952 } 2953 #endif 2954 } 2955 2956 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2957 /* Turn off the copper */ 2958 ixgbe_set_phy_power(hw, false); 2959 } else { 2960 /* Turn off the laser */ 2961 ixgbe_disable_tx_laser(hw); 2962 ixgbe_dev_link_update(dev, 0); 2963 } 2964 2965 return 0; 2966 } 2967 2968 /* 2969 * Reset and stop device. 2970 */ 2971 static int 2972 ixgbe_dev_close(struct rte_eth_dev *dev) 2973 { 2974 struct ixgbe_hw *hw = 2975 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2976 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2977 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2978 int retries = 0; 2979 int ret; 2980 2981 PMD_INIT_FUNC_TRACE(); 2982 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2983 return 0; 2984 2985 ixgbe_pf_reset_hw(hw); 2986 2987 ret = ixgbe_dev_stop(dev); 2988 2989 ixgbe_dev_free_queues(dev); 2990 2991 ixgbe_disable_pcie_master(hw); 2992 2993 /* reprogram the RAR[0] in case user changed it. */ 2994 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 2995 2996 /* Unlock any pending hardware semaphore */ 2997 ixgbe_swfw_lock_reset(hw); 2998 2999 /* disable uio intr before callback unregister */ 3000 rte_intr_disable(intr_handle); 3001 3002 do { 3003 ret = rte_intr_callback_unregister(intr_handle, 3004 ixgbe_dev_interrupt_handler, dev); 3005 if (ret >= 0 || ret == -ENOENT) { 3006 break; 3007 } else if (ret != -EAGAIN) { 3008 PMD_INIT_LOG(ERR, 3009 "intr callback unregister failed: %d", 3010 ret); 3011 } 3012 rte_delay_ms(100); 3013 } while (retries++ < (10 + IXGBE_LINK_UP_TIME)); 3014 3015 /* cancel the delay handler before remove dev */ 3016 rte_eal_alarm_cancel(ixgbe_dev_interrupt_delayed_handler, dev); 3017 3018 /* uninitialize PF if max_vfs not zero */ 3019 ixgbe_pf_host_uninit(dev); 3020 3021 /* remove all the fdir filters & hash */ 3022 ixgbe_fdir_filter_uninit(dev); 3023 3024 /* remove all the L2 tunnel filters & hash */ 3025 ixgbe_l2_tn_filter_uninit(dev); 3026 3027 /* Remove all ntuple filters of the device */ 3028 ixgbe_ntuple_filter_uninit(dev); 3029 3030 /* clear all the filters list */ 3031 ixgbe_filterlist_flush(); 3032 3033 /* Remove all Traffic Manager configuration */ 3034 ixgbe_tm_conf_uninit(dev); 3035 3036 #ifdef RTE_LIB_SECURITY 3037 rte_free(dev->security_ctx); 3038 #endif 3039 3040 return ret; 3041 } 3042 3043 /* 3044 * Reset PF device. 3045 */ 3046 static int 3047 ixgbe_dev_reset(struct rte_eth_dev *dev) 3048 { 3049 int ret; 3050 3051 /* When a DPDK PMD PF begin to reset PF port, it should notify all 3052 * its VF to make them align with it. The detailed notification 3053 * mechanism is PMD specific. As to ixgbe PF, it is rather complex. 3054 * To avoid unexpected behavior in VF, currently reset of PF with 3055 * SR-IOV activation is not supported. It might be supported later. 3056 */ 3057 if (dev->data->sriov.active) 3058 return -ENOTSUP; 3059 3060 ret = eth_ixgbe_dev_uninit(dev); 3061 if (ret) 3062 return ret; 3063 3064 ret = eth_ixgbe_dev_init(dev, NULL); 3065 3066 return ret; 3067 } 3068 3069 static void 3070 ixgbe_read_stats_registers(struct ixgbe_hw *hw, 3071 struct ixgbe_hw_stats *hw_stats, 3072 struct ixgbe_macsec_stats *macsec_stats, 3073 uint64_t *total_missed_rx, uint64_t *total_qbrc, 3074 uint64_t *total_qprc, uint64_t *total_qprdc) 3075 { 3076 uint32_t bprc, lxon, lxoff, total; 3077 uint32_t delta_gprc = 0; 3078 unsigned i; 3079 /* Workaround for RX byte count not including CRC bytes when CRC 3080 * strip is enabled. CRC bytes are removed from counters when crc_strip 3081 * is disabled. 3082 */ 3083 int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) & 3084 IXGBE_HLREG0_RXCRCSTRP); 3085 3086 hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3087 hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3088 hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3089 hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3090 3091 for (i = 0; i < 8; i++) { 3092 uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3093 3094 /* global total per queue */ 3095 hw_stats->mpc[i] += mp; 3096 /* Running comprehensive total for stats display */ 3097 *total_missed_rx += hw_stats->mpc[i]; 3098 if (hw->mac.type == ixgbe_mac_82598EB) { 3099 hw_stats->rnbc[i] += 3100 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3101 hw_stats->pxonrxc[i] += 3102 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 3103 hw_stats->pxoffrxc[i] += 3104 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 3105 } else { 3106 hw_stats->pxonrxc[i] += 3107 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 3108 hw_stats->pxoffrxc[i] += 3109 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 3110 hw_stats->pxon2offc[i] += 3111 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 3112 } 3113 hw_stats->pxontxc[i] += 3114 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 3115 hw_stats->pxofftxc[i] += 3116 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 3117 } 3118 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3119 uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3120 uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3121 uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3122 3123 delta_gprc += delta_qprc; 3124 3125 hw_stats->qprc[i] += delta_qprc; 3126 hw_stats->qptc[i] += delta_qptc; 3127 3128 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 3129 hw_stats->qbrc[i] += 3130 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32); 3131 if (crc_strip == 0) 3132 hw_stats->qbrc[i] -= delta_qprc * RTE_ETHER_CRC_LEN; 3133 3134 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 3135 hw_stats->qbtc[i] += 3136 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32); 3137 3138 hw_stats->qprdc[i] += delta_qprdc; 3139 *total_qprdc += hw_stats->qprdc[i]; 3140 3141 *total_qprc += hw_stats->qprc[i]; 3142 *total_qbrc += hw_stats->qbrc[i]; 3143 } 3144 hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3145 hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3146 hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3147 3148 /* 3149 * An errata states that gprc actually counts good + missed packets: 3150 * Workaround to set gprc to summated queue packet receives 3151 */ 3152 hw_stats->gprc = *total_qprc; 3153 3154 if (hw->mac.type != ixgbe_mac_82598EB) { 3155 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 3156 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3157 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 3158 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3159 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 3160 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3161 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3162 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3163 } else { 3164 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3165 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3166 /* 82598 only has a counter in the high register */ 3167 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3168 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3169 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3170 } 3171 uint64_t old_tpr = hw_stats->tpr; 3172 3173 hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3174 hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3175 3176 if (crc_strip == 0) 3177 hw_stats->gorc -= delta_gprc * RTE_ETHER_CRC_LEN; 3178 3179 uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC); 3180 hw_stats->gptc += delta_gptc; 3181 hw_stats->gotc -= delta_gptc * RTE_ETHER_CRC_LEN; 3182 hw_stats->tor -= (hw_stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 3183 3184 /* 3185 * Workaround: mprc hardware is incorrectly counting 3186 * broadcasts, so for now we subtract those. 3187 */ 3188 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3189 hw_stats->bprc += bprc; 3190 hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3191 if (hw->mac.type == ixgbe_mac_82598EB) 3192 hw_stats->mprc -= bprc; 3193 3194 hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3195 hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3196 hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3197 hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3198 hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3199 hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3200 3201 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3202 hw_stats->lxontxc += lxon; 3203 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3204 hw_stats->lxofftxc += lxoff; 3205 total = lxon + lxoff; 3206 3207 hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3208 hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3209 hw_stats->gptc -= total; 3210 hw_stats->mptc -= total; 3211 hw_stats->ptc64 -= total; 3212 hw_stats->gotc -= total * RTE_ETHER_MIN_LEN; 3213 3214 hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3215 hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3216 hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3217 hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3218 hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3219 hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3220 hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3221 hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3222 hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3223 hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3224 hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3225 hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3226 hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3227 hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3228 hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3229 hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3230 /* Only read FCOE on 82599 */ 3231 if (hw->mac.type != ixgbe_mac_82598EB) { 3232 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3233 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3234 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3235 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3236 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3237 } 3238 3239 /* Flow Director Stats registers */ 3240 if (hw->mac.type != ixgbe_mac_82598EB) { 3241 hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 3242 hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 3243 hw_stats->fdirustat_add += IXGBE_READ_REG(hw, 3244 IXGBE_FDIRUSTAT) & 0xFFFF; 3245 hw_stats->fdirustat_remove += (IXGBE_READ_REG(hw, 3246 IXGBE_FDIRUSTAT) >> 16) & 0xFFFF; 3247 hw_stats->fdirfstat_fadd += IXGBE_READ_REG(hw, 3248 IXGBE_FDIRFSTAT) & 0xFFFF; 3249 hw_stats->fdirfstat_fremove += (IXGBE_READ_REG(hw, 3250 IXGBE_FDIRFSTAT) >> 16) & 0xFFFF; 3251 } 3252 /* MACsec Stats registers */ 3253 macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT); 3254 macsec_stats->out_pkts_encrypted += 3255 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE); 3256 macsec_stats->out_pkts_protected += 3257 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP); 3258 macsec_stats->out_octets_encrypted += 3259 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE); 3260 macsec_stats->out_octets_protected += 3261 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP); 3262 macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT); 3263 macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD); 3264 macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI); 3265 macsec_stats->in_pkts_unknownsci += 3266 IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI); 3267 macsec_stats->in_octets_decrypted += 3268 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD); 3269 macsec_stats->in_octets_validated += 3270 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV); 3271 macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH); 3272 macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY); 3273 macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE); 3274 for (i = 0; i < 2; i++) { 3275 macsec_stats->in_pkts_ok += 3276 IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i)); 3277 macsec_stats->in_pkts_invalid += 3278 IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i)); 3279 macsec_stats->in_pkts_notvalid += 3280 IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i)); 3281 } 3282 macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA); 3283 macsec_stats->in_pkts_notusingsa += 3284 IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA); 3285 } 3286 3287 /* 3288 * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c 3289 */ 3290 static int 3291 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3292 { 3293 struct ixgbe_hw *hw = 3294 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3295 struct ixgbe_hw_stats *hw_stats = 3296 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3297 struct ixgbe_macsec_stats *macsec_stats = 3298 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3299 dev->data->dev_private); 3300 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3301 unsigned i; 3302 3303 total_missed_rx = 0; 3304 total_qbrc = 0; 3305 total_qprc = 0; 3306 total_qprdc = 0; 3307 3308 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3309 &total_qbrc, &total_qprc, &total_qprdc); 3310 3311 if (stats == NULL) 3312 return -EINVAL; 3313 3314 /* Fill out the rte_eth_stats statistics structure */ 3315 stats->ipackets = total_qprc; 3316 stats->ibytes = total_qbrc; 3317 stats->opackets = hw_stats->gptc; 3318 stats->obytes = hw_stats->gotc; 3319 3320 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3321 stats->q_ipackets[i] = hw_stats->qprc[i]; 3322 stats->q_opackets[i] = hw_stats->qptc[i]; 3323 stats->q_ibytes[i] = hw_stats->qbrc[i]; 3324 stats->q_obytes[i] = hw_stats->qbtc[i]; 3325 stats->q_errors[i] = hw_stats->qprdc[i]; 3326 } 3327 3328 /* Rx Errors */ 3329 stats->imissed = total_missed_rx; 3330 stats->ierrors = hw_stats->crcerrs + 3331 hw_stats->mspdc + 3332 hw_stats->rlec + 3333 hw_stats->ruc + 3334 hw_stats->roc + 3335 hw_stats->illerrc + 3336 hw_stats->errbc + 3337 hw_stats->rfc + 3338 hw_stats->fccrc + 3339 hw_stats->fclast; 3340 3341 /* Tx Errors */ 3342 stats->oerrors = 0; 3343 return 0; 3344 } 3345 3346 static int 3347 ixgbe_dev_stats_reset(struct rte_eth_dev *dev) 3348 { 3349 struct ixgbe_hw_stats *stats = 3350 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3351 3352 /* HW registers are cleared on read */ 3353 ixgbe_dev_stats_get(dev, NULL); 3354 3355 /* Reset software totals */ 3356 memset(stats, 0, sizeof(*stats)); 3357 3358 return 0; 3359 } 3360 3361 /* This function calculates the number of xstats based on the current config */ 3362 static unsigned 3363 ixgbe_xstats_calc_num(void) { 3364 return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS + 3365 (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) + 3366 (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES); 3367 } 3368 3369 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3370 struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int size) 3371 { 3372 const unsigned cnt_stats = ixgbe_xstats_calc_num(); 3373 unsigned stat, i, count; 3374 3375 if (xstats_names != NULL) { 3376 count = 0; 3377 3378 /* Note: limit >= cnt_stats checked upstream 3379 * in rte_eth_xstats_names() 3380 */ 3381 3382 /* Extended stats from ixgbe_hw_stats */ 3383 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3384 strlcpy(xstats_names[count].name, 3385 rte_ixgbe_stats_strings[i].name, 3386 sizeof(xstats_names[count].name)); 3387 count++; 3388 } 3389 3390 /* MACsec Stats */ 3391 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3392 strlcpy(xstats_names[count].name, 3393 rte_ixgbe_macsec_strings[i].name, 3394 sizeof(xstats_names[count].name)); 3395 count++; 3396 } 3397 3398 /* RX Priority Stats */ 3399 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3400 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3401 snprintf(xstats_names[count].name, 3402 sizeof(xstats_names[count].name), 3403 "rx_priority%u_%s", i, 3404 rte_ixgbe_rxq_strings[stat].name); 3405 count++; 3406 } 3407 } 3408 3409 /* TX Priority Stats */ 3410 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3411 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3412 snprintf(xstats_names[count].name, 3413 sizeof(xstats_names[count].name), 3414 "tx_priority%u_%s", i, 3415 rte_ixgbe_txq_strings[stat].name); 3416 count++; 3417 } 3418 } 3419 } 3420 return cnt_stats; 3421 } 3422 3423 static int ixgbe_dev_xstats_get_names_by_id( 3424 struct rte_eth_dev *dev, 3425 struct rte_eth_xstat_name *xstats_names, 3426 const uint64_t *ids, 3427 unsigned int limit) 3428 { 3429 if (!ids) { 3430 const unsigned int cnt_stats = ixgbe_xstats_calc_num(); 3431 unsigned int stat, i, count; 3432 3433 if (xstats_names != NULL) { 3434 count = 0; 3435 3436 /* Note: limit >= cnt_stats checked upstream 3437 * in rte_eth_xstats_names() 3438 */ 3439 3440 /* Extended stats from ixgbe_hw_stats */ 3441 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3442 strlcpy(xstats_names[count].name, 3443 rte_ixgbe_stats_strings[i].name, 3444 sizeof(xstats_names[count].name)); 3445 count++; 3446 } 3447 3448 /* MACsec Stats */ 3449 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3450 strlcpy(xstats_names[count].name, 3451 rte_ixgbe_macsec_strings[i].name, 3452 sizeof(xstats_names[count].name)); 3453 count++; 3454 } 3455 3456 /* RX Priority Stats */ 3457 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3458 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3459 snprintf(xstats_names[count].name, 3460 sizeof(xstats_names[count].name), 3461 "rx_priority%u_%s", i, 3462 rte_ixgbe_rxq_strings[stat].name); 3463 count++; 3464 } 3465 } 3466 3467 /* TX Priority Stats */ 3468 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3469 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3470 snprintf(xstats_names[count].name, 3471 sizeof(xstats_names[count].name), 3472 "tx_priority%u_%s", i, 3473 rte_ixgbe_txq_strings[stat].name); 3474 count++; 3475 } 3476 } 3477 } 3478 return cnt_stats; 3479 } 3480 3481 uint16_t i; 3482 uint16_t size = ixgbe_xstats_calc_num(); 3483 struct rte_eth_xstat_name xstats_names_copy[size]; 3484 3485 ixgbe_dev_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 3486 size); 3487 3488 for (i = 0; i < limit; i++) { 3489 if (ids[i] >= size) { 3490 PMD_INIT_LOG(ERR, "id value isn't valid"); 3491 return -1; 3492 } 3493 strcpy(xstats_names[i].name, 3494 xstats_names_copy[ids[i]].name); 3495 } 3496 return limit; 3497 } 3498 3499 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3500 struct rte_eth_xstat_name *xstats_names, unsigned limit) 3501 { 3502 unsigned i; 3503 3504 if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL) 3505 return -ENOMEM; 3506 3507 if (xstats_names != NULL) 3508 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) 3509 strlcpy(xstats_names[i].name, 3510 rte_ixgbevf_stats_strings[i].name, 3511 sizeof(xstats_names[i].name)); 3512 return IXGBEVF_NB_XSTATS; 3513 } 3514 3515 static int 3516 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3517 unsigned n) 3518 { 3519 struct ixgbe_hw *hw = 3520 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3521 struct ixgbe_hw_stats *hw_stats = 3522 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3523 struct ixgbe_macsec_stats *macsec_stats = 3524 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3525 dev->data->dev_private); 3526 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3527 unsigned i, stat, count = 0; 3528 3529 count = ixgbe_xstats_calc_num(); 3530 3531 if (n < count) 3532 return count; 3533 3534 total_missed_rx = 0; 3535 total_qbrc = 0; 3536 total_qprc = 0; 3537 total_qprdc = 0; 3538 3539 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3540 &total_qbrc, &total_qprc, &total_qprdc); 3541 3542 /* If this is a reset xstats is NULL, and we have cleared the 3543 * registers by reading them. 3544 */ 3545 if (!xstats) 3546 return 0; 3547 3548 /* Extended stats from ixgbe_hw_stats */ 3549 count = 0; 3550 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3551 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3552 rte_ixgbe_stats_strings[i].offset); 3553 xstats[count].id = count; 3554 count++; 3555 } 3556 3557 /* MACsec Stats */ 3558 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3559 xstats[count].value = *(uint64_t *)(((char *)macsec_stats) + 3560 rte_ixgbe_macsec_strings[i].offset); 3561 xstats[count].id = count; 3562 count++; 3563 } 3564 3565 /* RX Priority Stats */ 3566 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3567 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3568 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3569 rte_ixgbe_rxq_strings[stat].offset + 3570 (sizeof(uint64_t) * i)); 3571 xstats[count].id = count; 3572 count++; 3573 } 3574 } 3575 3576 /* TX Priority Stats */ 3577 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3578 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3579 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3580 rte_ixgbe_txq_strings[stat].offset + 3581 (sizeof(uint64_t) * i)); 3582 xstats[count].id = count; 3583 count++; 3584 } 3585 } 3586 return count; 3587 } 3588 3589 static int 3590 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 3591 uint64_t *values, unsigned int n) 3592 { 3593 if (!ids) { 3594 struct ixgbe_hw *hw = 3595 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3596 struct ixgbe_hw_stats *hw_stats = 3597 IXGBE_DEV_PRIVATE_TO_STATS( 3598 dev->data->dev_private); 3599 struct ixgbe_macsec_stats *macsec_stats = 3600 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3601 dev->data->dev_private); 3602 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3603 unsigned int i, stat, count = 0; 3604 3605 count = ixgbe_xstats_calc_num(); 3606 3607 if (!ids && n < count) 3608 return count; 3609 3610 total_missed_rx = 0; 3611 total_qbrc = 0; 3612 total_qprc = 0; 3613 total_qprdc = 0; 3614 3615 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, 3616 &total_missed_rx, &total_qbrc, &total_qprc, 3617 &total_qprdc); 3618 3619 /* If this is a reset xstats is NULL, and we have cleared the 3620 * registers by reading them. 3621 */ 3622 if (!ids && !values) 3623 return 0; 3624 3625 /* Extended stats from ixgbe_hw_stats */ 3626 count = 0; 3627 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3628 values[count] = *(uint64_t *)(((char *)hw_stats) + 3629 rte_ixgbe_stats_strings[i].offset); 3630 count++; 3631 } 3632 3633 /* MACsec Stats */ 3634 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3635 values[count] = *(uint64_t *)(((char *)macsec_stats) + 3636 rte_ixgbe_macsec_strings[i].offset); 3637 count++; 3638 } 3639 3640 /* RX Priority Stats */ 3641 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3642 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3643 values[count] = 3644 *(uint64_t *)(((char *)hw_stats) + 3645 rte_ixgbe_rxq_strings[stat].offset + 3646 (sizeof(uint64_t) * i)); 3647 count++; 3648 } 3649 } 3650 3651 /* TX Priority Stats */ 3652 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3653 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3654 values[count] = 3655 *(uint64_t *)(((char *)hw_stats) + 3656 rte_ixgbe_txq_strings[stat].offset + 3657 (sizeof(uint64_t) * i)); 3658 count++; 3659 } 3660 } 3661 return count; 3662 } 3663 3664 uint16_t i; 3665 uint16_t size = ixgbe_xstats_calc_num(); 3666 uint64_t values_copy[size]; 3667 3668 ixgbe_dev_xstats_get_by_id(dev, NULL, values_copy, size); 3669 3670 for (i = 0; i < n; i++) { 3671 if (ids[i] >= size) { 3672 PMD_INIT_LOG(ERR, "id value isn't valid"); 3673 return -1; 3674 } 3675 values[i] = values_copy[ids[i]]; 3676 } 3677 return n; 3678 } 3679 3680 static int 3681 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) 3682 { 3683 struct ixgbe_hw_stats *stats = 3684 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3685 struct ixgbe_macsec_stats *macsec_stats = 3686 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3687 dev->data->dev_private); 3688 3689 unsigned count = ixgbe_xstats_calc_num(); 3690 3691 /* HW registers are cleared on read */ 3692 ixgbe_dev_xstats_get(dev, NULL, count); 3693 3694 /* Reset software totals */ 3695 memset(stats, 0, sizeof(*stats)); 3696 memset(macsec_stats, 0, sizeof(*macsec_stats)); 3697 3698 return 0; 3699 } 3700 3701 static void 3702 ixgbevf_update_stats(struct rte_eth_dev *dev) 3703 { 3704 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3705 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3706 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3707 3708 /* Good Rx packet, include VF loopback */ 3709 UPDATE_VF_STAT(IXGBE_VFGPRC, 3710 hw_stats->last_vfgprc, hw_stats->vfgprc); 3711 3712 /* Good Rx octets, include VF loopback */ 3713 UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3714 hw_stats->last_vfgorc, hw_stats->vfgorc); 3715 3716 /* Good Tx packet, include VF loopback */ 3717 UPDATE_VF_STAT(IXGBE_VFGPTC, 3718 hw_stats->last_vfgptc, hw_stats->vfgptc); 3719 3720 /* Good Tx octets, include VF loopback */ 3721 UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3722 hw_stats->last_vfgotc, hw_stats->vfgotc); 3723 3724 /* Rx Multicst Packet */ 3725 UPDATE_VF_STAT(IXGBE_VFMPRC, 3726 hw_stats->last_vfmprc, hw_stats->vfmprc); 3727 } 3728 3729 static int 3730 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3731 unsigned n) 3732 { 3733 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3734 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3735 unsigned i; 3736 3737 if (n < IXGBEVF_NB_XSTATS) 3738 return IXGBEVF_NB_XSTATS; 3739 3740 ixgbevf_update_stats(dev); 3741 3742 if (!xstats) 3743 return 0; 3744 3745 /* Extended stats */ 3746 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) { 3747 xstats[i].id = i; 3748 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 3749 rte_ixgbevf_stats_strings[i].offset); 3750 } 3751 3752 return IXGBEVF_NB_XSTATS; 3753 } 3754 3755 static int 3756 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3757 { 3758 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3759 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3760 3761 ixgbevf_update_stats(dev); 3762 3763 if (stats == NULL) 3764 return -EINVAL; 3765 3766 stats->ipackets = hw_stats->vfgprc; 3767 stats->ibytes = hw_stats->vfgorc; 3768 stats->opackets = hw_stats->vfgptc; 3769 stats->obytes = hw_stats->vfgotc; 3770 return 0; 3771 } 3772 3773 static int 3774 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev) 3775 { 3776 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3777 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3778 3779 /* Sync HW register to the last stats */ 3780 ixgbevf_dev_stats_get(dev, NULL); 3781 3782 /* reset HW current stats*/ 3783 hw_stats->vfgprc = 0; 3784 hw_stats->vfgorc = 0; 3785 hw_stats->vfgptc = 0; 3786 hw_stats->vfgotc = 0; 3787 3788 return 0; 3789 } 3790 3791 static int 3792 ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 3793 { 3794 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3795 u16 eeprom_verh, eeprom_verl; 3796 u32 etrack_id; 3797 int ret; 3798 3799 ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); 3800 ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); 3801 3802 etrack_id = (eeprom_verh << 16) | eeprom_verl; 3803 ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); 3804 3805 ret += 1; /* add the size of '\0' */ 3806 if (fw_size < (u32)ret) 3807 return ret; 3808 else 3809 return 0; 3810 } 3811 3812 static int 3813 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 3814 { 3815 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3816 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3817 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3818 3819 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3820 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3821 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3822 /* 3823 * When DCB/VT is off, maximum number of queues changes, 3824 * except for 82598EB, which remains constant. 3825 */ 3826 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 3827 hw->mac.type != ixgbe_mac_82598EB) 3828 dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES; 3829 } 3830 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */ 3831 dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */ 3832 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3833 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3834 dev_info->max_vfs = pci_dev->max_vfs; 3835 if (hw->mac.type == ixgbe_mac_82598EB) 3836 dev_info->max_vmdq_pools = ETH_16_POOLS; 3837 else 3838 dev_info->max_vmdq_pools = ETH_64_POOLS; 3839 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3840 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 3841 dev_info->vmdq_queue_num = dev_info->max_rx_queues; 3842 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3843 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3844 dev_info->rx_queue_offload_capa); 3845 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3846 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3847 3848 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3849 .rx_thresh = { 3850 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3851 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3852 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3853 }, 3854 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3855 .rx_drop_en = 0, 3856 .offloads = 0, 3857 }; 3858 3859 dev_info->default_txconf = (struct rte_eth_txconf) { 3860 .tx_thresh = { 3861 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3862 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3863 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3864 }, 3865 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3866 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3867 .offloads = 0, 3868 }; 3869 3870 dev_info->rx_desc_lim = rx_desc_lim; 3871 dev_info->tx_desc_lim = tx_desc_lim; 3872 3873 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3874 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3875 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3876 3877 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; 3878 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 3879 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 3880 dev_info->speed_capa = ETH_LINK_SPEED_10M | 3881 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 3882 3883 if (hw->mac.type == ixgbe_mac_X540 || 3884 hw->mac.type == ixgbe_mac_X540_vf || 3885 hw->mac.type == ixgbe_mac_X550 || 3886 hw->mac.type == ixgbe_mac_X550_vf) { 3887 dev_info->speed_capa |= ETH_LINK_SPEED_100M; 3888 } 3889 if (hw->mac.type == ixgbe_mac_X550) { 3890 dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; 3891 dev_info->speed_capa |= ETH_LINK_SPEED_5G; 3892 } 3893 3894 /* Driver-preferred Rx/Tx parameters */ 3895 dev_info->default_rxportconf.burst_size = 32; 3896 dev_info->default_txportconf.burst_size = 32; 3897 dev_info->default_rxportconf.nb_queues = 1; 3898 dev_info->default_txportconf.nb_queues = 1; 3899 dev_info->default_rxportconf.ring_size = 256; 3900 dev_info->default_txportconf.ring_size = 256; 3901 3902 return 0; 3903 } 3904 3905 static const uint32_t * 3906 ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) 3907 { 3908 static const uint32_t ptypes[] = { 3909 /* For non-vec functions, 3910 * refers to ixgbe_rxd_pkt_info_to_pkt_type(); 3911 * for vec functions, 3912 * refers to _recv_raw_pkts_vec(). 3913 */ 3914 RTE_PTYPE_L2_ETHER, 3915 RTE_PTYPE_L3_IPV4, 3916 RTE_PTYPE_L3_IPV4_EXT, 3917 RTE_PTYPE_L3_IPV6, 3918 RTE_PTYPE_L3_IPV6_EXT, 3919 RTE_PTYPE_L4_SCTP, 3920 RTE_PTYPE_L4_TCP, 3921 RTE_PTYPE_L4_UDP, 3922 RTE_PTYPE_TUNNEL_IP, 3923 RTE_PTYPE_INNER_L3_IPV6, 3924 RTE_PTYPE_INNER_L3_IPV6_EXT, 3925 RTE_PTYPE_INNER_L4_TCP, 3926 RTE_PTYPE_INNER_L4_UDP, 3927 RTE_PTYPE_UNKNOWN 3928 }; 3929 3930 if (dev->rx_pkt_burst == ixgbe_recv_pkts || 3931 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc || 3932 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || 3933 dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) 3934 return ptypes; 3935 3936 #if defined(RTE_ARCH_X86) || defined(__ARM_NEON) 3937 if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || 3938 dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) 3939 return ptypes; 3940 #endif 3941 return NULL; 3942 } 3943 3944 static int 3945 ixgbevf_dev_info_get(struct rte_eth_dev *dev, 3946 struct rte_eth_dev_info *dev_info) 3947 { 3948 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3949 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3950 3951 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3952 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3953 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */ 3954 dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */ 3955 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3956 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3957 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3958 dev_info->max_vfs = pci_dev->max_vfs; 3959 if (hw->mac.type == ixgbe_mac_82598EB) 3960 dev_info->max_vmdq_pools = ETH_16_POOLS; 3961 else 3962 dev_info->max_vmdq_pools = ETH_64_POOLS; 3963 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3964 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3965 dev_info->rx_queue_offload_capa); 3966 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3967 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3968 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3969 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3970 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3971 3972 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3973 .rx_thresh = { 3974 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3975 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3976 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3977 }, 3978 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3979 .rx_drop_en = 0, 3980 .offloads = 0, 3981 }; 3982 3983 dev_info->default_txconf = (struct rte_eth_txconf) { 3984 .tx_thresh = { 3985 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3986 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3987 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3988 }, 3989 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3990 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3991 .offloads = 0, 3992 }; 3993 3994 dev_info->rx_desc_lim = rx_desc_lim; 3995 dev_info->tx_desc_lim = tx_desc_lim; 3996 3997 return 0; 3998 } 3999 4000 static int 4001 ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, 4002 bool *link_up, int wait_to_complete) 4003 { 4004 struct ixgbe_adapter *adapter = container_of(hw, 4005 struct ixgbe_adapter, hw); 4006 struct ixgbe_mbx_info *mbx = &hw->mbx; 4007 struct ixgbe_mac_info *mac = &hw->mac; 4008 uint32_t links_reg, in_msg; 4009 int ret_val = 0; 4010 4011 /* If we were hit with a reset drop the link */ 4012 if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) 4013 mac->get_link_status = true; 4014 4015 if (!mac->get_link_status) 4016 goto out; 4017 4018 /* if link status is down no point in checking to see if pf is up */ 4019 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 4020 if (!(links_reg & IXGBE_LINKS_UP)) 4021 goto out; 4022 4023 /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs 4024 * before the link status is correct 4025 */ 4026 if (mac->type == ixgbe_mac_82599_vf && wait_to_complete) { 4027 int i; 4028 4029 for (i = 0; i < 5; i++) { 4030 rte_delay_us(100); 4031 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 4032 4033 if (!(links_reg & IXGBE_LINKS_UP)) 4034 goto out; 4035 } 4036 } 4037 4038 switch (links_reg & IXGBE_LINKS_SPEED_82599) { 4039 case IXGBE_LINKS_SPEED_10G_82599: 4040 *speed = IXGBE_LINK_SPEED_10GB_FULL; 4041 if (hw->mac.type >= ixgbe_mac_X550) { 4042 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4043 *speed = IXGBE_LINK_SPEED_2_5GB_FULL; 4044 } 4045 break; 4046 case IXGBE_LINKS_SPEED_1G_82599: 4047 *speed = IXGBE_LINK_SPEED_1GB_FULL; 4048 break; 4049 case IXGBE_LINKS_SPEED_100_82599: 4050 *speed = IXGBE_LINK_SPEED_100_FULL; 4051 if (hw->mac.type == ixgbe_mac_X550) { 4052 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4053 *speed = IXGBE_LINK_SPEED_5GB_FULL; 4054 } 4055 break; 4056 case IXGBE_LINKS_SPEED_10_X550EM_A: 4057 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4058 /* Since Reserved in older MAC's */ 4059 if (hw->mac.type >= ixgbe_mac_X550) 4060 *speed = IXGBE_LINK_SPEED_10_FULL; 4061 break; 4062 default: 4063 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4064 } 4065 4066 if (wait_to_complete == 0 && adapter->pflink_fullchk == 0) { 4067 if (*speed == IXGBE_LINK_SPEED_UNKNOWN) 4068 mac->get_link_status = true; 4069 else 4070 mac->get_link_status = false; 4071 4072 goto out; 4073 } 4074 4075 /* if the read failed it could just be a mailbox collision, best wait 4076 * until we are called again and don't report an error 4077 */ 4078 if (mbx->ops.read(hw, &in_msg, 1, 0)) 4079 goto out; 4080 4081 if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { 4082 /* msg is not CTS and is NACK we must have lost CTS status */ 4083 if (in_msg & IXGBE_VT_MSGTYPE_NACK) 4084 mac->get_link_status = false; 4085 goto out; 4086 } 4087 4088 /* the pf is talking, if we timed out in the past we reinit */ 4089 if (!mbx->timeout) { 4090 ret_val = -1; 4091 goto out; 4092 } 4093 4094 /* if we passed all the tests above then the link is up and we no 4095 * longer need to check for link 4096 */ 4097 mac->get_link_status = false; 4098 4099 out: 4100 *link_up = !mac->get_link_status; 4101 return ret_val; 4102 } 4103 4104 /* 4105 * If @timeout_ms was 0, it means that it will not return until link complete. 4106 * It returns 1 on complete, return 0 on timeout. 4107 */ 4108 static int 4109 ixgbe_dev_wait_setup_link_complete(struct rte_eth_dev *dev, uint32_t timeout_ms) 4110 { 4111 #define WARNING_TIMEOUT 9000 /* 9s in total */ 4112 struct ixgbe_adapter *ad = dev->data->dev_private; 4113 uint32_t timeout = timeout_ms ? timeout_ms : WARNING_TIMEOUT; 4114 4115 while (rte_atomic32_read(&ad->link_thread_running)) { 4116 msec_delay(1); 4117 timeout--; 4118 4119 if (timeout_ms) { 4120 if (!timeout) 4121 return 0; 4122 } else if (!timeout) { 4123 /* It will not return until link complete */ 4124 timeout = WARNING_TIMEOUT; 4125 PMD_DRV_LOG(ERR, "IXGBE link thread not complete too long time!"); 4126 } 4127 } 4128 4129 return 1; 4130 } 4131 4132 static void * 4133 ixgbe_dev_setup_link_thread_handler(void *param) 4134 { 4135 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4136 struct ixgbe_adapter *ad = dev->data->dev_private; 4137 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4138 struct ixgbe_interrupt *intr = 4139 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4140 u32 speed; 4141 bool autoneg = false; 4142 4143 pthread_detach(pthread_self()); 4144 speed = hw->phy.autoneg_advertised; 4145 if (!speed) 4146 ixgbe_get_link_capabilities(hw, &speed, &autoneg); 4147 4148 ixgbe_setup_link(hw, speed, true); 4149 4150 intr->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4151 rte_atomic32_clear(&ad->link_thread_running); 4152 return NULL; 4153 } 4154 4155 /* 4156 * In freebsd environment, nic_uio drivers do not support interrupts, 4157 * rte_intr_callback_register() will fail to register interrupts. 4158 * We can not make link status to change from down to up by interrupt 4159 * callback. So we need to wait for the controller to acquire link 4160 * when ports start. 4161 * It returns 0 on link up. 4162 */ 4163 static int 4164 ixgbe_wait_for_link_up(struct ixgbe_hw *hw) 4165 { 4166 #ifdef RTE_EXEC_ENV_FREEBSD 4167 int err, i; 4168 bool link_up = false; 4169 uint32_t speed = 0; 4170 const int nb_iter = 25; 4171 4172 for (i = 0; i < nb_iter; i++) { 4173 err = ixgbe_check_link(hw, &speed, &link_up, 0); 4174 if (err) 4175 return err; 4176 if (link_up) 4177 return 0; 4178 msec_delay(200); 4179 } 4180 4181 return 0; 4182 #else 4183 RTE_SET_USED(hw); 4184 return 0; 4185 #endif 4186 } 4187 4188 /* return 0 means link status changed, -1 means not changed */ 4189 int 4190 ixgbe_dev_link_update_share(struct rte_eth_dev *dev, 4191 int wait_to_complete, int vf) 4192 { 4193 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4194 struct ixgbe_adapter *ad = dev->data->dev_private; 4195 struct rte_eth_link link; 4196 ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN; 4197 struct ixgbe_interrupt *intr = 4198 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4199 bool link_up; 4200 int diag; 4201 int wait = 1; 4202 u32 esdp_reg; 4203 4204 memset(&link, 0, sizeof(link)); 4205 link.link_status = ETH_LINK_DOWN; 4206 link.link_speed = ETH_SPEED_NUM_NONE; 4207 link.link_duplex = ETH_LINK_HALF_DUPLEX; 4208 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 4209 ETH_LINK_SPEED_FIXED); 4210 4211 hw->mac.get_link_status = true; 4212 4213 if (intr->flags & IXGBE_FLAG_NEED_LINK_CONFIG) 4214 return rte_eth_linkstatus_set(dev, &link); 4215 4216 /* check if it needs to wait to complete, if lsc interrupt is enabled */ 4217 if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) 4218 wait = 0; 4219 4220 /* BSD has no interrupt mechanism, so force NIC status synchronization. */ 4221 #ifdef RTE_EXEC_ENV_FREEBSD 4222 wait = 1; 4223 #endif 4224 4225 if (vf) 4226 diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); 4227 else 4228 diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); 4229 4230 if (diag != 0) { 4231 link.link_speed = ETH_SPEED_NUM_100M; 4232 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4233 return rte_eth_linkstatus_set(dev, &link); 4234 } 4235 4236 if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 4237 esdp_reg = IXGBE_READ_REG(hw, IXGBE_ESDP); 4238 if ((esdp_reg & IXGBE_ESDP_SDP3)) 4239 link_up = 0; 4240 } 4241 4242 if (link_up == 0) { 4243 if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 4244 ixgbe_dev_wait_setup_link_complete(dev, 0); 4245 if (rte_atomic32_test_and_set(&ad->link_thread_running)) { 4246 /* To avoid race condition between threads, set 4247 * the IXGBE_FLAG_NEED_LINK_CONFIG flag only 4248 * when there is no link thread running. 4249 */ 4250 intr->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 4251 if (rte_ctrl_thread_create(&ad->link_thread_tid, 4252 "ixgbe-link-handler", 4253 NULL, 4254 ixgbe_dev_setup_link_thread_handler, 4255 dev) < 0) { 4256 PMD_DRV_LOG(ERR, 4257 "Create link thread failed!"); 4258 rte_atomic32_clear(&ad->link_thread_running); 4259 } 4260 } else { 4261 PMD_DRV_LOG(ERR, 4262 "Other link thread is running now!"); 4263 } 4264 } 4265 return rte_eth_linkstatus_set(dev, &link); 4266 } 4267 4268 link.link_status = ETH_LINK_UP; 4269 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4270 4271 switch (link_speed) { 4272 default: 4273 case IXGBE_LINK_SPEED_UNKNOWN: 4274 link.link_speed = ETH_SPEED_NUM_UNKNOWN; 4275 break; 4276 4277 case IXGBE_LINK_SPEED_10_FULL: 4278 link.link_speed = ETH_SPEED_NUM_10M; 4279 break; 4280 4281 case IXGBE_LINK_SPEED_100_FULL: 4282 link.link_speed = ETH_SPEED_NUM_100M; 4283 break; 4284 4285 case IXGBE_LINK_SPEED_1GB_FULL: 4286 link.link_speed = ETH_SPEED_NUM_1G; 4287 break; 4288 4289 case IXGBE_LINK_SPEED_2_5GB_FULL: 4290 link.link_speed = ETH_SPEED_NUM_2_5G; 4291 break; 4292 4293 case IXGBE_LINK_SPEED_5GB_FULL: 4294 link.link_speed = ETH_SPEED_NUM_5G; 4295 break; 4296 4297 case IXGBE_LINK_SPEED_10GB_FULL: 4298 link.link_speed = ETH_SPEED_NUM_10G; 4299 break; 4300 } 4301 4302 return rte_eth_linkstatus_set(dev, &link); 4303 } 4304 4305 static int 4306 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4307 { 4308 return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); 4309 } 4310 4311 static int 4312 ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4313 { 4314 return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); 4315 } 4316 4317 static int 4318 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) 4319 { 4320 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4321 uint32_t fctrl; 4322 4323 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4324 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4325 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4326 4327 return 0; 4328 } 4329 4330 static int 4331 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev) 4332 { 4333 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4334 uint32_t fctrl; 4335 4336 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4337 fctrl &= (~IXGBE_FCTRL_UPE); 4338 if (dev->data->all_multicast == 1) 4339 fctrl |= IXGBE_FCTRL_MPE; 4340 else 4341 fctrl &= (~IXGBE_FCTRL_MPE); 4342 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4343 4344 return 0; 4345 } 4346 4347 static int 4348 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev) 4349 { 4350 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4351 uint32_t fctrl; 4352 4353 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4354 fctrl |= IXGBE_FCTRL_MPE; 4355 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4356 4357 return 0; 4358 } 4359 4360 static int 4361 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) 4362 { 4363 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4364 uint32_t fctrl; 4365 4366 if (dev->data->promiscuous == 1) 4367 return 0; /* must remain in all_multicast mode */ 4368 4369 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4370 fctrl &= (~IXGBE_FCTRL_MPE); 4371 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4372 4373 return 0; 4374 } 4375 4376 /** 4377 * It clears the interrupt causes and enables the interrupt. 4378 * It will be called once only during nic initialized. 4379 * 4380 * @param dev 4381 * Pointer to struct rte_eth_dev. 4382 * @param on 4383 * Enable or Disable. 4384 * 4385 * @return 4386 * - On success, zero. 4387 * - On failure, a negative value. 4388 */ 4389 static int 4390 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 4391 { 4392 struct ixgbe_interrupt *intr = 4393 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4394 4395 ixgbe_dev_link_status_print(dev); 4396 if (on) 4397 intr->mask |= IXGBE_EICR_LSC; 4398 else 4399 intr->mask &= ~IXGBE_EICR_LSC; 4400 4401 return 0; 4402 } 4403 4404 /** 4405 * It clears the interrupt causes and enables the interrupt. 4406 * It will be called once only during nic initialized. 4407 * 4408 * @param dev 4409 * Pointer to struct rte_eth_dev. 4410 * 4411 * @return 4412 * - On success, zero. 4413 * - On failure, a negative value. 4414 */ 4415 static int 4416 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) 4417 { 4418 struct ixgbe_interrupt *intr = 4419 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4420 4421 intr->mask |= IXGBE_EICR_RTX_QUEUE; 4422 4423 return 0; 4424 } 4425 4426 /** 4427 * It clears the interrupt causes and enables the interrupt. 4428 * It will be called once only during nic initialized. 4429 * 4430 * @param dev 4431 * Pointer to struct rte_eth_dev. 4432 * 4433 * @return 4434 * - On success, zero. 4435 * - On failure, a negative value. 4436 */ 4437 static int 4438 ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev) 4439 { 4440 struct ixgbe_interrupt *intr = 4441 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4442 4443 intr->mask |= IXGBE_EICR_LINKSEC; 4444 4445 return 0; 4446 } 4447 4448 /* 4449 * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update. 4450 * 4451 * @param dev 4452 * Pointer to struct rte_eth_dev. 4453 * 4454 * @return 4455 * - On success, zero. 4456 * - On failure, a negative value. 4457 */ 4458 static int 4459 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) 4460 { 4461 uint32_t eicr; 4462 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4463 struct ixgbe_interrupt *intr = 4464 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4465 4466 /* clear all cause mask */ 4467 ixgbe_disable_intr(hw); 4468 4469 /* read-on-clear nic registers here */ 4470 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4471 PMD_DRV_LOG(DEBUG, "eicr %x", eicr); 4472 4473 intr->flags = 0; 4474 4475 /* set flag for async link update */ 4476 if (eicr & IXGBE_EICR_LSC) 4477 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4478 4479 if (eicr & IXGBE_EICR_MAILBOX) 4480 intr->flags |= IXGBE_FLAG_MAILBOX; 4481 4482 if (eicr & IXGBE_EICR_LINKSEC) 4483 intr->flags |= IXGBE_FLAG_MACSEC; 4484 4485 if (hw->mac.type == ixgbe_mac_X550EM_x && 4486 hw->phy.type == ixgbe_phy_x550em_ext_t && 4487 (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x)) 4488 intr->flags |= IXGBE_FLAG_PHY_INTERRUPT; 4489 4490 return 0; 4491 } 4492 4493 /** 4494 * It gets and then prints the link status. 4495 * 4496 * @param dev 4497 * Pointer to struct rte_eth_dev. 4498 * 4499 * @return 4500 * - On success, zero. 4501 * - On failure, a negative value. 4502 */ 4503 static void 4504 ixgbe_dev_link_status_print(struct rte_eth_dev *dev) 4505 { 4506 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4507 struct rte_eth_link link; 4508 4509 rte_eth_linkstatus_get(dev, &link); 4510 4511 if (link.link_status) { 4512 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 4513 (int)(dev->data->port_id), 4514 (unsigned)link.link_speed, 4515 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 4516 "full-duplex" : "half-duplex"); 4517 } else { 4518 PMD_INIT_LOG(INFO, " Port %d: Link Down", 4519 (int)(dev->data->port_id)); 4520 } 4521 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 4522 pci_dev->addr.domain, 4523 pci_dev->addr.bus, 4524 pci_dev->addr.devid, 4525 pci_dev->addr.function); 4526 } 4527 4528 /* 4529 * It executes link_update after knowing an interrupt occurred. 4530 * 4531 * @param dev 4532 * Pointer to struct rte_eth_dev. 4533 * 4534 * @return 4535 * - On success, zero. 4536 * - On failure, a negative value. 4537 */ 4538 static int 4539 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) 4540 { 4541 struct ixgbe_interrupt *intr = 4542 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4543 int64_t timeout; 4544 struct ixgbe_hw *hw = 4545 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4546 4547 PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags); 4548 4549 if (intr->flags & IXGBE_FLAG_MAILBOX) { 4550 ixgbe_pf_mbx_process(dev); 4551 intr->flags &= ~IXGBE_FLAG_MAILBOX; 4552 } 4553 4554 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4555 ixgbe_handle_lasi(hw); 4556 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4557 } 4558 4559 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4560 struct rte_eth_link link; 4561 4562 /* get the link status before link update, for predicting later */ 4563 rte_eth_linkstatus_get(dev, &link); 4564 4565 ixgbe_dev_link_update(dev, 0); 4566 4567 /* likely to up */ 4568 if (!link.link_status) 4569 /* handle it 1 sec later, wait it being stable */ 4570 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT; 4571 /* likely to down */ 4572 else 4573 /* handle it 4 sec later, wait it being stable */ 4574 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; 4575 4576 ixgbe_dev_link_status_print(dev); 4577 if (rte_eal_alarm_set(timeout * 1000, 4578 ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) 4579 PMD_DRV_LOG(ERR, "Error setting alarm"); 4580 else { 4581 /* remember original mask */ 4582 intr->mask_original = intr->mask; 4583 /* only disable lsc interrupt */ 4584 intr->mask &= ~IXGBE_EIMS_LSC; 4585 } 4586 } 4587 4588 PMD_DRV_LOG(DEBUG, "enable intr immediately"); 4589 ixgbe_enable_intr(dev); 4590 4591 return 0; 4592 } 4593 4594 /** 4595 * Interrupt handler which shall be registered for alarm callback for delayed 4596 * handling specific interrupt to wait for the stable nic state. As the 4597 * NIC interrupt state is not stable for ixgbe after link is just down, 4598 * it needs to wait 4 seconds to get the stable status. 4599 * 4600 * @param handle 4601 * Pointer to interrupt handle. 4602 * @param param 4603 * The address of parameter (struct rte_eth_dev *) regsitered before. 4604 * 4605 * @return 4606 * void 4607 */ 4608 static void 4609 ixgbe_dev_interrupt_delayed_handler(void *param) 4610 { 4611 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4612 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4613 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 4614 struct ixgbe_interrupt *intr = 4615 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4616 struct ixgbe_hw *hw = 4617 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4618 uint32_t eicr; 4619 4620 ixgbe_disable_intr(hw); 4621 4622 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4623 if (eicr & IXGBE_EICR_MAILBOX) 4624 ixgbe_pf_mbx_process(dev); 4625 4626 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4627 ixgbe_handle_lasi(hw); 4628 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4629 } 4630 4631 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4632 ixgbe_dev_link_update(dev, 0); 4633 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4634 ixgbe_dev_link_status_print(dev); 4635 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 4636 } 4637 4638 if (intr->flags & IXGBE_FLAG_MACSEC) { 4639 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, NULL); 4640 intr->flags &= ~IXGBE_FLAG_MACSEC; 4641 } 4642 4643 /* restore original mask */ 4644 intr->mask = intr->mask_original; 4645 intr->mask_original = 0; 4646 4647 PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr); 4648 ixgbe_enable_intr(dev); 4649 rte_intr_ack(intr_handle); 4650 } 4651 4652 /** 4653 * Interrupt handler triggered by NIC for handling 4654 * specific interrupt. 4655 * 4656 * @param handle 4657 * Pointer to interrupt handle. 4658 * @param param 4659 * The address of parameter (struct rte_eth_dev *) regsitered before. 4660 * 4661 * @return 4662 * void 4663 */ 4664 static void 4665 ixgbe_dev_interrupt_handler(void *param) 4666 { 4667 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4668 4669 ixgbe_dev_interrupt_get_status(dev); 4670 ixgbe_dev_interrupt_action(dev); 4671 } 4672 4673 static int 4674 ixgbe_dev_led_on(struct rte_eth_dev *dev) 4675 { 4676 struct ixgbe_hw *hw; 4677 4678 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4679 return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4680 } 4681 4682 static int 4683 ixgbe_dev_led_off(struct rte_eth_dev *dev) 4684 { 4685 struct ixgbe_hw *hw; 4686 4687 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4688 return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4689 } 4690 4691 static int 4692 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4693 { 4694 struct ixgbe_hw *hw; 4695 uint32_t mflcn_reg; 4696 uint32_t fccfg_reg; 4697 int rx_pause; 4698 int tx_pause; 4699 4700 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4701 4702 fc_conf->pause_time = hw->fc.pause_time; 4703 fc_conf->high_water = hw->fc.high_water[0]; 4704 fc_conf->low_water = hw->fc.low_water[0]; 4705 fc_conf->send_xon = hw->fc.send_xon; 4706 fc_conf->autoneg = !hw->fc.disable_fc_autoneg; 4707 4708 /* 4709 * Return rx_pause status according to actual setting of 4710 * MFLCN register. 4711 */ 4712 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4713 if (mflcn_reg & IXGBE_MFLCN_PMCF) 4714 fc_conf->mac_ctrl_frame_fwd = 1; 4715 else 4716 fc_conf->mac_ctrl_frame_fwd = 0; 4717 4718 if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE)) 4719 rx_pause = 1; 4720 else 4721 rx_pause = 0; 4722 4723 /* 4724 * Return tx_pause status according to actual setting of 4725 * FCCFG register. 4726 */ 4727 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4728 if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY)) 4729 tx_pause = 1; 4730 else 4731 tx_pause = 0; 4732 4733 if (rx_pause && tx_pause) 4734 fc_conf->mode = RTE_FC_FULL; 4735 else if (rx_pause) 4736 fc_conf->mode = RTE_FC_RX_PAUSE; 4737 else if (tx_pause) 4738 fc_conf->mode = RTE_FC_TX_PAUSE; 4739 else 4740 fc_conf->mode = RTE_FC_NONE; 4741 4742 return 0; 4743 } 4744 4745 static int 4746 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4747 { 4748 struct ixgbe_hw *hw; 4749 struct ixgbe_adapter *adapter = dev->data->dev_private; 4750 int err; 4751 uint32_t rx_buf_size; 4752 uint32_t max_high_water; 4753 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4754 ixgbe_fc_none, 4755 ixgbe_fc_rx_pause, 4756 ixgbe_fc_tx_pause, 4757 ixgbe_fc_full 4758 }; 4759 4760 PMD_INIT_FUNC_TRACE(); 4761 4762 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4763 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)); 4764 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4765 4766 /* 4767 * At least reserve one Ethernet frame for watermark 4768 * high_water/low_water in kilo bytes for ixgbe 4769 */ 4770 max_high_water = (rx_buf_size - 4771 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4772 if ((fc_conf->high_water > max_high_water) || 4773 (fc_conf->high_water < fc_conf->low_water)) { 4774 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4775 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4776 return -EINVAL; 4777 } 4778 4779 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode]; 4780 hw->fc.pause_time = fc_conf->pause_time; 4781 hw->fc.high_water[0] = fc_conf->high_water; 4782 hw->fc.low_water[0] = fc_conf->low_water; 4783 hw->fc.send_xon = fc_conf->send_xon; 4784 hw->fc.disable_fc_autoneg = !fc_conf->autoneg; 4785 adapter->mac_ctrl_frame_fwd = fc_conf->mac_ctrl_frame_fwd; 4786 4787 err = ixgbe_flow_ctrl_enable(dev, hw); 4788 if (err < 0) { 4789 PMD_INIT_LOG(ERR, "ixgbe_flow_ctrl_enable = 0x%x", err); 4790 return -EIO; 4791 } 4792 return err; 4793 } 4794 4795 /** 4796 * ixgbe_pfc_enable_generic - Enable flow control 4797 * @hw: pointer to hardware structure 4798 * @tc_num: traffic class number 4799 * Enable flow control according to the current settings. 4800 */ 4801 static int 4802 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num) 4803 { 4804 int ret_val = 0; 4805 uint32_t mflcn_reg, fccfg_reg; 4806 uint32_t reg; 4807 uint32_t fcrtl, fcrth; 4808 uint8_t i; 4809 uint8_t nb_rx_en; 4810 4811 /* Validate the water mark configuration */ 4812 if (!hw->fc.pause_time) { 4813 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4814 goto out; 4815 } 4816 4817 /* Low water mark of zero causes XOFF floods */ 4818 if (hw->fc.current_mode & ixgbe_fc_tx_pause) { 4819 /* High/Low water can not be 0 */ 4820 if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) { 4821 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4822 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4823 goto out; 4824 } 4825 4826 if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) { 4827 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4828 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4829 goto out; 4830 } 4831 } 4832 /* Negotiate the fc mode to use */ 4833 ixgbe_fc_autoneg(hw); 4834 4835 /* Disable any previous flow control settings */ 4836 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4837 mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE); 4838 4839 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4840 fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY); 4841 4842 switch (hw->fc.current_mode) { 4843 case ixgbe_fc_none: 4844 /* 4845 * If the count of enabled RX Priority Flow control >1, 4846 * and the TX pause can not be disabled 4847 */ 4848 nb_rx_en = 0; 4849 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4850 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4851 if (reg & IXGBE_FCRTH_FCEN) 4852 nb_rx_en++; 4853 } 4854 if (nb_rx_en > 1) 4855 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4856 break; 4857 case ixgbe_fc_rx_pause: 4858 /* 4859 * Rx Flow control is enabled and Tx Flow control is 4860 * disabled by software override. Since there really 4861 * isn't a way to advertise that we are capable of RX 4862 * Pause ONLY, we will advertise that we support both 4863 * symmetric and asymmetric Rx PAUSE. Later, we will 4864 * disable the adapter's ability to send PAUSE frames. 4865 */ 4866 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4867 /* 4868 * If the count of enabled RX Priority Flow control >1, 4869 * and the TX pause can not be disabled 4870 */ 4871 nb_rx_en = 0; 4872 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4873 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4874 if (reg & IXGBE_FCRTH_FCEN) 4875 nb_rx_en++; 4876 } 4877 if (nb_rx_en > 1) 4878 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4879 break; 4880 case ixgbe_fc_tx_pause: 4881 /* 4882 * Tx Flow control is enabled, and Rx Flow control is 4883 * disabled by software override. 4884 */ 4885 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4886 break; 4887 case ixgbe_fc_full: 4888 /* Flow control (both Rx and Tx) is enabled by SW override. */ 4889 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4890 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4891 break; 4892 default: 4893 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly"); 4894 ret_val = IXGBE_ERR_CONFIG; 4895 goto out; 4896 } 4897 4898 /* Set 802.3x based flow control settings. */ 4899 mflcn_reg |= IXGBE_MFLCN_DPF; 4900 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg); 4901 IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg); 4902 4903 /* Set up and enable Rx high/low water mark thresholds, enable XON. */ 4904 if ((hw->fc.current_mode & ixgbe_fc_tx_pause) && 4905 hw->fc.high_water[tc_num]) { 4906 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE; 4907 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl); 4908 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN; 4909 } else { 4910 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0); 4911 /* 4912 * In order to prevent Tx hangs when the internal Tx 4913 * switch is enabled we must set the high water mark 4914 * to the maximum FCRTH value. This allows the Tx 4915 * switch to function even under heavy Rx workloads. 4916 */ 4917 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32; 4918 } 4919 IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth); 4920 4921 /* Configure pause time (2 TCs per register) */ 4922 reg = hw->fc.pause_time * 0x00010001; 4923 for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++) 4924 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg); 4925 4926 /* Configure flow control refresh threshold value */ 4927 IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2); 4928 4929 out: 4930 return ret_val; 4931 } 4932 4933 static int 4934 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num) 4935 { 4936 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4937 int32_t ret_val = IXGBE_NOT_IMPLEMENTED; 4938 4939 if (hw->mac.type != ixgbe_mac_82598EB) { 4940 ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num); 4941 } 4942 return ret_val; 4943 } 4944 4945 static int 4946 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf) 4947 { 4948 int err; 4949 uint32_t rx_buf_size; 4950 uint32_t max_high_water; 4951 uint8_t tc_num; 4952 uint8_t map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 }; 4953 struct ixgbe_hw *hw = 4954 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4955 struct ixgbe_dcb_config *dcb_config = 4956 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4957 4958 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4959 ixgbe_fc_none, 4960 ixgbe_fc_rx_pause, 4961 ixgbe_fc_tx_pause, 4962 ixgbe_fc_full 4963 }; 4964 4965 PMD_INIT_FUNC_TRACE(); 4966 4967 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4968 tc_num = map[pfc_conf->priority]; 4969 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)); 4970 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4971 /* 4972 * At least reserve one Ethernet frame for watermark 4973 * high_water/low_water in kilo bytes for ixgbe 4974 */ 4975 max_high_water = (rx_buf_size - 4976 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4977 if ((pfc_conf->fc.high_water > max_high_water) || 4978 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) { 4979 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4980 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4981 return -EINVAL; 4982 } 4983 4984 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode]; 4985 hw->fc.pause_time = pfc_conf->fc.pause_time; 4986 hw->fc.send_xon = pfc_conf->fc.send_xon; 4987 hw->fc.low_water[tc_num] = pfc_conf->fc.low_water; 4988 hw->fc.high_water[tc_num] = pfc_conf->fc.high_water; 4989 4990 err = ixgbe_dcb_pfc_enable(dev, tc_num); 4991 4992 /* Not negotiated is not an error case */ 4993 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) 4994 return 0; 4995 4996 PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err); 4997 return -EIO; 4998 } 4999 5000 static int 5001 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 5002 struct rte_eth_rss_reta_entry64 *reta_conf, 5003 uint16_t reta_size) 5004 { 5005 uint16_t i, sp_reta_size; 5006 uint8_t j, mask; 5007 uint32_t reta, r; 5008 uint16_t idx, shift; 5009 struct ixgbe_adapter *adapter = dev->data->dev_private; 5010 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5011 uint32_t reta_reg; 5012 5013 PMD_INIT_FUNC_TRACE(); 5014 5015 if (!ixgbe_rss_update_sp(hw->mac.type)) { 5016 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this " 5017 "NIC."); 5018 return -ENOTSUP; 5019 } 5020 5021 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 5022 if (reta_size != sp_reta_size) { 5023 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 5024 "(%d) doesn't match the number hardware can supported " 5025 "(%d)", reta_size, sp_reta_size); 5026 return -EINVAL; 5027 } 5028 5029 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 5030 idx = i / RTE_RETA_GROUP_SIZE; 5031 shift = i % RTE_RETA_GROUP_SIZE; 5032 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 5033 IXGBE_4_BIT_MASK); 5034 if (!mask) 5035 continue; 5036 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 5037 if (mask == IXGBE_4_BIT_MASK) 5038 r = 0; 5039 else 5040 r = IXGBE_READ_REG(hw, reta_reg); 5041 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) { 5042 if (mask & (0x1 << j)) 5043 reta |= reta_conf[idx].reta[shift + j] << 5044 (CHAR_BIT * j); 5045 else 5046 reta |= r & (IXGBE_8_BIT_MASK << 5047 (CHAR_BIT * j)); 5048 } 5049 IXGBE_WRITE_REG(hw, reta_reg, reta); 5050 } 5051 adapter->rss_reta_updated = 1; 5052 5053 return 0; 5054 } 5055 5056 static int 5057 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 5058 struct rte_eth_rss_reta_entry64 *reta_conf, 5059 uint16_t reta_size) 5060 { 5061 uint16_t i, sp_reta_size; 5062 uint8_t j, mask; 5063 uint32_t reta; 5064 uint16_t idx, shift; 5065 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5066 uint32_t reta_reg; 5067 5068 PMD_INIT_FUNC_TRACE(); 5069 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 5070 if (reta_size != sp_reta_size) { 5071 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 5072 "(%d) doesn't match the number hardware can supported " 5073 "(%d)", reta_size, sp_reta_size); 5074 return -EINVAL; 5075 } 5076 5077 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 5078 idx = i / RTE_RETA_GROUP_SIZE; 5079 shift = i % RTE_RETA_GROUP_SIZE; 5080 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 5081 IXGBE_4_BIT_MASK); 5082 if (!mask) 5083 continue; 5084 5085 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 5086 reta = IXGBE_READ_REG(hw, reta_reg); 5087 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) { 5088 if (mask & (0x1 << j)) 5089 reta_conf[idx].reta[shift + j] = 5090 ((reta >> (CHAR_BIT * j)) & 5091 IXGBE_8_BIT_MASK); 5092 } 5093 } 5094 5095 return 0; 5096 } 5097 5098 static int 5099 ixgbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 5100 uint32_t index, uint32_t pool) 5101 { 5102 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5103 uint32_t enable_addr = 1; 5104 5105 return ixgbe_set_rar(hw, index, mac_addr->addr_bytes, 5106 pool, enable_addr); 5107 } 5108 5109 static void 5110 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) 5111 { 5112 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5113 5114 ixgbe_clear_rar(hw, index); 5115 } 5116 5117 static int 5118 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 5119 { 5120 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5121 5122 ixgbe_remove_rar(dev, 0); 5123 ixgbe_add_rar(dev, addr, 0, pci_dev->max_vfs); 5124 5125 return 0; 5126 } 5127 5128 static bool 5129 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) 5130 { 5131 if (strcmp(dev->device->driver->name, drv->driver.name)) 5132 return false; 5133 5134 return true; 5135 } 5136 5137 bool 5138 is_ixgbe_supported(struct rte_eth_dev *dev) 5139 { 5140 return is_device_supported(dev, &rte_ixgbe_pmd); 5141 } 5142 5143 static int 5144 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 5145 { 5146 uint32_t hlreg0; 5147 uint32_t maxfrs; 5148 struct ixgbe_hw *hw; 5149 struct rte_eth_dev_info dev_info; 5150 uint32_t frame_size = mtu + IXGBE_ETH_OVERHEAD; 5151 struct rte_eth_dev_data *dev_data = dev->data; 5152 int ret; 5153 5154 ret = ixgbe_dev_info_get(dev, &dev_info); 5155 if (ret != 0) 5156 return ret; 5157 5158 /* check that mtu is within the allowed range */ 5159 if (mtu < RTE_ETHER_MIN_MTU || frame_size > dev_info.max_rx_pktlen) 5160 return -EINVAL; 5161 5162 /* If device is started, refuse mtu that requires the support of 5163 * scattered packets when this feature has not been enabled before. 5164 */ 5165 if (dev_data->dev_started && !dev_data->scattered_rx && 5166 (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > 5167 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 5168 PMD_INIT_LOG(ERR, "Stop port first."); 5169 return -EINVAL; 5170 } 5171 5172 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5173 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5174 5175 /* switch to jumbo mode if needed */ 5176 if (frame_size > RTE_ETHER_MAX_LEN) { 5177 dev->data->dev_conf.rxmode.offloads |= 5178 DEV_RX_OFFLOAD_JUMBO_FRAME; 5179 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 5180 } else { 5181 dev->data->dev_conf.rxmode.offloads &= 5182 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 5183 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 5184 } 5185 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5186 5187 /* update max frame size */ 5188 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 5189 5190 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 5191 maxfrs &= 0x0000FFFF; 5192 maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16); 5193 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 5194 5195 return 0; 5196 } 5197 5198 /* 5199 * Virtual Function operations 5200 */ 5201 static void 5202 ixgbevf_intr_disable(struct rte_eth_dev *dev) 5203 { 5204 struct ixgbe_interrupt *intr = 5205 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5206 struct ixgbe_hw *hw = 5207 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5208 5209 PMD_INIT_FUNC_TRACE(); 5210 5211 /* Clear interrupt mask to stop from interrupts being generated */ 5212 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK); 5213 5214 IXGBE_WRITE_FLUSH(hw); 5215 5216 /* Clear mask value. */ 5217 intr->mask = 0; 5218 } 5219 5220 static void 5221 ixgbevf_intr_enable(struct rte_eth_dev *dev) 5222 { 5223 struct ixgbe_interrupt *intr = 5224 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5225 struct ixgbe_hw *hw = 5226 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5227 5228 PMD_INIT_FUNC_TRACE(); 5229 5230 /* VF enable interrupt autoclean */ 5231 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK); 5232 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK); 5233 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK); 5234 5235 IXGBE_WRITE_FLUSH(hw); 5236 5237 /* Save IXGBE_VTEIMS value to mask. */ 5238 intr->mask = IXGBE_VF_IRQ_ENABLE_MASK; 5239 } 5240 5241 static int 5242 ixgbevf_dev_configure(struct rte_eth_dev *dev) 5243 { 5244 struct rte_eth_conf *conf = &dev->data->dev_conf; 5245 struct ixgbe_adapter *adapter = dev->data->dev_private; 5246 5247 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 5248 dev->data->port_id); 5249 5250 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 5251 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 5252 5253 /* 5254 * VF has no ability to enable/disable HW CRC 5255 * Keep the persistent behavior the same as Host PF 5256 */ 5257 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC 5258 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 5259 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 5260 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 5261 } 5262 #else 5263 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 5264 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 5265 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 5266 } 5267 #endif 5268 5269 /* 5270 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 5271 * allocation or vector Rx preconditions we will reset it. 5272 */ 5273 adapter->rx_bulk_alloc_allowed = true; 5274 adapter->rx_vec_allowed = true; 5275 5276 return 0; 5277 } 5278 5279 static int 5280 ixgbevf_dev_start(struct rte_eth_dev *dev) 5281 { 5282 struct ixgbe_hw *hw = 5283 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5284 uint32_t intr_vector = 0; 5285 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5286 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5287 5288 int err, mask = 0; 5289 5290 PMD_INIT_FUNC_TRACE(); 5291 5292 /* Stop the link setup handler before resetting the HW. */ 5293 ixgbe_dev_wait_setup_link_complete(dev, 0); 5294 5295 err = hw->mac.ops.reset_hw(hw); 5296 5297 /** 5298 * In this case, reuses the MAC address assigned by VF 5299 * initialization. 5300 */ 5301 if (err != IXGBE_SUCCESS && err != IXGBE_ERR_INVALID_MAC_ADDR) { 5302 PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); 5303 return err; 5304 } 5305 5306 hw->mac.get_link_status = true; 5307 5308 /* negotiate mailbox API version to use with the PF. */ 5309 ixgbevf_negotiate_api(hw); 5310 5311 ixgbevf_dev_tx_init(dev); 5312 5313 /* This can fail when allocating mbufs for descriptor rings */ 5314 err = ixgbevf_dev_rx_init(dev); 5315 if (err) { 5316 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err); 5317 ixgbe_dev_clear_queues(dev); 5318 return err; 5319 } 5320 5321 /* Set vfta */ 5322 ixgbevf_set_vfta_all(dev, 1); 5323 5324 /* Set HW strip */ 5325 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 5326 ETH_VLAN_EXTEND_MASK; 5327 err = ixgbevf_vlan_offload_config(dev, mask); 5328 if (err) { 5329 PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); 5330 ixgbe_dev_clear_queues(dev); 5331 return err; 5332 } 5333 5334 ixgbevf_dev_rxtx_start(dev); 5335 5336 /* check and configure queue intr-vector mapping */ 5337 if (rte_intr_cap_multiple(intr_handle) && 5338 dev->data->dev_conf.intr_conf.rxq) { 5339 /* According to datasheet, only vector 0/1/2 can be used, 5340 * now only one vector is used for Rx queue 5341 */ 5342 intr_vector = 1; 5343 if (rte_intr_efd_enable(intr_handle, intr_vector)) 5344 return -1; 5345 } 5346 5347 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 5348 intr_handle->intr_vec = 5349 rte_zmalloc("intr_vec", 5350 dev->data->nb_rx_queues * sizeof(int), 0); 5351 if (intr_handle->intr_vec == NULL) { 5352 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 5353 " intr_vec", dev->data->nb_rx_queues); 5354 return -ENOMEM; 5355 } 5356 } 5357 ixgbevf_configure_msix(dev); 5358 5359 /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt 5360 * is mapped to VFIO vector 0 in eth_ixgbevf_dev_init( ). 5361 * If previous VFIO interrupt mapping setting in eth_ixgbevf_dev_init( ) 5362 * is not cleared, it will fail when following rte_intr_enable( ) tries 5363 * to map Rx queue interrupt to other VFIO vectors. 5364 * So clear uio/vfio intr/evevnfd first to avoid failure. 5365 */ 5366 rte_intr_disable(intr_handle); 5367 5368 rte_intr_enable(intr_handle); 5369 5370 /* Re-enable interrupt for VF */ 5371 ixgbevf_intr_enable(dev); 5372 5373 /* 5374 * Update link status right before return, because it may 5375 * start link configuration process in a separate thread. 5376 */ 5377 ixgbevf_dev_link_update(dev, 0); 5378 5379 hw->adapter_stopped = false; 5380 5381 return 0; 5382 } 5383 5384 static int 5385 ixgbevf_dev_stop(struct rte_eth_dev *dev) 5386 { 5387 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5388 struct ixgbe_adapter *adapter = dev->data->dev_private; 5389 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5390 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5391 5392 if (hw->adapter_stopped) 5393 return 0; 5394 5395 PMD_INIT_FUNC_TRACE(); 5396 5397 ixgbe_dev_wait_setup_link_complete(dev, 0); 5398 5399 ixgbevf_intr_disable(dev); 5400 5401 dev->data->dev_started = 0; 5402 hw->adapter_stopped = 1; 5403 ixgbe_stop_adapter(hw); 5404 5405 /* 5406 * Clear what we set, but we still keep shadow_vfta to 5407 * restore after device starts 5408 */ 5409 ixgbevf_set_vfta_all(dev, 0); 5410 5411 /* Clear stored conf */ 5412 dev->data->scattered_rx = 0; 5413 5414 ixgbe_dev_clear_queues(dev); 5415 5416 /* Clean datapath event and queue/vec mapping */ 5417 rte_intr_efd_disable(intr_handle); 5418 if (intr_handle->intr_vec != NULL) { 5419 rte_free(intr_handle->intr_vec); 5420 intr_handle->intr_vec = NULL; 5421 } 5422 5423 adapter->rss_reta_updated = 0; 5424 5425 return 0; 5426 } 5427 5428 static int 5429 ixgbevf_dev_close(struct rte_eth_dev *dev) 5430 { 5431 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5432 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5433 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5434 int ret; 5435 5436 PMD_INIT_FUNC_TRACE(); 5437 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 5438 return 0; 5439 5440 ixgbe_reset_hw(hw); 5441 5442 ret = ixgbevf_dev_stop(dev); 5443 5444 ixgbe_dev_free_queues(dev); 5445 5446 /** 5447 * Remove the VF MAC address ro ensure 5448 * that the VF traffic goes to the PF 5449 * after stop, close and detach of the VF 5450 **/ 5451 ixgbevf_remove_mac_addr(dev, 0); 5452 5453 rte_intr_disable(intr_handle); 5454 rte_intr_callback_unregister(intr_handle, 5455 ixgbevf_dev_interrupt_handler, dev); 5456 5457 return ret; 5458 } 5459 5460 /* 5461 * Reset VF device 5462 */ 5463 static int 5464 ixgbevf_dev_reset(struct rte_eth_dev *dev) 5465 { 5466 int ret; 5467 5468 ret = eth_ixgbevf_dev_uninit(dev); 5469 if (ret) 5470 return ret; 5471 5472 ret = eth_ixgbevf_dev_init(dev); 5473 5474 return ret; 5475 } 5476 5477 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) 5478 { 5479 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5480 struct ixgbe_vfta *shadow_vfta = 5481 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5482 int i = 0, j = 0, vfta = 0, mask = 1; 5483 5484 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 5485 vfta = shadow_vfta->vfta[i]; 5486 if (vfta) { 5487 mask = 1; 5488 for (j = 0; j < 32; j++) { 5489 if (vfta & mask) 5490 ixgbe_set_vfta(hw, (i<<5)+j, 0, 5491 on, false); 5492 mask <<= 1; 5493 } 5494 } 5495 } 5496 5497 } 5498 5499 static int 5500 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 5501 { 5502 struct ixgbe_hw *hw = 5503 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5504 struct ixgbe_vfta *shadow_vfta = 5505 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5506 uint32_t vid_idx = 0; 5507 uint32_t vid_bit = 0; 5508 int ret = 0; 5509 5510 PMD_INIT_FUNC_TRACE(); 5511 5512 /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */ 5513 ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on, false); 5514 if (ret) { 5515 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 5516 return ret; 5517 } 5518 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 5519 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 5520 5521 /* Save what we set and retore it after device reset */ 5522 if (on) 5523 shadow_vfta->vfta[vid_idx] |= vid_bit; 5524 else 5525 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 5526 5527 return 0; 5528 } 5529 5530 static void 5531 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 5532 { 5533 struct ixgbe_hw *hw = 5534 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5535 uint32_t ctrl; 5536 5537 PMD_INIT_FUNC_TRACE(); 5538 5539 if (queue >= hw->mac.max_rx_queues) 5540 return; 5541 5542 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 5543 if (on) 5544 ctrl |= IXGBE_RXDCTL_VME; 5545 else 5546 ctrl &= ~IXGBE_RXDCTL_VME; 5547 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 5548 5549 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on); 5550 } 5551 5552 static int 5553 ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask) 5554 { 5555 struct ixgbe_rx_queue *rxq; 5556 uint16_t i; 5557 int on = 0; 5558 5559 /* VF function only support hw strip feature, others are not support */ 5560 if (mask & ETH_VLAN_STRIP_MASK) { 5561 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5562 rxq = dev->data->rx_queues[i]; 5563 on = !!(rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 5564 ixgbevf_vlan_strip_queue_set(dev, i, on); 5565 } 5566 } 5567 5568 return 0; 5569 } 5570 5571 static int 5572 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) 5573 { 5574 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 5575 5576 ixgbevf_vlan_offload_config(dev, mask); 5577 5578 return 0; 5579 } 5580 5581 int 5582 ixgbe_vt_check(struct ixgbe_hw *hw) 5583 { 5584 uint32_t reg_val; 5585 5586 /* if Virtualization Technology is enabled */ 5587 reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 5588 if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) { 5589 PMD_INIT_LOG(ERR, "VT must be enabled for this setting"); 5590 return -1; 5591 } 5592 5593 return 0; 5594 } 5595 5596 static uint32_t 5597 ixgbe_uta_vector(struct ixgbe_hw *hw, struct rte_ether_addr *uc_addr) 5598 { 5599 uint32_t vector = 0; 5600 5601 switch (hw->mac.mc_filter_type) { 5602 case 0: /* use bits [47:36] of the address */ 5603 vector = ((uc_addr->addr_bytes[4] >> 4) | 5604 (((uint16_t)uc_addr->addr_bytes[5]) << 4)); 5605 break; 5606 case 1: /* use bits [46:35] of the address */ 5607 vector = ((uc_addr->addr_bytes[4] >> 3) | 5608 (((uint16_t)uc_addr->addr_bytes[5]) << 5)); 5609 break; 5610 case 2: /* use bits [45:34] of the address */ 5611 vector = ((uc_addr->addr_bytes[4] >> 2) | 5612 (((uint16_t)uc_addr->addr_bytes[5]) << 6)); 5613 break; 5614 case 3: /* use bits [43:32] of the address */ 5615 vector = ((uc_addr->addr_bytes[4]) | 5616 (((uint16_t)uc_addr->addr_bytes[5]) << 8)); 5617 break; 5618 default: /* Invalid mc_filter_type */ 5619 break; 5620 } 5621 5622 /* vector can only be 12-bits or boundary will be exceeded */ 5623 vector &= 0xFFF; 5624 return vector; 5625 } 5626 5627 static int 5628 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, 5629 struct rte_ether_addr *mac_addr, uint8_t on) 5630 { 5631 uint32_t vector; 5632 uint32_t uta_idx; 5633 uint32_t reg_val; 5634 uint32_t uta_shift; 5635 uint32_t rc; 5636 const uint32_t ixgbe_uta_idx_mask = 0x7F; 5637 const uint32_t ixgbe_uta_bit_shift = 5; 5638 const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1; 5639 const uint32_t bit1 = 0x1; 5640 5641 struct ixgbe_hw *hw = 5642 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5643 struct ixgbe_uta_info *uta_info = 5644 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5645 5646 /* The UTA table only exists on 82599 hardware and newer */ 5647 if (hw->mac.type < ixgbe_mac_82599EB) 5648 return -ENOTSUP; 5649 5650 vector = ixgbe_uta_vector(hw, mac_addr); 5651 uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask; 5652 uta_shift = vector & ixgbe_uta_bit_mask; 5653 5654 rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0); 5655 if (rc == on) 5656 return 0; 5657 5658 reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx)); 5659 if (on) { 5660 uta_info->uta_in_use++; 5661 reg_val |= (bit1 << uta_shift); 5662 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift); 5663 } else { 5664 uta_info->uta_in_use--; 5665 reg_val &= ~(bit1 << uta_shift); 5666 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift); 5667 } 5668 5669 IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val); 5670 5671 if (uta_info->uta_in_use > 0) 5672 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, 5673 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type); 5674 else 5675 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type); 5676 5677 return 0; 5678 } 5679 5680 static int 5681 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on) 5682 { 5683 int i; 5684 struct ixgbe_hw *hw = 5685 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5686 struct ixgbe_uta_info *uta_info = 5687 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5688 5689 /* The UTA table only exists on 82599 hardware and newer */ 5690 if (hw->mac.type < ixgbe_mac_82599EB) 5691 return -ENOTSUP; 5692 5693 if (on) { 5694 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5695 uta_info->uta_shadow[i] = ~0; 5696 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0); 5697 } 5698 } else { 5699 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5700 uta_info->uta_shadow[i] = 0; 5701 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0); 5702 } 5703 } 5704 return 0; 5705 5706 } 5707 5708 uint32_t 5709 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val) 5710 { 5711 uint32_t new_val = orig_val; 5712 5713 if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG) 5714 new_val |= IXGBE_VMOLR_AUPE; 5715 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC) 5716 new_val |= IXGBE_VMOLR_ROMPE; 5717 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC) 5718 new_val |= IXGBE_VMOLR_ROPE; 5719 if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) 5720 new_val |= IXGBE_VMOLR_BAM; 5721 if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) 5722 new_val |= IXGBE_VMOLR_MPE; 5723 5724 return new_val; 5725 } 5726 5727 #define IXGBE_MRCTL_VPME 0x01 /* Virtual Pool Mirroring. */ 5728 #define IXGBE_MRCTL_UPME 0x02 /* Uplink Port Mirroring. */ 5729 #define IXGBE_MRCTL_DPME 0x04 /* Downlink Port Mirroring. */ 5730 #define IXGBE_MRCTL_VLME 0x08 /* VLAN Mirroring. */ 5731 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \ 5732 ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \ 5733 ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN)) 5734 5735 static int 5736 ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 5737 struct rte_eth_mirror_conf *mirror_conf, 5738 uint8_t rule_id, uint8_t on) 5739 { 5740 uint32_t mr_ctl, vlvf; 5741 uint32_t mp_lsb = 0; 5742 uint32_t mv_msb = 0; 5743 uint32_t mv_lsb = 0; 5744 uint32_t mp_msb = 0; 5745 uint8_t i = 0; 5746 int reg_index = 0; 5747 uint64_t vlan_mask = 0; 5748 5749 const uint8_t pool_mask_offset = 32; 5750 const uint8_t vlan_mask_offset = 32; 5751 const uint8_t dst_pool_offset = 8; 5752 const uint8_t rule_mr_offset = 4; 5753 const uint8_t mirror_rule_mask = 0x0F; 5754 5755 struct ixgbe_mirror_info *mr_info = 5756 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5757 struct ixgbe_hw *hw = 5758 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5759 uint8_t mirror_type = 0; 5760 5761 if (ixgbe_vt_check(hw) < 0) 5762 return -ENOTSUP; 5763 5764 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5765 return -EINVAL; 5766 5767 if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) { 5768 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.", 5769 mirror_conf->rule_type); 5770 return -EINVAL; 5771 } 5772 5773 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5774 mirror_type |= IXGBE_MRCTL_VLME; 5775 /* Check if vlan id is valid and find conresponding VLAN ID 5776 * index in VLVF 5777 */ 5778 for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) { 5779 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) { 5780 /* search vlan id related pool vlan filter 5781 * index 5782 */ 5783 reg_index = ixgbe_find_vlvf_slot( 5784 hw, 5785 mirror_conf->vlan.vlan_id[i], 5786 false); 5787 if (reg_index < 0) 5788 return -EINVAL; 5789 vlvf = IXGBE_READ_REG(hw, 5790 IXGBE_VLVF(reg_index)); 5791 if ((vlvf & IXGBE_VLVF_VIEN) && 5792 ((vlvf & IXGBE_VLVF_VLANID_MASK) == 5793 mirror_conf->vlan.vlan_id[i])) 5794 vlan_mask |= (1ULL << reg_index); 5795 else 5796 return -EINVAL; 5797 } 5798 } 5799 5800 if (on) { 5801 mv_lsb = vlan_mask & 0xFFFFFFFF; 5802 mv_msb = vlan_mask >> vlan_mask_offset; 5803 5804 mr_info->mr_conf[rule_id].vlan.vlan_mask = 5805 mirror_conf->vlan.vlan_mask; 5806 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) { 5807 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) 5808 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 5809 mirror_conf->vlan.vlan_id[i]; 5810 } 5811 } else { 5812 mv_lsb = 0; 5813 mv_msb = 0; 5814 mr_info->mr_conf[rule_id].vlan.vlan_mask = 0; 5815 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) 5816 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0; 5817 } 5818 } 5819 5820 /** 5821 * if enable pool mirror, write related pool mask register,if disable 5822 * pool mirror, clear PFMRVM register 5823 */ 5824 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5825 mirror_type |= IXGBE_MRCTL_VPME; 5826 if (on) { 5827 mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF; 5828 mp_msb = mirror_conf->pool_mask >> pool_mask_offset; 5829 mr_info->mr_conf[rule_id].pool_mask = 5830 mirror_conf->pool_mask; 5831 5832 } else { 5833 mp_lsb = 0; 5834 mp_msb = 0; 5835 mr_info->mr_conf[rule_id].pool_mask = 0; 5836 } 5837 } 5838 if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT) 5839 mirror_type |= IXGBE_MRCTL_UPME; 5840 if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT) 5841 mirror_type |= IXGBE_MRCTL_DPME; 5842 5843 /* read mirror control register and recalculate it */ 5844 mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id)); 5845 5846 if (on) { 5847 mr_ctl |= mirror_type; 5848 mr_ctl &= mirror_rule_mask; 5849 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset; 5850 } else { 5851 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask); 5852 } 5853 5854 mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type; 5855 mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool; 5856 5857 /* write mirrror control register */ 5858 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5859 5860 /* write pool mirrror control register */ 5861 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5862 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb); 5863 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), 5864 mp_msb); 5865 } 5866 /* write VLAN mirrror control register */ 5867 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5868 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb); 5869 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), 5870 mv_msb); 5871 } 5872 5873 return 0; 5874 } 5875 5876 static int 5877 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) 5878 { 5879 int mr_ctl = 0; 5880 uint32_t lsb_val = 0; 5881 uint32_t msb_val = 0; 5882 const uint8_t rule_mr_offset = 4; 5883 5884 struct ixgbe_hw *hw = 5885 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5886 struct ixgbe_mirror_info *mr_info = 5887 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5888 5889 if (ixgbe_vt_check(hw) < 0) 5890 return -ENOTSUP; 5891 5892 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5893 return -EINVAL; 5894 5895 memset(&mr_info->mr_conf[rule_id], 0, 5896 sizeof(struct rte_eth_mirror_conf)); 5897 5898 /* clear PFVMCTL register */ 5899 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5900 5901 /* clear pool mask register */ 5902 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val); 5903 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val); 5904 5905 /* clear vlan mask register */ 5906 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val); 5907 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val); 5908 5909 return 0; 5910 } 5911 5912 static int 5913 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5914 { 5915 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5916 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5917 struct ixgbe_interrupt *intr = 5918 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5919 struct ixgbe_hw *hw = 5920 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5921 uint32_t vec = IXGBE_MISC_VEC_ID; 5922 5923 if (rte_intr_allow_others(intr_handle)) 5924 vec = IXGBE_RX_VEC_START; 5925 intr->mask |= (1 << vec); 5926 RTE_SET_USED(queue_id); 5927 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5928 5929 rte_intr_ack(intr_handle); 5930 5931 return 0; 5932 } 5933 5934 static int 5935 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5936 { 5937 struct ixgbe_interrupt *intr = 5938 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5939 struct ixgbe_hw *hw = 5940 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5941 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5942 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5943 uint32_t vec = IXGBE_MISC_VEC_ID; 5944 5945 if (rte_intr_allow_others(intr_handle)) 5946 vec = IXGBE_RX_VEC_START; 5947 intr->mask &= ~(1 << vec); 5948 RTE_SET_USED(queue_id); 5949 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5950 5951 return 0; 5952 } 5953 5954 static int 5955 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5956 { 5957 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5958 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5959 uint32_t mask; 5960 struct ixgbe_hw *hw = 5961 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5962 struct ixgbe_interrupt *intr = 5963 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5964 5965 if (queue_id < 16) { 5966 ixgbe_disable_intr(hw); 5967 intr->mask |= (1 << queue_id); 5968 ixgbe_enable_intr(dev); 5969 } else if (queue_id < 32) { 5970 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5971 mask &= (1 << queue_id); 5972 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5973 } else if (queue_id < 64) { 5974 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5975 mask &= (1 << (queue_id - 32)); 5976 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5977 } 5978 rte_intr_ack(intr_handle); 5979 5980 return 0; 5981 } 5982 5983 static int 5984 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5985 { 5986 uint32_t mask; 5987 struct ixgbe_hw *hw = 5988 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5989 struct ixgbe_interrupt *intr = 5990 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5991 5992 if (queue_id < 16) { 5993 ixgbe_disable_intr(hw); 5994 intr->mask &= ~(1 << queue_id); 5995 ixgbe_enable_intr(dev); 5996 } else if (queue_id < 32) { 5997 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5998 mask &= ~(1 << queue_id); 5999 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 6000 } else if (queue_id < 64) { 6001 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 6002 mask &= ~(1 << (queue_id - 32)); 6003 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 6004 } 6005 6006 return 0; 6007 } 6008 6009 static void 6010 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 6011 uint8_t queue, uint8_t msix_vector) 6012 { 6013 uint32_t tmp, idx; 6014 6015 if (direction == -1) { 6016 /* other causes */ 6017 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6018 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 6019 tmp &= ~0xFF; 6020 tmp |= msix_vector; 6021 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp); 6022 } else { 6023 /* rx or tx cause */ 6024 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6025 idx = ((16 * (queue & 1)) + (8 * direction)); 6026 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 6027 tmp &= ~(0xFF << idx); 6028 tmp |= (msix_vector << idx); 6029 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp); 6030 } 6031 } 6032 6033 /** 6034 * set the IVAR registers, mapping interrupt causes to vectors 6035 * @param hw 6036 * pointer to ixgbe_hw struct 6037 * @direction 6038 * 0 for Rx, 1 for Tx, -1 for other causes 6039 * @queue 6040 * queue to map the corresponding interrupt to 6041 * @msix_vector 6042 * the vector to map to the corresponding queue 6043 */ 6044 static void 6045 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 6046 uint8_t queue, uint8_t msix_vector) 6047 { 6048 uint32_t tmp, idx; 6049 6050 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 6051 if (hw->mac.type == ixgbe_mac_82598EB) { 6052 if (direction == -1) 6053 direction = 0; 6054 idx = (((direction * 64) + queue) >> 2) & 0x1F; 6055 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx)); 6056 tmp &= ~(0xFF << (8 * (queue & 0x3))); 6057 tmp |= (msix_vector << (8 * (queue & 0x3))); 6058 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); 6059 } else if ((hw->mac.type == ixgbe_mac_82599EB) || 6060 (hw->mac.type == ixgbe_mac_X540) || 6061 (hw->mac.type == ixgbe_mac_X550) || 6062 (hw->mac.type == ixgbe_mac_X550EM_x)) { 6063 if (direction == -1) { 6064 /* other causes */ 6065 idx = ((queue & 1) * 8); 6066 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 6067 tmp &= ~(0xFF << idx); 6068 tmp |= (msix_vector << idx); 6069 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp); 6070 } else { 6071 /* rx or tx causes */ 6072 idx = ((16 * (queue & 1)) + (8 * direction)); 6073 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 6074 tmp &= ~(0xFF << idx); 6075 tmp |= (msix_vector << idx); 6076 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp); 6077 } 6078 } 6079 } 6080 6081 static void 6082 ixgbevf_configure_msix(struct rte_eth_dev *dev) 6083 { 6084 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 6085 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 6086 struct ixgbe_hw *hw = 6087 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6088 uint32_t q_idx; 6089 uint32_t vector_idx = IXGBE_MISC_VEC_ID; 6090 uint32_t base = IXGBE_MISC_VEC_ID; 6091 6092 /* Configure VF other cause ivar */ 6093 ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); 6094 6095 /* won't configure msix register if no mapping is done 6096 * between intr vector and event fd. 6097 */ 6098 if (!rte_intr_dp_is_en(intr_handle)) 6099 return; 6100 6101 if (rte_intr_allow_others(intr_handle)) { 6102 base = IXGBE_RX_VEC_START; 6103 vector_idx = IXGBE_RX_VEC_START; 6104 } 6105 6106 /* Configure all RX queues of VF */ 6107 for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) { 6108 /* Force all queue use vector 0, 6109 * as IXGBE_VF_MAXMSIVECOTR = 1 6110 */ 6111 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx); 6112 intr_handle->intr_vec[q_idx] = vector_idx; 6113 if (vector_idx < base + intr_handle->nb_efd - 1) 6114 vector_idx++; 6115 } 6116 6117 /* As RX queue setting above show, all queues use the vector 0. 6118 * Set only the ITR value of IXGBE_MISC_VEC_ID. 6119 */ 6120 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(IXGBE_MISC_VEC_ID), 6121 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6122 | IXGBE_EITR_CNT_WDIS); 6123 } 6124 6125 /** 6126 * Sets up the hardware to properly generate MSI-X interrupts 6127 * @hw 6128 * board private structure 6129 */ 6130 static void 6131 ixgbe_configure_msix(struct rte_eth_dev *dev) 6132 { 6133 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 6134 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 6135 struct ixgbe_hw *hw = 6136 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6137 uint32_t queue_id, base = IXGBE_MISC_VEC_ID; 6138 uint32_t vec = IXGBE_MISC_VEC_ID; 6139 uint32_t mask; 6140 uint32_t gpie; 6141 6142 /* won't configure msix register if no mapping is done 6143 * between intr vector and event fd 6144 * but if misx has been enabled already, need to configure 6145 * auto clean, auto mask and throttling. 6146 */ 6147 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6148 if (!rte_intr_dp_is_en(intr_handle) && 6149 !(gpie & (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT))) 6150 return; 6151 6152 if (rte_intr_allow_others(intr_handle)) 6153 vec = base = IXGBE_RX_VEC_START; 6154 6155 /* setup GPIE for MSI-x mode */ 6156 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6157 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 6158 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME; 6159 /* auto clearing and auto setting corresponding bits in EIMS 6160 * when MSI-X interrupt is triggered 6161 */ 6162 if (hw->mac.type == ixgbe_mac_82598EB) { 6163 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 6164 } else { 6165 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 6166 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 6167 } 6168 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 6169 6170 /* Populate the IVAR table and set the ITR values to the 6171 * corresponding register. 6172 */ 6173 if (rte_intr_dp_is_en(intr_handle)) { 6174 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; 6175 queue_id++) { 6176 /* by default, 1:1 mapping */ 6177 ixgbe_set_ivar_map(hw, 0, queue_id, vec); 6178 intr_handle->intr_vec[queue_id] = vec; 6179 if (vec < base + intr_handle->nb_efd - 1) 6180 vec++; 6181 } 6182 6183 switch (hw->mac.type) { 6184 case ixgbe_mac_82598EB: 6185 ixgbe_set_ivar_map(hw, -1, 6186 IXGBE_IVAR_OTHER_CAUSES_INDEX, 6187 IXGBE_MISC_VEC_ID); 6188 break; 6189 case ixgbe_mac_82599EB: 6190 case ixgbe_mac_X540: 6191 case ixgbe_mac_X550: 6192 case ixgbe_mac_X550EM_x: 6193 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); 6194 break; 6195 default: 6196 break; 6197 } 6198 } 6199 IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID), 6200 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6201 | IXGBE_EITR_CNT_WDIS); 6202 6203 /* set up to autoclear timer, and the vectors */ 6204 mask = IXGBE_EIMS_ENABLE_MASK; 6205 mask &= ~(IXGBE_EIMS_OTHER | 6206 IXGBE_EIMS_MAILBOX | 6207 IXGBE_EIMS_LSC); 6208 6209 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 6210 } 6211 6212 int 6213 ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, 6214 uint16_t queue_idx, uint16_t tx_rate) 6215 { 6216 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6217 struct rte_eth_rxmode *rxmode; 6218 uint32_t rf_dec, rf_int; 6219 uint32_t bcnrc_val; 6220 uint16_t link_speed = dev->data->dev_link.link_speed; 6221 6222 if (queue_idx >= hw->mac.max_tx_queues) 6223 return -EINVAL; 6224 6225 if (tx_rate != 0) { 6226 /* Calculate the rate factor values to set */ 6227 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate; 6228 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate; 6229 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate; 6230 6231 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA; 6232 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) & 6233 IXGBE_RTTBCNRC_RF_INT_MASK_M); 6234 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK); 6235 } else { 6236 bcnrc_val = 0; 6237 } 6238 6239 rxmode = &dev->data->dev_conf.rxmode; 6240 /* 6241 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 6242 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise 6243 * set as 0x4. 6244 */ 6245 if ((rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) && 6246 (rxmode->max_rx_pkt_len >= IXGBE_MAX_JUMBO_FRAME_SIZE)) 6247 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6248 IXGBE_MMW_SIZE_JUMBO_FRAME); 6249 else 6250 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6251 IXGBE_MMW_SIZE_DEFAULT); 6252 6253 /* Set RTTBCNRC of queue X */ 6254 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx); 6255 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val); 6256 IXGBE_WRITE_FLUSH(hw); 6257 6258 return 0; 6259 } 6260 6261 static int 6262 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 6263 __rte_unused uint32_t index, 6264 __rte_unused uint32_t pool) 6265 { 6266 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6267 int diag; 6268 6269 /* 6270 * On a 82599 VF, adding again the same MAC addr is not an idempotent 6271 * operation. Trap this case to avoid exhausting the [very limited] 6272 * set of PF resources used to store VF MAC addresses. 6273 */ 6274 if (memcmp(hw->mac.perm_addr, mac_addr, 6275 sizeof(struct rte_ether_addr)) == 0) 6276 return -1; 6277 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6278 if (diag != 0) 6279 PMD_DRV_LOG(ERR, "Unable to add MAC address " 6280 "%02x:%02x:%02x:%02x:%02x:%02x - diag=%d", 6281 mac_addr->addr_bytes[0], 6282 mac_addr->addr_bytes[1], 6283 mac_addr->addr_bytes[2], 6284 mac_addr->addr_bytes[3], 6285 mac_addr->addr_bytes[4], 6286 mac_addr->addr_bytes[5], 6287 diag); 6288 return diag; 6289 } 6290 6291 static void 6292 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index) 6293 { 6294 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6295 struct rte_ether_addr *perm_addr = 6296 (struct rte_ether_addr *)hw->mac.perm_addr; 6297 struct rte_ether_addr *mac_addr; 6298 uint32_t i; 6299 int diag; 6300 6301 /* 6302 * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does 6303 * not support the deletion of a given MAC address. 6304 * Instead, it imposes to delete all MAC addresses, then to add again 6305 * all MAC addresses with the exception of the one to be deleted. 6306 */ 6307 (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL); 6308 6309 /* 6310 * Add again all MAC addresses, with the exception of the deleted one 6311 * and of the permanent MAC address. 6312 */ 6313 for (i = 0, mac_addr = dev->data->mac_addrs; 6314 i < hw->mac.num_rar_entries; i++, mac_addr++) { 6315 /* Skip the deleted MAC address */ 6316 if (i == index) 6317 continue; 6318 /* Skip NULL MAC addresses */ 6319 if (rte_is_zero_ether_addr(mac_addr)) 6320 continue; 6321 /* Skip the permanent MAC address */ 6322 if (memcmp(perm_addr, mac_addr, 6323 sizeof(struct rte_ether_addr)) == 0) 6324 continue; 6325 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6326 if (diag != 0) 6327 PMD_DRV_LOG(ERR, 6328 "Adding again MAC address " 6329 "%02x:%02x:%02x:%02x:%02x:%02x failed " 6330 "diag=%d", 6331 mac_addr->addr_bytes[0], 6332 mac_addr->addr_bytes[1], 6333 mac_addr->addr_bytes[2], 6334 mac_addr->addr_bytes[3], 6335 mac_addr->addr_bytes[4], 6336 mac_addr->addr_bytes[5], 6337 diag); 6338 } 6339 } 6340 6341 static int 6342 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 6343 struct rte_ether_addr *addr) 6344 { 6345 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6346 6347 hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0); 6348 6349 return 0; 6350 } 6351 6352 int 6353 ixgbe_syn_filter_set(struct rte_eth_dev *dev, 6354 struct rte_eth_syn_filter *filter, 6355 bool add) 6356 { 6357 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6358 struct ixgbe_filter_info *filter_info = 6359 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6360 uint32_t syn_info; 6361 uint32_t synqf; 6362 6363 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6364 return -EINVAL; 6365 6366 syn_info = filter_info->syn_info; 6367 6368 if (add) { 6369 if (syn_info & IXGBE_SYN_FILTER_ENABLE) 6370 return -EINVAL; 6371 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) & 6372 IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE); 6373 6374 if (filter->hig_pri) 6375 synqf |= IXGBE_SYN_FILTER_SYNQFP; 6376 else 6377 synqf &= ~IXGBE_SYN_FILTER_SYNQFP; 6378 } else { 6379 synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6380 if (!(syn_info & IXGBE_SYN_FILTER_ENABLE)) 6381 return -ENOENT; 6382 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE); 6383 } 6384 6385 filter_info->syn_info = synqf; 6386 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 6387 IXGBE_WRITE_FLUSH(hw); 6388 return 0; 6389 } 6390 6391 6392 static inline enum ixgbe_5tuple_protocol 6393 convert_protocol_type(uint8_t protocol_value) 6394 { 6395 if (protocol_value == IPPROTO_TCP) 6396 return IXGBE_FILTER_PROTOCOL_TCP; 6397 else if (protocol_value == IPPROTO_UDP) 6398 return IXGBE_FILTER_PROTOCOL_UDP; 6399 else if (protocol_value == IPPROTO_SCTP) 6400 return IXGBE_FILTER_PROTOCOL_SCTP; 6401 else 6402 return IXGBE_FILTER_PROTOCOL_NONE; 6403 } 6404 6405 /* inject a 5-tuple filter to HW */ 6406 static inline void 6407 ixgbe_inject_5tuple_filter(struct rte_eth_dev *dev, 6408 struct ixgbe_5tuple_filter *filter) 6409 { 6410 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6411 int i; 6412 uint32_t ftqf, sdpqf; 6413 uint32_t l34timir = 0; 6414 uint8_t mask = 0xff; 6415 6416 i = filter->index; 6417 6418 sdpqf = (uint32_t)(filter->filter_info.dst_port << 6419 IXGBE_SDPQF_DSTPORT_SHIFT); 6420 sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT); 6421 6422 ftqf = (uint32_t)(filter->filter_info.proto & 6423 IXGBE_FTQF_PROTOCOL_MASK); 6424 ftqf |= (uint32_t)((filter->filter_info.priority & 6425 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT); 6426 if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */ 6427 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK; 6428 if (filter->filter_info.dst_ip_mask == 0) 6429 mask &= IXGBE_FTQF_DEST_ADDR_MASK; 6430 if (filter->filter_info.src_port_mask == 0) 6431 mask &= IXGBE_FTQF_SOURCE_PORT_MASK; 6432 if (filter->filter_info.dst_port_mask == 0) 6433 mask &= IXGBE_FTQF_DEST_PORT_MASK; 6434 if (filter->filter_info.proto_mask == 0) 6435 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK; 6436 ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT; 6437 ftqf |= IXGBE_FTQF_POOL_MASK_EN; 6438 ftqf |= IXGBE_FTQF_QUEUE_ENABLE; 6439 6440 IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip); 6441 IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip); 6442 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf); 6443 IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf); 6444 6445 l34timir |= IXGBE_L34T_IMIR_RESERVE; 6446 l34timir |= (uint32_t)(filter->queue << 6447 IXGBE_L34T_IMIR_QUEUE_SHIFT); 6448 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir); 6449 } 6450 6451 /* 6452 * add a 5tuple filter 6453 * 6454 * @param 6455 * dev: Pointer to struct rte_eth_dev. 6456 * index: the index the filter allocates. 6457 * filter: ponter to the filter that will be added. 6458 * rx_queue: the queue id the filter assigned to. 6459 * 6460 * @return 6461 * - On success, zero. 6462 * - On failure, a negative value. 6463 */ 6464 static int 6465 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 6466 struct ixgbe_5tuple_filter *filter) 6467 { 6468 struct ixgbe_filter_info *filter_info = 6469 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6470 int i, idx, shift; 6471 6472 /* 6473 * look for an unused 5tuple filter index, 6474 * and insert the filter to list. 6475 */ 6476 for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { 6477 idx = i / (sizeof(uint32_t) * NBBY); 6478 shift = i % (sizeof(uint32_t) * NBBY); 6479 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { 6480 filter_info->fivetuple_mask[idx] |= 1 << shift; 6481 filter->index = i; 6482 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 6483 filter, 6484 entries); 6485 break; 6486 } 6487 } 6488 if (i >= IXGBE_MAX_FTQF_FILTERS) { 6489 PMD_DRV_LOG(ERR, "5tuple filters are full."); 6490 return -ENOSYS; 6491 } 6492 6493 ixgbe_inject_5tuple_filter(dev, filter); 6494 6495 return 0; 6496 } 6497 6498 /* 6499 * remove a 5tuple filter 6500 * 6501 * @param 6502 * dev: Pointer to struct rte_eth_dev. 6503 * filter: the pointer of the filter will be removed. 6504 */ 6505 static void 6506 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 6507 struct ixgbe_5tuple_filter *filter) 6508 { 6509 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6510 struct ixgbe_filter_info *filter_info = 6511 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6512 uint16_t index = filter->index; 6513 6514 filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &= 6515 ~(1 << (index % (sizeof(uint32_t) * NBBY))); 6516 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 6517 rte_free(filter); 6518 6519 IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0); 6520 IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0); 6521 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0); 6522 IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0); 6523 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0); 6524 } 6525 6526 static int 6527 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 6528 { 6529 struct ixgbe_hw *hw; 6530 uint32_t max_frame = mtu + IXGBE_ETH_OVERHEAD; 6531 struct rte_eth_dev_data *dev_data = dev->data; 6532 6533 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6534 6535 if (mtu < RTE_ETHER_MIN_MTU || 6536 max_frame > RTE_ETHER_MAX_JUMBO_FRAME_LEN) 6537 return -EINVAL; 6538 6539 /* If device is started, refuse mtu that requires the support of 6540 * scattered packets when this feature has not been enabled before. 6541 */ 6542 if (dev_data->dev_started && !dev_data->scattered_rx && 6543 (max_frame + 2 * IXGBE_VLAN_TAG_SIZE > 6544 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 6545 PMD_INIT_LOG(ERR, "Stop port first."); 6546 return -EINVAL; 6547 } 6548 6549 /* 6550 * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU 6551 * request of the version 2.0 of the mailbox API. 6552 * For now, use the IXGBE_VF_SET_LPE request of the version 1.0 6553 * of the mailbox API. 6554 * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers 6555 * prior to 3.11.33 which contains the following change: 6556 * "ixgbe: Enable jumbo frames support w/ SR-IOV" 6557 */ 6558 ixgbevf_rlpml_set_vf(hw, max_frame); 6559 6560 /* update max frame size */ 6561 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame; 6562 return 0; 6563 } 6564 6565 static inline struct ixgbe_5tuple_filter * 6566 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list, 6567 struct ixgbe_5tuple_filter_info *key) 6568 { 6569 struct ixgbe_5tuple_filter *it; 6570 6571 TAILQ_FOREACH(it, filter_list, entries) { 6572 if (memcmp(key, &it->filter_info, 6573 sizeof(struct ixgbe_5tuple_filter_info)) == 0) { 6574 return it; 6575 } 6576 } 6577 return NULL; 6578 } 6579 6580 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/ 6581 static inline int 6582 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter, 6583 struct ixgbe_5tuple_filter_info *filter_info) 6584 { 6585 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM || 6586 filter->priority > IXGBE_5TUPLE_MAX_PRI || 6587 filter->priority < IXGBE_5TUPLE_MIN_PRI) 6588 return -EINVAL; 6589 6590 switch (filter->dst_ip_mask) { 6591 case UINT32_MAX: 6592 filter_info->dst_ip_mask = 0; 6593 filter_info->dst_ip = filter->dst_ip; 6594 break; 6595 case 0: 6596 filter_info->dst_ip_mask = 1; 6597 break; 6598 default: 6599 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 6600 return -EINVAL; 6601 } 6602 6603 switch (filter->src_ip_mask) { 6604 case UINT32_MAX: 6605 filter_info->src_ip_mask = 0; 6606 filter_info->src_ip = filter->src_ip; 6607 break; 6608 case 0: 6609 filter_info->src_ip_mask = 1; 6610 break; 6611 default: 6612 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 6613 return -EINVAL; 6614 } 6615 6616 switch (filter->dst_port_mask) { 6617 case UINT16_MAX: 6618 filter_info->dst_port_mask = 0; 6619 filter_info->dst_port = filter->dst_port; 6620 break; 6621 case 0: 6622 filter_info->dst_port_mask = 1; 6623 break; 6624 default: 6625 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 6626 return -EINVAL; 6627 } 6628 6629 switch (filter->src_port_mask) { 6630 case UINT16_MAX: 6631 filter_info->src_port_mask = 0; 6632 filter_info->src_port = filter->src_port; 6633 break; 6634 case 0: 6635 filter_info->src_port_mask = 1; 6636 break; 6637 default: 6638 PMD_DRV_LOG(ERR, "invalid src_port mask."); 6639 return -EINVAL; 6640 } 6641 6642 switch (filter->proto_mask) { 6643 case UINT8_MAX: 6644 filter_info->proto_mask = 0; 6645 filter_info->proto = 6646 convert_protocol_type(filter->proto); 6647 break; 6648 case 0: 6649 filter_info->proto_mask = 1; 6650 break; 6651 default: 6652 PMD_DRV_LOG(ERR, "invalid protocol mask."); 6653 return -EINVAL; 6654 } 6655 6656 filter_info->priority = (uint8_t)filter->priority; 6657 return 0; 6658 } 6659 6660 /* 6661 * add or delete a ntuple filter 6662 * 6663 * @param 6664 * dev: Pointer to struct rte_eth_dev. 6665 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6666 * add: if true, add filter, if false, remove filter 6667 * 6668 * @return 6669 * - On success, zero. 6670 * - On failure, a negative value. 6671 */ 6672 int 6673 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, 6674 struct rte_eth_ntuple_filter *ntuple_filter, 6675 bool add) 6676 { 6677 struct ixgbe_filter_info *filter_info = 6678 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6679 struct ixgbe_5tuple_filter_info filter_5tuple; 6680 struct ixgbe_5tuple_filter *filter; 6681 int ret; 6682 6683 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6684 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6685 return -EINVAL; 6686 } 6687 6688 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6689 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6690 if (ret < 0) 6691 return ret; 6692 6693 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6694 &filter_5tuple); 6695 if (filter != NULL && add) { 6696 PMD_DRV_LOG(ERR, "filter exists."); 6697 return -EEXIST; 6698 } 6699 if (filter == NULL && !add) { 6700 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6701 return -ENOENT; 6702 } 6703 6704 if (add) { 6705 filter = rte_zmalloc("ixgbe_5tuple_filter", 6706 sizeof(struct ixgbe_5tuple_filter), 0); 6707 if (filter == NULL) 6708 return -ENOMEM; 6709 rte_memcpy(&filter->filter_info, 6710 &filter_5tuple, 6711 sizeof(struct ixgbe_5tuple_filter_info)); 6712 filter->queue = ntuple_filter->queue; 6713 ret = ixgbe_add_5tuple_filter(dev, filter); 6714 if (ret < 0) { 6715 rte_free(filter); 6716 return ret; 6717 } 6718 } else 6719 ixgbe_remove_5tuple_filter(dev, filter); 6720 6721 return 0; 6722 } 6723 6724 int 6725 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, 6726 struct rte_eth_ethertype_filter *filter, 6727 bool add) 6728 { 6729 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6730 struct ixgbe_filter_info *filter_info = 6731 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6732 uint32_t etqf = 0; 6733 uint32_t etqs = 0; 6734 int ret; 6735 struct ixgbe_ethertype_filter ethertype_filter; 6736 6737 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6738 return -EINVAL; 6739 6740 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 6741 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 6742 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 6743 " ethertype filter.", filter->ether_type); 6744 return -EINVAL; 6745 } 6746 6747 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 6748 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 6749 return -EINVAL; 6750 } 6751 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 6752 PMD_DRV_LOG(ERR, "drop option is unsupported."); 6753 return -EINVAL; 6754 } 6755 6756 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6757 if (ret >= 0 && add) { 6758 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 6759 filter->ether_type); 6760 return -EEXIST; 6761 } 6762 if (ret < 0 && !add) { 6763 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6764 filter->ether_type); 6765 return -ENOENT; 6766 } 6767 6768 if (add) { 6769 etqf = IXGBE_ETQF_FILTER_EN; 6770 etqf |= (uint32_t)filter->ether_type; 6771 etqs |= (uint32_t)((filter->queue << 6772 IXGBE_ETQS_RX_QUEUE_SHIFT) & 6773 IXGBE_ETQS_RX_QUEUE); 6774 etqs |= IXGBE_ETQS_QUEUE_EN; 6775 6776 ethertype_filter.ethertype = filter->ether_type; 6777 ethertype_filter.etqf = etqf; 6778 ethertype_filter.etqs = etqs; 6779 ethertype_filter.conf = FALSE; 6780 ret = ixgbe_ethertype_filter_insert(filter_info, 6781 ðertype_filter); 6782 if (ret < 0) { 6783 PMD_DRV_LOG(ERR, "ethertype filters are full."); 6784 return -ENOSPC; 6785 } 6786 } else { 6787 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret); 6788 if (ret < 0) 6789 return -ENOSYS; 6790 } 6791 IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf); 6792 IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs); 6793 IXGBE_WRITE_FLUSH(hw); 6794 6795 return 0; 6796 } 6797 6798 static int 6799 ixgbe_dev_filter_ctrl(__rte_unused struct rte_eth_dev *dev, 6800 enum rte_filter_type filter_type, 6801 enum rte_filter_op filter_op, 6802 void *arg) 6803 { 6804 int ret = 0; 6805 6806 switch (filter_type) { 6807 case RTE_ETH_FILTER_GENERIC: 6808 if (filter_op != RTE_ETH_FILTER_GET) 6809 return -EINVAL; 6810 *(const void **)arg = &ixgbe_flow_ops; 6811 break; 6812 default: 6813 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 6814 filter_type); 6815 ret = -EINVAL; 6816 break; 6817 } 6818 6819 return ret; 6820 } 6821 6822 static u8 * 6823 ixgbe_dev_addr_list_itr(__rte_unused struct ixgbe_hw *hw, 6824 u8 **mc_addr_ptr, u32 *vmdq) 6825 { 6826 u8 *mc_addr; 6827 6828 *vmdq = 0; 6829 mc_addr = *mc_addr_ptr; 6830 *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr)); 6831 return mc_addr; 6832 } 6833 6834 static int 6835 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 6836 struct rte_ether_addr *mc_addr_set, 6837 uint32_t nb_mc_addr) 6838 { 6839 struct ixgbe_hw *hw; 6840 u8 *mc_addr_list; 6841 6842 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6843 mc_addr_list = (u8 *)mc_addr_set; 6844 return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr, 6845 ixgbe_dev_addr_list_itr, TRUE); 6846 } 6847 6848 static uint64_t 6849 ixgbe_read_systime_cyclecounter(struct rte_eth_dev *dev) 6850 { 6851 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6852 uint64_t systime_cycles; 6853 6854 switch (hw->mac.type) { 6855 case ixgbe_mac_X550: 6856 case ixgbe_mac_X550EM_x: 6857 case ixgbe_mac_X550EM_a: 6858 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 6859 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6860 systime_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6861 * NSEC_PER_SEC; 6862 break; 6863 default: 6864 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6865 systime_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6866 << 32; 6867 } 6868 6869 return systime_cycles; 6870 } 6871 6872 static uint64_t 6873 ixgbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6874 { 6875 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6876 uint64_t rx_tstamp_cycles; 6877 6878 switch (hw->mac.type) { 6879 case ixgbe_mac_X550: 6880 case ixgbe_mac_X550EM_x: 6881 case ixgbe_mac_X550EM_a: 6882 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6883 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6884 rx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6885 * NSEC_PER_SEC; 6886 break; 6887 default: 6888 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6889 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6890 rx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6891 << 32; 6892 } 6893 6894 return rx_tstamp_cycles; 6895 } 6896 6897 static uint64_t 6898 ixgbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6899 { 6900 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6901 uint64_t tx_tstamp_cycles; 6902 6903 switch (hw->mac.type) { 6904 case ixgbe_mac_X550: 6905 case ixgbe_mac_X550EM_x: 6906 case ixgbe_mac_X550EM_a: 6907 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6908 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6909 tx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6910 * NSEC_PER_SEC; 6911 break; 6912 default: 6913 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6914 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6915 tx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6916 << 32; 6917 } 6918 6919 return tx_tstamp_cycles; 6920 } 6921 6922 static void 6923 ixgbe_start_timecounters(struct rte_eth_dev *dev) 6924 { 6925 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6926 struct ixgbe_adapter *adapter = dev->data->dev_private; 6927 struct rte_eth_link link; 6928 uint32_t incval = 0; 6929 uint32_t shift = 0; 6930 6931 /* Get current link speed. */ 6932 ixgbe_dev_link_update(dev, 1); 6933 rte_eth_linkstatus_get(dev, &link); 6934 6935 switch (link.link_speed) { 6936 case ETH_SPEED_NUM_100M: 6937 incval = IXGBE_INCVAL_100; 6938 shift = IXGBE_INCVAL_SHIFT_100; 6939 break; 6940 case ETH_SPEED_NUM_1G: 6941 incval = IXGBE_INCVAL_1GB; 6942 shift = IXGBE_INCVAL_SHIFT_1GB; 6943 break; 6944 case ETH_SPEED_NUM_10G: 6945 default: 6946 incval = IXGBE_INCVAL_10GB; 6947 shift = IXGBE_INCVAL_SHIFT_10GB; 6948 break; 6949 } 6950 6951 switch (hw->mac.type) { 6952 case ixgbe_mac_X550: 6953 case ixgbe_mac_X550EM_x: 6954 case ixgbe_mac_X550EM_a: 6955 /* Independent of link speed. */ 6956 incval = 1; 6957 /* Cycles read will be interpreted as ns. */ 6958 shift = 0; 6959 /* Fall-through */ 6960 case ixgbe_mac_X540: 6961 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval); 6962 break; 6963 case ixgbe_mac_82599EB: 6964 incval >>= IXGBE_INCVAL_SHIFT_82599; 6965 shift -= IXGBE_INCVAL_SHIFT_82599; 6966 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 6967 (1 << IXGBE_INCPER_SHIFT_82599) | incval); 6968 break; 6969 default: 6970 /* Not supported. */ 6971 return; 6972 } 6973 6974 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 6975 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6976 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6977 6978 adapter->systime_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6979 adapter->systime_tc.cc_shift = shift; 6980 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 6981 6982 adapter->rx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6983 adapter->rx_tstamp_tc.cc_shift = shift; 6984 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6985 6986 adapter->tx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6987 adapter->tx_tstamp_tc.cc_shift = shift; 6988 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6989 } 6990 6991 static int 6992 ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 6993 { 6994 struct ixgbe_adapter *adapter = dev->data->dev_private; 6995 6996 adapter->systime_tc.nsec += delta; 6997 adapter->rx_tstamp_tc.nsec += delta; 6998 adapter->tx_tstamp_tc.nsec += delta; 6999 7000 return 0; 7001 } 7002 7003 static int 7004 ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 7005 { 7006 uint64_t ns; 7007 struct ixgbe_adapter *adapter = dev->data->dev_private; 7008 7009 ns = rte_timespec_to_ns(ts); 7010 /* Set the timecounters to a new value. */ 7011 adapter->systime_tc.nsec = ns; 7012 adapter->rx_tstamp_tc.nsec = ns; 7013 adapter->tx_tstamp_tc.nsec = ns; 7014 7015 return 0; 7016 } 7017 7018 static int 7019 ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 7020 { 7021 uint64_t ns, systime_cycles; 7022 struct ixgbe_adapter *adapter = dev->data->dev_private; 7023 7024 systime_cycles = ixgbe_read_systime_cyclecounter(dev); 7025 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 7026 *ts = rte_ns_to_timespec(ns); 7027 7028 return 0; 7029 } 7030 7031 static int 7032 ixgbe_timesync_enable(struct rte_eth_dev *dev) 7033 { 7034 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7035 uint32_t tsync_ctl; 7036 uint32_t tsauxc; 7037 7038 /* Stop the timesync system time. */ 7039 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0x0); 7040 /* Reset the timesync system time value. */ 7041 IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x0); 7042 IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x0); 7043 7044 /* Enable system time for platforms where it isn't on by default. */ 7045 tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); 7046 tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME; 7047 IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc); 7048 7049 ixgbe_start_timecounters(dev); 7050 7051 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7052 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 7053 (RTE_ETHER_TYPE_1588 | 7054 IXGBE_ETQF_FILTER_EN | 7055 IXGBE_ETQF_1588)); 7056 7057 /* Enable timestamping of received PTP packets. */ 7058 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7059 tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED; 7060 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7061 7062 /* Enable timestamping of transmitted PTP packets. */ 7063 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7064 tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED; 7065 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7066 7067 IXGBE_WRITE_FLUSH(hw); 7068 7069 return 0; 7070 } 7071 7072 static int 7073 ixgbe_timesync_disable(struct rte_eth_dev *dev) 7074 { 7075 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7076 uint32_t tsync_ctl; 7077 7078 /* Disable timestamping of transmitted PTP packets. */ 7079 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7080 tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED; 7081 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7082 7083 /* Disable timestamping of received PTP packets. */ 7084 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7085 tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED; 7086 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7087 7088 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7089 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0); 7090 7091 /* Stop incrementating the System Time registers. */ 7092 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0); 7093 7094 return 0; 7095 } 7096 7097 static int 7098 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 7099 struct timespec *timestamp, 7100 uint32_t flags __rte_unused) 7101 { 7102 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7103 struct ixgbe_adapter *adapter = dev->data->dev_private; 7104 uint32_t tsync_rxctl; 7105 uint64_t rx_tstamp_cycles; 7106 uint64_t ns; 7107 7108 tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7109 if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0) 7110 return -EINVAL; 7111 7112 rx_tstamp_cycles = ixgbe_read_rx_tstamp_cyclecounter(dev); 7113 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 7114 *timestamp = rte_ns_to_timespec(ns); 7115 7116 return 0; 7117 } 7118 7119 static int 7120 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 7121 struct timespec *timestamp) 7122 { 7123 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7124 struct ixgbe_adapter *adapter = dev->data->dev_private; 7125 uint32_t tsync_txctl; 7126 uint64_t tx_tstamp_cycles; 7127 uint64_t ns; 7128 7129 tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7130 if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0) 7131 return -EINVAL; 7132 7133 tx_tstamp_cycles = ixgbe_read_tx_tstamp_cyclecounter(dev); 7134 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 7135 *timestamp = rte_ns_to_timespec(ns); 7136 7137 return 0; 7138 } 7139 7140 static int 7141 ixgbe_get_reg_length(struct rte_eth_dev *dev) 7142 { 7143 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7144 int count = 0; 7145 int g_ind = 0; 7146 const struct reg_info *reg_group; 7147 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7148 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7149 7150 while ((reg_group = reg_set[g_ind++])) 7151 count += ixgbe_regs_group_count(reg_group); 7152 7153 return count; 7154 } 7155 7156 static int 7157 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 7158 { 7159 int count = 0; 7160 int g_ind = 0; 7161 const struct reg_info *reg_group; 7162 7163 while ((reg_group = ixgbevf_regs[g_ind++])) 7164 count += ixgbe_regs_group_count(reg_group); 7165 7166 return count; 7167 } 7168 7169 static int 7170 ixgbe_get_regs(struct rte_eth_dev *dev, 7171 struct rte_dev_reg_info *regs) 7172 { 7173 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7174 uint32_t *data = regs->data; 7175 int g_ind = 0; 7176 int count = 0; 7177 const struct reg_info *reg_group; 7178 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7179 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7180 7181 if (data == NULL) { 7182 regs->length = ixgbe_get_reg_length(dev); 7183 regs->width = sizeof(uint32_t); 7184 return 0; 7185 } 7186 7187 /* Support only full register dump */ 7188 if ((regs->length == 0) || 7189 (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) { 7190 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7191 hw->device_id; 7192 while ((reg_group = reg_set[g_ind++])) 7193 count += ixgbe_read_regs_group(dev, &data[count], 7194 reg_group); 7195 return 0; 7196 } 7197 7198 return -ENOTSUP; 7199 } 7200 7201 static int 7202 ixgbevf_get_regs(struct rte_eth_dev *dev, 7203 struct rte_dev_reg_info *regs) 7204 { 7205 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7206 uint32_t *data = regs->data; 7207 int g_ind = 0; 7208 int count = 0; 7209 const struct reg_info *reg_group; 7210 7211 if (data == NULL) { 7212 regs->length = ixgbevf_get_reg_length(dev); 7213 regs->width = sizeof(uint32_t); 7214 return 0; 7215 } 7216 7217 /* Support only full register dump */ 7218 if ((regs->length == 0) || 7219 (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) { 7220 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7221 hw->device_id; 7222 while ((reg_group = ixgbevf_regs[g_ind++])) 7223 count += ixgbe_read_regs_group(dev, &data[count], 7224 reg_group); 7225 return 0; 7226 } 7227 7228 return -ENOTSUP; 7229 } 7230 7231 static int 7232 ixgbe_get_eeprom_length(struct rte_eth_dev *dev) 7233 { 7234 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7235 7236 /* Return unit is byte count */ 7237 return hw->eeprom.word_size * 2; 7238 } 7239 7240 static int 7241 ixgbe_get_eeprom(struct rte_eth_dev *dev, 7242 struct rte_dev_eeprom_info *in_eeprom) 7243 { 7244 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7245 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7246 uint16_t *data = in_eeprom->data; 7247 int first, length; 7248 7249 first = in_eeprom->offset >> 1; 7250 length = in_eeprom->length >> 1; 7251 if ((first > hw->eeprom.word_size) || 7252 ((first + length) > hw->eeprom.word_size)) 7253 return -EINVAL; 7254 7255 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7256 7257 return eeprom->ops.read_buffer(hw, first, length, data); 7258 } 7259 7260 static int 7261 ixgbe_set_eeprom(struct rte_eth_dev *dev, 7262 struct rte_dev_eeprom_info *in_eeprom) 7263 { 7264 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7265 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7266 uint16_t *data = in_eeprom->data; 7267 int first, length; 7268 7269 first = in_eeprom->offset >> 1; 7270 length = in_eeprom->length >> 1; 7271 if ((first > hw->eeprom.word_size) || 7272 ((first + length) > hw->eeprom.word_size)) 7273 return -EINVAL; 7274 7275 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7276 7277 return eeprom->ops.write_buffer(hw, first, length, data); 7278 } 7279 7280 static int 7281 ixgbe_get_module_info(struct rte_eth_dev *dev, 7282 struct rte_eth_dev_module_info *modinfo) 7283 { 7284 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7285 uint32_t status; 7286 uint8_t sff8472_rev, addr_mode; 7287 bool page_swap = false; 7288 7289 /* Check whether we support SFF-8472 or not */ 7290 status = hw->phy.ops.read_i2c_eeprom(hw, 7291 IXGBE_SFF_SFF_8472_COMP, 7292 &sff8472_rev); 7293 if (status != 0) 7294 return -EIO; 7295 7296 /* addressing mode is not supported */ 7297 status = hw->phy.ops.read_i2c_eeprom(hw, 7298 IXGBE_SFF_SFF_8472_SWAP, 7299 &addr_mode); 7300 if (status != 0) 7301 return -EIO; 7302 7303 if (addr_mode & IXGBE_SFF_ADDRESSING_MODE) { 7304 PMD_DRV_LOG(ERR, 7305 "Address change required to access page 0xA2, " 7306 "but not supported. Please report the module " 7307 "type to the driver maintainers."); 7308 page_swap = true; 7309 } 7310 7311 if (sff8472_rev == IXGBE_SFF_SFF_8472_UNSUP || page_swap) { 7312 /* We have a SFP, but it does not support SFF-8472 */ 7313 modinfo->type = RTE_ETH_MODULE_SFF_8079; 7314 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 7315 } else { 7316 /* We have a SFP which supports a revision of SFF-8472. */ 7317 modinfo->type = RTE_ETH_MODULE_SFF_8472; 7318 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 7319 } 7320 7321 return 0; 7322 } 7323 7324 static int 7325 ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 7326 struct rte_dev_eeprom_info *info) 7327 { 7328 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7329 uint32_t status = IXGBE_ERR_PHY_ADDR_INVALID; 7330 uint8_t databyte = 0xFF; 7331 uint8_t *data = info->data; 7332 uint32_t i = 0; 7333 7334 if (info->length == 0) 7335 return -EINVAL; 7336 7337 for (i = info->offset; i < info->offset + info->length; i++) { 7338 if (i < RTE_ETH_MODULE_SFF_8079_LEN) 7339 status = hw->phy.ops.read_i2c_eeprom(hw, i, &databyte); 7340 else 7341 status = hw->phy.ops.read_i2c_sff8472(hw, i, &databyte); 7342 7343 if (status != 0) 7344 return -EIO; 7345 7346 data[i - info->offset] = databyte; 7347 } 7348 7349 return 0; 7350 } 7351 7352 uint16_t 7353 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) { 7354 switch (mac_type) { 7355 case ixgbe_mac_X550: 7356 case ixgbe_mac_X550EM_x: 7357 case ixgbe_mac_X550EM_a: 7358 return ETH_RSS_RETA_SIZE_512; 7359 case ixgbe_mac_X550_vf: 7360 case ixgbe_mac_X550EM_x_vf: 7361 case ixgbe_mac_X550EM_a_vf: 7362 return ETH_RSS_RETA_SIZE_64; 7363 case ixgbe_mac_X540_vf: 7364 case ixgbe_mac_82599_vf: 7365 return 0; 7366 default: 7367 return ETH_RSS_RETA_SIZE_128; 7368 } 7369 } 7370 7371 uint32_t 7372 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) { 7373 switch (mac_type) { 7374 case ixgbe_mac_X550: 7375 case ixgbe_mac_X550EM_x: 7376 case ixgbe_mac_X550EM_a: 7377 if (reta_idx < ETH_RSS_RETA_SIZE_128) 7378 return IXGBE_RETA(reta_idx >> 2); 7379 else 7380 return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2); 7381 case ixgbe_mac_X550_vf: 7382 case ixgbe_mac_X550EM_x_vf: 7383 case ixgbe_mac_X550EM_a_vf: 7384 return IXGBE_VFRETA(reta_idx >> 2); 7385 default: 7386 return IXGBE_RETA(reta_idx >> 2); 7387 } 7388 } 7389 7390 uint32_t 7391 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) { 7392 switch (mac_type) { 7393 case ixgbe_mac_X550_vf: 7394 case ixgbe_mac_X550EM_x_vf: 7395 case ixgbe_mac_X550EM_a_vf: 7396 return IXGBE_VFMRQC; 7397 default: 7398 return IXGBE_MRQC; 7399 } 7400 } 7401 7402 uint32_t 7403 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) { 7404 switch (mac_type) { 7405 case ixgbe_mac_X550_vf: 7406 case ixgbe_mac_X550EM_x_vf: 7407 case ixgbe_mac_X550EM_a_vf: 7408 return IXGBE_VFRSSRK(i); 7409 default: 7410 return IXGBE_RSSRK(i); 7411 } 7412 } 7413 7414 bool 7415 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) { 7416 switch (mac_type) { 7417 case ixgbe_mac_82599_vf: 7418 case ixgbe_mac_X540_vf: 7419 return 0; 7420 default: 7421 return 1; 7422 } 7423 } 7424 7425 static int 7426 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 7427 struct rte_eth_dcb_info *dcb_info) 7428 { 7429 struct ixgbe_dcb_config *dcb_config = 7430 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 7431 struct ixgbe_dcb_tc_config *tc; 7432 struct rte_eth_dcb_tc_queue_mapping *tc_queue; 7433 uint8_t nb_tcs; 7434 uint8_t i, j; 7435 7436 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) 7437 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs; 7438 else 7439 dcb_info->nb_tcs = 1; 7440 7441 tc_queue = &dcb_info->tc_queue; 7442 nb_tcs = dcb_info->nb_tcs; 7443 7444 if (dcb_config->vt_mode) { /* vt is enabled*/ 7445 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 7446 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 7447 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7448 dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i]; 7449 if (RTE_ETH_DEV_SRIOV(dev).active > 0) { 7450 for (j = 0; j < nb_tcs; j++) { 7451 tc_queue->tc_rxq[0][j].base = j; 7452 tc_queue->tc_rxq[0][j].nb_queue = 1; 7453 tc_queue->tc_txq[0][j].base = j; 7454 tc_queue->tc_txq[0][j].nb_queue = 1; 7455 } 7456 } else { 7457 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { 7458 for (j = 0; j < nb_tcs; j++) { 7459 tc_queue->tc_rxq[i][j].base = 7460 i * nb_tcs + j; 7461 tc_queue->tc_rxq[i][j].nb_queue = 1; 7462 tc_queue->tc_txq[i][j].base = 7463 i * nb_tcs + j; 7464 tc_queue->tc_txq[i][j].nb_queue = 1; 7465 } 7466 } 7467 } 7468 } else { /* vt is disabled*/ 7469 struct rte_eth_dcb_rx_conf *rx_conf = 7470 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 7471 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7472 dcb_info->prio_tc[i] = rx_conf->dcb_tc[i]; 7473 if (dcb_info->nb_tcs == ETH_4_TCS) { 7474 for (i = 0; i < dcb_info->nb_tcs; i++) { 7475 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32; 7476 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7477 } 7478 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7479 dcb_info->tc_queue.tc_txq[0][1].base = 64; 7480 dcb_info->tc_queue.tc_txq[0][2].base = 96; 7481 dcb_info->tc_queue.tc_txq[0][3].base = 112; 7482 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64; 7483 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7484 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7485 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7486 } else if (dcb_info->nb_tcs == ETH_8_TCS) { 7487 for (i = 0; i < dcb_info->nb_tcs; i++) { 7488 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16; 7489 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7490 } 7491 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7492 dcb_info->tc_queue.tc_txq[0][1].base = 32; 7493 dcb_info->tc_queue.tc_txq[0][2].base = 64; 7494 dcb_info->tc_queue.tc_txq[0][3].base = 80; 7495 dcb_info->tc_queue.tc_txq[0][4].base = 96; 7496 dcb_info->tc_queue.tc_txq[0][5].base = 104; 7497 dcb_info->tc_queue.tc_txq[0][6].base = 112; 7498 dcb_info->tc_queue.tc_txq[0][7].base = 120; 7499 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32; 7500 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7501 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7502 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7503 dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8; 7504 dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8; 7505 dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8; 7506 dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8; 7507 } 7508 } 7509 for (i = 0; i < dcb_info->nb_tcs; i++) { 7510 tc = &dcb_config->tc_config[i]; 7511 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent; 7512 } 7513 return 0; 7514 } 7515 7516 /* Update e-tag ether type */ 7517 static int 7518 ixgbe_update_e_tag_eth_type(struct ixgbe_hw *hw, 7519 uint16_t ether_type) 7520 { 7521 uint32_t etag_etype; 7522 7523 if (hw->mac.type != ixgbe_mac_X550 && 7524 hw->mac.type != ixgbe_mac_X550EM_x && 7525 hw->mac.type != ixgbe_mac_X550EM_a) { 7526 return -ENOTSUP; 7527 } 7528 7529 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7530 etag_etype &= ~IXGBE_ETAG_ETYPE_MASK; 7531 etag_etype |= ether_type; 7532 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7533 IXGBE_WRITE_FLUSH(hw); 7534 7535 return 0; 7536 } 7537 7538 /* Enable e-tag tunnel */ 7539 static int 7540 ixgbe_e_tag_enable(struct ixgbe_hw *hw) 7541 { 7542 uint32_t etag_etype; 7543 7544 if (hw->mac.type != ixgbe_mac_X550 && 7545 hw->mac.type != ixgbe_mac_X550EM_x && 7546 hw->mac.type != ixgbe_mac_X550EM_a) { 7547 return -ENOTSUP; 7548 } 7549 7550 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7551 etag_etype |= IXGBE_ETAG_ETYPE_VALID; 7552 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7553 IXGBE_WRITE_FLUSH(hw); 7554 7555 return 0; 7556 } 7557 7558 static int 7559 ixgbe_e_tag_filter_del(struct rte_eth_dev *dev, 7560 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7561 { 7562 int ret = 0; 7563 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7564 uint32_t i, rar_entries; 7565 uint32_t rar_low, rar_high; 7566 7567 if (hw->mac.type != ixgbe_mac_X550 && 7568 hw->mac.type != ixgbe_mac_X550EM_x && 7569 hw->mac.type != ixgbe_mac_X550EM_a) { 7570 return -ENOTSUP; 7571 } 7572 7573 rar_entries = ixgbe_get_num_rx_addrs(hw); 7574 7575 for (i = 1; i < rar_entries; i++) { 7576 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7577 rar_low = IXGBE_READ_REG(hw, IXGBE_RAL(i)); 7578 if ((rar_high & IXGBE_RAH_AV) && 7579 (rar_high & IXGBE_RAH_ADTYPE) && 7580 ((rar_low & IXGBE_RAL_ETAG_FILTER_MASK) == 7581 l2_tunnel->tunnel_id)) { 7582 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0); 7583 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0); 7584 7585 ixgbe_clear_vmdq(hw, i, IXGBE_CLEAR_VMDQ_ALL); 7586 7587 return ret; 7588 } 7589 } 7590 7591 return ret; 7592 } 7593 7594 static int 7595 ixgbe_e_tag_filter_add(struct rte_eth_dev *dev, 7596 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7597 { 7598 int ret = 0; 7599 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7600 uint32_t i, rar_entries; 7601 uint32_t rar_low, rar_high; 7602 7603 if (hw->mac.type != ixgbe_mac_X550 && 7604 hw->mac.type != ixgbe_mac_X550EM_x && 7605 hw->mac.type != ixgbe_mac_X550EM_a) { 7606 return -ENOTSUP; 7607 } 7608 7609 /* One entry for one tunnel. Try to remove potential existing entry. */ 7610 ixgbe_e_tag_filter_del(dev, l2_tunnel); 7611 7612 rar_entries = ixgbe_get_num_rx_addrs(hw); 7613 7614 for (i = 1; i < rar_entries; i++) { 7615 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7616 if (rar_high & IXGBE_RAH_AV) { 7617 continue; 7618 } else { 7619 ixgbe_set_vmdq(hw, i, l2_tunnel->pool); 7620 rar_high = IXGBE_RAH_AV | IXGBE_RAH_ADTYPE; 7621 rar_low = l2_tunnel->tunnel_id; 7622 7623 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), rar_low); 7624 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), rar_high); 7625 7626 return ret; 7627 } 7628 } 7629 7630 PMD_INIT_LOG(NOTICE, "The table of E-tag forwarding rule is full." 7631 " Please remove a rule before adding a new one."); 7632 return -EINVAL; 7633 } 7634 7635 static inline struct ixgbe_l2_tn_filter * 7636 ixgbe_l2_tn_filter_lookup(struct ixgbe_l2_tn_info *l2_tn_info, 7637 struct ixgbe_l2_tn_key *key) 7638 { 7639 int ret; 7640 7641 ret = rte_hash_lookup(l2_tn_info->hash_handle, (const void *)key); 7642 if (ret < 0) 7643 return NULL; 7644 7645 return l2_tn_info->hash_map[ret]; 7646 } 7647 7648 static inline int 7649 ixgbe_insert_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7650 struct ixgbe_l2_tn_filter *l2_tn_filter) 7651 { 7652 int ret; 7653 7654 ret = rte_hash_add_key(l2_tn_info->hash_handle, 7655 &l2_tn_filter->key); 7656 7657 if (ret < 0) { 7658 PMD_DRV_LOG(ERR, 7659 "Failed to insert L2 tunnel filter" 7660 " to hash table %d!", 7661 ret); 7662 return ret; 7663 } 7664 7665 l2_tn_info->hash_map[ret] = l2_tn_filter; 7666 7667 TAILQ_INSERT_TAIL(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7668 7669 return 0; 7670 } 7671 7672 static inline int 7673 ixgbe_remove_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7674 struct ixgbe_l2_tn_key *key) 7675 { 7676 int ret; 7677 struct ixgbe_l2_tn_filter *l2_tn_filter; 7678 7679 ret = rte_hash_del_key(l2_tn_info->hash_handle, key); 7680 7681 if (ret < 0) { 7682 PMD_DRV_LOG(ERR, 7683 "No such L2 tunnel filter to delete %d!", 7684 ret); 7685 return ret; 7686 } 7687 7688 l2_tn_filter = l2_tn_info->hash_map[ret]; 7689 l2_tn_info->hash_map[ret] = NULL; 7690 7691 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7692 rte_free(l2_tn_filter); 7693 7694 return 0; 7695 } 7696 7697 /* Add l2 tunnel filter */ 7698 int 7699 ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, 7700 struct ixgbe_l2_tunnel_conf *l2_tunnel, 7701 bool restore) 7702 { 7703 int ret; 7704 struct ixgbe_l2_tn_info *l2_tn_info = 7705 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7706 struct ixgbe_l2_tn_key key; 7707 struct ixgbe_l2_tn_filter *node; 7708 7709 if (!restore) { 7710 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7711 key.tn_id = l2_tunnel->tunnel_id; 7712 7713 node = ixgbe_l2_tn_filter_lookup(l2_tn_info, &key); 7714 7715 if (node) { 7716 PMD_DRV_LOG(ERR, 7717 "The L2 tunnel filter already exists!"); 7718 return -EINVAL; 7719 } 7720 7721 node = rte_zmalloc("ixgbe_l2_tn", 7722 sizeof(struct ixgbe_l2_tn_filter), 7723 0); 7724 if (!node) 7725 return -ENOMEM; 7726 7727 rte_memcpy(&node->key, 7728 &key, 7729 sizeof(struct ixgbe_l2_tn_key)); 7730 node->pool = l2_tunnel->pool; 7731 ret = ixgbe_insert_l2_tn_filter(l2_tn_info, node); 7732 if (ret < 0) { 7733 rte_free(node); 7734 return ret; 7735 } 7736 } 7737 7738 switch (l2_tunnel->l2_tunnel_type) { 7739 case RTE_L2_TUNNEL_TYPE_E_TAG: 7740 ret = ixgbe_e_tag_filter_add(dev, l2_tunnel); 7741 break; 7742 default: 7743 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7744 ret = -EINVAL; 7745 break; 7746 } 7747 7748 if ((!restore) && (ret < 0)) 7749 (void)ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7750 7751 return ret; 7752 } 7753 7754 /* Delete l2 tunnel filter */ 7755 int 7756 ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, 7757 struct ixgbe_l2_tunnel_conf *l2_tunnel) 7758 { 7759 int ret; 7760 struct ixgbe_l2_tn_info *l2_tn_info = 7761 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7762 struct ixgbe_l2_tn_key key; 7763 7764 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7765 key.tn_id = l2_tunnel->tunnel_id; 7766 ret = ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7767 if (ret < 0) 7768 return ret; 7769 7770 switch (l2_tunnel->l2_tunnel_type) { 7771 case RTE_L2_TUNNEL_TYPE_E_TAG: 7772 ret = ixgbe_e_tag_filter_del(dev, l2_tunnel); 7773 break; 7774 default: 7775 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7776 ret = -EINVAL; 7777 break; 7778 } 7779 7780 return ret; 7781 } 7782 7783 static int 7784 ixgbe_e_tag_forwarding_en_dis(struct rte_eth_dev *dev, bool en) 7785 { 7786 int ret = 0; 7787 uint32_t ctrl; 7788 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7789 7790 if (hw->mac.type != ixgbe_mac_X550 && 7791 hw->mac.type != ixgbe_mac_X550EM_x && 7792 hw->mac.type != ixgbe_mac_X550EM_a) { 7793 return -ENOTSUP; 7794 } 7795 7796 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 7797 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 7798 if (en) 7799 ctrl |= IXGBE_VT_CTL_POOLING_MODE_ETAG; 7800 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 7801 7802 return ret; 7803 } 7804 7805 static int 7806 ixgbe_update_vxlan_port(struct ixgbe_hw *hw, 7807 uint16_t port) 7808 { 7809 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, port); 7810 IXGBE_WRITE_FLUSH(hw); 7811 7812 return 0; 7813 } 7814 7815 /* There's only one register for VxLAN UDP port. 7816 * So, we cannot add several ports. Will update it. 7817 */ 7818 static int 7819 ixgbe_add_vxlan_port(struct ixgbe_hw *hw, 7820 uint16_t port) 7821 { 7822 if (port == 0) { 7823 PMD_DRV_LOG(ERR, "Add VxLAN port 0 is not allowed."); 7824 return -EINVAL; 7825 } 7826 7827 return ixgbe_update_vxlan_port(hw, port); 7828 } 7829 7830 /* We cannot delete the VxLAN port. For there's a register for VxLAN 7831 * UDP port, it must have a value. 7832 * So, will reset it to the original value 0. 7833 */ 7834 static int 7835 ixgbe_del_vxlan_port(struct ixgbe_hw *hw, 7836 uint16_t port) 7837 { 7838 uint16_t cur_port; 7839 7840 cur_port = (uint16_t)IXGBE_READ_REG(hw, IXGBE_VXLANCTRL); 7841 7842 if (cur_port != port) { 7843 PMD_DRV_LOG(ERR, "Port %u does not exist.", port); 7844 return -EINVAL; 7845 } 7846 7847 return ixgbe_update_vxlan_port(hw, 0); 7848 } 7849 7850 /* Add UDP tunneling port */ 7851 static int 7852 ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 7853 struct rte_eth_udp_tunnel *udp_tunnel) 7854 { 7855 int ret = 0; 7856 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7857 7858 if (hw->mac.type != ixgbe_mac_X550 && 7859 hw->mac.type != ixgbe_mac_X550EM_x && 7860 hw->mac.type != ixgbe_mac_X550EM_a) { 7861 return -ENOTSUP; 7862 } 7863 7864 if (udp_tunnel == NULL) 7865 return -EINVAL; 7866 7867 switch (udp_tunnel->prot_type) { 7868 case RTE_TUNNEL_TYPE_VXLAN: 7869 ret = ixgbe_add_vxlan_port(hw, udp_tunnel->udp_port); 7870 break; 7871 7872 case RTE_TUNNEL_TYPE_GENEVE: 7873 case RTE_TUNNEL_TYPE_TEREDO: 7874 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 7875 ret = -EINVAL; 7876 break; 7877 7878 default: 7879 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7880 ret = -EINVAL; 7881 break; 7882 } 7883 7884 return ret; 7885 } 7886 7887 /* Remove UDP tunneling port */ 7888 static int 7889 ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 7890 struct rte_eth_udp_tunnel *udp_tunnel) 7891 { 7892 int ret = 0; 7893 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7894 7895 if (hw->mac.type != ixgbe_mac_X550 && 7896 hw->mac.type != ixgbe_mac_X550EM_x && 7897 hw->mac.type != ixgbe_mac_X550EM_a) { 7898 return -ENOTSUP; 7899 } 7900 7901 if (udp_tunnel == NULL) 7902 return -EINVAL; 7903 7904 switch (udp_tunnel->prot_type) { 7905 case RTE_TUNNEL_TYPE_VXLAN: 7906 ret = ixgbe_del_vxlan_port(hw, udp_tunnel->udp_port); 7907 break; 7908 case RTE_TUNNEL_TYPE_GENEVE: 7909 case RTE_TUNNEL_TYPE_TEREDO: 7910 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 7911 ret = -EINVAL; 7912 break; 7913 default: 7914 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7915 ret = -EINVAL; 7916 break; 7917 } 7918 7919 return ret; 7920 } 7921 7922 static int 7923 ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev) 7924 { 7925 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7926 int ret; 7927 7928 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_PROMISC)) { 7929 case IXGBE_SUCCESS: 7930 ret = 0; 7931 break; 7932 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7933 ret = -ENOTSUP; 7934 break; 7935 default: 7936 ret = -EAGAIN; 7937 break; 7938 } 7939 7940 return ret; 7941 } 7942 7943 static int 7944 ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev) 7945 { 7946 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7947 int ret; 7948 7949 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_NONE)) { 7950 case IXGBE_SUCCESS: 7951 ret = 0; 7952 break; 7953 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7954 ret = -ENOTSUP; 7955 break; 7956 default: 7957 ret = -EAGAIN; 7958 break; 7959 } 7960 7961 return ret; 7962 } 7963 7964 static int 7965 ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) 7966 { 7967 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7968 int ret; 7969 int mode = IXGBEVF_XCAST_MODE_ALLMULTI; 7970 7971 switch (hw->mac.ops.update_xcast_mode(hw, mode)) { 7972 case IXGBE_SUCCESS: 7973 ret = 0; 7974 break; 7975 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7976 ret = -ENOTSUP; 7977 break; 7978 default: 7979 ret = -EAGAIN; 7980 break; 7981 } 7982 7983 return ret; 7984 } 7985 7986 static int 7987 ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) 7988 { 7989 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7990 int ret; 7991 7992 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI)) { 7993 case IXGBE_SUCCESS: 7994 ret = 0; 7995 break; 7996 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 7997 ret = -ENOTSUP; 7998 break; 7999 default: 8000 ret = -EAGAIN; 8001 break; 8002 } 8003 8004 return ret; 8005 } 8006 8007 static void ixgbevf_mbx_process(struct rte_eth_dev *dev) 8008 { 8009 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8010 u32 in_msg = 0; 8011 8012 /* peek the message first */ 8013 in_msg = IXGBE_READ_REG(hw, IXGBE_VFMBMEM); 8014 8015 /* PF reset VF event */ 8016 if (in_msg == IXGBE_PF_CONTROL_MSG) { 8017 /* dummy mbx read to ack pf */ 8018 if (ixgbe_read_mbx(hw, &in_msg, 1, 0)) 8019 return; 8020 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 8021 NULL); 8022 } 8023 } 8024 8025 static int 8026 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev) 8027 { 8028 uint32_t eicr; 8029 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8030 struct ixgbe_interrupt *intr = 8031 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8032 ixgbevf_intr_disable(dev); 8033 8034 /* read-on-clear nic registers here */ 8035 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR); 8036 intr->flags = 0; 8037 8038 /* only one misc vector supported - mailbox */ 8039 eicr &= IXGBE_VTEICR_MASK; 8040 if (eicr == IXGBE_MISC_VEC_ID) 8041 intr->flags |= IXGBE_FLAG_MAILBOX; 8042 8043 return 0; 8044 } 8045 8046 static int 8047 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev) 8048 { 8049 struct ixgbe_interrupt *intr = 8050 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8051 8052 if (intr->flags & IXGBE_FLAG_MAILBOX) { 8053 ixgbevf_mbx_process(dev); 8054 intr->flags &= ~IXGBE_FLAG_MAILBOX; 8055 } 8056 8057 ixgbevf_intr_enable(dev); 8058 8059 return 0; 8060 } 8061 8062 static void 8063 ixgbevf_dev_interrupt_handler(void *param) 8064 { 8065 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 8066 8067 ixgbevf_dev_interrupt_get_status(dev); 8068 ixgbevf_dev_interrupt_action(dev); 8069 } 8070 8071 /** 8072 * ixgbe_disable_sec_tx_path_generic - Stops the transmit data path 8073 * @hw: pointer to hardware structure 8074 * 8075 * Stops the transmit data path and waits for the HW to internally empty 8076 * the Tx security block 8077 **/ 8078 int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw) 8079 { 8080 #define IXGBE_MAX_SECTX_POLL 40 8081 8082 int i; 8083 int sectxreg; 8084 8085 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8086 sectxreg |= IXGBE_SECTXCTRL_TX_DIS; 8087 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8088 for (i = 0; i < IXGBE_MAX_SECTX_POLL; i++) { 8089 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT); 8090 if (sectxreg & IXGBE_SECTXSTAT_SECTX_RDY) 8091 break; 8092 /* Use interrupt-safe sleep just in case */ 8093 usec_delay(1000); 8094 } 8095 8096 /* For informational purposes only */ 8097 if (i >= IXGBE_MAX_SECTX_POLL) 8098 PMD_DRV_LOG(DEBUG, "Tx unit being enabled before security " 8099 "path fully disabled. Continuing with init."); 8100 8101 return IXGBE_SUCCESS; 8102 } 8103 8104 /** 8105 * ixgbe_enable_sec_tx_path_generic - Enables the transmit data path 8106 * @hw: pointer to hardware structure 8107 * 8108 * Enables the transmit data path. 8109 **/ 8110 int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw) 8111 { 8112 uint32_t sectxreg; 8113 8114 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8115 sectxreg &= ~IXGBE_SECTXCTRL_TX_DIS; 8116 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8117 IXGBE_WRITE_FLUSH(hw); 8118 8119 return IXGBE_SUCCESS; 8120 } 8121 8122 /* restore n-tuple filter */ 8123 static inline void 8124 ixgbe_ntuple_filter_restore(struct rte_eth_dev *dev) 8125 { 8126 struct ixgbe_filter_info *filter_info = 8127 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8128 struct ixgbe_5tuple_filter *node; 8129 8130 TAILQ_FOREACH(node, &filter_info->fivetuple_list, entries) { 8131 ixgbe_inject_5tuple_filter(dev, node); 8132 } 8133 } 8134 8135 /* restore ethernet type filter */ 8136 static inline void 8137 ixgbe_ethertype_filter_restore(struct rte_eth_dev *dev) 8138 { 8139 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8140 struct ixgbe_filter_info *filter_info = 8141 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8142 int i; 8143 8144 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8145 if (filter_info->ethertype_mask & (1 << i)) { 8146 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 8147 filter_info->ethertype_filters[i].etqf); 8148 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 8149 filter_info->ethertype_filters[i].etqs); 8150 IXGBE_WRITE_FLUSH(hw); 8151 } 8152 } 8153 } 8154 8155 /* restore SYN filter */ 8156 static inline void 8157 ixgbe_syn_filter_restore(struct rte_eth_dev *dev) 8158 { 8159 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8160 struct ixgbe_filter_info *filter_info = 8161 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8162 uint32_t synqf; 8163 8164 synqf = filter_info->syn_info; 8165 8166 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 8167 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 8168 IXGBE_WRITE_FLUSH(hw); 8169 } 8170 } 8171 8172 /* restore L2 tunnel filter */ 8173 static inline void 8174 ixgbe_l2_tn_filter_restore(struct rte_eth_dev *dev) 8175 { 8176 struct ixgbe_l2_tn_info *l2_tn_info = 8177 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8178 struct ixgbe_l2_tn_filter *node; 8179 struct ixgbe_l2_tunnel_conf l2_tn_conf; 8180 8181 TAILQ_FOREACH(node, &l2_tn_info->l2_tn_list, entries) { 8182 l2_tn_conf.l2_tunnel_type = node->key.l2_tn_type; 8183 l2_tn_conf.tunnel_id = node->key.tn_id; 8184 l2_tn_conf.pool = node->pool; 8185 (void)ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_conf, TRUE); 8186 } 8187 } 8188 8189 /* restore rss filter */ 8190 static inline void 8191 ixgbe_rss_filter_restore(struct rte_eth_dev *dev) 8192 { 8193 struct ixgbe_filter_info *filter_info = 8194 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8195 8196 if (filter_info->rss_info.conf.queue_num) 8197 ixgbe_config_rss_filter(dev, 8198 &filter_info->rss_info, TRUE); 8199 } 8200 8201 static int 8202 ixgbe_filter_restore(struct rte_eth_dev *dev) 8203 { 8204 ixgbe_ntuple_filter_restore(dev); 8205 ixgbe_ethertype_filter_restore(dev); 8206 ixgbe_syn_filter_restore(dev); 8207 ixgbe_fdir_filter_restore(dev); 8208 ixgbe_l2_tn_filter_restore(dev); 8209 ixgbe_rss_filter_restore(dev); 8210 8211 return 0; 8212 } 8213 8214 static void 8215 ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev) 8216 { 8217 struct ixgbe_l2_tn_info *l2_tn_info = 8218 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8219 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8220 8221 if (l2_tn_info->e_tag_en) 8222 (void)ixgbe_e_tag_enable(hw); 8223 8224 if (l2_tn_info->e_tag_fwd_en) 8225 (void)ixgbe_e_tag_forwarding_en_dis(dev, 1); 8226 8227 (void)ixgbe_update_e_tag_eth_type(hw, l2_tn_info->e_tag_ether_type); 8228 } 8229 8230 /* remove all the n-tuple filters */ 8231 void 8232 ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev) 8233 { 8234 struct ixgbe_filter_info *filter_info = 8235 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8236 struct ixgbe_5tuple_filter *p_5tuple; 8237 8238 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) 8239 ixgbe_remove_5tuple_filter(dev, p_5tuple); 8240 } 8241 8242 /* remove all the ether type filters */ 8243 void 8244 ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev) 8245 { 8246 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8247 struct ixgbe_filter_info *filter_info = 8248 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8249 int i; 8250 8251 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8252 if (filter_info->ethertype_mask & (1 << i) && 8253 !filter_info->ethertype_filters[i].conf) { 8254 (void)ixgbe_ethertype_filter_remove(filter_info, 8255 (uint8_t)i); 8256 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 0); 8257 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 0); 8258 IXGBE_WRITE_FLUSH(hw); 8259 } 8260 } 8261 } 8262 8263 /* remove the SYN filter */ 8264 void 8265 ixgbe_clear_syn_filter(struct rte_eth_dev *dev) 8266 { 8267 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8268 struct ixgbe_filter_info *filter_info = 8269 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8270 8271 if (filter_info->syn_info & IXGBE_SYN_FILTER_ENABLE) { 8272 filter_info->syn_info = 0; 8273 8274 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, 0); 8275 IXGBE_WRITE_FLUSH(hw); 8276 } 8277 } 8278 8279 /* remove all the L2 tunnel filters */ 8280 int 8281 ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev) 8282 { 8283 struct ixgbe_l2_tn_info *l2_tn_info = 8284 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8285 struct ixgbe_l2_tn_filter *l2_tn_filter; 8286 struct ixgbe_l2_tunnel_conf l2_tn_conf; 8287 int ret = 0; 8288 8289 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 8290 l2_tn_conf.l2_tunnel_type = l2_tn_filter->key.l2_tn_type; 8291 l2_tn_conf.tunnel_id = l2_tn_filter->key.tn_id; 8292 l2_tn_conf.pool = l2_tn_filter->pool; 8293 ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_conf); 8294 if (ret < 0) 8295 return ret; 8296 } 8297 8298 return 0; 8299 } 8300 8301 void 8302 ixgbe_dev_macsec_setting_save(struct rte_eth_dev *dev, 8303 struct ixgbe_macsec_setting *macsec_setting) 8304 { 8305 struct ixgbe_macsec_setting *macsec = 8306 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 8307 8308 macsec->offload_en = macsec_setting->offload_en; 8309 macsec->encrypt_en = macsec_setting->encrypt_en; 8310 macsec->replayprotect_en = macsec_setting->replayprotect_en; 8311 } 8312 8313 void 8314 ixgbe_dev_macsec_setting_reset(struct rte_eth_dev *dev) 8315 { 8316 struct ixgbe_macsec_setting *macsec = 8317 IXGBE_DEV_PRIVATE_TO_MACSEC_SETTING(dev->data->dev_private); 8318 8319 macsec->offload_en = 0; 8320 macsec->encrypt_en = 0; 8321 macsec->replayprotect_en = 0; 8322 } 8323 8324 void 8325 ixgbe_dev_macsec_register_enable(struct rte_eth_dev *dev, 8326 struct ixgbe_macsec_setting *macsec_setting) 8327 { 8328 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8329 uint32_t ctrl; 8330 uint8_t en = macsec_setting->encrypt_en; 8331 uint8_t rp = macsec_setting->replayprotect_en; 8332 8333 /** 8334 * Workaround: 8335 * As no ixgbe_disable_sec_rx_path equivalent is 8336 * implemented for tx in the base code, and we are 8337 * not allowed to modify the base code in DPDK, so 8338 * just call the hand-written one directly for now. 8339 * The hardware support has been checked by 8340 * ixgbe_disable_sec_rx_path(). 8341 */ 8342 ixgbe_disable_sec_tx_path_generic(hw); 8343 8344 /* Enable Ethernet CRC (required by MACsec offload) */ 8345 ctrl = IXGBE_READ_REG(hw, IXGBE_HLREG0); 8346 ctrl |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP; 8347 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, ctrl); 8348 8349 /* Enable the TX and RX crypto engines */ 8350 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8351 ctrl &= ~IXGBE_SECTXCTRL_SECTX_DIS; 8352 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); 8353 8354 ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); 8355 ctrl &= ~IXGBE_SECRXCTRL_SECRX_DIS; 8356 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); 8357 8358 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 8359 ctrl &= ~IXGBE_SECTX_MINSECIFG_MASK; 8360 ctrl |= 0x3; 8361 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, ctrl); 8362 8363 /* Enable SA lookup */ 8364 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); 8365 ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; 8366 ctrl |= en ? IXGBE_LSECTXCTRL_AUTH_ENCRYPT : 8367 IXGBE_LSECTXCTRL_AUTH; 8368 ctrl |= IXGBE_LSECTXCTRL_AISCI; 8369 ctrl &= ~IXGBE_LSECTXCTRL_PNTHRSH_MASK; 8370 ctrl |= IXGBE_MACSEC_PNTHRSH & IXGBE_LSECTXCTRL_PNTHRSH_MASK; 8371 IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); 8372 8373 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); 8374 ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; 8375 ctrl |= IXGBE_LSECRXCTRL_STRICT << IXGBE_LSECRXCTRL_EN_SHIFT; 8376 ctrl &= ~IXGBE_LSECRXCTRL_PLSH; 8377 if (rp) 8378 ctrl |= IXGBE_LSECRXCTRL_RP; 8379 else 8380 ctrl &= ~IXGBE_LSECRXCTRL_RP; 8381 IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); 8382 8383 /* Start the data paths */ 8384 ixgbe_enable_sec_rx_path(hw); 8385 /** 8386 * Workaround: 8387 * As no ixgbe_enable_sec_rx_path equivalent is 8388 * implemented for tx in the base code, and we are 8389 * not allowed to modify the base code in DPDK, so 8390 * just call the hand-written one directly for now. 8391 */ 8392 ixgbe_enable_sec_tx_path_generic(hw); 8393 } 8394 8395 void 8396 ixgbe_dev_macsec_register_disable(struct rte_eth_dev *dev) 8397 { 8398 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8399 uint32_t ctrl; 8400 8401 /** 8402 * Workaround: 8403 * As no ixgbe_disable_sec_rx_path equivalent is 8404 * implemented for tx in the base code, and we are 8405 * not allowed to modify the base code in DPDK, so 8406 * just call the hand-written one directly for now. 8407 * The hardware support has been checked by 8408 * ixgbe_disable_sec_rx_path(). 8409 */ 8410 ixgbe_disable_sec_tx_path_generic(hw); 8411 8412 /* Disable the TX and RX crypto engines */ 8413 ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8414 ctrl |= IXGBE_SECTXCTRL_SECTX_DIS; 8415 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); 8416 8417 ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); 8418 ctrl |= IXGBE_SECRXCTRL_SECRX_DIS; 8419 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); 8420 8421 /* Disable SA lookup */ 8422 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); 8423 ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; 8424 ctrl |= IXGBE_LSECTXCTRL_DISABLE; 8425 IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); 8426 8427 ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); 8428 ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; 8429 ctrl |= IXGBE_LSECRXCTRL_DISABLE << IXGBE_LSECRXCTRL_EN_SHIFT; 8430 IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); 8431 8432 /* Start the data paths */ 8433 ixgbe_enable_sec_rx_path(hw); 8434 /** 8435 * Workaround: 8436 * As no ixgbe_enable_sec_rx_path equivalent is 8437 * implemented for tx in the base code, and we are 8438 * not allowed to modify the base code in DPDK, so 8439 * just call the hand-written one directly for now. 8440 */ 8441 ixgbe_enable_sec_tx_path_generic(hw); 8442 } 8443 8444 RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 8445 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe, pci_id_ixgbe_map); 8446 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe, "* igb_uio | uio_pci_generic | vfio-pci"); 8447 RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd); 8448 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map); 8449 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci"); 8450 RTE_PMD_REGISTER_PARAM_STRING(net_ixgbe_vf, 8451 IXGBEVF_DEVARG_PFLINK_FULLCHK "=<0|1>"); 8452 8453 RTE_LOG_REGISTER(ixgbe_logtype_init, pmd.net.ixgbe.init, NOTICE); 8454 RTE_LOG_REGISTER(ixgbe_logtype_driver, pmd.net.ixgbe.driver, NOTICE); 8455 8456 #ifdef RTE_LIBRTE_IXGBE_DEBUG_RX 8457 RTE_LOG_REGISTER(ixgbe_logtype_rx, pmd.net.ixgbe.rx, DEBUG); 8458 #endif 8459 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX 8460 RTE_LOG_REGISTER(ixgbe_logtype_tx, pmd.net.ixgbe.tx, DEBUG); 8461 #endif 8462 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX_FREE 8463 RTE_LOG_REGISTER(ixgbe_logtype_tx_free, pmd.net.ixgbe.tx_free, DEBUG); 8464 #endif 8465