1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static int eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static int eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static int eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static int eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static int eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static int eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 100 unsigned int limit); 101 static int eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static int eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static int eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static int eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct rte_ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct rte_ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static int igbvf_dev_stop(struct rte_eth_dev *dev); 158 static int igbvf_dev_close(struct rte_eth_dev *dev); 159 static int igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static int igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static int igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static int igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static int eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct rte_ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 190 struct rte_eth_ntuple_filter *ntuple_filter); 191 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 192 struct rte_eth_ntuple_filter *ntuple_filter); 193 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 194 struct rte_eth_ntuple_filter *ntuple_filter); 195 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 196 struct rte_eth_ntuple_filter *ntuple_filter); 197 static int eth_igb_flow_ops_get(struct rte_eth_dev *dev, 198 const struct rte_flow_ops **ops); 199 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 200 static int eth_igb_get_regs(struct rte_eth_dev *dev, 201 struct rte_dev_reg_info *regs); 202 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 203 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 204 struct rte_dev_eeprom_info *eeprom); 205 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 206 struct rte_dev_eeprom_info *eeprom); 207 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 208 struct rte_eth_dev_module_info *modinfo); 209 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 210 struct rte_dev_eeprom_info *info); 211 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 212 struct rte_ether_addr *mc_addr_set, 213 uint32_t nb_mc_addr); 214 static int igb_timesync_enable(struct rte_eth_dev *dev); 215 static int igb_timesync_disable(struct rte_eth_dev *dev); 216 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 217 struct timespec *timestamp, 218 uint32_t flags); 219 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 220 struct timespec *timestamp); 221 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 222 static int igb_timesync_read_time(struct rte_eth_dev *dev, 223 struct timespec *timestamp); 224 static int igb_timesync_write_time(struct rte_eth_dev *dev, 225 const struct timespec *timestamp); 226 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 227 uint16_t queue_id); 228 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 229 uint16_t queue_id); 230 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 231 uint8_t queue, uint8_t msix_vector); 232 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 233 uint8_t index, uint8_t offset); 234 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 235 static void eth_igbvf_interrupt_handler(void *param); 236 static void igbvf_mbx_process(struct rte_eth_dev *dev); 237 static int igb_filter_restore(struct rte_eth_dev *dev); 238 239 /* 240 * Define VF Stats MACRO for Non "cleared on read" register 241 */ 242 #define UPDATE_VF_STAT(reg, last, cur) \ 243 { \ 244 u32 latest = E1000_READ_REG(hw, reg); \ 245 cur += (latest - last) & UINT_MAX; \ 246 last = latest; \ 247 } 248 249 #define IGB_FC_PAUSE_TIME 0x0680 250 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 251 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 252 253 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 254 255 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 256 257 /* 258 * The set of PCI devices this driver supports 259 */ 260 static const struct rte_pci_id pci_id_igb_map[] = { 261 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 262 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 263 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 264 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 265 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 266 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 267 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 268 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 269 270 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 271 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 272 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 273 274 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 275 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 276 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 277 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 278 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 279 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 280 281 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 291 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 295 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 302 { .vendor_id = 0, /* sentinel */ }, 303 }; 304 305 /* 306 * The set of PCI devices this driver supports (for 82576&I350 VF) 307 */ 308 static const struct rte_pci_id pci_id_igbvf_map[] = { 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 313 { .vendor_id = 0, /* sentinel */ }, 314 }; 315 316 static const struct rte_eth_desc_lim rx_desc_lim = { 317 .nb_max = E1000_MAX_RING_DESC, 318 .nb_min = E1000_MIN_RING_DESC, 319 .nb_align = IGB_RXD_ALIGN, 320 }; 321 322 static const struct rte_eth_desc_lim tx_desc_lim = { 323 .nb_max = E1000_MAX_RING_DESC, 324 .nb_min = E1000_MIN_RING_DESC, 325 .nb_align = IGB_RXD_ALIGN, 326 .nb_seg_max = IGB_TX_MAX_SEG, 327 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 328 }; 329 330 static const struct eth_dev_ops eth_igb_ops = { 331 .dev_configure = eth_igb_configure, 332 .dev_start = eth_igb_start, 333 .dev_stop = eth_igb_stop, 334 .dev_set_link_up = eth_igb_dev_set_link_up, 335 .dev_set_link_down = eth_igb_dev_set_link_down, 336 .dev_close = eth_igb_close, 337 .dev_reset = eth_igb_reset, 338 .promiscuous_enable = eth_igb_promiscuous_enable, 339 .promiscuous_disable = eth_igb_promiscuous_disable, 340 .allmulticast_enable = eth_igb_allmulticast_enable, 341 .allmulticast_disable = eth_igb_allmulticast_disable, 342 .link_update = eth_igb_link_update, 343 .stats_get = eth_igb_stats_get, 344 .xstats_get = eth_igb_xstats_get, 345 .xstats_get_by_id = eth_igb_xstats_get_by_id, 346 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 347 .xstats_get_names = eth_igb_xstats_get_names, 348 .stats_reset = eth_igb_stats_reset, 349 .xstats_reset = eth_igb_xstats_reset, 350 .fw_version_get = eth_igb_fw_version_get, 351 .dev_infos_get = eth_igb_infos_get, 352 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 353 .mtu_set = eth_igb_mtu_set, 354 .vlan_filter_set = eth_igb_vlan_filter_set, 355 .vlan_tpid_set = eth_igb_vlan_tpid_set, 356 .vlan_offload_set = eth_igb_vlan_offload_set, 357 .rx_queue_setup = eth_igb_rx_queue_setup, 358 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 359 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 360 .rx_queue_release = eth_igb_rx_queue_release, 361 .tx_queue_setup = eth_igb_tx_queue_setup, 362 .tx_queue_release = eth_igb_tx_queue_release, 363 .tx_done_cleanup = eth_igb_tx_done_cleanup, 364 .dev_led_on = eth_igb_led_on, 365 .dev_led_off = eth_igb_led_off, 366 .flow_ctrl_get = eth_igb_flow_ctrl_get, 367 .flow_ctrl_set = eth_igb_flow_ctrl_set, 368 .mac_addr_add = eth_igb_rar_set, 369 .mac_addr_remove = eth_igb_rar_clear, 370 .mac_addr_set = eth_igb_default_mac_addr_set, 371 .reta_update = eth_igb_rss_reta_update, 372 .reta_query = eth_igb_rss_reta_query, 373 .rss_hash_update = eth_igb_rss_hash_update, 374 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 375 .flow_ops_get = eth_igb_flow_ops_get, 376 .set_mc_addr_list = eth_igb_set_mc_addr_list, 377 .rxq_info_get = igb_rxq_info_get, 378 .txq_info_get = igb_txq_info_get, 379 .timesync_enable = igb_timesync_enable, 380 .timesync_disable = igb_timesync_disable, 381 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 382 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 383 .get_reg = eth_igb_get_regs, 384 .get_eeprom_length = eth_igb_get_eeprom_length, 385 .get_eeprom = eth_igb_get_eeprom, 386 .set_eeprom = eth_igb_set_eeprom, 387 .get_module_info = eth_igb_get_module_info, 388 .get_module_eeprom = eth_igb_get_module_eeprom, 389 .timesync_adjust_time = igb_timesync_adjust_time, 390 .timesync_read_time = igb_timesync_read_time, 391 .timesync_write_time = igb_timesync_write_time, 392 }; 393 394 /* 395 * dev_ops for virtual function, bare necessities for basic vf 396 * operation have been implemented 397 */ 398 static const struct eth_dev_ops igbvf_eth_dev_ops = { 399 .dev_configure = igbvf_dev_configure, 400 .dev_start = igbvf_dev_start, 401 .dev_stop = igbvf_dev_stop, 402 .dev_close = igbvf_dev_close, 403 .promiscuous_enable = igbvf_promiscuous_enable, 404 .promiscuous_disable = igbvf_promiscuous_disable, 405 .allmulticast_enable = igbvf_allmulticast_enable, 406 .allmulticast_disable = igbvf_allmulticast_disable, 407 .link_update = eth_igb_link_update, 408 .stats_get = eth_igbvf_stats_get, 409 .xstats_get = eth_igbvf_xstats_get, 410 .xstats_get_names = eth_igbvf_xstats_get_names, 411 .stats_reset = eth_igbvf_stats_reset, 412 .xstats_reset = eth_igbvf_stats_reset, 413 .vlan_filter_set = igbvf_vlan_filter_set, 414 .dev_infos_get = eth_igbvf_infos_get, 415 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 416 .rx_queue_setup = eth_igb_rx_queue_setup, 417 .rx_queue_release = eth_igb_rx_queue_release, 418 .tx_queue_setup = eth_igb_tx_queue_setup, 419 .tx_queue_release = eth_igb_tx_queue_release, 420 .tx_done_cleanup = eth_igb_tx_done_cleanup, 421 .set_mc_addr_list = eth_igb_set_mc_addr_list, 422 .rxq_info_get = igb_rxq_info_get, 423 .txq_info_get = igb_txq_info_get, 424 .mac_addr_set = igbvf_default_mac_addr_set, 425 .get_reg = igbvf_get_regs, 426 }; 427 428 /* store statistics names and its offset in stats structure */ 429 struct rte_igb_xstats_name_off { 430 char name[RTE_ETH_XSTATS_NAME_SIZE]; 431 unsigned offset; 432 }; 433 434 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 435 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 436 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 437 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 438 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 439 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 440 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 441 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 442 ecol)}, 443 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 444 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 445 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 446 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 447 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 448 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 449 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 450 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 451 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 452 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 453 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 454 fcruc)}, 455 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 456 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 457 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 458 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 459 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 460 prc1023)}, 461 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 462 prc1522)}, 463 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 464 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 465 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 466 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 467 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 468 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 469 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 470 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 471 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 472 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 473 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 474 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 475 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 476 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 477 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 478 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 479 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 480 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 481 ptc1023)}, 482 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 483 ptc1522)}, 484 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 485 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 486 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 487 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 488 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 489 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 490 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 491 492 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 493 }; 494 495 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 496 sizeof(rte_igb_stats_strings[0])) 497 498 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 499 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 500 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 501 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 502 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 503 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 504 }; 505 506 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 507 sizeof(rte_igbvf_stats_strings[0])) 508 509 510 static inline void 511 igb_intr_enable(struct rte_eth_dev *dev) 512 { 513 struct e1000_interrupt *intr = 514 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 515 struct e1000_hw *hw = 516 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 517 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 518 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 519 520 if (rte_intr_allow_others(intr_handle) && 521 dev->data->dev_conf.intr_conf.lsc != 0) { 522 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 523 } 524 525 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 526 E1000_WRITE_FLUSH(hw); 527 } 528 529 static void 530 igb_intr_disable(struct rte_eth_dev *dev) 531 { 532 struct e1000_hw *hw = 533 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 534 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 535 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 536 537 if (rte_intr_allow_others(intr_handle) && 538 dev->data->dev_conf.intr_conf.lsc != 0) { 539 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 540 } 541 542 E1000_WRITE_REG(hw, E1000_IMC, ~0); 543 E1000_WRITE_FLUSH(hw); 544 } 545 546 static inline void 547 igbvf_intr_enable(struct rte_eth_dev *dev) 548 { 549 struct e1000_hw *hw = 550 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 551 552 /* only for mailbox */ 553 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 554 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 555 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 556 E1000_WRITE_FLUSH(hw); 557 } 558 559 /* only for mailbox now. If RX/TX needed, should extend this function. */ 560 static void 561 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 562 { 563 uint32_t tmp = 0; 564 565 /* mailbox */ 566 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 567 tmp |= E1000_VTIVAR_VALID; 568 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 569 } 570 571 static void 572 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 573 { 574 struct e1000_hw *hw = 575 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 576 577 /* Configure VF other cause ivar */ 578 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 579 } 580 581 static inline int32_t 582 igb_pf_reset_hw(struct e1000_hw *hw) 583 { 584 uint32_t ctrl_ext; 585 int32_t status; 586 587 status = e1000_reset_hw(hw); 588 589 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 590 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 591 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 592 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 593 E1000_WRITE_FLUSH(hw); 594 595 return status; 596 } 597 598 static void 599 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 600 { 601 struct e1000_hw *hw = 602 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 603 604 605 hw->vendor_id = pci_dev->id.vendor_id; 606 hw->device_id = pci_dev->id.device_id; 607 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 608 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 609 610 e1000_set_mac_type(hw); 611 612 /* need to check if it is a vf device below */ 613 } 614 615 static int 616 igb_reset_swfw_lock(struct e1000_hw *hw) 617 { 618 int ret_val; 619 620 /* 621 * Do mac ops initialization manually here, since we will need 622 * some function pointers set by this call. 623 */ 624 ret_val = e1000_init_mac_params(hw); 625 if (ret_val) 626 return ret_val; 627 628 /* 629 * SMBI lock should not fail in this early stage. If this is the case, 630 * it is due to an improper exit of the application. 631 * So force the release of the faulty lock. 632 */ 633 if (e1000_get_hw_semaphore_generic(hw) < 0) { 634 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 635 } 636 e1000_put_hw_semaphore_generic(hw); 637 638 if (hw->mac.ops.acquire_swfw_sync != NULL) { 639 uint16_t mask; 640 641 /* 642 * Phy lock should not fail in this early stage. If this is the case, 643 * it is due to an improper exit of the application. 644 * So force the release of the faulty lock. 645 */ 646 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 647 if (hw->bus.func > E1000_FUNC_1) 648 mask <<= 2; 649 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 650 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 651 hw->bus.func); 652 } 653 hw->mac.ops.release_swfw_sync(hw, mask); 654 655 /* 656 * This one is more tricky since it is common to all ports; but 657 * swfw_sync retries last long enough (1s) to be almost sure that if 658 * lock can not be taken it is due to an improper lock of the 659 * semaphore. 660 */ 661 mask = E1000_SWFW_EEP_SM; 662 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 663 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 664 } 665 hw->mac.ops.release_swfw_sync(hw, mask); 666 } 667 668 return E1000_SUCCESS; 669 } 670 671 /* Remove all ntuple filters of the device */ 672 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 673 { 674 struct e1000_filter_info *filter_info = 675 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 676 struct e1000_5tuple_filter *p_5tuple; 677 struct e1000_2tuple_filter *p_2tuple; 678 679 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 680 TAILQ_REMOVE(&filter_info->fivetuple_list, 681 p_5tuple, entries); 682 rte_free(p_5tuple); 683 } 684 filter_info->fivetuple_mask = 0; 685 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 686 TAILQ_REMOVE(&filter_info->twotuple_list, 687 p_2tuple, entries); 688 rte_free(p_2tuple); 689 } 690 filter_info->twotuple_mask = 0; 691 692 return 0; 693 } 694 695 /* Remove all flex filters of the device */ 696 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 697 { 698 struct e1000_filter_info *filter_info = 699 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 700 struct e1000_flex_filter *p_flex; 701 702 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 703 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 704 rte_free(p_flex); 705 } 706 filter_info->flex_mask = 0; 707 708 return 0; 709 } 710 711 static int 712 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 713 { 714 int error = 0; 715 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 716 struct e1000_hw *hw = 717 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 718 struct e1000_vfta * shadow_vfta = 719 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 720 struct e1000_filter_info *filter_info = 721 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 722 struct e1000_adapter *adapter = 723 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 724 725 uint32_t ctrl_ext; 726 727 eth_dev->dev_ops = ð_igb_ops; 728 eth_dev->rx_queue_count = eth_igb_rx_queue_count; 729 eth_dev->rx_descriptor_done = eth_igb_rx_descriptor_done; 730 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 731 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 732 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 733 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 734 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 735 736 /* for secondary processes, we don't initialise any further as primary 737 * has already done this work. Only check we don't need a different 738 * RX function */ 739 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 740 if (eth_dev->data->scattered_rx) 741 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 742 return 0; 743 } 744 745 rte_eth_copy_pci_info(eth_dev, pci_dev); 746 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 747 748 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 749 750 igb_identify_hardware(eth_dev, pci_dev); 751 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 752 error = -EIO; 753 goto err_late; 754 } 755 756 e1000_get_bus_info(hw); 757 758 /* Reset any pending lock */ 759 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 760 error = -EIO; 761 goto err_late; 762 } 763 764 /* Finish initialization */ 765 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 766 error = -EIO; 767 goto err_late; 768 } 769 770 hw->mac.autoneg = 1; 771 hw->phy.autoneg_wait_to_complete = 0; 772 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 773 774 /* Copper options */ 775 if (hw->phy.media_type == e1000_media_type_copper) { 776 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 777 hw->phy.disable_polarity_correction = 0; 778 hw->phy.ms_type = e1000_ms_hw_default; 779 } 780 781 /* 782 * Start from a known state, this is important in reading the nvm 783 * and mac from that. 784 */ 785 igb_pf_reset_hw(hw); 786 787 /* Make sure we have a good EEPROM before we read from it */ 788 if (e1000_validate_nvm_checksum(hw) < 0) { 789 /* 790 * Some PCI-E parts fail the first check due to 791 * the link being in sleep state, call it again, 792 * if it fails a second time its a real issue. 793 */ 794 if (e1000_validate_nvm_checksum(hw) < 0) { 795 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 796 error = -EIO; 797 goto err_late; 798 } 799 } 800 801 /* Read the permanent MAC address out of the EEPROM */ 802 if (e1000_read_mac_addr(hw) != 0) { 803 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 804 error = -EIO; 805 goto err_late; 806 } 807 808 /* Allocate memory for storing MAC addresses */ 809 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 810 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 811 if (eth_dev->data->mac_addrs == NULL) { 812 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 813 "store MAC addresses", 814 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 815 error = -ENOMEM; 816 goto err_late; 817 } 818 819 /* Copy the permanent MAC address */ 820 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 821 ð_dev->data->mac_addrs[0]); 822 823 /* initialize the vfta */ 824 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 825 826 /* Now initialize the hardware */ 827 if (igb_hardware_init(hw) != 0) { 828 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 829 rte_free(eth_dev->data->mac_addrs); 830 eth_dev->data->mac_addrs = NULL; 831 error = -ENODEV; 832 goto err_late; 833 } 834 hw->mac.get_link_status = 1; 835 adapter->stopped = 0; 836 837 /* Indicate SOL/IDER usage */ 838 if (e1000_check_reset_block(hw) < 0) { 839 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 840 "SOL/IDER session"); 841 } 842 843 /* initialize PF if max_vfs not zero */ 844 igb_pf_host_init(eth_dev); 845 846 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 847 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 848 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 849 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 850 E1000_WRITE_FLUSH(hw); 851 852 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 853 eth_dev->data->port_id, pci_dev->id.vendor_id, 854 pci_dev->id.device_id); 855 856 rte_intr_callback_register(&pci_dev->intr_handle, 857 eth_igb_interrupt_handler, 858 (void *)eth_dev); 859 860 /* enable uio/vfio intr/eventfd mapping */ 861 rte_intr_enable(&pci_dev->intr_handle); 862 863 /* enable support intr */ 864 igb_intr_enable(eth_dev); 865 866 eth_igb_dev_set_link_down(eth_dev); 867 868 /* initialize filter info */ 869 memset(filter_info, 0, 870 sizeof(struct e1000_filter_info)); 871 872 TAILQ_INIT(&filter_info->flex_list); 873 TAILQ_INIT(&filter_info->twotuple_list); 874 TAILQ_INIT(&filter_info->fivetuple_list); 875 876 TAILQ_INIT(&igb_filter_ntuple_list); 877 TAILQ_INIT(&igb_filter_ethertype_list); 878 TAILQ_INIT(&igb_filter_syn_list); 879 TAILQ_INIT(&igb_filter_flex_list); 880 TAILQ_INIT(&igb_filter_rss_list); 881 TAILQ_INIT(&igb_flow_list); 882 883 return 0; 884 885 err_late: 886 igb_hw_control_release(hw); 887 888 return error; 889 } 890 891 static int 892 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 893 { 894 PMD_INIT_FUNC_TRACE(); 895 896 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 897 return 0; 898 899 eth_igb_close(eth_dev); 900 901 return 0; 902 } 903 904 /* 905 * Virtual Function device init 906 */ 907 static int 908 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 909 { 910 struct rte_pci_device *pci_dev; 911 struct rte_intr_handle *intr_handle; 912 struct e1000_adapter *adapter = 913 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 914 struct e1000_hw *hw = 915 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 916 int diag; 917 struct rte_ether_addr *perm_addr = 918 (struct rte_ether_addr *)hw->mac.perm_addr; 919 920 PMD_INIT_FUNC_TRACE(); 921 922 eth_dev->dev_ops = &igbvf_eth_dev_ops; 923 eth_dev->rx_descriptor_done = eth_igb_rx_descriptor_done; 924 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 925 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 926 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 927 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 928 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 929 930 /* for secondary processes, we don't initialise any further as primary 931 * has already done this work. Only check we don't need a different 932 * RX function */ 933 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 934 if (eth_dev->data->scattered_rx) 935 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 936 return 0; 937 } 938 939 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 940 rte_eth_copy_pci_info(eth_dev, pci_dev); 941 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 942 943 hw->device_id = pci_dev->id.device_id; 944 hw->vendor_id = pci_dev->id.vendor_id; 945 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 946 adapter->stopped = 0; 947 948 /* Initialize the shared code (base driver) */ 949 diag = e1000_setup_init_funcs(hw, TRUE); 950 if (diag != 0) { 951 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 952 diag); 953 return -EIO; 954 } 955 956 /* init_mailbox_params */ 957 hw->mbx.ops.init_params(hw); 958 959 /* Disable the interrupts for VF */ 960 igbvf_intr_disable(hw); 961 962 diag = hw->mac.ops.reset_hw(hw); 963 964 /* Allocate memory for storing MAC addresses */ 965 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", RTE_ETHER_ADDR_LEN * 966 hw->mac.rar_entry_count, 0); 967 if (eth_dev->data->mac_addrs == NULL) { 968 PMD_INIT_LOG(ERR, 969 "Failed to allocate %d bytes needed to store MAC " 970 "addresses", 971 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 972 return -ENOMEM; 973 } 974 975 /* Generate a random MAC address, if none was assigned by PF. */ 976 if (rte_is_zero_ether_addr(perm_addr)) { 977 rte_eth_random_addr(perm_addr->addr_bytes); 978 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 979 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 980 RTE_ETHER_ADDR_PRT_FMT, 981 RTE_ETHER_ADDR_BYTES(perm_addr)); 982 } 983 984 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 985 if (diag) { 986 rte_free(eth_dev->data->mac_addrs); 987 eth_dev->data->mac_addrs = NULL; 988 return diag; 989 } 990 /* Copy the permanent MAC address */ 991 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 992 ð_dev->data->mac_addrs[0]); 993 994 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 995 "mac.type=%s", 996 eth_dev->data->port_id, pci_dev->id.vendor_id, 997 pci_dev->id.device_id, "igb_mac_82576_vf"); 998 999 intr_handle = &pci_dev->intr_handle; 1000 rte_intr_callback_register(intr_handle, 1001 eth_igbvf_interrupt_handler, eth_dev); 1002 1003 return 0; 1004 } 1005 1006 static int 1007 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1008 { 1009 PMD_INIT_FUNC_TRACE(); 1010 1011 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1012 return 0; 1013 1014 igbvf_dev_close(eth_dev); 1015 1016 return 0; 1017 } 1018 1019 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1020 struct rte_pci_device *pci_dev) 1021 { 1022 return rte_eth_dev_pci_generic_probe(pci_dev, 1023 sizeof(struct e1000_adapter), eth_igb_dev_init); 1024 } 1025 1026 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1027 { 1028 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1029 } 1030 1031 static struct rte_pci_driver rte_igb_pmd = { 1032 .id_table = pci_id_igb_map, 1033 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1034 .probe = eth_igb_pci_probe, 1035 .remove = eth_igb_pci_remove, 1036 }; 1037 1038 1039 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1040 struct rte_pci_device *pci_dev) 1041 { 1042 return rte_eth_dev_pci_generic_probe(pci_dev, 1043 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1044 } 1045 1046 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1047 { 1048 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1049 } 1050 1051 /* 1052 * virtual function driver struct 1053 */ 1054 static struct rte_pci_driver rte_igbvf_pmd = { 1055 .id_table = pci_id_igbvf_map, 1056 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1057 .probe = eth_igbvf_pci_probe, 1058 .remove = eth_igbvf_pci_remove, 1059 }; 1060 1061 static void 1062 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1063 { 1064 struct e1000_hw *hw = 1065 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1066 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1067 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1068 rctl |= E1000_RCTL_VFE; 1069 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1070 } 1071 1072 static int 1073 igb_check_mq_mode(struct rte_eth_dev *dev) 1074 { 1075 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1076 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1077 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1078 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1079 1080 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1081 tx_mq_mode == ETH_MQ_TX_DCB || 1082 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1083 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1084 return -EINVAL; 1085 } 1086 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1087 /* Check multi-queue mode. 1088 * To no break software we accept ETH_MQ_RX_NONE as this might 1089 * be used to turn off VLAN filter. 1090 */ 1091 1092 if (rx_mq_mode == ETH_MQ_RX_NONE || 1093 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1094 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1095 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1096 } else { 1097 /* Only support one queue on VFs. 1098 * RSS together with SRIOV is not supported. 1099 */ 1100 PMD_INIT_LOG(ERR, "SRIOV is active," 1101 " wrong mq_mode rx %d.", 1102 rx_mq_mode); 1103 return -EINVAL; 1104 } 1105 /* TX mode is not used here, so mode might be ignored.*/ 1106 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1107 /* SRIOV only works in VMDq enable mode */ 1108 PMD_INIT_LOG(WARNING, "SRIOV is active," 1109 " TX mode %d is not supported. " 1110 " Driver will behave as %d mode.", 1111 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1112 } 1113 1114 /* check valid queue number */ 1115 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1116 PMD_INIT_LOG(ERR, "SRIOV is active," 1117 " only support one queue on VFs."); 1118 return -EINVAL; 1119 } 1120 } else { 1121 /* To no break software that set invalid mode, only display 1122 * warning if invalid mode is used. 1123 */ 1124 if (rx_mq_mode != ETH_MQ_RX_NONE && 1125 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1126 rx_mq_mode != ETH_MQ_RX_RSS) { 1127 /* RSS together with VMDq not supported*/ 1128 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1129 rx_mq_mode); 1130 return -EINVAL; 1131 } 1132 1133 if (tx_mq_mode != ETH_MQ_TX_NONE && 1134 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1135 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1136 " Due to txmode is meaningless in this" 1137 " driver, just ignore.", 1138 tx_mq_mode); 1139 } 1140 } 1141 return 0; 1142 } 1143 1144 static int 1145 eth_igb_configure(struct rte_eth_dev *dev) 1146 { 1147 struct e1000_interrupt *intr = 1148 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1149 int ret; 1150 1151 PMD_INIT_FUNC_TRACE(); 1152 1153 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 1154 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1155 1156 /* multipe queue mode checking */ 1157 ret = igb_check_mq_mode(dev); 1158 if (ret != 0) { 1159 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1160 ret); 1161 return ret; 1162 } 1163 1164 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1165 PMD_INIT_FUNC_TRACE(); 1166 1167 return 0; 1168 } 1169 1170 static void 1171 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1172 bool enable) 1173 { 1174 struct e1000_hw *hw = 1175 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1176 uint32_t tctl, rctl; 1177 1178 tctl = E1000_READ_REG(hw, E1000_TCTL); 1179 rctl = E1000_READ_REG(hw, E1000_RCTL); 1180 1181 if (enable) { 1182 /* enable Tx/Rx */ 1183 tctl |= E1000_TCTL_EN; 1184 rctl |= E1000_RCTL_EN; 1185 } else { 1186 /* disable Tx/Rx */ 1187 tctl &= ~E1000_TCTL_EN; 1188 rctl &= ~E1000_RCTL_EN; 1189 } 1190 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1191 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1192 E1000_WRITE_FLUSH(hw); 1193 } 1194 1195 static int 1196 eth_igb_start(struct rte_eth_dev *dev) 1197 { 1198 struct e1000_hw *hw = 1199 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1200 struct e1000_adapter *adapter = 1201 E1000_DEV_PRIVATE(dev->data->dev_private); 1202 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1203 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1204 int ret, mask; 1205 uint32_t intr_vector = 0; 1206 uint32_t ctrl_ext; 1207 uint32_t *speeds; 1208 int num_speeds; 1209 bool autoneg; 1210 1211 PMD_INIT_FUNC_TRACE(); 1212 1213 /* disable uio/vfio intr/eventfd mapping */ 1214 rte_intr_disable(intr_handle); 1215 1216 /* Power up the phy. Needed to make the link go Up */ 1217 eth_igb_dev_set_link_up(dev); 1218 1219 /* 1220 * Packet Buffer Allocation (PBA) 1221 * Writing PBA sets the receive portion of the buffer 1222 * the remainder is used for the transmit buffer. 1223 */ 1224 if (hw->mac.type == e1000_82575) { 1225 uint32_t pba; 1226 1227 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1228 E1000_WRITE_REG(hw, E1000_PBA, pba); 1229 } 1230 1231 /* Put the address into the Receive Address Array */ 1232 e1000_rar_set(hw, hw->mac.addr, 0); 1233 1234 /* Initialize the hardware */ 1235 if (igb_hardware_init(hw)) { 1236 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1237 return -EIO; 1238 } 1239 adapter->stopped = 0; 1240 1241 E1000_WRITE_REG(hw, E1000_VET, 1242 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1243 1244 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1245 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1246 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1247 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1248 E1000_WRITE_FLUSH(hw); 1249 1250 /* configure PF module if SRIOV enabled */ 1251 igb_pf_host_configure(dev); 1252 1253 /* check and configure queue intr-vector mapping */ 1254 if ((rte_intr_cap_multiple(intr_handle) || 1255 !RTE_ETH_DEV_SRIOV(dev).active) && 1256 dev->data->dev_conf.intr_conf.rxq != 0) { 1257 intr_vector = dev->data->nb_rx_queues; 1258 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1259 return -1; 1260 } 1261 1262 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1263 intr_handle->intr_vec = 1264 rte_zmalloc("intr_vec", 1265 dev->data->nb_rx_queues * sizeof(int), 0); 1266 if (intr_handle->intr_vec == NULL) { 1267 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1268 " intr_vec", dev->data->nb_rx_queues); 1269 return -ENOMEM; 1270 } 1271 } 1272 1273 /* confiugre msix for rx interrupt */ 1274 eth_igb_configure_msix_intr(dev); 1275 1276 /* Configure for OS presence */ 1277 igb_init_manageability(hw); 1278 1279 eth_igb_tx_init(dev); 1280 1281 /* This can fail when allocating mbufs for descriptor rings */ 1282 ret = eth_igb_rx_init(dev); 1283 if (ret) { 1284 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1285 igb_dev_clear_queues(dev); 1286 return ret; 1287 } 1288 1289 e1000_clear_hw_cntrs_base_generic(hw); 1290 1291 /* 1292 * VLAN Offload Settings 1293 */ 1294 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1295 ETH_VLAN_EXTEND_MASK; 1296 ret = eth_igb_vlan_offload_set(dev, mask); 1297 if (ret) { 1298 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1299 igb_dev_clear_queues(dev); 1300 return ret; 1301 } 1302 1303 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1304 /* Enable VLAN filter since VMDq always use VLAN filter */ 1305 igb_vmdq_vlan_hw_filter_enable(dev); 1306 } 1307 1308 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1309 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1310 (hw->mac.type == e1000_i211)) { 1311 /* Configure EITR with the maximum possible value (0xFFFF) */ 1312 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1313 } 1314 1315 /* Setup link speed and duplex */ 1316 speeds = &dev->data->dev_conf.link_speeds; 1317 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1318 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1319 hw->mac.autoneg = 1; 1320 } else { 1321 num_speeds = 0; 1322 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1323 1324 /* Reset */ 1325 hw->phy.autoneg_advertised = 0; 1326 1327 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1328 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1329 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1330 num_speeds = -1; 1331 goto error_invalid_config; 1332 } 1333 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1334 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1335 num_speeds++; 1336 } 1337 if (*speeds & ETH_LINK_SPEED_10M) { 1338 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1339 num_speeds++; 1340 } 1341 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1342 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1343 num_speeds++; 1344 } 1345 if (*speeds & ETH_LINK_SPEED_100M) { 1346 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1347 num_speeds++; 1348 } 1349 if (*speeds & ETH_LINK_SPEED_1G) { 1350 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1351 num_speeds++; 1352 } 1353 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1354 goto error_invalid_config; 1355 1356 /* Set/reset the mac.autoneg based on the link speed, 1357 * fixed or not 1358 */ 1359 if (!autoneg) { 1360 hw->mac.autoneg = 0; 1361 hw->mac.forced_speed_duplex = 1362 hw->phy.autoneg_advertised; 1363 } else { 1364 hw->mac.autoneg = 1; 1365 } 1366 } 1367 1368 e1000_setup_link(hw); 1369 1370 if (rte_intr_allow_others(intr_handle)) { 1371 /* check if lsc interrupt is enabled */ 1372 if (dev->data->dev_conf.intr_conf.lsc != 0) 1373 eth_igb_lsc_interrupt_setup(dev, TRUE); 1374 else 1375 eth_igb_lsc_interrupt_setup(dev, FALSE); 1376 } else { 1377 rte_intr_callback_unregister(intr_handle, 1378 eth_igb_interrupt_handler, 1379 (void *)dev); 1380 if (dev->data->dev_conf.intr_conf.lsc != 0) 1381 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1382 " no intr multiplex"); 1383 } 1384 1385 /* check if rxq interrupt is enabled */ 1386 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1387 rte_intr_dp_is_en(intr_handle)) 1388 eth_igb_rxq_interrupt_setup(dev); 1389 1390 /* enable uio/vfio intr/eventfd mapping */ 1391 rte_intr_enable(intr_handle); 1392 1393 /* resume enabled intr since hw reset */ 1394 igb_intr_enable(dev); 1395 1396 /* restore all types filter */ 1397 igb_filter_restore(dev); 1398 1399 eth_igb_rxtx_control(dev, true); 1400 eth_igb_link_update(dev, 0); 1401 1402 PMD_INIT_LOG(DEBUG, "<<"); 1403 1404 return 0; 1405 1406 error_invalid_config: 1407 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1408 dev->data->dev_conf.link_speeds, dev->data->port_id); 1409 igb_dev_clear_queues(dev); 1410 return -EINVAL; 1411 } 1412 1413 /********************************************************************* 1414 * 1415 * This routine disables all traffic on the adapter by issuing a 1416 * global reset on the MAC. 1417 * 1418 **********************************************************************/ 1419 static int 1420 eth_igb_stop(struct rte_eth_dev *dev) 1421 { 1422 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1423 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1424 struct rte_eth_link link; 1425 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1426 struct e1000_adapter *adapter = 1427 E1000_DEV_PRIVATE(dev->data->dev_private); 1428 1429 if (adapter->stopped) 1430 return 0; 1431 1432 eth_igb_rxtx_control(dev, false); 1433 1434 igb_intr_disable(dev); 1435 1436 /* disable intr eventfd mapping */ 1437 rte_intr_disable(intr_handle); 1438 1439 igb_pf_reset_hw(hw); 1440 E1000_WRITE_REG(hw, E1000_WUC, 0); 1441 1442 /* Set bit for Go Link disconnect if PHY reset is not blocked */ 1443 if (hw->mac.type >= e1000_82580 && 1444 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1445 uint32_t phpm_reg; 1446 1447 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1448 phpm_reg |= E1000_82580_PM_GO_LINKD; 1449 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1450 } 1451 1452 /* Power down the phy. Needed to make the link go Down */ 1453 eth_igb_dev_set_link_down(dev); 1454 1455 igb_dev_clear_queues(dev); 1456 1457 /* clear the recorded link status */ 1458 memset(&link, 0, sizeof(link)); 1459 rte_eth_linkstatus_set(dev, &link); 1460 1461 if (!rte_intr_allow_others(intr_handle)) 1462 /* resume to the default handler */ 1463 rte_intr_callback_register(intr_handle, 1464 eth_igb_interrupt_handler, 1465 (void *)dev); 1466 1467 /* Clean datapath event and queue/vec mapping */ 1468 rte_intr_efd_disable(intr_handle); 1469 if (intr_handle->intr_vec != NULL) { 1470 rte_free(intr_handle->intr_vec); 1471 intr_handle->intr_vec = NULL; 1472 } 1473 1474 adapter->stopped = true; 1475 dev->data->dev_started = 0; 1476 1477 return 0; 1478 } 1479 1480 static int 1481 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1482 { 1483 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1484 1485 if (hw->phy.media_type == e1000_media_type_copper) 1486 e1000_power_up_phy(hw); 1487 else 1488 e1000_power_up_fiber_serdes_link(hw); 1489 1490 return 0; 1491 } 1492 1493 static int 1494 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1495 { 1496 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1497 1498 if (hw->phy.media_type == e1000_media_type_copper) 1499 e1000_power_down_phy(hw); 1500 else 1501 e1000_shutdown_fiber_serdes_link(hw); 1502 1503 return 0; 1504 } 1505 1506 static int 1507 eth_igb_close(struct rte_eth_dev *dev) 1508 { 1509 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1510 struct rte_eth_link link; 1511 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1512 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1513 struct e1000_filter_info *filter_info = 1514 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 1515 int ret; 1516 1517 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1518 return 0; 1519 1520 ret = eth_igb_stop(dev); 1521 1522 e1000_phy_hw_reset(hw); 1523 igb_release_manageability(hw); 1524 igb_hw_control_release(hw); 1525 1526 /* Clear bit for Go Link disconnect if PHY reset is not blocked */ 1527 if (hw->mac.type >= e1000_82580 && 1528 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1529 uint32_t phpm_reg; 1530 1531 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1532 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1533 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1534 } 1535 1536 igb_dev_free_queues(dev); 1537 1538 if (intr_handle->intr_vec) { 1539 rte_free(intr_handle->intr_vec); 1540 intr_handle->intr_vec = NULL; 1541 } 1542 1543 memset(&link, 0, sizeof(link)); 1544 rte_eth_linkstatus_set(dev, &link); 1545 1546 /* Reset any pending lock */ 1547 igb_reset_swfw_lock(hw); 1548 1549 /* uninitialize PF if max_vfs not zero */ 1550 igb_pf_host_uninit(dev); 1551 1552 rte_intr_callback_unregister(intr_handle, 1553 eth_igb_interrupt_handler, dev); 1554 1555 /* clear the SYN filter info */ 1556 filter_info->syn_info = 0; 1557 1558 /* clear the ethertype filters info */ 1559 filter_info->ethertype_mask = 0; 1560 memset(filter_info->ethertype_filters, 0, 1561 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 1562 1563 /* clear the rss filter info */ 1564 memset(&filter_info->rss_info, 0, 1565 sizeof(struct igb_rte_flow_rss_conf)); 1566 1567 /* remove all ntuple filters of the device */ 1568 igb_ntuple_filter_uninit(dev); 1569 1570 /* remove all flex filters of the device */ 1571 igb_flex_filter_uninit(dev); 1572 1573 /* clear all the filters list */ 1574 igb_filterlist_flush(dev); 1575 1576 return ret; 1577 } 1578 1579 /* 1580 * Reset PF device. 1581 */ 1582 static int 1583 eth_igb_reset(struct rte_eth_dev *dev) 1584 { 1585 int ret; 1586 1587 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1588 * its VF to make them align with it. The detailed notification 1589 * mechanism is PMD specific and is currently not implemented. 1590 * To avoid unexpected behavior in VF, currently reset of PF with 1591 * SR-IOV activation is not supported. It might be supported later. 1592 */ 1593 if (dev->data->sriov.active) 1594 return -ENOTSUP; 1595 1596 ret = eth_igb_dev_uninit(dev); 1597 if (ret) 1598 return ret; 1599 1600 ret = eth_igb_dev_init(dev); 1601 1602 return ret; 1603 } 1604 1605 1606 static int 1607 igb_get_rx_buffer_size(struct e1000_hw *hw) 1608 { 1609 uint32_t rx_buf_size; 1610 if (hw->mac.type == e1000_82576) { 1611 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1612 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1613 /* PBS needs to be translated according to a lookup table */ 1614 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1615 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1616 rx_buf_size = (rx_buf_size << 10); 1617 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1618 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1619 } else { 1620 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1621 } 1622 1623 return rx_buf_size; 1624 } 1625 1626 /********************************************************************* 1627 * 1628 * Initialize the hardware 1629 * 1630 **********************************************************************/ 1631 static int 1632 igb_hardware_init(struct e1000_hw *hw) 1633 { 1634 uint32_t rx_buf_size; 1635 int diag; 1636 1637 /* Let the firmware know the OS is in control */ 1638 igb_hw_control_acquire(hw); 1639 1640 /* 1641 * These parameters control the automatic generation (Tx) and 1642 * response (Rx) to Ethernet PAUSE frames. 1643 * - High water mark should allow for at least two standard size (1518) 1644 * frames to be received after sending an XOFF. 1645 * - Low water mark works best when it is very near the high water mark. 1646 * This allows the receiver to restart by sending XON when it has 1647 * drained a bit. Here we use an arbitrary value of 1500 which will 1648 * restart after one full frame is pulled from the buffer. There 1649 * could be several smaller frames in the buffer and if so they will 1650 * not trigger the XON until their total number reduces the buffer 1651 * by 1500. 1652 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1653 */ 1654 rx_buf_size = igb_get_rx_buffer_size(hw); 1655 1656 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2); 1657 hw->fc.low_water = hw->fc.high_water - 1500; 1658 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1659 hw->fc.send_xon = 1; 1660 1661 /* Set Flow control, use the tunable location if sane */ 1662 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1663 hw->fc.requested_mode = igb_fc_setting; 1664 else 1665 hw->fc.requested_mode = e1000_fc_none; 1666 1667 /* Issue a global reset */ 1668 igb_pf_reset_hw(hw); 1669 E1000_WRITE_REG(hw, E1000_WUC, 0); 1670 1671 diag = e1000_init_hw(hw); 1672 if (diag < 0) 1673 return diag; 1674 1675 E1000_WRITE_REG(hw, E1000_VET, 1676 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1677 e1000_get_phy_info(hw); 1678 e1000_check_for_link(hw); 1679 1680 return 0; 1681 } 1682 1683 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1684 static void 1685 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1686 { 1687 int pause_frames; 1688 1689 uint64_t old_gprc = stats->gprc; 1690 uint64_t old_gptc = stats->gptc; 1691 uint64_t old_tpr = stats->tpr; 1692 uint64_t old_tpt = stats->tpt; 1693 uint64_t old_rpthc = stats->rpthc; 1694 uint64_t old_hgptc = stats->hgptc; 1695 1696 if(hw->phy.media_type == e1000_media_type_copper || 1697 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1698 stats->symerrs += 1699 E1000_READ_REG(hw,E1000_SYMERRS); 1700 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1701 } 1702 1703 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1704 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1705 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1706 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1707 1708 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1709 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1710 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1711 stats->dc += E1000_READ_REG(hw, E1000_DC); 1712 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1713 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1714 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1715 /* 1716 ** For watchdog management we need to know if we have been 1717 ** paused during the last interval, so capture that here. 1718 */ 1719 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1720 stats->xoffrxc += pause_frames; 1721 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1722 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1723 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1724 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1725 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1726 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1727 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1728 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1729 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1730 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1731 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1732 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1733 1734 /* For the 64-bit byte counters the low dword must be read first. */ 1735 /* Both registers clear on the read of the high dword */ 1736 1737 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1738 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1739 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1740 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN; 1741 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1742 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1743 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN; 1744 1745 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1746 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1747 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1748 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1749 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1750 1751 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1752 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1753 1754 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1755 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1756 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 1757 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1758 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1759 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN; 1760 1761 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1762 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1763 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1764 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1765 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1766 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1767 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1768 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1769 1770 /* Interrupt Counts */ 1771 1772 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1773 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1774 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1775 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1776 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1777 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1778 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1779 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1780 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1781 1782 /* Host to Card Statistics */ 1783 1784 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1785 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1786 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1787 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1788 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1789 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1790 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1791 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1792 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1793 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN; 1794 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1795 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1796 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN; 1797 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1798 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1799 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1800 1801 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1802 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1803 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1804 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1805 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1806 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1807 } 1808 1809 static int 1810 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1811 { 1812 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1813 struct e1000_hw_stats *stats = 1814 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1815 1816 igb_read_stats_registers(hw, stats); 1817 1818 if (rte_stats == NULL) 1819 return -EINVAL; 1820 1821 /* Rx Errors */ 1822 rte_stats->imissed = stats->mpc; 1823 rte_stats->ierrors = stats->crcerrs + stats->rlec + 1824 stats->rxerrc + stats->algnerrc + stats->cexterr; 1825 1826 /* Tx Errors */ 1827 rte_stats->oerrors = stats->ecol + stats->latecol; 1828 1829 rte_stats->ipackets = stats->gprc; 1830 rte_stats->opackets = stats->gptc; 1831 rte_stats->ibytes = stats->gorc; 1832 rte_stats->obytes = stats->gotc; 1833 return 0; 1834 } 1835 1836 static int 1837 eth_igb_stats_reset(struct rte_eth_dev *dev) 1838 { 1839 struct e1000_hw_stats *hw_stats = 1840 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1841 1842 /* HW registers are cleared on read */ 1843 eth_igb_stats_get(dev, NULL); 1844 1845 /* Reset software totals */ 1846 memset(hw_stats, 0, sizeof(*hw_stats)); 1847 1848 return 0; 1849 } 1850 1851 static int 1852 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1853 { 1854 struct e1000_hw_stats *stats = 1855 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1856 1857 /* HW registers are cleared on read */ 1858 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1859 1860 /* Reset software totals */ 1861 memset(stats, 0, sizeof(*stats)); 1862 1863 return 0; 1864 } 1865 1866 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1867 struct rte_eth_xstat_name *xstats_names, 1868 __rte_unused unsigned int size) 1869 { 1870 unsigned i; 1871 1872 if (xstats_names == NULL) 1873 return IGB_NB_XSTATS; 1874 1875 /* Note: limit checked in rte_eth_xstats_names() */ 1876 1877 for (i = 0; i < IGB_NB_XSTATS; i++) { 1878 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1879 sizeof(xstats_names[i].name)); 1880 } 1881 1882 return IGB_NB_XSTATS; 1883 } 1884 1885 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1886 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1887 unsigned int limit) 1888 { 1889 unsigned int i; 1890 1891 if (!ids) { 1892 if (xstats_names == NULL) 1893 return IGB_NB_XSTATS; 1894 1895 for (i = 0; i < IGB_NB_XSTATS; i++) 1896 strlcpy(xstats_names[i].name, 1897 rte_igb_stats_strings[i].name, 1898 sizeof(xstats_names[i].name)); 1899 1900 return IGB_NB_XSTATS; 1901 1902 } else { 1903 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1904 1905 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1906 IGB_NB_XSTATS); 1907 1908 for (i = 0; i < limit; i++) { 1909 if (ids[i] >= IGB_NB_XSTATS) { 1910 PMD_INIT_LOG(ERR, "id value isn't valid"); 1911 return -1; 1912 } 1913 strcpy(xstats_names[i].name, 1914 xstats_names_copy[ids[i]].name); 1915 } 1916 return limit; 1917 } 1918 } 1919 1920 static int 1921 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1922 unsigned n) 1923 { 1924 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1925 struct e1000_hw_stats *hw_stats = 1926 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1927 unsigned i; 1928 1929 if (n < IGB_NB_XSTATS) 1930 return IGB_NB_XSTATS; 1931 1932 igb_read_stats_registers(hw, hw_stats); 1933 1934 /* If this is a reset xstats is NULL, and we have cleared the 1935 * registers by reading them. 1936 */ 1937 if (!xstats) 1938 return 0; 1939 1940 /* Extended stats */ 1941 for (i = 0; i < IGB_NB_XSTATS; i++) { 1942 xstats[i].id = i; 1943 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1944 rte_igb_stats_strings[i].offset); 1945 } 1946 1947 return IGB_NB_XSTATS; 1948 } 1949 1950 static int 1951 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1952 uint64_t *values, unsigned int n) 1953 { 1954 unsigned int i; 1955 1956 if (!ids) { 1957 struct e1000_hw *hw = 1958 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1959 struct e1000_hw_stats *hw_stats = 1960 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1961 1962 if (n < IGB_NB_XSTATS) 1963 return IGB_NB_XSTATS; 1964 1965 igb_read_stats_registers(hw, hw_stats); 1966 1967 /* If this is a reset xstats is NULL, and we have cleared the 1968 * registers by reading them. 1969 */ 1970 if (!values) 1971 return 0; 1972 1973 /* Extended stats */ 1974 for (i = 0; i < IGB_NB_XSTATS; i++) 1975 values[i] = *(uint64_t *)(((char *)hw_stats) + 1976 rte_igb_stats_strings[i].offset); 1977 1978 return IGB_NB_XSTATS; 1979 1980 } else { 1981 uint64_t values_copy[IGB_NB_XSTATS]; 1982 1983 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 1984 IGB_NB_XSTATS); 1985 1986 for (i = 0; i < n; i++) { 1987 if (ids[i] >= IGB_NB_XSTATS) { 1988 PMD_INIT_LOG(ERR, "id value isn't valid"); 1989 return -1; 1990 } 1991 values[i] = values_copy[ids[i]]; 1992 } 1993 return n; 1994 } 1995 } 1996 1997 static void 1998 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 1999 { 2000 /* Good Rx packets, include VF loopback */ 2001 UPDATE_VF_STAT(E1000_VFGPRC, 2002 hw_stats->last_gprc, hw_stats->gprc); 2003 2004 /* Good Rx octets, include VF loopback */ 2005 UPDATE_VF_STAT(E1000_VFGORC, 2006 hw_stats->last_gorc, hw_stats->gorc); 2007 2008 /* Good Tx packets, include VF loopback */ 2009 UPDATE_VF_STAT(E1000_VFGPTC, 2010 hw_stats->last_gptc, hw_stats->gptc); 2011 2012 /* Good Tx octets, include VF loopback */ 2013 UPDATE_VF_STAT(E1000_VFGOTC, 2014 hw_stats->last_gotc, hw_stats->gotc); 2015 2016 /* Rx Multicst packets */ 2017 UPDATE_VF_STAT(E1000_VFMPRC, 2018 hw_stats->last_mprc, hw_stats->mprc); 2019 2020 /* Good Rx loopback packets */ 2021 UPDATE_VF_STAT(E1000_VFGPRLBC, 2022 hw_stats->last_gprlbc, hw_stats->gprlbc); 2023 2024 /* Good Rx loopback octets */ 2025 UPDATE_VF_STAT(E1000_VFGORLBC, 2026 hw_stats->last_gorlbc, hw_stats->gorlbc); 2027 2028 /* Good Tx loopback packets */ 2029 UPDATE_VF_STAT(E1000_VFGPTLBC, 2030 hw_stats->last_gptlbc, hw_stats->gptlbc); 2031 2032 /* Good Tx loopback octets */ 2033 UPDATE_VF_STAT(E1000_VFGOTLBC, 2034 hw_stats->last_gotlbc, hw_stats->gotlbc); 2035 } 2036 2037 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2038 struct rte_eth_xstat_name *xstats_names, 2039 __rte_unused unsigned limit) 2040 { 2041 unsigned i; 2042 2043 if (xstats_names != NULL) 2044 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2045 strlcpy(xstats_names[i].name, 2046 rte_igbvf_stats_strings[i].name, 2047 sizeof(xstats_names[i].name)); 2048 } 2049 return IGBVF_NB_XSTATS; 2050 } 2051 2052 static int 2053 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2054 unsigned n) 2055 { 2056 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2057 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2058 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2059 unsigned i; 2060 2061 if (n < IGBVF_NB_XSTATS) 2062 return IGBVF_NB_XSTATS; 2063 2064 igbvf_read_stats_registers(hw, hw_stats); 2065 2066 if (!xstats) 2067 return 0; 2068 2069 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2070 xstats[i].id = i; 2071 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2072 rte_igbvf_stats_strings[i].offset); 2073 } 2074 2075 return IGBVF_NB_XSTATS; 2076 } 2077 2078 static int 2079 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2080 { 2081 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2082 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2083 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2084 2085 igbvf_read_stats_registers(hw, hw_stats); 2086 2087 if (rte_stats == NULL) 2088 return -EINVAL; 2089 2090 rte_stats->ipackets = hw_stats->gprc; 2091 rte_stats->ibytes = hw_stats->gorc; 2092 rte_stats->opackets = hw_stats->gptc; 2093 rte_stats->obytes = hw_stats->gotc; 2094 return 0; 2095 } 2096 2097 static int 2098 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2099 { 2100 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2101 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2102 2103 /* Sync HW register to the last stats */ 2104 eth_igbvf_stats_get(dev, NULL); 2105 2106 /* reset HW current stats*/ 2107 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2108 offsetof(struct e1000_vf_stats, gprc)); 2109 2110 return 0; 2111 } 2112 2113 static int 2114 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2115 size_t fw_size) 2116 { 2117 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2118 struct e1000_fw_version fw; 2119 int ret; 2120 2121 e1000_get_fw_version(hw, &fw); 2122 2123 switch (hw->mac.type) { 2124 case e1000_i210: 2125 case e1000_i211: 2126 if (!(e1000_get_flash_presence_i210(hw))) { 2127 ret = snprintf(fw_version, fw_size, 2128 "%2d.%2d-%d", 2129 fw.invm_major, fw.invm_minor, 2130 fw.invm_img_type); 2131 break; 2132 } 2133 /* fall through */ 2134 default: 2135 /* if option rom is valid, display its version too */ 2136 if (fw.or_valid) { 2137 ret = snprintf(fw_version, fw_size, 2138 "%d.%d, 0x%08x, %d.%d.%d", 2139 fw.eep_major, fw.eep_minor, fw.etrack_id, 2140 fw.or_major, fw.or_build, fw.or_patch); 2141 /* no option rom */ 2142 } else { 2143 if (fw.etrack_id != 0X0000) { 2144 ret = snprintf(fw_version, fw_size, 2145 "%d.%d, 0x%08x", 2146 fw.eep_major, fw.eep_minor, 2147 fw.etrack_id); 2148 } else { 2149 ret = snprintf(fw_version, fw_size, 2150 "%d.%d.%d", 2151 fw.eep_major, fw.eep_minor, 2152 fw.eep_build); 2153 } 2154 } 2155 break; 2156 } 2157 if (ret < 0) 2158 return -EINVAL; 2159 2160 ret += 1; /* add the size of '\0' */ 2161 if (fw_size < (size_t)ret) 2162 return ret; 2163 else 2164 return 0; 2165 } 2166 2167 static int 2168 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2169 { 2170 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2171 2172 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2173 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2174 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2175 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2176 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2177 dev_info->rx_queue_offload_capa; 2178 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2179 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2180 dev_info->tx_queue_offload_capa; 2181 2182 switch (hw->mac.type) { 2183 case e1000_82575: 2184 dev_info->max_rx_queues = 4; 2185 dev_info->max_tx_queues = 4; 2186 dev_info->max_vmdq_pools = 0; 2187 break; 2188 2189 case e1000_82576: 2190 dev_info->max_rx_queues = 16; 2191 dev_info->max_tx_queues = 16; 2192 dev_info->max_vmdq_pools = ETH_8_POOLS; 2193 dev_info->vmdq_queue_num = 16; 2194 break; 2195 2196 case e1000_82580: 2197 dev_info->max_rx_queues = 8; 2198 dev_info->max_tx_queues = 8; 2199 dev_info->max_vmdq_pools = ETH_8_POOLS; 2200 dev_info->vmdq_queue_num = 8; 2201 break; 2202 2203 case e1000_i350: 2204 dev_info->max_rx_queues = 8; 2205 dev_info->max_tx_queues = 8; 2206 dev_info->max_vmdq_pools = ETH_8_POOLS; 2207 dev_info->vmdq_queue_num = 8; 2208 break; 2209 2210 case e1000_i354: 2211 dev_info->max_rx_queues = 8; 2212 dev_info->max_tx_queues = 8; 2213 break; 2214 2215 case e1000_i210: 2216 dev_info->max_rx_queues = 4; 2217 dev_info->max_tx_queues = 4; 2218 dev_info->max_vmdq_pools = 0; 2219 break; 2220 2221 case e1000_i211: 2222 dev_info->max_rx_queues = 2; 2223 dev_info->max_tx_queues = 2; 2224 dev_info->max_vmdq_pools = 0; 2225 break; 2226 2227 default: 2228 /* Should not happen */ 2229 return -EINVAL; 2230 } 2231 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2232 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2233 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2234 2235 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2236 .rx_thresh = { 2237 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2238 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2239 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2240 }, 2241 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2242 .rx_drop_en = 0, 2243 .offloads = 0, 2244 }; 2245 2246 dev_info->default_txconf = (struct rte_eth_txconf) { 2247 .tx_thresh = { 2248 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2249 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2250 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2251 }, 2252 .offloads = 0, 2253 }; 2254 2255 dev_info->rx_desc_lim = rx_desc_lim; 2256 dev_info->tx_desc_lim = tx_desc_lim; 2257 2258 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2259 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2260 ETH_LINK_SPEED_1G; 2261 2262 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2263 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 2264 2265 return 0; 2266 } 2267 2268 static const uint32_t * 2269 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2270 { 2271 static const uint32_t ptypes[] = { 2272 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2273 RTE_PTYPE_L2_ETHER, 2274 RTE_PTYPE_L3_IPV4, 2275 RTE_PTYPE_L3_IPV4_EXT, 2276 RTE_PTYPE_L3_IPV6, 2277 RTE_PTYPE_L3_IPV6_EXT, 2278 RTE_PTYPE_L4_TCP, 2279 RTE_PTYPE_L4_UDP, 2280 RTE_PTYPE_L4_SCTP, 2281 RTE_PTYPE_TUNNEL_IP, 2282 RTE_PTYPE_INNER_L3_IPV6, 2283 RTE_PTYPE_INNER_L3_IPV6_EXT, 2284 RTE_PTYPE_INNER_L4_TCP, 2285 RTE_PTYPE_INNER_L4_UDP, 2286 RTE_PTYPE_UNKNOWN 2287 }; 2288 2289 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2290 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2291 return ptypes; 2292 return NULL; 2293 } 2294 2295 static int 2296 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2297 { 2298 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2299 2300 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2301 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2302 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2303 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2304 DEV_TX_OFFLOAD_IPV4_CKSUM | 2305 DEV_TX_OFFLOAD_UDP_CKSUM | 2306 DEV_TX_OFFLOAD_TCP_CKSUM | 2307 DEV_TX_OFFLOAD_SCTP_CKSUM | 2308 DEV_TX_OFFLOAD_TCP_TSO; 2309 switch (hw->mac.type) { 2310 case e1000_vfadapt: 2311 dev_info->max_rx_queues = 2; 2312 dev_info->max_tx_queues = 2; 2313 break; 2314 case e1000_vfadapt_i350: 2315 dev_info->max_rx_queues = 1; 2316 dev_info->max_tx_queues = 1; 2317 break; 2318 default: 2319 /* Should not happen */ 2320 return -EINVAL; 2321 } 2322 2323 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2324 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2325 dev_info->rx_queue_offload_capa; 2326 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2327 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2328 dev_info->tx_queue_offload_capa; 2329 2330 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2331 .rx_thresh = { 2332 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2333 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2334 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2335 }, 2336 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2337 .rx_drop_en = 0, 2338 .offloads = 0, 2339 }; 2340 2341 dev_info->default_txconf = (struct rte_eth_txconf) { 2342 .tx_thresh = { 2343 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2344 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2345 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2346 }, 2347 .offloads = 0, 2348 }; 2349 2350 dev_info->rx_desc_lim = rx_desc_lim; 2351 dev_info->tx_desc_lim = tx_desc_lim; 2352 2353 return 0; 2354 } 2355 2356 /* return 0 means link status changed, -1 means not changed */ 2357 static int 2358 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2359 { 2360 struct e1000_hw *hw = 2361 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2362 struct rte_eth_link link; 2363 int link_check, count; 2364 2365 link_check = 0; 2366 hw->mac.get_link_status = 1; 2367 2368 /* possible wait-to-complete in up to 9 seconds */ 2369 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2370 /* Read the real link status */ 2371 switch (hw->phy.media_type) { 2372 case e1000_media_type_copper: 2373 /* Do the work to read phy */ 2374 e1000_check_for_link(hw); 2375 link_check = !hw->mac.get_link_status; 2376 break; 2377 2378 case e1000_media_type_fiber: 2379 e1000_check_for_link(hw); 2380 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2381 E1000_STATUS_LU); 2382 break; 2383 2384 case e1000_media_type_internal_serdes: 2385 e1000_check_for_link(hw); 2386 link_check = hw->mac.serdes_has_link; 2387 break; 2388 2389 /* VF device is type_unknown */ 2390 case e1000_media_type_unknown: 2391 eth_igbvf_link_update(hw); 2392 link_check = !hw->mac.get_link_status; 2393 break; 2394 2395 default: 2396 break; 2397 } 2398 if (link_check || wait_to_complete == 0) 2399 break; 2400 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2401 } 2402 memset(&link, 0, sizeof(link)); 2403 2404 /* Now we check if a transition has happened */ 2405 if (link_check) { 2406 uint16_t duplex, speed; 2407 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2408 link.link_duplex = (duplex == FULL_DUPLEX) ? 2409 ETH_LINK_FULL_DUPLEX : 2410 ETH_LINK_HALF_DUPLEX; 2411 link.link_speed = speed; 2412 link.link_status = ETH_LINK_UP; 2413 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2414 ETH_LINK_SPEED_FIXED); 2415 } else if (!link_check) { 2416 link.link_speed = 0; 2417 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2418 link.link_status = ETH_LINK_DOWN; 2419 link.link_autoneg = ETH_LINK_FIXED; 2420 } 2421 2422 return rte_eth_linkstatus_set(dev, &link); 2423 } 2424 2425 /* 2426 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2427 * For ASF and Pass Through versions of f/w this means 2428 * that the driver is loaded. 2429 */ 2430 static void 2431 igb_hw_control_acquire(struct e1000_hw *hw) 2432 { 2433 uint32_t ctrl_ext; 2434 2435 /* Let firmware know the driver has taken over */ 2436 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2437 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2438 } 2439 2440 /* 2441 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2442 * For ASF and Pass Through versions of f/w this means that the 2443 * driver is no longer loaded. 2444 */ 2445 static void 2446 igb_hw_control_release(struct e1000_hw *hw) 2447 { 2448 uint32_t ctrl_ext; 2449 2450 /* Let firmware taken over control of h/w */ 2451 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2452 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2453 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2454 } 2455 2456 /* 2457 * Bit of a misnomer, what this really means is 2458 * to enable OS management of the system... aka 2459 * to disable special hardware management features. 2460 */ 2461 static void 2462 igb_init_manageability(struct e1000_hw *hw) 2463 { 2464 if (e1000_enable_mng_pass_thru(hw)) { 2465 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2466 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2467 2468 /* disable hardware interception of ARP */ 2469 manc &= ~(E1000_MANC_ARP_EN); 2470 2471 /* enable receiving management packets to the host */ 2472 manc |= E1000_MANC_EN_MNG2HOST; 2473 manc2h |= 1 << 5; /* Mng Port 623 */ 2474 manc2h |= 1 << 6; /* Mng Port 664 */ 2475 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2476 E1000_WRITE_REG(hw, E1000_MANC, manc); 2477 } 2478 } 2479 2480 static void 2481 igb_release_manageability(struct e1000_hw *hw) 2482 { 2483 if (e1000_enable_mng_pass_thru(hw)) { 2484 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2485 2486 manc |= E1000_MANC_ARP_EN; 2487 manc &= ~E1000_MANC_EN_MNG2HOST; 2488 2489 E1000_WRITE_REG(hw, E1000_MANC, manc); 2490 } 2491 } 2492 2493 static int 2494 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2495 { 2496 struct e1000_hw *hw = 2497 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2498 uint32_t rctl; 2499 2500 rctl = E1000_READ_REG(hw, E1000_RCTL); 2501 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2502 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2503 2504 return 0; 2505 } 2506 2507 static int 2508 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2509 { 2510 struct e1000_hw *hw = 2511 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2512 uint32_t rctl; 2513 2514 rctl = E1000_READ_REG(hw, E1000_RCTL); 2515 rctl &= (~E1000_RCTL_UPE); 2516 if (dev->data->all_multicast == 1) 2517 rctl |= E1000_RCTL_MPE; 2518 else 2519 rctl &= (~E1000_RCTL_MPE); 2520 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2521 2522 return 0; 2523 } 2524 2525 static int 2526 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2527 { 2528 struct e1000_hw *hw = 2529 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2530 uint32_t rctl; 2531 2532 rctl = E1000_READ_REG(hw, E1000_RCTL); 2533 rctl |= E1000_RCTL_MPE; 2534 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2535 2536 return 0; 2537 } 2538 2539 static int 2540 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2541 { 2542 struct e1000_hw *hw = 2543 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2544 uint32_t rctl; 2545 2546 if (dev->data->promiscuous == 1) 2547 return 0; /* must remain in all_multicast mode */ 2548 rctl = E1000_READ_REG(hw, E1000_RCTL); 2549 rctl &= (~E1000_RCTL_MPE); 2550 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2551 2552 return 0; 2553 } 2554 2555 static int 2556 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2557 { 2558 struct e1000_hw *hw = 2559 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2560 struct e1000_vfta * shadow_vfta = 2561 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2562 uint32_t vfta; 2563 uint32_t vid_idx; 2564 uint32_t vid_bit; 2565 2566 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2567 E1000_VFTA_ENTRY_MASK); 2568 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2569 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2570 if (on) 2571 vfta |= vid_bit; 2572 else 2573 vfta &= ~vid_bit; 2574 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2575 2576 /* update local VFTA copy */ 2577 shadow_vfta->vfta[vid_idx] = vfta; 2578 2579 return 0; 2580 } 2581 2582 static int 2583 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2584 enum rte_vlan_type vlan_type, 2585 uint16_t tpid) 2586 { 2587 struct e1000_hw *hw = 2588 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2589 uint32_t reg, qinq; 2590 2591 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2592 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2593 2594 /* only outer TPID of double VLAN can be configured*/ 2595 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2596 reg = E1000_READ_REG(hw, E1000_VET); 2597 reg = (reg & (~E1000_VET_VET_EXT)) | 2598 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2599 E1000_WRITE_REG(hw, E1000_VET, reg); 2600 2601 return 0; 2602 } 2603 2604 /* all other TPID values are read-only*/ 2605 PMD_DRV_LOG(ERR, "Not supported"); 2606 2607 return -ENOTSUP; 2608 } 2609 2610 static void 2611 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2612 { 2613 struct e1000_hw *hw = 2614 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2615 uint32_t reg; 2616 2617 /* Filter Table Disable */ 2618 reg = E1000_READ_REG(hw, E1000_RCTL); 2619 reg &= ~E1000_RCTL_CFIEN; 2620 reg &= ~E1000_RCTL_VFE; 2621 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2622 } 2623 2624 static void 2625 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2626 { 2627 struct e1000_hw *hw = 2628 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2629 struct e1000_vfta * shadow_vfta = 2630 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2631 uint32_t reg; 2632 int i; 2633 2634 /* Filter Table Enable, CFI not used for packet acceptance */ 2635 reg = E1000_READ_REG(hw, E1000_RCTL); 2636 reg &= ~E1000_RCTL_CFIEN; 2637 reg |= E1000_RCTL_VFE; 2638 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2639 2640 /* restore VFTA table */ 2641 for (i = 0; i < IGB_VFTA_SIZE; i++) 2642 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2643 } 2644 2645 static void 2646 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2647 { 2648 struct e1000_hw *hw = 2649 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2650 uint32_t reg; 2651 2652 /* VLAN Mode Disable */ 2653 reg = E1000_READ_REG(hw, E1000_CTRL); 2654 reg &= ~E1000_CTRL_VME; 2655 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2656 } 2657 2658 static void 2659 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2660 { 2661 struct e1000_hw *hw = 2662 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2663 uint32_t reg; 2664 2665 /* VLAN Mode Enable */ 2666 reg = E1000_READ_REG(hw, E1000_CTRL); 2667 reg |= E1000_CTRL_VME; 2668 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2669 } 2670 2671 static void 2672 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2673 { 2674 struct e1000_hw *hw = 2675 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2676 uint32_t reg; 2677 2678 /* CTRL_EXT: Extended VLAN */ 2679 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2680 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2681 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2682 2683 /* Update maximum packet length */ 2684 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2685 E1000_WRITE_REG(hw, E1000_RLPML, 2686 dev->data->dev_conf.rxmode.max_rx_pkt_len); 2687 } 2688 2689 static void 2690 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2691 { 2692 struct e1000_hw *hw = 2693 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2694 uint32_t reg; 2695 2696 /* CTRL_EXT: Extended VLAN */ 2697 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2698 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2699 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2700 2701 /* Update maximum packet length */ 2702 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2703 E1000_WRITE_REG(hw, E1000_RLPML, 2704 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2705 VLAN_TAG_SIZE); 2706 } 2707 2708 static int 2709 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2710 { 2711 struct rte_eth_rxmode *rxmode; 2712 2713 rxmode = &dev->data->dev_conf.rxmode; 2714 if(mask & ETH_VLAN_STRIP_MASK){ 2715 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2716 igb_vlan_hw_strip_enable(dev); 2717 else 2718 igb_vlan_hw_strip_disable(dev); 2719 } 2720 2721 if(mask & ETH_VLAN_FILTER_MASK){ 2722 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2723 igb_vlan_hw_filter_enable(dev); 2724 else 2725 igb_vlan_hw_filter_disable(dev); 2726 } 2727 2728 if(mask & ETH_VLAN_EXTEND_MASK){ 2729 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2730 igb_vlan_hw_extend_enable(dev); 2731 else 2732 igb_vlan_hw_extend_disable(dev); 2733 } 2734 2735 return 0; 2736 } 2737 2738 2739 /** 2740 * It enables the interrupt mask and then enable the interrupt. 2741 * 2742 * @param dev 2743 * Pointer to struct rte_eth_dev. 2744 * @param on 2745 * Enable or Disable 2746 * 2747 * @return 2748 * - On success, zero. 2749 * - On failure, a negative value. 2750 */ 2751 static int 2752 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2753 { 2754 struct e1000_interrupt *intr = 2755 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2756 2757 if (on) 2758 intr->mask |= E1000_ICR_LSC; 2759 else 2760 intr->mask &= ~E1000_ICR_LSC; 2761 2762 return 0; 2763 } 2764 2765 /* It clears the interrupt causes and enables the interrupt. 2766 * It will be called once only during nic initialized. 2767 * 2768 * @param dev 2769 * Pointer to struct rte_eth_dev. 2770 * 2771 * @return 2772 * - On success, zero. 2773 * - On failure, a negative value. 2774 */ 2775 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2776 { 2777 uint32_t mask, regval; 2778 int ret; 2779 struct e1000_hw *hw = 2780 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2781 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2782 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2783 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2784 struct rte_eth_dev_info dev_info; 2785 2786 memset(&dev_info, 0, sizeof(dev_info)); 2787 ret = eth_igb_infos_get(dev, &dev_info); 2788 if (ret != 0) 2789 return ret; 2790 2791 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2792 regval = E1000_READ_REG(hw, E1000_EIMS); 2793 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2794 2795 return 0; 2796 } 2797 2798 /* 2799 * It reads ICR and gets interrupt causes, check it and set a bit flag 2800 * to update link status. 2801 * 2802 * @param dev 2803 * Pointer to struct rte_eth_dev. 2804 * 2805 * @return 2806 * - On success, zero. 2807 * - On failure, a negative value. 2808 */ 2809 static int 2810 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2811 { 2812 uint32_t icr; 2813 struct e1000_hw *hw = 2814 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2815 struct e1000_interrupt *intr = 2816 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2817 2818 igb_intr_disable(dev); 2819 2820 /* read-on-clear nic registers here */ 2821 icr = E1000_READ_REG(hw, E1000_ICR); 2822 2823 intr->flags = 0; 2824 if (icr & E1000_ICR_LSC) { 2825 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2826 } 2827 2828 if (icr & E1000_ICR_VMMB) 2829 intr->flags |= E1000_FLAG_MAILBOX; 2830 2831 return 0; 2832 } 2833 2834 /* 2835 * It executes link_update after knowing an interrupt is prsent. 2836 * 2837 * @param dev 2838 * Pointer to struct rte_eth_dev. 2839 * 2840 * @return 2841 * - On success, zero. 2842 * - On failure, a negative value. 2843 */ 2844 static int 2845 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2846 struct rte_intr_handle *intr_handle) 2847 { 2848 struct e1000_hw *hw = 2849 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2850 struct e1000_interrupt *intr = 2851 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2852 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2853 struct rte_eth_link link; 2854 int ret; 2855 2856 if (intr->flags & E1000_FLAG_MAILBOX) { 2857 igb_pf_mbx_process(dev); 2858 intr->flags &= ~E1000_FLAG_MAILBOX; 2859 } 2860 2861 igb_intr_enable(dev); 2862 rte_intr_ack(intr_handle); 2863 2864 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2865 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2866 2867 /* set get_link_status to check register later */ 2868 hw->mac.get_link_status = 1; 2869 ret = eth_igb_link_update(dev, 0); 2870 2871 /* check if link has changed */ 2872 if (ret < 0) 2873 return 0; 2874 2875 rte_eth_linkstatus_get(dev, &link); 2876 if (link.link_status) { 2877 PMD_INIT_LOG(INFO, 2878 " Port %d: Link Up - speed %u Mbps - %s", 2879 dev->data->port_id, 2880 (unsigned)link.link_speed, 2881 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2882 "full-duplex" : "half-duplex"); 2883 } else { 2884 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2885 dev->data->port_id); 2886 } 2887 2888 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 2889 pci_dev->addr.domain, 2890 pci_dev->addr.bus, 2891 pci_dev->addr.devid, 2892 pci_dev->addr.function); 2893 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2894 } 2895 2896 return 0; 2897 } 2898 2899 /** 2900 * Interrupt handler which shall be registered at first. 2901 * 2902 * @param handle 2903 * Pointer to interrupt handle. 2904 * @param param 2905 * The address of parameter (struct rte_eth_dev *) regsitered before. 2906 * 2907 * @return 2908 * void 2909 */ 2910 static void 2911 eth_igb_interrupt_handler(void *param) 2912 { 2913 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2914 2915 eth_igb_interrupt_get_status(dev); 2916 eth_igb_interrupt_action(dev, dev->intr_handle); 2917 } 2918 2919 static int 2920 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2921 { 2922 uint32_t eicr; 2923 struct e1000_hw *hw = 2924 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2925 struct e1000_interrupt *intr = 2926 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2927 2928 igbvf_intr_disable(hw); 2929 2930 /* read-on-clear nic registers here */ 2931 eicr = E1000_READ_REG(hw, E1000_EICR); 2932 intr->flags = 0; 2933 2934 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2935 intr->flags |= E1000_FLAG_MAILBOX; 2936 2937 return 0; 2938 } 2939 2940 void igbvf_mbx_process(struct rte_eth_dev *dev) 2941 { 2942 struct e1000_hw *hw = 2943 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2944 struct e1000_mbx_info *mbx = &hw->mbx; 2945 u32 in_msg = 0; 2946 2947 /* peek the message first */ 2948 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2949 2950 /* PF reset VF event */ 2951 if (in_msg == E1000_PF_CONTROL_MSG) { 2952 /* dummy mbx read to ack pf */ 2953 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2954 return; 2955 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2956 NULL); 2957 } 2958 } 2959 2960 static int 2961 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2962 { 2963 struct e1000_interrupt *intr = 2964 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2965 2966 if (intr->flags & E1000_FLAG_MAILBOX) { 2967 igbvf_mbx_process(dev); 2968 intr->flags &= ~E1000_FLAG_MAILBOX; 2969 } 2970 2971 igbvf_intr_enable(dev); 2972 rte_intr_ack(intr_handle); 2973 2974 return 0; 2975 } 2976 2977 static void 2978 eth_igbvf_interrupt_handler(void *param) 2979 { 2980 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2981 2982 eth_igbvf_interrupt_get_status(dev); 2983 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2984 } 2985 2986 static int 2987 eth_igb_led_on(struct rte_eth_dev *dev) 2988 { 2989 struct e1000_hw *hw; 2990 2991 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2992 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2993 } 2994 2995 static int 2996 eth_igb_led_off(struct rte_eth_dev *dev) 2997 { 2998 struct e1000_hw *hw; 2999 3000 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3001 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3002 } 3003 3004 static int 3005 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3006 { 3007 struct e1000_hw *hw; 3008 uint32_t ctrl; 3009 int tx_pause; 3010 int rx_pause; 3011 3012 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3013 fc_conf->pause_time = hw->fc.pause_time; 3014 fc_conf->high_water = hw->fc.high_water; 3015 fc_conf->low_water = hw->fc.low_water; 3016 fc_conf->send_xon = hw->fc.send_xon; 3017 fc_conf->autoneg = hw->mac.autoneg; 3018 3019 /* 3020 * Return rx_pause and tx_pause status according to actual setting of 3021 * the TFCE and RFCE bits in the CTRL register. 3022 */ 3023 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3024 if (ctrl & E1000_CTRL_TFCE) 3025 tx_pause = 1; 3026 else 3027 tx_pause = 0; 3028 3029 if (ctrl & E1000_CTRL_RFCE) 3030 rx_pause = 1; 3031 else 3032 rx_pause = 0; 3033 3034 if (rx_pause && tx_pause) 3035 fc_conf->mode = RTE_FC_FULL; 3036 else if (rx_pause) 3037 fc_conf->mode = RTE_FC_RX_PAUSE; 3038 else if (tx_pause) 3039 fc_conf->mode = RTE_FC_TX_PAUSE; 3040 else 3041 fc_conf->mode = RTE_FC_NONE; 3042 3043 return 0; 3044 } 3045 3046 static int 3047 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3048 { 3049 struct e1000_hw *hw; 3050 int err; 3051 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3052 e1000_fc_none, 3053 e1000_fc_rx_pause, 3054 e1000_fc_tx_pause, 3055 e1000_fc_full 3056 }; 3057 uint32_t rx_buf_size; 3058 uint32_t max_high_water; 3059 uint32_t rctl; 3060 uint32_t ctrl; 3061 3062 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3063 if (fc_conf->autoneg != hw->mac.autoneg) 3064 return -ENOTSUP; 3065 rx_buf_size = igb_get_rx_buffer_size(hw); 3066 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3067 3068 /* At least reserve one Ethernet frame for watermark */ 3069 max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN; 3070 if ((fc_conf->high_water > max_high_water) || 3071 (fc_conf->high_water < fc_conf->low_water)) { 3072 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3073 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3074 return -EINVAL; 3075 } 3076 3077 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3078 hw->fc.pause_time = fc_conf->pause_time; 3079 hw->fc.high_water = fc_conf->high_water; 3080 hw->fc.low_water = fc_conf->low_water; 3081 hw->fc.send_xon = fc_conf->send_xon; 3082 3083 err = e1000_setup_link_generic(hw); 3084 if (err == E1000_SUCCESS) { 3085 3086 /* check if we want to forward MAC frames - driver doesn't have native 3087 * capability to do that, so we'll write the registers ourselves */ 3088 3089 rctl = E1000_READ_REG(hw, E1000_RCTL); 3090 3091 /* set or clear MFLCN.PMCF bit depending on configuration */ 3092 if (fc_conf->mac_ctrl_frame_fwd != 0) 3093 rctl |= E1000_RCTL_PMCF; 3094 else 3095 rctl &= ~E1000_RCTL_PMCF; 3096 3097 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3098 3099 /* 3100 * check if we want to change flow control mode - driver doesn't have native 3101 * capability to do that, so we'll write the registers ourselves 3102 */ 3103 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3104 3105 /* 3106 * set or clear E1000_CTRL_RFCE and E1000_CTRL_TFCE bits depending 3107 * on configuration 3108 */ 3109 switch (fc_conf->mode) { 3110 case RTE_FC_NONE: 3111 ctrl &= ~E1000_CTRL_RFCE & ~E1000_CTRL_TFCE; 3112 break; 3113 case RTE_FC_RX_PAUSE: 3114 ctrl |= E1000_CTRL_RFCE; 3115 ctrl &= ~E1000_CTRL_TFCE; 3116 break; 3117 case RTE_FC_TX_PAUSE: 3118 ctrl |= E1000_CTRL_TFCE; 3119 ctrl &= ~E1000_CTRL_RFCE; 3120 break; 3121 case RTE_FC_FULL: 3122 ctrl |= E1000_CTRL_RFCE | E1000_CTRL_TFCE; 3123 break; 3124 default: 3125 PMD_INIT_LOG(ERR, "invalid flow control mode"); 3126 return -EINVAL; 3127 } 3128 3129 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3130 3131 E1000_WRITE_FLUSH(hw); 3132 3133 return 0; 3134 } 3135 3136 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3137 return -EIO; 3138 } 3139 3140 #define E1000_RAH_POOLSEL_SHIFT (18) 3141 static int 3142 eth_igb_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 3143 uint32_t index, uint32_t pool) 3144 { 3145 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3146 uint32_t rah; 3147 3148 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3149 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3150 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3151 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3152 return 0; 3153 } 3154 3155 static void 3156 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3157 { 3158 uint8_t addr[RTE_ETHER_ADDR_LEN]; 3159 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3160 3161 memset(addr, 0, sizeof(addr)); 3162 3163 e1000_rar_set(hw, addr, index); 3164 } 3165 3166 static int 3167 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3168 struct rte_ether_addr *addr) 3169 { 3170 eth_igb_rar_clear(dev, 0); 3171 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3172 3173 return 0; 3174 } 3175 /* 3176 * Virtual Function operations 3177 */ 3178 static void 3179 igbvf_intr_disable(struct e1000_hw *hw) 3180 { 3181 PMD_INIT_FUNC_TRACE(); 3182 3183 /* Clear interrupt mask to stop from interrupts being generated */ 3184 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3185 3186 E1000_WRITE_FLUSH(hw); 3187 } 3188 3189 static void 3190 igbvf_stop_adapter(struct rte_eth_dev *dev) 3191 { 3192 u32 reg_val; 3193 u16 i; 3194 struct rte_eth_dev_info dev_info; 3195 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3196 int ret; 3197 3198 memset(&dev_info, 0, sizeof(dev_info)); 3199 ret = eth_igbvf_infos_get(dev, &dev_info); 3200 if (ret != 0) 3201 return; 3202 3203 /* Clear interrupt mask to stop from interrupts being generated */ 3204 igbvf_intr_disable(hw); 3205 3206 /* Clear any pending interrupts, flush previous writes */ 3207 E1000_READ_REG(hw, E1000_EICR); 3208 3209 /* Disable the transmit unit. Each queue must be disabled. */ 3210 for (i = 0; i < dev_info.max_tx_queues; i++) 3211 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3212 3213 /* Disable the receive unit by stopping each queue */ 3214 for (i = 0; i < dev_info.max_rx_queues; i++) { 3215 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3216 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3217 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3218 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3219 ; 3220 } 3221 3222 /* flush all queues disables */ 3223 E1000_WRITE_FLUSH(hw); 3224 msec_delay(2); 3225 } 3226 3227 static int eth_igbvf_link_update(struct e1000_hw *hw) 3228 { 3229 struct e1000_mbx_info *mbx = &hw->mbx; 3230 struct e1000_mac_info *mac = &hw->mac; 3231 int ret_val = E1000_SUCCESS; 3232 3233 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3234 3235 /* 3236 * We only want to run this if there has been a rst asserted. 3237 * in this case that could mean a link change, device reset, 3238 * or a virtual function reset 3239 */ 3240 3241 /* If we were hit with a reset or timeout drop the link */ 3242 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3243 mac->get_link_status = TRUE; 3244 3245 if (!mac->get_link_status) 3246 goto out; 3247 3248 /* if link status is down no point in checking to see if pf is up */ 3249 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3250 goto out; 3251 3252 /* if we passed all the tests above then the link is up and we no 3253 * longer need to check for link */ 3254 mac->get_link_status = FALSE; 3255 3256 out: 3257 return ret_val; 3258 } 3259 3260 3261 static int 3262 igbvf_dev_configure(struct rte_eth_dev *dev) 3263 { 3264 struct rte_eth_conf* conf = &dev->data->dev_conf; 3265 3266 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3267 dev->data->port_id); 3268 3269 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 3270 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 3271 3272 /* 3273 * VF has no ability to enable/disable HW CRC 3274 * Keep the persistent behavior the same as Host PF 3275 */ 3276 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3277 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3278 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3279 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3280 } 3281 #else 3282 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3283 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3284 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3285 } 3286 #endif 3287 3288 return 0; 3289 } 3290 3291 static int 3292 igbvf_dev_start(struct rte_eth_dev *dev) 3293 { 3294 struct e1000_hw *hw = 3295 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3296 struct e1000_adapter *adapter = 3297 E1000_DEV_PRIVATE(dev->data->dev_private); 3298 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3299 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3300 int ret; 3301 uint32_t intr_vector = 0; 3302 3303 PMD_INIT_FUNC_TRACE(); 3304 3305 hw->mac.ops.reset_hw(hw); 3306 adapter->stopped = 0; 3307 3308 /* Set all vfta */ 3309 igbvf_set_vfta_all(dev,1); 3310 3311 eth_igbvf_tx_init(dev); 3312 3313 /* This can fail when allocating mbufs for descriptor rings */ 3314 ret = eth_igbvf_rx_init(dev); 3315 if (ret) { 3316 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3317 igb_dev_clear_queues(dev); 3318 return ret; 3319 } 3320 3321 /* check and configure queue intr-vector mapping */ 3322 if (rte_intr_cap_multiple(intr_handle) && 3323 dev->data->dev_conf.intr_conf.rxq) { 3324 intr_vector = dev->data->nb_rx_queues; 3325 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3326 if (ret) 3327 return ret; 3328 } 3329 3330 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3331 intr_handle->intr_vec = 3332 rte_zmalloc("intr_vec", 3333 dev->data->nb_rx_queues * sizeof(int), 0); 3334 if (!intr_handle->intr_vec) { 3335 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3336 " intr_vec", dev->data->nb_rx_queues); 3337 return -ENOMEM; 3338 } 3339 } 3340 3341 eth_igbvf_configure_msix_intr(dev); 3342 3343 /* enable uio/vfio intr/eventfd mapping */ 3344 rte_intr_enable(intr_handle); 3345 3346 /* resume enabled intr since hw reset */ 3347 igbvf_intr_enable(dev); 3348 3349 return 0; 3350 } 3351 3352 static int 3353 igbvf_dev_stop(struct rte_eth_dev *dev) 3354 { 3355 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3356 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3357 struct e1000_adapter *adapter = 3358 E1000_DEV_PRIVATE(dev->data->dev_private); 3359 3360 if (adapter->stopped) 3361 return 0; 3362 3363 PMD_INIT_FUNC_TRACE(); 3364 3365 igbvf_stop_adapter(dev); 3366 3367 /* 3368 * Clear what we set, but we still keep shadow_vfta to 3369 * restore after device starts 3370 */ 3371 igbvf_set_vfta_all(dev,0); 3372 3373 igb_dev_clear_queues(dev); 3374 3375 /* disable intr eventfd mapping */ 3376 rte_intr_disable(intr_handle); 3377 3378 /* Clean datapath event and queue/vec mapping */ 3379 rte_intr_efd_disable(intr_handle); 3380 if (intr_handle->intr_vec) { 3381 rte_free(intr_handle->intr_vec); 3382 intr_handle->intr_vec = NULL; 3383 } 3384 3385 adapter->stopped = true; 3386 dev->data->dev_started = 0; 3387 3388 return 0; 3389 } 3390 3391 static int 3392 igbvf_dev_close(struct rte_eth_dev *dev) 3393 { 3394 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3395 struct rte_ether_addr addr; 3396 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3397 int ret; 3398 3399 PMD_INIT_FUNC_TRACE(); 3400 3401 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 3402 return 0; 3403 3404 e1000_reset_hw(hw); 3405 3406 ret = igbvf_dev_stop(dev); 3407 if (ret != 0) 3408 return ret; 3409 3410 igb_dev_free_queues(dev); 3411 3412 /** 3413 * reprogram the RAR with a zero mac address, 3414 * to ensure that the VF traffic goes to the PF 3415 * after stop, close and detach of the VF. 3416 **/ 3417 3418 memset(&addr, 0, sizeof(addr)); 3419 igbvf_default_mac_addr_set(dev, &addr); 3420 3421 rte_intr_callback_unregister(&pci_dev->intr_handle, 3422 eth_igbvf_interrupt_handler, 3423 (void *)dev); 3424 3425 return 0; 3426 } 3427 3428 static int 3429 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3430 { 3431 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3432 3433 /* Set both unicast and multicast promisc */ 3434 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3435 3436 return 0; 3437 } 3438 3439 static int 3440 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3441 { 3442 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3443 3444 /* If in allmulticast mode leave multicast promisc */ 3445 if (dev->data->all_multicast == 1) 3446 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3447 else 3448 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3449 3450 return 0; 3451 } 3452 3453 static int 3454 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3455 { 3456 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3457 3458 /* In promiscuous mode multicast promisc already set */ 3459 if (dev->data->promiscuous == 0) 3460 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3461 3462 return 0; 3463 } 3464 3465 static int 3466 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3467 { 3468 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3469 3470 /* In promiscuous mode leave multicast promisc enabled */ 3471 if (dev->data->promiscuous == 0) 3472 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3473 3474 return 0; 3475 } 3476 3477 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3478 { 3479 struct e1000_mbx_info *mbx = &hw->mbx; 3480 uint32_t msgbuf[2]; 3481 s32 err; 3482 3483 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3484 msgbuf[0] = E1000_VF_SET_VLAN; 3485 msgbuf[1] = vid; 3486 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3487 if (on) 3488 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3489 3490 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3491 if (err) 3492 goto mbx_err; 3493 3494 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3495 if (err) 3496 goto mbx_err; 3497 3498 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3499 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3500 err = -EINVAL; 3501 3502 mbx_err: 3503 return err; 3504 } 3505 3506 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3507 { 3508 struct e1000_hw *hw = 3509 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3510 struct e1000_vfta * shadow_vfta = 3511 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3512 int i = 0, j = 0, vfta = 0, mask = 1; 3513 3514 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3515 vfta = shadow_vfta->vfta[i]; 3516 if(vfta){ 3517 mask = 1; 3518 for (j = 0; j < 32; j++){ 3519 if(vfta & mask) 3520 igbvf_set_vfta(hw, 3521 (uint16_t)((i<<5)+j), on); 3522 mask<<=1; 3523 } 3524 } 3525 } 3526 3527 } 3528 3529 static int 3530 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3531 { 3532 struct e1000_hw *hw = 3533 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3534 struct e1000_vfta * shadow_vfta = 3535 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3536 uint32_t vid_idx = 0; 3537 uint32_t vid_bit = 0; 3538 int ret = 0; 3539 3540 PMD_INIT_FUNC_TRACE(); 3541 3542 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3543 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3544 if(ret){ 3545 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3546 return ret; 3547 } 3548 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3549 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3550 3551 /*Save what we set and retore it after device reset*/ 3552 if (on) 3553 shadow_vfta->vfta[vid_idx] |= vid_bit; 3554 else 3555 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3556 3557 return 0; 3558 } 3559 3560 static int 3561 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 3562 { 3563 struct e1000_hw *hw = 3564 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3565 3566 /* index is not used by rar_set() */ 3567 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3568 return 0; 3569 } 3570 3571 3572 static int 3573 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3574 struct rte_eth_rss_reta_entry64 *reta_conf, 3575 uint16_t reta_size) 3576 { 3577 uint8_t i, j, mask; 3578 uint32_t reta, r; 3579 uint16_t idx, shift; 3580 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3581 3582 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3583 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3584 "(%d) doesn't match the number hardware can supported " 3585 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3586 return -EINVAL; 3587 } 3588 3589 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3590 idx = i / RTE_RETA_GROUP_SIZE; 3591 shift = i % RTE_RETA_GROUP_SIZE; 3592 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3593 IGB_4_BIT_MASK); 3594 if (!mask) 3595 continue; 3596 if (mask == IGB_4_BIT_MASK) 3597 r = 0; 3598 else 3599 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3600 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3601 if (mask & (0x1 << j)) 3602 reta |= reta_conf[idx].reta[shift + j] << 3603 (CHAR_BIT * j); 3604 else 3605 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3606 } 3607 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3608 } 3609 3610 return 0; 3611 } 3612 3613 static int 3614 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3615 struct rte_eth_rss_reta_entry64 *reta_conf, 3616 uint16_t reta_size) 3617 { 3618 uint8_t i, j, mask; 3619 uint32_t reta; 3620 uint16_t idx, shift; 3621 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3622 3623 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3624 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3625 "(%d) doesn't match the number hardware can supported " 3626 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3627 return -EINVAL; 3628 } 3629 3630 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3631 idx = i / RTE_RETA_GROUP_SIZE; 3632 shift = i % RTE_RETA_GROUP_SIZE; 3633 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3634 IGB_4_BIT_MASK); 3635 if (!mask) 3636 continue; 3637 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3638 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3639 if (mask & (0x1 << j)) 3640 reta_conf[idx].reta[shift + j] = 3641 ((reta >> (CHAR_BIT * j)) & 3642 IGB_8_BIT_MASK); 3643 } 3644 } 3645 3646 return 0; 3647 } 3648 3649 int 3650 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3651 struct rte_eth_syn_filter *filter, 3652 bool add) 3653 { 3654 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3655 struct e1000_filter_info *filter_info = 3656 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3657 uint32_t synqf, rfctl; 3658 3659 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3660 return -EINVAL; 3661 3662 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3663 3664 if (add) { 3665 if (synqf & E1000_SYN_FILTER_ENABLE) 3666 return -EINVAL; 3667 3668 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3669 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3670 3671 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3672 if (filter->hig_pri) 3673 rfctl |= E1000_RFCTL_SYNQFP; 3674 else 3675 rfctl &= ~E1000_RFCTL_SYNQFP; 3676 3677 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3678 } else { 3679 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3680 return -ENOENT; 3681 synqf = 0; 3682 } 3683 3684 filter_info->syn_info = synqf; 3685 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3686 E1000_WRITE_FLUSH(hw); 3687 return 0; 3688 } 3689 3690 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3691 static inline int 3692 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3693 struct e1000_2tuple_filter_info *filter_info) 3694 { 3695 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3696 return -EINVAL; 3697 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3698 return -EINVAL; /* filter index is out of range. */ 3699 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 3700 return -EINVAL; /* flags is invalid. */ 3701 3702 switch (filter->dst_port_mask) { 3703 case UINT16_MAX: 3704 filter_info->dst_port_mask = 0; 3705 filter_info->dst_port = filter->dst_port; 3706 break; 3707 case 0: 3708 filter_info->dst_port_mask = 1; 3709 break; 3710 default: 3711 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3712 return -EINVAL; 3713 } 3714 3715 switch (filter->proto_mask) { 3716 case UINT8_MAX: 3717 filter_info->proto_mask = 0; 3718 filter_info->proto = filter->proto; 3719 break; 3720 case 0: 3721 filter_info->proto_mask = 1; 3722 break; 3723 default: 3724 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3725 return -EINVAL; 3726 } 3727 3728 filter_info->priority = (uint8_t)filter->priority; 3729 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3730 filter_info->tcp_flags = filter->tcp_flags; 3731 else 3732 filter_info->tcp_flags = 0; 3733 3734 return 0; 3735 } 3736 3737 static inline struct e1000_2tuple_filter * 3738 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3739 struct e1000_2tuple_filter_info *key) 3740 { 3741 struct e1000_2tuple_filter *it; 3742 3743 TAILQ_FOREACH(it, filter_list, entries) { 3744 if (memcmp(key, &it->filter_info, 3745 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3746 return it; 3747 } 3748 } 3749 return NULL; 3750 } 3751 3752 /* inject a igb 2tuple filter to HW */ 3753 static inline void 3754 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3755 struct e1000_2tuple_filter *filter) 3756 { 3757 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3758 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3759 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3760 int i; 3761 3762 i = filter->index; 3763 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3764 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3765 imir |= E1000_IMIR_PORT_BP; 3766 else 3767 imir &= ~E1000_IMIR_PORT_BP; 3768 3769 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3770 3771 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3772 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3773 ttqf |= (uint32_t)(filter->filter_info.proto & 3774 E1000_TTQF_PROTOCOL_MASK); 3775 if (filter->filter_info.proto_mask == 0) 3776 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3777 3778 /* tcp flags bits setting. */ 3779 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 3780 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 3781 imir_ext |= E1000_IMIREXT_CTRL_URG; 3782 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 3783 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3784 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 3785 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3786 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 3787 imir_ext |= E1000_IMIREXT_CTRL_RST; 3788 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 3789 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3790 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 3791 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3792 } else { 3793 imir_ext |= E1000_IMIREXT_CTRL_BP; 3794 } 3795 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3796 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3797 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3798 } 3799 3800 /* 3801 * igb_add_2tuple_filter - add a 2tuple filter 3802 * 3803 * @param 3804 * dev: Pointer to struct rte_eth_dev. 3805 * ntuple_filter: ponter to the filter that will be added. 3806 * 3807 * @return 3808 * - On success, zero. 3809 * - On failure, a negative value. 3810 */ 3811 static int 3812 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3813 struct rte_eth_ntuple_filter *ntuple_filter) 3814 { 3815 struct e1000_filter_info *filter_info = 3816 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3817 struct e1000_2tuple_filter *filter; 3818 int i, ret; 3819 3820 filter = rte_zmalloc("e1000_2tuple_filter", 3821 sizeof(struct e1000_2tuple_filter), 0); 3822 if (filter == NULL) 3823 return -ENOMEM; 3824 3825 ret = ntuple_filter_to_2tuple(ntuple_filter, 3826 &filter->filter_info); 3827 if (ret < 0) { 3828 rte_free(filter); 3829 return ret; 3830 } 3831 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3832 &filter->filter_info) != NULL) { 3833 PMD_DRV_LOG(ERR, "filter exists."); 3834 rte_free(filter); 3835 return -EEXIST; 3836 } 3837 filter->queue = ntuple_filter->queue; 3838 3839 /* 3840 * look for an unused 2tuple filter index, 3841 * and insert the filter to list. 3842 */ 3843 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3844 if (!(filter_info->twotuple_mask & (1 << i))) { 3845 filter_info->twotuple_mask |= 1 << i; 3846 filter->index = i; 3847 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3848 filter, 3849 entries); 3850 break; 3851 } 3852 } 3853 if (i >= E1000_MAX_TTQF_FILTERS) { 3854 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3855 rte_free(filter); 3856 return -ENOSYS; 3857 } 3858 3859 igb_inject_2uple_filter(dev, filter); 3860 return 0; 3861 } 3862 3863 int 3864 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3865 struct e1000_2tuple_filter *filter) 3866 { 3867 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3868 struct e1000_filter_info *filter_info = 3869 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3870 3871 filter_info->twotuple_mask &= ~(1 << filter->index); 3872 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3873 rte_free(filter); 3874 3875 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3876 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3877 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3878 return 0; 3879 } 3880 3881 /* 3882 * igb_remove_2tuple_filter - remove a 2tuple filter 3883 * 3884 * @param 3885 * dev: Pointer to struct rte_eth_dev. 3886 * ntuple_filter: ponter to the filter that will be removed. 3887 * 3888 * @return 3889 * - On success, zero. 3890 * - On failure, a negative value. 3891 */ 3892 static int 3893 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3894 struct rte_eth_ntuple_filter *ntuple_filter) 3895 { 3896 struct e1000_filter_info *filter_info = 3897 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3898 struct e1000_2tuple_filter_info filter_2tuple; 3899 struct e1000_2tuple_filter *filter; 3900 int ret; 3901 3902 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3903 ret = ntuple_filter_to_2tuple(ntuple_filter, 3904 &filter_2tuple); 3905 if (ret < 0) 3906 return ret; 3907 3908 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3909 &filter_2tuple); 3910 if (filter == NULL) { 3911 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3912 return -ENOENT; 3913 } 3914 3915 igb_delete_2tuple_filter(dev, filter); 3916 3917 return 0; 3918 } 3919 3920 /* inject a igb flex filter to HW */ 3921 static inline void 3922 igb_inject_flex_filter(struct rte_eth_dev *dev, 3923 struct e1000_flex_filter *filter) 3924 { 3925 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3926 uint32_t wufc, queueing; 3927 uint32_t reg_off; 3928 uint8_t i, j = 0; 3929 3930 wufc = E1000_READ_REG(hw, E1000_WUFC); 3931 if (filter->index < E1000_MAX_FHFT) 3932 reg_off = E1000_FHFT(filter->index); 3933 else 3934 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3935 3936 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3937 (E1000_WUFC_FLX0 << filter->index)); 3938 queueing = filter->filter_info.len | 3939 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3940 (filter->filter_info.priority << 3941 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3942 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3943 queueing); 3944 3945 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3946 E1000_WRITE_REG(hw, reg_off, 3947 filter->filter_info.dwords[j]); 3948 reg_off += sizeof(uint32_t); 3949 E1000_WRITE_REG(hw, reg_off, 3950 filter->filter_info.dwords[++j]); 3951 reg_off += sizeof(uint32_t); 3952 E1000_WRITE_REG(hw, reg_off, 3953 (uint32_t)filter->filter_info.mask[i]); 3954 reg_off += sizeof(uint32_t) * 2; 3955 ++j; 3956 } 3957 } 3958 3959 static inline struct e1000_flex_filter * 3960 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3961 struct e1000_flex_filter_info *key) 3962 { 3963 struct e1000_flex_filter *it; 3964 3965 TAILQ_FOREACH(it, filter_list, entries) { 3966 if (memcmp(key, &it->filter_info, 3967 sizeof(struct e1000_flex_filter_info)) == 0) 3968 return it; 3969 } 3970 3971 return NULL; 3972 } 3973 3974 /* remove a flex byte filter 3975 * @param 3976 * dev: Pointer to struct rte_eth_dev. 3977 * filter: the pointer of the filter will be removed. 3978 */ 3979 void 3980 igb_remove_flex_filter(struct rte_eth_dev *dev, 3981 struct e1000_flex_filter *filter) 3982 { 3983 struct e1000_filter_info *filter_info = 3984 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3985 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3986 uint32_t wufc, i; 3987 uint32_t reg_off; 3988 3989 wufc = E1000_READ_REG(hw, E1000_WUFC); 3990 if (filter->index < E1000_MAX_FHFT) 3991 reg_off = E1000_FHFT(filter->index); 3992 else 3993 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3994 3995 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 3996 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 3997 3998 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 3999 (~(E1000_WUFC_FLX0 << filter->index))); 4000 4001 filter_info->flex_mask &= ~(1 << filter->index); 4002 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4003 rte_free(filter); 4004 } 4005 4006 int 4007 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4008 struct igb_flex_filter *filter, 4009 bool add) 4010 { 4011 struct e1000_filter_info *filter_info = 4012 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4013 struct e1000_flex_filter *flex_filter, *it; 4014 uint32_t mask; 4015 uint8_t shift, i; 4016 4017 flex_filter = rte_zmalloc("e1000_flex_filter", 4018 sizeof(struct e1000_flex_filter), 0); 4019 if (flex_filter == NULL) 4020 return -ENOMEM; 4021 4022 flex_filter->filter_info.len = filter->len; 4023 flex_filter->filter_info.priority = filter->priority; 4024 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4025 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4026 mask = 0; 4027 /* reverse bits in flex filter's mask*/ 4028 for (shift = 0; shift < CHAR_BIT; shift++) { 4029 if (filter->mask[i] & (0x01 << shift)) 4030 mask |= (0x80 >> shift); 4031 } 4032 flex_filter->filter_info.mask[i] = mask; 4033 } 4034 4035 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4036 &flex_filter->filter_info); 4037 if (it == NULL && !add) { 4038 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4039 rte_free(flex_filter); 4040 return -ENOENT; 4041 } 4042 if (it != NULL && add) { 4043 PMD_DRV_LOG(ERR, "filter exists."); 4044 rte_free(flex_filter); 4045 return -EEXIST; 4046 } 4047 4048 if (add) { 4049 flex_filter->queue = filter->queue; 4050 /* 4051 * look for an unused flex filter index 4052 * and insert the filter into the list. 4053 */ 4054 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4055 if (!(filter_info->flex_mask & (1 << i))) { 4056 filter_info->flex_mask |= 1 << i; 4057 flex_filter->index = i; 4058 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4059 flex_filter, 4060 entries); 4061 break; 4062 } 4063 } 4064 if (i >= E1000_MAX_FLEX_FILTERS) { 4065 PMD_DRV_LOG(ERR, "flex filters are full."); 4066 rte_free(flex_filter); 4067 return -ENOSYS; 4068 } 4069 4070 igb_inject_flex_filter(dev, flex_filter); 4071 4072 } else { 4073 igb_remove_flex_filter(dev, it); 4074 rte_free(flex_filter); 4075 } 4076 4077 return 0; 4078 } 4079 4080 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4081 static inline int 4082 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4083 struct e1000_5tuple_filter_info *filter_info) 4084 { 4085 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4086 return -EINVAL; 4087 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4088 return -EINVAL; /* filter index is out of range. */ 4089 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 4090 return -EINVAL; /* flags is invalid. */ 4091 4092 switch (filter->dst_ip_mask) { 4093 case UINT32_MAX: 4094 filter_info->dst_ip_mask = 0; 4095 filter_info->dst_ip = filter->dst_ip; 4096 break; 4097 case 0: 4098 filter_info->dst_ip_mask = 1; 4099 break; 4100 default: 4101 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4102 return -EINVAL; 4103 } 4104 4105 switch (filter->src_ip_mask) { 4106 case UINT32_MAX: 4107 filter_info->src_ip_mask = 0; 4108 filter_info->src_ip = filter->src_ip; 4109 break; 4110 case 0: 4111 filter_info->src_ip_mask = 1; 4112 break; 4113 default: 4114 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4115 return -EINVAL; 4116 } 4117 4118 switch (filter->dst_port_mask) { 4119 case UINT16_MAX: 4120 filter_info->dst_port_mask = 0; 4121 filter_info->dst_port = filter->dst_port; 4122 break; 4123 case 0: 4124 filter_info->dst_port_mask = 1; 4125 break; 4126 default: 4127 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4128 return -EINVAL; 4129 } 4130 4131 switch (filter->src_port_mask) { 4132 case UINT16_MAX: 4133 filter_info->src_port_mask = 0; 4134 filter_info->src_port = filter->src_port; 4135 break; 4136 case 0: 4137 filter_info->src_port_mask = 1; 4138 break; 4139 default: 4140 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4141 return -EINVAL; 4142 } 4143 4144 switch (filter->proto_mask) { 4145 case UINT8_MAX: 4146 filter_info->proto_mask = 0; 4147 filter_info->proto = filter->proto; 4148 break; 4149 case 0: 4150 filter_info->proto_mask = 1; 4151 break; 4152 default: 4153 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4154 return -EINVAL; 4155 } 4156 4157 filter_info->priority = (uint8_t)filter->priority; 4158 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4159 filter_info->tcp_flags = filter->tcp_flags; 4160 else 4161 filter_info->tcp_flags = 0; 4162 4163 return 0; 4164 } 4165 4166 static inline struct e1000_5tuple_filter * 4167 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4168 struct e1000_5tuple_filter_info *key) 4169 { 4170 struct e1000_5tuple_filter *it; 4171 4172 TAILQ_FOREACH(it, filter_list, entries) { 4173 if (memcmp(key, &it->filter_info, 4174 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4175 return it; 4176 } 4177 } 4178 return NULL; 4179 } 4180 4181 /* inject a igb 5-tuple filter to HW */ 4182 static inline void 4183 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4184 struct e1000_5tuple_filter *filter) 4185 { 4186 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4187 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4188 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4189 uint8_t i; 4190 4191 i = filter->index; 4192 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4193 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4194 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4195 if (filter->filter_info.dst_ip_mask == 0) 4196 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4197 if (filter->filter_info.src_port_mask == 0) 4198 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4199 if (filter->filter_info.proto_mask == 0) 4200 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4201 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4202 E1000_FTQF_QUEUE_MASK; 4203 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4204 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4205 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4206 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4207 4208 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4209 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4210 4211 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4212 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4213 imir |= E1000_IMIR_PORT_BP; 4214 else 4215 imir &= ~E1000_IMIR_PORT_BP; 4216 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4217 4218 /* tcp flags bits setting. */ 4219 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 4220 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 4221 imir_ext |= E1000_IMIREXT_CTRL_URG; 4222 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 4223 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4224 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 4225 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4226 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 4227 imir_ext |= E1000_IMIREXT_CTRL_RST; 4228 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 4229 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4230 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 4231 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4232 } else { 4233 imir_ext |= E1000_IMIREXT_CTRL_BP; 4234 } 4235 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4236 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4237 } 4238 4239 /* 4240 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4241 * 4242 * @param 4243 * dev: Pointer to struct rte_eth_dev. 4244 * ntuple_filter: ponter to the filter that will be added. 4245 * 4246 * @return 4247 * - On success, zero. 4248 * - On failure, a negative value. 4249 */ 4250 static int 4251 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4252 struct rte_eth_ntuple_filter *ntuple_filter) 4253 { 4254 struct e1000_filter_info *filter_info = 4255 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4256 struct e1000_5tuple_filter *filter; 4257 uint8_t i; 4258 int ret; 4259 4260 filter = rte_zmalloc("e1000_5tuple_filter", 4261 sizeof(struct e1000_5tuple_filter), 0); 4262 if (filter == NULL) 4263 return -ENOMEM; 4264 4265 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4266 &filter->filter_info); 4267 if (ret < 0) { 4268 rte_free(filter); 4269 return ret; 4270 } 4271 4272 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4273 &filter->filter_info) != NULL) { 4274 PMD_DRV_LOG(ERR, "filter exists."); 4275 rte_free(filter); 4276 return -EEXIST; 4277 } 4278 filter->queue = ntuple_filter->queue; 4279 4280 /* 4281 * look for an unused 5tuple filter index, 4282 * and insert the filter to list. 4283 */ 4284 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4285 if (!(filter_info->fivetuple_mask & (1 << i))) { 4286 filter_info->fivetuple_mask |= 1 << i; 4287 filter->index = i; 4288 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4289 filter, 4290 entries); 4291 break; 4292 } 4293 } 4294 if (i >= E1000_MAX_FTQF_FILTERS) { 4295 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4296 rte_free(filter); 4297 return -ENOSYS; 4298 } 4299 4300 igb_inject_5tuple_filter_82576(dev, filter); 4301 return 0; 4302 } 4303 4304 int 4305 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4306 struct e1000_5tuple_filter *filter) 4307 { 4308 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4309 struct e1000_filter_info *filter_info = 4310 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4311 4312 filter_info->fivetuple_mask &= ~(1 << filter->index); 4313 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4314 rte_free(filter); 4315 4316 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4317 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4318 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4319 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4320 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4321 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4322 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4323 return 0; 4324 } 4325 4326 /* 4327 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4328 * 4329 * @param 4330 * dev: Pointer to struct rte_eth_dev. 4331 * ntuple_filter: ponter to the filter that will be removed. 4332 * 4333 * @return 4334 * - On success, zero. 4335 * - On failure, a negative value. 4336 */ 4337 static int 4338 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4339 struct rte_eth_ntuple_filter *ntuple_filter) 4340 { 4341 struct e1000_filter_info *filter_info = 4342 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4343 struct e1000_5tuple_filter_info filter_5tuple; 4344 struct e1000_5tuple_filter *filter; 4345 int ret; 4346 4347 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4348 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4349 &filter_5tuple); 4350 if (ret < 0) 4351 return ret; 4352 4353 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4354 &filter_5tuple); 4355 if (filter == NULL) { 4356 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4357 return -ENOENT; 4358 } 4359 4360 igb_delete_5tuple_filter_82576(dev, filter); 4361 4362 return 0; 4363 } 4364 4365 static int 4366 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4367 { 4368 uint32_t rctl; 4369 struct e1000_hw *hw; 4370 struct rte_eth_dev_info dev_info; 4371 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4372 int ret; 4373 4374 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4375 4376 #ifdef RTE_LIBRTE_82571_SUPPORT 4377 /* XXX: not bigger than max_rx_pktlen */ 4378 if (hw->mac.type == e1000_82571) 4379 return -ENOTSUP; 4380 #endif 4381 ret = eth_igb_infos_get(dev, &dev_info); 4382 if (ret != 0) 4383 return ret; 4384 4385 /* check that mtu is within the allowed range */ 4386 if (mtu < RTE_ETHER_MIN_MTU || 4387 frame_size > dev_info.max_rx_pktlen) 4388 return -EINVAL; 4389 4390 /* 4391 * If device is started, refuse mtu that requires the support of 4392 * scattered packets when this feature has not been enabled before. 4393 */ 4394 if (dev->data->dev_started && !dev->data->scattered_rx && 4395 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) { 4396 PMD_INIT_LOG(ERR, "Stop port first."); 4397 return -EINVAL; 4398 } 4399 4400 rctl = E1000_READ_REG(hw, E1000_RCTL); 4401 4402 /* switch to jumbo mode if needed */ 4403 if (frame_size > E1000_ETH_MAX_LEN) { 4404 dev->data->dev_conf.rxmode.offloads |= 4405 DEV_RX_OFFLOAD_JUMBO_FRAME; 4406 rctl |= E1000_RCTL_LPE; 4407 } else { 4408 dev->data->dev_conf.rxmode.offloads &= 4409 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4410 rctl &= ~E1000_RCTL_LPE; 4411 } 4412 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4413 4414 /* update max frame size */ 4415 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4416 4417 E1000_WRITE_REG(hw, E1000_RLPML, 4418 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4419 4420 return 0; 4421 } 4422 4423 /* 4424 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4425 * 4426 * @param 4427 * dev: Pointer to struct rte_eth_dev. 4428 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4429 * add: if true, add filter, if false, remove filter 4430 * 4431 * @return 4432 * - On success, zero. 4433 * - On failure, a negative value. 4434 */ 4435 int 4436 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4437 struct rte_eth_ntuple_filter *ntuple_filter, 4438 bool add) 4439 { 4440 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4441 int ret; 4442 4443 switch (ntuple_filter->flags) { 4444 case RTE_5TUPLE_FLAGS: 4445 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4446 if (hw->mac.type != e1000_82576) 4447 return -ENOTSUP; 4448 if (add) 4449 ret = igb_add_5tuple_filter_82576(dev, 4450 ntuple_filter); 4451 else 4452 ret = igb_remove_5tuple_filter_82576(dev, 4453 ntuple_filter); 4454 break; 4455 case RTE_2TUPLE_FLAGS: 4456 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4457 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4458 hw->mac.type != e1000_i210 && 4459 hw->mac.type != e1000_i211) 4460 return -ENOTSUP; 4461 if (add) 4462 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4463 else 4464 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4465 break; 4466 default: 4467 ret = -EINVAL; 4468 break; 4469 } 4470 4471 return ret; 4472 } 4473 4474 static inline int 4475 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4476 uint16_t ethertype) 4477 { 4478 int i; 4479 4480 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4481 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4482 (filter_info->ethertype_mask & (1 << i))) 4483 return i; 4484 } 4485 return -1; 4486 } 4487 4488 static inline int 4489 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4490 uint16_t ethertype, uint32_t etqf) 4491 { 4492 int i; 4493 4494 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4495 if (!(filter_info->ethertype_mask & (1 << i))) { 4496 filter_info->ethertype_mask |= 1 << i; 4497 filter_info->ethertype_filters[i].ethertype = ethertype; 4498 filter_info->ethertype_filters[i].etqf = etqf; 4499 return i; 4500 } 4501 } 4502 return -1; 4503 } 4504 4505 int 4506 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4507 uint8_t idx) 4508 { 4509 if (idx >= E1000_MAX_ETQF_FILTERS) 4510 return -1; 4511 filter_info->ethertype_mask &= ~(1 << idx); 4512 filter_info->ethertype_filters[idx].ethertype = 0; 4513 filter_info->ethertype_filters[idx].etqf = 0; 4514 return idx; 4515 } 4516 4517 4518 int 4519 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4520 struct rte_eth_ethertype_filter *filter, 4521 bool add) 4522 { 4523 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4524 struct e1000_filter_info *filter_info = 4525 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4526 uint32_t etqf = 0; 4527 int ret; 4528 4529 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 4530 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 4531 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4532 " ethertype filter.", filter->ether_type); 4533 return -EINVAL; 4534 } 4535 4536 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4537 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4538 return -EINVAL; 4539 } 4540 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4541 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4542 return -EINVAL; 4543 } 4544 4545 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4546 if (ret >= 0 && add) { 4547 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4548 filter->ether_type); 4549 return -EEXIST; 4550 } 4551 if (ret < 0 && !add) { 4552 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4553 filter->ether_type); 4554 return -ENOENT; 4555 } 4556 4557 if (add) { 4558 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4559 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4560 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4561 ret = igb_ethertype_filter_insert(filter_info, 4562 filter->ether_type, etqf); 4563 if (ret < 0) { 4564 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4565 return -ENOSYS; 4566 } 4567 } else { 4568 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4569 if (ret < 0) 4570 return -ENOSYS; 4571 } 4572 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4573 E1000_WRITE_FLUSH(hw); 4574 4575 return 0; 4576 } 4577 4578 static int 4579 eth_igb_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 4580 const struct rte_flow_ops **ops) 4581 { 4582 *ops = &igb_flow_ops; 4583 return 0; 4584 } 4585 4586 static int 4587 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4588 struct rte_ether_addr *mc_addr_set, 4589 uint32_t nb_mc_addr) 4590 { 4591 struct e1000_hw *hw; 4592 4593 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4594 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4595 return 0; 4596 } 4597 4598 static uint64_t 4599 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4600 { 4601 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4602 uint64_t systime_cycles; 4603 4604 switch (hw->mac.type) { 4605 case e1000_i210: 4606 case e1000_i211: 4607 /* 4608 * Need to read System Time Residue Register to be able 4609 * to read the other two registers. 4610 */ 4611 E1000_READ_REG(hw, E1000_SYSTIMR); 4612 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4613 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4614 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4615 * NSEC_PER_SEC; 4616 break; 4617 case e1000_82580: 4618 case e1000_i350: 4619 case e1000_i354: 4620 /* 4621 * Need to read System Time Residue Register to be able 4622 * to read the other two registers. 4623 */ 4624 E1000_READ_REG(hw, E1000_SYSTIMR); 4625 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4626 /* Only the 8 LSB are valid. */ 4627 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4628 & 0xff) << 32; 4629 break; 4630 default: 4631 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4632 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4633 << 32; 4634 break; 4635 } 4636 4637 return systime_cycles; 4638 } 4639 4640 static uint64_t 4641 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4642 { 4643 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4644 uint64_t rx_tstamp_cycles; 4645 4646 switch (hw->mac.type) { 4647 case e1000_i210: 4648 case e1000_i211: 4649 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4650 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4651 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4652 * NSEC_PER_SEC; 4653 break; 4654 case e1000_82580: 4655 case e1000_i350: 4656 case e1000_i354: 4657 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4658 /* Only the 8 LSB are valid. */ 4659 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4660 & 0xff) << 32; 4661 break; 4662 default: 4663 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4664 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4665 << 32; 4666 break; 4667 } 4668 4669 return rx_tstamp_cycles; 4670 } 4671 4672 static uint64_t 4673 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4674 { 4675 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4676 uint64_t tx_tstamp_cycles; 4677 4678 switch (hw->mac.type) { 4679 case e1000_i210: 4680 case e1000_i211: 4681 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4682 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4683 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4684 * NSEC_PER_SEC; 4685 break; 4686 case e1000_82580: 4687 case e1000_i350: 4688 case e1000_i354: 4689 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4690 /* Only the 8 LSB are valid. */ 4691 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 4692 & 0xff) << 32; 4693 break; 4694 default: 4695 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4696 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4697 << 32; 4698 break; 4699 } 4700 4701 return tx_tstamp_cycles; 4702 } 4703 4704 static void 4705 igb_start_timecounters(struct rte_eth_dev *dev) 4706 { 4707 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4708 struct e1000_adapter *adapter = dev->data->dev_private; 4709 uint32_t incval = 1; 4710 uint32_t shift = 0; 4711 uint64_t mask = E1000_CYCLECOUNTER_MASK; 4712 4713 switch (hw->mac.type) { 4714 case e1000_82580: 4715 case e1000_i350: 4716 case e1000_i354: 4717 /* 32 LSB bits + 8 MSB bits = 40 bits */ 4718 mask = (1ULL << 40) - 1; 4719 /* fall-through */ 4720 case e1000_i210: 4721 case e1000_i211: 4722 /* 4723 * Start incrementing the register 4724 * used to timestamp PTP packets. 4725 */ 4726 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 4727 break; 4728 case e1000_82576: 4729 incval = E1000_INCVALUE_82576; 4730 shift = IGB_82576_TSYNC_SHIFT; 4731 E1000_WRITE_REG(hw, E1000_TIMINCA, 4732 E1000_INCPERIOD_82576 | incval); 4733 break; 4734 default: 4735 /* Not supported */ 4736 return; 4737 } 4738 4739 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 4740 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 4741 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 4742 4743 adapter->systime_tc.cc_mask = mask; 4744 adapter->systime_tc.cc_shift = shift; 4745 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 4746 4747 adapter->rx_tstamp_tc.cc_mask = mask; 4748 adapter->rx_tstamp_tc.cc_shift = shift; 4749 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 4750 4751 adapter->tx_tstamp_tc.cc_mask = mask; 4752 adapter->tx_tstamp_tc.cc_shift = shift; 4753 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 4754 } 4755 4756 static int 4757 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 4758 { 4759 struct e1000_adapter *adapter = dev->data->dev_private; 4760 4761 adapter->systime_tc.nsec += delta; 4762 adapter->rx_tstamp_tc.nsec += delta; 4763 adapter->tx_tstamp_tc.nsec += delta; 4764 4765 return 0; 4766 } 4767 4768 static int 4769 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 4770 { 4771 uint64_t ns; 4772 struct e1000_adapter *adapter = dev->data->dev_private; 4773 4774 ns = rte_timespec_to_ns(ts); 4775 4776 /* Set the timecounters to a new value. */ 4777 adapter->systime_tc.nsec = ns; 4778 adapter->rx_tstamp_tc.nsec = ns; 4779 adapter->tx_tstamp_tc.nsec = ns; 4780 4781 return 0; 4782 } 4783 4784 static int 4785 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 4786 { 4787 uint64_t ns, systime_cycles; 4788 struct e1000_adapter *adapter = dev->data->dev_private; 4789 4790 systime_cycles = igb_read_systime_cyclecounter(dev); 4791 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 4792 *ts = rte_ns_to_timespec(ns); 4793 4794 return 0; 4795 } 4796 4797 static int 4798 igb_timesync_enable(struct rte_eth_dev *dev) 4799 { 4800 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4801 uint32_t tsync_ctl; 4802 uint32_t tsauxc; 4803 4804 /* Stop the timesync system time. */ 4805 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 4806 /* Reset the timesync system time value. */ 4807 switch (hw->mac.type) { 4808 case e1000_82580: 4809 case e1000_i350: 4810 case e1000_i354: 4811 case e1000_i210: 4812 case e1000_i211: 4813 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 4814 /* fall-through */ 4815 case e1000_82576: 4816 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 4817 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 4818 break; 4819 default: 4820 /* Not supported. */ 4821 return -ENOTSUP; 4822 } 4823 4824 /* Enable system time for it isn't on by default. */ 4825 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 4826 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 4827 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 4828 4829 igb_start_timecounters(dev); 4830 4831 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 4832 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 4833 (RTE_ETHER_TYPE_1588 | 4834 E1000_ETQF_FILTER_ENABLE | 4835 E1000_ETQF_1588)); 4836 4837 /* Enable timestamping of received PTP packets. */ 4838 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4839 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 4840 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 4841 4842 /* Enable Timestamping of transmitted PTP packets. */ 4843 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4844 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 4845 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 4846 4847 return 0; 4848 } 4849 4850 static int 4851 igb_timesync_disable(struct rte_eth_dev *dev) 4852 { 4853 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4854 uint32_t tsync_ctl; 4855 4856 /* Disable timestamping of transmitted PTP packets. */ 4857 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4858 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 4859 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 4860 4861 /* Disable timestamping of received PTP packets. */ 4862 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4863 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 4864 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 4865 4866 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 4867 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 4868 4869 /* Stop incrementating the System Time registers. */ 4870 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 4871 4872 return 0; 4873 } 4874 4875 static int 4876 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 4877 struct timespec *timestamp, 4878 uint32_t flags __rte_unused) 4879 { 4880 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4881 struct e1000_adapter *adapter = dev->data->dev_private; 4882 uint32_t tsync_rxctl; 4883 uint64_t rx_tstamp_cycles; 4884 uint64_t ns; 4885 4886 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4887 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 4888 return -EINVAL; 4889 4890 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 4891 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 4892 *timestamp = rte_ns_to_timespec(ns); 4893 4894 return 0; 4895 } 4896 4897 static int 4898 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 4899 struct timespec *timestamp) 4900 { 4901 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4902 struct e1000_adapter *adapter = dev->data->dev_private; 4903 uint32_t tsync_txctl; 4904 uint64_t tx_tstamp_cycles; 4905 uint64_t ns; 4906 4907 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4908 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 4909 return -EINVAL; 4910 4911 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 4912 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 4913 *timestamp = rte_ns_to_timespec(ns); 4914 4915 return 0; 4916 } 4917 4918 static int 4919 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 4920 { 4921 int count = 0; 4922 int g_ind = 0; 4923 const struct reg_info *reg_group; 4924 4925 while ((reg_group = igb_regs[g_ind++])) 4926 count += igb_reg_group_count(reg_group); 4927 4928 return count; 4929 } 4930 4931 static int 4932 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 4933 { 4934 int count = 0; 4935 int g_ind = 0; 4936 const struct reg_info *reg_group; 4937 4938 while ((reg_group = igbvf_regs[g_ind++])) 4939 count += igb_reg_group_count(reg_group); 4940 4941 return count; 4942 } 4943 4944 static int 4945 eth_igb_get_regs(struct rte_eth_dev *dev, 4946 struct rte_dev_reg_info *regs) 4947 { 4948 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4949 uint32_t *data = regs->data; 4950 int g_ind = 0; 4951 int count = 0; 4952 const struct reg_info *reg_group; 4953 4954 if (data == NULL) { 4955 regs->length = eth_igb_get_reg_length(dev); 4956 regs->width = sizeof(uint32_t); 4957 return 0; 4958 } 4959 4960 /* Support only full register dump */ 4961 if ((regs->length == 0) || 4962 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 4963 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 4964 hw->device_id; 4965 while ((reg_group = igb_regs[g_ind++])) 4966 count += igb_read_regs_group(dev, &data[count], 4967 reg_group); 4968 return 0; 4969 } 4970 4971 return -ENOTSUP; 4972 } 4973 4974 static int 4975 igbvf_get_regs(struct rte_eth_dev *dev, 4976 struct rte_dev_reg_info *regs) 4977 { 4978 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4979 uint32_t *data = regs->data; 4980 int g_ind = 0; 4981 int count = 0; 4982 const struct reg_info *reg_group; 4983 4984 if (data == NULL) { 4985 regs->length = igbvf_get_reg_length(dev); 4986 regs->width = sizeof(uint32_t); 4987 return 0; 4988 } 4989 4990 /* Support only full register dump */ 4991 if ((regs->length == 0) || 4992 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 4993 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 4994 hw->device_id; 4995 while ((reg_group = igbvf_regs[g_ind++])) 4996 count += igb_read_regs_group(dev, &data[count], 4997 reg_group); 4998 return 0; 4999 } 5000 5001 return -ENOTSUP; 5002 } 5003 5004 static int 5005 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5006 { 5007 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5008 5009 /* Return unit is byte count */ 5010 return hw->nvm.word_size * 2; 5011 } 5012 5013 static int 5014 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5015 struct rte_dev_eeprom_info *in_eeprom) 5016 { 5017 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5018 struct e1000_nvm_info *nvm = &hw->nvm; 5019 uint16_t *data = in_eeprom->data; 5020 int first, length; 5021 5022 first = in_eeprom->offset >> 1; 5023 length = in_eeprom->length >> 1; 5024 if ((first >= hw->nvm.word_size) || 5025 ((first + length) >= hw->nvm.word_size)) 5026 return -EINVAL; 5027 5028 in_eeprom->magic = hw->vendor_id | 5029 ((uint32_t)hw->device_id << 16); 5030 5031 if ((nvm->ops.read) == NULL) 5032 return -ENOTSUP; 5033 5034 return nvm->ops.read(hw, first, length, data); 5035 } 5036 5037 static int 5038 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5039 struct rte_dev_eeprom_info *in_eeprom) 5040 { 5041 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5042 struct e1000_nvm_info *nvm = &hw->nvm; 5043 uint16_t *data = in_eeprom->data; 5044 int first, length; 5045 5046 first = in_eeprom->offset >> 1; 5047 length = in_eeprom->length >> 1; 5048 if ((first >= hw->nvm.word_size) || 5049 ((first + length) >= hw->nvm.word_size)) 5050 return -EINVAL; 5051 5052 in_eeprom->magic = (uint32_t)hw->vendor_id | 5053 ((uint32_t)hw->device_id << 16); 5054 5055 if ((nvm->ops.write) == NULL) 5056 return -ENOTSUP; 5057 return nvm->ops.write(hw, first, length, data); 5058 } 5059 5060 static int 5061 eth_igb_get_module_info(struct rte_eth_dev *dev, 5062 struct rte_eth_dev_module_info *modinfo) 5063 { 5064 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5065 5066 uint32_t status = 0; 5067 uint16_t sff8472_rev, addr_mode; 5068 bool page_swap = false; 5069 5070 if (hw->phy.media_type == e1000_media_type_copper || 5071 hw->phy.media_type == e1000_media_type_unknown) 5072 return -EOPNOTSUPP; 5073 5074 /* Check whether we support SFF-8472 or not */ 5075 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5076 if (status) 5077 return -EIO; 5078 5079 /* addressing mode is not supported */ 5080 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5081 if (status) 5082 return -EIO; 5083 5084 /* addressing mode is not supported */ 5085 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5086 PMD_DRV_LOG(ERR, 5087 "Address change required to access page 0xA2, " 5088 "but not supported. Please report the module " 5089 "type to the driver maintainers.\n"); 5090 page_swap = true; 5091 } 5092 5093 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5094 /* We have an SFP, but it does not support SFF-8472 */ 5095 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5096 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5097 } else { 5098 /* We have an SFP which supports a revision of SFF-8472 */ 5099 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5100 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5101 } 5102 5103 return 0; 5104 } 5105 5106 static int 5107 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5108 struct rte_dev_eeprom_info *info) 5109 { 5110 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5111 5112 uint32_t status = 0; 5113 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5114 u16 first_word, last_word; 5115 int i = 0; 5116 5117 first_word = info->offset >> 1; 5118 last_word = (info->offset + info->length - 1) >> 1; 5119 5120 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5121 for (i = 0; i < last_word - first_word + 1; i++) { 5122 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5123 &dataword[i]); 5124 if (status) { 5125 /* Error occurred while reading module */ 5126 return -EIO; 5127 } 5128 5129 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5130 } 5131 5132 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5133 5134 return 0; 5135 } 5136 5137 static int 5138 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5139 { 5140 struct e1000_hw *hw = 5141 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5142 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5143 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5144 uint32_t vec = E1000_MISC_VEC_ID; 5145 5146 if (rte_intr_allow_others(intr_handle)) 5147 vec = E1000_RX_VEC_START; 5148 5149 uint32_t mask = 1 << (queue_id + vec); 5150 5151 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5152 E1000_WRITE_FLUSH(hw); 5153 5154 return 0; 5155 } 5156 5157 static int 5158 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5159 { 5160 struct e1000_hw *hw = 5161 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5162 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5163 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5164 uint32_t vec = E1000_MISC_VEC_ID; 5165 5166 if (rte_intr_allow_others(intr_handle)) 5167 vec = E1000_RX_VEC_START; 5168 5169 uint32_t mask = 1 << (queue_id + vec); 5170 uint32_t regval; 5171 5172 regval = E1000_READ_REG(hw, E1000_EIMS); 5173 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5174 E1000_WRITE_FLUSH(hw); 5175 5176 rte_intr_ack(intr_handle); 5177 5178 return 0; 5179 } 5180 5181 static void 5182 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5183 uint8_t index, uint8_t offset) 5184 { 5185 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5186 5187 /* clear bits */ 5188 val &= ~((uint32_t)0xFF << offset); 5189 5190 /* write vector and valid bit */ 5191 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5192 5193 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5194 } 5195 5196 static void 5197 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5198 uint8_t queue, uint8_t msix_vector) 5199 { 5200 uint32_t tmp = 0; 5201 5202 if (hw->mac.type == e1000_82575) { 5203 if (direction == 0) 5204 tmp = E1000_EICR_RX_QUEUE0 << queue; 5205 else if (direction == 1) 5206 tmp = E1000_EICR_TX_QUEUE0 << queue; 5207 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5208 } else if (hw->mac.type == e1000_82576) { 5209 if ((direction == 0) || (direction == 1)) 5210 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5211 ((queue & 0x8) << 1) + 5212 8 * direction); 5213 } else if ((hw->mac.type == e1000_82580) || 5214 (hw->mac.type == e1000_i350) || 5215 (hw->mac.type == e1000_i354) || 5216 (hw->mac.type == e1000_i210) || 5217 (hw->mac.type == e1000_i211)) { 5218 if ((direction == 0) || (direction == 1)) 5219 eth_igb_write_ivar(hw, msix_vector, 5220 queue >> 1, 5221 ((queue & 0x1) << 4) + 5222 8 * direction); 5223 } 5224 } 5225 5226 /* Sets up the hardware to generate MSI-X interrupts properly 5227 * @hw 5228 * board private structure 5229 */ 5230 static void 5231 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5232 { 5233 int queue_id; 5234 uint32_t tmpval, regval, intr_mask; 5235 struct e1000_hw *hw = 5236 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5237 uint32_t vec = E1000_MISC_VEC_ID; 5238 uint32_t base = E1000_MISC_VEC_ID; 5239 uint32_t misc_shift = 0; 5240 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5241 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5242 5243 /* won't configure msix register if no mapping is done 5244 * between intr vector and event fd 5245 */ 5246 if (!rte_intr_dp_is_en(intr_handle)) 5247 return; 5248 5249 if (rte_intr_allow_others(intr_handle)) { 5250 vec = base = E1000_RX_VEC_START; 5251 misc_shift = 1; 5252 } 5253 5254 /* set interrupt vector for other causes */ 5255 if (hw->mac.type == e1000_82575) { 5256 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5257 /* enable MSI-X PBA support */ 5258 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5259 5260 /* Auto-Mask interrupts upon ICR read */ 5261 tmpval |= E1000_CTRL_EXT_EIAME; 5262 tmpval |= E1000_CTRL_EXT_IRCA; 5263 5264 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5265 5266 /* enable msix_other interrupt */ 5267 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5268 regval = E1000_READ_REG(hw, E1000_EIAC); 5269 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5270 regval = E1000_READ_REG(hw, E1000_EIAM); 5271 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5272 } else if ((hw->mac.type == e1000_82576) || 5273 (hw->mac.type == e1000_82580) || 5274 (hw->mac.type == e1000_i350) || 5275 (hw->mac.type == e1000_i354) || 5276 (hw->mac.type == e1000_i210) || 5277 (hw->mac.type == e1000_i211)) { 5278 /* turn on MSI-X capability first */ 5279 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5280 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5281 E1000_GPIE_NSICR); 5282 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5283 misc_shift; 5284 5285 if (dev->data->dev_conf.intr_conf.lsc != 0) 5286 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5287 5288 regval = E1000_READ_REG(hw, E1000_EIAC); 5289 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5290 5291 /* enable msix_other interrupt */ 5292 regval = E1000_READ_REG(hw, E1000_EIMS); 5293 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5294 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5295 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5296 } 5297 5298 /* use EIAM to auto-mask when MSI-X interrupt 5299 * is asserted, this saves a register write for every interrupt 5300 */ 5301 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5302 misc_shift; 5303 5304 if (dev->data->dev_conf.intr_conf.lsc != 0) 5305 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5306 5307 regval = E1000_READ_REG(hw, E1000_EIAM); 5308 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5309 5310 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5311 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5312 intr_handle->intr_vec[queue_id] = vec; 5313 if (vec < base + intr_handle->nb_efd - 1) 5314 vec++; 5315 } 5316 5317 E1000_WRITE_FLUSH(hw); 5318 } 5319 5320 /* restore n-tuple filter */ 5321 static inline void 5322 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5323 { 5324 struct e1000_filter_info *filter_info = 5325 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5326 struct e1000_5tuple_filter *p_5tuple; 5327 struct e1000_2tuple_filter *p_2tuple; 5328 5329 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5330 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5331 } 5332 5333 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5334 igb_inject_2uple_filter(dev, p_2tuple); 5335 } 5336 } 5337 5338 /* restore SYN filter */ 5339 static inline void 5340 igb_syn_filter_restore(struct rte_eth_dev *dev) 5341 { 5342 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5343 struct e1000_filter_info *filter_info = 5344 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5345 uint32_t synqf; 5346 5347 synqf = filter_info->syn_info; 5348 5349 if (synqf & E1000_SYN_FILTER_ENABLE) { 5350 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5351 E1000_WRITE_FLUSH(hw); 5352 } 5353 } 5354 5355 /* restore ethernet type filter */ 5356 static inline void 5357 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5358 { 5359 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5360 struct e1000_filter_info *filter_info = 5361 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5362 int i; 5363 5364 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5365 if (filter_info->ethertype_mask & (1 << i)) { 5366 E1000_WRITE_REG(hw, E1000_ETQF(i), 5367 filter_info->ethertype_filters[i].etqf); 5368 E1000_WRITE_FLUSH(hw); 5369 } 5370 } 5371 } 5372 5373 /* restore flex byte filter */ 5374 static inline void 5375 igb_flex_filter_restore(struct rte_eth_dev *dev) 5376 { 5377 struct e1000_filter_info *filter_info = 5378 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5379 struct e1000_flex_filter *flex_filter; 5380 5381 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5382 igb_inject_flex_filter(dev, flex_filter); 5383 } 5384 } 5385 5386 /* restore rss filter */ 5387 static inline void 5388 igb_rss_filter_restore(struct rte_eth_dev *dev) 5389 { 5390 struct e1000_filter_info *filter_info = 5391 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5392 5393 if (filter_info->rss_info.conf.queue_num) 5394 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5395 } 5396 5397 /* restore all types filter */ 5398 static int 5399 igb_filter_restore(struct rte_eth_dev *dev) 5400 { 5401 igb_ntuple_filter_restore(dev); 5402 igb_ethertype_filter_restore(dev); 5403 igb_syn_filter_restore(dev); 5404 igb_flex_filter_restore(dev); 5405 igb_rss_filter_restore(dev); 5406 5407 return 0; 5408 } 5409 5410 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5411 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5412 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5413 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5414 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5415 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5416