1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static int eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static int eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static int eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static int eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static int eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static int eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 const uint64_t *ids, struct rte_eth_xstat_name *xstats_names, 100 unsigned int limit); 101 static int eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static int eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static int eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static int eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct rte_ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct rte_ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static int igbvf_dev_stop(struct rte_eth_dev *dev); 158 static int igbvf_dev_close(struct rte_eth_dev *dev); 159 static int igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static int igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static int igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static int igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static int eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct rte_ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 190 struct rte_eth_ntuple_filter *ntuple_filter); 191 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 192 struct rte_eth_ntuple_filter *ntuple_filter); 193 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 194 struct rte_eth_ntuple_filter *ntuple_filter); 195 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 196 struct rte_eth_ntuple_filter *ntuple_filter); 197 static int eth_igb_flow_ops_get(struct rte_eth_dev *dev, 198 const struct rte_flow_ops **ops); 199 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 200 static int eth_igb_get_regs(struct rte_eth_dev *dev, 201 struct rte_dev_reg_info *regs); 202 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 203 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 204 struct rte_dev_eeprom_info *eeprom); 205 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 206 struct rte_dev_eeprom_info *eeprom); 207 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 208 struct rte_eth_dev_module_info *modinfo); 209 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 210 struct rte_dev_eeprom_info *info); 211 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 212 struct rte_ether_addr *mc_addr_set, 213 uint32_t nb_mc_addr); 214 static int igb_timesync_enable(struct rte_eth_dev *dev); 215 static int igb_timesync_disable(struct rte_eth_dev *dev); 216 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 217 struct timespec *timestamp, 218 uint32_t flags); 219 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 220 struct timespec *timestamp); 221 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 222 static int igb_timesync_read_time(struct rte_eth_dev *dev, 223 struct timespec *timestamp); 224 static int igb_timesync_write_time(struct rte_eth_dev *dev, 225 const struct timespec *timestamp); 226 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 227 uint16_t queue_id); 228 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 229 uint16_t queue_id); 230 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 231 uint8_t queue, uint8_t msix_vector); 232 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 233 uint8_t index, uint8_t offset); 234 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 235 static void eth_igbvf_interrupt_handler(void *param); 236 static void igbvf_mbx_process(struct rte_eth_dev *dev); 237 static int igb_filter_restore(struct rte_eth_dev *dev); 238 239 /* 240 * Define VF Stats MACRO for Non "cleared on read" register 241 */ 242 #define UPDATE_VF_STAT(reg, last, cur) \ 243 { \ 244 u32 latest = E1000_READ_REG(hw, reg); \ 245 cur += (latest - last) & UINT_MAX; \ 246 last = latest; \ 247 } 248 249 #define IGB_FC_PAUSE_TIME 0x0680 250 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 251 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 252 253 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 254 255 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 256 257 /* 258 * The set of PCI devices this driver supports 259 */ 260 static const struct rte_pci_id pci_id_igb_map[] = { 261 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 262 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 263 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 264 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 265 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 266 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 267 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 268 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 269 270 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 271 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 272 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 273 274 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 275 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 276 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 277 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 278 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 279 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 280 281 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 291 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 295 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 302 { .vendor_id = 0, /* sentinel */ }, 303 }; 304 305 /* 306 * The set of PCI devices this driver supports (for 82576&I350 VF) 307 */ 308 static const struct rte_pci_id pci_id_igbvf_map[] = { 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 313 { .vendor_id = 0, /* sentinel */ }, 314 }; 315 316 static const struct rte_eth_desc_lim rx_desc_lim = { 317 .nb_max = E1000_MAX_RING_DESC, 318 .nb_min = E1000_MIN_RING_DESC, 319 .nb_align = IGB_RXD_ALIGN, 320 }; 321 322 static const struct rte_eth_desc_lim tx_desc_lim = { 323 .nb_max = E1000_MAX_RING_DESC, 324 .nb_min = E1000_MIN_RING_DESC, 325 .nb_align = IGB_RXD_ALIGN, 326 .nb_seg_max = IGB_TX_MAX_SEG, 327 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 328 }; 329 330 static const struct eth_dev_ops eth_igb_ops = { 331 .dev_configure = eth_igb_configure, 332 .dev_start = eth_igb_start, 333 .dev_stop = eth_igb_stop, 334 .dev_set_link_up = eth_igb_dev_set_link_up, 335 .dev_set_link_down = eth_igb_dev_set_link_down, 336 .dev_close = eth_igb_close, 337 .dev_reset = eth_igb_reset, 338 .promiscuous_enable = eth_igb_promiscuous_enable, 339 .promiscuous_disable = eth_igb_promiscuous_disable, 340 .allmulticast_enable = eth_igb_allmulticast_enable, 341 .allmulticast_disable = eth_igb_allmulticast_disable, 342 .link_update = eth_igb_link_update, 343 .stats_get = eth_igb_stats_get, 344 .xstats_get = eth_igb_xstats_get, 345 .xstats_get_by_id = eth_igb_xstats_get_by_id, 346 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 347 .xstats_get_names = eth_igb_xstats_get_names, 348 .stats_reset = eth_igb_stats_reset, 349 .xstats_reset = eth_igb_xstats_reset, 350 .fw_version_get = eth_igb_fw_version_get, 351 .dev_infos_get = eth_igb_infos_get, 352 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 353 .mtu_set = eth_igb_mtu_set, 354 .vlan_filter_set = eth_igb_vlan_filter_set, 355 .vlan_tpid_set = eth_igb_vlan_tpid_set, 356 .vlan_offload_set = eth_igb_vlan_offload_set, 357 .rx_queue_setup = eth_igb_rx_queue_setup, 358 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 359 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 360 .rx_queue_release = eth_igb_rx_queue_release, 361 .tx_queue_setup = eth_igb_tx_queue_setup, 362 .tx_queue_release = eth_igb_tx_queue_release, 363 .tx_done_cleanup = eth_igb_tx_done_cleanup, 364 .dev_led_on = eth_igb_led_on, 365 .dev_led_off = eth_igb_led_off, 366 .flow_ctrl_get = eth_igb_flow_ctrl_get, 367 .flow_ctrl_set = eth_igb_flow_ctrl_set, 368 .mac_addr_add = eth_igb_rar_set, 369 .mac_addr_remove = eth_igb_rar_clear, 370 .mac_addr_set = eth_igb_default_mac_addr_set, 371 .reta_update = eth_igb_rss_reta_update, 372 .reta_query = eth_igb_rss_reta_query, 373 .rss_hash_update = eth_igb_rss_hash_update, 374 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 375 .flow_ops_get = eth_igb_flow_ops_get, 376 .set_mc_addr_list = eth_igb_set_mc_addr_list, 377 .rxq_info_get = igb_rxq_info_get, 378 .txq_info_get = igb_txq_info_get, 379 .timesync_enable = igb_timesync_enable, 380 .timesync_disable = igb_timesync_disable, 381 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 382 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 383 .get_reg = eth_igb_get_regs, 384 .get_eeprom_length = eth_igb_get_eeprom_length, 385 .get_eeprom = eth_igb_get_eeprom, 386 .set_eeprom = eth_igb_set_eeprom, 387 .get_module_info = eth_igb_get_module_info, 388 .get_module_eeprom = eth_igb_get_module_eeprom, 389 .timesync_adjust_time = igb_timesync_adjust_time, 390 .timesync_read_time = igb_timesync_read_time, 391 .timesync_write_time = igb_timesync_write_time, 392 }; 393 394 /* 395 * dev_ops for virtual function, bare necessities for basic vf 396 * operation have been implemented 397 */ 398 static const struct eth_dev_ops igbvf_eth_dev_ops = { 399 .dev_configure = igbvf_dev_configure, 400 .dev_start = igbvf_dev_start, 401 .dev_stop = igbvf_dev_stop, 402 .dev_close = igbvf_dev_close, 403 .promiscuous_enable = igbvf_promiscuous_enable, 404 .promiscuous_disable = igbvf_promiscuous_disable, 405 .allmulticast_enable = igbvf_allmulticast_enable, 406 .allmulticast_disable = igbvf_allmulticast_disable, 407 .link_update = eth_igb_link_update, 408 .stats_get = eth_igbvf_stats_get, 409 .xstats_get = eth_igbvf_xstats_get, 410 .xstats_get_names = eth_igbvf_xstats_get_names, 411 .stats_reset = eth_igbvf_stats_reset, 412 .xstats_reset = eth_igbvf_stats_reset, 413 .vlan_filter_set = igbvf_vlan_filter_set, 414 .dev_infos_get = eth_igbvf_infos_get, 415 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 416 .rx_queue_setup = eth_igb_rx_queue_setup, 417 .rx_queue_release = eth_igb_rx_queue_release, 418 .tx_queue_setup = eth_igb_tx_queue_setup, 419 .tx_queue_release = eth_igb_tx_queue_release, 420 .tx_done_cleanup = eth_igb_tx_done_cleanup, 421 .set_mc_addr_list = eth_igb_set_mc_addr_list, 422 .rxq_info_get = igb_rxq_info_get, 423 .txq_info_get = igb_txq_info_get, 424 .mac_addr_set = igbvf_default_mac_addr_set, 425 .get_reg = igbvf_get_regs, 426 }; 427 428 /* store statistics names and its offset in stats structure */ 429 struct rte_igb_xstats_name_off { 430 char name[RTE_ETH_XSTATS_NAME_SIZE]; 431 unsigned offset; 432 }; 433 434 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 435 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 436 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 437 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 438 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 439 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 440 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 441 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 442 ecol)}, 443 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 444 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 445 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 446 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 447 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 448 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 449 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 450 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 451 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 452 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 453 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 454 fcruc)}, 455 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 456 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 457 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 458 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 459 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 460 prc1023)}, 461 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 462 prc1522)}, 463 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 464 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 465 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 466 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 467 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 468 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 469 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 470 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 471 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 472 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 473 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 474 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 475 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 476 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 477 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 478 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 479 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 480 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 481 ptc1023)}, 482 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 483 ptc1522)}, 484 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 485 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 486 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 487 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 488 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 489 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 490 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 491 492 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 493 }; 494 495 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 496 sizeof(rte_igb_stats_strings[0])) 497 498 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 499 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 500 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 501 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 502 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 503 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 504 }; 505 506 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 507 sizeof(rte_igbvf_stats_strings[0])) 508 509 510 static inline void 511 igb_intr_enable(struct rte_eth_dev *dev) 512 { 513 struct e1000_interrupt *intr = 514 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 515 struct e1000_hw *hw = 516 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 517 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 518 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 519 520 if (rte_intr_allow_others(intr_handle) && 521 dev->data->dev_conf.intr_conf.lsc != 0) { 522 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 523 } 524 525 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 526 E1000_WRITE_FLUSH(hw); 527 } 528 529 static void 530 igb_intr_disable(struct rte_eth_dev *dev) 531 { 532 struct e1000_hw *hw = 533 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 534 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 535 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 536 537 if (rte_intr_allow_others(intr_handle) && 538 dev->data->dev_conf.intr_conf.lsc != 0) { 539 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 540 } 541 542 E1000_WRITE_REG(hw, E1000_IMC, ~0); 543 E1000_WRITE_FLUSH(hw); 544 } 545 546 static inline void 547 igbvf_intr_enable(struct rte_eth_dev *dev) 548 { 549 struct e1000_hw *hw = 550 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 551 552 /* only for mailbox */ 553 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 554 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 555 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 556 E1000_WRITE_FLUSH(hw); 557 } 558 559 /* only for mailbox now. If RX/TX needed, should extend this function. */ 560 static void 561 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 562 { 563 uint32_t tmp = 0; 564 565 /* mailbox */ 566 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 567 tmp |= E1000_VTIVAR_VALID; 568 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 569 } 570 571 static void 572 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 573 { 574 struct e1000_hw *hw = 575 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 576 577 /* Configure VF other cause ivar */ 578 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 579 } 580 581 static inline int32_t 582 igb_pf_reset_hw(struct e1000_hw *hw) 583 { 584 uint32_t ctrl_ext; 585 int32_t status; 586 587 status = e1000_reset_hw(hw); 588 589 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 590 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 591 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 592 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 593 E1000_WRITE_FLUSH(hw); 594 595 return status; 596 } 597 598 static void 599 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 600 { 601 struct e1000_hw *hw = 602 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 603 604 605 hw->vendor_id = pci_dev->id.vendor_id; 606 hw->device_id = pci_dev->id.device_id; 607 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 608 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 609 610 e1000_set_mac_type(hw); 611 612 /* need to check if it is a vf device below */ 613 } 614 615 static int 616 igb_reset_swfw_lock(struct e1000_hw *hw) 617 { 618 int ret_val; 619 620 /* 621 * Do mac ops initialization manually here, since we will need 622 * some function pointers set by this call. 623 */ 624 ret_val = e1000_init_mac_params(hw); 625 if (ret_val) 626 return ret_val; 627 628 /* 629 * SMBI lock should not fail in this early stage. If this is the case, 630 * it is due to an improper exit of the application. 631 * So force the release of the faulty lock. 632 */ 633 if (e1000_get_hw_semaphore_generic(hw) < 0) { 634 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 635 } 636 e1000_put_hw_semaphore_generic(hw); 637 638 if (hw->mac.ops.acquire_swfw_sync != NULL) { 639 uint16_t mask; 640 641 /* 642 * Phy lock should not fail in this early stage. If this is the case, 643 * it is due to an improper exit of the application. 644 * So force the release of the faulty lock. 645 */ 646 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 647 if (hw->bus.func > E1000_FUNC_1) 648 mask <<= 2; 649 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 650 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 651 hw->bus.func); 652 } 653 hw->mac.ops.release_swfw_sync(hw, mask); 654 655 /* 656 * This one is more tricky since it is common to all ports; but 657 * swfw_sync retries last long enough (1s) to be almost sure that if 658 * lock can not be taken it is due to an improper lock of the 659 * semaphore. 660 */ 661 mask = E1000_SWFW_EEP_SM; 662 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 663 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 664 } 665 hw->mac.ops.release_swfw_sync(hw, mask); 666 } 667 668 return E1000_SUCCESS; 669 } 670 671 /* Remove all ntuple filters of the device */ 672 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 673 { 674 struct e1000_filter_info *filter_info = 675 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 676 struct e1000_5tuple_filter *p_5tuple; 677 struct e1000_2tuple_filter *p_2tuple; 678 679 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 680 TAILQ_REMOVE(&filter_info->fivetuple_list, 681 p_5tuple, entries); 682 rte_free(p_5tuple); 683 } 684 filter_info->fivetuple_mask = 0; 685 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 686 TAILQ_REMOVE(&filter_info->twotuple_list, 687 p_2tuple, entries); 688 rte_free(p_2tuple); 689 } 690 filter_info->twotuple_mask = 0; 691 692 return 0; 693 } 694 695 /* Remove all flex filters of the device */ 696 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 697 { 698 struct e1000_filter_info *filter_info = 699 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 700 struct e1000_flex_filter *p_flex; 701 702 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 703 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 704 rte_free(p_flex); 705 } 706 filter_info->flex_mask = 0; 707 708 return 0; 709 } 710 711 static int 712 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 713 { 714 int error = 0; 715 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 716 struct e1000_hw *hw = 717 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 718 struct e1000_vfta * shadow_vfta = 719 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 720 struct e1000_filter_info *filter_info = 721 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 722 struct e1000_adapter *adapter = 723 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 724 725 uint32_t ctrl_ext; 726 727 eth_dev->dev_ops = ð_igb_ops; 728 eth_dev->rx_queue_count = eth_igb_rx_queue_count; 729 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 730 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 731 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 732 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 733 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 734 735 /* for secondary processes, we don't initialise any further as primary 736 * has already done this work. Only check we don't need a different 737 * RX function */ 738 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 739 if (eth_dev->data->scattered_rx) 740 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 741 return 0; 742 } 743 744 rte_eth_copy_pci_info(eth_dev, pci_dev); 745 746 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 747 748 igb_identify_hardware(eth_dev, pci_dev); 749 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 750 error = -EIO; 751 goto err_late; 752 } 753 754 e1000_get_bus_info(hw); 755 756 /* Reset any pending lock */ 757 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 758 error = -EIO; 759 goto err_late; 760 } 761 762 /* Finish initialization */ 763 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 764 error = -EIO; 765 goto err_late; 766 } 767 768 hw->mac.autoneg = 1; 769 hw->phy.autoneg_wait_to_complete = 0; 770 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 771 772 /* Copper options */ 773 if (hw->phy.media_type == e1000_media_type_copper) { 774 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 775 hw->phy.disable_polarity_correction = 0; 776 hw->phy.ms_type = e1000_ms_hw_default; 777 } 778 779 /* 780 * Start from a known state, this is important in reading the nvm 781 * and mac from that. 782 */ 783 igb_pf_reset_hw(hw); 784 785 /* Make sure we have a good EEPROM before we read from it */ 786 if (e1000_validate_nvm_checksum(hw) < 0) { 787 /* 788 * Some PCI-E parts fail the first check due to 789 * the link being in sleep state, call it again, 790 * if it fails a second time its a real issue. 791 */ 792 if (e1000_validate_nvm_checksum(hw) < 0) { 793 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 794 error = -EIO; 795 goto err_late; 796 } 797 } 798 799 /* Read the permanent MAC address out of the EEPROM */ 800 if (e1000_read_mac_addr(hw) != 0) { 801 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 802 error = -EIO; 803 goto err_late; 804 } 805 806 /* Allocate memory for storing MAC addresses */ 807 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 808 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 809 if (eth_dev->data->mac_addrs == NULL) { 810 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 811 "store MAC addresses", 812 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 813 error = -ENOMEM; 814 goto err_late; 815 } 816 817 /* Copy the permanent MAC address */ 818 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 819 ð_dev->data->mac_addrs[0]); 820 821 /* initialize the vfta */ 822 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 823 824 /* Now initialize the hardware */ 825 if (igb_hardware_init(hw) != 0) { 826 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 827 rte_free(eth_dev->data->mac_addrs); 828 eth_dev->data->mac_addrs = NULL; 829 error = -ENODEV; 830 goto err_late; 831 } 832 hw->mac.get_link_status = 1; 833 adapter->stopped = 0; 834 835 /* Indicate SOL/IDER usage */ 836 if (e1000_check_reset_block(hw) < 0) { 837 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 838 "SOL/IDER session"); 839 } 840 841 /* initialize PF if max_vfs not zero */ 842 igb_pf_host_init(eth_dev); 843 844 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 845 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 846 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 847 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 848 E1000_WRITE_FLUSH(hw); 849 850 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 851 eth_dev->data->port_id, pci_dev->id.vendor_id, 852 pci_dev->id.device_id); 853 854 rte_intr_callback_register(&pci_dev->intr_handle, 855 eth_igb_interrupt_handler, 856 (void *)eth_dev); 857 858 /* enable uio/vfio intr/eventfd mapping */ 859 rte_intr_enable(&pci_dev->intr_handle); 860 861 /* enable support intr */ 862 igb_intr_enable(eth_dev); 863 864 eth_igb_dev_set_link_down(eth_dev); 865 866 /* initialize filter info */ 867 memset(filter_info, 0, 868 sizeof(struct e1000_filter_info)); 869 870 TAILQ_INIT(&filter_info->flex_list); 871 TAILQ_INIT(&filter_info->twotuple_list); 872 TAILQ_INIT(&filter_info->fivetuple_list); 873 874 TAILQ_INIT(&igb_filter_ntuple_list); 875 TAILQ_INIT(&igb_filter_ethertype_list); 876 TAILQ_INIT(&igb_filter_syn_list); 877 TAILQ_INIT(&igb_filter_flex_list); 878 TAILQ_INIT(&igb_filter_rss_list); 879 TAILQ_INIT(&igb_flow_list); 880 881 return 0; 882 883 err_late: 884 igb_hw_control_release(hw); 885 886 return error; 887 } 888 889 static int 890 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 891 { 892 PMD_INIT_FUNC_TRACE(); 893 894 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 895 return 0; 896 897 eth_igb_close(eth_dev); 898 899 return 0; 900 } 901 902 /* 903 * Virtual Function device init 904 */ 905 static int 906 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 907 { 908 struct rte_pci_device *pci_dev; 909 struct rte_intr_handle *intr_handle; 910 struct e1000_adapter *adapter = 911 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 912 struct e1000_hw *hw = 913 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 914 int diag; 915 struct rte_ether_addr *perm_addr = 916 (struct rte_ether_addr *)hw->mac.perm_addr; 917 918 PMD_INIT_FUNC_TRACE(); 919 920 eth_dev->dev_ops = &igbvf_eth_dev_ops; 921 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 922 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 923 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 924 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 925 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 926 927 /* for secondary processes, we don't initialise any further as primary 928 * has already done this work. Only check we don't need a different 929 * RX function */ 930 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 931 if (eth_dev->data->scattered_rx) 932 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 933 return 0; 934 } 935 936 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 937 rte_eth_copy_pci_info(eth_dev, pci_dev); 938 939 hw->device_id = pci_dev->id.device_id; 940 hw->vendor_id = pci_dev->id.vendor_id; 941 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 942 adapter->stopped = 0; 943 944 /* Initialize the shared code (base driver) */ 945 diag = e1000_setup_init_funcs(hw, TRUE); 946 if (diag != 0) { 947 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 948 diag); 949 return -EIO; 950 } 951 952 /* init_mailbox_params */ 953 hw->mbx.ops.init_params(hw); 954 955 /* Disable the interrupts for VF */ 956 igbvf_intr_disable(hw); 957 958 diag = hw->mac.ops.reset_hw(hw); 959 960 /* Allocate memory for storing MAC addresses */ 961 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", RTE_ETHER_ADDR_LEN * 962 hw->mac.rar_entry_count, 0); 963 if (eth_dev->data->mac_addrs == NULL) { 964 PMD_INIT_LOG(ERR, 965 "Failed to allocate %d bytes needed to store MAC " 966 "addresses", 967 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 968 return -ENOMEM; 969 } 970 971 /* Generate a random MAC address, if none was assigned by PF. */ 972 if (rte_is_zero_ether_addr(perm_addr)) { 973 rte_eth_random_addr(perm_addr->addr_bytes); 974 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 975 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 976 RTE_ETHER_ADDR_PRT_FMT, 977 RTE_ETHER_ADDR_BYTES(perm_addr)); 978 } 979 980 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 981 if (diag) { 982 rte_free(eth_dev->data->mac_addrs); 983 eth_dev->data->mac_addrs = NULL; 984 return diag; 985 } 986 /* Copy the permanent MAC address */ 987 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 988 ð_dev->data->mac_addrs[0]); 989 990 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 991 "mac.type=%s", 992 eth_dev->data->port_id, pci_dev->id.vendor_id, 993 pci_dev->id.device_id, "igb_mac_82576_vf"); 994 995 intr_handle = &pci_dev->intr_handle; 996 rte_intr_callback_register(intr_handle, 997 eth_igbvf_interrupt_handler, eth_dev); 998 999 return 0; 1000 } 1001 1002 static int 1003 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1004 { 1005 PMD_INIT_FUNC_TRACE(); 1006 1007 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1008 return 0; 1009 1010 igbvf_dev_close(eth_dev); 1011 1012 return 0; 1013 } 1014 1015 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1016 struct rte_pci_device *pci_dev) 1017 { 1018 return rte_eth_dev_pci_generic_probe(pci_dev, 1019 sizeof(struct e1000_adapter), eth_igb_dev_init); 1020 } 1021 1022 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1023 { 1024 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1025 } 1026 1027 static struct rte_pci_driver rte_igb_pmd = { 1028 .id_table = pci_id_igb_map, 1029 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1030 .probe = eth_igb_pci_probe, 1031 .remove = eth_igb_pci_remove, 1032 }; 1033 1034 1035 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1036 struct rte_pci_device *pci_dev) 1037 { 1038 return rte_eth_dev_pci_generic_probe(pci_dev, 1039 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1040 } 1041 1042 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1043 { 1044 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1045 } 1046 1047 /* 1048 * virtual function driver struct 1049 */ 1050 static struct rte_pci_driver rte_igbvf_pmd = { 1051 .id_table = pci_id_igbvf_map, 1052 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1053 .probe = eth_igbvf_pci_probe, 1054 .remove = eth_igbvf_pci_remove, 1055 }; 1056 1057 static void 1058 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1059 { 1060 struct e1000_hw *hw = 1061 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1062 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1063 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1064 rctl |= E1000_RCTL_VFE; 1065 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1066 } 1067 1068 static int 1069 igb_check_mq_mode(struct rte_eth_dev *dev) 1070 { 1071 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1072 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1073 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1074 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1075 1076 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1077 tx_mq_mode == ETH_MQ_TX_DCB || 1078 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1079 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1080 return -EINVAL; 1081 } 1082 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1083 /* Check multi-queue mode. 1084 * To no break software we accept ETH_MQ_RX_NONE as this might 1085 * be used to turn off VLAN filter. 1086 */ 1087 1088 if (rx_mq_mode == ETH_MQ_RX_NONE || 1089 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1090 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1091 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1092 } else { 1093 /* Only support one queue on VFs. 1094 * RSS together with SRIOV is not supported. 1095 */ 1096 PMD_INIT_LOG(ERR, "SRIOV is active," 1097 " wrong mq_mode rx %d.", 1098 rx_mq_mode); 1099 return -EINVAL; 1100 } 1101 /* TX mode is not used here, so mode might be ignored.*/ 1102 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1103 /* SRIOV only works in VMDq enable mode */ 1104 PMD_INIT_LOG(WARNING, "SRIOV is active," 1105 " TX mode %d is not supported. " 1106 " Driver will behave as %d mode.", 1107 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1108 } 1109 1110 /* check valid queue number */ 1111 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1112 PMD_INIT_LOG(ERR, "SRIOV is active," 1113 " only support one queue on VFs."); 1114 return -EINVAL; 1115 } 1116 } else { 1117 /* To no break software that set invalid mode, only display 1118 * warning if invalid mode is used. 1119 */ 1120 if (rx_mq_mode != ETH_MQ_RX_NONE && 1121 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1122 rx_mq_mode != ETH_MQ_RX_RSS) { 1123 /* RSS together with VMDq not supported*/ 1124 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1125 rx_mq_mode); 1126 return -EINVAL; 1127 } 1128 1129 if (tx_mq_mode != ETH_MQ_TX_NONE && 1130 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1131 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1132 " Due to txmode is meaningless in this" 1133 " driver, just ignore.", 1134 tx_mq_mode); 1135 } 1136 } 1137 return 0; 1138 } 1139 1140 static int 1141 eth_igb_configure(struct rte_eth_dev *dev) 1142 { 1143 struct e1000_interrupt *intr = 1144 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1145 int ret; 1146 1147 PMD_INIT_FUNC_TRACE(); 1148 1149 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 1150 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1151 1152 /* multipe queue mode checking */ 1153 ret = igb_check_mq_mode(dev); 1154 if (ret != 0) { 1155 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1156 ret); 1157 return ret; 1158 } 1159 1160 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1161 PMD_INIT_FUNC_TRACE(); 1162 1163 return 0; 1164 } 1165 1166 static void 1167 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1168 bool enable) 1169 { 1170 struct e1000_hw *hw = 1171 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1172 uint32_t tctl, rctl; 1173 1174 tctl = E1000_READ_REG(hw, E1000_TCTL); 1175 rctl = E1000_READ_REG(hw, E1000_RCTL); 1176 1177 if (enable) { 1178 /* enable Tx/Rx */ 1179 tctl |= E1000_TCTL_EN; 1180 rctl |= E1000_RCTL_EN; 1181 } else { 1182 /* disable Tx/Rx */ 1183 tctl &= ~E1000_TCTL_EN; 1184 rctl &= ~E1000_RCTL_EN; 1185 } 1186 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1187 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1188 E1000_WRITE_FLUSH(hw); 1189 } 1190 1191 static int 1192 eth_igb_start(struct rte_eth_dev *dev) 1193 { 1194 struct e1000_hw *hw = 1195 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1196 struct e1000_adapter *adapter = 1197 E1000_DEV_PRIVATE(dev->data->dev_private); 1198 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1199 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1200 int ret, mask; 1201 uint32_t intr_vector = 0; 1202 uint32_t ctrl_ext; 1203 uint32_t *speeds; 1204 int num_speeds; 1205 bool autoneg; 1206 1207 PMD_INIT_FUNC_TRACE(); 1208 1209 /* disable uio/vfio intr/eventfd mapping */ 1210 rte_intr_disable(intr_handle); 1211 1212 /* Power up the phy. Needed to make the link go Up */ 1213 eth_igb_dev_set_link_up(dev); 1214 1215 /* 1216 * Packet Buffer Allocation (PBA) 1217 * Writing PBA sets the receive portion of the buffer 1218 * the remainder is used for the transmit buffer. 1219 */ 1220 if (hw->mac.type == e1000_82575) { 1221 uint32_t pba; 1222 1223 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1224 E1000_WRITE_REG(hw, E1000_PBA, pba); 1225 } 1226 1227 /* Put the address into the Receive Address Array */ 1228 e1000_rar_set(hw, hw->mac.addr, 0); 1229 1230 /* Initialize the hardware */ 1231 if (igb_hardware_init(hw)) { 1232 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1233 return -EIO; 1234 } 1235 adapter->stopped = 0; 1236 1237 E1000_WRITE_REG(hw, E1000_VET, 1238 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1239 1240 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1241 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1242 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1243 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1244 E1000_WRITE_FLUSH(hw); 1245 1246 /* configure PF module if SRIOV enabled */ 1247 igb_pf_host_configure(dev); 1248 1249 /* check and configure queue intr-vector mapping */ 1250 if ((rte_intr_cap_multiple(intr_handle) || 1251 !RTE_ETH_DEV_SRIOV(dev).active) && 1252 dev->data->dev_conf.intr_conf.rxq != 0) { 1253 intr_vector = dev->data->nb_rx_queues; 1254 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1255 return -1; 1256 } 1257 1258 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1259 intr_handle->intr_vec = 1260 rte_zmalloc("intr_vec", 1261 dev->data->nb_rx_queues * sizeof(int), 0); 1262 if (intr_handle->intr_vec == NULL) { 1263 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1264 " intr_vec", dev->data->nb_rx_queues); 1265 return -ENOMEM; 1266 } 1267 } 1268 1269 /* confiugre msix for rx interrupt */ 1270 eth_igb_configure_msix_intr(dev); 1271 1272 /* Configure for OS presence */ 1273 igb_init_manageability(hw); 1274 1275 eth_igb_tx_init(dev); 1276 1277 /* This can fail when allocating mbufs for descriptor rings */ 1278 ret = eth_igb_rx_init(dev); 1279 if (ret) { 1280 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1281 igb_dev_clear_queues(dev); 1282 return ret; 1283 } 1284 1285 e1000_clear_hw_cntrs_base_generic(hw); 1286 1287 /* 1288 * VLAN Offload Settings 1289 */ 1290 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1291 ETH_VLAN_EXTEND_MASK; 1292 ret = eth_igb_vlan_offload_set(dev, mask); 1293 if (ret) { 1294 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1295 igb_dev_clear_queues(dev); 1296 return ret; 1297 } 1298 1299 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1300 /* Enable VLAN filter since VMDq always use VLAN filter */ 1301 igb_vmdq_vlan_hw_filter_enable(dev); 1302 } 1303 1304 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1305 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1306 (hw->mac.type == e1000_i211)) { 1307 /* Configure EITR with the maximum possible value (0xFFFF) */ 1308 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1309 } 1310 1311 /* Setup link speed and duplex */ 1312 speeds = &dev->data->dev_conf.link_speeds; 1313 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1314 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1315 hw->mac.autoneg = 1; 1316 } else { 1317 num_speeds = 0; 1318 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1319 1320 /* Reset */ 1321 hw->phy.autoneg_advertised = 0; 1322 1323 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1324 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1325 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1326 num_speeds = -1; 1327 goto error_invalid_config; 1328 } 1329 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1330 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1331 num_speeds++; 1332 } 1333 if (*speeds & ETH_LINK_SPEED_10M) { 1334 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1335 num_speeds++; 1336 } 1337 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1338 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1339 num_speeds++; 1340 } 1341 if (*speeds & ETH_LINK_SPEED_100M) { 1342 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1343 num_speeds++; 1344 } 1345 if (*speeds & ETH_LINK_SPEED_1G) { 1346 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1347 num_speeds++; 1348 } 1349 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1350 goto error_invalid_config; 1351 1352 /* Set/reset the mac.autoneg based on the link speed, 1353 * fixed or not 1354 */ 1355 if (!autoneg) { 1356 hw->mac.autoneg = 0; 1357 hw->mac.forced_speed_duplex = 1358 hw->phy.autoneg_advertised; 1359 } else { 1360 hw->mac.autoneg = 1; 1361 } 1362 } 1363 1364 e1000_setup_link(hw); 1365 1366 if (rte_intr_allow_others(intr_handle)) { 1367 /* check if lsc interrupt is enabled */ 1368 if (dev->data->dev_conf.intr_conf.lsc != 0) 1369 eth_igb_lsc_interrupt_setup(dev, TRUE); 1370 else 1371 eth_igb_lsc_interrupt_setup(dev, FALSE); 1372 } else { 1373 rte_intr_callback_unregister(intr_handle, 1374 eth_igb_interrupt_handler, 1375 (void *)dev); 1376 if (dev->data->dev_conf.intr_conf.lsc != 0) 1377 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1378 " no intr multiplex"); 1379 } 1380 1381 /* check if rxq interrupt is enabled */ 1382 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1383 rte_intr_dp_is_en(intr_handle)) 1384 eth_igb_rxq_interrupt_setup(dev); 1385 1386 /* enable uio/vfio intr/eventfd mapping */ 1387 rte_intr_enable(intr_handle); 1388 1389 /* resume enabled intr since hw reset */ 1390 igb_intr_enable(dev); 1391 1392 /* restore all types filter */ 1393 igb_filter_restore(dev); 1394 1395 eth_igb_rxtx_control(dev, true); 1396 eth_igb_link_update(dev, 0); 1397 1398 PMD_INIT_LOG(DEBUG, "<<"); 1399 1400 return 0; 1401 1402 error_invalid_config: 1403 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1404 dev->data->dev_conf.link_speeds, dev->data->port_id); 1405 igb_dev_clear_queues(dev); 1406 return -EINVAL; 1407 } 1408 1409 /********************************************************************* 1410 * 1411 * This routine disables all traffic on the adapter by issuing a 1412 * global reset on the MAC. 1413 * 1414 **********************************************************************/ 1415 static int 1416 eth_igb_stop(struct rte_eth_dev *dev) 1417 { 1418 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1419 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1420 struct rte_eth_link link; 1421 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1422 struct e1000_adapter *adapter = 1423 E1000_DEV_PRIVATE(dev->data->dev_private); 1424 1425 if (adapter->stopped) 1426 return 0; 1427 1428 eth_igb_rxtx_control(dev, false); 1429 1430 igb_intr_disable(dev); 1431 1432 /* disable intr eventfd mapping */ 1433 rte_intr_disable(intr_handle); 1434 1435 igb_pf_reset_hw(hw); 1436 E1000_WRITE_REG(hw, E1000_WUC, 0); 1437 1438 /* Set bit for Go Link disconnect if PHY reset is not blocked */ 1439 if (hw->mac.type >= e1000_82580 && 1440 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1441 uint32_t phpm_reg; 1442 1443 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1444 phpm_reg |= E1000_82580_PM_GO_LINKD; 1445 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1446 } 1447 1448 /* Power down the phy. Needed to make the link go Down */ 1449 eth_igb_dev_set_link_down(dev); 1450 1451 igb_dev_clear_queues(dev); 1452 1453 /* clear the recorded link status */ 1454 memset(&link, 0, sizeof(link)); 1455 rte_eth_linkstatus_set(dev, &link); 1456 1457 if (!rte_intr_allow_others(intr_handle)) 1458 /* resume to the default handler */ 1459 rte_intr_callback_register(intr_handle, 1460 eth_igb_interrupt_handler, 1461 (void *)dev); 1462 1463 /* Clean datapath event and queue/vec mapping */ 1464 rte_intr_efd_disable(intr_handle); 1465 if (intr_handle->intr_vec != NULL) { 1466 rte_free(intr_handle->intr_vec); 1467 intr_handle->intr_vec = NULL; 1468 } 1469 1470 adapter->stopped = true; 1471 dev->data->dev_started = 0; 1472 1473 return 0; 1474 } 1475 1476 static int 1477 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1478 { 1479 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1480 1481 if (hw->phy.media_type == e1000_media_type_copper) 1482 e1000_power_up_phy(hw); 1483 else 1484 e1000_power_up_fiber_serdes_link(hw); 1485 1486 return 0; 1487 } 1488 1489 static int 1490 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1491 { 1492 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1493 1494 if (hw->phy.media_type == e1000_media_type_copper) 1495 e1000_power_down_phy(hw); 1496 else 1497 e1000_shutdown_fiber_serdes_link(hw); 1498 1499 return 0; 1500 } 1501 1502 static int 1503 eth_igb_close(struct rte_eth_dev *dev) 1504 { 1505 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1506 struct rte_eth_link link; 1507 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1508 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1509 struct e1000_filter_info *filter_info = 1510 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 1511 int ret; 1512 1513 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1514 return 0; 1515 1516 ret = eth_igb_stop(dev); 1517 1518 e1000_phy_hw_reset(hw); 1519 igb_release_manageability(hw); 1520 igb_hw_control_release(hw); 1521 1522 /* Clear bit for Go Link disconnect if PHY reset is not blocked */ 1523 if (hw->mac.type >= e1000_82580 && 1524 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1525 uint32_t phpm_reg; 1526 1527 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1528 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1529 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1530 } 1531 1532 igb_dev_free_queues(dev); 1533 1534 if (intr_handle->intr_vec) { 1535 rte_free(intr_handle->intr_vec); 1536 intr_handle->intr_vec = NULL; 1537 } 1538 1539 memset(&link, 0, sizeof(link)); 1540 rte_eth_linkstatus_set(dev, &link); 1541 1542 /* Reset any pending lock */ 1543 igb_reset_swfw_lock(hw); 1544 1545 /* uninitialize PF if max_vfs not zero */ 1546 igb_pf_host_uninit(dev); 1547 1548 rte_intr_callback_unregister(intr_handle, 1549 eth_igb_interrupt_handler, dev); 1550 1551 /* clear the SYN filter info */ 1552 filter_info->syn_info = 0; 1553 1554 /* clear the ethertype filters info */ 1555 filter_info->ethertype_mask = 0; 1556 memset(filter_info->ethertype_filters, 0, 1557 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 1558 1559 /* clear the rss filter info */ 1560 memset(&filter_info->rss_info, 0, 1561 sizeof(struct igb_rte_flow_rss_conf)); 1562 1563 /* remove all ntuple filters of the device */ 1564 igb_ntuple_filter_uninit(dev); 1565 1566 /* remove all flex filters of the device */ 1567 igb_flex_filter_uninit(dev); 1568 1569 /* clear all the filters list */ 1570 igb_filterlist_flush(dev); 1571 1572 return ret; 1573 } 1574 1575 /* 1576 * Reset PF device. 1577 */ 1578 static int 1579 eth_igb_reset(struct rte_eth_dev *dev) 1580 { 1581 int ret; 1582 1583 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1584 * its VF to make them align with it. The detailed notification 1585 * mechanism is PMD specific and is currently not implemented. 1586 * To avoid unexpected behavior in VF, currently reset of PF with 1587 * SR-IOV activation is not supported. It might be supported later. 1588 */ 1589 if (dev->data->sriov.active) 1590 return -ENOTSUP; 1591 1592 ret = eth_igb_dev_uninit(dev); 1593 if (ret) 1594 return ret; 1595 1596 ret = eth_igb_dev_init(dev); 1597 1598 return ret; 1599 } 1600 1601 1602 static int 1603 igb_get_rx_buffer_size(struct e1000_hw *hw) 1604 { 1605 uint32_t rx_buf_size; 1606 if (hw->mac.type == e1000_82576) { 1607 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1608 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1609 /* PBS needs to be translated according to a lookup table */ 1610 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1611 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1612 rx_buf_size = (rx_buf_size << 10); 1613 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1614 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1615 } else { 1616 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1617 } 1618 1619 return rx_buf_size; 1620 } 1621 1622 /********************************************************************* 1623 * 1624 * Initialize the hardware 1625 * 1626 **********************************************************************/ 1627 static int 1628 igb_hardware_init(struct e1000_hw *hw) 1629 { 1630 uint32_t rx_buf_size; 1631 int diag; 1632 1633 /* Let the firmware know the OS is in control */ 1634 igb_hw_control_acquire(hw); 1635 1636 /* 1637 * These parameters control the automatic generation (Tx) and 1638 * response (Rx) to Ethernet PAUSE frames. 1639 * - High water mark should allow for at least two standard size (1518) 1640 * frames to be received after sending an XOFF. 1641 * - Low water mark works best when it is very near the high water mark. 1642 * This allows the receiver to restart by sending XON when it has 1643 * drained a bit. Here we use an arbitrary value of 1500 which will 1644 * restart after one full frame is pulled from the buffer. There 1645 * could be several smaller frames in the buffer and if so they will 1646 * not trigger the XON until their total number reduces the buffer 1647 * by 1500. 1648 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1649 */ 1650 rx_buf_size = igb_get_rx_buffer_size(hw); 1651 1652 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2); 1653 hw->fc.low_water = hw->fc.high_water - 1500; 1654 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1655 hw->fc.send_xon = 1; 1656 1657 /* Set Flow control, use the tunable location if sane */ 1658 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1659 hw->fc.requested_mode = igb_fc_setting; 1660 else 1661 hw->fc.requested_mode = e1000_fc_none; 1662 1663 /* Issue a global reset */ 1664 igb_pf_reset_hw(hw); 1665 E1000_WRITE_REG(hw, E1000_WUC, 0); 1666 1667 diag = e1000_init_hw(hw); 1668 if (diag < 0) 1669 return diag; 1670 1671 E1000_WRITE_REG(hw, E1000_VET, 1672 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1673 e1000_get_phy_info(hw); 1674 e1000_check_for_link(hw); 1675 1676 return 0; 1677 } 1678 1679 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1680 static void 1681 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1682 { 1683 int pause_frames; 1684 1685 uint64_t old_gprc = stats->gprc; 1686 uint64_t old_gptc = stats->gptc; 1687 uint64_t old_tpr = stats->tpr; 1688 uint64_t old_tpt = stats->tpt; 1689 uint64_t old_rpthc = stats->rpthc; 1690 uint64_t old_hgptc = stats->hgptc; 1691 1692 if(hw->phy.media_type == e1000_media_type_copper || 1693 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1694 stats->symerrs += 1695 E1000_READ_REG(hw,E1000_SYMERRS); 1696 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1697 } 1698 1699 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1700 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1701 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1702 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1703 1704 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1705 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1706 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1707 stats->dc += E1000_READ_REG(hw, E1000_DC); 1708 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1709 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1710 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1711 /* 1712 ** For watchdog management we need to know if we have been 1713 ** paused during the last interval, so capture that here. 1714 */ 1715 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1716 stats->xoffrxc += pause_frames; 1717 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1718 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1719 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1720 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1721 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1722 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1723 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1724 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1725 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1726 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1727 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1728 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1729 1730 /* For the 64-bit byte counters the low dword must be read first. */ 1731 /* Both registers clear on the read of the high dword */ 1732 1733 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1734 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1735 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1736 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN; 1737 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1738 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1739 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN; 1740 1741 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1742 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1743 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1744 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1745 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1746 1747 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1748 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1749 1750 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1751 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1752 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 1753 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1754 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1755 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN; 1756 1757 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1758 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1759 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1760 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1761 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1762 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1763 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1764 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1765 1766 /* Interrupt Counts */ 1767 1768 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1769 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1770 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1771 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1772 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1773 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1774 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1775 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1776 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1777 1778 /* Host to Card Statistics */ 1779 1780 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1781 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1782 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1783 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1784 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1785 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1786 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1787 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1788 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1789 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN; 1790 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1791 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1792 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN; 1793 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1794 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1795 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1796 1797 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1798 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1799 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1800 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1801 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1802 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1803 } 1804 1805 static int 1806 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1807 { 1808 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1809 struct e1000_hw_stats *stats = 1810 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1811 1812 igb_read_stats_registers(hw, stats); 1813 1814 if (rte_stats == NULL) 1815 return -EINVAL; 1816 1817 /* Rx Errors */ 1818 rte_stats->imissed = stats->mpc; 1819 rte_stats->ierrors = stats->crcerrs + stats->rlec + 1820 stats->rxerrc + stats->algnerrc + stats->cexterr; 1821 1822 /* Tx Errors */ 1823 rte_stats->oerrors = stats->ecol + stats->latecol; 1824 1825 rte_stats->ipackets = stats->gprc; 1826 rte_stats->opackets = stats->gptc; 1827 rte_stats->ibytes = stats->gorc; 1828 rte_stats->obytes = stats->gotc; 1829 return 0; 1830 } 1831 1832 static int 1833 eth_igb_stats_reset(struct rte_eth_dev *dev) 1834 { 1835 struct e1000_hw_stats *hw_stats = 1836 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1837 1838 /* HW registers are cleared on read */ 1839 eth_igb_stats_get(dev, NULL); 1840 1841 /* Reset software totals */ 1842 memset(hw_stats, 0, sizeof(*hw_stats)); 1843 1844 return 0; 1845 } 1846 1847 static int 1848 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1849 { 1850 struct e1000_hw_stats *stats = 1851 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1852 1853 /* HW registers are cleared on read */ 1854 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1855 1856 /* Reset software totals */ 1857 memset(stats, 0, sizeof(*stats)); 1858 1859 return 0; 1860 } 1861 1862 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1863 struct rte_eth_xstat_name *xstats_names, 1864 __rte_unused unsigned int size) 1865 { 1866 unsigned i; 1867 1868 if (xstats_names == NULL) 1869 return IGB_NB_XSTATS; 1870 1871 /* Note: limit checked in rte_eth_xstats_names() */ 1872 1873 for (i = 0; i < IGB_NB_XSTATS; i++) { 1874 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1875 sizeof(xstats_names[i].name)); 1876 } 1877 1878 return IGB_NB_XSTATS; 1879 } 1880 1881 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1882 const uint64_t *ids, struct rte_eth_xstat_name *xstats_names, 1883 unsigned int limit) 1884 { 1885 unsigned int i; 1886 1887 if (!ids) { 1888 if (xstats_names == NULL) 1889 return IGB_NB_XSTATS; 1890 1891 for (i = 0; i < IGB_NB_XSTATS; i++) 1892 strlcpy(xstats_names[i].name, 1893 rte_igb_stats_strings[i].name, 1894 sizeof(xstats_names[i].name)); 1895 1896 return IGB_NB_XSTATS; 1897 1898 } else { 1899 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1900 1901 eth_igb_xstats_get_names_by_id(dev, NULL, xstats_names_copy, 1902 IGB_NB_XSTATS); 1903 1904 for (i = 0; i < limit; i++) { 1905 if (ids[i] >= IGB_NB_XSTATS) { 1906 PMD_INIT_LOG(ERR, "id value isn't valid"); 1907 return -1; 1908 } 1909 strcpy(xstats_names[i].name, 1910 xstats_names_copy[ids[i]].name); 1911 } 1912 return limit; 1913 } 1914 } 1915 1916 static int 1917 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1918 unsigned n) 1919 { 1920 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1921 struct e1000_hw_stats *hw_stats = 1922 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1923 unsigned i; 1924 1925 if (n < IGB_NB_XSTATS) 1926 return IGB_NB_XSTATS; 1927 1928 igb_read_stats_registers(hw, hw_stats); 1929 1930 /* If this is a reset xstats is NULL, and we have cleared the 1931 * registers by reading them. 1932 */ 1933 if (!xstats) 1934 return 0; 1935 1936 /* Extended stats */ 1937 for (i = 0; i < IGB_NB_XSTATS; i++) { 1938 xstats[i].id = i; 1939 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1940 rte_igb_stats_strings[i].offset); 1941 } 1942 1943 return IGB_NB_XSTATS; 1944 } 1945 1946 static int 1947 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1948 uint64_t *values, unsigned int n) 1949 { 1950 unsigned int i; 1951 1952 if (!ids) { 1953 struct e1000_hw *hw = 1954 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1955 struct e1000_hw_stats *hw_stats = 1956 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1957 1958 if (n < IGB_NB_XSTATS) 1959 return IGB_NB_XSTATS; 1960 1961 igb_read_stats_registers(hw, hw_stats); 1962 1963 /* If this is a reset xstats is NULL, and we have cleared the 1964 * registers by reading them. 1965 */ 1966 if (!values) 1967 return 0; 1968 1969 /* Extended stats */ 1970 for (i = 0; i < IGB_NB_XSTATS; i++) 1971 values[i] = *(uint64_t *)(((char *)hw_stats) + 1972 rte_igb_stats_strings[i].offset); 1973 1974 return IGB_NB_XSTATS; 1975 1976 } else { 1977 uint64_t values_copy[IGB_NB_XSTATS]; 1978 1979 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 1980 IGB_NB_XSTATS); 1981 1982 for (i = 0; i < n; i++) { 1983 if (ids[i] >= IGB_NB_XSTATS) { 1984 PMD_INIT_LOG(ERR, "id value isn't valid"); 1985 return -1; 1986 } 1987 values[i] = values_copy[ids[i]]; 1988 } 1989 return n; 1990 } 1991 } 1992 1993 static void 1994 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 1995 { 1996 /* Good Rx packets, include VF loopback */ 1997 UPDATE_VF_STAT(E1000_VFGPRC, 1998 hw_stats->last_gprc, hw_stats->gprc); 1999 2000 /* Good Rx octets, include VF loopback */ 2001 UPDATE_VF_STAT(E1000_VFGORC, 2002 hw_stats->last_gorc, hw_stats->gorc); 2003 2004 /* Good Tx packets, include VF loopback */ 2005 UPDATE_VF_STAT(E1000_VFGPTC, 2006 hw_stats->last_gptc, hw_stats->gptc); 2007 2008 /* Good Tx octets, include VF loopback */ 2009 UPDATE_VF_STAT(E1000_VFGOTC, 2010 hw_stats->last_gotc, hw_stats->gotc); 2011 2012 /* Rx Multicst packets */ 2013 UPDATE_VF_STAT(E1000_VFMPRC, 2014 hw_stats->last_mprc, hw_stats->mprc); 2015 2016 /* Good Rx loopback packets */ 2017 UPDATE_VF_STAT(E1000_VFGPRLBC, 2018 hw_stats->last_gprlbc, hw_stats->gprlbc); 2019 2020 /* Good Rx loopback octets */ 2021 UPDATE_VF_STAT(E1000_VFGORLBC, 2022 hw_stats->last_gorlbc, hw_stats->gorlbc); 2023 2024 /* Good Tx loopback packets */ 2025 UPDATE_VF_STAT(E1000_VFGPTLBC, 2026 hw_stats->last_gptlbc, hw_stats->gptlbc); 2027 2028 /* Good Tx loopback octets */ 2029 UPDATE_VF_STAT(E1000_VFGOTLBC, 2030 hw_stats->last_gotlbc, hw_stats->gotlbc); 2031 } 2032 2033 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2034 struct rte_eth_xstat_name *xstats_names, 2035 __rte_unused unsigned limit) 2036 { 2037 unsigned i; 2038 2039 if (xstats_names != NULL) 2040 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2041 strlcpy(xstats_names[i].name, 2042 rte_igbvf_stats_strings[i].name, 2043 sizeof(xstats_names[i].name)); 2044 } 2045 return IGBVF_NB_XSTATS; 2046 } 2047 2048 static int 2049 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2050 unsigned n) 2051 { 2052 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2053 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2054 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2055 unsigned i; 2056 2057 if (n < IGBVF_NB_XSTATS) 2058 return IGBVF_NB_XSTATS; 2059 2060 igbvf_read_stats_registers(hw, hw_stats); 2061 2062 if (!xstats) 2063 return 0; 2064 2065 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2066 xstats[i].id = i; 2067 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2068 rte_igbvf_stats_strings[i].offset); 2069 } 2070 2071 return IGBVF_NB_XSTATS; 2072 } 2073 2074 static int 2075 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2076 { 2077 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2078 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2079 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2080 2081 igbvf_read_stats_registers(hw, hw_stats); 2082 2083 if (rte_stats == NULL) 2084 return -EINVAL; 2085 2086 rte_stats->ipackets = hw_stats->gprc; 2087 rte_stats->ibytes = hw_stats->gorc; 2088 rte_stats->opackets = hw_stats->gptc; 2089 rte_stats->obytes = hw_stats->gotc; 2090 return 0; 2091 } 2092 2093 static int 2094 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2095 { 2096 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2097 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2098 2099 /* Sync HW register to the last stats */ 2100 eth_igbvf_stats_get(dev, NULL); 2101 2102 /* reset HW current stats*/ 2103 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2104 offsetof(struct e1000_vf_stats, gprc)); 2105 2106 return 0; 2107 } 2108 2109 static int 2110 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2111 size_t fw_size) 2112 { 2113 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2114 struct e1000_fw_version fw; 2115 int ret; 2116 2117 e1000_get_fw_version(hw, &fw); 2118 2119 switch (hw->mac.type) { 2120 case e1000_i210: 2121 case e1000_i211: 2122 if (!(e1000_get_flash_presence_i210(hw))) { 2123 ret = snprintf(fw_version, fw_size, 2124 "%2d.%2d-%d", 2125 fw.invm_major, fw.invm_minor, 2126 fw.invm_img_type); 2127 break; 2128 } 2129 /* fall through */ 2130 default: 2131 /* if option rom is valid, display its version too */ 2132 if (fw.or_valid) { 2133 ret = snprintf(fw_version, fw_size, 2134 "%d.%d, 0x%08x, %d.%d.%d", 2135 fw.eep_major, fw.eep_minor, fw.etrack_id, 2136 fw.or_major, fw.or_build, fw.or_patch); 2137 /* no option rom */ 2138 } else { 2139 if (fw.etrack_id != 0X0000) { 2140 ret = snprintf(fw_version, fw_size, 2141 "%d.%d, 0x%08x", 2142 fw.eep_major, fw.eep_minor, 2143 fw.etrack_id); 2144 } else { 2145 ret = snprintf(fw_version, fw_size, 2146 "%d.%d.%d", 2147 fw.eep_major, fw.eep_minor, 2148 fw.eep_build); 2149 } 2150 } 2151 break; 2152 } 2153 if (ret < 0) 2154 return -EINVAL; 2155 2156 ret += 1; /* add the size of '\0' */ 2157 if (fw_size < (size_t)ret) 2158 return ret; 2159 else 2160 return 0; 2161 } 2162 2163 static int 2164 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2165 { 2166 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2167 2168 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2169 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2170 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2171 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2172 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2173 dev_info->rx_queue_offload_capa; 2174 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2175 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2176 dev_info->tx_queue_offload_capa; 2177 2178 switch (hw->mac.type) { 2179 case e1000_82575: 2180 dev_info->max_rx_queues = 4; 2181 dev_info->max_tx_queues = 4; 2182 dev_info->max_vmdq_pools = 0; 2183 break; 2184 2185 case e1000_82576: 2186 dev_info->max_rx_queues = 16; 2187 dev_info->max_tx_queues = 16; 2188 dev_info->max_vmdq_pools = ETH_8_POOLS; 2189 dev_info->vmdq_queue_num = 16; 2190 break; 2191 2192 case e1000_82580: 2193 dev_info->max_rx_queues = 8; 2194 dev_info->max_tx_queues = 8; 2195 dev_info->max_vmdq_pools = ETH_8_POOLS; 2196 dev_info->vmdq_queue_num = 8; 2197 break; 2198 2199 case e1000_i350: 2200 dev_info->max_rx_queues = 8; 2201 dev_info->max_tx_queues = 8; 2202 dev_info->max_vmdq_pools = ETH_8_POOLS; 2203 dev_info->vmdq_queue_num = 8; 2204 break; 2205 2206 case e1000_i354: 2207 dev_info->max_rx_queues = 8; 2208 dev_info->max_tx_queues = 8; 2209 break; 2210 2211 case e1000_i210: 2212 dev_info->max_rx_queues = 4; 2213 dev_info->max_tx_queues = 4; 2214 dev_info->max_vmdq_pools = 0; 2215 break; 2216 2217 case e1000_i211: 2218 dev_info->max_rx_queues = 2; 2219 dev_info->max_tx_queues = 2; 2220 dev_info->max_vmdq_pools = 0; 2221 break; 2222 2223 default: 2224 /* Should not happen */ 2225 return -EINVAL; 2226 } 2227 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2228 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2229 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2230 2231 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2232 .rx_thresh = { 2233 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2234 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2235 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2236 }, 2237 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2238 .rx_drop_en = 0, 2239 .offloads = 0, 2240 }; 2241 2242 dev_info->default_txconf = (struct rte_eth_txconf) { 2243 .tx_thresh = { 2244 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2245 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2246 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2247 }, 2248 .offloads = 0, 2249 }; 2250 2251 dev_info->rx_desc_lim = rx_desc_lim; 2252 dev_info->tx_desc_lim = tx_desc_lim; 2253 2254 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2255 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2256 ETH_LINK_SPEED_1G; 2257 2258 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2259 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 2260 2261 return 0; 2262 } 2263 2264 static const uint32_t * 2265 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2266 { 2267 static const uint32_t ptypes[] = { 2268 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2269 RTE_PTYPE_L2_ETHER, 2270 RTE_PTYPE_L3_IPV4, 2271 RTE_PTYPE_L3_IPV4_EXT, 2272 RTE_PTYPE_L3_IPV6, 2273 RTE_PTYPE_L3_IPV6_EXT, 2274 RTE_PTYPE_L4_TCP, 2275 RTE_PTYPE_L4_UDP, 2276 RTE_PTYPE_L4_SCTP, 2277 RTE_PTYPE_TUNNEL_IP, 2278 RTE_PTYPE_INNER_L3_IPV6, 2279 RTE_PTYPE_INNER_L3_IPV6_EXT, 2280 RTE_PTYPE_INNER_L4_TCP, 2281 RTE_PTYPE_INNER_L4_UDP, 2282 RTE_PTYPE_UNKNOWN 2283 }; 2284 2285 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2286 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2287 return ptypes; 2288 return NULL; 2289 } 2290 2291 static int 2292 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2293 { 2294 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2295 2296 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2297 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2298 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2299 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2300 DEV_TX_OFFLOAD_IPV4_CKSUM | 2301 DEV_TX_OFFLOAD_UDP_CKSUM | 2302 DEV_TX_OFFLOAD_TCP_CKSUM | 2303 DEV_TX_OFFLOAD_SCTP_CKSUM | 2304 DEV_TX_OFFLOAD_TCP_TSO; 2305 switch (hw->mac.type) { 2306 case e1000_vfadapt: 2307 dev_info->max_rx_queues = 2; 2308 dev_info->max_tx_queues = 2; 2309 break; 2310 case e1000_vfadapt_i350: 2311 dev_info->max_rx_queues = 1; 2312 dev_info->max_tx_queues = 1; 2313 break; 2314 default: 2315 /* Should not happen */ 2316 return -EINVAL; 2317 } 2318 2319 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2320 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2321 dev_info->rx_queue_offload_capa; 2322 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2323 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2324 dev_info->tx_queue_offload_capa; 2325 2326 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2327 .rx_thresh = { 2328 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2329 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2330 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2331 }, 2332 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2333 .rx_drop_en = 0, 2334 .offloads = 0, 2335 }; 2336 2337 dev_info->default_txconf = (struct rte_eth_txconf) { 2338 .tx_thresh = { 2339 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2340 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2341 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2342 }, 2343 .offloads = 0, 2344 }; 2345 2346 dev_info->rx_desc_lim = rx_desc_lim; 2347 dev_info->tx_desc_lim = tx_desc_lim; 2348 2349 return 0; 2350 } 2351 2352 /* return 0 means link status changed, -1 means not changed */ 2353 static int 2354 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2355 { 2356 struct e1000_hw *hw = 2357 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2358 struct rte_eth_link link; 2359 int link_check, count; 2360 2361 link_check = 0; 2362 hw->mac.get_link_status = 1; 2363 2364 /* possible wait-to-complete in up to 9 seconds */ 2365 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2366 /* Read the real link status */ 2367 switch (hw->phy.media_type) { 2368 case e1000_media_type_copper: 2369 /* Do the work to read phy */ 2370 e1000_check_for_link(hw); 2371 link_check = !hw->mac.get_link_status; 2372 break; 2373 2374 case e1000_media_type_fiber: 2375 e1000_check_for_link(hw); 2376 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2377 E1000_STATUS_LU); 2378 break; 2379 2380 case e1000_media_type_internal_serdes: 2381 e1000_check_for_link(hw); 2382 link_check = hw->mac.serdes_has_link; 2383 break; 2384 2385 /* VF device is type_unknown */ 2386 case e1000_media_type_unknown: 2387 eth_igbvf_link_update(hw); 2388 link_check = !hw->mac.get_link_status; 2389 break; 2390 2391 default: 2392 break; 2393 } 2394 if (link_check || wait_to_complete == 0) 2395 break; 2396 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2397 } 2398 memset(&link, 0, sizeof(link)); 2399 2400 /* Now we check if a transition has happened */ 2401 if (link_check) { 2402 uint16_t duplex, speed; 2403 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2404 link.link_duplex = (duplex == FULL_DUPLEX) ? 2405 ETH_LINK_FULL_DUPLEX : 2406 ETH_LINK_HALF_DUPLEX; 2407 link.link_speed = speed; 2408 link.link_status = ETH_LINK_UP; 2409 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2410 ETH_LINK_SPEED_FIXED); 2411 } else if (!link_check) { 2412 link.link_speed = 0; 2413 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2414 link.link_status = ETH_LINK_DOWN; 2415 link.link_autoneg = ETH_LINK_FIXED; 2416 } 2417 2418 return rte_eth_linkstatus_set(dev, &link); 2419 } 2420 2421 /* 2422 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2423 * For ASF and Pass Through versions of f/w this means 2424 * that the driver is loaded. 2425 */ 2426 static void 2427 igb_hw_control_acquire(struct e1000_hw *hw) 2428 { 2429 uint32_t ctrl_ext; 2430 2431 /* Let firmware know the driver has taken over */ 2432 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2433 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2434 } 2435 2436 /* 2437 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2438 * For ASF and Pass Through versions of f/w this means that the 2439 * driver is no longer loaded. 2440 */ 2441 static void 2442 igb_hw_control_release(struct e1000_hw *hw) 2443 { 2444 uint32_t ctrl_ext; 2445 2446 /* Let firmware taken over control of h/w */ 2447 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2448 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2449 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2450 } 2451 2452 /* 2453 * Bit of a misnomer, what this really means is 2454 * to enable OS management of the system... aka 2455 * to disable special hardware management features. 2456 */ 2457 static void 2458 igb_init_manageability(struct e1000_hw *hw) 2459 { 2460 if (e1000_enable_mng_pass_thru(hw)) { 2461 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2462 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2463 2464 /* disable hardware interception of ARP */ 2465 manc &= ~(E1000_MANC_ARP_EN); 2466 2467 /* enable receiving management packets to the host */ 2468 manc |= E1000_MANC_EN_MNG2HOST; 2469 manc2h |= 1 << 5; /* Mng Port 623 */ 2470 manc2h |= 1 << 6; /* Mng Port 664 */ 2471 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2472 E1000_WRITE_REG(hw, E1000_MANC, manc); 2473 } 2474 } 2475 2476 static void 2477 igb_release_manageability(struct e1000_hw *hw) 2478 { 2479 if (e1000_enable_mng_pass_thru(hw)) { 2480 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2481 2482 manc |= E1000_MANC_ARP_EN; 2483 manc &= ~E1000_MANC_EN_MNG2HOST; 2484 2485 E1000_WRITE_REG(hw, E1000_MANC, manc); 2486 } 2487 } 2488 2489 static int 2490 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2491 { 2492 struct e1000_hw *hw = 2493 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2494 uint32_t rctl; 2495 2496 rctl = E1000_READ_REG(hw, E1000_RCTL); 2497 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2498 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2499 2500 return 0; 2501 } 2502 2503 static int 2504 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2505 { 2506 struct e1000_hw *hw = 2507 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2508 uint32_t rctl; 2509 2510 rctl = E1000_READ_REG(hw, E1000_RCTL); 2511 rctl &= (~E1000_RCTL_UPE); 2512 if (dev->data->all_multicast == 1) 2513 rctl |= E1000_RCTL_MPE; 2514 else 2515 rctl &= (~E1000_RCTL_MPE); 2516 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2517 2518 return 0; 2519 } 2520 2521 static int 2522 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2523 { 2524 struct e1000_hw *hw = 2525 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2526 uint32_t rctl; 2527 2528 rctl = E1000_READ_REG(hw, E1000_RCTL); 2529 rctl |= E1000_RCTL_MPE; 2530 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2531 2532 return 0; 2533 } 2534 2535 static int 2536 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2537 { 2538 struct e1000_hw *hw = 2539 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2540 uint32_t rctl; 2541 2542 if (dev->data->promiscuous == 1) 2543 return 0; /* must remain in all_multicast mode */ 2544 rctl = E1000_READ_REG(hw, E1000_RCTL); 2545 rctl &= (~E1000_RCTL_MPE); 2546 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2547 2548 return 0; 2549 } 2550 2551 static int 2552 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2553 { 2554 struct e1000_hw *hw = 2555 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2556 struct e1000_vfta * shadow_vfta = 2557 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2558 uint32_t vfta; 2559 uint32_t vid_idx; 2560 uint32_t vid_bit; 2561 2562 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2563 E1000_VFTA_ENTRY_MASK); 2564 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2565 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2566 if (on) 2567 vfta |= vid_bit; 2568 else 2569 vfta &= ~vid_bit; 2570 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2571 2572 /* update local VFTA copy */ 2573 shadow_vfta->vfta[vid_idx] = vfta; 2574 2575 return 0; 2576 } 2577 2578 static int 2579 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2580 enum rte_vlan_type vlan_type, 2581 uint16_t tpid) 2582 { 2583 struct e1000_hw *hw = 2584 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2585 uint32_t reg, qinq; 2586 2587 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2588 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2589 2590 /* only outer TPID of double VLAN can be configured*/ 2591 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2592 reg = E1000_READ_REG(hw, E1000_VET); 2593 reg = (reg & (~E1000_VET_VET_EXT)) | 2594 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2595 E1000_WRITE_REG(hw, E1000_VET, reg); 2596 2597 return 0; 2598 } 2599 2600 /* all other TPID values are read-only*/ 2601 PMD_DRV_LOG(ERR, "Not supported"); 2602 2603 return -ENOTSUP; 2604 } 2605 2606 static void 2607 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2608 { 2609 struct e1000_hw *hw = 2610 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2611 uint32_t reg; 2612 2613 /* Filter Table Disable */ 2614 reg = E1000_READ_REG(hw, E1000_RCTL); 2615 reg &= ~E1000_RCTL_CFIEN; 2616 reg &= ~E1000_RCTL_VFE; 2617 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2618 } 2619 2620 static void 2621 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2622 { 2623 struct e1000_hw *hw = 2624 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2625 struct e1000_vfta * shadow_vfta = 2626 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2627 uint32_t reg; 2628 int i; 2629 2630 /* Filter Table Enable, CFI not used for packet acceptance */ 2631 reg = E1000_READ_REG(hw, E1000_RCTL); 2632 reg &= ~E1000_RCTL_CFIEN; 2633 reg |= E1000_RCTL_VFE; 2634 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2635 2636 /* restore VFTA table */ 2637 for (i = 0; i < IGB_VFTA_SIZE; i++) 2638 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2639 } 2640 2641 static void 2642 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2643 { 2644 struct e1000_hw *hw = 2645 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2646 uint32_t reg; 2647 2648 /* VLAN Mode Disable */ 2649 reg = E1000_READ_REG(hw, E1000_CTRL); 2650 reg &= ~E1000_CTRL_VME; 2651 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2652 } 2653 2654 static void 2655 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2656 { 2657 struct e1000_hw *hw = 2658 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2659 uint32_t reg; 2660 2661 /* VLAN Mode Enable */ 2662 reg = E1000_READ_REG(hw, E1000_CTRL); 2663 reg |= E1000_CTRL_VME; 2664 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2665 } 2666 2667 static void 2668 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2669 { 2670 struct e1000_hw *hw = 2671 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2672 uint32_t reg; 2673 2674 /* CTRL_EXT: Extended VLAN */ 2675 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2676 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2677 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2678 2679 /* Update maximum packet length */ 2680 E1000_WRITE_REG(hw, E1000_RLPML, dev->data->mtu + E1000_ETH_OVERHEAD); 2681 } 2682 2683 static void 2684 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2685 { 2686 struct e1000_hw *hw = 2687 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2688 uint32_t reg; 2689 2690 /* CTRL_EXT: Extended VLAN */ 2691 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2692 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2693 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2694 2695 /* Update maximum packet length */ 2696 E1000_WRITE_REG(hw, E1000_RLPML, 2697 dev->data->mtu + E1000_ETH_OVERHEAD + VLAN_TAG_SIZE); 2698 } 2699 2700 static int 2701 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2702 { 2703 struct rte_eth_rxmode *rxmode; 2704 2705 rxmode = &dev->data->dev_conf.rxmode; 2706 if(mask & ETH_VLAN_STRIP_MASK){ 2707 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2708 igb_vlan_hw_strip_enable(dev); 2709 else 2710 igb_vlan_hw_strip_disable(dev); 2711 } 2712 2713 if(mask & ETH_VLAN_FILTER_MASK){ 2714 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2715 igb_vlan_hw_filter_enable(dev); 2716 else 2717 igb_vlan_hw_filter_disable(dev); 2718 } 2719 2720 if(mask & ETH_VLAN_EXTEND_MASK){ 2721 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2722 igb_vlan_hw_extend_enable(dev); 2723 else 2724 igb_vlan_hw_extend_disable(dev); 2725 } 2726 2727 return 0; 2728 } 2729 2730 2731 /** 2732 * It enables the interrupt mask and then enable the interrupt. 2733 * 2734 * @param dev 2735 * Pointer to struct rte_eth_dev. 2736 * @param on 2737 * Enable or Disable 2738 * 2739 * @return 2740 * - On success, zero. 2741 * - On failure, a negative value. 2742 */ 2743 static int 2744 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2745 { 2746 struct e1000_interrupt *intr = 2747 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2748 2749 if (on) 2750 intr->mask |= E1000_ICR_LSC; 2751 else 2752 intr->mask &= ~E1000_ICR_LSC; 2753 2754 return 0; 2755 } 2756 2757 /* It clears the interrupt causes and enables the interrupt. 2758 * It will be called once only during nic initialized. 2759 * 2760 * @param dev 2761 * Pointer to struct rte_eth_dev. 2762 * 2763 * @return 2764 * - On success, zero. 2765 * - On failure, a negative value. 2766 */ 2767 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2768 { 2769 uint32_t mask, regval; 2770 int ret; 2771 struct e1000_hw *hw = 2772 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2773 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2774 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2775 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2776 struct rte_eth_dev_info dev_info; 2777 2778 memset(&dev_info, 0, sizeof(dev_info)); 2779 ret = eth_igb_infos_get(dev, &dev_info); 2780 if (ret != 0) 2781 return ret; 2782 2783 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2784 regval = E1000_READ_REG(hw, E1000_EIMS); 2785 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2786 2787 return 0; 2788 } 2789 2790 /* 2791 * It reads ICR and gets interrupt causes, check it and set a bit flag 2792 * to update link status. 2793 * 2794 * @param dev 2795 * Pointer to struct rte_eth_dev. 2796 * 2797 * @return 2798 * - On success, zero. 2799 * - On failure, a negative value. 2800 */ 2801 static int 2802 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2803 { 2804 uint32_t icr; 2805 struct e1000_hw *hw = 2806 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2807 struct e1000_interrupt *intr = 2808 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2809 2810 igb_intr_disable(dev); 2811 2812 /* read-on-clear nic registers here */ 2813 icr = E1000_READ_REG(hw, E1000_ICR); 2814 2815 intr->flags = 0; 2816 if (icr & E1000_ICR_LSC) { 2817 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2818 } 2819 2820 if (icr & E1000_ICR_VMMB) 2821 intr->flags |= E1000_FLAG_MAILBOX; 2822 2823 return 0; 2824 } 2825 2826 /* 2827 * It executes link_update after knowing an interrupt is prsent. 2828 * 2829 * @param dev 2830 * Pointer to struct rte_eth_dev. 2831 * 2832 * @return 2833 * - On success, zero. 2834 * - On failure, a negative value. 2835 */ 2836 static int 2837 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2838 struct rte_intr_handle *intr_handle) 2839 { 2840 struct e1000_hw *hw = 2841 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2842 struct e1000_interrupt *intr = 2843 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2844 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2845 struct rte_eth_link link; 2846 int ret; 2847 2848 if (intr->flags & E1000_FLAG_MAILBOX) { 2849 igb_pf_mbx_process(dev); 2850 intr->flags &= ~E1000_FLAG_MAILBOX; 2851 } 2852 2853 igb_intr_enable(dev); 2854 rte_intr_ack(intr_handle); 2855 2856 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2857 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2858 2859 /* set get_link_status to check register later */ 2860 hw->mac.get_link_status = 1; 2861 ret = eth_igb_link_update(dev, 0); 2862 2863 /* check if link has changed */ 2864 if (ret < 0) 2865 return 0; 2866 2867 rte_eth_linkstatus_get(dev, &link); 2868 if (link.link_status) { 2869 PMD_INIT_LOG(INFO, 2870 " Port %d: Link Up - speed %u Mbps - %s", 2871 dev->data->port_id, 2872 (unsigned)link.link_speed, 2873 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2874 "full-duplex" : "half-duplex"); 2875 } else { 2876 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2877 dev->data->port_id); 2878 } 2879 2880 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 2881 pci_dev->addr.domain, 2882 pci_dev->addr.bus, 2883 pci_dev->addr.devid, 2884 pci_dev->addr.function); 2885 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2886 } 2887 2888 return 0; 2889 } 2890 2891 /** 2892 * Interrupt handler which shall be registered at first. 2893 * 2894 * @param handle 2895 * Pointer to interrupt handle. 2896 * @param param 2897 * The address of parameter (struct rte_eth_dev *) regsitered before. 2898 * 2899 * @return 2900 * void 2901 */ 2902 static void 2903 eth_igb_interrupt_handler(void *param) 2904 { 2905 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2906 2907 eth_igb_interrupt_get_status(dev); 2908 eth_igb_interrupt_action(dev, dev->intr_handle); 2909 } 2910 2911 static int 2912 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2913 { 2914 uint32_t eicr; 2915 struct e1000_hw *hw = 2916 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2917 struct e1000_interrupt *intr = 2918 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2919 2920 igbvf_intr_disable(hw); 2921 2922 /* read-on-clear nic registers here */ 2923 eicr = E1000_READ_REG(hw, E1000_EICR); 2924 intr->flags = 0; 2925 2926 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2927 intr->flags |= E1000_FLAG_MAILBOX; 2928 2929 return 0; 2930 } 2931 2932 void igbvf_mbx_process(struct rte_eth_dev *dev) 2933 { 2934 struct e1000_hw *hw = 2935 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2936 struct e1000_mbx_info *mbx = &hw->mbx; 2937 u32 in_msg = 0; 2938 2939 /* peek the message first */ 2940 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2941 2942 /* PF reset VF event */ 2943 if (in_msg == E1000_PF_CONTROL_MSG) { 2944 /* dummy mbx read to ack pf */ 2945 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2946 return; 2947 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2948 NULL); 2949 } 2950 } 2951 2952 static int 2953 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2954 { 2955 struct e1000_interrupt *intr = 2956 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2957 2958 if (intr->flags & E1000_FLAG_MAILBOX) { 2959 igbvf_mbx_process(dev); 2960 intr->flags &= ~E1000_FLAG_MAILBOX; 2961 } 2962 2963 igbvf_intr_enable(dev); 2964 rte_intr_ack(intr_handle); 2965 2966 return 0; 2967 } 2968 2969 static void 2970 eth_igbvf_interrupt_handler(void *param) 2971 { 2972 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2973 2974 eth_igbvf_interrupt_get_status(dev); 2975 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2976 } 2977 2978 static int 2979 eth_igb_led_on(struct rte_eth_dev *dev) 2980 { 2981 struct e1000_hw *hw; 2982 2983 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2984 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2985 } 2986 2987 static int 2988 eth_igb_led_off(struct rte_eth_dev *dev) 2989 { 2990 struct e1000_hw *hw; 2991 2992 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2993 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2994 } 2995 2996 static int 2997 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 2998 { 2999 struct e1000_hw *hw; 3000 uint32_t ctrl; 3001 int tx_pause; 3002 int rx_pause; 3003 3004 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3005 fc_conf->pause_time = hw->fc.pause_time; 3006 fc_conf->high_water = hw->fc.high_water; 3007 fc_conf->low_water = hw->fc.low_water; 3008 fc_conf->send_xon = hw->fc.send_xon; 3009 fc_conf->autoneg = hw->mac.autoneg; 3010 3011 /* 3012 * Return rx_pause and tx_pause status according to actual setting of 3013 * the TFCE and RFCE bits in the CTRL register. 3014 */ 3015 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3016 if (ctrl & E1000_CTRL_TFCE) 3017 tx_pause = 1; 3018 else 3019 tx_pause = 0; 3020 3021 if (ctrl & E1000_CTRL_RFCE) 3022 rx_pause = 1; 3023 else 3024 rx_pause = 0; 3025 3026 if (rx_pause && tx_pause) 3027 fc_conf->mode = RTE_FC_FULL; 3028 else if (rx_pause) 3029 fc_conf->mode = RTE_FC_RX_PAUSE; 3030 else if (tx_pause) 3031 fc_conf->mode = RTE_FC_TX_PAUSE; 3032 else 3033 fc_conf->mode = RTE_FC_NONE; 3034 3035 return 0; 3036 } 3037 3038 static int 3039 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3040 { 3041 struct e1000_hw *hw; 3042 int err; 3043 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3044 e1000_fc_none, 3045 e1000_fc_rx_pause, 3046 e1000_fc_tx_pause, 3047 e1000_fc_full 3048 }; 3049 uint32_t rx_buf_size; 3050 uint32_t max_high_water; 3051 uint32_t rctl; 3052 uint32_t ctrl; 3053 3054 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3055 if (fc_conf->autoneg != hw->mac.autoneg) 3056 return -ENOTSUP; 3057 rx_buf_size = igb_get_rx_buffer_size(hw); 3058 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3059 3060 /* At least reserve one Ethernet frame for watermark */ 3061 max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN; 3062 if ((fc_conf->high_water > max_high_water) || 3063 (fc_conf->high_water < fc_conf->low_water)) { 3064 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3065 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3066 return -EINVAL; 3067 } 3068 3069 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3070 hw->fc.pause_time = fc_conf->pause_time; 3071 hw->fc.high_water = fc_conf->high_water; 3072 hw->fc.low_water = fc_conf->low_water; 3073 hw->fc.send_xon = fc_conf->send_xon; 3074 3075 err = e1000_setup_link_generic(hw); 3076 if (err == E1000_SUCCESS) { 3077 3078 /* check if we want to forward MAC frames - driver doesn't have native 3079 * capability to do that, so we'll write the registers ourselves */ 3080 3081 rctl = E1000_READ_REG(hw, E1000_RCTL); 3082 3083 /* set or clear MFLCN.PMCF bit depending on configuration */ 3084 if (fc_conf->mac_ctrl_frame_fwd != 0) 3085 rctl |= E1000_RCTL_PMCF; 3086 else 3087 rctl &= ~E1000_RCTL_PMCF; 3088 3089 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3090 3091 /* 3092 * check if we want to change flow control mode - driver doesn't have native 3093 * capability to do that, so we'll write the registers ourselves 3094 */ 3095 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3096 3097 /* 3098 * set or clear E1000_CTRL_RFCE and E1000_CTRL_TFCE bits depending 3099 * on configuration 3100 */ 3101 switch (fc_conf->mode) { 3102 case RTE_FC_NONE: 3103 ctrl &= ~E1000_CTRL_RFCE & ~E1000_CTRL_TFCE; 3104 break; 3105 case RTE_FC_RX_PAUSE: 3106 ctrl |= E1000_CTRL_RFCE; 3107 ctrl &= ~E1000_CTRL_TFCE; 3108 break; 3109 case RTE_FC_TX_PAUSE: 3110 ctrl |= E1000_CTRL_TFCE; 3111 ctrl &= ~E1000_CTRL_RFCE; 3112 break; 3113 case RTE_FC_FULL: 3114 ctrl |= E1000_CTRL_RFCE | E1000_CTRL_TFCE; 3115 break; 3116 default: 3117 PMD_INIT_LOG(ERR, "invalid flow control mode"); 3118 return -EINVAL; 3119 } 3120 3121 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3122 3123 E1000_WRITE_FLUSH(hw); 3124 3125 return 0; 3126 } 3127 3128 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3129 return -EIO; 3130 } 3131 3132 #define E1000_RAH_POOLSEL_SHIFT (18) 3133 static int 3134 eth_igb_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 3135 uint32_t index, uint32_t pool) 3136 { 3137 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3138 uint32_t rah; 3139 3140 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3141 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3142 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3143 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3144 return 0; 3145 } 3146 3147 static void 3148 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3149 { 3150 uint8_t addr[RTE_ETHER_ADDR_LEN]; 3151 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3152 3153 memset(addr, 0, sizeof(addr)); 3154 3155 e1000_rar_set(hw, addr, index); 3156 } 3157 3158 static int 3159 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3160 struct rte_ether_addr *addr) 3161 { 3162 eth_igb_rar_clear(dev, 0); 3163 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3164 3165 return 0; 3166 } 3167 /* 3168 * Virtual Function operations 3169 */ 3170 static void 3171 igbvf_intr_disable(struct e1000_hw *hw) 3172 { 3173 PMD_INIT_FUNC_TRACE(); 3174 3175 /* Clear interrupt mask to stop from interrupts being generated */ 3176 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3177 3178 E1000_WRITE_FLUSH(hw); 3179 } 3180 3181 static void 3182 igbvf_stop_adapter(struct rte_eth_dev *dev) 3183 { 3184 u32 reg_val; 3185 u16 i; 3186 struct rte_eth_dev_info dev_info; 3187 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3188 int ret; 3189 3190 memset(&dev_info, 0, sizeof(dev_info)); 3191 ret = eth_igbvf_infos_get(dev, &dev_info); 3192 if (ret != 0) 3193 return; 3194 3195 /* Clear interrupt mask to stop from interrupts being generated */ 3196 igbvf_intr_disable(hw); 3197 3198 /* Clear any pending interrupts, flush previous writes */ 3199 E1000_READ_REG(hw, E1000_EICR); 3200 3201 /* Disable the transmit unit. Each queue must be disabled. */ 3202 for (i = 0; i < dev_info.max_tx_queues; i++) 3203 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3204 3205 /* Disable the receive unit by stopping each queue */ 3206 for (i = 0; i < dev_info.max_rx_queues; i++) { 3207 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3208 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3209 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3210 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3211 ; 3212 } 3213 3214 /* flush all queues disables */ 3215 E1000_WRITE_FLUSH(hw); 3216 msec_delay(2); 3217 } 3218 3219 static int eth_igbvf_link_update(struct e1000_hw *hw) 3220 { 3221 struct e1000_mbx_info *mbx = &hw->mbx; 3222 struct e1000_mac_info *mac = &hw->mac; 3223 int ret_val = E1000_SUCCESS; 3224 3225 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3226 3227 /* 3228 * We only want to run this if there has been a rst asserted. 3229 * in this case that could mean a link change, device reset, 3230 * or a virtual function reset 3231 */ 3232 3233 /* If we were hit with a reset or timeout drop the link */ 3234 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3235 mac->get_link_status = TRUE; 3236 3237 if (!mac->get_link_status) 3238 goto out; 3239 3240 /* if link status is down no point in checking to see if pf is up */ 3241 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3242 goto out; 3243 3244 /* if we passed all the tests above then the link is up and we no 3245 * longer need to check for link */ 3246 mac->get_link_status = FALSE; 3247 3248 out: 3249 return ret_val; 3250 } 3251 3252 3253 static int 3254 igbvf_dev_configure(struct rte_eth_dev *dev) 3255 { 3256 struct rte_eth_conf* conf = &dev->data->dev_conf; 3257 3258 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3259 dev->data->port_id); 3260 3261 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 3262 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 3263 3264 /* 3265 * VF has no ability to enable/disable HW CRC 3266 * Keep the persistent behavior the same as Host PF 3267 */ 3268 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3269 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3270 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3271 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3272 } 3273 #else 3274 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3275 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3276 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3277 } 3278 #endif 3279 3280 return 0; 3281 } 3282 3283 static int 3284 igbvf_dev_start(struct rte_eth_dev *dev) 3285 { 3286 struct e1000_hw *hw = 3287 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3288 struct e1000_adapter *adapter = 3289 E1000_DEV_PRIVATE(dev->data->dev_private); 3290 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3291 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3292 int ret; 3293 uint32_t intr_vector = 0; 3294 3295 PMD_INIT_FUNC_TRACE(); 3296 3297 hw->mac.ops.reset_hw(hw); 3298 adapter->stopped = 0; 3299 3300 /* Set all vfta */ 3301 igbvf_set_vfta_all(dev,1); 3302 3303 eth_igbvf_tx_init(dev); 3304 3305 /* This can fail when allocating mbufs for descriptor rings */ 3306 ret = eth_igbvf_rx_init(dev); 3307 if (ret) { 3308 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3309 igb_dev_clear_queues(dev); 3310 return ret; 3311 } 3312 3313 /* check and configure queue intr-vector mapping */ 3314 if (rte_intr_cap_multiple(intr_handle) && 3315 dev->data->dev_conf.intr_conf.rxq) { 3316 intr_vector = dev->data->nb_rx_queues; 3317 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3318 if (ret) 3319 return ret; 3320 } 3321 3322 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3323 intr_handle->intr_vec = 3324 rte_zmalloc("intr_vec", 3325 dev->data->nb_rx_queues * sizeof(int), 0); 3326 if (!intr_handle->intr_vec) { 3327 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3328 " intr_vec", dev->data->nb_rx_queues); 3329 return -ENOMEM; 3330 } 3331 } 3332 3333 eth_igbvf_configure_msix_intr(dev); 3334 3335 /* enable uio/vfio intr/eventfd mapping */ 3336 rte_intr_enable(intr_handle); 3337 3338 /* resume enabled intr since hw reset */ 3339 igbvf_intr_enable(dev); 3340 3341 return 0; 3342 } 3343 3344 static int 3345 igbvf_dev_stop(struct rte_eth_dev *dev) 3346 { 3347 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3348 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3349 struct e1000_adapter *adapter = 3350 E1000_DEV_PRIVATE(dev->data->dev_private); 3351 3352 if (adapter->stopped) 3353 return 0; 3354 3355 PMD_INIT_FUNC_TRACE(); 3356 3357 igbvf_stop_adapter(dev); 3358 3359 /* 3360 * Clear what we set, but we still keep shadow_vfta to 3361 * restore after device starts 3362 */ 3363 igbvf_set_vfta_all(dev,0); 3364 3365 igb_dev_clear_queues(dev); 3366 3367 /* disable intr eventfd mapping */ 3368 rte_intr_disable(intr_handle); 3369 3370 /* Clean datapath event and queue/vec mapping */ 3371 rte_intr_efd_disable(intr_handle); 3372 if (intr_handle->intr_vec) { 3373 rte_free(intr_handle->intr_vec); 3374 intr_handle->intr_vec = NULL; 3375 } 3376 3377 adapter->stopped = true; 3378 dev->data->dev_started = 0; 3379 3380 return 0; 3381 } 3382 3383 static int 3384 igbvf_dev_close(struct rte_eth_dev *dev) 3385 { 3386 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3387 struct rte_ether_addr addr; 3388 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3389 int ret; 3390 3391 PMD_INIT_FUNC_TRACE(); 3392 3393 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 3394 return 0; 3395 3396 e1000_reset_hw(hw); 3397 3398 ret = igbvf_dev_stop(dev); 3399 if (ret != 0) 3400 return ret; 3401 3402 igb_dev_free_queues(dev); 3403 3404 /** 3405 * reprogram the RAR with a zero mac address, 3406 * to ensure that the VF traffic goes to the PF 3407 * after stop, close and detach of the VF. 3408 **/ 3409 3410 memset(&addr, 0, sizeof(addr)); 3411 igbvf_default_mac_addr_set(dev, &addr); 3412 3413 rte_intr_callback_unregister(&pci_dev->intr_handle, 3414 eth_igbvf_interrupt_handler, 3415 (void *)dev); 3416 3417 return 0; 3418 } 3419 3420 static int 3421 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3422 { 3423 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3424 3425 /* Set both unicast and multicast promisc */ 3426 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3427 3428 return 0; 3429 } 3430 3431 static int 3432 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3433 { 3434 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3435 3436 /* If in allmulticast mode leave multicast promisc */ 3437 if (dev->data->all_multicast == 1) 3438 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3439 else 3440 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3441 3442 return 0; 3443 } 3444 3445 static int 3446 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3447 { 3448 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3449 3450 /* In promiscuous mode multicast promisc already set */ 3451 if (dev->data->promiscuous == 0) 3452 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3453 3454 return 0; 3455 } 3456 3457 static int 3458 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3459 { 3460 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3461 3462 /* In promiscuous mode leave multicast promisc enabled */ 3463 if (dev->data->promiscuous == 0) 3464 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3465 3466 return 0; 3467 } 3468 3469 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3470 { 3471 struct e1000_mbx_info *mbx = &hw->mbx; 3472 uint32_t msgbuf[2]; 3473 s32 err; 3474 3475 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3476 msgbuf[0] = E1000_VF_SET_VLAN; 3477 msgbuf[1] = vid; 3478 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3479 if (on) 3480 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3481 3482 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3483 if (err) 3484 goto mbx_err; 3485 3486 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3487 if (err) 3488 goto mbx_err; 3489 3490 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3491 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3492 err = -EINVAL; 3493 3494 mbx_err: 3495 return err; 3496 } 3497 3498 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3499 { 3500 struct e1000_hw *hw = 3501 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3502 struct e1000_vfta * shadow_vfta = 3503 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3504 int i = 0, j = 0, vfta = 0, mask = 1; 3505 3506 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3507 vfta = shadow_vfta->vfta[i]; 3508 if(vfta){ 3509 mask = 1; 3510 for (j = 0; j < 32; j++){ 3511 if(vfta & mask) 3512 igbvf_set_vfta(hw, 3513 (uint16_t)((i<<5)+j), on); 3514 mask<<=1; 3515 } 3516 } 3517 } 3518 3519 } 3520 3521 static int 3522 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3523 { 3524 struct e1000_hw *hw = 3525 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3526 struct e1000_vfta * shadow_vfta = 3527 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3528 uint32_t vid_idx = 0; 3529 uint32_t vid_bit = 0; 3530 int ret = 0; 3531 3532 PMD_INIT_FUNC_TRACE(); 3533 3534 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3535 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3536 if(ret){ 3537 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3538 return ret; 3539 } 3540 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3541 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3542 3543 /*Save what we set and retore it after device reset*/ 3544 if (on) 3545 shadow_vfta->vfta[vid_idx] |= vid_bit; 3546 else 3547 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3548 3549 return 0; 3550 } 3551 3552 static int 3553 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 3554 { 3555 struct e1000_hw *hw = 3556 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3557 3558 /* index is not used by rar_set() */ 3559 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3560 return 0; 3561 } 3562 3563 3564 static int 3565 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3566 struct rte_eth_rss_reta_entry64 *reta_conf, 3567 uint16_t reta_size) 3568 { 3569 uint8_t i, j, mask; 3570 uint32_t reta, r; 3571 uint16_t idx, shift; 3572 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3573 3574 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3575 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3576 "(%d) doesn't match the number hardware can supported " 3577 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3578 return -EINVAL; 3579 } 3580 3581 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3582 idx = i / RTE_RETA_GROUP_SIZE; 3583 shift = i % RTE_RETA_GROUP_SIZE; 3584 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3585 IGB_4_BIT_MASK); 3586 if (!mask) 3587 continue; 3588 if (mask == IGB_4_BIT_MASK) 3589 r = 0; 3590 else 3591 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3592 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3593 if (mask & (0x1 << j)) 3594 reta |= reta_conf[idx].reta[shift + j] << 3595 (CHAR_BIT * j); 3596 else 3597 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3598 } 3599 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3600 } 3601 3602 return 0; 3603 } 3604 3605 static int 3606 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3607 struct rte_eth_rss_reta_entry64 *reta_conf, 3608 uint16_t reta_size) 3609 { 3610 uint8_t i, j, mask; 3611 uint32_t reta; 3612 uint16_t idx, shift; 3613 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3614 3615 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3616 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3617 "(%d) doesn't match the number hardware can supported " 3618 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3619 return -EINVAL; 3620 } 3621 3622 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3623 idx = i / RTE_RETA_GROUP_SIZE; 3624 shift = i % RTE_RETA_GROUP_SIZE; 3625 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3626 IGB_4_BIT_MASK); 3627 if (!mask) 3628 continue; 3629 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3630 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3631 if (mask & (0x1 << j)) 3632 reta_conf[idx].reta[shift + j] = 3633 ((reta >> (CHAR_BIT * j)) & 3634 IGB_8_BIT_MASK); 3635 } 3636 } 3637 3638 return 0; 3639 } 3640 3641 int 3642 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3643 struct rte_eth_syn_filter *filter, 3644 bool add) 3645 { 3646 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3647 struct e1000_filter_info *filter_info = 3648 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3649 uint32_t synqf, rfctl; 3650 3651 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3652 return -EINVAL; 3653 3654 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3655 3656 if (add) { 3657 if (synqf & E1000_SYN_FILTER_ENABLE) 3658 return -EINVAL; 3659 3660 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3661 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3662 3663 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3664 if (filter->hig_pri) 3665 rfctl |= E1000_RFCTL_SYNQFP; 3666 else 3667 rfctl &= ~E1000_RFCTL_SYNQFP; 3668 3669 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3670 } else { 3671 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3672 return -ENOENT; 3673 synqf = 0; 3674 } 3675 3676 filter_info->syn_info = synqf; 3677 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3678 E1000_WRITE_FLUSH(hw); 3679 return 0; 3680 } 3681 3682 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3683 static inline int 3684 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3685 struct e1000_2tuple_filter_info *filter_info) 3686 { 3687 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3688 return -EINVAL; 3689 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3690 return -EINVAL; /* filter index is out of range. */ 3691 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 3692 return -EINVAL; /* flags is invalid. */ 3693 3694 switch (filter->dst_port_mask) { 3695 case UINT16_MAX: 3696 filter_info->dst_port_mask = 0; 3697 filter_info->dst_port = filter->dst_port; 3698 break; 3699 case 0: 3700 filter_info->dst_port_mask = 1; 3701 break; 3702 default: 3703 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3704 return -EINVAL; 3705 } 3706 3707 switch (filter->proto_mask) { 3708 case UINT8_MAX: 3709 filter_info->proto_mask = 0; 3710 filter_info->proto = filter->proto; 3711 break; 3712 case 0: 3713 filter_info->proto_mask = 1; 3714 break; 3715 default: 3716 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3717 return -EINVAL; 3718 } 3719 3720 filter_info->priority = (uint8_t)filter->priority; 3721 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3722 filter_info->tcp_flags = filter->tcp_flags; 3723 else 3724 filter_info->tcp_flags = 0; 3725 3726 return 0; 3727 } 3728 3729 static inline struct e1000_2tuple_filter * 3730 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3731 struct e1000_2tuple_filter_info *key) 3732 { 3733 struct e1000_2tuple_filter *it; 3734 3735 TAILQ_FOREACH(it, filter_list, entries) { 3736 if (memcmp(key, &it->filter_info, 3737 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3738 return it; 3739 } 3740 } 3741 return NULL; 3742 } 3743 3744 /* inject a igb 2tuple filter to HW */ 3745 static inline void 3746 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3747 struct e1000_2tuple_filter *filter) 3748 { 3749 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3750 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3751 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3752 int i; 3753 3754 i = filter->index; 3755 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3756 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3757 imir |= E1000_IMIR_PORT_BP; 3758 else 3759 imir &= ~E1000_IMIR_PORT_BP; 3760 3761 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3762 3763 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3764 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3765 ttqf |= (uint32_t)(filter->filter_info.proto & 3766 E1000_TTQF_PROTOCOL_MASK); 3767 if (filter->filter_info.proto_mask == 0) 3768 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3769 3770 /* tcp flags bits setting. */ 3771 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 3772 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 3773 imir_ext |= E1000_IMIREXT_CTRL_URG; 3774 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 3775 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3776 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 3777 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3778 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 3779 imir_ext |= E1000_IMIREXT_CTRL_RST; 3780 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 3781 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3782 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 3783 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3784 } else { 3785 imir_ext |= E1000_IMIREXT_CTRL_BP; 3786 } 3787 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3788 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3789 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3790 } 3791 3792 /* 3793 * igb_add_2tuple_filter - add a 2tuple filter 3794 * 3795 * @param 3796 * dev: Pointer to struct rte_eth_dev. 3797 * ntuple_filter: ponter to the filter that will be added. 3798 * 3799 * @return 3800 * - On success, zero. 3801 * - On failure, a negative value. 3802 */ 3803 static int 3804 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3805 struct rte_eth_ntuple_filter *ntuple_filter) 3806 { 3807 struct e1000_filter_info *filter_info = 3808 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3809 struct e1000_2tuple_filter *filter; 3810 int i, ret; 3811 3812 filter = rte_zmalloc("e1000_2tuple_filter", 3813 sizeof(struct e1000_2tuple_filter), 0); 3814 if (filter == NULL) 3815 return -ENOMEM; 3816 3817 ret = ntuple_filter_to_2tuple(ntuple_filter, 3818 &filter->filter_info); 3819 if (ret < 0) { 3820 rte_free(filter); 3821 return ret; 3822 } 3823 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3824 &filter->filter_info) != NULL) { 3825 PMD_DRV_LOG(ERR, "filter exists."); 3826 rte_free(filter); 3827 return -EEXIST; 3828 } 3829 filter->queue = ntuple_filter->queue; 3830 3831 /* 3832 * look for an unused 2tuple filter index, 3833 * and insert the filter to list. 3834 */ 3835 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3836 if (!(filter_info->twotuple_mask & (1 << i))) { 3837 filter_info->twotuple_mask |= 1 << i; 3838 filter->index = i; 3839 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3840 filter, 3841 entries); 3842 break; 3843 } 3844 } 3845 if (i >= E1000_MAX_TTQF_FILTERS) { 3846 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3847 rte_free(filter); 3848 return -ENOSYS; 3849 } 3850 3851 igb_inject_2uple_filter(dev, filter); 3852 return 0; 3853 } 3854 3855 int 3856 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3857 struct e1000_2tuple_filter *filter) 3858 { 3859 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3860 struct e1000_filter_info *filter_info = 3861 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3862 3863 filter_info->twotuple_mask &= ~(1 << filter->index); 3864 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3865 rte_free(filter); 3866 3867 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3868 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3869 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3870 return 0; 3871 } 3872 3873 /* 3874 * igb_remove_2tuple_filter - remove a 2tuple filter 3875 * 3876 * @param 3877 * dev: Pointer to struct rte_eth_dev. 3878 * ntuple_filter: ponter to the filter that will be removed. 3879 * 3880 * @return 3881 * - On success, zero. 3882 * - On failure, a negative value. 3883 */ 3884 static int 3885 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3886 struct rte_eth_ntuple_filter *ntuple_filter) 3887 { 3888 struct e1000_filter_info *filter_info = 3889 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3890 struct e1000_2tuple_filter_info filter_2tuple; 3891 struct e1000_2tuple_filter *filter; 3892 int ret; 3893 3894 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3895 ret = ntuple_filter_to_2tuple(ntuple_filter, 3896 &filter_2tuple); 3897 if (ret < 0) 3898 return ret; 3899 3900 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3901 &filter_2tuple); 3902 if (filter == NULL) { 3903 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3904 return -ENOENT; 3905 } 3906 3907 igb_delete_2tuple_filter(dev, filter); 3908 3909 return 0; 3910 } 3911 3912 /* inject a igb flex filter to HW */ 3913 static inline void 3914 igb_inject_flex_filter(struct rte_eth_dev *dev, 3915 struct e1000_flex_filter *filter) 3916 { 3917 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3918 uint32_t wufc, queueing; 3919 uint32_t reg_off; 3920 uint8_t i, j = 0; 3921 3922 wufc = E1000_READ_REG(hw, E1000_WUFC); 3923 if (filter->index < E1000_MAX_FHFT) 3924 reg_off = E1000_FHFT(filter->index); 3925 else 3926 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3927 3928 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3929 (E1000_WUFC_FLX0 << filter->index)); 3930 queueing = filter->filter_info.len | 3931 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3932 (filter->filter_info.priority << 3933 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3934 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3935 queueing); 3936 3937 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3938 E1000_WRITE_REG(hw, reg_off, 3939 filter->filter_info.dwords[j]); 3940 reg_off += sizeof(uint32_t); 3941 E1000_WRITE_REG(hw, reg_off, 3942 filter->filter_info.dwords[++j]); 3943 reg_off += sizeof(uint32_t); 3944 E1000_WRITE_REG(hw, reg_off, 3945 (uint32_t)filter->filter_info.mask[i]); 3946 reg_off += sizeof(uint32_t) * 2; 3947 ++j; 3948 } 3949 } 3950 3951 static inline struct e1000_flex_filter * 3952 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3953 struct e1000_flex_filter_info *key) 3954 { 3955 struct e1000_flex_filter *it; 3956 3957 TAILQ_FOREACH(it, filter_list, entries) { 3958 if (memcmp(key, &it->filter_info, 3959 sizeof(struct e1000_flex_filter_info)) == 0) 3960 return it; 3961 } 3962 3963 return NULL; 3964 } 3965 3966 /* remove a flex byte filter 3967 * @param 3968 * dev: Pointer to struct rte_eth_dev. 3969 * filter: the pointer of the filter will be removed. 3970 */ 3971 void 3972 igb_remove_flex_filter(struct rte_eth_dev *dev, 3973 struct e1000_flex_filter *filter) 3974 { 3975 struct e1000_filter_info *filter_info = 3976 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3977 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3978 uint32_t wufc, i; 3979 uint32_t reg_off; 3980 3981 wufc = E1000_READ_REG(hw, E1000_WUFC); 3982 if (filter->index < E1000_MAX_FHFT) 3983 reg_off = E1000_FHFT(filter->index); 3984 else 3985 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3986 3987 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 3988 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 3989 3990 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 3991 (~(E1000_WUFC_FLX0 << filter->index))); 3992 3993 filter_info->flex_mask &= ~(1 << filter->index); 3994 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 3995 rte_free(filter); 3996 } 3997 3998 int 3999 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4000 struct igb_flex_filter *filter, 4001 bool add) 4002 { 4003 struct e1000_filter_info *filter_info = 4004 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4005 struct e1000_flex_filter *flex_filter, *it; 4006 uint32_t mask; 4007 uint8_t shift, i; 4008 4009 flex_filter = rte_zmalloc("e1000_flex_filter", 4010 sizeof(struct e1000_flex_filter), 0); 4011 if (flex_filter == NULL) 4012 return -ENOMEM; 4013 4014 flex_filter->filter_info.len = filter->len; 4015 flex_filter->filter_info.priority = filter->priority; 4016 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4017 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4018 mask = 0; 4019 /* reverse bits in flex filter's mask*/ 4020 for (shift = 0; shift < CHAR_BIT; shift++) { 4021 if (filter->mask[i] & (0x01 << shift)) 4022 mask |= (0x80 >> shift); 4023 } 4024 flex_filter->filter_info.mask[i] = mask; 4025 } 4026 4027 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4028 &flex_filter->filter_info); 4029 if (it == NULL && !add) { 4030 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4031 rte_free(flex_filter); 4032 return -ENOENT; 4033 } 4034 if (it != NULL && add) { 4035 PMD_DRV_LOG(ERR, "filter exists."); 4036 rte_free(flex_filter); 4037 return -EEXIST; 4038 } 4039 4040 if (add) { 4041 flex_filter->queue = filter->queue; 4042 /* 4043 * look for an unused flex filter index 4044 * and insert the filter into the list. 4045 */ 4046 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4047 if (!(filter_info->flex_mask & (1 << i))) { 4048 filter_info->flex_mask |= 1 << i; 4049 flex_filter->index = i; 4050 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4051 flex_filter, 4052 entries); 4053 break; 4054 } 4055 } 4056 if (i >= E1000_MAX_FLEX_FILTERS) { 4057 PMD_DRV_LOG(ERR, "flex filters are full."); 4058 rte_free(flex_filter); 4059 return -ENOSYS; 4060 } 4061 4062 igb_inject_flex_filter(dev, flex_filter); 4063 4064 } else { 4065 igb_remove_flex_filter(dev, it); 4066 rte_free(flex_filter); 4067 } 4068 4069 return 0; 4070 } 4071 4072 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4073 static inline int 4074 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4075 struct e1000_5tuple_filter_info *filter_info) 4076 { 4077 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4078 return -EINVAL; 4079 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4080 return -EINVAL; /* filter index is out of range. */ 4081 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 4082 return -EINVAL; /* flags is invalid. */ 4083 4084 switch (filter->dst_ip_mask) { 4085 case UINT32_MAX: 4086 filter_info->dst_ip_mask = 0; 4087 filter_info->dst_ip = filter->dst_ip; 4088 break; 4089 case 0: 4090 filter_info->dst_ip_mask = 1; 4091 break; 4092 default: 4093 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4094 return -EINVAL; 4095 } 4096 4097 switch (filter->src_ip_mask) { 4098 case UINT32_MAX: 4099 filter_info->src_ip_mask = 0; 4100 filter_info->src_ip = filter->src_ip; 4101 break; 4102 case 0: 4103 filter_info->src_ip_mask = 1; 4104 break; 4105 default: 4106 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4107 return -EINVAL; 4108 } 4109 4110 switch (filter->dst_port_mask) { 4111 case UINT16_MAX: 4112 filter_info->dst_port_mask = 0; 4113 filter_info->dst_port = filter->dst_port; 4114 break; 4115 case 0: 4116 filter_info->dst_port_mask = 1; 4117 break; 4118 default: 4119 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4120 return -EINVAL; 4121 } 4122 4123 switch (filter->src_port_mask) { 4124 case UINT16_MAX: 4125 filter_info->src_port_mask = 0; 4126 filter_info->src_port = filter->src_port; 4127 break; 4128 case 0: 4129 filter_info->src_port_mask = 1; 4130 break; 4131 default: 4132 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4133 return -EINVAL; 4134 } 4135 4136 switch (filter->proto_mask) { 4137 case UINT8_MAX: 4138 filter_info->proto_mask = 0; 4139 filter_info->proto = filter->proto; 4140 break; 4141 case 0: 4142 filter_info->proto_mask = 1; 4143 break; 4144 default: 4145 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4146 return -EINVAL; 4147 } 4148 4149 filter_info->priority = (uint8_t)filter->priority; 4150 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4151 filter_info->tcp_flags = filter->tcp_flags; 4152 else 4153 filter_info->tcp_flags = 0; 4154 4155 return 0; 4156 } 4157 4158 static inline struct e1000_5tuple_filter * 4159 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4160 struct e1000_5tuple_filter_info *key) 4161 { 4162 struct e1000_5tuple_filter *it; 4163 4164 TAILQ_FOREACH(it, filter_list, entries) { 4165 if (memcmp(key, &it->filter_info, 4166 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4167 return it; 4168 } 4169 } 4170 return NULL; 4171 } 4172 4173 /* inject a igb 5-tuple filter to HW */ 4174 static inline void 4175 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4176 struct e1000_5tuple_filter *filter) 4177 { 4178 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4179 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4180 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4181 uint8_t i; 4182 4183 i = filter->index; 4184 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4185 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4186 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4187 if (filter->filter_info.dst_ip_mask == 0) 4188 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4189 if (filter->filter_info.src_port_mask == 0) 4190 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4191 if (filter->filter_info.proto_mask == 0) 4192 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4193 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4194 E1000_FTQF_QUEUE_MASK; 4195 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4196 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4197 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4198 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4199 4200 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4201 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4202 4203 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4204 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4205 imir |= E1000_IMIR_PORT_BP; 4206 else 4207 imir &= ~E1000_IMIR_PORT_BP; 4208 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4209 4210 /* tcp flags bits setting. */ 4211 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 4212 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 4213 imir_ext |= E1000_IMIREXT_CTRL_URG; 4214 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 4215 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4216 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 4217 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4218 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 4219 imir_ext |= E1000_IMIREXT_CTRL_RST; 4220 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 4221 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4222 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 4223 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4224 } else { 4225 imir_ext |= E1000_IMIREXT_CTRL_BP; 4226 } 4227 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4228 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4229 } 4230 4231 /* 4232 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4233 * 4234 * @param 4235 * dev: Pointer to struct rte_eth_dev. 4236 * ntuple_filter: ponter to the filter that will be added. 4237 * 4238 * @return 4239 * - On success, zero. 4240 * - On failure, a negative value. 4241 */ 4242 static int 4243 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4244 struct rte_eth_ntuple_filter *ntuple_filter) 4245 { 4246 struct e1000_filter_info *filter_info = 4247 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4248 struct e1000_5tuple_filter *filter; 4249 uint8_t i; 4250 int ret; 4251 4252 filter = rte_zmalloc("e1000_5tuple_filter", 4253 sizeof(struct e1000_5tuple_filter), 0); 4254 if (filter == NULL) 4255 return -ENOMEM; 4256 4257 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4258 &filter->filter_info); 4259 if (ret < 0) { 4260 rte_free(filter); 4261 return ret; 4262 } 4263 4264 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4265 &filter->filter_info) != NULL) { 4266 PMD_DRV_LOG(ERR, "filter exists."); 4267 rte_free(filter); 4268 return -EEXIST; 4269 } 4270 filter->queue = ntuple_filter->queue; 4271 4272 /* 4273 * look for an unused 5tuple filter index, 4274 * and insert the filter to list. 4275 */ 4276 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4277 if (!(filter_info->fivetuple_mask & (1 << i))) { 4278 filter_info->fivetuple_mask |= 1 << i; 4279 filter->index = i; 4280 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4281 filter, 4282 entries); 4283 break; 4284 } 4285 } 4286 if (i >= E1000_MAX_FTQF_FILTERS) { 4287 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4288 rte_free(filter); 4289 return -ENOSYS; 4290 } 4291 4292 igb_inject_5tuple_filter_82576(dev, filter); 4293 return 0; 4294 } 4295 4296 int 4297 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4298 struct e1000_5tuple_filter *filter) 4299 { 4300 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4301 struct e1000_filter_info *filter_info = 4302 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4303 4304 filter_info->fivetuple_mask &= ~(1 << filter->index); 4305 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4306 rte_free(filter); 4307 4308 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4309 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4310 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4311 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4312 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4313 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4314 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4315 return 0; 4316 } 4317 4318 /* 4319 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4320 * 4321 * @param 4322 * dev: Pointer to struct rte_eth_dev. 4323 * ntuple_filter: ponter to the filter that will be removed. 4324 * 4325 * @return 4326 * - On success, zero. 4327 * - On failure, a negative value. 4328 */ 4329 static int 4330 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4331 struct rte_eth_ntuple_filter *ntuple_filter) 4332 { 4333 struct e1000_filter_info *filter_info = 4334 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4335 struct e1000_5tuple_filter_info filter_5tuple; 4336 struct e1000_5tuple_filter *filter; 4337 int ret; 4338 4339 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4340 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4341 &filter_5tuple); 4342 if (ret < 0) 4343 return ret; 4344 4345 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4346 &filter_5tuple); 4347 if (filter == NULL) { 4348 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4349 return -ENOENT; 4350 } 4351 4352 igb_delete_5tuple_filter_82576(dev, filter); 4353 4354 return 0; 4355 } 4356 4357 static int 4358 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4359 { 4360 uint32_t rctl; 4361 struct e1000_hw *hw; 4362 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4363 4364 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4365 4366 #ifdef RTE_LIBRTE_82571_SUPPORT 4367 /* XXX: not bigger than max_rx_pktlen */ 4368 if (hw->mac.type == e1000_82571) 4369 return -ENOTSUP; 4370 #endif 4371 /* 4372 * If device is started, refuse mtu that requires the support of 4373 * scattered packets when this feature has not been enabled before. 4374 */ 4375 if (dev->data->dev_started && !dev->data->scattered_rx && 4376 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) { 4377 PMD_INIT_LOG(ERR, "Stop port first."); 4378 return -EINVAL; 4379 } 4380 4381 rctl = E1000_READ_REG(hw, E1000_RCTL); 4382 4383 /* switch to jumbo mode if needed */ 4384 if (mtu > RTE_ETHER_MTU) 4385 rctl |= E1000_RCTL_LPE; 4386 else 4387 rctl &= ~E1000_RCTL_LPE; 4388 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4389 4390 E1000_WRITE_REG(hw, E1000_RLPML, frame_size); 4391 4392 return 0; 4393 } 4394 4395 /* 4396 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4397 * 4398 * @param 4399 * dev: Pointer to struct rte_eth_dev. 4400 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4401 * add: if true, add filter, if false, remove filter 4402 * 4403 * @return 4404 * - On success, zero. 4405 * - On failure, a negative value. 4406 */ 4407 int 4408 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4409 struct rte_eth_ntuple_filter *ntuple_filter, 4410 bool add) 4411 { 4412 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4413 int ret; 4414 4415 switch (ntuple_filter->flags) { 4416 case RTE_5TUPLE_FLAGS: 4417 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4418 if (hw->mac.type != e1000_82576) 4419 return -ENOTSUP; 4420 if (add) 4421 ret = igb_add_5tuple_filter_82576(dev, 4422 ntuple_filter); 4423 else 4424 ret = igb_remove_5tuple_filter_82576(dev, 4425 ntuple_filter); 4426 break; 4427 case RTE_2TUPLE_FLAGS: 4428 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4429 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4430 hw->mac.type != e1000_i210 && 4431 hw->mac.type != e1000_i211) 4432 return -ENOTSUP; 4433 if (add) 4434 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4435 else 4436 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4437 break; 4438 default: 4439 ret = -EINVAL; 4440 break; 4441 } 4442 4443 return ret; 4444 } 4445 4446 static inline int 4447 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4448 uint16_t ethertype) 4449 { 4450 int i; 4451 4452 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4453 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4454 (filter_info->ethertype_mask & (1 << i))) 4455 return i; 4456 } 4457 return -1; 4458 } 4459 4460 static inline int 4461 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4462 uint16_t ethertype, uint32_t etqf) 4463 { 4464 int i; 4465 4466 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4467 if (!(filter_info->ethertype_mask & (1 << i))) { 4468 filter_info->ethertype_mask |= 1 << i; 4469 filter_info->ethertype_filters[i].ethertype = ethertype; 4470 filter_info->ethertype_filters[i].etqf = etqf; 4471 return i; 4472 } 4473 } 4474 return -1; 4475 } 4476 4477 int 4478 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4479 uint8_t idx) 4480 { 4481 if (idx >= E1000_MAX_ETQF_FILTERS) 4482 return -1; 4483 filter_info->ethertype_mask &= ~(1 << idx); 4484 filter_info->ethertype_filters[idx].ethertype = 0; 4485 filter_info->ethertype_filters[idx].etqf = 0; 4486 return idx; 4487 } 4488 4489 4490 int 4491 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4492 struct rte_eth_ethertype_filter *filter, 4493 bool add) 4494 { 4495 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4496 struct e1000_filter_info *filter_info = 4497 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4498 uint32_t etqf = 0; 4499 int ret; 4500 4501 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 4502 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 4503 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4504 " ethertype filter.", filter->ether_type); 4505 return -EINVAL; 4506 } 4507 4508 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4509 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4510 return -EINVAL; 4511 } 4512 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4513 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4514 return -EINVAL; 4515 } 4516 4517 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4518 if (ret >= 0 && add) { 4519 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4520 filter->ether_type); 4521 return -EEXIST; 4522 } 4523 if (ret < 0 && !add) { 4524 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4525 filter->ether_type); 4526 return -ENOENT; 4527 } 4528 4529 if (add) { 4530 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4531 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4532 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4533 ret = igb_ethertype_filter_insert(filter_info, 4534 filter->ether_type, etqf); 4535 if (ret < 0) { 4536 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4537 return -ENOSYS; 4538 } 4539 } else { 4540 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4541 if (ret < 0) 4542 return -ENOSYS; 4543 } 4544 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4545 E1000_WRITE_FLUSH(hw); 4546 4547 return 0; 4548 } 4549 4550 static int 4551 eth_igb_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 4552 const struct rte_flow_ops **ops) 4553 { 4554 *ops = &igb_flow_ops; 4555 return 0; 4556 } 4557 4558 static int 4559 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4560 struct rte_ether_addr *mc_addr_set, 4561 uint32_t nb_mc_addr) 4562 { 4563 struct e1000_hw *hw; 4564 4565 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4566 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4567 return 0; 4568 } 4569 4570 static uint64_t 4571 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4572 { 4573 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4574 uint64_t systime_cycles; 4575 4576 switch (hw->mac.type) { 4577 case e1000_i210: 4578 case e1000_i211: 4579 /* 4580 * Need to read System Time Residue Register to be able 4581 * to read the other two registers. 4582 */ 4583 E1000_READ_REG(hw, E1000_SYSTIMR); 4584 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4585 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4586 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4587 * NSEC_PER_SEC; 4588 break; 4589 case e1000_82580: 4590 case e1000_i350: 4591 case e1000_i354: 4592 /* 4593 * Need to read System Time Residue Register to be able 4594 * to read the other two registers. 4595 */ 4596 E1000_READ_REG(hw, E1000_SYSTIMR); 4597 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4598 /* Only the 8 LSB are valid. */ 4599 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4600 & 0xff) << 32; 4601 break; 4602 default: 4603 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4604 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4605 << 32; 4606 break; 4607 } 4608 4609 return systime_cycles; 4610 } 4611 4612 static uint64_t 4613 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4614 { 4615 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4616 uint64_t rx_tstamp_cycles; 4617 4618 switch (hw->mac.type) { 4619 case e1000_i210: 4620 case e1000_i211: 4621 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4622 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4623 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4624 * NSEC_PER_SEC; 4625 break; 4626 case e1000_82580: 4627 case e1000_i350: 4628 case e1000_i354: 4629 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4630 /* Only the 8 LSB are valid. */ 4631 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4632 & 0xff) << 32; 4633 break; 4634 default: 4635 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4636 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4637 << 32; 4638 break; 4639 } 4640 4641 return rx_tstamp_cycles; 4642 } 4643 4644 static uint64_t 4645 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4646 { 4647 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4648 uint64_t tx_tstamp_cycles; 4649 4650 switch (hw->mac.type) { 4651 case e1000_i210: 4652 case e1000_i211: 4653 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4654 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4655 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4656 * NSEC_PER_SEC; 4657 break; 4658 case e1000_82580: 4659 case e1000_i350: 4660 case e1000_i354: 4661 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4662 /* Only the 8 LSB are valid. */ 4663 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 4664 & 0xff) << 32; 4665 break; 4666 default: 4667 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4668 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4669 << 32; 4670 break; 4671 } 4672 4673 return tx_tstamp_cycles; 4674 } 4675 4676 static void 4677 igb_start_timecounters(struct rte_eth_dev *dev) 4678 { 4679 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4680 struct e1000_adapter *adapter = dev->data->dev_private; 4681 uint32_t incval = 1; 4682 uint32_t shift = 0; 4683 uint64_t mask = E1000_CYCLECOUNTER_MASK; 4684 4685 switch (hw->mac.type) { 4686 case e1000_82580: 4687 case e1000_i350: 4688 case e1000_i354: 4689 /* 32 LSB bits + 8 MSB bits = 40 bits */ 4690 mask = (1ULL << 40) - 1; 4691 /* fall-through */ 4692 case e1000_i210: 4693 case e1000_i211: 4694 /* 4695 * Start incrementing the register 4696 * used to timestamp PTP packets. 4697 */ 4698 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 4699 break; 4700 case e1000_82576: 4701 incval = E1000_INCVALUE_82576; 4702 shift = IGB_82576_TSYNC_SHIFT; 4703 E1000_WRITE_REG(hw, E1000_TIMINCA, 4704 E1000_INCPERIOD_82576 | incval); 4705 break; 4706 default: 4707 /* Not supported */ 4708 return; 4709 } 4710 4711 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 4712 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 4713 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 4714 4715 adapter->systime_tc.cc_mask = mask; 4716 adapter->systime_tc.cc_shift = shift; 4717 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 4718 4719 adapter->rx_tstamp_tc.cc_mask = mask; 4720 adapter->rx_tstamp_tc.cc_shift = shift; 4721 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 4722 4723 adapter->tx_tstamp_tc.cc_mask = mask; 4724 adapter->tx_tstamp_tc.cc_shift = shift; 4725 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 4726 } 4727 4728 static int 4729 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 4730 { 4731 struct e1000_adapter *adapter = dev->data->dev_private; 4732 4733 adapter->systime_tc.nsec += delta; 4734 adapter->rx_tstamp_tc.nsec += delta; 4735 adapter->tx_tstamp_tc.nsec += delta; 4736 4737 return 0; 4738 } 4739 4740 static int 4741 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 4742 { 4743 uint64_t ns; 4744 struct e1000_adapter *adapter = dev->data->dev_private; 4745 4746 ns = rte_timespec_to_ns(ts); 4747 4748 /* Set the timecounters to a new value. */ 4749 adapter->systime_tc.nsec = ns; 4750 adapter->rx_tstamp_tc.nsec = ns; 4751 adapter->tx_tstamp_tc.nsec = ns; 4752 4753 return 0; 4754 } 4755 4756 static int 4757 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 4758 { 4759 uint64_t ns, systime_cycles; 4760 struct e1000_adapter *adapter = dev->data->dev_private; 4761 4762 systime_cycles = igb_read_systime_cyclecounter(dev); 4763 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 4764 *ts = rte_ns_to_timespec(ns); 4765 4766 return 0; 4767 } 4768 4769 static int 4770 igb_timesync_enable(struct rte_eth_dev *dev) 4771 { 4772 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4773 uint32_t tsync_ctl; 4774 uint32_t tsauxc; 4775 4776 /* Stop the timesync system time. */ 4777 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 4778 /* Reset the timesync system time value. */ 4779 switch (hw->mac.type) { 4780 case e1000_82580: 4781 case e1000_i350: 4782 case e1000_i354: 4783 case e1000_i210: 4784 case e1000_i211: 4785 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 4786 /* fall-through */ 4787 case e1000_82576: 4788 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 4789 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 4790 break; 4791 default: 4792 /* Not supported. */ 4793 return -ENOTSUP; 4794 } 4795 4796 /* Enable system time for it isn't on by default. */ 4797 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 4798 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 4799 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 4800 4801 igb_start_timecounters(dev); 4802 4803 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 4804 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 4805 (RTE_ETHER_TYPE_1588 | 4806 E1000_ETQF_FILTER_ENABLE | 4807 E1000_ETQF_1588)); 4808 4809 /* Enable timestamping of received PTP packets. */ 4810 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4811 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 4812 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 4813 4814 /* Enable Timestamping of transmitted PTP packets. */ 4815 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4816 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 4817 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 4818 4819 return 0; 4820 } 4821 4822 static int 4823 igb_timesync_disable(struct rte_eth_dev *dev) 4824 { 4825 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4826 uint32_t tsync_ctl; 4827 4828 /* Disable timestamping of transmitted PTP packets. */ 4829 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4830 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 4831 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 4832 4833 /* Disable timestamping of received PTP packets. */ 4834 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4835 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 4836 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 4837 4838 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 4839 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 4840 4841 /* Stop incrementating the System Time registers. */ 4842 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 4843 4844 return 0; 4845 } 4846 4847 static int 4848 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 4849 struct timespec *timestamp, 4850 uint32_t flags __rte_unused) 4851 { 4852 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4853 struct e1000_adapter *adapter = dev->data->dev_private; 4854 uint32_t tsync_rxctl; 4855 uint64_t rx_tstamp_cycles; 4856 uint64_t ns; 4857 4858 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 4859 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 4860 return -EINVAL; 4861 4862 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 4863 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 4864 *timestamp = rte_ns_to_timespec(ns); 4865 4866 return 0; 4867 } 4868 4869 static int 4870 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 4871 struct timespec *timestamp) 4872 { 4873 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4874 struct e1000_adapter *adapter = dev->data->dev_private; 4875 uint32_t tsync_txctl; 4876 uint64_t tx_tstamp_cycles; 4877 uint64_t ns; 4878 4879 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 4880 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 4881 return -EINVAL; 4882 4883 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 4884 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 4885 *timestamp = rte_ns_to_timespec(ns); 4886 4887 return 0; 4888 } 4889 4890 static int 4891 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 4892 { 4893 int count = 0; 4894 int g_ind = 0; 4895 const struct reg_info *reg_group; 4896 4897 while ((reg_group = igb_regs[g_ind++])) 4898 count += igb_reg_group_count(reg_group); 4899 4900 return count; 4901 } 4902 4903 static int 4904 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 4905 { 4906 int count = 0; 4907 int g_ind = 0; 4908 const struct reg_info *reg_group; 4909 4910 while ((reg_group = igbvf_regs[g_ind++])) 4911 count += igb_reg_group_count(reg_group); 4912 4913 return count; 4914 } 4915 4916 static int 4917 eth_igb_get_regs(struct rte_eth_dev *dev, 4918 struct rte_dev_reg_info *regs) 4919 { 4920 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4921 uint32_t *data = regs->data; 4922 int g_ind = 0; 4923 int count = 0; 4924 const struct reg_info *reg_group; 4925 4926 if (data == NULL) { 4927 regs->length = eth_igb_get_reg_length(dev); 4928 regs->width = sizeof(uint32_t); 4929 return 0; 4930 } 4931 4932 /* Support only full register dump */ 4933 if ((regs->length == 0) || 4934 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 4935 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 4936 hw->device_id; 4937 while ((reg_group = igb_regs[g_ind++])) 4938 count += igb_read_regs_group(dev, &data[count], 4939 reg_group); 4940 return 0; 4941 } 4942 4943 return -ENOTSUP; 4944 } 4945 4946 static int 4947 igbvf_get_regs(struct rte_eth_dev *dev, 4948 struct rte_dev_reg_info *regs) 4949 { 4950 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4951 uint32_t *data = regs->data; 4952 int g_ind = 0; 4953 int count = 0; 4954 const struct reg_info *reg_group; 4955 4956 if (data == NULL) { 4957 regs->length = igbvf_get_reg_length(dev); 4958 regs->width = sizeof(uint32_t); 4959 return 0; 4960 } 4961 4962 /* Support only full register dump */ 4963 if ((regs->length == 0) || 4964 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 4965 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 4966 hw->device_id; 4967 while ((reg_group = igbvf_regs[g_ind++])) 4968 count += igb_read_regs_group(dev, &data[count], 4969 reg_group); 4970 return 0; 4971 } 4972 4973 return -ENOTSUP; 4974 } 4975 4976 static int 4977 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 4978 { 4979 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4980 4981 /* Return unit is byte count */ 4982 return hw->nvm.word_size * 2; 4983 } 4984 4985 static int 4986 eth_igb_get_eeprom(struct rte_eth_dev *dev, 4987 struct rte_dev_eeprom_info *in_eeprom) 4988 { 4989 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4990 struct e1000_nvm_info *nvm = &hw->nvm; 4991 uint16_t *data = in_eeprom->data; 4992 int first, length; 4993 4994 first = in_eeprom->offset >> 1; 4995 length = in_eeprom->length >> 1; 4996 if ((first >= hw->nvm.word_size) || 4997 ((first + length) >= hw->nvm.word_size)) 4998 return -EINVAL; 4999 5000 in_eeprom->magic = hw->vendor_id | 5001 ((uint32_t)hw->device_id << 16); 5002 5003 if ((nvm->ops.read) == NULL) 5004 return -ENOTSUP; 5005 5006 return nvm->ops.read(hw, first, length, data); 5007 } 5008 5009 static int 5010 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5011 struct rte_dev_eeprom_info *in_eeprom) 5012 { 5013 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5014 struct e1000_nvm_info *nvm = &hw->nvm; 5015 uint16_t *data = in_eeprom->data; 5016 int first, length; 5017 5018 first = in_eeprom->offset >> 1; 5019 length = in_eeprom->length >> 1; 5020 if ((first >= hw->nvm.word_size) || 5021 ((first + length) >= hw->nvm.word_size)) 5022 return -EINVAL; 5023 5024 in_eeprom->magic = (uint32_t)hw->vendor_id | 5025 ((uint32_t)hw->device_id << 16); 5026 5027 if ((nvm->ops.write) == NULL) 5028 return -ENOTSUP; 5029 return nvm->ops.write(hw, first, length, data); 5030 } 5031 5032 static int 5033 eth_igb_get_module_info(struct rte_eth_dev *dev, 5034 struct rte_eth_dev_module_info *modinfo) 5035 { 5036 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5037 5038 uint32_t status = 0; 5039 uint16_t sff8472_rev, addr_mode; 5040 bool page_swap = false; 5041 5042 if (hw->phy.media_type == e1000_media_type_copper || 5043 hw->phy.media_type == e1000_media_type_unknown) 5044 return -EOPNOTSUPP; 5045 5046 /* Check whether we support SFF-8472 or not */ 5047 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5048 if (status) 5049 return -EIO; 5050 5051 /* addressing mode is not supported */ 5052 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5053 if (status) 5054 return -EIO; 5055 5056 /* addressing mode is not supported */ 5057 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5058 PMD_DRV_LOG(ERR, 5059 "Address change required to access page 0xA2, " 5060 "but not supported. Please report the module " 5061 "type to the driver maintainers.\n"); 5062 page_swap = true; 5063 } 5064 5065 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5066 /* We have an SFP, but it does not support SFF-8472 */ 5067 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5068 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5069 } else { 5070 /* We have an SFP which supports a revision of SFF-8472 */ 5071 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5072 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5073 } 5074 5075 return 0; 5076 } 5077 5078 static int 5079 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5080 struct rte_dev_eeprom_info *info) 5081 { 5082 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5083 5084 uint32_t status = 0; 5085 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5086 u16 first_word, last_word; 5087 int i = 0; 5088 5089 first_word = info->offset >> 1; 5090 last_word = (info->offset + info->length - 1) >> 1; 5091 5092 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5093 for (i = 0; i < last_word - first_word + 1; i++) { 5094 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5095 &dataword[i]); 5096 if (status) { 5097 /* Error occurred while reading module */ 5098 return -EIO; 5099 } 5100 5101 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5102 } 5103 5104 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5105 5106 return 0; 5107 } 5108 5109 static int 5110 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5111 { 5112 struct e1000_hw *hw = 5113 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5114 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5115 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5116 uint32_t vec = E1000_MISC_VEC_ID; 5117 5118 if (rte_intr_allow_others(intr_handle)) 5119 vec = E1000_RX_VEC_START; 5120 5121 uint32_t mask = 1 << (queue_id + vec); 5122 5123 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5124 E1000_WRITE_FLUSH(hw); 5125 5126 return 0; 5127 } 5128 5129 static int 5130 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5131 { 5132 struct e1000_hw *hw = 5133 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5134 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5135 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5136 uint32_t vec = E1000_MISC_VEC_ID; 5137 5138 if (rte_intr_allow_others(intr_handle)) 5139 vec = E1000_RX_VEC_START; 5140 5141 uint32_t mask = 1 << (queue_id + vec); 5142 uint32_t regval; 5143 5144 regval = E1000_READ_REG(hw, E1000_EIMS); 5145 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5146 E1000_WRITE_FLUSH(hw); 5147 5148 rte_intr_ack(intr_handle); 5149 5150 return 0; 5151 } 5152 5153 static void 5154 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5155 uint8_t index, uint8_t offset) 5156 { 5157 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5158 5159 /* clear bits */ 5160 val &= ~((uint32_t)0xFF << offset); 5161 5162 /* write vector and valid bit */ 5163 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5164 5165 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5166 } 5167 5168 static void 5169 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5170 uint8_t queue, uint8_t msix_vector) 5171 { 5172 uint32_t tmp = 0; 5173 5174 if (hw->mac.type == e1000_82575) { 5175 if (direction == 0) 5176 tmp = E1000_EICR_RX_QUEUE0 << queue; 5177 else if (direction == 1) 5178 tmp = E1000_EICR_TX_QUEUE0 << queue; 5179 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5180 } else if (hw->mac.type == e1000_82576) { 5181 if ((direction == 0) || (direction == 1)) 5182 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5183 ((queue & 0x8) << 1) + 5184 8 * direction); 5185 } else if ((hw->mac.type == e1000_82580) || 5186 (hw->mac.type == e1000_i350) || 5187 (hw->mac.type == e1000_i354) || 5188 (hw->mac.type == e1000_i210) || 5189 (hw->mac.type == e1000_i211)) { 5190 if ((direction == 0) || (direction == 1)) 5191 eth_igb_write_ivar(hw, msix_vector, 5192 queue >> 1, 5193 ((queue & 0x1) << 4) + 5194 8 * direction); 5195 } 5196 } 5197 5198 /* Sets up the hardware to generate MSI-X interrupts properly 5199 * @hw 5200 * board private structure 5201 */ 5202 static void 5203 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5204 { 5205 int queue_id; 5206 uint32_t tmpval, regval, intr_mask; 5207 struct e1000_hw *hw = 5208 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5209 uint32_t vec = E1000_MISC_VEC_ID; 5210 uint32_t base = E1000_MISC_VEC_ID; 5211 uint32_t misc_shift = 0; 5212 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5213 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5214 5215 /* won't configure msix register if no mapping is done 5216 * between intr vector and event fd 5217 */ 5218 if (!rte_intr_dp_is_en(intr_handle)) 5219 return; 5220 5221 if (rte_intr_allow_others(intr_handle)) { 5222 vec = base = E1000_RX_VEC_START; 5223 misc_shift = 1; 5224 } 5225 5226 /* set interrupt vector for other causes */ 5227 if (hw->mac.type == e1000_82575) { 5228 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5229 /* enable MSI-X PBA support */ 5230 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5231 5232 /* Auto-Mask interrupts upon ICR read */ 5233 tmpval |= E1000_CTRL_EXT_EIAME; 5234 tmpval |= E1000_CTRL_EXT_IRCA; 5235 5236 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5237 5238 /* enable msix_other interrupt */ 5239 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5240 regval = E1000_READ_REG(hw, E1000_EIAC); 5241 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5242 regval = E1000_READ_REG(hw, E1000_EIAM); 5243 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5244 } else if ((hw->mac.type == e1000_82576) || 5245 (hw->mac.type == e1000_82580) || 5246 (hw->mac.type == e1000_i350) || 5247 (hw->mac.type == e1000_i354) || 5248 (hw->mac.type == e1000_i210) || 5249 (hw->mac.type == e1000_i211)) { 5250 /* turn on MSI-X capability first */ 5251 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5252 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5253 E1000_GPIE_NSICR); 5254 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5255 misc_shift; 5256 5257 if (dev->data->dev_conf.intr_conf.lsc != 0) 5258 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5259 5260 regval = E1000_READ_REG(hw, E1000_EIAC); 5261 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5262 5263 /* enable msix_other interrupt */ 5264 regval = E1000_READ_REG(hw, E1000_EIMS); 5265 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5266 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5267 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5268 } 5269 5270 /* use EIAM to auto-mask when MSI-X interrupt 5271 * is asserted, this saves a register write for every interrupt 5272 */ 5273 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5274 misc_shift; 5275 5276 if (dev->data->dev_conf.intr_conf.lsc != 0) 5277 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5278 5279 regval = E1000_READ_REG(hw, E1000_EIAM); 5280 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5281 5282 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5283 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5284 intr_handle->intr_vec[queue_id] = vec; 5285 if (vec < base + intr_handle->nb_efd - 1) 5286 vec++; 5287 } 5288 5289 E1000_WRITE_FLUSH(hw); 5290 } 5291 5292 /* restore n-tuple filter */ 5293 static inline void 5294 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5295 { 5296 struct e1000_filter_info *filter_info = 5297 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5298 struct e1000_5tuple_filter *p_5tuple; 5299 struct e1000_2tuple_filter *p_2tuple; 5300 5301 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5302 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5303 } 5304 5305 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5306 igb_inject_2uple_filter(dev, p_2tuple); 5307 } 5308 } 5309 5310 /* restore SYN filter */ 5311 static inline void 5312 igb_syn_filter_restore(struct rte_eth_dev *dev) 5313 { 5314 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5315 struct e1000_filter_info *filter_info = 5316 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5317 uint32_t synqf; 5318 5319 synqf = filter_info->syn_info; 5320 5321 if (synqf & E1000_SYN_FILTER_ENABLE) { 5322 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5323 E1000_WRITE_FLUSH(hw); 5324 } 5325 } 5326 5327 /* restore ethernet type filter */ 5328 static inline void 5329 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5330 { 5331 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5332 struct e1000_filter_info *filter_info = 5333 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5334 int i; 5335 5336 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5337 if (filter_info->ethertype_mask & (1 << i)) { 5338 E1000_WRITE_REG(hw, E1000_ETQF(i), 5339 filter_info->ethertype_filters[i].etqf); 5340 E1000_WRITE_FLUSH(hw); 5341 } 5342 } 5343 } 5344 5345 /* restore flex byte filter */ 5346 static inline void 5347 igb_flex_filter_restore(struct rte_eth_dev *dev) 5348 { 5349 struct e1000_filter_info *filter_info = 5350 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5351 struct e1000_flex_filter *flex_filter; 5352 5353 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5354 igb_inject_flex_filter(dev, flex_filter); 5355 } 5356 } 5357 5358 /* restore rss filter */ 5359 static inline void 5360 igb_rss_filter_restore(struct rte_eth_dev *dev) 5361 { 5362 struct e1000_filter_info *filter_info = 5363 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5364 5365 if (filter_info->rss_info.conf.queue_num) 5366 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5367 } 5368 5369 /* restore all types filter */ 5370 static int 5371 igb_filter_restore(struct rte_eth_dev *dev) 5372 { 5373 igb_ntuple_filter_restore(dev); 5374 igb_ethertype_filter_restore(dev); 5375 igb_syn_filter_restore(dev); 5376 igb_flex_filter_restore(dev); 5377 igb_rss_filter_restore(dev); 5378 5379 return 0; 5380 } 5381 5382 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5383 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5384 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5385 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5386 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5387 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5388