1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static void eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static void eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static int eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static int eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static int eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static int eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 100 unsigned int limit); 101 static int eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static int eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static int eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static int eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct rte_ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct rte_ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static void igbvf_dev_stop(struct rte_eth_dev *dev); 158 static void igbvf_dev_close(struct rte_eth_dev *dev); 159 static int igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static int igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static int igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static int igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static int eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct rte_ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 190 struct rte_eth_syn_filter *filter); 191 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 192 enum rte_filter_op filter_op, 193 void *arg); 194 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 195 struct rte_eth_ntuple_filter *ntuple_filter); 196 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 197 struct rte_eth_ntuple_filter *ntuple_filter); 198 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 199 struct rte_eth_flex_filter *filter); 200 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 201 enum rte_filter_op filter_op, 202 void *arg); 203 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *ntuple_filter); 205 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 206 struct rte_eth_ntuple_filter *ntuple_filter); 207 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 208 struct rte_eth_ntuple_filter *filter); 209 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 210 enum rte_filter_op filter_op, 211 void *arg); 212 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 213 enum rte_filter_op filter_op, 214 void *arg); 215 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 216 struct rte_eth_ethertype_filter *filter); 217 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 218 enum rte_filter_type filter_type, 219 enum rte_filter_op filter_op, 220 void *arg); 221 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 222 static int eth_igb_get_regs(struct rte_eth_dev *dev, 223 struct rte_dev_reg_info *regs); 224 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 225 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 226 struct rte_dev_eeprom_info *eeprom); 227 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 228 struct rte_dev_eeprom_info *eeprom); 229 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 230 struct rte_eth_dev_module_info *modinfo); 231 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 232 struct rte_dev_eeprom_info *info); 233 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 234 struct rte_ether_addr *mc_addr_set, 235 uint32_t nb_mc_addr); 236 static int igb_timesync_enable(struct rte_eth_dev *dev); 237 static int igb_timesync_disable(struct rte_eth_dev *dev); 238 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 239 struct timespec *timestamp, 240 uint32_t flags); 241 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 242 struct timespec *timestamp); 243 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 244 static int igb_timesync_read_time(struct rte_eth_dev *dev, 245 struct timespec *timestamp); 246 static int igb_timesync_write_time(struct rte_eth_dev *dev, 247 const struct timespec *timestamp); 248 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 249 uint16_t queue_id); 250 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 251 uint16_t queue_id); 252 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 253 uint8_t queue, uint8_t msix_vector); 254 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 255 uint8_t index, uint8_t offset); 256 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 257 static void eth_igbvf_interrupt_handler(void *param); 258 static void igbvf_mbx_process(struct rte_eth_dev *dev); 259 static int igb_filter_restore(struct rte_eth_dev *dev); 260 261 /* 262 * Define VF Stats MACRO for Non "cleared on read" register 263 */ 264 #define UPDATE_VF_STAT(reg, last, cur) \ 265 { \ 266 u32 latest = E1000_READ_REG(hw, reg); \ 267 cur += (latest - last) & UINT_MAX; \ 268 last = latest; \ 269 } 270 271 #define IGB_FC_PAUSE_TIME 0x0680 272 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 273 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 274 275 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 276 277 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 278 279 /* 280 * The set of PCI devices this driver supports 281 */ 282 static const struct rte_pci_id pci_id_igb_map[] = { 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 291 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 295 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 302 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 320 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 321 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 322 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 323 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 324 { .vendor_id = 0, /* sentinel */ }, 325 }; 326 327 /* 328 * The set of PCI devices this driver supports (for 82576&I350 VF) 329 */ 330 static const struct rte_pci_id pci_id_igbvf_map[] = { 331 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 332 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 333 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 334 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 335 { .vendor_id = 0, /* sentinel */ }, 336 }; 337 338 static const struct rte_eth_desc_lim rx_desc_lim = { 339 .nb_max = E1000_MAX_RING_DESC, 340 .nb_min = E1000_MIN_RING_DESC, 341 .nb_align = IGB_RXD_ALIGN, 342 }; 343 344 static const struct rte_eth_desc_lim tx_desc_lim = { 345 .nb_max = E1000_MAX_RING_DESC, 346 .nb_min = E1000_MIN_RING_DESC, 347 .nb_align = IGB_RXD_ALIGN, 348 .nb_seg_max = IGB_TX_MAX_SEG, 349 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 350 }; 351 352 static const struct eth_dev_ops eth_igb_ops = { 353 .dev_configure = eth_igb_configure, 354 .dev_start = eth_igb_start, 355 .dev_stop = eth_igb_stop, 356 .dev_set_link_up = eth_igb_dev_set_link_up, 357 .dev_set_link_down = eth_igb_dev_set_link_down, 358 .dev_close = eth_igb_close, 359 .dev_reset = eth_igb_reset, 360 .promiscuous_enable = eth_igb_promiscuous_enable, 361 .promiscuous_disable = eth_igb_promiscuous_disable, 362 .allmulticast_enable = eth_igb_allmulticast_enable, 363 .allmulticast_disable = eth_igb_allmulticast_disable, 364 .link_update = eth_igb_link_update, 365 .stats_get = eth_igb_stats_get, 366 .xstats_get = eth_igb_xstats_get, 367 .xstats_get_by_id = eth_igb_xstats_get_by_id, 368 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 369 .xstats_get_names = eth_igb_xstats_get_names, 370 .stats_reset = eth_igb_stats_reset, 371 .xstats_reset = eth_igb_xstats_reset, 372 .fw_version_get = eth_igb_fw_version_get, 373 .dev_infos_get = eth_igb_infos_get, 374 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 375 .mtu_set = eth_igb_mtu_set, 376 .vlan_filter_set = eth_igb_vlan_filter_set, 377 .vlan_tpid_set = eth_igb_vlan_tpid_set, 378 .vlan_offload_set = eth_igb_vlan_offload_set, 379 .rx_queue_setup = eth_igb_rx_queue_setup, 380 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 381 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 382 .rx_queue_release = eth_igb_rx_queue_release, 383 .rx_queue_count = eth_igb_rx_queue_count, 384 .rx_descriptor_done = eth_igb_rx_descriptor_done, 385 .rx_descriptor_status = eth_igb_rx_descriptor_status, 386 .tx_descriptor_status = eth_igb_tx_descriptor_status, 387 .tx_queue_setup = eth_igb_tx_queue_setup, 388 .tx_queue_release = eth_igb_tx_queue_release, 389 .tx_done_cleanup = eth_igb_tx_done_cleanup, 390 .dev_led_on = eth_igb_led_on, 391 .dev_led_off = eth_igb_led_off, 392 .flow_ctrl_get = eth_igb_flow_ctrl_get, 393 .flow_ctrl_set = eth_igb_flow_ctrl_set, 394 .mac_addr_add = eth_igb_rar_set, 395 .mac_addr_remove = eth_igb_rar_clear, 396 .mac_addr_set = eth_igb_default_mac_addr_set, 397 .reta_update = eth_igb_rss_reta_update, 398 .reta_query = eth_igb_rss_reta_query, 399 .rss_hash_update = eth_igb_rss_hash_update, 400 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 401 .filter_ctrl = eth_igb_filter_ctrl, 402 .set_mc_addr_list = eth_igb_set_mc_addr_list, 403 .rxq_info_get = igb_rxq_info_get, 404 .txq_info_get = igb_txq_info_get, 405 .timesync_enable = igb_timesync_enable, 406 .timesync_disable = igb_timesync_disable, 407 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 408 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 409 .get_reg = eth_igb_get_regs, 410 .get_eeprom_length = eth_igb_get_eeprom_length, 411 .get_eeprom = eth_igb_get_eeprom, 412 .set_eeprom = eth_igb_set_eeprom, 413 .get_module_info = eth_igb_get_module_info, 414 .get_module_eeprom = eth_igb_get_module_eeprom, 415 .timesync_adjust_time = igb_timesync_adjust_time, 416 .timesync_read_time = igb_timesync_read_time, 417 .timesync_write_time = igb_timesync_write_time, 418 }; 419 420 /* 421 * dev_ops for virtual function, bare necessities for basic vf 422 * operation have been implemented 423 */ 424 static const struct eth_dev_ops igbvf_eth_dev_ops = { 425 .dev_configure = igbvf_dev_configure, 426 .dev_start = igbvf_dev_start, 427 .dev_stop = igbvf_dev_stop, 428 .dev_close = igbvf_dev_close, 429 .promiscuous_enable = igbvf_promiscuous_enable, 430 .promiscuous_disable = igbvf_promiscuous_disable, 431 .allmulticast_enable = igbvf_allmulticast_enable, 432 .allmulticast_disable = igbvf_allmulticast_disable, 433 .link_update = eth_igb_link_update, 434 .stats_get = eth_igbvf_stats_get, 435 .xstats_get = eth_igbvf_xstats_get, 436 .xstats_get_names = eth_igbvf_xstats_get_names, 437 .stats_reset = eth_igbvf_stats_reset, 438 .xstats_reset = eth_igbvf_stats_reset, 439 .vlan_filter_set = igbvf_vlan_filter_set, 440 .dev_infos_get = eth_igbvf_infos_get, 441 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 442 .rx_queue_setup = eth_igb_rx_queue_setup, 443 .rx_queue_release = eth_igb_rx_queue_release, 444 .rx_descriptor_done = eth_igb_rx_descriptor_done, 445 .rx_descriptor_status = eth_igb_rx_descriptor_status, 446 .tx_descriptor_status = eth_igb_tx_descriptor_status, 447 .tx_queue_setup = eth_igb_tx_queue_setup, 448 .tx_queue_release = eth_igb_tx_queue_release, 449 .set_mc_addr_list = eth_igb_set_mc_addr_list, 450 .rxq_info_get = igb_rxq_info_get, 451 .txq_info_get = igb_txq_info_get, 452 .mac_addr_set = igbvf_default_mac_addr_set, 453 .get_reg = igbvf_get_regs, 454 }; 455 456 /* store statistics names and its offset in stats structure */ 457 struct rte_igb_xstats_name_off { 458 char name[RTE_ETH_XSTATS_NAME_SIZE]; 459 unsigned offset; 460 }; 461 462 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 463 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 464 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 465 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 466 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 467 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 468 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 469 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 470 ecol)}, 471 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 472 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 473 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 474 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 475 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 476 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 477 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 478 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 479 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 480 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 481 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 482 fcruc)}, 483 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 484 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 485 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 486 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 487 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 488 prc1023)}, 489 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 490 prc1522)}, 491 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 492 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 493 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 494 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 495 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 496 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 497 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 498 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 499 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 500 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 501 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 502 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 503 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 504 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 505 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 506 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 507 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 508 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 509 ptc1023)}, 510 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 511 ptc1522)}, 512 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 513 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 514 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 515 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 516 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 517 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 518 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 519 520 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 521 }; 522 523 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 524 sizeof(rte_igb_stats_strings[0])) 525 526 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 527 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 528 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 529 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 530 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 531 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 532 }; 533 534 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 535 sizeof(rte_igbvf_stats_strings[0])) 536 537 538 static inline void 539 igb_intr_enable(struct rte_eth_dev *dev) 540 { 541 struct e1000_interrupt *intr = 542 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 543 struct e1000_hw *hw = 544 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 545 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 546 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 547 548 if (rte_intr_allow_others(intr_handle) && 549 dev->data->dev_conf.intr_conf.lsc != 0) { 550 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 551 } 552 553 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 554 E1000_WRITE_FLUSH(hw); 555 } 556 557 static void 558 igb_intr_disable(struct rte_eth_dev *dev) 559 { 560 struct e1000_hw *hw = 561 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 562 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 563 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 564 565 if (rte_intr_allow_others(intr_handle) && 566 dev->data->dev_conf.intr_conf.lsc != 0) { 567 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 568 } 569 570 E1000_WRITE_REG(hw, E1000_IMC, ~0); 571 E1000_WRITE_FLUSH(hw); 572 } 573 574 static inline void 575 igbvf_intr_enable(struct rte_eth_dev *dev) 576 { 577 struct e1000_hw *hw = 578 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 579 580 /* only for mailbox */ 581 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 582 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 583 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 584 E1000_WRITE_FLUSH(hw); 585 } 586 587 /* only for mailbox now. If RX/TX needed, should extend this function. */ 588 static void 589 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 590 { 591 uint32_t tmp = 0; 592 593 /* mailbox */ 594 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 595 tmp |= E1000_VTIVAR_VALID; 596 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 597 } 598 599 static void 600 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 601 { 602 struct e1000_hw *hw = 603 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 604 605 /* Configure VF other cause ivar */ 606 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 607 } 608 609 static inline int32_t 610 igb_pf_reset_hw(struct e1000_hw *hw) 611 { 612 uint32_t ctrl_ext; 613 int32_t status; 614 615 status = e1000_reset_hw(hw); 616 617 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 618 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 619 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 620 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 621 E1000_WRITE_FLUSH(hw); 622 623 return status; 624 } 625 626 static void 627 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 628 { 629 struct e1000_hw *hw = 630 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 631 632 633 hw->vendor_id = pci_dev->id.vendor_id; 634 hw->device_id = pci_dev->id.device_id; 635 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 636 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 637 638 e1000_set_mac_type(hw); 639 640 /* need to check if it is a vf device below */ 641 } 642 643 static int 644 igb_reset_swfw_lock(struct e1000_hw *hw) 645 { 646 int ret_val; 647 648 /* 649 * Do mac ops initialization manually here, since we will need 650 * some function pointers set by this call. 651 */ 652 ret_val = e1000_init_mac_params(hw); 653 if (ret_val) 654 return ret_val; 655 656 /* 657 * SMBI lock should not fail in this early stage. If this is the case, 658 * it is due to an improper exit of the application. 659 * So force the release of the faulty lock. 660 */ 661 if (e1000_get_hw_semaphore_generic(hw) < 0) { 662 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 663 } 664 e1000_put_hw_semaphore_generic(hw); 665 666 if (hw->mac.ops.acquire_swfw_sync != NULL) { 667 uint16_t mask; 668 669 /* 670 * Phy lock should not fail in this early stage. If this is the case, 671 * it is due to an improper exit of the application. 672 * So force the release of the faulty lock. 673 */ 674 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 675 if (hw->bus.func > E1000_FUNC_1) 676 mask <<= 2; 677 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 678 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 679 hw->bus.func); 680 } 681 hw->mac.ops.release_swfw_sync(hw, mask); 682 683 /* 684 * This one is more tricky since it is common to all ports; but 685 * swfw_sync retries last long enough (1s) to be almost sure that if 686 * lock can not be taken it is due to an improper lock of the 687 * semaphore. 688 */ 689 mask = E1000_SWFW_EEP_SM; 690 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 691 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 692 } 693 hw->mac.ops.release_swfw_sync(hw, mask); 694 } 695 696 return E1000_SUCCESS; 697 } 698 699 /* Remove all ntuple filters of the device */ 700 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 701 { 702 struct e1000_filter_info *filter_info = 703 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 704 struct e1000_5tuple_filter *p_5tuple; 705 struct e1000_2tuple_filter *p_2tuple; 706 707 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 708 TAILQ_REMOVE(&filter_info->fivetuple_list, 709 p_5tuple, entries); 710 rte_free(p_5tuple); 711 } 712 filter_info->fivetuple_mask = 0; 713 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 714 TAILQ_REMOVE(&filter_info->twotuple_list, 715 p_2tuple, entries); 716 rte_free(p_2tuple); 717 } 718 filter_info->twotuple_mask = 0; 719 720 return 0; 721 } 722 723 /* Remove all flex filters of the device */ 724 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 725 { 726 struct e1000_filter_info *filter_info = 727 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 728 struct e1000_flex_filter *p_flex; 729 730 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 731 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 732 rte_free(p_flex); 733 } 734 filter_info->flex_mask = 0; 735 736 return 0; 737 } 738 739 static int 740 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 741 { 742 int error = 0; 743 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 744 struct e1000_hw *hw = 745 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 746 struct e1000_vfta * shadow_vfta = 747 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 748 struct e1000_filter_info *filter_info = 749 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 750 struct e1000_adapter *adapter = 751 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 752 753 uint32_t ctrl_ext; 754 755 eth_dev->dev_ops = ð_igb_ops; 756 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 757 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 758 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 759 760 /* for secondary processes, we don't initialise any further as primary 761 * has already done this work. Only check we don't need a different 762 * RX function */ 763 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 764 if (eth_dev->data->scattered_rx) 765 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 766 return 0; 767 } 768 769 rte_eth_copy_pci_info(eth_dev, pci_dev); 770 771 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 772 773 igb_identify_hardware(eth_dev, pci_dev); 774 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 775 error = -EIO; 776 goto err_late; 777 } 778 779 e1000_get_bus_info(hw); 780 781 /* Reset any pending lock */ 782 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 783 error = -EIO; 784 goto err_late; 785 } 786 787 /* Finish initialization */ 788 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 789 error = -EIO; 790 goto err_late; 791 } 792 793 hw->mac.autoneg = 1; 794 hw->phy.autoneg_wait_to_complete = 0; 795 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 796 797 /* Copper options */ 798 if (hw->phy.media_type == e1000_media_type_copper) { 799 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 800 hw->phy.disable_polarity_correction = 0; 801 hw->phy.ms_type = e1000_ms_hw_default; 802 } 803 804 /* 805 * Start from a known state, this is important in reading the nvm 806 * and mac from that. 807 */ 808 igb_pf_reset_hw(hw); 809 810 /* Make sure we have a good EEPROM before we read from it */ 811 if (e1000_validate_nvm_checksum(hw) < 0) { 812 /* 813 * Some PCI-E parts fail the first check due to 814 * the link being in sleep state, call it again, 815 * if it fails a second time its a real issue. 816 */ 817 if (e1000_validate_nvm_checksum(hw) < 0) { 818 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 819 error = -EIO; 820 goto err_late; 821 } 822 } 823 824 /* Read the permanent MAC address out of the EEPROM */ 825 if (e1000_read_mac_addr(hw) != 0) { 826 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 827 error = -EIO; 828 goto err_late; 829 } 830 831 /* Allocate memory for storing MAC addresses */ 832 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 833 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 834 if (eth_dev->data->mac_addrs == NULL) { 835 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 836 "store MAC addresses", 837 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 838 error = -ENOMEM; 839 goto err_late; 840 } 841 842 /* Copy the permanent MAC address */ 843 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 844 ð_dev->data->mac_addrs[0]); 845 846 /* Pass the information to the rte_eth_dev_close() that it should also 847 * release the private port resources. 848 */ 849 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 850 851 /* initialize the vfta */ 852 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 853 854 /* Now initialize the hardware */ 855 if (igb_hardware_init(hw) != 0) { 856 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 857 rte_free(eth_dev->data->mac_addrs); 858 eth_dev->data->mac_addrs = NULL; 859 error = -ENODEV; 860 goto err_late; 861 } 862 hw->mac.get_link_status = 1; 863 adapter->stopped = 0; 864 865 /* Indicate SOL/IDER usage */ 866 if (e1000_check_reset_block(hw) < 0) { 867 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 868 "SOL/IDER session"); 869 } 870 871 /* initialize PF if max_vfs not zero */ 872 igb_pf_host_init(eth_dev); 873 874 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 875 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 876 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 877 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 878 E1000_WRITE_FLUSH(hw); 879 880 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 881 eth_dev->data->port_id, pci_dev->id.vendor_id, 882 pci_dev->id.device_id); 883 884 rte_intr_callback_register(&pci_dev->intr_handle, 885 eth_igb_interrupt_handler, 886 (void *)eth_dev); 887 888 /* enable uio/vfio intr/eventfd mapping */ 889 rte_intr_enable(&pci_dev->intr_handle); 890 891 /* enable support intr */ 892 igb_intr_enable(eth_dev); 893 894 /* initialize filter info */ 895 memset(filter_info, 0, 896 sizeof(struct e1000_filter_info)); 897 898 TAILQ_INIT(&filter_info->flex_list); 899 TAILQ_INIT(&filter_info->twotuple_list); 900 TAILQ_INIT(&filter_info->fivetuple_list); 901 902 TAILQ_INIT(&igb_filter_ntuple_list); 903 TAILQ_INIT(&igb_filter_ethertype_list); 904 TAILQ_INIT(&igb_filter_syn_list); 905 TAILQ_INIT(&igb_filter_flex_list); 906 TAILQ_INIT(&igb_filter_rss_list); 907 TAILQ_INIT(&igb_flow_list); 908 909 return 0; 910 911 err_late: 912 igb_hw_control_release(hw); 913 914 return error; 915 } 916 917 static int 918 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 919 { 920 PMD_INIT_FUNC_TRACE(); 921 922 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 923 return -EPERM; 924 925 eth_igb_close(eth_dev); 926 927 return 0; 928 } 929 930 /* 931 * Virtual Function device init 932 */ 933 static int 934 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 935 { 936 struct rte_pci_device *pci_dev; 937 struct rte_intr_handle *intr_handle; 938 struct e1000_adapter *adapter = 939 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 940 struct e1000_hw *hw = 941 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 942 int diag; 943 struct rte_ether_addr *perm_addr = 944 (struct rte_ether_addr *)hw->mac.perm_addr; 945 946 PMD_INIT_FUNC_TRACE(); 947 948 eth_dev->dev_ops = &igbvf_eth_dev_ops; 949 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 950 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 951 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 952 953 /* for secondary processes, we don't initialise any further as primary 954 * has already done this work. Only check we don't need a different 955 * RX function */ 956 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 957 if (eth_dev->data->scattered_rx) 958 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 959 return 0; 960 } 961 962 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 963 rte_eth_copy_pci_info(eth_dev, pci_dev); 964 965 hw->device_id = pci_dev->id.device_id; 966 hw->vendor_id = pci_dev->id.vendor_id; 967 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 968 adapter->stopped = 0; 969 970 /* Initialize the shared code (base driver) */ 971 diag = e1000_setup_init_funcs(hw, TRUE); 972 if (diag != 0) { 973 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 974 diag); 975 return -EIO; 976 } 977 978 /* init_mailbox_params */ 979 hw->mbx.ops.init_params(hw); 980 981 /* Disable the interrupts for VF */ 982 igbvf_intr_disable(hw); 983 984 diag = hw->mac.ops.reset_hw(hw); 985 986 /* Allocate memory for storing MAC addresses */ 987 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", RTE_ETHER_ADDR_LEN * 988 hw->mac.rar_entry_count, 0); 989 if (eth_dev->data->mac_addrs == NULL) { 990 PMD_INIT_LOG(ERR, 991 "Failed to allocate %d bytes needed to store MAC " 992 "addresses", 993 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 994 return -ENOMEM; 995 } 996 997 /* Pass the information to the rte_eth_dev_close() that it should also 998 * release the private port resources. 999 */ 1000 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1001 1002 /* Generate a random MAC address, if none was assigned by PF. */ 1003 if (rte_is_zero_ether_addr(perm_addr)) { 1004 rte_eth_random_addr(perm_addr->addr_bytes); 1005 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1006 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1007 "%02x:%02x:%02x:%02x:%02x:%02x", 1008 perm_addr->addr_bytes[0], 1009 perm_addr->addr_bytes[1], 1010 perm_addr->addr_bytes[2], 1011 perm_addr->addr_bytes[3], 1012 perm_addr->addr_bytes[4], 1013 perm_addr->addr_bytes[5]); 1014 } 1015 1016 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1017 if (diag) { 1018 rte_free(eth_dev->data->mac_addrs); 1019 eth_dev->data->mac_addrs = NULL; 1020 return diag; 1021 } 1022 /* Copy the permanent MAC address */ 1023 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1024 ð_dev->data->mac_addrs[0]); 1025 1026 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1027 "mac.type=%s", 1028 eth_dev->data->port_id, pci_dev->id.vendor_id, 1029 pci_dev->id.device_id, "igb_mac_82576_vf"); 1030 1031 intr_handle = &pci_dev->intr_handle; 1032 rte_intr_callback_register(intr_handle, 1033 eth_igbvf_interrupt_handler, eth_dev); 1034 1035 return 0; 1036 } 1037 1038 static int 1039 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1040 { 1041 PMD_INIT_FUNC_TRACE(); 1042 1043 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1044 return -EPERM; 1045 1046 igbvf_dev_close(eth_dev); 1047 1048 return 0; 1049 } 1050 1051 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1052 struct rte_pci_device *pci_dev) 1053 { 1054 return rte_eth_dev_pci_generic_probe(pci_dev, 1055 sizeof(struct e1000_adapter), eth_igb_dev_init); 1056 } 1057 1058 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1059 { 1060 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1061 } 1062 1063 static struct rte_pci_driver rte_igb_pmd = { 1064 .id_table = pci_id_igb_map, 1065 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1066 .probe = eth_igb_pci_probe, 1067 .remove = eth_igb_pci_remove, 1068 }; 1069 1070 1071 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1072 struct rte_pci_device *pci_dev) 1073 { 1074 return rte_eth_dev_pci_generic_probe(pci_dev, 1075 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1076 } 1077 1078 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1079 { 1080 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1081 } 1082 1083 /* 1084 * virtual function driver struct 1085 */ 1086 static struct rte_pci_driver rte_igbvf_pmd = { 1087 .id_table = pci_id_igbvf_map, 1088 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1089 .probe = eth_igbvf_pci_probe, 1090 .remove = eth_igbvf_pci_remove, 1091 }; 1092 1093 static void 1094 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1095 { 1096 struct e1000_hw *hw = 1097 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1098 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1099 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1100 rctl |= E1000_RCTL_VFE; 1101 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1102 } 1103 1104 static int 1105 igb_check_mq_mode(struct rte_eth_dev *dev) 1106 { 1107 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1108 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1109 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1110 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1111 1112 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1113 tx_mq_mode == ETH_MQ_TX_DCB || 1114 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1115 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1116 return -EINVAL; 1117 } 1118 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1119 /* Check multi-queue mode. 1120 * To no break software we accept ETH_MQ_RX_NONE as this might 1121 * be used to turn off VLAN filter. 1122 */ 1123 1124 if (rx_mq_mode == ETH_MQ_RX_NONE || 1125 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1126 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1127 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1128 } else { 1129 /* Only support one queue on VFs. 1130 * RSS together with SRIOV is not supported. 1131 */ 1132 PMD_INIT_LOG(ERR, "SRIOV is active," 1133 " wrong mq_mode rx %d.", 1134 rx_mq_mode); 1135 return -EINVAL; 1136 } 1137 /* TX mode is not used here, so mode might be ignored.*/ 1138 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1139 /* SRIOV only works in VMDq enable mode */ 1140 PMD_INIT_LOG(WARNING, "SRIOV is active," 1141 " TX mode %d is not supported. " 1142 " Driver will behave as %d mode.", 1143 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1144 } 1145 1146 /* check valid queue number */ 1147 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1148 PMD_INIT_LOG(ERR, "SRIOV is active," 1149 " only support one queue on VFs."); 1150 return -EINVAL; 1151 } 1152 } else { 1153 /* To no break software that set invalid mode, only display 1154 * warning if invalid mode is used. 1155 */ 1156 if (rx_mq_mode != ETH_MQ_RX_NONE && 1157 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1158 rx_mq_mode != ETH_MQ_RX_RSS) { 1159 /* RSS together with VMDq not supported*/ 1160 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1161 rx_mq_mode); 1162 return -EINVAL; 1163 } 1164 1165 if (tx_mq_mode != ETH_MQ_TX_NONE && 1166 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1167 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1168 " Due to txmode is meaningless in this" 1169 " driver, just ignore.", 1170 tx_mq_mode); 1171 } 1172 } 1173 return 0; 1174 } 1175 1176 static int 1177 eth_igb_configure(struct rte_eth_dev *dev) 1178 { 1179 struct e1000_interrupt *intr = 1180 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1181 int ret; 1182 1183 PMD_INIT_FUNC_TRACE(); 1184 1185 /* multipe queue mode checking */ 1186 ret = igb_check_mq_mode(dev); 1187 if (ret != 0) { 1188 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1189 ret); 1190 return ret; 1191 } 1192 1193 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1194 PMD_INIT_FUNC_TRACE(); 1195 1196 return 0; 1197 } 1198 1199 static void 1200 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1201 bool enable) 1202 { 1203 struct e1000_hw *hw = 1204 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1205 uint32_t tctl, rctl; 1206 1207 tctl = E1000_READ_REG(hw, E1000_TCTL); 1208 rctl = E1000_READ_REG(hw, E1000_RCTL); 1209 1210 if (enable) { 1211 /* enable Tx/Rx */ 1212 tctl |= E1000_TCTL_EN; 1213 rctl |= E1000_RCTL_EN; 1214 } else { 1215 /* disable Tx/Rx */ 1216 tctl &= ~E1000_TCTL_EN; 1217 rctl &= ~E1000_RCTL_EN; 1218 } 1219 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1220 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1221 E1000_WRITE_FLUSH(hw); 1222 } 1223 1224 static int 1225 eth_igb_start(struct rte_eth_dev *dev) 1226 { 1227 struct e1000_hw *hw = 1228 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1229 struct e1000_adapter *adapter = 1230 E1000_DEV_PRIVATE(dev->data->dev_private); 1231 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1232 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1233 int ret, mask; 1234 uint32_t intr_vector = 0; 1235 uint32_t ctrl_ext; 1236 uint32_t *speeds; 1237 int num_speeds; 1238 bool autoneg; 1239 1240 PMD_INIT_FUNC_TRACE(); 1241 1242 /* disable uio/vfio intr/eventfd mapping */ 1243 rte_intr_disable(intr_handle); 1244 1245 /* Power up the phy. Needed to make the link go Up */ 1246 eth_igb_dev_set_link_up(dev); 1247 1248 /* 1249 * Packet Buffer Allocation (PBA) 1250 * Writing PBA sets the receive portion of the buffer 1251 * the remainder is used for the transmit buffer. 1252 */ 1253 if (hw->mac.type == e1000_82575) { 1254 uint32_t pba; 1255 1256 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1257 E1000_WRITE_REG(hw, E1000_PBA, pba); 1258 } 1259 1260 /* Put the address into the Receive Address Array */ 1261 e1000_rar_set(hw, hw->mac.addr, 0); 1262 1263 /* Initialize the hardware */ 1264 if (igb_hardware_init(hw)) { 1265 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1266 return -EIO; 1267 } 1268 adapter->stopped = 0; 1269 1270 E1000_WRITE_REG(hw, E1000_VET, 1271 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1272 1273 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1274 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1275 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1276 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1277 E1000_WRITE_FLUSH(hw); 1278 1279 /* configure PF module if SRIOV enabled */ 1280 igb_pf_host_configure(dev); 1281 1282 /* check and configure queue intr-vector mapping */ 1283 if ((rte_intr_cap_multiple(intr_handle) || 1284 !RTE_ETH_DEV_SRIOV(dev).active) && 1285 dev->data->dev_conf.intr_conf.rxq != 0) { 1286 intr_vector = dev->data->nb_rx_queues; 1287 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1288 return -1; 1289 } 1290 1291 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1292 intr_handle->intr_vec = 1293 rte_zmalloc("intr_vec", 1294 dev->data->nb_rx_queues * sizeof(int), 0); 1295 if (intr_handle->intr_vec == NULL) { 1296 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1297 " intr_vec", dev->data->nb_rx_queues); 1298 return -ENOMEM; 1299 } 1300 } 1301 1302 /* confiugre msix for rx interrupt */ 1303 eth_igb_configure_msix_intr(dev); 1304 1305 /* Configure for OS presence */ 1306 igb_init_manageability(hw); 1307 1308 eth_igb_tx_init(dev); 1309 1310 /* This can fail when allocating mbufs for descriptor rings */ 1311 ret = eth_igb_rx_init(dev); 1312 if (ret) { 1313 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1314 igb_dev_clear_queues(dev); 1315 return ret; 1316 } 1317 1318 e1000_clear_hw_cntrs_base_generic(hw); 1319 1320 /* 1321 * VLAN Offload Settings 1322 */ 1323 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1324 ETH_VLAN_EXTEND_MASK; 1325 ret = eth_igb_vlan_offload_set(dev, mask); 1326 if (ret) { 1327 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1328 igb_dev_clear_queues(dev); 1329 return ret; 1330 } 1331 1332 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1333 /* Enable VLAN filter since VMDq always use VLAN filter */ 1334 igb_vmdq_vlan_hw_filter_enable(dev); 1335 } 1336 1337 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1338 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1339 (hw->mac.type == e1000_i211)) { 1340 /* Configure EITR with the maximum possible value (0xFFFF) */ 1341 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1342 } 1343 1344 /* Setup link speed and duplex */ 1345 speeds = &dev->data->dev_conf.link_speeds; 1346 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1347 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1348 hw->mac.autoneg = 1; 1349 } else { 1350 num_speeds = 0; 1351 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1352 1353 /* Reset */ 1354 hw->phy.autoneg_advertised = 0; 1355 1356 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1357 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1358 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1359 num_speeds = -1; 1360 goto error_invalid_config; 1361 } 1362 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1363 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1364 num_speeds++; 1365 } 1366 if (*speeds & ETH_LINK_SPEED_10M) { 1367 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1368 num_speeds++; 1369 } 1370 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1371 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1372 num_speeds++; 1373 } 1374 if (*speeds & ETH_LINK_SPEED_100M) { 1375 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1376 num_speeds++; 1377 } 1378 if (*speeds & ETH_LINK_SPEED_1G) { 1379 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1380 num_speeds++; 1381 } 1382 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1383 goto error_invalid_config; 1384 1385 /* Set/reset the mac.autoneg based on the link speed, 1386 * fixed or not 1387 */ 1388 if (!autoneg) { 1389 hw->mac.autoneg = 0; 1390 hw->mac.forced_speed_duplex = 1391 hw->phy.autoneg_advertised; 1392 } else { 1393 hw->mac.autoneg = 1; 1394 } 1395 } 1396 1397 e1000_setup_link(hw); 1398 1399 if (rte_intr_allow_others(intr_handle)) { 1400 /* check if lsc interrupt is enabled */ 1401 if (dev->data->dev_conf.intr_conf.lsc != 0) 1402 eth_igb_lsc_interrupt_setup(dev, TRUE); 1403 else 1404 eth_igb_lsc_interrupt_setup(dev, FALSE); 1405 } else { 1406 rte_intr_callback_unregister(intr_handle, 1407 eth_igb_interrupt_handler, 1408 (void *)dev); 1409 if (dev->data->dev_conf.intr_conf.lsc != 0) 1410 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1411 " no intr multiplex"); 1412 } 1413 1414 /* check if rxq interrupt is enabled */ 1415 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1416 rte_intr_dp_is_en(intr_handle)) 1417 eth_igb_rxq_interrupt_setup(dev); 1418 1419 /* enable uio/vfio intr/eventfd mapping */ 1420 rte_intr_enable(intr_handle); 1421 1422 /* resume enabled intr since hw reset */ 1423 igb_intr_enable(dev); 1424 1425 /* restore all types filter */ 1426 igb_filter_restore(dev); 1427 1428 eth_igb_rxtx_control(dev, true); 1429 eth_igb_link_update(dev, 0); 1430 1431 PMD_INIT_LOG(DEBUG, "<<"); 1432 1433 return 0; 1434 1435 error_invalid_config: 1436 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1437 dev->data->dev_conf.link_speeds, dev->data->port_id); 1438 igb_dev_clear_queues(dev); 1439 return -EINVAL; 1440 } 1441 1442 /********************************************************************* 1443 * 1444 * This routine disables all traffic on the adapter by issuing a 1445 * global reset on the MAC. 1446 * 1447 **********************************************************************/ 1448 static void 1449 eth_igb_stop(struct rte_eth_dev *dev) 1450 { 1451 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1452 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1453 struct rte_eth_link link; 1454 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1455 struct e1000_adapter *adapter = 1456 E1000_DEV_PRIVATE(dev->data->dev_private); 1457 1458 if (adapter->stopped) 1459 return; 1460 1461 eth_igb_rxtx_control(dev, false); 1462 1463 igb_intr_disable(dev); 1464 1465 /* disable intr eventfd mapping */ 1466 rte_intr_disable(intr_handle); 1467 1468 igb_pf_reset_hw(hw); 1469 E1000_WRITE_REG(hw, E1000_WUC, 0); 1470 1471 /* Set bit for Go Link disconnect */ 1472 if (hw->mac.type >= e1000_82580) { 1473 uint32_t phpm_reg; 1474 1475 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1476 phpm_reg |= E1000_82580_PM_GO_LINKD; 1477 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1478 } 1479 1480 /* Power down the phy. Needed to make the link go Down */ 1481 eth_igb_dev_set_link_down(dev); 1482 1483 igb_dev_clear_queues(dev); 1484 1485 /* clear the recorded link status */ 1486 memset(&link, 0, sizeof(link)); 1487 rte_eth_linkstatus_set(dev, &link); 1488 1489 if (!rte_intr_allow_others(intr_handle)) 1490 /* resume to the default handler */ 1491 rte_intr_callback_register(intr_handle, 1492 eth_igb_interrupt_handler, 1493 (void *)dev); 1494 1495 /* Clean datapath event and queue/vec mapping */ 1496 rte_intr_efd_disable(intr_handle); 1497 if (intr_handle->intr_vec != NULL) { 1498 rte_free(intr_handle->intr_vec); 1499 intr_handle->intr_vec = NULL; 1500 } 1501 1502 adapter->stopped = true; 1503 } 1504 1505 static int 1506 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1507 { 1508 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1509 1510 if (hw->phy.media_type == e1000_media_type_copper) 1511 e1000_power_up_phy(hw); 1512 else 1513 e1000_power_up_fiber_serdes_link(hw); 1514 1515 return 0; 1516 } 1517 1518 static int 1519 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1520 { 1521 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1522 1523 if (hw->phy.media_type == e1000_media_type_copper) 1524 e1000_power_down_phy(hw); 1525 else 1526 e1000_shutdown_fiber_serdes_link(hw); 1527 1528 return 0; 1529 } 1530 1531 static void 1532 eth_igb_close(struct rte_eth_dev *dev) 1533 { 1534 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1535 struct rte_eth_link link; 1536 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1537 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1538 struct e1000_filter_info *filter_info = 1539 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 1540 1541 eth_igb_stop(dev); 1542 1543 e1000_phy_hw_reset(hw); 1544 igb_release_manageability(hw); 1545 igb_hw_control_release(hw); 1546 1547 /* Clear bit for Go Link disconnect */ 1548 if (hw->mac.type >= e1000_82580) { 1549 uint32_t phpm_reg; 1550 1551 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1552 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1553 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1554 } 1555 1556 igb_dev_free_queues(dev); 1557 1558 if (intr_handle->intr_vec) { 1559 rte_free(intr_handle->intr_vec); 1560 intr_handle->intr_vec = NULL; 1561 } 1562 1563 memset(&link, 0, sizeof(link)); 1564 rte_eth_linkstatus_set(dev, &link); 1565 1566 dev->dev_ops = NULL; 1567 dev->rx_pkt_burst = NULL; 1568 dev->tx_pkt_burst = NULL; 1569 1570 /* Reset any pending lock */ 1571 igb_reset_swfw_lock(hw); 1572 1573 /* uninitialize PF if max_vfs not zero */ 1574 igb_pf_host_uninit(dev); 1575 1576 rte_intr_callback_unregister(intr_handle, 1577 eth_igb_interrupt_handler, dev); 1578 1579 /* clear the SYN filter info */ 1580 filter_info->syn_info = 0; 1581 1582 /* clear the ethertype filters info */ 1583 filter_info->ethertype_mask = 0; 1584 memset(filter_info->ethertype_filters, 0, 1585 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 1586 1587 /* clear the rss filter info */ 1588 memset(&filter_info->rss_info, 0, 1589 sizeof(struct igb_rte_flow_rss_conf)); 1590 1591 /* remove all ntuple filters of the device */ 1592 igb_ntuple_filter_uninit(dev); 1593 1594 /* remove all flex filters of the device */ 1595 igb_flex_filter_uninit(dev); 1596 1597 /* clear all the filters list */ 1598 igb_filterlist_flush(dev); 1599 } 1600 1601 /* 1602 * Reset PF device. 1603 */ 1604 static int 1605 eth_igb_reset(struct rte_eth_dev *dev) 1606 { 1607 int ret; 1608 1609 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1610 * its VF to make them align with it. The detailed notification 1611 * mechanism is PMD specific and is currently not implemented. 1612 * To avoid unexpected behavior in VF, currently reset of PF with 1613 * SR-IOV activation is not supported. It might be supported later. 1614 */ 1615 if (dev->data->sriov.active) 1616 return -ENOTSUP; 1617 1618 ret = eth_igb_dev_uninit(dev); 1619 if (ret) 1620 return ret; 1621 1622 ret = eth_igb_dev_init(dev); 1623 1624 return ret; 1625 } 1626 1627 1628 static int 1629 igb_get_rx_buffer_size(struct e1000_hw *hw) 1630 { 1631 uint32_t rx_buf_size; 1632 if (hw->mac.type == e1000_82576) { 1633 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1634 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1635 /* PBS needs to be translated according to a lookup table */ 1636 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1637 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1638 rx_buf_size = (rx_buf_size << 10); 1639 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1640 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1641 } else { 1642 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1643 } 1644 1645 return rx_buf_size; 1646 } 1647 1648 /********************************************************************* 1649 * 1650 * Initialize the hardware 1651 * 1652 **********************************************************************/ 1653 static int 1654 igb_hardware_init(struct e1000_hw *hw) 1655 { 1656 uint32_t rx_buf_size; 1657 int diag; 1658 1659 /* Let the firmware know the OS is in control */ 1660 igb_hw_control_acquire(hw); 1661 1662 /* 1663 * These parameters control the automatic generation (Tx) and 1664 * response (Rx) to Ethernet PAUSE frames. 1665 * - High water mark should allow for at least two standard size (1518) 1666 * frames to be received after sending an XOFF. 1667 * - Low water mark works best when it is very near the high water mark. 1668 * This allows the receiver to restart by sending XON when it has 1669 * drained a bit. Here we use an arbitrary value of 1500 which will 1670 * restart after one full frame is pulled from the buffer. There 1671 * could be several smaller frames in the buffer and if so they will 1672 * not trigger the XON until their total number reduces the buffer 1673 * by 1500. 1674 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1675 */ 1676 rx_buf_size = igb_get_rx_buffer_size(hw); 1677 1678 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2); 1679 hw->fc.low_water = hw->fc.high_water - 1500; 1680 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1681 hw->fc.send_xon = 1; 1682 1683 /* Set Flow control, use the tunable location if sane */ 1684 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1685 hw->fc.requested_mode = igb_fc_setting; 1686 else 1687 hw->fc.requested_mode = e1000_fc_none; 1688 1689 /* Issue a global reset */ 1690 igb_pf_reset_hw(hw); 1691 E1000_WRITE_REG(hw, E1000_WUC, 0); 1692 1693 diag = e1000_init_hw(hw); 1694 if (diag < 0) 1695 return diag; 1696 1697 E1000_WRITE_REG(hw, E1000_VET, 1698 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1699 e1000_get_phy_info(hw); 1700 e1000_check_for_link(hw); 1701 1702 return 0; 1703 } 1704 1705 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1706 static void 1707 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1708 { 1709 int pause_frames; 1710 1711 uint64_t old_gprc = stats->gprc; 1712 uint64_t old_gptc = stats->gptc; 1713 uint64_t old_tpr = stats->tpr; 1714 uint64_t old_tpt = stats->tpt; 1715 uint64_t old_rpthc = stats->rpthc; 1716 uint64_t old_hgptc = stats->hgptc; 1717 1718 if(hw->phy.media_type == e1000_media_type_copper || 1719 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1720 stats->symerrs += 1721 E1000_READ_REG(hw,E1000_SYMERRS); 1722 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1723 } 1724 1725 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1726 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1727 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1728 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1729 1730 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1731 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1732 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1733 stats->dc += E1000_READ_REG(hw, E1000_DC); 1734 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1735 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1736 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1737 /* 1738 ** For watchdog management we need to know if we have been 1739 ** paused during the last interval, so capture that here. 1740 */ 1741 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1742 stats->xoffrxc += pause_frames; 1743 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1744 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1745 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1746 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1747 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1748 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1749 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1750 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1751 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1752 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1753 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1754 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1755 1756 /* For the 64-bit byte counters the low dword must be read first. */ 1757 /* Both registers clear on the read of the high dword */ 1758 1759 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1760 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1761 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1762 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN; 1763 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1764 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1765 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN; 1766 1767 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1768 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1769 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1770 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1771 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1772 1773 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1774 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1775 1776 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1777 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1778 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 1779 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1780 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1781 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN; 1782 1783 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1784 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1785 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1786 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1787 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1788 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1789 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1790 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1791 1792 /* Interrupt Counts */ 1793 1794 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1795 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1796 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1797 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1798 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1799 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1800 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1801 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1802 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1803 1804 /* Host to Card Statistics */ 1805 1806 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1807 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1808 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1809 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1810 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1811 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1812 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1813 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1814 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1815 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN; 1816 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1817 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1818 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN; 1819 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1820 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1821 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1822 1823 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1824 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1825 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1826 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1827 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1828 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1829 } 1830 1831 static int 1832 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1833 { 1834 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1835 struct e1000_hw_stats *stats = 1836 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1837 1838 igb_read_stats_registers(hw, stats); 1839 1840 if (rte_stats == NULL) 1841 return -EINVAL; 1842 1843 /* Rx Errors */ 1844 rte_stats->imissed = stats->mpc; 1845 rte_stats->ierrors = stats->crcerrs + 1846 stats->rlec + stats->ruc + stats->roc + 1847 stats->rxerrc + stats->algnerrc + stats->cexterr; 1848 1849 /* Tx Errors */ 1850 rte_stats->oerrors = stats->ecol + stats->latecol; 1851 1852 rte_stats->ipackets = stats->gprc; 1853 rte_stats->opackets = stats->gptc; 1854 rte_stats->ibytes = stats->gorc; 1855 rte_stats->obytes = stats->gotc; 1856 return 0; 1857 } 1858 1859 static int 1860 eth_igb_stats_reset(struct rte_eth_dev *dev) 1861 { 1862 struct e1000_hw_stats *hw_stats = 1863 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1864 1865 /* HW registers are cleared on read */ 1866 eth_igb_stats_get(dev, NULL); 1867 1868 /* Reset software totals */ 1869 memset(hw_stats, 0, sizeof(*hw_stats)); 1870 1871 return 0; 1872 } 1873 1874 static int 1875 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1876 { 1877 struct e1000_hw_stats *stats = 1878 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1879 1880 /* HW registers are cleared on read */ 1881 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1882 1883 /* Reset software totals */ 1884 memset(stats, 0, sizeof(*stats)); 1885 1886 return 0; 1887 } 1888 1889 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1890 struct rte_eth_xstat_name *xstats_names, 1891 __rte_unused unsigned int size) 1892 { 1893 unsigned i; 1894 1895 if (xstats_names == NULL) 1896 return IGB_NB_XSTATS; 1897 1898 /* Note: limit checked in rte_eth_xstats_names() */ 1899 1900 for (i = 0; i < IGB_NB_XSTATS; i++) { 1901 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1902 sizeof(xstats_names[i].name)); 1903 } 1904 1905 return IGB_NB_XSTATS; 1906 } 1907 1908 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1909 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1910 unsigned int limit) 1911 { 1912 unsigned int i; 1913 1914 if (!ids) { 1915 if (xstats_names == NULL) 1916 return IGB_NB_XSTATS; 1917 1918 for (i = 0; i < IGB_NB_XSTATS; i++) 1919 strlcpy(xstats_names[i].name, 1920 rte_igb_stats_strings[i].name, 1921 sizeof(xstats_names[i].name)); 1922 1923 return IGB_NB_XSTATS; 1924 1925 } else { 1926 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1927 1928 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1929 IGB_NB_XSTATS); 1930 1931 for (i = 0; i < limit; i++) { 1932 if (ids[i] >= IGB_NB_XSTATS) { 1933 PMD_INIT_LOG(ERR, "id value isn't valid"); 1934 return -1; 1935 } 1936 strcpy(xstats_names[i].name, 1937 xstats_names_copy[ids[i]].name); 1938 } 1939 return limit; 1940 } 1941 } 1942 1943 static int 1944 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1945 unsigned n) 1946 { 1947 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1948 struct e1000_hw_stats *hw_stats = 1949 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1950 unsigned i; 1951 1952 if (n < IGB_NB_XSTATS) 1953 return IGB_NB_XSTATS; 1954 1955 igb_read_stats_registers(hw, hw_stats); 1956 1957 /* If this is a reset xstats is NULL, and we have cleared the 1958 * registers by reading them. 1959 */ 1960 if (!xstats) 1961 return 0; 1962 1963 /* Extended stats */ 1964 for (i = 0; i < IGB_NB_XSTATS; i++) { 1965 xstats[i].id = i; 1966 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1967 rte_igb_stats_strings[i].offset); 1968 } 1969 1970 return IGB_NB_XSTATS; 1971 } 1972 1973 static int 1974 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1975 uint64_t *values, unsigned int n) 1976 { 1977 unsigned int i; 1978 1979 if (!ids) { 1980 struct e1000_hw *hw = 1981 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1982 struct e1000_hw_stats *hw_stats = 1983 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1984 1985 if (n < IGB_NB_XSTATS) 1986 return IGB_NB_XSTATS; 1987 1988 igb_read_stats_registers(hw, hw_stats); 1989 1990 /* If this is a reset xstats is NULL, and we have cleared the 1991 * registers by reading them. 1992 */ 1993 if (!values) 1994 return 0; 1995 1996 /* Extended stats */ 1997 for (i = 0; i < IGB_NB_XSTATS; i++) 1998 values[i] = *(uint64_t *)(((char *)hw_stats) + 1999 rte_igb_stats_strings[i].offset); 2000 2001 return IGB_NB_XSTATS; 2002 2003 } else { 2004 uint64_t values_copy[IGB_NB_XSTATS]; 2005 2006 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2007 IGB_NB_XSTATS); 2008 2009 for (i = 0; i < n; i++) { 2010 if (ids[i] >= IGB_NB_XSTATS) { 2011 PMD_INIT_LOG(ERR, "id value isn't valid"); 2012 return -1; 2013 } 2014 values[i] = values_copy[ids[i]]; 2015 } 2016 return n; 2017 } 2018 } 2019 2020 static void 2021 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2022 { 2023 /* Good Rx packets, include VF loopback */ 2024 UPDATE_VF_STAT(E1000_VFGPRC, 2025 hw_stats->last_gprc, hw_stats->gprc); 2026 2027 /* Good Rx octets, include VF loopback */ 2028 UPDATE_VF_STAT(E1000_VFGORC, 2029 hw_stats->last_gorc, hw_stats->gorc); 2030 2031 /* Good Tx packets, include VF loopback */ 2032 UPDATE_VF_STAT(E1000_VFGPTC, 2033 hw_stats->last_gptc, hw_stats->gptc); 2034 2035 /* Good Tx octets, include VF loopback */ 2036 UPDATE_VF_STAT(E1000_VFGOTC, 2037 hw_stats->last_gotc, hw_stats->gotc); 2038 2039 /* Rx Multicst packets */ 2040 UPDATE_VF_STAT(E1000_VFMPRC, 2041 hw_stats->last_mprc, hw_stats->mprc); 2042 2043 /* Good Rx loopback packets */ 2044 UPDATE_VF_STAT(E1000_VFGPRLBC, 2045 hw_stats->last_gprlbc, hw_stats->gprlbc); 2046 2047 /* Good Rx loopback octets */ 2048 UPDATE_VF_STAT(E1000_VFGORLBC, 2049 hw_stats->last_gorlbc, hw_stats->gorlbc); 2050 2051 /* Good Tx loopback packets */ 2052 UPDATE_VF_STAT(E1000_VFGPTLBC, 2053 hw_stats->last_gptlbc, hw_stats->gptlbc); 2054 2055 /* Good Tx loopback octets */ 2056 UPDATE_VF_STAT(E1000_VFGOTLBC, 2057 hw_stats->last_gotlbc, hw_stats->gotlbc); 2058 } 2059 2060 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2061 struct rte_eth_xstat_name *xstats_names, 2062 __rte_unused unsigned limit) 2063 { 2064 unsigned i; 2065 2066 if (xstats_names != NULL) 2067 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2068 strlcpy(xstats_names[i].name, 2069 rte_igbvf_stats_strings[i].name, 2070 sizeof(xstats_names[i].name)); 2071 } 2072 return IGBVF_NB_XSTATS; 2073 } 2074 2075 static int 2076 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2077 unsigned n) 2078 { 2079 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2080 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2081 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2082 unsigned i; 2083 2084 if (n < IGBVF_NB_XSTATS) 2085 return IGBVF_NB_XSTATS; 2086 2087 igbvf_read_stats_registers(hw, hw_stats); 2088 2089 if (!xstats) 2090 return 0; 2091 2092 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2093 xstats[i].id = i; 2094 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2095 rte_igbvf_stats_strings[i].offset); 2096 } 2097 2098 return IGBVF_NB_XSTATS; 2099 } 2100 2101 static int 2102 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2103 { 2104 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2105 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2106 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2107 2108 igbvf_read_stats_registers(hw, hw_stats); 2109 2110 if (rte_stats == NULL) 2111 return -EINVAL; 2112 2113 rte_stats->ipackets = hw_stats->gprc; 2114 rte_stats->ibytes = hw_stats->gorc; 2115 rte_stats->opackets = hw_stats->gptc; 2116 rte_stats->obytes = hw_stats->gotc; 2117 return 0; 2118 } 2119 2120 static int 2121 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2122 { 2123 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2124 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2125 2126 /* Sync HW register to the last stats */ 2127 eth_igbvf_stats_get(dev, NULL); 2128 2129 /* reset HW current stats*/ 2130 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2131 offsetof(struct e1000_vf_stats, gprc)); 2132 2133 return 0; 2134 } 2135 2136 static int 2137 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2138 size_t fw_size) 2139 { 2140 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2141 struct e1000_fw_version fw; 2142 int ret; 2143 2144 e1000_get_fw_version(hw, &fw); 2145 2146 switch (hw->mac.type) { 2147 case e1000_i210: 2148 case e1000_i211: 2149 if (!(e1000_get_flash_presence_i210(hw))) { 2150 ret = snprintf(fw_version, fw_size, 2151 "%2d.%2d-%d", 2152 fw.invm_major, fw.invm_minor, 2153 fw.invm_img_type); 2154 break; 2155 } 2156 /* fall through */ 2157 default: 2158 /* if option rom is valid, display its version too */ 2159 if (fw.or_valid) { 2160 ret = snprintf(fw_version, fw_size, 2161 "%d.%d, 0x%08x, %d.%d.%d", 2162 fw.eep_major, fw.eep_minor, fw.etrack_id, 2163 fw.or_major, fw.or_build, fw.or_patch); 2164 /* no option rom */ 2165 } else { 2166 if (fw.etrack_id != 0X0000) { 2167 ret = snprintf(fw_version, fw_size, 2168 "%d.%d, 0x%08x", 2169 fw.eep_major, fw.eep_minor, 2170 fw.etrack_id); 2171 } else { 2172 ret = snprintf(fw_version, fw_size, 2173 "%d.%d.%d", 2174 fw.eep_major, fw.eep_minor, 2175 fw.eep_build); 2176 } 2177 } 2178 break; 2179 } 2180 2181 ret += 1; /* add the size of '\0' */ 2182 if (fw_size < (u32)ret) 2183 return ret; 2184 else 2185 return 0; 2186 } 2187 2188 static int 2189 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2190 { 2191 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2192 2193 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2194 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2195 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2196 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2197 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2198 dev_info->rx_queue_offload_capa; 2199 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2200 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2201 dev_info->tx_queue_offload_capa; 2202 2203 switch (hw->mac.type) { 2204 case e1000_82575: 2205 dev_info->max_rx_queues = 4; 2206 dev_info->max_tx_queues = 4; 2207 dev_info->max_vmdq_pools = 0; 2208 break; 2209 2210 case e1000_82576: 2211 dev_info->max_rx_queues = 16; 2212 dev_info->max_tx_queues = 16; 2213 dev_info->max_vmdq_pools = ETH_8_POOLS; 2214 dev_info->vmdq_queue_num = 16; 2215 break; 2216 2217 case e1000_82580: 2218 dev_info->max_rx_queues = 8; 2219 dev_info->max_tx_queues = 8; 2220 dev_info->max_vmdq_pools = ETH_8_POOLS; 2221 dev_info->vmdq_queue_num = 8; 2222 break; 2223 2224 case e1000_i350: 2225 dev_info->max_rx_queues = 8; 2226 dev_info->max_tx_queues = 8; 2227 dev_info->max_vmdq_pools = ETH_8_POOLS; 2228 dev_info->vmdq_queue_num = 8; 2229 break; 2230 2231 case e1000_i354: 2232 dev_info->max_rx_queues = 8; 2233 dev_info->max_tx_queues = 8; 2234 break; 2235 2236 case e1000_i210: 2237 dev_info->max_rx_queues = 4; 2238 dev_info->max_tx_queues = 4; 2239 dev_info->max_vmdq_pools = 0; 2240 break; 2241 2242 case e1000_i211: 2243 dev_info->max_rx_queues = 2; 2244 dev_info->max_tx_queues = 2; 2245 dev_info->max_vmdq_pools = 0; 2246 break; 2247 2248 default: 2249 /* Should not happen */ 2250 return -EINVAL; 2251 } 2252 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2253 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2254 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2255 2256 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2257 .rx_thresh = { 2258 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2259 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2260 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2261 }, 2262 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2263 .rx_drop_en = 0, 2264 .offloads = 0, 2265 }; 2266 2267 dev_info->default_txconf = (struct rte_eth_txconf) { 2268 .tx_thresh = { 2269 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2270 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2271 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2272 }, 2273 .offloads = 0, 2274 }; 2275 2276 dev_info->rx_desc_lim = rx_desc_lim; 2277 dev_info->tx_desc_lim = tx_desc_lim; 2278 2279 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2280 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2281 ETH_LINK_SPEED_1G; 2282 2283 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2284 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 2285 2286 return 0; 2287 } 2288 2289 static const uint32_t * 2290 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2291 { 2292 static const uint32_t ptypes[] = { 2293 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2294 RTE_PTYPE_L2_ETHER, 2295 RTE_PTYPE_L3_IPV4, 2296 RTE_PTYPE_L3_IPV4_EXT, 2297 RTE_PTYPE_L3_IPV6, 2298 RTE_PTYPE_L3_IPV6_EXT, 2299 RTE_PTYPE_L4_TCP, 2300 RTE_PTYPE_L4_UDP, 2301 RTE_PTYPE_L4_SCTP, 2302 RTE_PTYPE_TUNNEL_IP, 2303 RTE_PTYPE_INNER_L3_IPV6, 2304 RTE_PTYPE_INNER_L3_IPV6_EXT, 2305 RTE_PTYPE_INNER_L4_TCP, 2306 RTE_PTYPE_INNER_L4_UDP, 2307 RTE_PTYPE_UNKNOWN 2308 }; 2309 2310 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2311 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2312 return ptypes; 2313 return NULL; 2314 } 2315 2316 static int 2317 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2318 { 2319 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2320 2321 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2322 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2323 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2324 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2325 DEV_TX_OFFLOAD_IPV4_CKSUM | 2326 DEV_TX_OFFLOAD_UDP_CKSUM | 2327 DEV_TX_OFFLOAD_TCP_CKSUM | 2328 DEV_TX_OFFLOAD_SCTP_CKSUM | 2329 DEV_TX_OFFLOAD_TCP_TSO; 2330 switch (hw->mac.type) { 2331 case e1000_vfadapt: 2332 dev_info->max_rx_queues = 2; 2333 dev_info->max_tx_queues = 2; 2334 break; 2335 case e1000_vfadapt_i350: 2336 dev_info->max_rx_queues = 1; 2337 dev_info->max_tx_queues = 1; 2338 break; 2339 default: 2340 /* Should not happen */ 2341 return -EINVAL; 2342 } 2343 2344 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2345 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2346 dev_info->rx_queue_offload_capa; 2347 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2348 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2349 dev_info->tx_queue_offload_capa; 2350 2351 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2352 .rx_thresh = { 2353 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2354 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2355 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2356 }, 2357 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2358 .rx_drop_en = 0, 2359 .offloads = 0, 2360 }; 2361 2362 dev_info->default_txconf = (struct rte_eth_txconf) { 2363 .tx_thresh = { 2364 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2365 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2366 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2367 }, 2368 .offloads = 0, 2369 }; 2370 2371 dev_info->rx_desc_lim = rx_desc_lim; 2372 dev_info->tx_desc_lim = tx_desc_lim; 2373 2374 return 0; 2375 } 2376 2377 /* return 0 means link status changed, -1 means not changed */ 2378 static int 2379 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2380 { 2381 struct e1000_hw *hw = 2382 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2383 struct rte_eth_link link; 2384 int link_check, count; 2385 2386 link_check = 0; 2387 hw->mac.get_link_status = 1; 2388 2389 /* possible wait-to-complete in up to 9 seconds */ 2390 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2391 /* Read the real link status */ 2392 switch (hw->phy.media_type) { 2393 case e1000_media_type_copper: 2394 /* Do the work to read phy */ 2395 e1000_check_for_link(hw); 2396 link_check = !hw->mac.get_link_status; 2397 break; 2398 2399 case e1000_media_type_fiber: 2400 e1000_check_for_link(hw); 2401 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2402 E1000_STATUS_LU); 2403 break; 2404 2405 case e1000_media_type_internal_serdes: 2406 e1000_check_for_link(hw); 2407 link_check = hw->mac.serdes_has_link; 2408 break; 2409 2410 /* VF device is type_unknown */ 2411 case e1000_media_type_unknown: 2412 eth_igbvf_link_update(hw); 2413 link_check = !hw->mac.get_link_status; 2414 break; 2415 2416 default: 2417 break; 2418 } 2419 if (link_check || wait_to_complete == 0) 2420 break; 2421 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2422 } 2423 memset(&link, 0, sizeof(link)); 2424 2425 /* Now we check if a transition has happened */ 2426 if (link_check) { 2427 uint16_t duplex, speed; 2428 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2429 link.link_duplex = (duplex == FULL_DUPLEX) ? 2430 ETH_LINK_FULL_DUPLEX : 2431 ETH_LINK_HALF_DUPLEX; 2432 link.link_speed = speed; 2433 link.link_status = ETH_LINK_UP; 2434 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2435 ETH_LINK_SPEED_FIXED); 2436 } else if (!link_check) { 2437 link.link_speed = 0; 2438 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2439 link.link_status = ETH_LINK_DOWN; 2440 link.link_autoneg = ETH_LINK_FIXED; 2441 } 2442 2443 return rte_eth_linkstatus_set(dev, &link); 2444 } 2445 2446 /* 2447 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2448 * For ASF and Pass Through versions of f/w this means 2449 * that the driver is loaded. 2450 */ 2451 static void 2452 igb_hw_control_acquire(struct e1000_hw *hw) 2453 { 2454 uint32_t ctrl_ext; 2455 2456 /* Let firmware know the driver has taken over */ 2457 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2458 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2459 } 2460 2461 /* 2462 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2463 * For ASF and Pass Through versions of f/w this means that the 2464 * driver is no longer loaded. 2465 */ 2466 static void 2467 igb_hw_control_release(struct e1000_hw *hw) 2468 { 2469 uint32_t ctrl_ext; 2470 2471 /* Let firmware taken over control of h/w */ 2472 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2473 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2474 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2475 } 2476 2477 /* 2478 * Bit of a misnomer, what this really means is 2479 * to enable OS management of the system... aka 2480 * to disable special hardware management features. 2481 */ 2482 static void 2483 igb_init_manageability(struct e1000_hw *hw) 2484 { 2485 if (e1000_enable_mng_pass_thru(hw)) { 2486 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2487 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2488 2489 /* disable hardware interception of ARP */ 2490 manc &= ~(E1000_MANC_ARP_EN); 2491 2492 /* enable receiving management packets to the host */ 2493 manc |= E1000_MANC_EN_MNG2HOST; 2494 manc2h |= 1 << 5; /* Mng Port 623 */ 2495 manc2h |= 1 << 6; /* Mng Port 664 */ 2496 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2497 E1000_WRITE_REG(hw, E1000_MANC, manc); 2498 } 2499 } 2500 2501 static void 2502 igb_release_manageability(struct e1000_hw *hw) 2503 { 2504 if (e1000_enable_mng_pass_thru(hw)) { 2505 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2506 2507 manc |= E1000_MANC_ARP_EN; 2508 manc &= ~E1000_MANC_EN_MNG2HOST; 2509 2510 E1000_WRITE_REG(hw, E1000_MANC, manc); 2511 } 2512 } 2513 2514 static int 2515 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2516 { 2517 struct e1000_hw *hw = 2518 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2519 uint32_t rctl; 2520 2521 rctl = E1000_READ_REG(hw, E1000_RCTL); 2522 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2523 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2524 2525 return 0; 2526 } 2527 2528 static int 2529 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2530 { 2531 struct e1000_hw *hw = 2532 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2533 uint32_t rctl; 2534 2535 rctl = E1000_READ_REG(hw, E1000_RCTL); 2536 rctl &= (~E1000_RCTL_UPE); 2537 if (dev->data->all_multicast == 1) 2538 rctl |= E1000_RCTL_MPE; 2539 else 2540 rctl &= (~E1000_RCTL_MPE); 2541 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2542 2543 return 0; 2544 } 2545 2546 static int 2547 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2548 { 2549 struct e1000_hw *hw = 2550 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2551 uint32_t rctl; 2552 2553 rctl = E1000_READ_REG(hw, E1000_RCTL); 2554 rctl |= E1000_RCTL_MPE; 2555 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2556 2557 return 0; 2558 } 2559 2560 static int 2561 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2562 { 2563 struct e1000_hw *hw = 2564 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2565 uint32_t rctl; 2566 2567 if (dev->data->promiscuous == 1) 2568 return 0; /* must remain in all_multicast mode */ 2569 rctl = E1000_READ_REG(hw, E1000_RCTL); 2570 rctl &= (~E1000_RCTL_MPE); 2571 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2572 2573 return 0; 2574 } 2575 2576 static int 2577 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2578 { 2579 struct e1000_hw *hw = 2580 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2581 struct e1000_vfta * shadow_vfta = 2582 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2583 uint32_t vfta; 2584 uint32_t vid_idx; 2585 uint32_t vid_bit; 2586 2587 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2588 E1000_VFTA_ENTRY_MASK); 2589 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2590 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2591 if (on) 2592 vfta |= vid_bit; 2593 else 2594 vfta &= ~vid_bit; 2595 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2596 2597 /* update local VFTA copy */ 2598 shadow_vfta->vfta[vid_idx] = vfta; 2599 2600 return 0; 2601 } 2602 2603 static int 2604 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2605 enum rte_vlan_type vlan_type, 2606 uint16_t tpid) 2607 { 2608 struct e1000_hw *hw = 2609 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2610 uint32_t reg, qinq; 2611 2612 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2613 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2614 2615 /* only outer TPID of double VLAN can be configured*/ 2616 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2617 reg = E1000_READ_REG(hw, E1000_VET); 2618 reg = (reg & (~E1000_VET_VET_EXT)) | 2619 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2620 E1000_WRITE_REG(hw, E1000_VET, reg); 2621 2622 return 0; 2623 } 2624 2625 /* all other TPID values are read-only*/ 2626 PMD_DRV_LOG(ERR, "Not supported"); 2627 2628 return -ENOTSUP; 2629 } 2630 2631 static void 2632 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2633 { 2634 struct e1000_hw *hw = 2635 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2636 uint32_t reg; 2637 2638 /* Filter Table Disable */ 2639 reg = E1000_READ_REG(hw, E1000_RCTL); 2640 reg &= ~E1000_RCTL_CFIEN; 2641 reg &= ~E1000_RCTL_VFE; 2642 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2643 } 2644 2645 static void 2646 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2647 { 2648 struct e1000_hw *hw = 2649 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2650 struct e1000_vfta * shadow_vfta = 2651 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2652 uint32_t reg; 2653 int i; 2654 2655 /* Filter Table Enable, CFI not used for packet acceptance */ 2656 reg = E1000_READ_REG(hw, E1000_RCTL); 2657 reg &= ~E1000_RCTL_CFIEN; 2658 reg |= E1000_RCTL_VFE; 2659 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2660 2661 /* restore VFTA table */ 2662 for (i = 0; i < IGB_VFTA_SIZE; i++) 2663 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2664 } 2665 2666 static void 2667 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2668 { 2669 struct e1000_hw *hw = 2670 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2671 uint32_t reg; 2672 2673 /* VLAN Mode Disable */ 2674 reg = E1000_READ_REG(hw, E1000_CTRL); 2675 reg &= ~E1000_CTRL_VME; 2676 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2677 } 2678 2679 static void 2680 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2681 { 2682 struct e1000_hw *hw = 2683 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2684 uint32_t reg; 2685 2686 /* VLAN Mode Enable */ 2687 reg = E1000_READ_REG(hw, E1000_CTRL); 2688 reg |= E1000_CTRL_VME; 2689 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2690 } 2691 2692 static void 2693 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2694 { 2695 struct e1000_hw *hw = 2696 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2697 uint32_t reg; 2698 2699 /* CTRL_EXT: Extended VLAN */ 2700 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2701 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2702 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2703 2704 /* Update maximum packet length */ 2705 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2706 E1000_WRITE_REG(hw, E1000_RLPML, 2707 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2708 VLAN_TAG_SIZE); 2709 } 2710 2711 static void 2712 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2713 { 2714 struct e1000_hw *hw = 2715 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2716 uint32_t reg; 2717 2718 /* CTRL_EXT: Extended VLAN */ 2719 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2720 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2721 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2722 2723 /* Update maximum packet length */ 2724 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2725 E1000_WRITE_REG(hw, E1000_RLPML, 2726 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2727 2 * VLAN_TAG_SIZE); 2728 } 2729 2730 static int 2731 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2732 { 2733 struct rte_eth_rxmode *rxmode; 2734 2735 rxmode = &dev->data->dev_conf.rxmode; 2736 if(mask & ETH_VLAN_STRIP_MASK){ 2737 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2738 igb_vlan_hw_strip_enable(dev); 2739 else 2740 igb_vlan_hw_strip_disable(dev); 2741 } 2742 2743 if(mask & ETH_VLAN_FILTER_MASK){ 2744 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2745 igb_vlan_hw_filter_enable(dev); 2746 else 2747 igb_vlan_hw_filter_disable(dev); 2748 } 2749 2750 if(mask & ETH_VLAN_EXTEND_MASK){ 2751 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2752 igb_vlan_hw_extend_enable(dev); 2753 else 2754 igb_vlan_hw_extend_disable(dev); 2755 } 2756 2757 return 0; 2758 } 2759 2760 2761 /** 2762 * It enables the interrupt mask and then enable the interrupt. 2763 * 2764 * @param dev 2765 * Pointer to struct rte_eth_dev. 2766 * @param on 2767 * Enable or Disable 2768 * 2769 * @return 2770 * - On success, zero. 2771 * - On failure, a negative value. 2772 */ 2773 static int 2774 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2775 { 2776 struct e1000_interrupt *intr = 2777 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2778 2779 if (on) 2780 intr->mask |= E1000_ICR_LSC; 2781 else 2782 intr->mask &= ~E1000_ICR_LSC; 2783 2784 return 0; 2785 } 2786 2787 /* It clears the interrupt causes and enables the interrupt. 2788 * It will be called once only during nic initialized. 2789 * 2790 * @param dev 2791 * Pointer to struct rte_eth_dev. 2792 * 2793 * @return 2794 * - On success, zero. 2795 * - On failure, a negative value. 2796 */ 2797 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2798 { 2799 uint32_t mask, regval; 2800 int ret; 2801 struct e1000_hw *hw = 2802 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2803 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2804 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2805 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2806 struct rte_eth_dev_info dev_info; 2807 2808 memset(&dev_info, 0, sizeof(dev_info)); 2809 ret = eth_igb_infos_get(dev, &dev_info); 2810 if (ret != 0) 2811 return ret; 2812 2813 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2814 regval = E1000_READ_REG(hw, E1000_EIMS); 2815 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2816 2817 return 0; 2818 } 2819 2820 /* 2821 * It reads ICR and gets interrupt causes, check it and set a bit flag 2822 * to update link status. 2823 * 2824 * @param dev 2825 * Pointer to struct rte_eth_dev. 2826 * 2827 * @return 2828 * - On success, zero. 2829 * - On failure, a negative value. 2830 */ 2831 static int 2832 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2833 { 2834 uint32_t icr; 2835 struct e1000_hw *hw = 2836 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2837 struct e1000_interrupt *intr = 2838 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2839 2840 igb_intr_disable(dev); 2841 2842 /* read-on-clear nic registers here */ 2843 icr = E1000_READ_REG(hw, E1000_ICR); 2844 2845 intr->flags = 0; 2846 if (icr & E1000_ICR_LSC) { 2847 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2848 } 2849 2850 if (icr & E1000_ICR_VMMB) 2851 intr->flags |= E1000_FLAG_MAILBOX; 2852 2853 return 0; 2854 } 2855 2856 /* 2857 * It executes link_update after knowing an interrupt is prsent. 2858 * 2859 * @param dev 2860 * Pointer to struct rte_eth_dev. 2861 * 2862 * @return 2863 * - On success, zero. 2864 * - On failure, a negative value. 2865 */ 2866 static int 2867 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2868 struct rte_intr_handle *intr_handle) 2869 { 2870 struct e1000_hw *hw = 2871 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2872 struct e1000_interrupt *intr = 2873 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2874 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2875 struct rte_eth_link link; 2876 int ret; 2877 2878 if (intr->flags & E1000_FLAG_MAILBOX) { 2879 igb_pf_mbx_process(dev); 2880 intr->flags &= ~E1000_FLAG_MAILBOX; 2881 } 2882 2883 igb_intr_enable(dev); 2884 rte_intr_ack(intr_handle); 2885 2886 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2887 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2888 2889 /* set get_link_status to check register later */ 2890 hw->mac.get_link_status = 1; 2891 ret = eth_igb_link_update(dev, 0); 2892 2893 /* check if link has changed */ 2894 if (ret < 0) 2895 return 0; 2896 2897 rte_eth_linkstatus_get(dev, &link); 2898 if (link.link_status) { 2899 PMD_INIT_LOG(INFO, 2900 " Port %d: Link Up - speed %u Mbps - %s", 2901 dev->data->port_id, 2902 (unsigned)link.link_speed, 2903 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2904 "full-duplex" : "half-duplex"); 2905 } else { 2906 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2907 dev->data->port_id); 2908 } 2909 2910 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2911 pci_dev->addr.domain, 2912 pci_dev->addr.bus, 2913 pci_dev->addr.devid, 2914 pci_dev->addr.function); 2915 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2916 NULL); 2917 } 2918 2919 return 0; 2920 } 2921 2922 /** 2923 * Interrupt handler which shall be registered at first. 2924 * 2925 * @param handle 2926 * Pointer to interrupt handle. 2927 * @param param 2928 * The address of parameter (struct rte_eth_dev *) regsitered before. 2929 * 2930 * @return 2931 * void 2932 */ 2933 static void 2934 eth_igb_interrupt_handler(void *param) 2935 { 2936 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2937 2938 eth_igb_interrupt_get_status(dev); 2939 eth_igb_interrupt_action(dev, dev->intr_handle); 2940 } 2941 2942 static int 2943 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2944 { 2945 uint32_t eicr; 2946 struct e1000_hw *hw = 2947 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2948 struct e1000_interrupt *intr = 2949 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2950 2951 igbvf_intr_disable(hw); 2952 2953 /* read-on-clear nic registers here */ 2954 eicr = E1000_READ_REG(hw, E1000_EICR); 2955 intr->flags = 0; 2956 2957 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2958 intr->flags |= E1000_FLAG_MAILBOX; 2959 2960 return 0; 2961 } 2962 2963 void igbvf_mbx_process(struct rte_eth_dev *dev) 2964 { 2965 struct e1000_hw *hw = 2966 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2967 struct e1000_mbx_info *mbx = &hw->mbx; 2968 u32 in_msg = 0; 2969 2970 /* peek the message first */ 2971 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2972 2973 /* PF reset VF event */ 2974 if (in_msg == E1000_PF_CONTROL_MSG) { 2975 /* dummy mbx read to ack pf */ 2976 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2977 return; 2978 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2979 NULL); 2980 } 2981 } 2982 2983 static int 2984 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2985 { 2986 struct e1000_interrupt *intr = 2987 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2988 2989 if (intr->flags & E1000_FLAG_MAILBOX) { 2990 igbvf_mbx_process(dev); 2991 intr->flags &= ~E1000_FLAG_MAILBOX; 2992 } 2993 2994 igbvf_intr_enable(dev); 2995 rte_intr_ack(intr_handle); 2996 2997 return 0; 2998 } 2999 3000 static void 3001 eth_igbvf_interrupt_handler(void *param) 3002 { 3003 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 3004 3005 eth_igbvf_interrupt_get_status(dev); 3006 eth_igbvf_interrupt_action(dev, dev->intr_handle); 3007 } 3008 3009 static int 3010 eth_igb_led_on(struct rte_eth_dev *dev) 3011 { 3012 struct e1000_hw *hw; 3013 3014 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3015 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3016 } 3017 3018 static int 3019 eth_igb_led_off(struct rte_eth_dev *dev) 3020 { 3021 struct e1000_hw *hw; 3022 3023 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3024 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3025 } 3026 3027 static int 3028 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3029 { 3030 struct e1000_hw *hw; 3031 uint32_t ctrl; 3032 int tx_pause; 3033 int rx_pause; 3034 3035 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3036 fc_conf->pause_time = hw->fc.pause_time; 3037 fc_conf->high_water = hw->fc.high_water; 3038 fc_conf->low_water = hw->fc.low_water; 3039 fc_conf->send_xon = hw->fc.send_xon; 3040 fc_conf->autoneg = hw->mac.autoneg; 3041 3042 /* 3043 * Return rx_pause and tx_pause status according to actual setting of 3044 * the TFCE and RFCE bits in the CTRL register. 3045 */ 3046 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3047 if (ctrl & E1000_CTRL_TFCE) 3048 tx_pause = 1; 3049 else 3050 tx_pause = 0; 3051 3052 if (ctrl & E1000_CTRL_RFCE) 3053 rx_pause = 1; 3054 else 3055 rx_pause = 0; 3056 3057 if (rx_pause && tx_pause) 3058 fc_conf->mode = RTE_FC_FULL; 3059 else if (rx_pause) 3060 fc_conf->mode = RTE_FC_RX_PAUSE; 3061 else if (tx_pause) 3062 fc_conf->mode = RTE_FC_TX_PAUSE; 3063 else 3064 fc_conf->mode = RTE_FC_NONE; 3065 3066 return 0; 3067 } 3068 3069 static int 3070 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3071 { 3072 struct e1000_hw *hw; 3073 int err; 3074 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3075 e1000_fc_none, 3076 e1000_fc_rx_pause, 3077 e1000_fc_tx_pause, 3078 e1000_fc_full 3079 }; 3080 uint32_t rx_buf_size; 3081 uint32_t max_high_water; 3082 uint32_t rctl; 3083 3084 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3085 if (fc_conf->autoneg != hw->mac.autoneg) 3086 return -ENOTSUP; 3087 rx_buf_size = igb_get_rx_buffer_size(hw); 3088 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3089 3090 /* At least reserve one Ethernet frame for watermark */ 3091 max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN; 3092 if ((fc_conf->high_water > max_high_water) || 3093 (fc_conf->high_water < fc_conf->low_water)) { 3094 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3095 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3096 return -EINVAL; 3097 } 3098 3099 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3100 hw->fc.pause_time = fc_conf->pause_time; 3101 hw->fc.high_water = fc_conf->high_water; 3102 hw->fc.low_water = fc_conf->low_water; 3103 hw->fc.send_xon = fc_conf->send_xon; 3104 3105 err = e1000_setup_link_generic(hw); 3106 if (err == E1000_SUCCESS) { 3107 3108 /* check if we want to forward MAC frames - driver doesn't have native 3109 * capability to do that, so we'll write the registers ourselves */ 3110 3111 rctl = E1000_READ_REG(hw, E1000_RCTL); 3112 3113 /* set or clear MFLCN.PMCF bit depending on configuration */ 3114 if (fc_conf->mac_ctrl_frame_fwd != 0) 3115 rctl |= E1000_RCTL_PMCF; 3116 else 3117 rctl &= ~E1000_RCTL_PMCF; 3118 3119 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3120 E1000_WRITE_FLUSH(hw); 3121 3122 return 0; 3123 } 3124 3125 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3126 return -EIO; 3127 } 3128 3129 #define E1000_RAH_POOLSEL_SHIFT (18) 3130 static int 3131 eth_igb_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 3132 uint32_t index, uint32_t pool) 3133 { 3134 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3135 uint32_t rah; 3136 3137 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3138 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3139 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3140 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3141 return 0; 3142 } 3143 3144 static void 3145 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3146 { 3147 uint8_t addr[RTE_ETHER_ADDR_LEN]; 3148 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3149 3150 memset(addr, 0, sizeof(addr)); 3151 3152 e1000_rar_set(hw, addr, index); 3153 } 3154 3155 static int 3156 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3157 struct rte_ether_addr *addr) 3158 { 3159 eth_igb_rar_clear(dev, 0); 3160 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3161 3162 return 0; 3163 } 3164 /* 3165 * Virtual Function operations 3166 */ 3167 static void 3168 igbvf_intr_disable(struct e1000_hw *hw) 3169 { 3170 PMD_INIT_FUNC_TRACE(); 3171 3172 /* Clear interrupt mask to stop from interrupts being generated */ 3173 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3174 3175 E1000_WRITE_FLUSH(hw); 3176 } 3177 3178 static void 3179 igbvf_stop_adapter(struct rte_eth_dev *dev) 3180 { 3181 u32 reg_val; 3182 u16 i; 3183 struct rte_eth_dev_info dev_info; 3184 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3185 int ret; 3186 3187 memset(&dev_info, 0, sizeof(dev_info)); 3188 ret = eth_igbvf_infos_get(dev, &dev_info); 3189 if (ret != 0) 3190 return; 3191 3192 /* Clear interrupt mask to stop from interrupts being generated */ 3193 igbvf_intr_disable(hw); 3194 3195 /* Clear any pending interrupts, flush previous writes */ 3196 E1000_READ_REG(hw, E1000_EICR); 3197 3198 /* Disable the transmit unit. Each queue must be disabled. */ 3199 for (i = 0; i < dev_info.max_tx_queues; i++) 3200 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3201 3202 /* Disable the receive unit by stopping each queue */ 3203 for (i = 0; i < dev_info.max_rx_queues; i++) { 3204 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3205 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3206 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3207 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3208 ; 3209 } 3210 3211 /* flush all queues disables */ 3212 E1000_WRITE_FLUSH(hw); 3213 msec_delay(2); 3214 } 3215 3216 static int eth_igbvf_link_update(struct e1000_hw *hw) 3217 { 3218 struct e1000_mbx_info *mbx = &hw->mbx; 3219 struct e1000_mac_info *mac = &hw->mac; 3220 int ret_val = E1000_SUCCESS; 3221 3222 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3223 3224 /* 3225 * We only want to run this if there has been a rst asserted. 3226 * in this case that could mean a link change, device reset, 3227 * or a virtual function reset 3228 */ 3229 3230 /* If we were hit with a reset or timeout drop the link */ 3231 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3232 mac->get_link_status = TRUE; 3233 3234 if (!mac->get_link_status) 3235 goto out; 3236 3237 /* if link status is down no point in checking to see if pf is up */ 3238 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3239 goto out; 3240 3241 /* if we passed all the tests above then the link is up and we no 3242 * longer need to check for link */ 3243 mac->get_link_status = FALSE; 3244 3245 out: 3246 return ret_val; 3247 } 3248 3249 3250 static int 3251 igbvf_dev_configure(struct rte_eth_dev *dev) 3252 { 3253 struct rte_eth_conf* conf = &dev->data->dev_conf; 3254 3255 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3256 dev->data->port_id); 3257 3258 /* 3259 * VF has no ability to enable/disable HW CRC 3260 * Keep the persistent behavior the same as Host PF 3261 */ 3262 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3263 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3264 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3265 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3266 } 3267 #else 3268 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3269 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3270 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3271 } 3272 #endif 3273 3274 return 0; 3275 } 3276 3277 static int 3278 igbvf_dev_start(struct rte_eth_dev *dev) 3279 { 3280 struct e1000_hw *hw = 3281 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3282 struct e1000_adapter *adapter = 3283 E1000_DEV_PRIVATE(dev->data->dev_private); 3284 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3285 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3286 int ret; 3287 uint32_t intr_vector = 0; 3288 3289 PMD_INIT_FUNC_TRACE(); 3290 3291 hw->mac.ops.reset_hw(hw); 3292 adapter->stopped = 0; 3293 3294 /* Set all vfta */ 3295 igbvf_set_vfta_all(dev,1); 3296 3297 eth_igbvf_tx_init(dev); 3298 3299 /* This can fail when allocating mbufs for descriptor rings */ 3300 ret = eth_igbvf_rx_init(dev); 3301 if (ret) { 3302 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3303 igb_dev_clear_queues(dev); 3304 return ret; 3305 } 3306 3307 /* check and configure queue intr-vector mapping */ 3308 if (rte_intr_cap_multiple(intr_handle) && 3309 dev->data->dev_conf.intr_conf.rxq) { 3310 intr_vector = dev->data->nb_rx_queues; 3311 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3312 if (ret) 3313 return ret; 3314 } 3315 3316 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3317 intr_handle->intr_vec = 3318 rte_zmalloc("intr_vec", 3319 dev->data->nb_rx_queues * sizeof(int), 0); 3320 if (!intr_handle->intr_vec) { 3321 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3322 " intr_vec", dev->data->nb_rx_queues); 3323 return -ENOMEM; 3324 } 3325 } 3326 3327 eth_igbvf_configure_msix_intr(dev); 3328 3329 /* enable uio/vfio intr/eventfd mapping */ 3330 rte_intr_enable(intr_handle); 3331 3332 /* resume enabled intr since hw reset */ 3333 igbvf_intr_enable(dev); 3334 3335 return 0; 3336 } 3337 3338 static void 3339 igbvf_dev_stop(struct rte_eth_dev *dev) 3340 { 3341 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3342 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3343 struct e1000_adapter *adapter = 3344 E1000_DEV_PRIVATE(dev->data->dev_private); 3345 3346 if (adapter->stopped) 3347 return; 3348 3349 PMD_INIT_FUNC_TRACE(); 3350 3351 igbvf_stop_adapter(dev); 3352 3353 /* 3354 * Clear what we set, but we still keep shadow_vfta to 3355 * restore after device starts 3356 */ 3357 igbvf_set_vfta_all(dev,0); 3358 3359 igb_dev_clear_queues(dev); 3360 3361 /* disable intr eventfd mapping */ 3362 rte_intr_disable(intr_handle); 3363 3364 /* Clean datapath event and queue/vec mapping */ 3365 rte_intr_efd_disable(intr_handle); 3366 if (intr_handle->intr_vec) { 3367 rte_free(intr_handle->intr_vec); 3368 intr_handle->intr_vec = NULL; 3369 } 3370 3371 adapter->stopped = true; 3372 } 3373 3374 static void 3375 igbvf_dev_close(struct rte_eth_dev *dev) 3376 { 3377 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3378 struct rte_ether_addr addr; 3379 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3380 3381 PMD_INIT_FUNC_TRACE(); 3382 3383 e1000_reset_hw(hw); 3384 3385 igbvf_dev_stop(dev); 3386 3387 igb_dev_free_queues(dev); 3388 3389 /** 3390 * reprogram the RAR with a zero mac address, 3391 * to ensure that the VF traffic goes to the PF 3392 * after stop, close and detach of the VF. 3393 **/ 3394 3395 memset(&addr, 0, sizeof(addr)); 3396 igbvf_default_mac_addr_set(dev, &addr); 3397 3398 dev->dev_ops = NULL; 3399 dev->rx_pkt_burst = NULL; 3400 dev->tx_pkt_burst = NULL; 3401 3402 rte_intr_callback_unregister(&pci_dev->intr_handle, 3403 eth_igbvf_interrupt_handler, 3404 (void *)dev); 3405 } 3406 3407 static int 3408 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3409 { 3410 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3411 3412 /* Set both unicast and multicast promisc */ 3413 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3414 3415 return 0; 3416 } 3417 3418 static int 3419 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3420 { 3421 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3422 3423 /* If in allmulticast mode leave multicast promisc */ 3424 if (dev->data->all_multicast == 1) 3425 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3426 else 3427 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3428 3429 return 0; 3430 } 3431 3432 static int 3433 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3434 { 3435 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3436 3437 /* In promiscuous mode multicast promisc already set */ 3438 if (dev->data->promiscuous == 0) 3439 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3440 3441 return 0; 3442 } 3443 3444 static int 3445 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3446 { 3447 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3448 3449 /* In promiscuous mode leave multicast promisc enabled */ 3450 if (dev->data->promiscuous == 0) 3451 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3452 3453 return 0; 3454 } 3455 3456 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3457 { 3458 struct e1000_mbx_info *mbx = &hw->mbx; 3459 uint32_t msgbuf[2]; 3460 s32 err; 3461 3462 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3463 msgbuf[0] = E1000_VF_SET_VLAN; 3464 msgbuf[1] = vid; 3465 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3466 if (on) 3467 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3468 3469 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3470 if (err) 3471 goto mbx_err; 3472 3473 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3474 if (err) 3475 goto mbx_err; 3476 3477 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3478 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3479 err = -EINVAL; 3480 3481 mbx_err: 3482 return err; 3483 } 3484 3485 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3486 { 3487 struct e1000_hw *hw = 3488 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3489 struct e1000_vfta * shadow_vfta = 3490 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3491 int i = 0, j = 0, vfta = 0, mask = 1; 3492 3493 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3494 vfta = shadow_vfta->vfta[i]; 3495 if(vfta){ 3496 mask = 1; 3497 for (j = 0; j < 32; j++){ 3498 if(vfta & mask) 3499 igbvf_set_vfta(hw, 3500 (uint16_t)((i<<5)+j), on); 3501 mask<<=1; 3502 } 3503 } 3504 } 3505 3506 } 3507 3508 static int 3509 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3510 { 3511 struct e1000_hw *hw = 3512 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3513 struct e1000_vfta * shadow_vfta = 3514 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3515 uint32_t vid_idx = 0; 3516 uint32_t vid_bit = 0; 3517 int ret = 0; 3518 3519 PMD_INIT_FUNC_TRACE(); 3520 3521 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3522 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3523 if(ret){ 3524 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3525 return ret; 3526 } 3527 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3528 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3529 3530 /*Save what we set and retore it after device reset*/ 3531 if (on) 3532 shadow_vfta->vfta[vid_idx] |= vid_bit; 3533 else 3534 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3535 3536 return 0; 3537 } 3538 3539 static int 3540 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 3541 { 3542 struct e1000_hw *hw = 3543 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3544 3545 /* index is not used by rar_set() */ 3546 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3547 return 0; 3548 } 3549 3550 3551 static int 3552 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3553 struct rte_eth_rss_reta_entry64 *reta_conf, 3554 uint16_t reta_size) 3555 { 3556 uint8_t i, j, mask; 3557 uint32_t reta, r; 3558 uint16_t idx, shift; 3559 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3560 3561 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3562 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3563 "(%d) doesn't match the number hardware can supported " 3564 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3565 return -EINVAL; 3566 } 3567 3568 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3569 idx = i / RTE_RETA_GROUP_SIZE; 3570 shift = i % RTE_RETA_GROUP_SIZE; 3571 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3572 IGB_4_BIT_MASK); 3573 if (!mask) 3574 continue; 3575 if (mask == IGB_4_BIT_MASK) 3576 r = 0; 3577 else 3578 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3579 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3580 if (mask & (0x1 << j)) 3581 reta |= reta_conf[idx].reta[shift + j] << 3582 (CHAR_BIT * j); 3583 else 3584 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3585 } 3586 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3587 } 3588 3589 return 0; 3590 } 3591 3592 static int 3593 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3594 struct rte_eth_rss_reta_entry64 *reta_conf, 3595 uint16_t reta_size) 3596 { 3597 uint8_t i, j, mask; 3598 uint32_t reta; 3599 uint16_t idx, shift; 3600 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3601 3602 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3603 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3604 "(%d) doesn't match the number hardware can supported " 3605 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3606 return -EINVAL; 3607 } 3608 3609 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3610 idx = i / RTE_RETA_GROUP_SIZE; 3611 shift = i % RTE_RETA_GROUP_SIZE; 3612 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3613 IGB_4_BIT_MASK); 3614 if (!mask) 3615 continue; 3616 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3617 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3618 if (mask & (0x1 << j)) 3619 reta_conf[idx].reta[shift + j] = 3620 ((reta >> (CHAR_BIT * j)) & 3621 IGB_8_BIT_MASK); 3622 } 3623 } 3624 3625 return 0; 3626 } 3627 3628 int 3629 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3630 struct rte_eth_syn_filter *filter, 3631 bool add) 3632 { 3633 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3634 struct e1000_filter_info *filter_info = 3635 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3636 uint32_t synqf, rfctl; 3637 3638 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3639 return -EINVAL; 3640 3641 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3642 3643 if (add) { 3644 if (synqf & E1000_SYN_FILTER_ENABLE) 3645 return -EINVAL; 3646 3647 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3648 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3649 3650 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3651 if (filter->hig_pri) 3652 rfctl |= E1000_RFCTL_SYNQFP; 3653 else 3654 rfctl &= ~E1000_RFCTL_SYNQFP; 3655 3656 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3657 } else { 3658 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3659 return -ENOENT; 3660 synqf = 0; 3661 } 3662 3663 filter_info->syn_info = synqf; 3664 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3665 E1000_WRITE_FLUSH(hw); 3666 return 0; 3667 } 3668 3669 static int 3670 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3671 struct rte_eth_syn_filter *filter) 3672 { 3673 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3674 uint32_t synqf, rfctl; 3675 3676 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3677 if (synqf & E1000_SYN_FILTER_ENABLE) { 3678 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3679 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3680 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3681 E1000_SYN_FILTER_QUEUE_SHIFT); 3682 return 0; 3683 } 3684 3685 return -ENOENT; 3686 } 3687 3688 static int 3689 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3690 enum rte_filter_op filter_op, 3691 void *arg) 3692 { 3693 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3694 int ret; 3695 3696 MAC_TYPE_FILTER_SUP(hw->mac.type); 3697 3698 if (filter_op == RTE_ETH_FILTER_NOP) 3699 return 0; 3700 3701 if (arg == NULL) { 3702 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3703 filter_op); 3704 return -EINVAL; 3705 } 3706 3707 switch (filter_op) { 3708 case RTE_ETH_FILTER_ADD: 3709 ret = eth_igb_syn_filter_set(dev, 3710 (struct rte_eth_syn_filter *)arg, 3711 TRUE); 3712 break; 3713 case RTE_ETH_FILTER_DELETE: 3714 ret = eth_igb_syn_filter_set(dev, 3715 (struct rte_eth_syn_filter *)arg, 3716 FALSE); 3717 break; 3718 case RTE_ETH_FILTER_GET: 3719 ret = eth_igb_syn_filter_get(dev, 3720 (struct rte_eth_syn_filter *)arg); 3721 break; 3722 default: 3723 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3724 ret = -EINVAL; 3725 break; 3726 } 3727 3728 return ret; 3729 } 3730 3731 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3732 static inline int 3733 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3734 struct e1000_2tuple_filter_info *filter_info) 3735 { 3736 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3737 return -EINVAL; 3738 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3739 return -EINVAL; /* filter index is out of range. */ 3740 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 3741 return -EINVAL; /* flags is invalid. */ 3742 3743 switch (filter->dst_port_mask) { 3744 case UINT16_MAX: 3745 filter_info->dst_port_mask = 0; 3746 filter_info->dst_port = filter->dst_port; 3747 break; 3748 case 0: 3749 filter_info->dst_port_mask = 1; 3750 break; 3751 default: 3752 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3753 return -EINVAL; 3754 } 3755 3756 switch (filter->proto_mask) { 3757 case UINT8_MAX: 3758 filter_info->proto_mask = 0; 3759 filter_info->proto = filter->proto; 3760 break; 3761 case 0: 3762 filter_info->proto_mask = 1; 3763 break; 3764 default: 3765 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3766 return -EINVAL; 3767 } 3768 3769 filter_info->priority = (uint8_t)filter->priority; 3770 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3771 filter_info->tcp_flags = filter->tcp_flags; 3772 else 3773 filter_info->tcp_flags = 0; 3774 3775 return 0; 3776 } 3777 3778 static inline struct e1000_2tuple_filter * 3779 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3780 struct e1000_2tuple_filter_info *key) 3781 { 3782 struct e1000_2tuple_filter *it; 3783 3784 TAILQ_FOREACH(it, filter_list, entries) { 3785 if (memcmp(key, &it->filter_info, 3786 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3787 return it; 3788 } 3789 } 3790 return NULL; 3791 } 3792 3793 /* inject a igb 2tuple filter to HW */ 3794 static inline void 3795 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3796 struct e1000_2tuple_filter *filter) 3797 { 3798 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3799 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3800 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3801 int i; 3802 3803 i = filter->index; 3804 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3805 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3806 imir |= E1000_IMIR_PORT_BP; 3807 else 3808 imir &= ~E1000_IMIR_PORT_BP; 3809 3810 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3811 3812 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3813 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3814 ttqf |= (uint32_t)(filter->filter_info.proto & 3815 E1000_TTQF_PROTOCOL_MASK); 3816 if (filter->filter_info.proto_mask == 0) 3817 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3818 3819 /* tcp flags bits setting. */ 3820 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 3821 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 3822 imir_ext |= E1000_IMIREXT_CTRL_URG; 3823 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 3824 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3825 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 3826 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3827 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 3828 imir_ext |= E1000_IMIREXT_CTRL_RST; 3829 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 3830 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3831 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 3832 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3833 } else { 3834 imir_ext |= E1000_IMIREXT_CTRL_BP; 3835 } 3836 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3837 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3838 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3839 } 3840 3841 /* 3842 * igb_add_2tuple_filter - add a 2tuple filter 3843 * 3844 * @param 3845 * dev: Pointer to struct rte_eth_dev. 3846 * ntuple_filter: ponter to the filter that will be added. 3847 * 3848 * @return 3849 * - On success, zero. 3850 * - On failure, a negative value. 3851 */ 3852 static int 3853 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3854 struct rte_eth_ntuple_filter *ntuple_filter) 3855 { 3856 struct e1000_filter_info *filter_info = 3857 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3858 struct e1000_2tuple_filter *filter; 3859 int i, ret; 3860 3861 filter = rte_zmalloc("e1000_2tuple_filter", 3862 sizeof(struct e1000_2tuple_filter), 0); 3863 if (filter == NULL) 3864 return -ENOMEM; 3865 3866 ret = ntuple_filter_to_2tuple(ntuple_filter, 3867 &filter->filter_info); 3868 if (ret < 0) { 3869 rte_free(filter); 3870 return ret; 3871 } 3872 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3873 &filter->filter_info) != NULL) { 3874 PMD_DRV_LOG(ERR, "filter exists."); 3875 rte_free(filter); 3876 return -EEXIST; 3877 } 3878 filter->queue = ntuple_filter->queue; 3879 3880 /* 3881 * look for an unused 2tuple filter index, 3882 * and insert the filter to list. 3883 */ 3884 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3885 if (!(filter_info->twotuple_mask & (1 << i))) { 3886 filter_info->twotuple_mask |= 1 << i; 3887 filter->index = i; 3888 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3889 filter, 3890 entries); 3891 break; 3892 } 3893 } 3894 if (i >= E1000_MAX_TTQF_FILTERS) { 3895 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3896 rte_free(filter); 3897 return -ENOSYS; 3898 } 3899 3900 igb_inject_2uple_filter(dev, filter); 3901 return 0; 3902 } 3903 3904 int 3905 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3906 struct e1000_2tuple_filter *filter) 3907 { 3908 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3909 struct e1000_filter_info *filter_info = 3910 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3911 3912 filter_info->twotuple_mask &= ~(1 << filter->index); 3913 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3914 rte_free(filter); 3915 3916 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3917 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3918 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3919 return 0; 3920 } 3921 3922 /* 3923 * igb_remove_2tuple_filter - remove a 2tuple filter 3924 * 3925 * @param 3926 * dev: Pointer to struct rte_eth_dev. 3927 * ntuple_filter: ponter to the filter that will be removed. 3928 * 3929 * @return 3930 * - On success, zero. 3931 * - On failure, a negative value. 3932 */ 3933 static int 3934 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3935 struct rte_eth_ntuple_filter *ntuple_filter) 3936 { 3937 struct e1000_filter_info *filter_info = 3938 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3939 struct e1000_2tuple_filter_info filter_2tuple; 3940 struct e1000_2tuple_filter *filter; 3941 int ret; 3942 3943 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3944 ret = ntuple_filter_to_2tuple(ntuple_filter, 3945 &filter_2tuple); 3946 if (ret < 0) 3947 return ret; 3948 3949 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3950 &filter_2tuple); 3951 if (filter == NULL) { 3952 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3953 return -ENOENT; 3954 } 3955 3956 igb_delete_2tuple_filter(dev, filter); 3957 3958 return 0; 3959 } 3960 3961 /* inject a igb flex filter to HW */ 3962 static inline void 3963 igb_inject_flex_filter(struct rte_eth_dev *dev, 3964 struct e1000_flex_filter *filter) 3965 { 3966 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3967 uint32_t wufc, queueing; 3968 uint32_t reg_off; 3969 uint8_t i, j = 0; 3970 3971 wufc = E1000_READ_REG(hw, E1000_WUFC); 3972 if (filter->index < E1000_MAX_FHFT) 3973 reg_off = E1000_FHFT(filter->index); 3974 else 3975 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3976 3977 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3978 (E1000_WUFC_FLX0 << filter->index)); 3979 queueing = filter->filter_info.len | 3980 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3981 (filter->filter_info.priority << 3982 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3983 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3984 queueing); 3985 3986 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3987 E1000_WRITE_REG(hw, reg_off, 3988 filter->filter_info.dwords[j]); 3989 reg_off += sizeof(uint32_t); 3990 E1000_WRITE_REG(hw, reg_off, 3991 filter->filter_info.dwords[++j]); 3992 reg_off += sizeof(uint32_t); 3993 E1000_WRITE_REG(hw, reg_off, 3994 (uint32_t)filter->filter_info.mask[i]); 3995 reg_off += sizeof(uint32_t) * 2; 3996 ++j; 3997 } 3998 } 3999 4000 static inline struct e1000_flex_filter * 4001 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 4002 struct e1000_flex_filter_info *key) 4003 { 4004 struct e1000_flex_filter *it; 4005 4006 TAILQ_FOREACH(it, filter_list, entries) { 4007 if (memcmp(key, &it->filter_info, 4008 sizeof(struct e1000_flex_filter_info)) == 0) 4009 return it; 4010 } 4011 4012 return NULL; 4013 } 4014 4015 /* remove a flex byte filter 4016 * @param 4017 * dev: Pointer to struct rte_eth_dev. 4018 * filter: the pointer of the filter will be removed. 4019 */ 4020 void 4021 igb_remove_flex_filter(struct rte_eth_dev *dev, 4022 struct e1000_flex_filter *filter) 4023 { 4024 struct e1000_filter_info *filter_info = 4025 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4026 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4027 uint32_t wufc, i; 4028 uint32_t reg_off; 4029 4030 wufc = E1000_READ_REG(hw, E1000_WUFC); 4031 if (filter->index < E1000_MAX_FHFT) 4032 reg_off = E1000_FHFT(filter->index); 4033 else 4034 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4035 4036 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4037 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4038 4039 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4040 (~(E1000_WUFC_FLX0 << filter->index))); 4041 4042 filter_info->flex_mask &= ~(1 << filter->index); 4043 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4044 rte_free(filter); 4045 } 4046 4047 int 4048 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4049 struct rte_eth_flex_filter *filter, 4050 bool add) 4051 { 4052 struct e1000_filter_info *filter_info = 4053 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4054 struct e1000_flex_filter *flex_filter, *it; 4055 uint32_t mask; 4056 uint8_t shift, i; 4057 4058 flex_filter = rte_zmalloc("e1000_flex_filter", 4059 sizeof(struct e1000_flex_filter), 0); 4060 if (flex_filter == NULL) 4061 return -ENOMEM; 4062 4063 flex_filter->filter_info.len = filter->len; 4064 flex_filter->filter_info.priority = filter->priority; 4065 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4066 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4067 mask = 0; 4068 /* reverse bits in flex filter's mask*/ 4069 for (shift = 0; shift < CHAR_BIT; shift++) { 4070 if (filter->mask[i] & (0x01 << shift)) 4071 mask |= (0x80 >> shift); 4072 } 4073 flex_filter->filter_info.mask[i] = mask; 4074 } 4075 4076 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4077 &flex_filter->filter_info); 4078 if (it == NULL && !add) { 4079 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4080 rte_free(flex_filter); 4081 return -ENOENT; 4082 } 4083 if (it != NULL && add) { 4084 PMD_DRV_LOG(ERR, "filter exists."); 4085 rte_free(flex_filter); 4086 return -EEXIST; 4087 } 4088 4089 if (add) { 4090 flex_filter->queue = filter->queue; 4091 /* 4092 * look for an unused flex filter index 4093 * and insert the filter into the list. 4094 */ 4095 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4096 if (!(filter_info->flex_mask & (1 << i))) { 4097 filter_info->flex_mask |= 1 << i; 4098 flex_filter->index = i; 4099 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4100 flex_filter, 4101 entries); 4102 break; 4103 } 4104 } 4105 if (i >= E1000_MAX_FLEX_FILTERS) { 4106 PMD_DRV_LOG(ERR, "flex filters are full."); 4107 rte_free(flex_filter); 4108 return -ENOSYS; 4109 } 4110 4111 igb_inject_flex_filter(dev, flex_filter); 4112 4113 } else { 4114 igb_remove_flex_filter(dev, it); 4115 rte_free(flex_filter); 4116 } 4117 4118 return 0; 4119 } 4120 4121 static int 4122 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4123 struct rte_eth_flex_filter *filter) 4124 { 4125 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4126 struct e1000_filter_info *filter_info = 4127 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4128 struct e1000_flex_filter flex_filter, *it; 4129 uint32_t wufc, queueing, wufc_en = 0; 4130 4131 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4132 flex_filter.filter_info.len = filter->len; 4133 flex_filter.filter_info.priority = filter->priority; 4134 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4135 memcpy(flex_filter.filter_info.mask, filter->mask, 4136 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4137 4138 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4139 &flex_filter.filter_info); 4140 if (it == NULL) { 4141 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4142 return -ENOENT; 4143 } 4144 4145 wufc = E1000_READ_REG(hw, E1000_WUFC); 4146 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4147 4148 if ((wufc & wufc_en) == wufc_en) { 4149 uint32_t reg_off = 0; 4150 if (it->index < E1000_MAX_FHFT) 4151 reg_off = E1000_FHFT(it->index); 4152 else 4153 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4154 4155 queueing = E1000_READ_REG(hw, 4156 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4157 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4158 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4159 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4160 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4161 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4162 return 0; 4163 } 4164 return -ENOENT; 4165 } 4166 4167 static int 4168 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4169 enum rte_filter_op filter_op, 4170 void *arg) 4171 { 4172 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4173 struct rte_eth_flex_filter *filter; 4174 int ret = 0; 4175 4176 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4177 4178 if (filter_op == RTE_ETH_FILTER_NOP) 4179 return ret; 4180 4181 if (arg == NULL) { 4182 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4183 filter_op); 4184 return -EINVAL; 4185 } 4186 4187 filter = (struct rte_eth_flex_filter *)arg; 4188 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4189 || filter->len % sizeof(uint64_t) != 0) { 4190 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4191 return -EINVAL; 4192 } 4193 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4194 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4195 return -EINVAL; 4196 } 4197 4198 switch (filter_op) { 4199 case RTE_ETH_FILTER_ADD: 4200 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4201 break; 4202 case RTE_ETH_FILTER_DELETE: 4203 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4204 break; 4205 case RTE_ETH_FILTER_GET: 4206 ret = eth_igb_get_flex_filter(dev, filter); 4207 break; 4208 default: 4209 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4210 ret = -EINVAL; 4211 break; 4212 } 4213 4214 return ret; 4215 } 4216 4217 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4218 static inline int 4219 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4220 struct e1000_5tuple_filter_info *filter_info) 4221 { 4222 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4223 return -EINVAL; 4224 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4225 return -EINVAL; /* filter index is out of range. */ 4226 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 4227 return -EINVAL; /* flags is invalid. */ 4228 4229 switch (filter->dst_ip_mask) { 4230 case UINT32_MAX: 4231 filter_info->dst_ip_mask = 0; 4232 filter_info->dst_ip = filter->dst_ip; 4233 break; 4234 case 0: 4235 filter_info->dst_ip_mask = 1; 4236 break; 4237 default: 4238 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4239 return -EINVAL; 4240 } 4241 4242 switch (filter->src_ip_mask) { 4243 case UINT32_MAX: 4244 filter_info->src_ip_mask = 0; 4245 filter_info->src_ip = filter->src_ip; 4246 break; 4247 case 0: 4248 filter_info->src_ip_mask = 1; 4249 break; 4250 default: 4251 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4252 return -EINVAL; 4253 } 4254 4255 switch (filter->dst_port_mask) { 4256 case UINT16_MAX: 4257 filter_info->dst_port_mask = 0; 4258 filter_info->dst_port = filter->dst_port; 4259 break; 4260 case 0: 4261 filter_info->dst_port_mask = 1; 4262 break; 4263 default: 4264 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4265 return -EINVAL; 4266 } 4267 4268 switch (filter->src_port_mask) { 4269 case UINT16_MAX: 4270 filter_info->src_port_mask = 0; 4271 filter_info->src_port = filter->src_port; 4272 break; 4273 case 0: 4274 filter_info->src_port_mask = 1; 4275 break; 4276 default: 4277 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4278 return -EINVAL; 4279 } 4280 4281 switch (filter->proto_mask) { 4282 case UINT8_MAX: 4283 filter_info->proto_mask = 0; 4284 filter_info->proto = filter->proto; 4285 break; 4286 case 0: 4287 filter_info->proto_mask = 1; 4288 break; 4289 default: 4290 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4291 return -EINVAL; 4292 } 4293 4294 filter_info->priority = (uint8_t)filter->priority; 4295 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4296 filter_info->tcp_flags = filter->tcp_flags; 4297 else 4298 filter_info->tcp_flags = 0; 4299 4300 return 0; 4301 } 4302 4303 static inline struct e1000_5tuple_filter * 4304 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4305 struct e1000_5tuple_filter_info *key) 4306 { 4307 struct e1000_5tuple_filter *it; 4308 4309 TAILQ_FOREACH(it, filter_list, entries) { 4310 if (memcmp(key, &it->filter_info, 4311 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4312 return it; 4313 } 4314 } 4315 return NULL; 4316 } 4317 4318 /* inject a igb 5-tuple filter to HW */ 4319 static inline void 4320 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4321 struct e1000_5tuple_filter *filter) 4322 { 4323 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4324 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4325 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4326 uint8_t i; 4327 4328 i = filter->index; 4329 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4330 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4331 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4332 if (filter->filter_info.dst_ip_mask == 0) 4333 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4334 if (filter->filter_info.src_port_mask == 0) 4335 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4336 if (filter->filter_info.proto_mask == 0) 4337 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4338 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4339 E1000_FTQF_QUEUE_MASK; 4340 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4341 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4342 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4343 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4344 4345 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4346 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4347 4348 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4349 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4350 imir |= E1000_IMIR_PORT_BP; 4351 else 4352 imir &= ~E1000_IMIR_PORT_BP; 4353 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4354 4355 /* tcp flags bits setting. */ 4356 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 4357 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 4358 imir_ext |= E1000_IMIREXT_CTRL_URG; 4359 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 4360 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4361 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 4362 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4363 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 4364 imir_ext |= E1000_IMIREXT_CTRL_RST; 4365 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 4366 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4367 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 4368 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4369 } else { 4370 imir_ext |= E1000_IMIREXT_CTRL_BP; 4371 } 4372 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4373 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4374 } 4375 4376 /* 4377 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4378 * 4379 * @param 4380 * dev: Pointer to struct rte_eth_dev. 4381 * ntuple_filter: ponter to the filter that will be added. 4382 * 4383 * @return 4384 * - On success, zero. 4385 * - On failure, a negative value. 4386 */ 4387 static int 4388 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4389 struct rte_eth_ntuple_filter *ntuple_filter) 4390 { 4391 struct e1000_filter_info *filter_info = 4392 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4393 struct e1000_5tuple_filter *filter; 4394 uint8_t i; 4395 int ret; 4396 4397 filter = rte_zmalloc("e1000_5tuple_filter", 4398 sizeof(struct e1000_5tuple_filter), 0); 4399 if (filter == NULL) 4400 return -ENOMEM; 4401 4402 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4403 &filter->filter_info); 4404 if (ret < 0) { 4405 rte_free(filter); 4406 return ret; 4407 } 4408 4409 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4410 &filter->filter_info) != NULL) { 4411 PMD_DRV_LOG(ERR, "filter exists."); 4412 rte_free(filter); 4413 return -EEXIST; 4414 } 4415 filter->queue = ntuple_filter->queue; 4416 4417 /* 4418 * look for an unused 5tuple filter index, 4419 * and insert the filter to list. 4420 */ 4421 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4422 if (!(filter_info->fivetuple_mask & (1 << i))) { 4423 filter_info->fivetuple_mask |= 1 << i; 4424 filter->index = i; 4425 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4426 filter, 4427 entries); 4428 break; 4429 } 4430 } 4431 if (i >= E1000_MAX_FTQF_FILTERS) { 4432 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4433 rte_free(filter); 4434 return -ENOSYS; 4435 } 4436 4437 igb_inject_5tuple_filter_82576(dev, filter); 4438 return 0; 4439 } 4440 4441 int 4442 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4443 struct e1000_5tuple_filter *filter) 4444 { 4445 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4446 struct e1000_filter_info *filter_info = 4447 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4448 4449 filter_info->fivetuple_mask &= ~(1 << filter->index); 4450 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4451 rte_free(filter); 4452 4453 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4454 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4455 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4456 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4457 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4458 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4459 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4460 return 0; 4461 } 4462 4463 /* 4464 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4465 * 4466 * @param 4467 * dev: Pointer to struct rte_eth_dev. 4468 * ntuple_filter: ponter to the filter that will be removed. 4469 * 4470 * @return 4471 * - On success, zero. 4472 * - On failure, a negative value. 4473 */ 4474 static int 4475 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4476 struct rte_eth_ntuple_filter *ntuple_filter) 4477 { 4478 struct e1000_filter_info *filter_info = 4479 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4480 struct e1000_5tuple_filter_info filter_5tuple; 4481 struct e1000_5tuple_filter *filter; 4482 int ret; 4483 4484 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4485 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4486 &filter_5tuple); 4487 if (ret < 0) 4488 return ret; 4489 4490 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4491 &filter_5tuple); 4492 if (filter == NULL) { 4493 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4494 return -ENOENT; 4495 } 4496 4497 igb_delete_5tuple_filter_82576(dev, filter); 4498 4499 return 0; 4500 } 4501 4502 static int 4503 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4504 { 4505 uint32_t rctl; 4506 struct e1000_hw *hw; 4507 struct rte_eth_dev_info dev_info; 4508 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4509 int ret; 4510 4511 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4512 4513 #ifdef RTE_LIBRTE_82571_SUPPORT 4514 /* XXX: not bigger than max_rx_pktlen */ 4515 if (hw->mac.type == e1000_82571) 4516 return -ENOTSUP; 4517 #endif 4518 ret = eth_igb_infos_get(dev, &dev_info); 4519 if (ret != 0) 4520 return ret; 4521 4522 /* check that mtu is within the allowed range */ 4523 if (mtu < RTE_ETHER_MIN_MTU || 4524 frame_size > dev_info.max_rx_pktlen) 4525 return -EINVAL; 4526 4527 /* refuse mtu that requires the support of scattered packets when this 4528 * feature has not been enabled before. */ 4529 if (!dev->data->scattered_rx && 4530 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4531 return -EINVAL; 4532 4533 rctl = E1000_READ_REG(hw, E1000_RCTL); 4534 4535 /* switch to jumbo mode if needed */ 4536 if (frame_size > RTE_ETHER_MAX_LEN) { 4537 dev->data->dev_conf.rxmode.offloads |= 4538 DEV_RX_OFFLOAD_JUMBO_FRAME; 4539 rctl |= E1000_RCTL_LPE; 4540 } else { 4541 dev->data->dev_conf.rxmode.offloads &= 4542 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4543 rctl &= ~E1000_RCTL_LPE; 4544 } 4545 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4546 4547 /* update max frame size */ 4548 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4549 4550 E1000_WRITE_REG(hw, E1000_RLPML, 4551 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4552 4553 return 0; 4554 } 4555 4556 /* 4557 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4558 * 4559 * @param 4560 * dev: Pointer to struct rte_eth_dev. 4561 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4562 * add: if true, add filter, if false, remove filter 4563 * 4564 * @return 4565 * - On success, zero. 4566 * - On failure, a negative value. 4567 */ 4568 int 4569 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4570 struct rte_eth_ntuple_filter *ntuple_filter, 4571 bool add) 4572 { 4573 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4574 int ret; 4575 4576 switch (ntuple_filter->flags) { 4577 case RTE_5TUPLE_FLAGS: 4578 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4579 if (hw->mac.type != e1000_82576) 4580 return -ENOTSUP; 4581 if (add) 4582 ret = igb_add_5tuple_filter_82576(dev, 4583 ntuple_filter); 4584 else 4585 ret = igb_remove_5tuple_filter_82576(dev, 4586 ntuple_filter); 4587 break; 4588 case RTE_2TUPLE_FLAGS: 4589 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4590 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4591 hw->mac.type != e1000_i210 && 4592 hw->mac.type != e1000_i211) 4593 return -ENOTSUP; 4594 if (add) 4595 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4596 else 4597 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4598 break; 4599 default: 4600 ret = -EINVAL; 4601 break; 4602 } 4603 4604 return ret; 4605 } 4606 4607 /* 4608 * igb_get_ntuple_filter - get a ntuple filter 4609 * 4610 * @param 4611 * dev: Pointer to struct rte_eth_dev. 4612 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4613 * 4614 * @return 4615 * - On success, zero. 4616 * - On failure, a negative value. 4617 */ 4618 static int 4619 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4620 struct rte_eth_ntuple_filter *ntuple_filter) 4621 { 4622 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4623 struct e1000_filter_info *filter_info = 4624 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4625 struct e1000_5tuple_filter_info filter_5tuple; 4626 struct e1000_2tuple_filter_info filter_2tuple; 4627 struct e1000_5tuple_filter *p_5tuple_filter; 4628 struct e1000_2tuple_filter *p_2tuple_filter; 4629 int ret; 4630 4631 switch (ntuple_filter->flags) { 4632 case RTE_5TUPLE_FLAGS: 4633 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4634 if (hw->mac.type != e1000_82576) 4635 return -ENOTSUP; 4636 memset(&filter_5tuple, 4637 0, 4638 sizeof(struct e1000_5tuple_filter_info)); 4639 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4640 &filter_5tuple); 4641 if (ret < 0) 4642 return ret; 4643 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4644 &filter_info->fivetuple_list, 4645 &filter_5tuple); 4646 if (p_5tuple_filter == NULL) { 4647 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4648 return -ENOENT; 4649 } 4650 ntuple_filter->queue = p_5tuple_filter->queue; 4651 break; 4652 case RTE_2TUPLE_FLAGS: 4653 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4654 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4655 return -ENOTSUP; 4656 memset(&filter_2tuple, 4657 0, 4658 sizeof(struct e1000_2tuple_filter_info)); 4659 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4660 if (ret < 0) 4661 return ret; 4662 p_2tuple_filter = igb_2tuple_filter_lookup( 4663 &filter_info->twotuple_list, 4664 &filter_2tuple); 4665 if (p_2tuple_filter == NULL) { 4666 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4667 return -ENOENT; 4668 } 4669 ntuple_filter->queue = p_2tuple_filter->queue; 4670 break; 4671 default: 4672 ret = -EINVAL; 4673 break; 4674 } 4675 4676 return 0; 4677 } 4678 4679 /* 4680 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4681 * @dev: pointer to rte_eth_dev structure 4682 * @filter_op:operation will be taken. 4683 * @arg: a pointer to specific structure corresponding to the filter_op 4684 */ 4685 static int 4686 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4687 enum rte_filter_op filter_op, 4688 void *arg) 4689 { 4690 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4691 int ret; 4692 4693 MAC_TYPE_FILTER_SUP(hw->mac.type); 4694 4695 if (filter_op == RTE_ETH_FILTER_NOP) 4696 return 0; 4697 4698 if (arg == NULL) { 4699 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4700 filter_op); 4701 return -EINVAL; 4702 } 4703 4704 switch (filter_op) { 4705 case RTE_ETH_FILTER_ADD: 4706 ret = igb_add_del_ntuple_filter(dev, 4707 (struct rte_eth_ntuple_filter *)arg, 4708 TRUE); 4709 break; 4710 case RTE_ETH_FILTER_DELETE: 4711 ret = igb_add_del_ntuple_filter(dev, 4712 (struct rte_eth_ntuple_filter *)arg, 4713 FALSE); 4714 break; 4715 case RTE_ETH_FILTER_GET: 4716 ret = igb_get_ntuple_filter(dev, 4717 (struct rte_eth_ntuple_filter *)arg); 4718 break; 4719 default: 4720 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4721 ret = -EINVAL; 4722 break; 4723 } 4724 return ret; 4725 } 4726 4727 static inline int 4728 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4729 uint16_t ethertype) 4730 { 4731 int i; 4732 4733 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4734 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4735 (filter_info->ethertype_mask & (1 << i))) 4736 return i; 4737 } 4738 return -1; 4739 } 4740 4741 static inline int 4742 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4743 uint16_t ethertype, uint32_t etqf) 4744 { 4745 int i; 4746 4747 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4748 if (!(filter_info->ethertype_mask & (1 << i))) { 4749 filter_info->ethertype_mask |= 1 << i; 4750 filter_info->ethertype_filters[i].ethertype = ethertype; 4751 filter_info->ethertype_filters[i].etqf = etqf; 4752 return i; 4753 } 4754 } 4755 return -1; 4756 } 4757 4758 int 4759 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4760 uint8_t idx) 4761 { 4762 if (idx >= E1000_MAX_ETQF_FILTERS) 4763 return -1; 4764 filter_info->ethertype_mask &= ~(1 << idx); 4765 filter_info->ethertype_filters[idx].ethertype = 0; 4766 filter_info->ethertype_filters[idx].etqf = 0; 4767 return idx; 4768 } 4769 4770 4771 int 4772 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4773 struct rte_eth_ethertype_filter *filter, 4774 bool add) 4775 { 4776 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4777 struct e1000_filter_info *filter_info = 4778 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4779 uint32_t etqf = 0; 4780 int ret; 4781 4782 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 4783 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 4784 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4785 " ethertype filter.", filter->ether_type); 4786 return -EINVAL; 4787 } 4788 4789 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4790 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4791 return -EINVAL; 4792 } 4793 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4794 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4795 return -EINVAL; 4796 } 4797 4798 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4799 if (ret >= 0 && add) { 4800 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4801 filter->ether_type); 4802 return -EEXIST; 4803 } 4804 if (ret < 0 && !add) { 4805 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4806 filter->ether_type); 4807 return -ENOENT; 4808 } 4809 4810 if (add) { 4811 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4812 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4813 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4814 ret = igb_ethertype_filter_insert(filter_info, 4815 filter->ether_type, etqf); 4816 if (ret < 0) { 4817 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4818 return -ENOSYS; 4819 } 4820 } else { 4821 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4822 if (ret < 0) 4823 return -ENOSYS; 4824 } 4825 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4826 E1000_WRITE_FLUSH(hw); 4827 4828 return 0; 4829 } 4830 4831 static int 4832 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4833 struct rte_eth_ethertype_filter *filter) 4834 { 4835 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4836 struct e1000_filter_info *filter_info = 4837 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4838 uint32_t etqf; 4839 int ret; 4840 4841 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4842 if (ret < 0) { 4843 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4844 filter->ether_type); 4845 return -ENOENT; 4846 } 4847 4848 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4849 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4850 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4851 filter->flags = 0; 4852 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4853 E1000_ETQF_QUEUE_SHIFT; 4854 return 0; 4855 } 4856 4857 return -ENOENT; 4858 } 4859 4860 /* 4861 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4862 * @dev: pointer to rte_eth_dev structure 4863 * @filter_op:operation will be taken. 4864 * @arg: a pointer to specific structure corresponding to the filter_op 4865 */ 4866 static int 4867 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4868 enum rte_filter_op filter_op, 4869 void *arg) 4870 { 4871 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4872 int ret; 4873 4874 MAC_TYPE_FILTER_SUP(hw->mac.type); 4875 4876 if (filter_op == RTE_ETH_FILTER_NOP) 4877 return 0; 4878 4879 if (arg == NULL) { 4880 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4881 filter_op); 4882 return -EINVAL; 4883 } 4884 4885 switch (filter_op) { 4886 case RTE_ETH_FILTER_ADD: 4887 ret = igb_add_del_ethertype_filter(dev, 4888 (struct rte_eth_ethertype_filter *)arg, 4889 TRUE); 4890 break; 4891 case RTE_ETH_FILTER_DELETE: 4892 ret = igb_add_del_ethertype_filter(dev, 4893 (struct rte_eth_ethertype_filter *)arg, 4894 FALSE); 4895 break; 4896 case RTE_ETH_FILTER_GET: 4897 ret = igb_get_ethertype_filter(dev, 4898 (struct rte_eth_ethertype_filter *)arg); 4899 break; 4900 default: 4901 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4902 ret = -EINVAL; 4903 break; 4904 } 4905 return ret; 4906 } 4907 4908 static int 4909 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4910 enum rte_filter_type filter_type, 4911 enum rte_filter_op filter_op, 4912 void *arg) 4913 { 4914 int ret = 0; 4915 4916 switch (filter_type) { 4917 case RTE_ETH_FILTER_NTUPLE: 4918 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4919 break; 4920 case RTE_ETH_FILTER_ETHERTYPE: 4921 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4922 break; 4923 case RTE_ETH_FILTER_SYN: 4924 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4925 break; 4926 case RTE_ETH_FILTER_FLEXIBLE: 4927 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4928 break; 4929 case RTE_ETH_FILTER_GENERIC: 4930 if (filter_op != RTE_ETH_FILTER_GET) 4931 return -EINVAL; 4932 *(const void **)arg = &igb_flow_ops; 4933 break; 4934 default: 4935 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4936 filter_type); 4937 break; 4938 } 4939 4940 return ret; 4941 } 4942 4943 static int 4944 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4945 struct rte_ether_addr *mc_addr_set, 4946 uint32_t nb_mc_addr) 4947 { 4948 struct e1000_hw *hw; 4949 4950 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4951 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4952 return 0; 4953 } 4954 4955 static uint64_t 4956 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4957 { 4958 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4959 uint64_t systime_cycles; 4960 4961 switch (hw->mac.type) { 4962 case e1000_i210: 4963 case e1000_i211: 4964 /* 4965 * Need to read System Time Residue Register to be able 4966 * to read the other two registers. 4967 */ 4968 E1000_READ_REG(hw, E1000_SYSTIMR); 4969 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4970 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4971 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4972 * NSEC_PER_SEC; 4973 break; 4974 case e1000_82580: 4975 case e1000_i350: 4976 case e1000_i354: 4977 /* 4978 * Need to read System Time Residue Register to be able 4979 * to read the other two registers. 4980 */ 4981 E1000_READ_REG(hw, E1000_SYSTIMR); 4982 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4983 /* Only the 8 LSB are valid. */ 4984 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4985 & 0xff) << 32; 4986 break; 4987 default: 4988 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4989 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4990 << 32; 4991 break; 4992 } 4993 4994 return systime_cycles; 4995 } 4996 4997 static uint64_t 4998 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4999 { 5000 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5001 uint64_t rx_tstamp_cycles; 5002 5003 switch (hw->mac.type) { 5004 case e1000_i210: 5005 case e1000_i211: 5006 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5007 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5008 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 5009 * NSEC_PER_SEC; 5010 break; 5011 case e1000_82580: 5012 case e1000_i350: 5013 case e1000_i354: 5014 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5015 /* Only the 8 LSB are valid. */ 5016 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 5017 & 0xff) << 32; 5018 break; 5019 default: 5020 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5021 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 5022 << 32; 5023 break; 5024 } 5025 5026 return rx_tstamp_cycles; 5027 } 5028 5029 static uint64_t 5030 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 5031 { 5032 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5033 uint64_t tx_tstamp_cycles; 5034 5035 switch (hw->mac.type) { 5036 case e1000_i210: 5037 case e1000_i211: 5038 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5039 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5040 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5041 * NSEC_PER_SEC; 5042 break; 5043 case e1000_82580: 5044 case e1000_i350: 5045 case e1000_i354: 5046 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5047 /* Only the 8 LSB are valid. */ 5048 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5049 & 0xff) << 32; 5050 break; 5051 default: 5052 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5053 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5054 << 32; 5055 break; 5056 } 5057 5058 return tx_tstamp_cycles; 5059 } 5060 5061 static void 5062 igb_start_timecounters(struct rte_eth_dev *dev) 5063 { 5064 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5065 struct e1000_adapter *adapter = dev->data->dev_private; 5066 uint32_t incval = 1; 5067 uint32_t shift = 0; 5068 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5069 5070 switch (hw->mac.type) { 5071 case e1000_82580: 5072 case e1000_i350: 5073 case e1000_i354: 5074 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5075 mask = (1ULL << 40) - 1; 5076 /* fall-through */ 5077 case e1000_i210: 5078 case e1000_i211: 5079 /* 5080 * Start incrementing the register 5081 * used to timestamp PTP packets. 5082 */ 5083 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5084 break; 5085 case e1000_82576: 5086 incval = E1000_INCVALUE_82576; 5087 shift = IGB_82576_TSYNC_SHIFT; 5088 E1000_WRITE_REG(hw, E1000_TIMINCA, 5089 E1000_INCPERIOD_82576 | incval); 5090 break; 5091 default: 5092 /* Not supported */ 5093 return; 5094 } 5095 5096 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5097 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5098 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5099 5100 adapter->systime_tc.cc_mask = mask; 5101 adapter->systime_tc.cc_shift = shift; 5102 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5103 5104 adapter->rx_tstamp_tc.cc_mask = mask; 5105 adapter->rx_tstamp_tc.cc_shift = shift; 5106 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5107 5108 adapter->tx_tstamp_tc.cc_mask = mask; 5109 adapter->tx_tstamp_tc.cc_shift = shift; 5110 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5111 } 5112 5113 static int 5114 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5115 { 5116 struct e1000_adapter *adapter = dev->data->dev_private; 5117 5118 adapter->systime_tc.nsec += delta; 5119 adapter->rx_tstamp_tc.nsec += delta; 5120 adapter->tx_tstamp_tc.nsec += delta; 5121 5122 return 0; 5123 } 5124 5125 static int 5126 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5127 { 5128 uint64_t ns; 5129 struct e1000_adapter *adapter = dev->data->dev_private; 5130 5131 ns = rte_timespec_to_ns(ts); 5132 5133 /* Set the timecounters to a new value. */ 5134 adapter->systime_tc.nsec = ns; 5135 adapter->rx_tstamp_tc.nsec = ns; 5136 adapter->tx_tstamp_tc.nsec = ns; 5137 5138 return 0; 5139 } 5140 5141 static int 5142 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5143 { 5144 uint64_t ns, systime_cycles; 5145 struct e1000_adapter *adapter = dev->data->dev_private; 5146 5147 systime_cycles = igb_read_systime_cyclecounter(dev); 5148 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5149 *ts = rte_ns_to_timespec(ns); 5150 5151 return 0; 5152 } 5153 5154 static int 5155 igb_timesync_enable(struct rte_eth_dev *dev) 5156 { 5157 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5158 uint32_t tsync_ctl; 5159 uint32_t tsauxc; 5160 5161 /* Stop the timesync system time. */ 5162 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5163 /* Reset the timesync system time value. */ 5164 switch (hw->mac.type) { 5165 case e1000_82580: 5166 case e1000_i350: 5167 case e1000_i354: 5168 case e1000_i210: 5169 case e1000_i211: 5170 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5171 /* fall-through */ 5172 case e1000_82576: 5173 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5174 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5175 break; 5176 default: 5177 /* Not supported. */ 5178 return -ENOTSUP; 5179 } 5180 5181 /* Enable system time for it isn't on by default. */ 5182 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5183 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5184 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5185 5186 igb_start_timecounters(dev); 5187 5188 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5189 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5190 (RTE_ETHER_TYPE_1588 | 5191 E1000_ETQF_FILTER_ENABLE | 5192 E1000_ETQF_1588)); 5193 5194 /* Enable timestamping of received PTP packets. */ 5195 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5196 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5197 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5198 5199 /* Enable Timestamping of transmitted PTP packets. */ 5200 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5201 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5202 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5203 5204 return 0; 5205 } 5206 5207 static int 5208 igb_timesync_disable(struct rte_eth_dev *dev) 5209 { 5210 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5211 uint32_t tsync_ctl; 5212 5213 /* Disable timestamping of transmitted PTP packets. */ 5214 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5215 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5216 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5217 5218 /* Disable timestamping of received PTP packets. */ 5219 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5220 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5221 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5222 5223 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5224 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5225 5226 /* Stop incrementating the System Time registers. */ 5227 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5228 5229 return 0; 5230 } 5231 5232 static int 5233 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5234 struct timespec *timestamp, 5235 uint32_t flags __rte_unused) 5236 { 5237 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5238 struct e1000_adapter *adapter = dev->data->dev_private; 5239 uint32_t tsync_rxctl; 5240 uint64_t rx_tstamp_cycles; 5241 uint64_t ns; 5242 5243 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5244 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5245 return -EINVAL; 5246 5247 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5248 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5249 *timestamp = rte_ns_to_timespec(ns); 5250 5251 return 0; 5252 } 5253 5254 static int 5255 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5256 struct timespec *timestamp) 5257 { 5258 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5259 struct e1000_adapter *adapter = dev->data->dev_private; 5260 uint32_t tsync_txctl; 5261 uint64_t tx_tstamp_cycles; 5262 uint64_t ns; 5263 5264 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5265 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5266 return -EINVAL; 5267 5268 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5269 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5270 *timestamp = rte_ns_to_timespec(ns); 5271 5272 return 0; 5273 } 5274 5275 static int 5276 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5277 { 5278 int count = 0; 5279 int g_ind = 0; 5280 const struct reg_info *reg_group; 5281 5282 while ((reg_group = igb_regs[g_ind++])) 5283 count += igb_reg_group_count(reg_group); 5284 5285 return count; 5286 } 5287 5288 static int 5289 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5290 { 5291 int count = 0; 5292 int g_ind = 0; 5293 const struct reg_info *reg_group; 5294 5295 while ((reg_group = igbvf_regs[g_ind++])) 5296 count += igb_reg_group_count(reg_group); 5297 5298 return count; 5299 } 5300 5301 static int 5302 eth_igb_get_regs(struct rte_eth_dev *dev, 5303 struct rte_dev_reg_info *regs) 5304 { 5305 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5306 uint32_t *data = regs->data; 5307 int g_ind = 0; 5308 int count = 0; 5309 const struct reg_info *reg_group; 5310 5311 if (data == NULL) { 5312 regs->length = eth_igb_get_reg_length(dev); 5313 regs->width = sizeof(uint32_t); 5314 return 0; 5315 } 5316 5317 /* Support only full register dump */ 5318 if ((regs->length == 0) || 5319 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5320 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5321 hw->device_id; 5322 while ((reg_group = igb_regs[g_ind++])) 5323 count += igb_read_regs_group(dev, &data[count], 5324 reg_group); 5325 return 0; 5326 } 5327 5328 return -ENOTSUP; 5329 } 5330 5331 static int 5332 igbvf_get_regs(struct rte_eth_dev *dev, 5333 struct rte_dev_reg_info *regs) 5334 { 5335 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5336 uint32_t *data = regs->data; 5337 int g_ind = 0; 5338 int count = 0; 5339 const struct reg_info *reg_group; 5340 5341 if (data == NULL) { 5342 regs->length = igbvf_get_reg_length(dev); 5343 regs->width = sizeof(uint32_t); 5344 return 0; 5345 } 5346 5347 /* Support only full register dump */ 5348 if ((regs->length == 0) || 5349 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5350 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5351 hw->device_id; 5352 while ((reg_group = igbvf_regs[g_ind++])) 5353 count += igb_read_regs_group(dev, &data[count], 5354 reg_group); 5355 return 0; 5356 } 5357 5358 return -ENOTSUP; 5359 } 5360 5361 static int 5362 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5363 { 5364 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5365 5366 /* Return unit is byte count */ 5367 return hw->nvm.word_size * 2; 5368 } 5369 5370 static int 5371 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5372 struct rte_dev_eeprom_info *in_eeprom) 5373 { 5374 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5375 struct e1000_nvm_info *nvm = &hw->nvm; 5376 uint16_t *data = in_eeprom->data; 5377 int first, length; 5378 5379 first = in_eeprom->offset >> 1; 5380 length = in_eeprom->length >> 1; 5381 if ((first >= hw->nvm.word_size) || 5382 ((first + length) >= hw->nvm.word_size)) 5383 return -EINVAL; 5384 5385 in_eeprom->magic = hw->vendor_id | 5386 ((uint32_t)hw->device_id << 16); 5387 5388 if ((nvm->ops.read) == NULL) 5389 return -ENOTSUP; 5390 5391 return nvm->ops.read(hw, first, length, data); 5392 } 5393 5394 static int 5395 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5396 struct rte_dev_eeprom_info *in_eeprom) 5397 { 5398 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5399 struct e1000_nvm_info *nvm = &hw->nvm; 5400 uint16_t *data = in_eeprom->data; 5401 int first, length; 5402 5403 first = in_eeprom->offset >> 1; 5404 length = in_eeprom->length >> 1; 5405 if ((first >= hw->nvm.word_size) || 5406 ((first + length) >= hw->nvm.word_size)) 5407 return -EINVAL; 5408 5409 in_eeprom->magic = (uint32_t)hw->vendor_id | 5410 ((uint32_t)hw->device_id << 16); 5411 5412 if ((nvm->ops.write) == NULL) 5413 return -ENOTSUP; 5414 return nvm->ops.write(hw, first, length, data); 5415 } 5416 5417 static int 5418 eth_igb_get_module_info(struct rte_eth_dev *dev, 5419 struct rte_eth_dev_module_info *modinfo) 5420 { 5421 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5422 5423 uint32_t status = 0; 5424 uint16_t sff8472_rev, addr_mode; 5425 bool page_swap = false; 5426 5427 if (hw->phy.media_type == e1000_media_type_copper || 5428 hw->phy.media_type == e1000_media_type_unknown) 5429 return -EOPNOTSUPP; 5430 5431 /* Check whether we support SFF-8472 or not */ 5432 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5433 if (status) 5434 return -EIO; 5435 5436 /* addressing mode is not supported */ 5437 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5438 if (status) 5439 return -EIO; 5440 5441 /* addressing mode is not supported */ 5442 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5443 PMD_DRV_LOG(ERR, 5444 "Address change required to access page 0xA2, " 5445 "but not supported. Please report the module " 5446 "type to the driver maintainers.\n"); 5447 page_swap = true; 5448 } 5449 5450 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5451 /* We have an SFP, but it does not support SFF-8472 */ 5452 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5453 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5454 } else { 5455 /* We have an SFP which supports a revision of SFF-8472 */ 5456 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5457 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5458 } 5459 5460 return 0; 5461 } 5462 5463 static int 5464 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5465 struct rte_dev_eeprom_info *info) 5466 { 5467 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5468 5469 uint32_t status = 0; 5470 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5471 u16 first_word, last_word; 5472 int i = 0; 5473 5474 if (info->length == 0) 5475 return -EINVAL; 5476 5477 first_word = info->offset >> 1; 5478 last_word = (info->offset + info->length - 1) >> 1; 5479 5480 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5481 for (i = 0; i < last_word - first_word + 1; i++) { 5482 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5483 &dataword[i]); 5484 if (status) { 5485 /* Error occurred while reading module */ 5486 return -EIO; 5487 } 5488 5489 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5490 } 5491 5492 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5493 5494 return 0; 5495 } 5496 5497 static int 5498 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5499 { 5500 struct e1000_hw *hw = 5501 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5502 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5503 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5504 uint32_t vec = E1000_MISC_VEC_ID; 5505 5506 if (rte_intr_allow_others(intr_handle)) 5507 vec = E1000_RX_VEC_START; 5508 5509 uint32_t mask = 1 << (queue_id + vec); 5510 5511 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5512 E1000_WRITE_FLUSH(hw); 5513 5514 return 0; 5515 } 5516 5517 static int 5518 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5519 { 5520 struct e1000_hw *hw = 5521 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5522 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5523 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5524 uint32_t vec = E1000_MISC_VEC_ID; 5525 5526 if (rte_intr_allow_others(intr_handle)) 5527 vec = E1000_RX_VEC_START; 5528 5529 uint32_t mask = 1 << (queue_id + vec); 5530 uint32_t regval; 5531 5532 regval = E1000_READ_REG(hw, E1000_EIMS); 5533 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5534 E1000_WRITE_FLUSH(hw); 5535 5536 rte_intr_ack(intr_handle); 5537 5538 return 0; 5539 } 5540 5541 static void 5542 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5543 uint8_t index, uint8_t offset) 5544 { 5545 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5546 5547 /* clear bits */ 5548 val &= ~((uint32_t)0xFF << offset); 5549 5550 /* write vector and valid bit */ 5551 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5552 5553 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5554 } 5555 5556 static void 5557 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5558 uint8_t queue, uint8_t msix_vector) 5559 { 5560 uint32_t tmp = 0; 5561 5562 if (hw->mac.type == e1000_82575) { 5563 if (direction == 0) 5564 tmp = E1000_EICR_RX_QUEUE0 << queue; 5565 else if (direction == 1) 5566 tmp = E1000_EICR_TX_QUEUE0 << queue; 5567 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5568 } else if (hw->mac.type == e1000_82576) { 5569 if ((direction == 0) || (direction == 1)) 5570 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5571 ((queue & 0x8) << 1) + 5572 8 * direction); 5573 } else if ((hw->mac.type == e1000_82580) || 5574 (hw->mac.type == e1000_i350) || 5575 (hw->mac.type == e1000_i354) || 5576 (hw->mac.type == e1000_i210) || 5577 (hw->mac.type == e1000_i211)) { 5578 if ((direction == 0) || (direction == 1)) 5579 eth_igb_write_ivar(hw, msix_vector, 5580 queue >> 1, 5581 ((queue & 0x1) << 4) + 5582 8 * direction); 5583 } 5584 } 5585 5586 /* Sets up the hardware to generate MSI-X interrupts properly 5587 * @hw 5588 * board private structure 5589 */ 5590 static void 5591 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5592 { 5593 int queue_id; 5594 uint32_t tmpval, regval, intr_mask; 5595 struct e1000_hw *hw = 5596 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5597 uint32_t vec = E1000_MISC_VEC_ID; 5598 uint32_t base = E1000_MISC_VEC_ID; 5599 uint32_t misc_shift = 0; 5600 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5601 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5602 5603 /* won't configure msix register if no mapping is done 5604 * between intr vector and event fd 5605 */ 5606 if (!rte_intr_dp_is_en(intr_handle)) 5607 return; 5608 5609 if (rte_intr_allow_others(intr_handle)) { 5610 vec = base = E1000_RX_VEC_START; 5611 misc_shift = 1; 5612 } 5613 5614 /* set interrupt vector for other causes */ 5615 if (hw->mac.type == e1000_82575) { 5616 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5617 /* enable MSI-X PBA support */ 5618 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5619 5620 /* Auto-Mask interrupts upon ICR read */ 5621 tmpval |= E1000_CTRL_EXT_EIAME; 5622 tmpval |= E1000_CTRL_EXT_IRCA; 5623 5624 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5625 5626 /* enable msix_other interrupt */ 5627 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5628 regval = E1000_READ_REG(hw, E1000_EIAC); 5629 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5630 regval = E1000_READ_REG(hw, E1000_EIAM); 5631 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5632 } else if ((hw->mac.type == e1000_82576) || 5633 (hw->mac.type == e1000_82580) || 5634 (hw->mac.type == e1000_i350) || 5635 (hw->mac.type == e1000_i354) || 5636 (hw->mac.type == e1000_i210) || 5637 (hw->mac.type == e1000_i211)) { 5638 /* turn on MSI-X capability first */ 5639 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5640 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5641 E1000_GPIE_NSICR); 5642 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5643 misc_shift; 5644 5645 if (dev->data->dev_conf.intr_conf.lsc != 0) 5646 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5647 5648 regval = E1000_READ_REG(hw, E1000_EIAC); 5649 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5650 5651 /* enable msix_other interrupt */ 5652 regval = E1000_READ_REG(hw, E1000_EIMS); 5653 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5654 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5655 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5656 } 5657 5658 /* use EIAM to auto-mask when MSI-X interrupt 5659 * is asserted, this saves a register write for every interrupt 5660 */ 5661 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5662 misc_shift; 5663 5664 if (dev->data->dev_conf.intr_conf.lsc != 0) 5665 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5666 5667 regval = E1000_READ_REG(hw, E1000_EIAM); 5668 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5669 5670 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5671 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5672 intr_handle->intr_vec[queue_id] = vec; 5673 if (vec < base + intr_handle->nb_efd - 1) 5674 vec++; 5675 } 5676 5677 E1000_WRITE_FLUSH(hw); 5678 } 5679 5680 /* restore n-tuple filter */ 5681 static inline void 5682 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5683 { 5684 struct e1000_filter_info *filter_info = 5685 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5686 struct e1000_5tuple_filter *p_5tuple; 5687 struct e1000_2tuple_filter *p_2tuple; 5688 5689 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5690 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5691 } 5692 5693 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5694 igb_inject_2uple_filter(dev, p_2tuple); 5695 } 5696 } 5697 5698 /* restore SYN filter */ 5699 static inline void 5700 igb_syn_filter_restore(struct rte_eth_dev *dev) 5701 { 5702 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5703 struct e1000_filter_info *filter_info = 5704 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5705 uint32_t synqf; 5706 5707 synqf = filter_info->syn_info; 5708 5709 if (synqf & E1000_SYN_FILTER_ENABLE) { 5710 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5711 E1000_WRITE_FLUSH(hw); 5712 } 5713 } 5714 5715 /* restore ethernet type filter */ 5716 static inline void 5717 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5718 { 5719 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5720 struct e1000_filter_info *filter_info = 5721 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5722 int i; 5723 5724 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5725 if (filter_info->ethertype_mask & (1 << i)) { 5726 E1000_WRITE_REG(hw, E1000_ETQF(i), 5727 filter_info->ethertype_filters[i].etqf); 5728 E1000_WRITE_FLUSH(hw); 5729 } 5730 } 5731 } 5732 5733 /* restore flex byte filter */ 5734 static inline void 5735 igb_flex_filter_restore(struct rte_eth_dev *dev) 5736 { 5737 struct e1000_filter_info *filter_info = 5738 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5739 struct e1000_flex_filter *flex_filter; 5740 5741 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5742 igb_inject_flex_filter(dev, flex_filter); 5743 } 5744 } 5745 5746 /* restore rss filter */ 5747 static inline void 5748 igb_rss_filter_restore(struct rte_eth_dev *dev) 5749 { 5750 struct e1000_filter_info *filter_info = 5751 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5752 5753 if (filter_info->rss_info.conf.queue_num) 5754 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5755 } 5756 5757 /* restore all types filter */ 5758 static int 5759 igb_filter_restore(struct rte_eth_dev *dev) 5760 { 5761 igb_ntuple_filter_restore(dev); 5762 igb_ethertype_filter_restore(dev); 5763 igb_syn_filter_restore(dev); 5764 igb_flex_filter_restore(dev); 5765 igb_rss_filter_restore(dev); 5766 5767 return 0; 5768 } 5769 5770 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5771 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5772 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5773 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5774 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5775 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5776 5777 /* see e1000_logs.c */ 5778 RTE_INIT(e1000_init_log) 5779 { 5780 e1000_igb_init_log(); 5781 } 5782