1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static void eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static void eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static int eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static int eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static int eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static int eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 100 unsigned int limit); 101 static int eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static int eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static int eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static int eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct rte_ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct rte_ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static void igbvf_dev_stop(struct rte_eth_dev *dev); 158 static void igbvf_dev_close(struct rte_eth_dev *dev); 159 static int igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static int igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static int igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static int igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static int eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct rte_ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 190 struct rte_eth_syn_filter *filter); 191 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 192 enum rte_filter_op filter_op, 193 void *arg); 194 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 195 struct rte_eth_ntuple_filter *ntuple_filter); 196 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 197 struct rte_eth_ntuple_filter *ntuple_filter); 198 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 199 struct rte_eth_flex_filter *filter); 200 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 201 enum rte_filter_op filter_op, 202 void *arg); 203 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *ntuple_filter); 205 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 206 struct rte_eth_ntuple_filter *ntuple_filter); 207 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 208 struct rte_eth_ntuple_filter *filter); 209 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 210 enum rte_filter_op filter_op, 211 void *arg); 212 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 213 enum rte_filter_op filter_op, 214 void *arg); 215 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 216 struct rte_eth_ethertype_filter *filter); 217 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 218 enum rte_filter_type filter_type, 219 enum rte_filter_op filter_op, 220 void *arg); 221 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 222 static int eth_igb_get_regs(struct rte_eth_dev *dev, 223 struct rte_dev_reg_info *regs); 224 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 225 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 226 struct rte_dev_eeprom_info *eeprom); 227 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 228 struct rte_dev_eeprom_info *eeprom); 229 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 230 struct rte_eth_dev_module_info *modinfo); 231 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 232 struct rte_dev_eeprom_info *info); 233 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 234 struct rte_ether_addr *mc_addr_set, 235 uint32_t nb_mc_addr); 236 static int igb_timesync_enable(struct rte_eth_dev *dev); 237 static int igb_timesync_disable(struct rte_eth_dev *dev); 238 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 239 struct timespec *timestamp, 240 uint32_t flags); 241 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 242 struct timespec *timestamp); 243 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 244 static int igb_timesync_read_time(struct rte_eth_dev *dev, 245 struct timespec *timestamp); 246 static int igb_timesync_write_time(struct rte_eth_dev *dev, 247 const struct timespec *timestamp); 248 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 249 uint16_t queue_id); 250 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 251 uint16_t queue_id); 252 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 253 uint8_t queue, uint8_t msix_vector); 254 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 255 uint8_t index, uint8_t offset); 256 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 257 static void eth_igbvf_interrupt_handler(void *param); 258 static void igbvf_mbx_process(struct rte_eth_dev *dev); 259 static int igb_filter_restore(struct rte_eth_dev *dev); 260 261 /* 262 * Define VF Stats MACRO for Non "cleared on read" register 263 */ 264 #define UPDATE_VF_STAT(reg, last, cur) \ 265 { \ 266 u32 latest = E1000_READ_REG(hw, reg); \ 267 cur += (latest - last) & UINT_MAX; \ 268 last = latest; \ 269 } 270 271 #define IGB_FC_PAUSE_TIME 0x0680 272 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 273 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 274 275 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 276 277 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 278 279 /* 280 * The set of PCI devices this driver supports 281 */ 282 static const struct rte_pci_id pci_id_igb_map[] = { 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 291 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 295 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 302 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 320 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 321 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 322 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 323 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 324 { .vendor_id = 0, /* sentinel */ }, 325 }; 326 327 /* 328 * The set of PCI devices this driver supports (for 82576&I350 VF) 329 */ 330 static const struct rte_pci_id pci_id_igbvf_map[] = { 331 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 332 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 333 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 334 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 335 { .vendor_id = 0, /* sentinel */ }, 336 }; 337 338 static const struct rte_eth_desc_lim rx_desc_lim = { 339 .nb_max = E1000_MAX_RING_DESC, 340 .nb_min = E1000_MIN_RING_DESC, 341 .nb_align = IGB_RXD_ALIGN, 342 }; 343 344 static const struct rte_eth_desc_lim tx_desc_lim = { 345 .nb_max = E1000_MAX_RING_DESC, 346 .nb_min = E1000_MIN_RING_DESC, 347 .nb_align = IGB_RXD_ALIGN, 348 .nb_seg_max = IGB_TX_MAX_SEG, 349 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 350 }; 351 352 static const struct eth_dev_ops eth_igb_ops = { 353 .dev_configure = eth_igb_configure, 354 .dev_start = eth_igb_start, 355 .dev_stop = eth_igb_stop, 356 .dev_set_link_up = eth_igb_dev_set_link_up, 357 .dev_set_link_down = eth_igb_dev_set_link_down, 358 .dev_close = eth_igb_close, 359 .dev_reset = eth_igb_reset, 360 .promiscuous_enable = eth_igb_promiscuous_enable, 361 .promiscuous_disable = eth_igb_promiscuous_disable, 362 .allmulticast_enable = eth_igb_allmulticast_enable, 363 .allmulticast_disable = eth_igb_allmulticast_disable, 364 .link_update = eth_igb_link_update, 365 .stats_get = eth_igb_stats_get, 366 .xstats_get = eth_igb_xstats_get, 367 .xstats_get_by_id = eth_igb_xstats_get_by_id, 368 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 369 .xstats_get_names = eth_igb_xstats_get_names, 370 .stats_reset = eth_igb_stats_reset, 371 .xstats_reset = eth_igb_xstats_reset, 372 .fw_version_get = eth_igb_fw_version_get, 373 .dev_infos_get = eth_igb_infos_get, 374 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 375 .mtu_set = eth_igb_mtu_set, 376 .vlan_filter_set = eth_igb_vlan_filter_set, 377 .vlan_tpid_set = eth_igb_vlan_tpid_set, 378 .vlan_offload_set = eth_igb_vlan_offload_set, 379 .rx_queue_setup = eth_igb_rx_queue_setup, 380 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 381 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 382 .rx_queue_release = eth_igb_rx_queue_release, 383 .tx_queue_setup = eth_igb_tx_queue_setup, 384 .tx_queue_release = eth_igb_tx_queue_release, 385 .tx_done_cleanup = eth_igb_tx_done_cleanup, 386 .dev_led_on = eth_igb_led_on, 387 .dev_led_off = eth_igb_led_off, 388 .flow_ctrl_get = eth_igb_flow_ctrl_get, 389 .flow_ctrl_set = eth_igb_flow_ctrl_set, 390 .mac_addr_add = eth_igb_rar_set, 391 .mac_addr_remove = eth_igb_rar_clear, 392 .mac_addr_set = eth_igb_default_mac_addr_set, 393 .reta_update = eth_igb_rss_reta_update, 394 .reta_query = eth_igb_rss_reta_query, 395 .rss_hash_update = eth_igb_rss_hash_update, 396 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 397 .filter_ctrl = eth_igb_filter_ctrl, 398 .set_mc_addr_list = eth_igb_set_mc_addr_list, 399 .rxq_info_get = igb_rxq_info_get, 400 .txq_info_get = igb_txq_info_get, 401 .timesync_enable = igb_timesync_enable, 402 .timesync_disable = igb_timesync_disable, 403 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 404 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 405 .get_reg = eth_igb_get_regs, 406 .get_eeprom_length = eth_igb_get_eeprom_length, 407 .get_eeprom = eth_igb_get_eeprom, 408 .set_eeprom = eth_igb_set_eeprom, 409 .get_module_info = eth_igb_get_module_info, 410 .get_module_eeprom = eth_igb_get_module_eeprom, 411 .timesync_adjust_time = igb_timesync_adjust_time, 412 .timesync_read_time = igb_timesync_read_time, 413 .timesync_write_time = igb_timesync_write_time, 414 }; 415 416 /* 417 * dev_ops for virtual function, bare necessities for basic vf 418 * operation have been implemented 419 */ 420 static const struct eth_dev_ops igbvf_eth_dev_ops = { 421 .dev_configure = igbvf_dev_configure, 422 .dev_start = igbvf_dev_start, 423 .dev_stop = igbvf_dev_stop, 424 .dev_close = igbvf_dev_close, 425 .promiscuous_enable = igbvf_promiscuous_enable, 426 .promiscuous_disable = igbvf_promiscuous_disable, 427 .allmulticast_enable = igbvf_allmulticast_enable, 428 .allmulticast_disable = igbvf_allmulticast_disable, 429 .link_update = eth_igb_link_update, 430 .stats_get = eth_igbvf_stats_get, 431 .xstats_get = eth_igbvf_xstats_get, 432 .xstats_get_names = eth_igbvf_xstats_get_names, 433 .stats_reset = eth_igbvf_stats_reset, 434 .xstats_reset = eth_igbvf_stats_reset, 435 .vlan_filter_set = igbvf_vlan_filter_set, 436 .dev_infos_get = eth_igbvf_infos_get, 437 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 438 .rx_queue_setup = eth_igb_rx_queue_setup, 439 .rx_queue_release = eth_igb_rx_queue_release, 440 .tx_queue_setup = eth_igb_tx_queue_setup, 441 .tx_queue_release = eth_igb_tx_queue_release, 442 .tx_done_cleanup = eth_igb_tx_done_cleanup, 443 .set_mc_addr_list = eth_igb_set_mc_addr_list, 444 .rxq_info_get = igb_rxq_info_get, 445 .txq_info_get = igb_txq_info_get, 446 .mac_addr_set = igbvf_default_mac_addr_set, 447 .get_reg = igbvf_get_regs, 448 }; 449 450 /* store statistics names and its offset in stats structure */ 451 struct rte_igb_xstats_name_off { 452 char name[RTE_ETH_XSTATS_NAME_SIZE]; 453 unsigned offset; 454 }; 455 456 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 457 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 458 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 459 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 460 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 461 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 462 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 463 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 464 ecol)}, 465 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 466 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 467 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 468 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 469 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 470 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 471 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 472 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 473 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 474 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 475 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 476 fcruc)}, 477 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 478 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 479 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 480 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 481 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 482 prc1023)}, 483 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 484 prc1522)}, 485 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 486 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 487 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 488 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 489 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 490 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 491 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 492 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 493 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 494 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 495 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 496 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 497 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 498 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 499 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 500 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 501 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 502 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 503 ptc1023)}, 504 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 505 ptc1522)}, 506 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 507 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 508 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 509 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 510 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 511 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 512 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 513 514 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 515 }; 516 517 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 518 sizeof(rte_igb_stats_strings[0])) 519 520 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 521 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 522 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 523 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 524 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 525 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 526 }; 527 528 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 529 sizeof(rte_igbvf_stats_strings[0])) 530 531 532 static inline void 533 igb_intr_enable(struct rte_eth_dev *dev) 534 { 535 struct e1000_interrupt *intr = 536 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 537 struct e1000_hw *hw = 538 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 539 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 540 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 541 542 if (rte_intr_allow_others(intr_handle) && 543 dev->data->dev_conf.intr_conf.lsc != 0) { 544 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 545 } 546 547 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 548 E1000_WRITE_FLUSH(hw); 549 } 550 551 static void 552 igb_intr_disable(struct rte_eth_dev *dev) 553 { 554 struct e1000_hw *hw = 555 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 556 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 557 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 558 559 if (rte_intr_allow_others(intr_handle) && 560 dev->data->dev_conf.intr_conf.lsc != 0) { 561 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 562 } 563 564 E1000_WRITE_REG(hw, E1000_IMC, ~0); 565 E1000_WRITE_FLUSH(hw); 566 } 567 568 static inline void 569 igbvf_intr_enable(struct rte_eth_dev *dev) 570 { 571 struct e1000_hw *hw = 572 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 573 574 /* only for mailbox */ 575 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 576 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 577 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 578 E1000_WRITE_FLUSH(hw); 579 } 580 581 /* only for mailbox now. If RX/TX needed, should extend this function. */ 582 static void 583 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 584 { 585 uint32_t tmp = 0; 586 587 /* mailbox */ 588 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 589 tmp |= E1000_VTIVAR_VALID; 590 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 591 } 592 593 static void 594 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 595 { 596 struct e1000_hw *hw = 597 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 598 599 /* Configure VF other cause ivar */ 600 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 601 } 602 603 static inline int32_t 604 igb_pf_reset_hw(struct e1000_hw *hw) 605 { 606 uint32_t ctrl_ext; 607 int32_t status; 608 609 status = e1000_reset_hw(hw); 610 611 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 612 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 613 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 614 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 615 E1000_WRITE_FLUSH(hw); 616 617 return status; 618 } 619 620 static void 621 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 622 { 623 struct e1000_hw *hw = 624 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 625 626 627 hw->vendor_id = pci_dev->id.vendor_id; 628 hw->device_id = pci_dev->id.device_id; 629 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 630 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 631 632 e1000_set_mac_type(hw); 633 634 /* need to check if it is a vf device below */ 635 } 636 637 static int 638 igb_reset_swfw_lock(struct e1000_hw *hw) 639 { 640 int ret_val; 641 642 /* 643 * Do mac ops initialization manually here, since we will need 644 * some function pointers set by this call. 645 */ 646 ret_val = e1000_init_mac_params(hw); 647 if (ret_val) 648 return ret_val; 649 650 /* 651 * SMBI lock should not fail in this early stage. If this is the case, 652 * it is due to an improper exit of the application. 653 * So force the release of the faulty lock. 654 */ 655 if (e1000_get_hw_semaphore_generic(hw) < 0) { 656 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 657 } 658 e1000_put_hw_semaphore_generic(hw); 659 660 if (hw->mac.ops.acquire_swfw_sync != NULL) { 661 uint16_t mask; 662 663 /* 664 * Phy lock should not fail in this early stage. If this is the case, 665 * it is due to an improper exit of the application. 666 * So force the release of the faulty lock. 667 */ 668 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 669 if (hw->bus.func > E1000_FUNC_1) 670 mask <<= 2; 671 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 672 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 673 hw->bus.func); 674 } 675 hw->mac.ops.release_swfw_sync(hw, mask); 676 677 /* 678 * This one is more tricky since it is common to all ports; but 679 * swfw_sync retries last long enough (1s) to be almost sure that if 680 * lock can not be taken it is due to an improper lock of the 681 * semaphore. 682 */ 683 mask = E1000_SWFW_EEP_SM; 684 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 685 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 686 } 687 hw->mac.ops.release_swfw_sync(hw, mask); 688 } 689 690 return E1000_SUCCESS; 691 } 692 693 /* Remove all ntuple filters of the device */ 694 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 695 { 696 struct e1000_filter_info *filter_info = 697 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 698 struct e1000_5tuple_filter *p_5tuple; 699 struct e1000_2tuple_filter *p_2tuple; 700 701 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 702 TAILQ_REMOVE(&filter_info->fivetuple_list, 703 p_5tuple, entries); 704 rte_free(p_5tuple); 705 } 706 filter_info->fivetuple_mask = 0; 707 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 708 TAILQ_REMOVE(&filter_info->twotuple_list, 709 p_2tuple, entries); 710 rte_free(p_2tuple); 711 } 712 filter_info->twotuple_mask = 0; 713 714 return 0; 715 } 716 717 /* Remove all flex filters of the device */ 718 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 719 { 720 struct e1000_filter_info *filter_info = 721 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 722 struct e1000_flex_filter *p_flex; 723 724 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 725 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 726 rte_free(p_flex); 727 } 728 filter_info->flex_mask = 0; 729 730 return 0; 731 } 732 733 static int 734 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 735 { 736 int error = 0; 737 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 738 struct e1000_hw *hw = 739 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 740 struct e1000_vfta * shadow_vfta = 741 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 742 struct e1000_filter_info *filter_info = 743 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 744 struct e1000_adapter *adapter = 745 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 746 747 uint32_t ctrl_ext; 748 749 eth_dev->dev_ops = ð_igb_ops; 750 eth_dev->rx_queue_count = eth_igb_rx_queue_count; 751 eth_dev->rx_descriptor_done = eth_igb_rx_descriptor_done; 752 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 753 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 754 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 755 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 756 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 757 758 /* for secondary processes, we don't initialise any further as primary 759 * has already done this work. Only check we don't need a different 760 * RX function */ 761 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 762 if (eth_dev->data->scattered_rx) 763 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 764 return 0; 765 } 766 767 rte_eth_copy_pci_info(eth_dev, pci_dev); 768 769 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 770 771 igb_identify_hardware(eth_dev, pci_dev); 772 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 773 error = -EIO; 774 goto err_late; 775 } 776 777 e1000_get_bus_info(hw); 778 779 /* Reset any pending lock */ 780 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 781 error = -EIO; 782 goto err_late; 783 } 784 785 /* Finish initialization */ 786 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 787 error = -EIO; 788 goto err_late; 789 } 790 791 hw->mac.autoneg = 1; 792 hw->phy.autoneg_wait_to_complete = 0; 793 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 794 795 /* Copper options */ 796 if (hw->phy.media_type == e1000_media_type_copper) { 797 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 798 hw->phy.disable_polarity_correction = 0; 799 hw->phy.ms_type = e1000_ms_hw_default; 800 } 801 802 /* 803 * Start from a known state, this is important in reading the nvm 804 * and mac from that. 805 */ 806 igb_pf_reset_hw(hw); 807 808 /* Make sure we have a good EEPROM before we read from it */ 809 if (e1000_validate_nvm_checksum(hw) < 0) { 810 /* 811 * Some PCI-E parts fail the first check due to 812 * the link being in sleep state, call it again, 813 * if it fails a second time its a real issue. 814 */ 815 if (e1000_validate_nvm_checksum(hw) < 0) { 816 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 817 error = -EIO; 818 goto err_late; 819 } 820 } 821 822 /* Read the permanent MAC address out of the EEPROM */ 823 if (e1000_read_mac_addr(hw) != 0) { 824 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 825 error = -EIO; 826 goto err_late; 827 } 828 829 /* Allocate memory for storing MAC addresses */ 830 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 831 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 832 if (eth_dev->data->mac_addrs == NULL) { 833 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 834 "store MAC addresses", 835 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 836 error = -ENOMEM; 837 goto err_late; 838 } 839 840 /* Copy the permanent MAC address */ 841 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 842 ð_dev->data->mac_addrs[0]); 843 844 /* Pass the information to the rte_eth_dev_close() that it should also 845 * release the private port resources. 846 */ 847 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 848 849 /* initialize the vfta */ 850 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 851 852 /* Now initialize the hardware */ 853 if (igb_hardware_init(hw) != 0) { 854 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 855 rte_free(eth_dev->data->mac_addrs); 856 eth_dev->data->mac_addrs = NULL; 857 error = -ENODEV; 858 goto err_late; 859 } 860 hw->mac.get_link_status = 1; 861 adapter->stopped = 0; 862 863 /* Indicate SOL/IDER usage */ 864 if (e1000_check_reset_block(hw) < 0) { 865 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 866 "SOL/IDER session"); 867 } 868 869 /* initialize PF if max_vfs not zero */ 870 igb_pf_host_init(eth_dev); 871 872 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 873 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 874 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 875 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 876 E1000_WRITE_FLUSH(hw); 877 878 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 879 eth_dev->data->port_id, pci_dev->id.vendor_id, 880 pci_dev->id.device_id); 881 882 rte_intr_callback_register(&pci_dev->intr_handle, 883 eth_igb_interrupt_handler, 884 (void *)eth_dev); 885 886 /* enable uio/vfio intr/eventfd mapping */ 887 rte_intr_enable(&pci_dev->intr_handle); 888 889 /* enable support intr */ 890 igb_intr_enable(eth_dev); 891 892 eth_igb_dev_set_link_down(eth_dev); 893 894 /* initialize filter info */ 895 memset(filter_info, 0, 896 sizeof(struct e1000_filter_info)); 897 898 TAILQ_INIT(&filter_info->flex_list); 899 TAILQ_INIT(&filter_info->twotuple_list); 900 TAILQ_INIT(&filter_info->fivetuple_list); 901 902 TAILQ_INIT(&igb_filter_ntuple_list); 903 TAILQ_INIT(&igb_filter_ethertype_list); 904 TAILQ_INIT(&igb_filter_syn_list); 905 TAILQ_INIT(&igb_filter_flex_list); 906 TAILQ_INIT(&igb_filter_rss_list); 907 TAILQ_INIT(&igb_flow_list); 908 909 return 0; 910 911 err_late: 912 igb_hw_control_release(hw); 913 914 return error; 915 } 916 917 static int 918 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 919 { 920 PMD_INIT_FUNC_TRACE(); 921 922 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 923 return 0; 924 925 eth_igb_close(eth_dev); 926 927 return 0; 928 } 929 930 /* 931 * Virtual Function device init 932 */ 933 static int 934 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 935 { 936 struct rte_pci_device *pci_dev; 937 struct rte_intr_handle *intr_handle; 938 struct e1000_adapter *adapter = 939 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 940 struct e1000_hw *hw = 941 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 942 int diag; 943 struct rte_ether_addr *perm_addr = 944 (struct rte_ether_addr *)hw->mac.perm_addr; 945 946 PMD_INIT_FUNC_TRACE(); 947 948 eth_dev->dev_ops = &igbvf_eth_dev_ops; 949 eth_dev->rx_descriptor_done = eth_igb_rx_descriptor_done; 950 eth_dev->rx_descriptor_status = eth_igb_rx_descriptor_status; 951 eth_dev->tx_descriptor_status = eth_igb_tx_descriptor_status; 952 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 953 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 954 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 955 956 /* for secondary processes, we don't initialise any further as primary 957 * has already done this work. Only check we don't need a different 958 * RX function */ 959 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 960 if (eth_dev->data->scattered_rx) 961 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 962 return 0; 963 } 964 965 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 966 rte_eth_copy_pci_info(eth_dev, pci_dev); 967 968 hw->device_id = pci_dev->id.device_id; 969 hw->vendor_id = pci_dev->id.vendor_id; 970 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 971 adapter->stopped = 0; 972 973 /* Initialize the shared code (base driver) */ 974 diag = e1000_setup_init_funcs(hw, TRUE); 975 if (diag != 0) { 976 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 977 diag); 978 return -EIO; 979 } 980 981 /* init_mailbox_params */ 982 hw->mbx.ops.init_params(hw); 983 984 /* Disable the interrupts for VF */ 985 igbvf_intr_disable(hw); 986 987 diag = hw->mac.ops.reset_hw(hw); 988 989 /* Allocate memory for storing MAC addresses */ 990 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", RTE_ETHER_ADDR_LEN * 991 hw->mac.rar_entry_count, 0); 992 if (eth_dev->data->mac_addrs == NULL) { 993 PMD_INIT_LOG(ERR, 994 "Failed to allocate %d bytes needed to store MAC " 995 "addresses", 996 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 997 return -ENOMEM; 998 } 999 1000 /* Pass the information to the rte_eth_dev_close() that it should also 1001 * release the private port resources. 1002 */ 1003 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1004 1005 /* Generate a random MAC address, if none was assigned by PF. */ 1006 if (rte_is_zero_ether_addr(perm_addr)) { 1007 rte_eth_random_addr(perm_addr->addr_bytes); 1008 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1009 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1010 "%02x:%02x:%02x:%02x:%02x:%02x", 1011 perm_addr->addr_bytes[0], 1012 perm_addr->addr_bytes[1], 1013 perm_addr->addr_bytes[2], 1014 perm_addr->addr_bytes[3], 1015 perm_addr->addr_bytes[4], 1016 perm_addr->addr_bytes[5]); 1017 } 1018 1019 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1020 if (diag) { 1021 rte_free(eth_dev->data->mac_addrs); 1022 eth_dev->data->mac_addrs = NULL; 1023 return diag; 1024 } 1025 /* Copy the permanent MAC address */ 1026 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1027 ð_dev->data->mac_addrs[0]); 1028 1029 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1030 "mac.type=%s", 1031 eth_dev->data->port_id, pci_dev->id.vendor_id, 1032 pci_dev->id.device_id, "igb_mac_82576_vf"); 1033 1034 intr_handle = &pci_dev->intr_handle; 1035 rte_intr_callback_register(intr_handle, 1036 eth_igbvf_interrupt_handler, eth_dev); 1037 1038 return 0; 1039 } 1040 1041 static int 1042 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1043 { 1044 PMD_INIT_FUNC_TRACE(); 1045 1046 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1047 return 0; 1048 1049 igbvf_dev_close(eth_dev); 1050 1051 return 0; 1052 } 1053 1054 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1055 struct rte_pci_device *pci_dev) 1056 { 1057 return rte_eth_dev_pci_generic_probe(pci_dev, 1058 sizeof(struct e1000_adapter), eth_igb_dev_init); 1059 } 1060 1061 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1062 { 1063 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1064 } 1065 1066 static struct rte_pci_driver rte_igb_pmd = { 1067 .id_table = pci_id_igb_map, 1068 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1069 .probe = eth_igb_pci_probe, 1070 .remove = eth_igb_pci_remove, 1071 }; 1072 1073 1074 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1075 struct rte_pci_device *pci_dev) 1076 { 1077 return rte_eth_dev_pci_generic_probe(pci_dev, 1078 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1079 } 1080 1081 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1082 { 1083 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1084 } 1085 1086 /* 1087 * virtual function driver struct 1088 */ 1089 static struct rte_pci_driver rte_igbvf_pmd = { 1090 .id_table = pci_id_igbvf_map, 1091 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1092 .probe = eth_igbvf_pci_probe, 1093 .remove = eth_igbvf_pci_remove, 1094 }; 1095 1096 static void 1097 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1098 { 1099 struct e1000_hw *hw = 1100 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1101 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1102 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1103 rctl |= E1000_RCTL_VFE; 1104 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1105 } 1106 1107 static int 1108 igb_check_mq_mode(struct rte_eth_dev *dev) 1109 { 1110 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1111 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1112 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1113 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1114 1115 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1116 tx_mq_mode == ETH_MQ_TX_DCB || 1117 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1118 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1119 return -EINVAL; 1120 } 1121 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1122 /* Check multi-queue mode. 1123 * To no break software we accept ETH_MQ_RX_NONE as this might 1124 * be used to turn off VLAN filter. 1125 */ 1126 1127 if (rx_mq_mode == ETH_MQ_RX_NONE || 1128 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1129 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1130 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1131 } else { 1132 /* Only support one queue on VFs. 1133 * RSS together with SRIOV is not supported. 1134 */ 1135 PMD_INIT_LOG(ERR, "SRIOV is active," 1136 " wrong mq_mode rx %d.", 1137 rx_mq_mode); 1138 return -EINVAL; 1139 } 1140 /* TX mode is not used here, so mode might be ignored.*/ 1141 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1142 /* SRIOV only works in VMDq enable mode */ 1143 PMD_INIT_LOG(WARNING, "SRIOV is active," 1144 " TX mode %d is not supported. " 1145 " Driver will behave as %d mode.", 1146 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1147 } 1148 1149 /* check valid queue number */ 1150 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1151 PMD_INIT_LOG(ERR, "SRIOV is active," 1152 " only support one queue on VFs."); 1153 return -EINVAL; 1154 } 1155 } else { 1156 /* To no break software that set invalid mode, only display 1157 * warning if invalid mode is used. 1158 */ 1159 if (rx_mq_mode != ETH_MQ_RX_NONE && 1160 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1161 rx_mq_mode != ETH_MQ_RX_RSS) { 1162 /* RSS together with VMDq not supported*/ 1163 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1164 rx_mq_mode); 1165 return -EINVAL; 1166 } 1167 1168 if (tx_mq_mode != ETH_MQ_TX_NONE && 1169 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1170 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1171 " Due to txmode is meaningless in this" 1172 " driver, just ignore.", 1173 tx_mq_mode); 1174 } 1175 } 1176 return 0; 1177 } 1178 1179 static int 1180 eth_igb_configure(struct rte_eth_dev *dev) 1181 { 1182 struct e1000_interrupt *intr = 1183 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1184 int ret; 1185 1186 PMD_INIT_FUNC_TRACE(); 1187 1188 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 1189 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1190 1191 /* multipe queue mode checking */ 1192 ret = igb_check_mq_mode(dev); 1193 if (ret != 0) { 1194 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1195 ret); 1196 return ret; 1197 } 1198 1199 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1200 PMD_INIT_FUNC_TRACE(); 1201 1202 return 0; 1203 } 1204 1205 static void 1206 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1207 bool enable) 1208 { 1209 struct e1000_hw *hw = 1210 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1211 uint32_t tctl, rctl; 1212 1213 tctl = E1000_READ_REG(hw, E1000_TCTL); 1214 rctl = E1000_READ_REG(hw, E1000_RCTL); 1215 1216 if (enable) { 1217 /* enable Tx/Rx */ 1218 tctl |= E1000_TCTL_EN; 1219 rctl |= E1000_RCTL_EN; 1220 } else { 1221 /* disable Tx/Rx */ 1222 tctl &= ~E1000_TCTL_EN; 1223 rctl &= ~E1000_RCTL_EN; 1224 } 1225 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1226 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1227 E1000_WRITE_FLUSH(hw); 1228 } 1229 1230 static int 1231 eth_igb_start(struct rte_eth_dev *dev) 1232 { 1233 struct e1000_hw *hw = 1234 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1235 struct e1000_adapter *adapter = 1236 E1000_DEV_PRIVATE(dev->data->dev_private); 1237 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1238 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1239 int ret, mask; 1240 uint32_t intr_vector = 0; 1241 uint32_t ctrl_ext; 1242 uint32_t *speeds; 1243 int num_speeds; 1244 bool autoneg; 1245 1246 PMD_INIT_FUNC_TRACE(); 1247 1248 /* disable uio/vfio intr/eventfd mapping */ 1249 rte_intr_disable(intr_handle); 1250 1251 /* Power up the phy. Needed to make the link go Up */ 1252 eth_igb_dev_set_link_up(dev); 1253 1254 /* 1255 * Packet Buffer Allocation (PBA) 1256 * Writing PBA sets the receive portion of the buffer 1257 * the remainder is used for the transmit buffer. 1258 */ 1259 if (hw->mac.type == e1000_82575) { 1260 uint32_t pba; 1261 1262 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1263 E1000_WRITE_REG(hw, E1000_PBA, pba); 1264 } 1265 1266 /* Put the address into the Receive Address Array */ 1267 e1000_rar_set(hw, hw->mac.addr, 0); 1268 1269 /* Initialize the hardware */ 1270 if (igb_hardware_init(hw)) { 1271 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1272 return -EIO; 1273 } 1274 adapter->stopped = 0; 1275 1276 E1000_WRITE_REG(hw, E1000_VET, 1277 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1278 1279 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1280 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1281 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1282 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1283 E1000_WRITE_FLUSH(hw); 1284 1285 /* configure PF module if SRIOV enabled */ 1286 igb_pf_host_configure(dev); 1287 1288 /* check and configure queue intr-vector mapping */ 1289 if ((rte_intr_cap_multiple(intr_handle) || 1290 !RTE_ETH_DEV_SRIOV(dev).active) && 1291 dev->data->dev_conf.intr_conf.rxq != 0) { 1292 intr_vector = dev->data->nb_rx_queues; 1293 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1294 return -1; 1295 } 1296 1297 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1298 intr_handle->intr_vec = 1299 rte_zmalloc("intr_vec", 1300 dev->data->nb_rx_queues * sizeof(int), 0); 1301 if (intr_handle->intr_vec == NULL) { 1302 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1303 " intr_vec", dev->data->nb_rx_queues); 1304 return -ENOMEM; 1305 } 1306 } 1307 1308 /* confiugre msix for rx interrupt */ 1309 eth_igb_configure_msix_intr(dev); 1310 1311 /* Configure for OS presence */ 1312 igb_init_manageability(hw); 1313 1314 eth_igb_tx_init(dev); 1315 1316 /* This can fail when allocating mbufs for descriptor rings */ 1317 ret = eth_igb_rx_init(dev); 1318 if (ret) { 1319 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1320 igb_dev_clear_queues(dev); 1321 return ret; 1322 } 1323 1324 e1000_clear_hw_cntrs_base_generic(hw); 1325 1326 /* 1327 * VLAN Offload Settings 1328 */ 1329 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1330 ETH_VLAN_EXTEND_MASK; 1331 ret = eth_igb_vlan_offload_set(dev, mask); 1332 if (ret) { 1333 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1334 igb_dev_clear_queues(dev); 1335 return ret; 1336 } 1337 1338 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1339 /* Enable VLAN filter since VMDq always use VLAN filter */ 1340 igb_vmdq_vlan_hw_filter_enable(dev); 1341 } 1342 1343 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1344 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1345 (hw->mac.type == e1000_i211)) { 1346 /* Configure EITR with the maximum possible value (0xFFFF) */ 1347 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1348 } 1349 1350 /* Setup link speed and duplex */ 1351 speeds = &dev->data->dev_conf.link_speeds; 1352 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1353 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1354 hw->mac.autoneg = 1; 1355 } else { 1356 num_speeds = 0; 1357 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1358 1359 /* Reset */ 1360 hw->phy.autoneg_advertised = 0; 1361 1362 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1363 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1364 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1365 num_speeds = -1; 1366 goto error_invalid_config; 1367 } 1368 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1369 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1370 num_speeds++; 1371 } 1372 if (*speeds & ETH_LINK_SPEED_10M) { 1373 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1374 num_speeds++; 1375 } 1376 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1377 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1378 num_speeds++; 1379 } 1380 if (*speeds & ETH_LINK_SPEED_100M) { 1381 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1382 num_speeds++; 1383 } 1384 if (*speeds & ETH_LINK_SPEED_1G) { 1385 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1386 num_speeds++; 1387 } 1388 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1389 goto error_invalid_config; 1390 1391 /* Set/reset the mac.autoneg based on the link speed, 1392 * fixed or not 1393 */ 1394 if (!autoneg) { 1395 hw->mac.autoneg = 0; 1396 hw->mac.forced_speed_duplex = 1397 hw->phy.autoneg_advertised; 1398 } else { 1399 hw->mac.autoneg = 1; 1400 } 1401 } 1402 1403 e1000_setup_link(hw); 1404 1405 if (rte_intr_allow_others(intr_handle)) { 1406 /* check if lsc interrupt is enabled */ 1407 if (dev->data->dev_conf.intr_conf.lsc != 0) 1408 eth_igb_lsc_interrupt_setup(dev, TRUE); 1409 else 1410 eth_igb_lsc_interrupt_setup(dev, FALSE); 1411 } else { 1412 rte_intr_callback_unregister(intr_handle, 1413 eth_igb_interrupt_handler, 1414 (void *)dev); 1415 if (dev->data->dev_conf.intr_conf.lsc != 0) 1416 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1417 " no intr multiplex"); 1418 } 1419 1420 /* check if rxq interrupt is enabled */ 1421 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1422 rte_intr_dp_is_en(intr_handle)) 1423 eth_igb_rxq_interrupt_setup(dev); 1424 1425 /* enable uio/vfio intr/eventfd mapping */ 1426 rte_intr_enable(intr_handle); 1427 1428 /* resume enabled intr since hw reset */ 1429 igb_intr_enable(dev); 1430 1431 /* restore all types filter */ 1432 igb_filter_restore(dev); 1433 1434 eth_igb_rxtx_control(dev, true); 1435 eth_igb_link_update(dev, 0); 1436 1437 PMD_INIT_LOG(DEBUG, "<<"); 1438 1439 return 0; 1440 1441 error_invalid_config: 1442 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1443 dev->data->dev_conf.link_speeds, dev->data->port_id); 1444 igb_dev_clear_queues(dev); 1445 return -EINVAL; 1446 } 1447 1448 /********************************************************************* 1449 * 1450 * This routine disables all traffic on the adapter by issuing a 1451 * global reset on the MAC. 1452 * 1453 **********************************************************************/ 1454 static void 1455 eth_igb_stop(struct rte_eth_dev *dev) 1456 { 1457 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1458 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1459 struct rte_eth_link link; 1460 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1461 struct e1000_adapter *adapter = 1462 E1000_DEV_PRIVATE(dev->data->dev_private); 1463 1464 if (adapter->stopped) 1465 return; 1466 1467 eth_igb_rxtx_control(dev, false); 1468 1469 igb_intr_disable(dev); 1470 1471 /* disable intr eventfd mapping */ 1472 rte_intr_disable(intr_handle); 1473 1474 igb_pf_reset_hw(hw); 1475 E1000_WRITE_REG(hw, E1000_WUC, 0); 1476 1477 /* Set bit for Go Link disconnect if PHY reset is not blocked */ 1478 if (hw->mac.type >= e1000_82580 && 1479 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1480 uint32_t phpm_reg; 1481 1482 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1483 phpm_reg |= E1000_82580_PM_GO_LINKD; 1484 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1485 } 1486 1487 /* Power down the phy. Needed to make the link go Down */ 1488 eth_igb_dev_set_link_down(dev); 1489 1490 igb_dev_clear_queues(dev); 1491 1492 /* clear the recorded link status */ 1493 memset(&link, 0, sizeof(link)); 1494 rte_eth_linkstatus_set(dev, &link); 1495 1496 if (!rte_intr_allow_others(intr_handle)) 1497 /* resume to the default handler */ 1498 rte_intr_callback_register(intr_handle, 1499 eth_igb_interrupt_handler, 1500 (void *)dev); 1501 1502 /* Clean datapath event and queue/vec mapping */ 1503 rte_intr_efd_disable(intr_handle); 1504 if (intr_handle->intr_vec != NULL) { 1505 rte_free(intr_handle->intr_vec); 1506 intr_handle->intr_vec = NULL; 1507 } 1508 1509 adapter->stopped = true; 1510 } 1511 1512 static int 1513 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1514 { 1515 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1516 1517 if (hw->phy.media_type == e1000_media_type_copper) 1518 e1000_power_up_phy(hw); 1519 else 1520 e1000_power_up_fiber_serdes_link(hw); 1521 1522 return 0; 1523 } 1524 1525 static int 1526 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1527 { 1528 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1529 1530 if (hw->phy.media_type == e1000_media_type_copper) 1531 e1000_power_down_phy(hw); 1532 else 1533 e1000_shutdown_fiber_serdes_link(hw); 1534 1535 return 0; 1536 } 1537 1538 static void 1539 eth_igb_close(struct rte_eth_dev *dev) 1540 { 1541 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1542 struct rte_eth_link link; 1543 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1544 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1545 struct e1000_filter_info *filter_info = 1546 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 1547 1548 eth_igb_stop(dev); 1549 1550 e1000_phy_hw_reset(hw); 1551 igb_release_manageability(hw); 1552 igb_hw_control_release(hw); 1553 1554 /* Clear bit for Go Link disconnect if PHY reset is not blocked */ 1555 if (hw->mac.type >= e1000_82580 && 1556 (e1000_check_reset_block(hw) != E1000_BLK_PHY_RESET)) { 1557 uint32_t phpm_reg; 1558 1559 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1560 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1561 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1562 } 1563 1564 igb_dev_free_queues(dev); 1565 1566 if (intr_handle->intr_vec) { 1567 rte_free(intr_handle->intr_vec); 1568 intr_handle->intr_vec = NULL; 1569 } 1570 1571 memset(&link, 0, sizeof(link)); 1572 rte_eth_linkstatus_set(dev, &link); 1573 1574 dev->dev_ops = NULL; 1575 dev->rx_pkt_burst = NULL; 1576 dev->tx_pkt_burst = NULL; 1577 1578 /* Reset any pending lock */ 1579 igb_reset_swfw_lock(hw); 1580 1581 /* uninitialize PF if max_vfs not zero */ 1582 igb_pf_host_uninit(dev); 1583 1584 rte_intr_callback_unregister(intr_handle, 1585 eth_igb_interrupt_handler, dev); 1586 1587 /* clear the SYN filter info */ 1588 filter_info->syn_info = 0; 1589 1590 /* clear the ethertype filters info */ 1591 filter_info->ethertype_mask = 0; 1592 memset(filter_info->ethertype_filters, 0, 1593 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 1594 1595 /* clear the rss filter info */ 1596 memset(&filter_info->rss_info, 0, 1597 sizeof(struct igb_rte_flow_rss_conf)); 1598 1599 /* remove all ntuple filters of the device */ 1600 igb_ntuple_filter_uninit(dev); 1601 1602 /* remove all flex filters of the device */ 1603 igb_flex_filter_uninit(dev); 1604 1605 /* clear all the filters list */ 1606 igb_filterlist_flush(dev); 1607 } 1608 1609 /* 1610 * Reset PF device. 1611 */ 1612 static int 1613 eth_igb_reset(struct rte_eth_dev *dev) 1614 { 1615 int ret; 1616 1617 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1618 * its VF to make them align with it. The detailed notification 1619 * mechanism is PMD specific and is currently not implemented. 1620 * To avoid unexpected behavior in VF, currently reset of PF with 1621 * SR-IOV activation is not supported. It might be supported later. 1622 */ 1623 if (dev->data->sriov.active) 1624 return -ENOTSUP; 1625 1626 ret = eth_igb_dev_uninit(dev); 1627 if (ret) 1628 return ret; 1629 1630 ret = eth_igb_dev_init(dev); 1631 1632 return ret; 1633 } 1634 1635 1636 static int 1637 igb_get_rx_buffer_size(struct e1000_hw *hw) 1638 { 1639 uint32_t rx_buf_size; 1640 if (hw->mac.type == e1000_82576) { 1641 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1642 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1643 /* PBS needs to be translated according to a lookup table */ 1644 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1645 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1646 rx_buf_size = (rx_buf_size << 10); 1647 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1648 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1649 } else { 1650 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1651 } 1652 1653 return rx_buf_size; 1654 } 1655 1656 /********************************************************************* 1657 * 1658 * Initialize the hardware 1659 * 1660 **********************************************************************/ 1661 static int 1662 igb_hardware_init(struct e1000_hw *hw) 1663 { 1664 uint32_t rx_buf_size; 1665 int diag; 1666 1667 /* Let the firmware know the OS is in control */ 1668 igb_hw_control_acquire(hw); 1669 1670 /* 1671 * These parameters control the automatic generation (Tx) and 1672 * response (Rx) to Ethernet PAUSE frames. 1673 * - High water mark should allow for at least two standard size (1518) 1674 * frames to be received after sending an XOFF. 1675 * - Low water mark works best when it is very near the high water mark. 1676 * This allows the receiver to restart by sending XON when it has 1677 * drained a bit. Here we use an arbitrary value of 1500 which will 1678 * restart after one full frame is pulled from the buffer. There 1679 * could be several smaller frames in the buffer and if so they will 1680 * not trigger the XON until their total number reduces the buffer 1681 * by 1500. 1682 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1683 */ 1684 rx_buf_size = igb_get_rx_buffer_size(hw); 1685 1686 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2); 1687 hw->fc.low_water = hw->fc.high_water - 1500; 1688 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1689 hw->fc.send_xon = 1; 1690 1691 /* Set Flow control, use the tunable location if sane */ 1692 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1693 hw->fc.requested_mode = igb_fc_setting; 1694 else 1695 hw->fc.requested_mode = e1000_fc_none; 1696 1697 /* Issue a global reset */ 1698 igb_pf_reset_hw(hw); 1699 E1000_WRITE_REG(hw, E1000_WUC, 0); 1700 1701 diag = e1000_init_hw(hw); 1702 if (diag < 0) 1703 return diag; 1704 1705 E1000_WRITE_REG(hw, E1000_VET, 1706 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1707 e1000_get_phy_info(hw); 1708 e1000_check_for_link(hw); 1709 1710 return 0; 1711 } 1712 1713 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1714 static void 1715 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1716 { 1717 int pause_frames; 1718 1719 uint64_t old_gprc = stats->gprc; 1720 uint64_t old_gptc = stats->gptc; 1721 uint64_t old_tpr = stats->tpr; 1722 uint64_t old_tpt = stats->tpt; 1723 uint64_t old_rpthc = stats->rpthc; 1724 uint64_t old_hgptc = stats->hgptc; 1725 1726 if(hw->phy.media_type == e1000_media_type_copper || 1727 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1728 stats->symerrs += 1729 E1000_READ_REG(hw,E1000_SYMERRS); 1730 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1731 } 1732 1733 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1734 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1735 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1736 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1737 1738 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1739 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1740 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1741 stats->dc += E1000_READ_REG(hw, E1000_DC); 1742 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1743 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1744 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1745 /* 1746 ** For watchdog management we need to know if we have been 1747 ** paused during the last interval, so capture that here. 1748 */ 1749 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1750 stats->xoffrxc += pause_frames; 1751 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1752 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1753 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1754 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1755 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1756 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1757 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1758 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1759 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1760 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1761 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1762 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1763 1764 /* For the 64-bit byte counters the low dword must be read first. */ 1765 /* Both registers clear on the read of the high dword */ 1766 1767 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1768 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1769 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1770 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN; 1771 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1772 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1773 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN; 1774 1775 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1776 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1777 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1778 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1779 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1780 1781 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1782 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1783 1784 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1785 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1786 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 1787 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1788 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1789 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN; 1790 1791 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1792 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1793 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1794 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1795 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1796 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1797 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1798 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1799 1800 /* Interrupt Counts */ 1801 1802 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1803 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1804 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1805 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1806 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1807 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1808 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1809 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1810 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1811 1812 /* Host to Card Statistics */ 1813 1814 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1815 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1816 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1817 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1818 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1819 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1820 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1821 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1822 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1823 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN; 1824 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1825 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1826 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN; 1827 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1828 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1829 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1830 1831 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1832 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1833 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1834 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1835 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1836 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1837 } 1838 1839 static int 1840 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1841 { 1842 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1843 struct e1000_hw_stats *stats = 1844 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1845 1846 igb_read_stats_registers(hw, stats); 1847 1848 if (rte_stats == NULL) 1849 return -EINVAL; 1850 1851 /* Rx Errors */ 1852 rte_stats->imissed = stats->mpc; 1853 rte_stats->ierrors = stats->crcerrs + 1854 stats->rlec + stats->ruc + stats->roc + 1855 stats->rxerrc + stats->algnerrc + stats->cexterr; 1856 1857 /* Tx Errors */ 1858 rte_stats->oerrors = stats->ecol + stats->latecol; 1859 1860 rte_stats->ipackets = stats->gprc; 1861 rte_stats->opackets = stats->gptc; 1862 rte_stats->ibytes = stats->gorc; 1863 rte_stats->obytes = stats->gotc; 1864 return 0; 1865 } 1866 1867 static int 1868 eth_igb_stats_reset(struct rte_eth_dev *dev) 1869 { 1870 struct e1000_hw_stats *hw_stats = 1871 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1872 1873 /* HW registers are cleared on read */ 1874 eth_igb_stats_get(dev, NULL); 1875 1876 /* Reset software totals */ 1877 memset(hw_stats, 0, sizeof(*hw_stats)); 1878 1879 return 0; 1880 } 1881 1882 static int 1883 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1884 { 1885 struct e1000_hw_stats *stats = 1886 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1887 1888 /* HW registers are cleared on read */ 1889 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1890 1891 /* Reset software totals */ 1892 memset(stats, 0, sizeof(*stats)); 1893 1894 return 0; 1895 } 1896 1897 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1898 struct rte_eth_xstat_name *xstats_names, 1899 __rte_unused unsigned int size) 1900 { 1901 unsigned i; 1902 1903 if (xstats_names == NULL) 1904 return IGB_NB_XSTATS; 1905 1906 /* Note: limit checked in rte_eth_xstats_names() */ 1907 1908 for (i = 0; i < IGB_NB_XSTATS; i++) { 1909 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1910 sizeof(xstats_names[i].name)); 1911 } 1912 1913 return IGB_NB_XSTATS; 1914 } 1915 1916 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1917 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1918 unsigned int limit) 1919 { 1920 unsigned int i; 1921 1922 if (!ids) { 1923 if (xstats_names == NULL) 1924 return IGB_NB_XSTATS; 1925 1926 for (i = 0; i < IGB_NB_XSTATS; i++) 1927 strlcpy(xstats_names[i].name, 1928 rte_igb_stats_strings[i].name, 1929 sizeof(xstats_names[i].name)); 1930 1931 return IGB_NB_XSTATS; 1932 1933 } else { 1934 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1935 1936 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1937 IGB_NB_XSTATS); 1938 1939 for (i = 0; i < limit; i++) { 1940 if (ids[i] >= IGB_NB_XSTATS) { 1941 PMD_INIT_LOG(ERR, "id value isn't valid"); 1942 return -1; 1943 } 1944 strcpy(xstats_names[i].name, 1945 xstats_names_copy[ids[i]].name); 1946 } 1947 return limit; 1948 } 1949 } 1950 1951 static int 1952 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1953 unsigned n) 1954 { 1955 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1956 struct e1000_hw_stats *hw_stats = 1957 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1958 unsigned i; 1959 1960 if (n < IGB_NB_XSTATS) 1961 return IGB_NB_XSTATS; 1962 1963 igb_read_stats_registers(hw, hw_stats); 1964 1965 /* If this is a reset xstats is NULL, and we have cleared the 1966 * registers by reading them. 1967 */ 1968 if (!xstats) 1969 return 0; 1970 1971 /* Extended stats */ 1972 for (i = 0; i < IGB_NB_XSTATS; i++) { 1973 xstats[i].id = i; 1974 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1975 rte_igb_stats_strings[i].offset); 1976 } 1977 1978 return IGB_NB_XSTATS; 1979 } 1980 1981 static int 1982 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1983 uint64_t *values, unsigned int n) 1984 { 1985 unsigned int i; 1986 1987 if (!ids) { 1988 struct e1000_hw *hw = 1989 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1990 struct e1000_hw_stats *hw_stats = 1991 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1992 1993 if (n < IGB_NB_XSTATS) 1994 return IGB_NB_XSTATS; 1995 1996 igb_read_stats_registers(hw, hw_stats); 1997 1998 /* If this is a reset xstats is NULL, and we have cleared the 1999 * registers by reading them. 2000 */ 2001 if (!values) 2002 return 0; 2003 2004 /* Extended stats */ 2005 for (i = 0; i < IGB_NB_XSTATS; i++) 2006 values[i] = *(uint64_t *)(((char *)hw_stats) + 2007 rte_igb_stats_strings[i].offset); 2008 2009 return IGB_NB_XSTATS; 2010 2011 } else { 2012 uint64_t values_copy[IGB_NB_XSTATS]; 2013 2014 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2015 IGB_NB_XSTATS); 2016 2017 for (i = 0; i < n; i++) { 2018 if (ids[i] >= IGB_NB_XSTATS) { 2019 PMD_INIT_LOG(ERR, "id value isn't valid"); 2020 return -1; 2021 } 2022 values[i] = values_copy[ids[i]]; 2023 } 2024 return n; 2025 } 2026 } 2027 2028 static void 2029 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2030 { 2031 /* Good Rx packets, include VF loopback */ 2032 UPDATE_VF_STAT(E1000_VFGPRC, 2033 hw_stats->last_gprc, hw_stats->gprc); 2034 2035 /* Good Rx octets, include VF loopback */ 2036 UPDATE_VF_STAT(E1000_VFGORC, 2037 hw_stats->last_gorc, hw_stats->gorc); 2038 2039 /* Good Tx packets, include VF loopback */ 2040 UPDATE_VF_STAT(E1000_VFGPTC, 2041 hw_stats->last_gptc, hw_stats->gptc); 2042 2043 /* Good Tx octets, include VF loopback */ 2044 UPDATE_VF_STAT(E1000_VFGOTC, 2045 hw_stats->last_gotc, hw_stats->gotc); 2046 2047 /* Rx Multicst packets */ 2048 UPDATE_VF_STAT(E1000_VFMPRC, 2049 hw_stats->last_mprc, hw_stats->mprc); 2050 2051 /* Good Rx loopback packets */ 2052 UPDATE_VF_STAT(E1000_VFGPRLBC, 2053 hw_stats->last_gprlbc, hw_stats->gprlbc); 2054 2055 /* Good Rx loopback octets */ 2056 UPDATE_VF_STAT(E1000_VFGORLBC, 2057 hw_stats->last_gorlbc, hw_stats->gorlbc); 2058 2059 /* Good Tx loopback packets */ 2060 UPDATE_VF_STAT(E1000_VFGPTLBC, 2061 hw_stats->last_gptlbc, hw_stats->gptlbc); 2062 2063 /* Good Tx loopback octets */ 2064 UPDATE_VF_STAT(E1000_VFGOTLBC, 2065 hw_stats->last_gotlbc, hw_stats->gotlbc); 2066 } 2067 2068 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2069 struct rte_eth_xstat_name *xstats_names, 2070 __rte_unused unsigned limit) 2071 { 2072 unsigned i; 2073 2074 if (xstats_names != NULL) 2075 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2076 strlcpy(xstats_names[i].name, 2077 rte_igbvf_stats_strings[i].name, 2078 sizeof(xstats_names[i].name)); 2079 } 2080 return IGBVF_NB_XSTATS; 2081 } 2082 2083 static int 2084 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2085 unsigned n) 2086 { 2087 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2088 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2089 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2090 unsigned i; 2091 2092 if (n < IGBVF_NB_XSTATS) 2093 return IGBVF_NB_XSTATS; 2094 2095 igbvf_read_stats_registers(hw, hw_stats); 2096 2097 if (!xstats) 2098 return 0; 2099 2100 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2101 xstats[i].id = i; 2102 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2103 rte_igbvf_stats_strings[i].offset); 2104 } 2105 2106 return IGBVF_NB_XSTATS; 2107 } 2108 2109 static int 2110 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2111 { 2112 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2113 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2114 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2115 2116 igbvf_read_stats_registers(hw, hw_stats); 2117 2118 if (rte_stats == NULL) 2119 return -EINVAL; 2120 2121 rte_stats->ipackets = hw_stats->gprc; 2122 rte_stats->ibytes = hw_stats->gorc; 2123 rte_stats->opackets = hw_stats->gptc; 2124 rte_stats->obytes = hw_stats->gotc; 2125 return 0; 2126 } 2127 2128 static int 2129 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2130 { 2131 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2132 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2133 2134 /* Sync HW register to the last stats */ 2135 eth_igbvf_stats_get(dev, NULL); 2136 2137 /* reset HW current stats*/ 2138 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2139 offsetof(struct e1000_vf_stats, gprc)); 2140 2141 return 0; 2142 } 2143 2144 static int 2145 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2146 size_t fw_size) 2147 { 2148 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2149 struct e1000_fw_version fw; 2150 int ret; 2151 2152 e1000_get_fw_version(hw, &fw); 2153 2154 switch (hw->mac.type) { 2155 case e1000_i210: 2156 case e1000_i211: 2157 if (!(e1000_get_flash_presence_i210(hw))) { 2158 ret = snprintf(fw_version, fw_size, 2159 "%2d.%2d-%d", 2160 fw.invm_major, fw.invm_minor, 2161 fw.invm_img_type); 2162 break; 2163 } 2164 /* fall through */ 2165 default: 2166 /* if option rom is valid, display its version too */ 2167 if (fw.or_valid) { 2168 ret = snprintf(fw_version, fw_size, 2169 "%d.%d, 0x%08x, %d.%d.%d", 2170 fw.eep_major, fw.eep_minor, fw.etrack_id, 2171 fw.or_major, fw.or_build, fw.or_patch); 2172 /* no option rom */ 2173 } else { 2174 if (fw.etrack_id != 0X0000) { 2175 ret = snprintf(fw_version, fw_size, 2176 "%d.%d, 0x%08x", 2177 fw.eep_major, fw.eep_minor, 2178 fw.etrack_id); 2179 } else { 2180 ret = snprintf(fw_version, fw_size, 2181 "%d.%d.%d", 2182 fw.eep_major, fw.eep_minor, 2183 fw.eep_build); 2184 } 2185 } 2186 break; 2187 } 2188 2189 ret += 1; /* add the size of '\0' */ 2190 if (fw_size < (u32)ret) 2191 return ret; 2192 else 2193 return 0; 2194 } 2195 2196 static int 2197 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2198 { 2199 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2200 2201 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2202 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2203 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2204 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2205 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2206 dev_info->rx_queue_offload_capa; 2207 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2208 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2209 dev_info->tx_queue_offload_capa; 2210 2211 switch (hw->mac.type) { 2212 case e1000_82575: 2213 dev_info->max_rx_queues = 4; 2214 dev_info->max_tx_queues = 4; 2215 dev_info->max_vmdq_pools = 0; 2216 break; 2217 2218 case e1000_82576: 2219 dev_info->max_rx_queues = 16; 2220 dev_info->max_tx_queues = 16; 2221 dev_info->max_vmdq_pools = ETH_8_POOLS; 2222 dev_info->vmdq_queue_num = 16; 2223 break; 2224 2225 case e1000_82580: 2226 dev_info->max_rx_queues = 8; 2227 dev_info->max_tx_queues = 8; 2228 dev_info->max_vmdq_pools = ETH_8_POOLS; 2229 dev_info->vmdq_queue_num = 8; 2230 break; 2231 2232 case e1000_i350: 2233 dev_info->max_rx_queues = 8; 2234 dev_info->max_tx_queues = 8; 2235 dev_info->max_vmdq_pools = ETH_8_POOLS; 2236 dev_info->vmdq_queue_num = 8; 2237 break; 2238 2239 case e1000_i354: 2240 dev_info->max_rx_queues = 8; 2241 dev_info->max_tx_queues = 8; 2242 break; 2243 2244 case e1000_i210: 2245 dev_info->max_rx_queues = 4; 2246 dev_info->max_tx_queues = 4; 2247 dev_info->max_vmdq_pools = 0; 2248 break; 2249 2250 case e1000_i211: 2251 dev_info->max_rx_queues = 2; 2252 dev_info->max_tx_queues = 2; 2253 dev_info->max_vmdq_pools = 0; 2254 break; 2255 2256 default: 2257 /* Should not happen */ 2258 return -EINVAL; 2259 } 2260 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2261 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2262 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2263 2264 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2265 .rx_thresh = { 2266 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2267 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2268 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2269 }, 2270 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2271 .rx_drop_en = 0, 2272 .offloads = 0, 2273 }; 2274 2275 dev_info->default_txconf = (struct rte_eth_txconf) { 2276 .tx_thresh = { 2277 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2278 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2279 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2280 }, 2281 .offloads = 0, 2282 }; 2283 2284 dev_info->rx_desc_lim = rx_desc_lim; 2285 dev_info->tx_desc_lim = tx_desc_lim; 2286 2287 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2288 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2289 ETH_LINK_SPEED_1G; 2290 2291 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2292 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 2293 2294 return 0; 2295 } 2296 2297 static const uint32_t * 2298 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2299 { 2300 static const uint32_t ptypes[] = { 2301 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2302 RTE_PTYPE_L2_ETHER, 2303 RTE_PTYPE_L3_IPV4, 2304 RTE_PTYPE_L3_IPV4_EXT, 2305 RTE_PTYPE_L3_IPV6, 2306 RTE_PTYPE_L3_IPV6_EXT, 2307 RTE_PTYPE_L4_TCP, 2308 RTE_PTYPE_L4_UDP, 2309 RTE_PTYPE_L4_SCTP, 2310 RTE_PTYPE_TUNNEL_IP, 2311 RTE_PTYPE_INNER_L3_IPV6, 2312 RTE_PTYPE_INNER_L3_IPV6_EXT, 2313 RTE_PTYPE_INNER_L4_TCP, 2314 RTE_PTYPE_INNER_L4_UDP, 2315 RTE_PTYPE_UNKNOWN 2316 }; 2317 2318 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2319 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2320 return ptypes; 2321 return NULL; 2322 } 2323 2324 static int 2325 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2326 { 2327 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2328 2329 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2330 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2331 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2332 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2333 DEV_TX_OFFLOAD_IPV4_CKSUM | 2334 DEV_TX_OFFLOAD_UDP_CKSUM | 2335 DEV_TX_OFFLOAD_TCP_CKSUM | 2336 DEV_TX_OFFLOAD_SCTP_CKSUM | 2337 DEV_TX_OFFLOAD_TCP_TSO; 2338 switch (hw->mac.type) { 2339 case e1000_vfadapt: 2340 dev_info->max_rx_queues = 2; 2341 dev_info->max_tx_queues = 2; 2342 break; 2343 case e1000_vfadapt_i350: 2344 dev_info->max_rx_queues = 1; 2345 dev_info->max_tx_queues = 1; 2346 break; 2347 default: 2348 /* Should not happen */ 2349 return -EINVAL; 2350 } 2351 2352 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2353 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2354 dev_info->rx_queue_offload_capa; 2355 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2356 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2357 dev_info->tx_queue_offload_capa; 2358 2359 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2360 .rx_thresh = { 2361 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2362 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2363 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2364 }, 2365 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2366 .rx_drop_en = 0, 2367 .offloads = 0, 2368 }; 2369 2370 dev_info->default_txconf = (struct rte_eth_txconf) { 2371 .tx_thresh = { 2372 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2373 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2374 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2375 }, 2376 .offloads = 0, 2377 }; 2378 2379 dev_info->rx_desc_lim = rx_desc_lim; 2380 dev_info->tx_desc_lim = tx_desc_lim; 2381 2382 return 0; 2383 } 2384 2385 /* return 0 means link status changed, -1 means not changed */ 2386 static int 2387 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2388 { 2389 struct e1000_hw *hw = 2390 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2391 struct rte_eth_link link; 2392 int link_check, count; 2393 2394 link_check = 0; 2395 hw->mac.get_link_status = 1; 2396 2397 /* possible wait-to-complete in up to 9 seconds */ 2398 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2399 /* Read the real link status */ 2400 switch (hw->phy.media_type) { 2401 case e1000_media_type_copper: 2402 /* Do the work to read phy */ 2403 e1000_check_for_link(hw); 2404 link_check = !hw->mac.get_link_status; 2405 break; 2406 2407 case e1000_media_type_fiber: 2408 e1000_check_for_link(hw); 2409 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2410 E1000_STATUS_LU); 2411 break; 2412 2413 case e1000_media_type_internal_serdes: 2414 e1000_check_for_link(hw); 2415 link_check = hw->mac.serdes_has_link; 2416 break; 2417 2418 /* VF device is type_unknown */ 2419 case e1000_media_type_unknown: 2420 eth_igbvf_link_update(hw); 2421 link_check = !hw->mac.get_link_status; 2422 break; 2423 2424 default: 2425 break; 2426 } 2427 if (link_check || wait_to_complete == 0) 2428 break; 2429 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2430 } 2431 memset(&link, 0, sizeof(link)); 2432 2433 /* Now we check if a transition has happened */ 2434 if (link_check) { 2435 uint16_t duplex, speed; 2436 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2437 link.link_duplex = (duplex == FULL_DUPLEX) ? 2438 ETH_LINK_FULL_DUPLEX : 2439 ETH_LINK_HALF_DUPLEX; 2440 link.link_speed = speed; 2441 link.link_status = ETH_LINK_UP; 2442 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2443 ETH_LINK_SPEED_FIXED); 2444 } else if (!link_check) { 2445 link.link_speed = 0; 2446 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2447 link.link_status = ETH_LINK_DOWN; 2448 link.link_autoneg = ETH_LINK_FIXED; 2449 } 2450 2451 return rte_eth_linkstatus_set(dev, &link); 2452 } 2453 2454 /* 2455 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2456 * For ASF and Pass Through versions of f/w this means 2457 * that the driver is loaded. 2458 */ 2459 static void 2460 igb_hw_control_acquire(struct e1000_hw *hw) 2461 { 2462 uint32_t ctrl_ext; 2463 2464 /* Let firmware know the driver has taken over */ 2465 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2466 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2467 } 2468 2469 /* 2470 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2471 * For ASF and Pass Through versions of f/w this means that the 2472 * driver is no longer loaded. 2473 */ 2474 static void 2475 igb_hw_control_release(struct e1000_hw *hw) 2476 { 2477 uint32_t ctrl_ext; 2478 2479 /* Let firmware taken over control of h/w */ 2480 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2481 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2482 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2483 } 2484 2485 /* 2486 * Bit of a misnomer, what this really means is 2487 * to enable OS management of the system... aka 2488 * to disable special hardware management features. 2489 */ 2490 static void 2491 igb_init_manageability(struct e1000_hw *hw) 2492 { 2493 if (e1000_enable_mng_pass_thru(hw)) { 2494 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2495 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2496 2497 /* disable hardware interception of ARP */ 2498 manc &= ~(E1000_MANC_ARP_EN); 2499 2500 /* enable receiving management packets to the host */ 2501 manc |= E1000_MANC_EN_MNG2HOST; 2502 manc2h |= 1 << 5; /* Mng Port 623 */ 2503 manc2h |= 1 << 6; /* Mng Port 664 */ 2504 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2505 E1000_WRITE_REG(hw, E1000_MANC, manc); 2506 } 2507 } 2508 2509 static void 2510 igb_release_manageability(struct e1000_hw *hw) 2511 { 2512 if (e1000_enable_mng_pass_thru(hw)) { 2513 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2514 2515 manc |= E1000_MANC_ARP_EN; 2516 manc &= ~E1000_MANC_EN_MNG2HOST; 2517 2518 E1000_WRITE_REG(hw, E1000_MANC, manc); 2519 } 2520 } 2521 2522 static int 2523 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2524 { 2525 struct e1000_hw *hw = 2526 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2527 uint32_t rctl; 2528 2529 rctl = E1000_READ_REG(hw, E1000_RCTL); 2530 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2531 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2532 2533 return 0; 2534 } 2535 2536 static int 2537 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2538 { 2539 struct e1000_hw *hw = 2540 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2541 uint32_t rctl; 2542 2543 rctl = E1000_READ_REG(hw, E1000_RCTL); 2544 rctl &= (~E1000_RCTL_UPE); 2545 if (dev->data->all_multicast == 1) 2546 rctl |= E1000_RCTL_MPE; 2547 else 2548 rctl &= (~E1000_RCTL_MPE); 2549 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2550 2551 return 0; 2552 } 2553 2554 static int 2555 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2556 { 2557 struct e1000_hw *hw = 2558 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2559 uint32_t rctl; 2560 2561 rctl = E1000_READ_REG(hw, E1000_RCTL); 2562 rctl |= E1000_RCTL_MPE; 2563 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2564 2565 return 0; 2566 } 2567 2568 static int 2569 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2570 { 2571 struct e1000_hw *hw = 2572 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2573 uint32_t rctl; 2574 2575 if (dev->data->promiscuous == 1) 2576 return 0; /* must remain in all_multicast mode */ 2577 rctl = E1000_READ_REG(hw, E1000_RCTL); 2578 rctl &= (~E1000_RCTL_MPE); 2579 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2580 2581 return 0; 2582 } 2583 2584 static int 2585 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2586 { 2587 struct e1000_hw *hw = 2588 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2589 struct e1000_vfta * shadow_vfta = 2590 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2591 uint32_t vfta; 2592 uint32_t vid_idx; 2593 uint32_t vid_bit; 2594 2595 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2596 E1000_VFTA_ENTRY_MASK); 2597 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2598 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2599 if (on) 2600 vfta |= vid_bit; 2601 else 2602 vfta &= ~vid_bit; 2603 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2604 2605 /* update local VFTA copy */ 2606 shadow_vfta->vfta[vid_idx] = vfta; 2607 2608 return 0; 2609 } 2610 2611 static int 2612 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2613 enum rte_vlan_type vlan_type, 2614 uint16_t tpid) 2615 { 2616 struct e1000_hw *hw = 2617 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2618 uint32_t reg, qinq; 2619 2620 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2621 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2622 2623 /* only outer TPID of double VLAN can be configured*/ 2624 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2625 reg = E1000_READ_REG(hw, E1000_VET); 2626 reg = (reg & (~E1000_VET_VET_EXT)) | 2627 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2628 E1000_WRITE_REG(hw, E1000_VET, reg); 2629 2630 return 0; 2631 } 2632 2633 /* all other TPID values are read-only*/ 2634 PMD_DRV_LOG(ERR, "Not supported"); 2635 2636 return -ENOTSUP; 2637 } 2638 2639 static void 2640 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2641 { 2642 struct e1000_hw *hw = 2643 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2644 uint32_t reg; 2645 2646 /* Filter Table Disable */ 2647 reg = E1000_READ_REG(hw, E1000_RCTL); 2648 reg &= ~E1000_RCTL_CFIEN; 2649 reg &= ~E1000_RCTL_VFE; 2650 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2651 } 2652 2653 static void 2654 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2655 { 2656 struct e1000_hw *hw = 2657 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2658 struct e1000_vfta * shadow_vfta = 2659 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2660 uint32_t reg; 2661 int i; 2662 2663 /* Filter Table Enable, CFI not used for packet acceptance */ 2664 reg = E1000_READ_REG(hw, E1000_RCTL); 2665 reg &= ~E1000_RCTL_CFIEN; 2666 reg |= E1000_RCTL_VFE; 2667 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2668 2669 /* restore VFTA table */ 2670 for (i = 0; i < IGB_VFTA_SIZE; i++) 2671 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2672 } 2673 2674 static void 2675 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2676 { 2677 struct e1000_hw *hw = 2678 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2679 uint32_t reg; 2680 2681 /* VLAN Mode Disable */ 2682 reg = E1000_READ_REG(hw, E1000_CTRL); 2683 reg &= ~E1000_CTRL_VME; 2684 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2685 } 2686 2687 static void 2688 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2689 { 2690 struct e1000_hw *hw = 2691 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2692 uint32_t reg; 2693 2694 /* VLAN Mode Enable */ 2695 reg = E1000_READ_REG(hw, E1000_CTRL); 2696 reg |= E1000_CTRL_VME; 2697 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2698 } 2699 2700 static void 2701 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2702 { 2703 struct e1000_hw *hw = 2704 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2705 uint32_t reg; 2706 2707 /* CTRL_EXT: Extended VLAN */ 2708 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2709 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2710 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2711 2712 /* Update maximum packet length */ 2713 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2714 E1000_WRITE_REG(hw, E1000_RLPML, 2715 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2716 VLAN_TAG_SIZE); 2717 } 2718 2719 static void 2720 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2721 { 2722 struct e1000_hw *hw = 2723 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2724 uint32_t reg; 2725 2726 /* CTRL_EXT: Extended VLAN */ 2727 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2728 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2729 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2730 2731 /* Update maximum packet length */ 2732 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2733 E1000_WRITE_REG(hw, E1000_RLPML, 2734 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2735 2 * VLAN_TAG_SIZE); 2736 } 2737 2738 static int 2739 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2740 { 2741 struct rte_eth_rxmode *rxmode; 2742 2743 rxmode = &dev->data->dev_conf.rxmode; 2744 if(mask & ETH_VLAN_STRIP_MASK){ 2745 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2746 igb_vlan_hw_strip_enable(dev); 2747 else 2748 igb_vlan_hw_strip_disable(dev); 2749 } 2750 2751 if(mask & ETH_VLAN_FILTER_MASK){ 2752 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2753 igb_vlan_hw_filter_enable(dev); 2754 else 2755 igb_vlan_hw_filter_disable(dev); 2756 } 2757 2758 if(mask & ETH_VLAN_EXTEND_MASK){ 2759 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2760 igb_vlan_hw_extend_enable(dev); 2761 else 2762 igb_vlan_hw_extend_disable(dev); 2763 } 2764 2765 return 0; 2766 } 2767 2768 2769 /** 2770 * It enables the interrupt mask and then enable the interrupt. 2771 * 2772 * @param dev 2773 * Pointer to struct rte_eth_dev. 2774 * @param on 2775 * Enable or Disable 2776 * 2777 * @return 2778 * - On success, zero. 2779 * - On failure, a negative value. 2780 */ 2781 static int 2782 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2783 { 2784 struct e1000_interrupt *intr = 2785 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2786 2787 if (on) 2788 intr->mask |= E1000_ICR_LSC; 2789 else 2790 intr->mask &= ~E1000_ICR_LSC; 2791 2792 return 0; 2793 } 2794 2795 /* It clears the interrupt causes and enables the interrupt. 2796 * It will be called once only during nic initialized. 2797 * 2798 * @param dev 2799 * Pointer to struct rte_eth_dev. 2800 * 2801 * @return 2802 * - On success, zero. 2803 * - On failure, a negative value. 2804 */ 2805 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2806 { 2807 uint32_t mask, regval; 2808 int ret; 2809 struct e1000_hw *hw = 2810 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2811 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2812 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2813 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2814 struct rte_eth_dev_info dev_info; 2815 2816 memset(&dev_info, 0, sizeof(dev_info)); 2817 ret = eth_igb_infos_get(dev, &dev_info); 2818 if (ret != 0) 2819 return ret; 2820 2821 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2822 regval = E1000_READ_REG(hw, E1000_EIMS); 2823 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2824 2825 return 0; 2826 } 2827 2828 /* 2829 * It reads ICR and gets interrupt causes, check it and set a bit flag 2830 * to update link status. 2831 * 2832 * @param dev 2833 * Pointer to struct rte_eth_dev. 2834 * 2835 * @return 2836 * - On success, zero. 2837 * - On failure, a negative value. 2838 */ 2839 static int 2840 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2841 { 2842 uint32_t icr; 2843 struct e1000_hw *hw = 2844 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2845 struct e1000_interrupt *intr = 2846 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2847 2848 igb_intr_disable(dev); 2849 2850 /* read-on-clear nic registers here */ 2851 icr = E1000_READ_REG(hw, E1000_ICR); 2852 2853 intr->flags = 0; 2854 if (icr & E1000_ICR_LSC) { 2855 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2856 } 2857 2858 if (icr & E1000_ICR_VMMB) 2859 intr->flags |= E1000_FLAG_MAILBOX; 2860 2861 return 0; 2862 } 2863 2864 /* 2865 * It executes link_update after knowing an interrupt is prsent. 2866 * 2867 * @param dev 2868 * Pointer to struct rte_eth_dev. 2869 * 2870 * @return 2871 * - On success, zero. 2872 * - On failure, a negative value. 2873 */ 2874 static int 2875 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2876 struct rte_intr_handle *intr_handle) 2877 { 2878 struct e1000_hw *hw = 2879 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2880 struct e1000_interrupt *intr = 2881 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2882 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2883 struct rte_eth_link link; 2884 int ret; 2885 2886 if (intr->flags & E1000_FLAG_MAILBOX) { 2887 igb_pf_mbx_process(dev); 2888 intr->flags &= ~E1000_FLAG_MAILBOX; 2889 } 2890 2891 igb_intr_enable(dev); 2892 rte_intr_ack(intr_handle); 2893 2894 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2895 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2896 2897 /* set get_link_status to check register later */ 2898 hw->mac.get_link_status = 1; 2899 ret = eth_igb_link_update(dev, 0); 2900 2901 /* check if link has changed */ 2902 if (ret < 0) 2903 return 0; 2904 2905 rte_eth_linkstatus_get(dev, &link); 2906 if (link.link_status) { 2907 PMD_INIT_LOG(INFO, 2908 " Port %d: Link Up - speed %u Mbps - %s", 2909 dev->data->port_id, 2910 (unsigned)link.link_speed, 2911 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2912 "full-duplex" : "half-duplex"); 2913 } else { 2914 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2915 dev->data->port_id); 2916 } 2917 2918 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 2919 pci_dev->addr.domain, 2920 pci_dev->addr.bus, 2921 pci_dev->addr.devid, 2922 pci_dev->addr.function); 2923 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2924 } 2925 2926 return 0; 2927 } 2928 2929 /** 2930 * Interrupt handler which shall be registered at first. 2931 * 2932 * @param handle 2933 * Pointer to interrupt handle. 2934 * @param param 2935 * The address of parameter (struct rte_eth_dev *) regsitered before. 2936 * 2937 * @return 2938 * void 2939 */ 2940 static void 2941 eth_igb_interrupt_handler(void *param) 2942 { 2943 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2944 2945 eth_igb_interrupt_get_status(dev); 2946 eth_igb_interrupt_action(dev, dev->intr_handle); 2947 } 2948 2949 static int 2950 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2951 { 2952 uint32_t eicr; 2953 struct e1000_hw *hw = 2954 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2955 struct e1000_interrupt *intr = 2956 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2957 2958 igbvf_intr_disable(hw); 2959 2960 /* read-on-clear nic registers here */ 2961 eicr = E1000_READ_REG(hw, E1000_EICR); 2962 intr->flags = 0; 2963 2964 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2965 intr->flags |= E1000_FLAG_MAILBOX; 2966 2967 return 0; 2968 } 2969 2970 void igbvf_mbx_process(struct rte_eth_dev *dev) 2971 { 2972 struct e1000_hw *hw = 2973 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2974 struct e1000_mbx_info *mbx = &hw->mbx; 2975 u32 in_msg = 0; 2976 2977 /* peek the message first */ 2978 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2979 2980 /* PF reset VF event */ 2981 if (in_msg == E1000_PF_CONTROL_MSG) { 2982 /* dummy mbx read to ack pf */ 2983 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2984 return; 2985 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2986 NULL); 2987 } 2988 } 2989 2990 static int 2991 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2992 { 2993 struct e1000_interrupt *intr = 2994 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2995 2996 if (intr->flags & E1000_FLAG_MAILBOX) { 2997 igbvf_mbx_process(dev); 2998 intr->flags &= ~E1000_FLAG_MAILBOX; 2999 } 3000 3001 igbvf_intr_enable(dev); 3002 rte_intr_ack(intr_handle); 3003 3004 return 0; 3005 } 3006 3007 static void 3008 eth_igbvf_interrupt_handler(void *param) 3009 { 3010 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 3011 3012 eth_igbvf_interrupt_get_status(dev); 3013 eth_igbvf_interrupt_action(dev, dev->intr_handle); 3014 } 3015 3016 static int 3017 eth_igb_led_on(struct rte_eth_dev *dev) 3018 { 3019 struct e1000_hw *hw; 3020 3021 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3022 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3023 } 3024 3025 static int 3026 eth_igb_led_off(struct rte_eth_dev *dev) 3027 { 3028 struct e1000_hw *hw; 3029 3030 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3031 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3032 } 3033 3034 static int 3035 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3036 { 3037 struct e1000_hw *hw; 3038 uint32_t ctrl; 3039 int tx_pause; 3040 int rx_pause; 3041 3042 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3043 fc_conf->pause_time = hw->fc.pause_time; 3044 fc_conf->high_water = hw->fc.high_water; 3045 fc_conf->low_water = hw->fc.low_water; 3046 fc_conf->send_xon = hw->fc.send_xon; 3047 fc_conf->autoneg = hw->mac.autoneg; 3048 3049 /* 3050 * Return rx_pause and tx_pause status according to actual setting of 3051 * the TFCE and RFCE bits in the CTRL register. 3052 */ 3053 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3054 if (ctrl & E1000_CTRL_TFCE) 3055 tx_pause = 1; 3056 else 3057 tx_pause = 0; 3058 3059 if (ctrl & E1000_CTRL_RFCE) 3060 rx_pause = 1; 3061 else 3062 rx_pause = 0; 3063 3064 if (rx_pause && tx_pause) 3065 fc_conf->mode = RTE_FC_FULL; 3066 else if (rx_pause) 3067 fc_conf->mode = RTE_FC_RX_PAUSE; 3068 else if (tx_pause) 3069 fc_conf->mode = RTE_FC_TX_PAUSE; 3070 else 3071 fc_conf->mode = RTE_FC_NONE; 3072 3073 return 0; 3074 } 3075 3076 static int 3077 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3078 { 3079 struct e1000_hw *hw; 3080 int err; 3081 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3082 e1000_fc_none, 3083 e1000_fc_rx_pause, 3084 e1000_fc_tx_pause, 3085 e1000_fc_full 3086 }; 3087 uint32_t rx_buf_size; 3088 uint32_t max_high_water; 3089 uint32_t rctl; 3090 3091 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3092 if (fc_conf->autoneg != hw->mac.autoneg) 3093 return -ENOTSUP; 3094 rx_buf_size = igb_get_rx_buffer_size(hw); 3095 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3096 3097 /* At least reserve one Ethernet frame for watermark */ 3098 max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN; 3099 if ((fc_conf->high_water > max_high_water) || 3100 (fc_conf->high_water < fc_conf->low_water)) { 3101 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3102 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3103 return -EINVAL; 3104 } 3105 3106 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3107 hw->fc.pause_time = fc_conf->pause_time; 3108 hw->fc.high_water = fc_conf->high_water; 3109 hw->fc.low_water = fc_conf->low_water; 3110 hw->fc.send_xon = fc_conf->send_xon; 3111 3112 err = e1000_setup_link_generic(hw); 3113 if (err == E1000_SUCCESS) { 3114 3115 /* check if we want to forward MAC frames - driver doesn't have native 3116 * capability to do that, so we'll write the registers ourselves */ 3117 3118 rctl = E1000_READ_REG(hw, E1000_RCTL); 3119 3120 /* set or clear MFLCN.PMCF bit depending on configuration */ 3121 if (fc_conf->mac_ctrl_frame_fwd != 0) 3122 rctl |= E1000_RCTL_PMCF; 3123 else 3124 rctl &= ~E1000_RCTL_PMCF; 3125 3126 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3127 E1000_WRITE_FLUSH(hw); 3128 3129 return 0; 3130 } 3131 3132 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3133 return -EIO; 3134 } 3135 3136 #define E1000_RAH_POOLSEL_SHIFT (18) 3137 static int 3138 eth_igb_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 3139 uint32_t index, uint32_t pool) 3140 { 3141 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3142 uint32_t rah; 3143 3144 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3145 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3146 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3147 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3148 return 0; 3149 } 3150 3151 static void 3152 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3153 { 3154 uint8_t addr[RTE_ETHER_ADDR_LEN]; 3155 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3156 3157 memset(addr, 0, sizeof(addr)); 3158 3159 e1000_rar_set(hw, addr, index); 3160 } 3161 3162 static int 3163 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3164 struct rte_ether_addr *addr) 3165 { 3166 eth_igb_rar_clear(dev, 0); 3167 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3168 3169 return 0; 3170 } 3171 /* 3172 * Virtual Function operations 3173 */ 3174 static void 3175 igbvf_intr_disable(struct e1000_hw *hw) 3176 { 3177 PMD_INIT_FUNC_TRACE(); 3178 3179 /* Clear interrupt mask to stop from interrupts being generated */ 3180 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3181 3182 E1000_WRITE_FLUSH(hw); 3183 } 3184 3185 static void 3186 igbvf_stop_adapter(struct rte_eth_dev *dev) 3187 { 3188 u32 reg_val; 3189 u16 i; 3190 struct rte_eth_dev_info dev_info; 3191 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3192 int ret; 3193 3194 memset(&dev_info, 0, sizeof(dev_info)); 3195 ret = eth_igbvf_infos_get(dev, &dev_info); 3196 if (ret != 0) 3197 return; 3198 3199 /* Clear interrupt mask to stop from interrupts being generated */ 3200 igbvf_intr_disable(hw); 3201 3202 /* Clear any pending interrupts, flush previous writes */ 3203 E1000_READ_REG(hw, E1000_EICR); 3204 3205 /* Disable the transmit unit. Each queue must be disabled. */ 3206 for (i = 0; i < dev_info.max_tx_queues; i++) 3207 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3208 3209 /* Disable the receive unit by stopping each queue */ 3210 for (i = 0; i < dev_info.max_rx_queues; i++) { 3211 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3212 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3213 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3214 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3215 ; 3216 } 3217 3218 /* flush all queues disables */ 3219 E1000_WRITE_FLUSH(hw); 3220 msec_delay(2); 3221 } 3222 3223 static int eth_igbvf_link_update(struct e1000_hw *hw) 3224 { 3225 struct e1000_mbx_info *mbx = &hw->mbx; 3226 struct e1000_mac_info *mac = &hw->mac; 3227 int ret_val = E1000_SUCCESS; 3228 3229 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3230 3231 /* 3232 * We only want to run this if there has been a rst asserted. 3233 * in this case that could mean a link change, device reset, 3234 * or a virtual function reset 3235 */ 3236 3237 /* If we were hit with a reset or timeout drop the link */ 3238 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3239 mac->get_link_status = TRUE; 3240 3241 if (!mac->get_link_status) 3242 goto out; 3243 3244 /* if link status is down no point in checking to see if pf is up */ 3245 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3246 goto out; 3247 3248 /* if we passed all the tests above then the link is up and we no 3249 * longer need to check for link */ 3250 mac->get_link_status = FALSE; 3251 3252 out: 3253 return ret_val; 3254 } 3255 3256 3257 static int 3258 igbvf_dev_configure(struct rte_eth_dev *dev) 3259 { 3260 struct rte_eth_conf* conf = &dev->data->dev_conf; 3261 3262 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3263 dev->data->port_id); 3264 3265 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 3266 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 3267 3268 /* 3269 * VF has no ability to enable/disable HW CRC 3270 * Keep the persistent behavior the same as Host PF 3271 */ 3272 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3273 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3274 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3275 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3276 } 3277 #else 3278 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3279 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3280 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3281 } 3282 #endif 3283 3284 return 0; 3285 } 3286 3287 static int 3288 igbvf_dev_start(struct rte_eth_dev *dev) 3289 { 3290 struct e1000_hw *hw = 3291 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3292 struct e1000_adapter *adapter = 3293 E1000_DEV_PRIVATE(dev->data->dev_private); 3294 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3295 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3296 int ret; 3297 uint32_t intr_vector = 0; 3298 3299 PMD_INIT_FUNC_TRACE(); 3300 3301 hw->mac.ops.reset_hw(hw); 3302 adapter->stopped = 0; 3303 3304 /* Set all vfta */ 3305 igbvf_set_vfta_all(dev,1); 3306 3307 eth_igbvf_tx_init(dev); 3308 3309 /* This can fail when allocating mbufs for descriptor rings */ 3310 ret = eth_igbvf_rx_init(dev); 3311 if (ret) { 3312 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3313 igb_dev_clear_queues(dev); 3314 return ret; 3315 } 3316 3317 /* check and configure queue intr-vector mapping */ 3318 if (rte_intr_cap_multiple(intr_handle) && 3319 dev->data->dev_conf.intr_conf.rxq) { 3320 intr_vector = dev->data->nb_rx_queues; 3321 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3322 if (ret) 3323 return ret; 3324 } 3325 3326 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3327 intr_handle->intr_vec = 3328 rte_zmalloc("intr_vec", 3329 dev->data->nb_rx_queues * sizeof(int), 0); 3330 if (!intr_handle->intr_vec) { 3331 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3332 " intr_vec", dev->data->nb_rx_queues); 3333 return -ENOMEM; 3334 } 3335 } 3336 3337 eth_igbvf_configure_msix_intr(dev); 3338 3339 /* enable uio/vfio intr/eventfd mapping */ 3340 rte_intr_enable(intr_handle); 3341 3342 /* resume enabled intr since hw reset */ 3343 igbvf_intr_enable(dev); 3344 3345 return 0; 3346 } 3347 3348 static void 3349 igbvf_dev_stop(struct rte_eth_dev *dev) 3350 { 3351 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3352 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3353 struct e1000_adapter *adapter = 3354 E1000_DEV_PRIVATE(dev->data->dev_private); 3355 3356 if (adapter->stopped) 3357 return; 3358 3359 PMD_INIT_FUNC_TRACE(); 3360 3361 igbvf_stop_adapter(dev); 3362 3363 /* 3364 * Clear what we set, but we still keep shadow_vfta to 3365 * restore after device starts 3366 */ 3367 igbvf_set_vfta_all(dev,0); 3368 3369 igb_dev_clear_queues(dev); 3370 3371 /* disable intr eventfd mapping */ 3372 rte_intr_disable(intr_handle); 3373 3374 /* Clean datapath event and queue/vec mapping */ 3375 rte_intr_efd_disable(intr_handle); 3376 if (intr_handle->intr_vec) { 3377 rte_free(intr_handle->intr_vec); 3378 intr_handle->intr_vec = NULL; 3379 } 3380 3381 adapter->stopped = true; 3382 } 3383 3384 static void 3385 igbvf_dev_close(struct rte_eth_dev *dev) 3386 { 3387 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3388 struct rte_ether_addr addr; 3389 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3390 3391 PMD_INIT_FUNC_TRACE(); 3392 3393 e1000_reset_hw(hw); 3394 3395 igbvf_dev_stop(dev); 3396 3397 igb_dev_free_queues(dev); 3398 3399 /** 3400 * reprogram the RAR with a zero mac address, 3401 * to ensure that the VF traffic goes to the PF 3402 * after stop, close and detach of the VF. 3403 **/ 3404 3405 memset(&addr, 0, sizeof(addr)); 3406 igbvf_default_mac_addr_set(dev, &addr); 3407 3408 dev->dev_ops = NULL; 3409 dev->rx_pkt_burst = NULL; 3410 dev->tx_pkt_burst = NULL; 3411 3412 rte_intr_callback_unregister(&pci_dev->intr_handle, 3413 eth_igbvf_interrupt_handler, 3414 (void *)dev); 3415 } 3416 3417 static int 3418 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3419 { 3420 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3421 3422 /* Set both unicast and multicast promisc */ 3423 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3424 3425 return 0; 3426 } 3427 3428 static int 3429 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3430 { 3431 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3432 3433 /* If in allmulticast mode leave multicast promisc */ 3434 if (dev->data->all_multicast == 1) 3435 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3436 else 3437 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3438 3439 return 0; 3440 } 3441 3442 static int 3443 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3444 { 3445 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3446 3447 /* In promiscuous mode multicast promisc already set */ 3448 if (dev->data->promiscuous == 0) 3449 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3450 3451 return 0; 3452 } 3453 3454 static int 3455 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3456 { 3457 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3458 3459 /* In promiscuous mode leave multicast promisc enabled */ 3460 if (dev->data->promiscuous == 0) 3461 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3462 3463 return 0; 3464 } 3465 3466 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3467 { 3468 struct e1000_mbx_info *mbx = &hw->mbx; 3469 uint32_t msgbuf[2]; 3470 s32 err; 3471 3472 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3473 msgbuf[0] = E1000_VF_SET_VLAN; 3474 msgbuf[1] = vid; 3475 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3476 if (on) 3477 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3478 3479 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3480 if (err) 3481 goto mbx_err; 3482 3483 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3484 if (err) 3485 goto mbx_err; 3486 3487 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3488 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3489 err = -EINVAL; 3490 3491 mbx_err: 3492 return err; 3493 } 3494 3495 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3496 { 3497 struct e1000_hw *hw = 3498 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3499 struct e1000_vfta * shadow_vfta = 3500 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3501 int i = 0, j = 0, vfta = 0, mask = 1; 3502 3503 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3504 vfta = shadow_vfta->vfta[i]; 3505 if(vfta){ 3506 mask = 1; 3507 for (j = 0; j < 32; j++){ 3508 if(vfta & mask) 3509 igbvf_set_vfta(hw, 3510 (uint16_t)((i<<5)+j), on); 3511 mask<<=1; 3512 } 3513 } 3514 } 3515 3516 } 3517 3518 static int 3519 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3520 { 3521 struct e1000_hw *hw = 3522 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3523 struct e1000_vfta * shadow_vfta = 3524 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3525 uint32_t vid_idx = 0; 3526 uint32_t vid_bit = 0; 3527 int ret = 0; 3528 3529 PMD_INIT_FUNC_TRACE(); 3530 3531 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3532 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3533 if(ret){ 3534 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3535 return ret; 3536 } 3537 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3538 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3539 3540 /*Save what we set and retore it after device reset*/ 3541 if (on) 3542 shadow_vfta->vfta[vid_idx] |= vid_bit; 3543 else 3544 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3545 3546 return 0; 3547 } 3548 3549 static int 3550 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 3551 { 3552 struct e1000_hw *hw = 3553 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3554 3555 /* index is not used by rar_set() */ 3556 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3557 return 0; 3558 } 3559 3560 3561 static int 3562 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3563 struct rte_eth_rss_reta_entry64 *reta_conf, 3564 uint16_t reta_size) 3565 { 3566 uint8_t i, j, mask; 3567 uint32_t reta, r; 3568 uint16_t idx, shift; 3569 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3570 3571 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3572 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3573 "(%d) doesn't match the number hardware can supported " 3574 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3575 return -EINVAL; 3576 } 3577 3578 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3579 idx = i / RTE_RETA_GROUP_SIZE; 3580 shift = i % RTE_RETA_GROUP_SIZE; 3581 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3582 IGB_4_BIT_MASK); 3583 if (!mask) 3584 continue; 3585 if (mask == IGB_4_BIT_MASK) 3586 r = 0; 3587 else 3588 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3589 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3590 if (mask & (0x1 << j)) 3591 reta |= reta_conf[idx].reta[shift + j] << 3592 (CHAR_BIT * j); 3593 else 3594 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3595 } 3596 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3597 } 3598 3599 return 0; 3600 } 3601 3602 static int 3603 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3604 struct rte_eth_rss_reta_entry64 *reta_conf, 3605 uint16_t reta_size) 3606 { 3607 uint8_t i, j, mask; 3608 uint32_t reta; 3609 uint16_t idx, shift; 3610 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3611 3612 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3613 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3614 "(%d) doesn't match the number hardware can supported " 3615 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3616 return -EINVAL; 3617 } 3618 3619 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3620 idx = i / RTE_RETA_GROUP_SIZE; 3621 shift = i % RTE_RETA_GROUP_SIZE; 3622 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3623 IGB_4_BIT_MASK); 3624 if (!mask) 3625 continue; 3626 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3627 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3628 if (mask & (0x1 << j)) 3629 reta_conf[idx].reta[shift + j] = 3630 ((reta >> (CHAR_BIT * j)) & 3631 IGB_8_BIT_MASK); 3632 } 3633 } 3634 3635 return 0; 3636 } 3637 3638 int 3639 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3640 struct rte_eth_syn_filter *filter, 3641 bool add) 3642 { 3643 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3644 struct e1000_filter_info *filter_info = 3645 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3646 uint32_t synqf, rfctl; 3647 3648 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3649 return -EINVAL; 3650 3651 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3652 3653 if (add) { 3654 if (synqf & E1000_SYN_FILTER_ENABLE) 3655 return -EINVAL; 3656 3657 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3658 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3659 3660 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3661 if (filter->hig_pri) 3662 rfctl |= E1000_RFCTL_SYNQFP; 3663 else 3664 rfctl &= ~E1000_RFCTL_SYNQFP; 3665 3666 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3667 } else { 3668 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3669 return -ENOENT; 3670 synqf = 0; 3671 } 3672 3673 filter_info->syn_info = synqf; 3674 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3675 E1000_WRITE_FLUSH(hw); 3676 return 0; 3677 } 3678 3679 static int 3680 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3681 struct rte_eth_syn_filter *filter) 3682 { 3683 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3684 uint32_t synqf, rfctl; 3685 3686 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3687 if (synqf & E1000_SYN_FILTER_ENABLE) { 3688 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3689 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3690 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3691 E1000_SYN_FILTER_QUEUE_SHIFT); 3692 return 0; 3693 } 3694 3695 return -ENOENT; 3696 } 3697 3698 static int 3699 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3700 enum rte_filter_op filter_op, 3701 void *arg) 3702 { 3703 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3704 int ret; 3705 3706 MAC_TYPE_FILTER_SUP(hw->mac.type); 3707 3708 if (filter_op == RTE_ETH_FILTER_NOP) 3709 return 0; 3710 3711 if (arg == NULL) { 3712 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3713 filter_op); 3714 return -EINVAL; 3715 } 3716 3717 switch (filter_op) { 3718 case RTE_ETH_FILTER_ADD: 3719 ret = eth_igb_syn_filter_set(dev, 3720 (struct rte_eth_syn_filter *)arg, 3721 TRUE); 3722 break; 3723 case RTE_ETH_FILTER_DELETE: 3724 ret = eth_igb_syn_filter_set(dev, 3725 (struct rte_eth_syn_filter *)arg, 3726 FALSE); 3727 break; 3728 case RTE_ETH_FILTER_GET: 3729 ret = eth_igb_syn_filter_get(dev, 3730 (struct rte_eth_syn_filter *)arg); 3731 break; 3732 default: 3733 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3734 ret = -EINVAL; 3735 break; 3736 } 3737 3738 return ret; 3739 } 3740 3741 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3742 static inline int 3743 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3744 struct e1000_2tuple_filter_info *filter_info) 3745 { 3746 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3747 return -EINVAL; 3748 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3749 return -EINVAL; /* filter index is out of range. */ 3750 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 3751 return -EINVAL; /* flags is invalid. */ 3752 3753 switch (filter->dst_port_mask) { 3754 case UINT16_MAX: 3755 filter_info->dst_port_mask = 0; 3756 filter_info->dst_port = filter->dst_port; 3757 break; 3758 case 0: 3759 filter_info->dst_port_mask = 1; 3760 break; 3761 default: 3762 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3763 return -EINVAL; 3764 } 3765 3766 switch (filter->proto_mask) { 3767 case UINT8_MAX: 3768 filter_info->proto_mask = 0; 3769 filter_info->proto = filter->proto; 3770 break; 3771 case 0: 3772 filter_info->proto_mask = 1; 3773 break; 3774 default: 3775 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3776 return -EINVAL; 3777 } 3778 3779 filter_info->priority = (uint8_t)filter->priority; 3780 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3781 filter_info->tcp_flags = filter->tcp_flags; 3782 else 3783 filter_info->tcp_flags = 0; 3784 3785 return 0; 3786 } 3787 3788 static inline struct e1000_2tuple_filter * 3789 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3790 struct e1000_2tuple_filter_info *key) 3791 { 3792 struct e1000_2tuple_filter *it; 3793 3794 TAILQ_FOREACH(it, filter_list, entries) { 3795 if (memcmp(key, &it->filter_info, 3796 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3797 return it; 3798 } 3799 } 3800 return NULL; 3801 } 3802 3803 /* inject a igb 2tuple filter to HW */ 3804 static inline void 3805 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3806 struct e1000_2tuple_filter *filter) 3807 { 3808 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3809 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3810 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3811 int i; 3812 3813 i = filter->index; 3814 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3815 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3816 imir |= E1000_IMIR_PORT_BP; 3817 else 3818 imir &= ~E1000_IMIR_PORT_BP; 3819 3820 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3821 3822 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3823 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3824 ttqf |= (uint32_t)(filter->filter_info.proto & 3825 E1000_TTQF_PROTOCOL_MASK); 3826 if (filter->filter_info.proto_mask == 0) 3827 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3828 3829 /* tcp flags bits setting. */ 3830 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 3831 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 3832 imir_ext |= E1000_IMIREXT_CTRL_URG; 3833 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 3834 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3835 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 3836 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3837 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 3838 imir_ext |= E1000_IMIREXT_CTRL_RST; 3839 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 3840 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3841 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 3842 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3843 } else { 3844 imir_ext |= E1000_IMIREXT_CTRL_BP; 3845 } 3846 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3847 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3848 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3849 } 3850 3851 /* 3852 * igb_add_2tuple_filter - add a 2tuple filter 3853 * 3854 * @param 3855 * dev: Pointer to struct rte_eth_dev. 3856 * ntuple_filter: ponter to the filter that will be added. 3857 * 3858 * @return 3859 * - On success, zero. 3860 * - On failure, a negative value. 3861 */ 3862 static int 3863 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3864 struct rte_eth_ntuple_filter *ntuple_filter) 3865 { 3866 struct e1000_filter_info *filter_info = 3867 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3868 struct e1000_2tuple_filter *filter; 3869 int i, ret; 3870 3871 filter = rte_zmalloc("e1000_2tuple_filter", 3872 sizeof(struct e1000_2tuple_filter), 0); 3873 if (filter == NULL) 3874 return -ENOMEM; 3875 3876 ret = ntuple_filter_to_2tuple(ntuple_filter, 3877 &filter->filter_info); 3878 if (ret < 0) { 3879 rte_free(filter); 3880 return ret; 3881 } 3882 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3883 &filter->filter_info) != NULL) { 3884 PMD_DRV_LOG(ERR, "filter exists."); 3885 rte_free(filter); 3886 return -EEXIST; 3887 } 3888 filter->queue = ntuple_filter->queue; 3889 3890 /* 3891 * look for an unused 2tuple filter index, 3892 * and insert the filter to list. 3893 */ 3894 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3895 if (!(filter_info->twotuple_mask & (1 << i))) { 3896 filter_info->twotuple_mask |= 1 << i; 3897 filter->index = i; 3898 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3899 filter, 3900 entries); 3901 break; 3902 } 3903 } 3904 if (i >= E1000_MAX_TTQF_FILTERS) { 3905 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3906 rte_free(filter); 3907 return -ENOSYS; 3908 } 3909 3910 igb_inject_2uple_filter(dev, filter); 3911 return 0; 3912 } 3913 3914 int 3915 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3916 struct e1000_2tuple_filter *filter) 3917 { 3918 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3919 struct e1000_filter_info *filter_info = 3920 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3921 3922 filter_info->twotuple_mask &= ~(1 << filter->index); 3923 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3924 rte_free(filter); 3925 3926 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3927 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3928 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3929 return 0; 3930 } 3931 3932 /* 3933 * igb_remove_2tuple_filter - remove a 2tuple filter 3934 * 3935 * @param 3936 * dev: Pointer to struct rte_eth_dev. 3937 * ntuple_filter: ponter to the filter that will be removed. 3938 * 3939 * @return 3940 * - On success, zero. 3941 * - On failure, a negative value. 3942 */ 3943 static int 3944 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3945 struct rte_eth_ntuple_filter *ntuple_filter) 3946 { 3947 struct e1000_filter_info *filter_info = 3948 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3949 struct e1000_2tuple_filter_info filter_2tuple; 3950 struct e1000_2tuple_filter *filter; 3951 int ret; 3952 3953 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3954 ret = ntuple_filter_to_2tuple(ntuple_filter, 3955 &filter_2tuple); 3956 if (ret < 0) 3957 return ret; 3958 3959 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3960 &filter_2tuple); 3961 if (filter == NULL) { 3962 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3963 return -ENOENT; 3964 } 3965 3966 igb_delete_2tuple_filter(dev, filter); 3967 3968 return 0; 3969 } 3970 3971 /* inject a igb flex filter to HW */ 3972 static inline void 3973 igb_inject_flex_filter(struct rte_eth_dev *dev, 3974 struct e1000_flex_filter *filter) 3975 { 3976 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3977 uint32_t wufc, queueing; 3978 uint32_t reg_off; 3979 uint8_t i, j = 0; 3980 3981 wufc = E1000_READ_REG(hw, E1000_WUFC); 3982 if (filter->index < E1000_MAX_FHFT) 3983 reg_off = E1000_FHFT(filter->index); 3984 else 3985 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3986 3987 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3988 (E1000_WUFC_FLX0 << filter->index)); 3989 queueing = filter->filter_info.len | 3990 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3991 (filter->filter_info.priority << 3992 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3993 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3994 queueing); 3995 3996 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3997 E1000_WRITE_REG(hw, reg_off, 3998 filter->filter_info.dwords[j]); 3999 reg_off += sizeof(uint32_t); 4000 E1000_WRITE_REG(hw, reg_off, 4001 filter->filter_info.dwords[++j]); 4002 reg_off += sizeof(uint32_t); 4003 E1000_WRITE_REG(hw, reg_off, 4004 (uint32_t)filter->filter_info.mask[i]); 4005 reg_off += sizeof(uint32_t) * 2; 4006 ++j; 4007 } 4008 } 4009 4010 static inline struct e1000_flex_filter * 4011 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 4012 struct e1000_flex_filter_info *key) 4013 { 4014 struct e1000_flex_filter *it; 4015 4016 TAILQ_FOREACH(it, filter_list, entries) { 4017 if (memcmp(key, &it->filter_info, 4018 sizeof(struct e1000_flex_filter_info)) == 0) 4019 return it; 4020 } 4021 4022 return NULL; 4023 } 4024 4025 /* remove a flex byte filter 4026 * @param 4027 * dev: Pointer to struct rte_eth_dev. 4028 * filter: the pointer of the filter will be removed. 4029 */ 4030 void 4031 igb_remove_flex_filter(struct rte_eth_dev *dev, 4032 struct e1000_flex_filter *filter) 4033 { 4034 struct e1000_filter_info *filter_info = 4035 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4036 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4037 uint32_t wufc, i; 4038 uint32_t reg_off; 4039 4040 wufc = E1000_READ_REG(hw, E1000_WUFC); 4041 if (filter->index < E1000_MAX_FHFT) 4042 reg_off = E1000_FHFT(filter->index); 4043 else 4044 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4045 4046 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4047 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4048 4049 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4050 (~(E1000_WUFC_FLX0 << filter->index))); 4051 4052 filter_info->flex_mask &= ~(1 << filter->index); 4053 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4054 rte_free(filter); 4055 } 4056 4057 int 4058 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4059 struct rte_eth_flex_filter *filter, 4060 bool add) 4061 { 4062 struct e1000_filter_info *filter_info = 4063 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4064 struct e1000_flex_filter *flex_filter, *it; 4065 uint32_t mask; 4066 uint8_t shift, i; 4067 4068 flex_filter = rte_zmalloc("e1000_flex_filter", 4069 sizeof(struct e1000_flex_filter), 0); 4070 if (flex_filter == NULL) 4071 return -ENOMEM; 4072 4073 flex_filter->filter_info.len = filter->len; 4074 flex_filter->filter_info.priority = filter->priority; 4075 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4076 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4077 mask = 0; 4078 /* reverse bits in flex filter's mask*/ 4079 for (shift = 0; shift < CHAR_BIT; shift++) { 4080 if (filter->mask[i] & (0x01 << shift)) 4081 mask |= (0x80 >> shift); 4082 } 4083 flex_filter->filter_info.mask[i] = mask; 4084 } 4085 4086 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4087 &flex_filter->filter_info); 4088 if (it == NULL && !add) { 4089 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4090 rte_free(flex_filter); 4091 return -ENOENT; 4092 } 4093 if (it != NULL && add) { 4094 PMD_DRV_LOG(ERR, "filter exists."); 4095 rte_free(flex_filter); 4096 return -EEXIST; 4097 } 4098 4099 if (add) { 4100 flex_filter->queue = filter->queue; 4101 /* 4102 * look for an unused flex filter index 4103 * and insert the filter into the list. 4104 */ 4105 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4106 if (!(filter_info->flex_mask & (1 << i))) { 4107 filter_info->flex_mask |= 1 << i; 4108 flex_filter->index = i; 4109 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4110 flex_filter, 4111 entries); 4112 break; 4113 } 4114 } 4115 if (i >= E1000_MAX_FLEX_FILTERS) { 4116 PMD_DRV_LOG(ERR, "flex filters are full."); 4117 rte_free(flex_filter); 4118 return -ENOSYS; 4119 } 4120 4121 igb_inject_flex_filter(dev, flex_filter); 4122 4123 } else { 4124 igb_remove_flex_filter(dev, it); 4125 rte_free(flex_filter); 4126 } 4127 4128 return 0; 4129 } 4130 4131 static int 4132 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4133 struct rte_eth_flex_filter *filter) 4134 { 4135 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4136 struct e1000_filter_info *filter_info = 4137 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4138 struct e1000_flex_filter flex_filter, *it; 4139 uint32_t wufc, queueing, wufc_en = 0; 4140 4141 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4142 flex_filter.filter_info.len = filter->len; 4143 flex_filter.filter_info.priority = filter->priority; 4144 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4145 memcpy(flex_filter.filter_info.mask, filter->mask, 4146 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4147 4148 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4149 &flex_filter.filter_info); 4150 if (it == NULL) { 4151 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4152 return -ENOENT; 4153 } 4154 4155 wufc = E1000_READ_REG(hw, E1000_WUFC); 4156 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4157 4158 if ((wufc & wufc_en) == wufc_en) { 4159 uint32_t reg_off = 0; 4160 if (it->index < E1000_MAX_FHFT) 4161 reg_off = E1000_FHFT(it->index); 4162 else 4163 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4164 4165 queueing = E1000_READ_REG(hw, 4166 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4167 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4168 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4169 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4170 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4171 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4172 return 0; 4173 } 4174 return -ENOENT; 4175 } 4176 4177 static int 4178 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4179 enum rte_filter_op filter_op, 4180 void *arg) 4181 { 4182 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4183 struct rte_eth_flex_filter *filter; 4184 int ret = 0; 4185 4186 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4187 4188 if (filter_op == RTE_ETH_FILTER_NOP) 4189 return ret; 4190 4191 if (arg == NULL) { 4192 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4193 filter_op); 4194 return -EINVAL; 4195 } 4196 4197 filter = (struct rte_eth_flex_filter *)arg; 4198 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4199 || filter->len % sizeof(uint64_t) != 0) { 4200 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4201 return -EINVAL; 4202 } 4203 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4204 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4205 return -EINVAL; 4206 } 4207 4208 switch (filter_op) { 4209 case RTE_ETH_FILTER_ADD: 4210 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4211 break; 4212 case RTE_ETH_FILTER_DELETE: 4213 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4214 break; 4215 case RTE_ETH_FILTER_GET: 4216 ret = eth_igb_get_flex_filter(dev, filter); 4217 break; 4218 default: 4219 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4220 ret = -EINVAL; 4221 break; 4222 } 4223 4224 return ret; 4225 } 4226 4227 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4228 static inline int 4229 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4230 struct e1000_5tuple_filter_info *filter_info) 4231 { 4232 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4233 return -EINVAL; 4234 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4235 return -EINVAL; /* filter index is out of range. */ 4236 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 4237 return -EINVAL; /* flags is invalid. */ 4238 4239 switch (filter->dst_ip_mask) { 4240 case UINT32_MAX: 4241 filter_info->dst_ip_mask = 0; 4242 filter_info->dst_ip = filter->dst_ip; 4243 break; 4244 case 0: 4245 filter_info->dst_ip_mask = 1; 4246 break; 4247 default: 4248 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4249 return -EINVAL; 4250 } 4251 4252 switch (filter->src_ip_mask) { 4253 case UINT32_MAX: 4254 filter_info->src_ip_mask = 0; 4255 filter_info->src_ip = filter->src_ip; 4256 break; 4257 case 0: 4258 filter_info->src_ip_mask = 1; 4259 break; 4260 default: 4261 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4262 return -EINVAL; 4263 } 4264 4265 switch (filter->dst_port_mask) { 4266 case UINT16_MAX: 4267 filter_info->dst_port_mask = 0; 4268 filter_info->dst_port = filter->dst_port; 4269 break; 4270 case 0: 4271 filter_info->dst_port_mask = 1; 4272 break; 4273 default: 4274 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4275 return -EINVAL; 4276 } 4277 4278 switch (filter->src_port_mask) { 4279 case UINT16_MAX: 4280 filter_info->src_port_mask = 0; 4281 filter_info->src_port = filter->src_port; 4282 break; 4283 case 0: 4284 filter_info->src_port_mask = 1; 4285 break; 4286 default: 4287 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4288 return -EINVAL; 4289 } 4290 4291 switch (filter->proto_mask) { 4292 case UINT8_MAX: 4293 filter_info->proto_mask = 0; 4294 filter_info->proto = filter->proto; 4295 break; 4296 case 0: 4297 filter_info->proto_mask = 1; 4298 break; 4299 default: 4300 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4301 return -EINVAL; 4302 } 4303 4304 filter_info->priority = (uint8_t)filter->priority; 4305 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4306 filter_info->tcp_flags = filter->tcp_flags; 4307 else 4308 filter_info->tcp_flags = 0; 4309 4310 return 0; 4311 } 4312 4313 static inline struct e1000_5tuple_filter * 4314 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4315 struct e1000_5tuple_filter_info *key) 4316 { 4317 struct e1000_5tuple_filter *it; 4318 4319 TAILQ_FOREACH(it, filter_list, entries) { 4320 if (memcmp(key, &it->filter_info, 4321 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4322 return it; 4323 } 4324 } 4325 return NULL; 4326 } 4327 4328 /* inject a igb 5-tuple filter to HW */ 4329 static inline void 4330 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4331 struct e1000_5tuple_filter *filter) 4332 { 4333 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4334 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4335 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4336 uint8_t i; 4337 4338 i = filter->index; 4339 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4340 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4341 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4342 if (filter->filter_info.dst_ip_mask == 0) 4343 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4344 if (filter->filter_info.src_port_mask == 0) 4345 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4346 if (filter->filter_info.proto_mask == 0) 4347 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4348 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4349 E1000_FTQF_QUEUE_MASK; 4350 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4351 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4352 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4353 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4354 4355 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4356 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4357 4358 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4359 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4360 imir |= E1000_IMIR_PORT_BP; 4361 else 4362 imir &= ~E1000_IMIR_PORT_BP; 4363 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4364 4365 /* tcp flags bits setting. */ 4366 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 4367 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 4368 imir_ext |= E1000_IMIREXT_CTRL_URG; 4369 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 4370 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4371 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 4372 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4373 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 4374 imir_ext |= E1000_IMIREXT_CTRL_RST; 4375 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 4376 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4377 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 4378 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4379 } else { 4380 imir_ext |= E1000_IMIREXT_CTRL_BP; 4381 } 4382 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4383 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4384 } 4385 4386 /* 4387 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4388 * 4389 * @param 4390 * dev: Pointer to struct rte_eth_dev. 4391 * ntuple_filter: ponter to the filter that will be added. 4392 * 4393 * @return 4394 * - On success, zero. 4395 * - On failure, a negative value. 4396 */ 4397 static int 4398 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4399 struct rte_eth_ntuple_filter *ntuple_filter) 4400 { 4401 struct e1000_filter_info *filter_info = 4402 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4403 struct e1000_5tuple_filter *filter; 4404 uint8_t i; 4405 int ret; 4406 4407 filter = rte_zmalloc("e1000_5tuple_filter", 4408 sizeof(struct e1000_5tuple_filter), 0); 4409 if (filter == NULL) 4410 return -ENOMEM; 4411 4412 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4413 &filter->filter_info); 4414 if (ret < 0) { 4415 rte_free(filter); 4416 return ret; 4417 } 4418 4419 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4420 &filter->filter_info) != NULL) { 4421 PMD_DRV_LOG(ERR, "filter exists."); 4422 rte_free(filter); 4423 return -EEXIST; 4424 } 4425 filter->queue = ntuple_filter->queue; 4426 4427 /* 4428 * look for an unused 5tuple filter index, 4429 * and insert the filter to list. 4430 */ 4431 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4432 if (!(filter_info->fivetuple_mask & (1 << i))) { 4433 filter_info->fivetuple_mask |= 1 << i; 4434 filter->index = i; 4435 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4436 filter, 4437 entries); 4438 break; 4439 } 4440 } 4441 if (i >= E1000_MAX_FTQF_FILTERS) { 4442 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4443 rte_free(filter); 4444 return -ENOSYS; 4445 } 4446 4447 igb_inject_5tuple_filter_82576(dev, filter); 4448 return 0; 4449 } 4450 4451 int 4452 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4453 struct e1000_5tuple_filter *filter) 4454 { 4455 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4456 struct e1000_filter_info *filter_info = 4457 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4458 4459 filter_info->fivetuple_mask &= ~(1 << filter->index); 4460 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4461 rte_free(filter); 4462 4463 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4464 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4465 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4466 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4467 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4468 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4469 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4470 return 0; 4471 } 4472 4473 /* 4474 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4475 * 4476 * @param 4477 * dev: Pointer to struct rte_eth_dev. 4478 * ntuple_filter: ponter to the filter that will be removed. 4479 * 4480 * @return 4481 * - On success, zero. 4482 * - On failure, a negative value. 4483 */ 4484 static int 4485 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4486 struct rte_eth_ntuple_filter *ntuple_filter) 4487 { 4488 struct e1000_filter_info *filter_info = 4489 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4490 struct e1000_5tuple_filter_info filter_5tuple; 4491 struct e1000_5tuple_filter *filter; 4492 int ret; 4493 4494 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4495 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4496 &filter_5tuple); 4497 if (ret < 0) 4498 return ret; 4499 4500 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4501 &filter_5tuple); 4502 if (filter == NULL) { 4503 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4504 return -ENOENT; 4505 } 4506 4507 igb_delete_5tuple_filter_82576(dev, filter); 4508 4509 return 0; 4510 } 4511 4512 static int 4513 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4514 { 4515 uint32_t rctl; 4516 struct e1000_hw *hw; 4517 struct rte_eth_dev_info dev_info; 4518 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4519 int ret; 4520 4521 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4522 4523 #ifdef RTE_LIBRTE_82571_SUPPORT 4524 /* XXX: not bigger than max_rx_pktlen */ 4525 if (hw->mac.type == e1000_82571) 4526 return -ENOTSUP; 4527 #endif 4528 ret = eth_igb_infos_get(dev, &dev_info); 4529 if (ret != 0) 4530 return ret; 4531 4532 /* check that mtu is within the allowed range */ 4533 if (mtu < RTE_ETHER_MIN_MTU || 4534 frame_size > dev_info.max_rx_pktlen) 4535 return -EINVAL; 4536 4537 /* refuse mtu that requires the support of scattered packets when this 4538 * feature has not been enabled before. */ 4539 if (!dev->data->scattered_rx && 4540 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4541 return -EINVAL; 4542 4543 rctl = E1000_READ_REG(hw, E1000_RCTL); 4544 4545 /* switch to jumbo mode if needed */ 4546 if (frame_size > RTE_ETHER_MAX_LEN) { 4547 dev->data->dev_conf.rxmode.offloads |= 4548 DEV_RX_OFFLOAD_JUMBO_FRAME; 4549 rctl |= E1000_RCTL_LPE; 4550 } else { 4551 dev->data->dev_conf.rxmode.offloads &= 4552 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4553 rctl &= ~E1000_RCTL_LPE; 4554 } 4555 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4556 4557 /* update max frame size */ 4558 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4559 4560 E1000_WRITE_REG(hw, E1000_RLPML, 4561 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4562 4563 return 0; 4564 } 4565 4566 /* 4567 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4568 * 4569 * @param 4570 * dev: Pointer to struct rte_eth_dev. 4571 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4572 * add: if true, add filter, if false, remove filter 4573 * 4574 * @return 4575 * - On success, zero. 4576 * - On failure, a negative value. 4577 */ 4578 int 4579 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4580 struct rte_eth_ntuple_filter *ntuple_filter, 4581 bool add) 4582 { 4583 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4584 int ret; 4585 4586 switch (ntuple_filter->flags) { 4587 case RTE_5TUPLE_FLAGS: 4588 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4589 if (hw->mac.type != e1000_82576) 4590 return -ENOTSUP; 4591 if (add) 4592 ret = igb_add_5tuple_filter_82576(dev, 4593 ntuple_filter); 4594 else 4595 ret = igb_remove_5tuple_filter_82576(dev, 4596 ntuple_filter); 4597 break; 4598 case RTE_2TUPLE_FLAGS: 4599 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4600 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4601 hw->mac.type != e1000_i210 && 4602 hw->mac.type != e1000_i211) 4603 return -ENOTSUP; 4604 if (add) 4605 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4606 else 4607 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4608 break; 4609 default: 4610 ret = -EINVAL; 4611 break; 4612 } 4613 4614 return ret; 4615 } 4616 4617 /* 4618 * igb_get_ntuple_filter - get a ntuple filter 4619 * 4620 * @param 4621 * dev: Pointer to struct rte_eth_dev. 4622 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4623 * 4624 * @return 4625 * - On success, zero. 4626 * - On failure, a negative value. 4627 */ 4628 static int 4629 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4630 struct rte_eth_ntuple_filter *ntuple_filter) 4631 { 4632 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4633 struct e1000_filter_info *filter_info = 4634 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4635 struct e1000_5tuple_filter_info filter_5tuple; 4636 struct e1000_2tuple_filter_info filter_2tuple; 4637 struct e1000_5tuple_filter *p_5tuple_filter; 4638 struct e1000_2tuple_filter *p_2tuple_filter; 4639 int ret; 4640 4641 switch (ntuple_filter->flags) { 4642 case RTE_5TUPLE_FLAGS: 4643 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4644 if (hw->mac.type != e1000_82576) 4645 return -ENOTSUP; 4646 memset(&filter_5tuple, 4647 0, 4648 sizeof(struct e1000_5tuple_filter_info)); 4649 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4650 &filter_5tuple); 4651 if (ret < 0) 4652 return ret; 4653 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4654 &filter_info->fivetuple_list, 4655 &filter_5tuple); 4656 if (p_5tuple_filter == NULL) { 4657 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4658 return -ENOENT; 4659 } 4660 ntuple_filter->queue = p_5tuple_filter->queue; 4661 break; 4662 case RTE_2TUPLE_FLAGS: 4663 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4664 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4665 return -ENOTSUP; 4666 memset(&filter_2tuple, 4667 0, 4668 sizeof(struct e1000_2tuple_filter_info)); 4669 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4670 if (ret < 0) 4671 return ret; 4672 p_2tuple_filter = igb_2tuple_filter_lookup( 4673 &filter_info->twotuple_list, 4674 &filter_2tuple); 4675 if (p_2tuple_filter == NULL) { 4676 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4677 return -ENOENT; 4678 } 4679 ntuple_filter->queue = p_2tuple_filter->queue; 4680 break; 4681 default: 4682 ret = -EINVAL; 4683 break; 4684 } 4685 4686 return 0; 4687 } 4688 4689 /* 4690 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4691 * @dev: pointer to rte_eth_dev structure 4692 * @filter_op:operation will be taken. 4693 * @arg: a pointer to specific structure corresponding to the filter_op 4694 */ 4695 static int 4696 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4697 enum rte_filter_op filter_op, 4698 void *arg) 4699 { 4700 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4701 int ret; 4702 4703 MAC_TYPE_FILTER_SUP(hw->mac.type); 4704 4705 if (filter_op == RTE_ETH_FILTER_NOP) 4706 return 0; 4707 4708 if (arg == NULL) { 4709 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4710 filter_op); 4711 return -EINVAL; 4712 } 4713 4714 switch (filter_op) { 4715 case RTE_ETH_FILTER_ADD: 4716 ret = igb_add_del_ntuple_filter(dev, 4717 (struct rte_eth_ntuple_filter *)arg, 4718 TRUE); 4719 break; 4720 case RTE_ETH_FILTER_DELETE: 4721 ret = igb_add_del_ntuple_filter(dev, 4722 (struct rte_eth_ntuple_filter *)arg, 4723 FALSE); 4724 break; 4725 case RTE_ETH_FILTER_GET: 4726 ret = igb_get_ntuple_filter(dev, 4727 (struct rte_eth_ntuple_filter *)arg); 4728 break; 4729 default: 4730 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4731 ret = -EINVAL; 4732 break; 4733 } 4734 return ret; 4735 } 4736 4737 static inline int 4738 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4739 uint16_t ethertype) 4740 { 4741 int i; 4742 4743 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4744 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4745 (filter_info->ethertype_mask & (1 << i))) 4746 return i; 4747 } 4748 return -1; 4749 } 4750 4751 static inline int 4752 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4753 uint16_t ethertype, uint32_t etqf) 4754 { 4755 int i; 4756 4757 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4758 if (!(filter_info->ethertype_mask & (1 << i))) { 4759 filter_info->ethertype_mask |= 1 << i; 4760 filter_info->ethertype_filters[i].ethertype = ethertype; 4761 filter_info->ethertype_filters[i].etqf = etqf; 4762 return i; 4763 } 4764 } 4765 return -1; 4766 } 4767 4768 int 4769 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4770 uint8_t idx) 4771 { 4772 if (idx >= E1000_MAX_ETQF_FILTERS) 4773 return -1; 4774 filter_info->ethertype_mask &= ~(1 << idx); 4775 filter_info->ethertype_filters[idx].ethertype = 0; 4776 filter_info->ethertype_filters[idx].etqf = 0; 4777 return idx; 4778 } 4779 4780 4781 int 4782 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4783 struct rte_eth_ethertype_filter *filter, 4784 bool add) 4785 { 4786 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4787 struct e1000_filter_info *filter_info = 4788 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4789 uint32_t etqf = 0; 4790 int ret; 4791 4792 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 4793 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 4794 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4795 " ethertype filter.", filter->ether_type); 4796 return -EINVAL; 4797 } 4798 4799 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4800 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4801 return -EINVAL; 4802 } 4803 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4804 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4805 return -EINVAL; 4806 } 4807 4808 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4809 if (ret >= 0 && add) { 4810 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4811 filter->ether_type); 4812 return -EEXIST; 4813 } 4814 if (ret < 0 && !add) { 4815 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4816 filter->ether_type); 4817 return -ENOENT; 4818 } 4819 4820 if (add) { 4821 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4822 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4823 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4824 ret = igb_ethertype_filter_insert(filter_info, 4825 filter->ether_type, etqf); 4826 if (ret < 0) { 4827 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4828 return -ENOSYS; 4829 } 4830 } else { 4831 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4832 if (ret < 0) 4833 return -ENOSYS; 4834 } 4835 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4836 E1000_WRITE_FLUSH(hw); 4837 4838 return 0; 4839 } 4840 4841 static int 4842 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4843 struct rte_eth_ethertype_filter *filter) 4844 { 4845 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4846 struct e1000_filter_info *filter_info = 4847 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4848 uint32_t etqf; 4849 int ret; 4850 4851 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4852 if (ret < 0) { 4853 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4854 filter->ether_type); 4855 return -ENOENT; 4856 } 4857 4858 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4859 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4860 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4861 filter->flags = 0; 4862 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4863 E1000_ETQF_QUEUE_SHIFT; 4864 return 0; 4865 } 4866 4867 return -ENOENT; 4868 } 4869 4870 /* 4871 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4872 * @dev: pointer to rte_eth_dev structure 4873 * @filter_op:operation will be taken. 4874 * @arg: a pointer to specific structure corresponding to the filter_op 4875 */ 4876 static int 4877 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4878 enum rte_filter_op filter_op, 4879 void *arg) 4880 { 4881 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4882 int ret; 4883 4884 MAC_TYPE_FILTER_SUP(hw->mac.type); 4885 4886 if (filter_op == RTE_ETH_FILTER_NOP) 4887 return 0; 4888 4889 if (arg == NULL) { 4890 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4891 filter_op); 4892 return -EINVAL; 4893 } 4894 4895 switch (filter_op) { 4896 case RTE_ETH_FILTER_ADD: 4897 ret = igb_add_del_ethertype_filter(dev, 4898 (struct rte_eth_ethertype_filter *)arg, 4899 TRUE); 4900 break; 4901 case RTE_ETH_FILTER_DELETE: 4902 ret = igb_add_del_ethertype_filter(dev, 4903 (struct rte_eth_ethertype_filter *)arg, 4904 FALSE); 4905 break; 4906 case RTE_ETH_FILTER_GET: 4907 ret = igb_get_ethertype_filter(dev, 4908 (struct rte_eth_ethertype_filter *)arg); 4909 break; 4910 default: 4911 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4912 ret = -EINVAL; 4913 break; 4914 } 4915 return ret; 4916 } 4917 4918 static int 4919 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4920 enum rte_filter_type filter_type, 4921 enum rte_filter_op filter_op, 4922 void *arg) 4923 { 4924 int ret = 0; 4925 4926 switch (filter_type) { 4927 case RTE_ETH_FILTER_NTUPLE: 4928 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4929 break; 4930 case RTE_ETH_FILTER_ETHERTYPE: 4931 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4932 break; 4933 case RTE_ETH_FILTER_SYN: 4934 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4935 break; 4936 case RTE_ETH_FILTER_FLEXIBLE: 4937 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4938 break; 4939 case RTE_ETH_FILTER_GENERIC: 4940 if (filter_op != RTE_ETH_FILTER_GET) 4941 return -EINVAL; 4942 *(const void **)arg = &igb_flow_ops; 4943 break; 4944 default: 4945 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4946 filter_type); 4947 break; 4948 } 4949 4950 return ret; 4951 } 4952 4953 static int 4954 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4955 struct rte_ether_addr *mc_addr_set, 4956 uint32_t nb_mc_addr) 4957 { 4958 struct e1000_hw *hw; 4959 4960 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4961 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4962 return 0; 4963 } 4964 4965 static uint64_t 4966 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4967 { 4968 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4969 uint64_t systime_cycles; 4970 4971 switch (hw->mac.type) { 4972 case e1000_i210: 4973 case e1000_i211: 4974 /* 4975 * Need to read System Time Residue Register to be able 4976 * to read the other two registers. 4977 */ 4978 E1000_READ_REG(hw, E1000_SYSTIMR); 4979 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4980 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4981 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4982 * NSEC_PER_SEC; 4983 break; 4984 case e1000_82580: 4985 case e1000_i350: 4986 case e1000_i354: 4987 /* 4988 * Need to read System Time Residue Register to be able 4989 * to read the other two registers. 4990 */ 4991 E1000_READ_REG(hw, E1000_SYSTIMR); 4992 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4993 /* Only the 8 LSB are valid. */ 4994 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4995 & 0xff) << 32; 4996 break; 4997 default: 4998 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4999 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 5000 << 32; 5001 break; 5002 } 5003 5004 return systime_cycles; 5005 } 5006 5007 static uint64_t 5008 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 5009 { 5010 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5011 uint64_t rx_tstamp_cycles; 5012 5013 switch (hw->mac.type) { 5014 case e1000_i210: 5015 case e1000_i211: 5016 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5017 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5018 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 5019 * NSEC_PER_SEC; 5020 break; 5021 case e1000_82580: 5022 case e1000_i350: 5023 case e1000_i354: 5024 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5025 /* Only the 8 LSB are valid. */ 5026 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 5027 & 0xff) << 32; 5028 break; 5029 default: 5030 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 5031 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 5032 << 32; 5033 break; 5034 } 5035 5036 return rx_tstamp_cycles; 5037 } 5038 5039 static uint64_t 5040 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 5041 { 5042 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5043 uint64_t tx_tstamp_cycles; 5044 5045 switch (hw->mac.type) { 5046 case e1000_i210: 5047 case e1000_i211: 5048 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5049 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5050 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5051 * NSEC_PER_SEC; 5052 break; 5053 case e1000_82580: 5054 case e1000_i350: 5055 case e1000_i354: 5056 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5057 /* Only the 8 LSB are valid. */ 5058 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5059 & 0xff) << 32; 5060 break; 5061 default: 5062 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5063 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5064 << 32; 5065 break; 5066 } 5067 5068 return tx_tstamp_cycles; 5069 } 5070 5071 static void 5072 igb_start_timecounters(struct rte_eth_dev *dev) 5073 { 5074 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5075 struct e1000_adapter *adapter = dev->data->dev_private; 5076 uint32_t incval = 1; 5077 uint32_t shift = 0; 5078 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5079 5080 switch (hw->mac.type) { 5081 case e1000_82580: 5082 case e1000_i350: 5083 case e1000_i354: 5084 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5085 mask = (1ULL << 40) - 1; 5086 /* fall-through */ 5087 case e1000_i210: 5088 case e1000_i211: 5089 /* 5090 * Start incrementing the register 5091 * used to timestamp PTP packets. 5092 */ 5093 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5094 break; 5095 case e1000_82576: 5096 incval = E1000_INCVALUE_82576; 5097 shift = IGB_82576_TSYNC_SHIFT; 5098 E1000_WRITE_REG(hw, E1000_TIMINCA, 5099 E1000_INCPERIOD_82576 | incval); 5100 break; 5101 default: 5102 /* Not supported */ 5103 return; 5104 } 5105 5106 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5107 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5108 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5109 5110 adapter->systime_tc.cc_mask = mask; 5111 adapter->systime_tc.cc_shift = shift; 5112 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5113 5114 adapter->rx_tstamp_tc.cc_mask = mask; 5115 adapter->rx_tstamp_tc.cc_shift = shift; 5116 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5117 5118 adapter->tx_tstamp_tc.cc_mask = mask; 5119 adapter->tx_tstamp_tc.cc_shift = shift; 5120 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5121 } 5122 5123 static int 5124 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5125 { 5126 struct e1000_adapter *adapter = dev->data->dev_private; 5127 5128 adapter->systime_tc.nsec += delta; 5129 adapter->rx_tstamp_tc.nsec += delta; 5130 adapter->tx_tstamp_tc.nsec += delta; 5131 5132 return 0; 5133 } 5134 5135 static int 5136 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5137 { 5138 uint64_t ns; 5139 struct e1000_adapter *adapter = dev->data->dev_private; 5140 5141 ns = rte_timespec_to_ns(ts); 5142 5143 /* Set the timecounters to a new value. */ 5144 adapter->systime_tc.nsec = ns; 5145 adapter->rx_tstamp_tc.nsec = ns; 5146 adapter->tx_tstamp_tc.nsec = ns; 5147 5148 return 0; 5149 } 5150 5151 static int 5152 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5153 { 5154 uint64_t ns, systime_cycles; 5155 struct e1000_adapter *adapter = dev->data->dev_private; 5156 5157 systime_cycles = igb_read_systime_cyclecounter(dev); 5158 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5159 *ts = rte_ns_to_timespec(ns); 5160 5161 return 0; 5162 } 5163 5164 static int 5165 igb_timesync_enable(struct rte_eth_dev *dev) 5166 { 5167 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5168 uint32_t tsync_ctl; 5169 uint32_t tsauxc; 5170 5171 /* Stop the timesync system time. */ 5172 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5173 /* Reset the timesync system time value. */ 5174 switch (hw->mac.type) { 5175 case e1000_82580: 5176 case e1000_i350: 5177 case e1000_i354: 5178 case e1000_i210: 5179 case e1000_i211: 5180 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5181 /* fall-through */ 5182 case e1000_82576: 5183 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5184 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5185 break; 5186 default: 5187 /* Not supported. */ 5188 return -ENOTSUP; 5189 } 5190 5191 /* Enable system time for it isn't on by default. */ 5192 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5193 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5194 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5195 5196 igb_start_timecounters(dev); 5197 5198 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5199 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5200 (RTE_ETHER_TYPE_1588 | 5201 E1000_ETQF_FILTER_ENABLE | 5202 E1000_ETQF_1588)); 5203 5204 /* Enable timestamping of received PTP packets. */ 5205 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5206 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5207 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5208 5209 /* Enable Timestamping of transmitted PTP packets. */ 5210 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5211 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5212 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5213 5214 return 0; 5215 } 5216 5217 static int 5218 igb_timesync_disable(struct rte_eth_dev *dev) 5219 { 5220 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5221 uint32_t tsync_ctl; 5222 5223 /* Disable timestamping of transmitted PTP packets. */ 5224 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5225 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5226 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5227 5228 /* Disable timestamping of received PTP packets. */ 5229 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5230 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5231 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5232 5233 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5234 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5235 5236 /* Stop incrementating the System Time registers. */ 5237 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5238 5239 return 0; 5240 } 5241 5242 static int 5243 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5244 struct timespec *timestamp, 5245 uint32_t flags __rte_unused) 5246 { 5247 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5248 struct e1000_adapter *adapter = dev->data->dev_private; 5249 uint32_t tsync_rxctl; 5250 uint64_t rx_tstamp_cycles; 5251 uint64_t ns; 5252 5253 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5254 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5255 return -EINVAL; 5256 5257 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5258 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5259 *timestamp = rte_ns_to_timespec(ns); 5260 5261 return 0; 5262 } 5263 5264 static int 5265 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5266 struct timespec *timestamp) 5267 { 5268 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5269 struct e1000_adapter *adapter = dev->data->dev_private; 5270 uint32_t tsync_txctl; 5271 uint64_t tx_tstamp_cycles; 5272 uint64_t ns; 5273 5274 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5275 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5276 return -EINVAL; 5277 5278 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5279 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5280 *timestamp = rte_ns_to_timespec(ns); 5281 5282 return 0; 5283 } 5284 5285 static int 5286 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5287 { 5288 int count = 0; 5289 int g_ind = 0; 5290 const struct reg_info *reg_group; 5291 5292 while ((reg_group = igb_regs[g_ind++])) 5293 count += igb_reg_group_count(reg_group); 5294 5295 return count; 5296 } 5297 5298 static int 5299 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5300 { 5301 int count = 0; 5302 int g_ind = 0; 5303 const struct reg_info *reg_group; 5304 5305 while ((reg_group = igbvf_regs[g_ind++])) 5306 count += igb_reg_group_count(reg_group); 5307 5308 return count; 5309 } 5310 5311 static int 5312 eth_igb_get_regs(struct rte_eth_dev *dev, 5313 struct rte_dev_reg_info *regs) 5314 { 5315 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5316 uint32_t *data = regs->data; 5317 int g_ind = 0; 5318 int count = 0; 5319 const struct reg_info *reg_group; 5320 5321 if (data == NULL) { 5322 regs->length = eth_igb_get_reg_length(dev); 5323 regs->width = sizeof(uint32_t); 5324 return 0; 5325 } 5326 5327 /* Support only full register dump */ 5328 if ((regs->length == 0) || 5329 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5330 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5331 hw->device_id; 5332 while ((reg_group = igb_regs[g_ind++])) 5333 count += igb_read_regs_group(dev, &data[count], 5334 reg_group); 5335 return 0; 5336 } 5337 5338 return -ENOTSUP; 5339 } 5340 5341 static int 5342 igbvf_get_regs(struct rte_eth_dev *dev, 5343 struct rte_dev_reg_info *regs) 5344 { 5345 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5346 uint32_t *data = regs->data; 5347 int g_ind = 0; 5348 int count = 0; 5349 const struct reg_info *reg_group; 5350 5351 if (data == NULL) { 5352 regs->length = igbvf_get_reg_length(dev); 5353 regs->width = sizeof(uint32_t); 5354 return 0; 5355 } 5356 5357 /* Support only full register dump */ 5358 if ((regs->length == 0) || 5359 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5360 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5361 hw->device_id; 5362 while ((reg_group = igbvf_regs[g_ind++])) 5363 count += igb_read_regs_group(dev, &data[count], 5364 reg_group); 5365 return 0; 5366 } 5367 5368 return -ENOTSUP; 5369 } 5370 5371 static int 5372 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5373 { 5374 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5375 5376 /* Return unit is byte count */ 5377 return hw->nvm.word_size * 2; 5378 } 5379 5380 static int 5381 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5382 struct rte_dev_eeprom_info *in_eeprom) 5383 { 5384 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5385 struct e1000_nvm_info *nvm = &hw->nvm; 5386 uint16_t *data = in_eeprom->data; 5387 int first, length; 5388 5389 first = in_eeprom->offset >> 1; 5390 length = in_eeprom->length >> 1; 5391 if ((first >= hw->nvm.word_size) || 5392 ((first + length) >= hw->nvm.word_size)) 5393 return -EINVAL; 5394 5395 in_eeprom->magic = hw->vendor_id | 5396 ((uint32_t)hw->device_id << 16); 5397 5398 if ((nvm->ops.read) == NULL) 5399 return -ENOTSUP; 5400 5401 return nvm->ops.read(hw, first, length, data); 5402 } 5403 5404 static int 5405 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5406 struct rte_dev_eeprom_info *in_eeprom) 5407 { 5408 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5409 struct e1000_nvm_info *nvm = &hw->nvm; 5410 uint16_t *data = in_eeprom->data; 5411 int first, length; 5412 5413 first = in_eeprom->offset >> 1; 5414 length = in_eeprom->length >> 1; 5415 if ((first >= hw->nvm.word_size) || 5416 ((first + length) >= hw->nvm.word_size)) 5417 return -EINVAL; 5418 5419 in_eeprom->magic = (uint32_t)hw->vendor_id | 5420 ((uint32_t)hw->device_id << 16); 5421 5422 if ((nvm->ops.write) == NULL) 5423 return -ENOTSUP; 5424 return nvm->ops.write(hw, first, length, data); 5425 } 5426 5427 static int 5428 eth_igb_get_module_info(struct rte_eth_dev *dev, 5429 struct rte_eth_dev_module_info *modinfo) 5430 { 5431 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5432 5433 uint32_t status = 0; 5434 uint16_t sff8472_rev, addr_mode; 5435 bool page_swap = false; 5436 5437 if (hw->phy.media_type == e1000_media_type_copper || 5438 hw->phy.media_type == e1000_media_type_unknown) 5439 return -EOPNOTSUPP; 5440 5441 /* Check whether we support SFF-8472 or not */ 5442 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5443 if (status) 5444 return -EIO; 5445 5446 /* addressing mode is not supported */ 5447 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5448 if (status) 5449 return -EIO; 5450 5451 /* addressing mode is not supported */ 5452 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5453 PMD_DRV_LOG(ERR, 5454 "Address change required to access page 0xA2, " 5455 "but not supported. Please report the module " 5456 "type to the driver maintainers.\n"); 5457 page_swap = true; 5458 } 5459 5460 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5461 /* We have an SFP, but it does not support SFF-8472 */ 5462 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5463 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5464 } else { 5465 /* We have an SFP which supports a revision of SFF-8472 */ 5466 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5467 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5468 } 5469 5470 return 0; 5471 } 5472 5473 static int 5474 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5475 struct rte_dev_eeprom_info *info) 5476 { 5477 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5478 5479 uint32_t status = 0; 5480 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5481 u16 first_word, last_word; 5482 int i = 0; 5483 5484 if (info->length == 0) 5485 return -EINVAL; 5486 5487 first_word = info->offset >> 1; 5488 last_word = (info->offset + info->length - 1) >> 1; 5489 5490 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5491 for (i = 0; i < last_word - first_word + 1; i++) { 5492 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5493 &dataword[i]); 5494 if (status) { 5495 /* Error occurred while reading module */ 5496 return -EIO; 5497 } 5498 5499 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5500 } 5501 5502 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5503 5504 return 0; 5505 } 5506 5507 static int 5508 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5509 { 5510 struct e1000_hw *hw = 5511 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5512 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5513 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5514 uint32_t vec = E1000_MISC_VEC_ID; 5515 5516 if (rte_intr_allow_others(intr_handle)) 5517 vec = E1000_RX_VEC_START; 5518 5519 uint32_t mask = 1 << (queue_id + vec); 5520 5521 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5522 E1000_WRITE_FLUSH(hw); 5523 5524 return 0; 5525 } 5526 5527 static int 5528 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5529 { 5530 struct e1000_hw *hw = 5531 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5532 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5533 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5534 uint32_t vec = E1000_MISC_VEC_ID; 5535 5536 if (rte_intr_allow_others(intr_handle)) 5537 vec = E1000_RX_VEC_START; 5538 5539 uint32_t mask = 1 << (queue_id + vec); 5540 uint32_t regval; 5541 5542 regval = E1000_READ_REG(hw, E1000_EIMS); 5543 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5544 E1000_WRITE_FLUSH(hw); 5545 5546 rte_intr_ack(intr_handle); 5547 5548 return 0; 5549 } 5550 5551 static void 5552 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5553 uint8_t index, uint8_t offset) 5554 { 5555 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5556 5557 /* clear bits */ 5558 val &= ~((uint32_t)0xFF << offset); 5559 5560 /* write vector and valid bit */ 5561 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5562 5563 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5564 } 5565 5566 static void 5567 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5568 uint8_t queue, uint8_t msix_vector) 5569 { 5570 uint32_t tmp = 0; 5571 5572 if (hw->mac.type == e1000_82575) { 5573 if (direction == 0) 5574 tmp = E1000_EICR_RX_QUEUE0 << queue; 5575 else if (direction == 1) 5576 tmp = E1000_EICR_TX_QUEUE0 << queue; 5577 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5578 } else if (hw->mac.type == e1000_82576) { 5579 if ((direction == 0) || (direction == 1)) 5580 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5581 ((queue & 0x8) << 1) + 5582 8 * direction); 5583 } else if ((hw->mac.type == e1000_82580) || 5584 (hw->mac.type == e1000_i350) || 5585 (hw->mac.type == e1000_i354) || 5586 (hw->mac.type == e1000_i210) || 5587 (hw->mac.type == e1000_i211)) { 5588 if ((direction == 0) || (direction == 1)) 5589 eth_igb_write_ivar(hw, msix_vector, 5590 queue >> 1, 5591 ((queue & 0x1) << 4) + 5592 8 * direction); 5593 } 5594 } 5595 5596 /* Sets up the hardware to generate MSI-X interrupts properly 5597 * @hw 5598 * board private structure 5599 */ 5600 static void 5601 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5602 { 5603 int queue_id; 5604 uint32_t tmpval, regval, intr_mask; 5605 struct e1000_hw *hw = 5606 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5607 uint32_t vec = E1000_MISC_VEC_ID; 5608 uint32_t base = E1000_MISC_VEC_ID; 5609 uint32_t misc_shift = 0; 5610 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5611 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5612 5613 /* won't configure msix register if no mapping is done 5614 * between intr vector and event fd 5615 */ 5616 if (!rte_intr_dp_is_en(intr_handle)) 5617 return; 5618 5619 if (rte_intr_allow_others(intr_handle)) { 5620 vec = base = E1000_RX_VEC_START; 5621 misc_shift = 1; 5622 } 5623 5624 /* set interrupt vector for other causes */ 5625 if (hw->mac.type == e1000_82575) { 5626 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5627 /* enable MSI-X PBA support */ 5628 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5629 5630 /* Auto-Mask interrupts upon ICR read */ 5631 tmpval |= E1000_CTRL_EXT_EIAME; 5632 tmpval |= E1000_CTRL_EXT_IRCA; 5633 5634 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5635 5636 /* enable msix_other interrupt */ 5637 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5638 regval = E1000_READ_REG(hw, E1000_EIAC); 5639 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5640 regval = E1000_READ_REG(hw, E1000_EIAM); 5641 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5642 } else if ((hw->mac.type == e1000_82576) || 5643 (hw->mac.type == e1000_82580) || 5644 (hw->mac.type == e1000_i350) || 5645 (hw->mac.type == e1000_i354) || 5646 (hw->mac.type == e1000_i210) || 5647 (hw->mac.type == e1000_i211)) { 5648 /* turn on MSI-X capability first */ 5649 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5650 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5651 E1000_GPIE_NSICR); 5652 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5653 misc_shift; 5654 5655 if (dev->data->dev_conf.intr_conf.lsc != 0) 5656 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5657 5658 regval = E1000_READ_REG(hw, E1000_EIAC); 5659 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5660 5661 /* enable msix_other interrupt */ 5662 regval = E1000_READ_REG(hw, E1000_EIMS); 5663 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5664 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5665 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5666 } 5667 5668 /* use EIAM to auto-mask when MSI-X interrupt 5669 * is asserted, this saves a register write for every interrupt 5670 */ 5671 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5672 misc_shift; 5673 5674 if (dev->data->dev_conf.intr_conf.lsc != 0) 5675 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5676 5677 regval = E1000_READ_REG(hw, E1000_EIAM); 5678 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5679 5680 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5681 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5682 intr_handle->intr_vec[queue_id] = vec; 5683 if (vec < base + intr_handle->nb_efd - 1) 5684 vec++; 5685 } 5686 5687 E1000_WRITE_FLUSH(hw); 5688 } 5689 5690 /* restore n-tuple filter */ 5691 static inline void 5692 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5693 { 5694 struct e1000_filter_info *filter_info = 5695 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5696 struct e1000_5tuple_filter *p_5tuple; 5697 struct e1000_2tuple_filter *p_2tuple; 5698 5699 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5700 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5701 } 5702 5703 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5704 igb_inject_2uple_filter(dev, p_2tuple); 5705 } 5706 } 5707 5708 /* restore SYN filter */ 5709 static inline void 5710 igb_syn_filter_restore(struct rte_eth_dev *dev) 5711 { 5712 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5713 struct e1000_filter_info *filter_info = 5714 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5715 uint32_t synqf; 5716 5717 synqf = filter_info->syn_info; 5718 5719 if (synqf & E1000_SYN_FILTER_ENABLE) { 5720 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5721 E1000_WRITE_FLUSH(hw); 5722 } 5723 } 5724 5725 /* restore ethernet type filter */ 5726 static inline void 5727 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5728 { 5729 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5730 struct e1000_filter_info *filter_info = 5731 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5732 int i; 5733 5734 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5735 if (filter_info->ethertype_mask & (1 << i)) { 5736 E1000_WRITE_REG(hw, E1000_ETQF(i), 5737 filter_info->ethertype_filters[i].etqf); 5738 E1000_WRITE_FLUSH(hw); 5739 } 5740 } 5741 } 5742 5743 /* restore flex byte filter */ 5744 static inline void 5745 igb_flex_filter_restore(struct rte_eth_dev *dev) 5746 { 5747 struct e1000_filter_info *filter_info = 5748 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5749 struct e1000_flex_filter *flex_filter; 5750 5751 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5752 igb_inject_flex_filter(dev, flex_filter); 5753 } 5754 } 5755 5756 /* restore rss filter */ 5757 static inline void 5758 igb_rss_filter_restore(struct rte_eth_dev *dev) 5759 { 5760 struct e1000_filter_info *filter_info = 5761 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5762 5763 if (filter_info->rss_info.conf.queue_num) 5764 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5765 } 5766 5767 /* restore all types filter */ 5768 static int 5769 igb_filter_restore(struct rte_eth_dev *dev) 5770 { 5771 igb_ntuple_filter_restore(dev); 5772 igb_ethertype_filter_restore(dev); 5773 igb_syn_filter_restore(dev); 5774 igb_flex_filter_restore(dev); 5775 igb_rss_filter_restore(dev); 5776 5777 return 0; 5778 } 5779 5780 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5781 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5782 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5783 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5784 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5785 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5786 5787 /* see e1000_logs.c */ 5788 RTE_INIT(e1000_init_log) 5789 { 5790 e1000_igb_init_log(); 5791 } 5792