1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_common.h> 12 #include <rte_interrupts.h> 13 #include <rte_byteorder.h> 14 #include <rte_log.h> 15 #include <rte_debug.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_ether.h> 19 #include <rte_ethdev_driver.h> 20 #include <rte_ethdev_pci.h> 21 #include <rte_memory.h> 22 #include <rte_eal.h> 23 #include <rte_malloc.h> 24 #include <rte_dev.h> 25 26 #include "e1000_logs.h" 27 #include "base/e1000_api.h" 28 #include "e1000_ethdev.h" 29 #include "igb_regs.h" 30 31 /* 32 * Default values for port configuration 33 */ 34 #define IGB_DEFAULT_RX_FREE_THRESH 32 35 36 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 37 #define IGB_DEFAULT_RX_HTHRESH 8 38 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 39 40 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 41 #define IGB_DEFAULT_TX_HTHRESH 1 42 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 43 44 /* Bit shift and mask */ 45 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 46 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 47 #define IGB_8_BIT_WIDTH CHAR_BIT 48 #define IGB_8_BIT_MASK UINT8_MAX 49 50 /* Additional timesync values. */ 51 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 52 #define E1000_ETQF_FILTER_1588 3 53 #define IGB_82576_TSYNC_SHIFT 16 54 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 55 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 56 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 57 58 #define E1000_VTIVAR_MISC 0x01740 59 #define E1000_VTIVAR_MISC_MASK 0xFF 60 #define E1000_VTIVAR_VALID 0x80 61 #define E1000_VTIVAR_MISC_MAILBOX 0 62 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 63 64 /* External VLAN Enable bit mask */ 65 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 66 67 /* External VLAN Ether Type bit mask and shift */ 68 #define E1000_VET_VET_EXT 0xFFFF0000 69 #define E1000_VET_VET_EXT_SHIFT 16 70 71 /* MSI-X other interrupt vector */ 72 #define IGB_MSIX_OTHER_INTR_VEC 0 73 74 static int eth_igb_configure(struct rte_eth_dev *dev); 75 static int eth_igb_start(struct rte_eth_dev *dev); 76 static void eth_igb_stop(struct rte_eth_dev *dev); 77 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 79 static void eth_igb_close(struct rte_eth_dev *dev); 80 static int eth_igb_reset(struct rte_eth_dev *dev); 81 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 82 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 83 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 84 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 85 static int eth_igb_link_update(struct rte_eth_dev *dev, 86 int wait_to_complete); 87 static int eth_igb_stats_get(struct rte_eth_dev *dev, 88 struct rte_eth_stats *rte_stats); 89 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 90 struct rte_eth_xstat *xstats, unsigned n); 91 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 92 const uint64_t *ids, 93 uint64_t *values, unsigned int n); 94 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 95 struct rte_eth_xstat_name *xstats_names, 96 unsigned int size); 97 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 98 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 99 unsigned int limit); 100 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 101 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 102 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 103 char *fw_version, size_t fw_size); 104 static void eth_igb_infos_get(struct rte_eth_dev *dev, 105 struct rte_eth_dev_info *dev_info); 106 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 107 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 108 struct rte_eth_dev_info *dev_info); 109 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 110 struct rte_eth_fc_conf *fc_conf); 111 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 112 struct rte_eth_fc_conf *fc_conf); 113 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 114 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 115 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 117 struct rte_intr_handle *handle); 118 static void eth_igb_interrupt_handler(void *param); 119 static int igb_hardware_init(struct e1000_hw *hw); 120 static void igb_hw_control_acquire(struct e1000_hw *hw); 121 static void igb_hw_control_release(struct e1000_hw *hw); 122 static void igb_init_manageability(struct e1000_hw *hw); 123 static void igb_release_manageability(struct e1000_hw *hw); 124 125 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 126 127 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 128 uint16_t vlan_id, int on); 129 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 130 enum rte_vlan_type vlan_type, 131 uint16_t tpid_id); 132 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 133 134 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 135 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 140 141 static int eth_igb_led_on(struct rte_eth_dev *dev); 142 static int eth_igb_led_off(struct rte_eth_dev *dev); 143 144 static void igb_intr_disable(struct rte_eth_dev *dev); 145 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 146 static int eth_igb_rar_set(struct rte_eth_dev *dev, 147 struct ether_addr *mac_addr, 148 uint32_t index, uint32_t pool); 149 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 150 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 151 struct ether_addr *addr); 152 153 static void igbvf_intr_disable(struct e1000_hw *hw); 154 static int igbvf_dev_configure(struct rte_eth_dev *dev); 155 static int igbvf_dev_start(struct rte_eth_dev *dev); 156 static void igbvf_dev_stop(struct rte_eth_dev *dev); 157 static void igbvf_dev_close(struct rte_eth_dev *dev); 158 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 159 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 160 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 161 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 162 static int eth_igbvf_link_update(struct e1000_hw *hw); 163 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 164 struct rte_eth_stats *rte_stats); 165 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 166 struct rte_eth_xstat *xstats, unsigned n); 167 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 168 struct rte_eth_xstat_name *xstats_names, 169 unsigned limit); 170 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 171 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 172 uint16_t vlan_id, int on); 173 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 174 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 175 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 176 struct ether_addr *addr); 177 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 178 static int igbvf_get_regs(struct rte_eth_dev *dev, 179 struct rte_dev_reg_info *regs); 180 181 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 182 struct rte_eth_rss_reta_entry64 *reta_conf, 183 uint16_t reta_size); 184 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 185 struct rte_eth_rss_reta_entry64 *reta_conf, 186 uint16_t reta_size); 187 188 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 189 struct rte_eth_syn_filter *filter); 190 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 191 enum rte_filter_op filter_op, 192 void *arg); 193 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 194 struct rte_eth_ntuple_filter *ntuple_filter); 195 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 196 struct rte_eth_ntuple_filter *ntuple_filter); 197 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 198 struct rte_eth_flex_filter *filter); 199 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 200 enum rte_filter_op filter_op, 201 void *arg); 202 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 203 struct rte_eth_ntuple_filter *ntuple_filter); 204 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 205 struct rte_eth_ntuple_filter *ntuple_filter); 206 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 207 struct rte_eth_ntuple_filter *filter); 208 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 209 enum rte_filter_op filter_op, 210 void *arg); 211 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 212 enum rte_filter_op filter_op, 213 void *arg); 214 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 215 struct rte_eth_ethertype_filter *filter); 216 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 217 enum rte_filter_type filter_type, 218 enum rte_filter_op filter_op, 219 void *arg); 220 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 221 static int eth_igb_get_regs(struct rte_eth_dev *dev, 222 struct rte_dev_reg_info *regs); 223 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 224 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 225 struct rte_dev_eeprom_info *eeprom); 226 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 227 struct rte_dev_eeprom_info *eeprom); 228 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 229 struct rte_eth_dev_module_info *modinfo); 230 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 231 struct rte_dev_eeprom_info *info); 232 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 233 struct ether_addr *mc_addr_set, 234 uint32_t nb_mc_addr); 235 static int igb_timesync_enable(struct rte_eth_dev *dev); 236 static int igb_timesync_disable(struct rte_eth_dev *dev); 237 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 238 struct timespec *timestamp, 239 uint32_t flags); 240 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 241 struct timespec *timestamp); 242 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 243 static int igb_timesync_read_time(struct rte_eth_dev *dev, 244 struct timespec *timestamp); 245 static int igb_timesync_write_time(struct rte_eth_dev *dev, 246 const struct timespec *timestamp); 247 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 248 uint16_t queue_id); 249 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 250 uint16_t queue_id); 251 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 252 uint8_t queue, uint8_t msix_vector); 253 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 254 uint8_t index, uint8_t offset); 255 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 256 static void eth_igbvf_interrupt_handler(void *param); 257 static void igbvf_mbx_process(struct rte_eth_dev *dev); 258 static int igb_filter_restore(struct rte_eth_dev *dev); 259 260 /* 261 * Define VF Stats MACRO for Non "cleared on read" register 262 */ 263 #define UPDATE_VF_STAT(reg, last, cur) \ 264 { \ 265 u32 latest = E1000_READ_REG(hw, reg); \ 266 cur += (latest - last) & UINT_MAX; \ 267 last = latest; \ 268 } 269 270 #define IGB_FC_PAUSE_TIME 0x0680 271 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 272 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 273 274 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 275 276 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 277 278 /* 279 * The set of PCI devices this driver supports 280 */ 281 static const struct rte_pci_id pci_id_igb_map[] = { 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 290 291 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 294 295 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 301 302 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 320 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 321 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 322 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 323 { .vendor_id = 0, /* sentinel */ }, 324 }; 325 326 /* 327 * The set of PCI devices this driver supports (for 82576&I350 VF) 328 */ 329 static const struct rte_pci_id pci_id_igbvf_map[] = { 330 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 331 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 332 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 333 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 334 { .vendor_id = 0, /* sentinel */ }, 335 }; 336 337 static const struct rte_eth_desc_lim rx_desc_lim = { 338 .nb_max = E1000_MAX_RING_DESC, 339 .nb_min = E1000_MIN_RING_DESC, 340 .nb_align = IGB_RXD_ALIGN, 341 }; 342 343 static const struct rte_eth_desc_lim tx_desc_lim = { 344 .nb_max = E1000_MAX_RING_DESC, 345 .nb_min = E1000_MIN_RING_DESC, 346 .nb_align = IGB_RXD_ALIGN, 347 .nb_seg_max = IGB_TX_MAX_SEG, 348 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 349 }; 350 351 static const struct eth_dev_ops eth_igb_ops = { 352 .dev_configure = eth_igb_configure, 353 .dev_start = eth_igb_start, 354 .dev_stop = eth_igb_stop, 355 .dev_set_link_up = eth_igb_dev_set_link_up, 356 .dev_set_link_down = eth_igb_dev_set_link_down, 357 .dev_close = eth_igb_close, 358 .dev_reset = eth_igb_reset, 359 .promiscuous_enable = eth_igb_promiscuous_enable, 360 .promiscuous_disable = eth_igb_promiscuous_disable, 361 .allmulticast_enable = eth_igb_allmulticast_enable, 362 .allmulticast_disable = eth_igb_allmulticast_disable, 363 .link_update = eth_igb_link_update, 364 .stats_get = eth_igb_stats_get, 365 .xstats_get = eth_igb_xstats_get, 366 .xstats_get_by_id = eth_igb_xstats_get_by_id, 367 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 368 .xstats_get_names = eth_igb_xstats_get_names, 369 .stats_reset = eth_igb_stats_reset, 370 .xstats_reset = eth_igb_xstats_reset, 371 .fw_version_get = eth_igb_fw_version_get, 372 .dev_infos_get = eth_igb_infos_get, 373 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 374 .mtu_set = eth_igb_mtu_set, 375 .vlan_filter_set = eth_igb_vlan_filter_set, 376 .vlan_tpid_set = eth_igb_vlan_tpid_set, 377 .vlan_offload_set = eth_igb_vlan_offload_set, 378 .rx_queue_setup = eth_igb_rx_queue_setup, 379 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 380 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 381 .rx_queue_release = eth_igb_rx_queue_release, 382 .rx_queue_count = eth_igb_rx_queue_count, 383 .rx_descriptor_done = eth_igb_rx_descriptor_done, 384 .rx_descriptor_status = eth_igb_rx_descriptor_status, 385 .tx_descriptor_status = eth_igb_tx_descriptor_status, 386 .tx_queue_setup = eth_igb_tx_queue_setup, 387 .tx_queue_release = eth_igb_tx_queue_release, 388 .tx_done_cleanup = eth_igb_tx_done_cleanup, 389 .dev_led_on = eth_igb_led_on, 390 .dev_led_off = eth_igb_led_off, 391 .flow_ctrl_get = eth_igb_flow_ctrl_get, 392 .flow_ctrl_set = eth_igb_flow_ctrl_set, 393 .mac_addr_add = eth_igb_rar_set, 394 .mac_addr_remove = eth_igb_rar_clear, 395 .mac_addr_set = eth_igb_default_mac_addr_set, 396 .reta_update = eth_igb_rss_reta_update, 397 .reta_query = eth_igb_rss_reta_query, 398 .rss_hash_update = eth_igb_rss_hash_update, 399 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 400 .filter_ctrl = eth_igb_filter_ctrl, 401 .set_mc_addr_list = eth_igb_set_mc_addr_list, 402 .rxq_info_get = igb_rxq_info_get, 403 .txq_info_get = igb_txq_info_get, 404 .timesync_enable = igb_timesync_enable, 405 .timesync_disable = igb_timesync_disable, 406 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 407 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 408 .get_reg = eth_igb_get_regs, 409 .get_eeprom_length = eth_igb_get_eeprom_length, 410 .get_eeprom = eth_igb_get_eeprom, 411 .set_eeprom = eth_igb_set_eeprom, 412 .get_module_info = eth_igb_get_module_info, 413 .get_module_eeprom = eth_igb_get_module_eeprom, 414 .timesync_adjust_time = igb_timesync_adjust_time, 415 .timesync_read_time = igb_timesync_read_time, 416 .timesync_write_time = igb_timesync_write_time, 417 }; 418 419 /* 420 * dev_ops for virtual function, bare necessities for basic vf 421 * operation have been implemented 422 */ 423 static const struct eth_dev_ops igbvf_eth_dev_ops = { 424 .dev_configure = igbvf_dev_configure, 425 .dev_start = igbvf_dev_start, 426 .dev_stop = igbvf_dev_stop, 427 .dev_close = igbvf_dev_close, 428 .promiscuous_enable = igbvf_promiscuous_enable, 429 .promiscuous_disable = igbvf_promiscuous_disable, 430 .allmulticast_enable = igbvf_allmulticast_enable, 431 .allmulticast_disable = igbvf_allmulticast_disable, 432 .link_update = eth_igb_link_update, 433 .stats_get = eth_igbvf_stats_get, 434 .xstats_get = eth_igbvf_xstats_get, 435 .xstats_get_names = eth_igbvf_xstats_get_names, 436 .stats_reset = eth_igbvf_stats_reset, 437 .xstats_reset = eth_igbvf_stats_reset, 438 .vlan_filter_set = igbvf_vlan_filter_set, 439 .dev_infos_get = eth_igbvf_infos_get, 440 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 441 .rx_queue_setup = eth_igb_rx_queue_setup, 442 .rx_queue_release = eth_igb_rx_queue_release, 443 .rx_descriptor_done = eth_igb_rx_descriptor_done, 444 .rx_descriptor_status = eth_igb_rx_descriptor_status, 445 .tx_descriptor_status = eth_igb_tx_descriptor_status, 446 .tx_queue_setup = eth_igb_tx_queue_setup, 447 .tx_queue_release = eth_igb_tx_queue_release, 448 .set_mc_addr_list = eth_igb_set_mc_addr_list, 449 .rxq_info_get = igb_rxq_info_get, 450 .txq_info_get = igb_txq_info_get, 451 .mac_addr_set = igbvf_default_mac_addr_set, 452 .get_reg = igbvf_get_regs, 453 }; 454 455 /* store statistics names and its offset in stats structure */ 456 struct rte_igb_xstats_name_off { 457 char name[RTE_ETH_XSTATS_NAME_SIZE]; 458 unsigned offset; 459 }; 460 461 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 462 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 463 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 464 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 465 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 466 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 467 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 468 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 469 ecol)}, 470 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 471 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 472 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 473 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 474 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 475 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 476 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 477 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 478 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 479 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 480 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 481 fcruc)}, 482 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 483 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 484 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 485 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 486 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 487 prc1023)}, 488 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 489 prc1522)}, 490 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 491 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 492 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 493 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 494 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 495 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 496 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 497 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 498 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 499 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 500 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 501 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 502 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 503 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 504 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 505 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 506 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 507 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 508 ptc1023)}, 509 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 510 ptc1522)}, 511 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 512 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 513 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 514 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 515 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 516 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 517 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 518 519 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 520 }; 521 522 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 523 sizeof(rte_igb_stats_strings[0])) 524 525 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 526 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 527 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 528 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 529 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 530 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 531 }; 532 533 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 534 sizeof(rte_igbvf_stats_strings[0])) 535 536 537 static inline void 538 igb_intr_enable(struct rte_eth_dev *dev) 539 { 540 struct e1000_interrupt *intr = 541 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 542 struct e1000_hw *hw = 543 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 544 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 545 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 546 547 if (rte_intr_allow_others(intr_handle) && 548 dev->data->dev_conf.intr_conf.lsc != 0) { 549 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 550 } 551 552 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 553 E1000_WRITE_FLUSH(hw); 554 } 555 556 static void 557 igb_intr_disable(struct rte_eth_dev *dev) 558 { 559 struct e1000_hw *hw = 560 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 561 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 562 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 563 564 if (rte_intr_allow_others(intr_handle) && 565 dev->data->dev_conf.intr_conf.lsc != 0) { 566 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 567 } 568 569 E1000_WRITE_REG(hw, E1000_IMC, ~0); 570 E1000_WRITE_FLUSH(hw); 571 } 572 573 static inline void 574 igbvf_intr_enable(struct rte_eth_dev *dev) 575 { 576 struct e1000_hw *hw = 577 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 578 579 /* only for mailbox */ 580 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 581 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 582 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 583 E1000_WRITE_FLUSH(hw); 584 } 585 586 /* only for mailbox now. If RX/TX needed, should extend this function. */ 587 static void 588 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 589 { 590 uint32_t tmp = 0; 591 592 /* mailbox */ 593 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 594 tmp |= E1000_VTIVAR_VALID; 595 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 596 } 597 598 static void 599 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 600 { 601 struct e1000_hw *hw = 602 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 603 604 /* Configure VF other cause ivar */ 605 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 606 } 607 608 static inline int32_t 609 igb_pf_reset_hw(struct e1000_hw *hw) 610 { 611 uint32_t ctrl_ext; 612 int32_t status; 613 614 status = e1000_reset_hw(hw); 615 616 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 617 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 618 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 619 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 620 E1000_WRITE_FLUSH(hw); 621 622 return status; 623 } 624 625 static void 626 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 627 { 628 struct e1000_hw *hw = 629 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 630 631 632 hw->vendor_id = pci_dev->id.vendor_id; 633 hw->device_id = pci_dev->id.device_id; 634 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 635 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 636 637 e1000_set_mac_type(hw); 638 639 /* need to check if it is a vf device below */ 640 } 641 642 static int 643 igb_reset_swfw_lock(struct e1000_hw *hw) 644 { 645 int ret_val; 646 647 /* 648 * Do mac ops initialization manually here, since we will need 649 * some function pointers set by this call. 650 */ 651 ret_val = e1000_init_mac_params(hw); 652 if (ret_val) 653 return ret_val; 654 655 /* 656 * SMBI lock should not fail in this early stage. If this is the case, 657 * it is due to an improper exit of the application. 658 * So force the release of the faulty lock. 659 */ 660 if (e1000_get_hw_semaphore_generic(hw) < 0) { 661 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 662 } 663 e1000_put_hw_semaphore_generic(hw); 664 665 if (hw->mac.ops.acquire_swfw_sync != NULL) { 666 uint16_t mask; 667 668 /* 669 * Phy lock should not fail in this early stage. If this is the case, 670 * it is due to an improper exit of the application. 671 * So force the release of the faulty lock. 672 */ 673 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 674 if (hw->bus.func > E1000_FUNC_1) 675 mask <<= 2; 676 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 677 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 678 hw->bus.func); 679 } 680 hw->mac.ops.release_swfw_sync(hw, mask); 681 682 /* 683 * This one is more tricky since it is common to all ports; but 684 * swfw_sync retries last long enough (1s) to be almost sure that if 685 * lock can not be taken it is due to an improper lock of the 686 * semaphore. 687 */ 688 mask = E1000_SWFW_EEP_SM; 689 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 690 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 691 } 692 hw->mac.ops.release_swfw_sync(hw, mask); 693 } 694 695 return E1000_SUCCESS; 696 } 697 698 /* Remove all ntuple filters of the device */ 699 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 700 { 701 struct e1000_filter_info *filter_info = 702 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 703 struct e1000_5tuple_filter *p_5tuple; 704 struct e1000_2tuple_filter *p_2tuple; 705 706 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 707 TAILQ_REMOVE(&filter_info->fivetuple_list, 708 p_5tuple, entries); 709 rte_free(p_5tuple); 710 } 711 filter_info->fivetuple_mask = 0; 712 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 713 TAILQ_REMOVE(&filter_info->twotuple_list, 714 p_2tuple, entries); 715 rte_free(p_2tuple); 716 } 717 filter_info->twotuple_mask = 0; 718 719 return 0; 720 } 721 722 /* Remove all flex filters of the device */ 723 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 724 { 725 struct e1000_filter_info *filter_info = 726 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 727 struct e1000_flex_filter *p_flex; 728 729 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 730 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 731 rte_free(p_flex); 732 } 733 filter_info->flex_mask = 0; 734 735 return 0; 736 } 737 738 static int 739 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 740 { 741 int error = 0; 742 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 743 struct e1000_hw *hw = 744 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 745 struct e1000_vfta * shadow_vfta = 746 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 747 struct e1000_filter_info *filter_info = 748 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 749 struct e1000_adapter *adapter = 750 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 751 752 uint32_t ctrl_ext; 753 754 eth_dev->dev_ops = ð_igb_ops; 755 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 756 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 757 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 758 759 /* for secondary processes, we don't initialise any further as primary 760 * has already done this work. Only check we don't need a different 761 * RX function */ 762 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 763 if (eth_dev->data->scattered_rx) 764 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 765 return 0; 766 } 767 768 rte_eth_copy_pci_info(eth_dev, pci_dev); 769 770 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 771 772 igb_identify_hardware(eth_dev, pci_dev); 773 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 774 error = -EIO; 775 goto err_late; 776 } 777 778 e1000_get_bus_info(hw); 779 780 /* Reset any pending lock */ 781 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 782 error = -EIO; 783 goto err_late; 784 } 785 786 /* Finish initialization */ 787 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 788 error = -EIO; 789 goto err_late; 790 } 791 792 hw->mac.autoneg = 1; 793 hw->phy.autoneg_wait_to_complete = 0; 794 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 795 796 /* Copper options */ 797 if (hw->phy.media_type == e1000_media_type_copper) { 798 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 799 hw->phy.disable_polarity_correction = 0; 800 hw->phy.ms_type = e1000_ms_hw_default; 801 } 802 803 /* 804 * Start from a known state, this is important in reading the nvm 805 * and mac from that. 806 */ 807 igb_pf_reset_hw(hw); 808 809 /* Make sure we have a good EEPROM before we read from it */ 810 if (e1000_validate_nvm_checksum(hw) < 0) { 811 /* 812 * Some PCI-E parts fail the first check due to 813 * the link being in sleep state, call it again, 814 * if it fails a second time its a real issue. 815 */ 816 if (e1000_validate_nvm_checksum(hw) < 0) { 817 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 818 error = -EIO; 819 goto err_late; 820 } 821 } 822 823 /* Read the permanent MAC address out of the EEPROM */ 824 if (e1000_read_mac_addr(hw) != 0) { 825 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 826 error = -EIO; 827 goto err_late; 828 } 829 830 /* Allocate memory for storing MAC addresses */ 831 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 832 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 833 if (eth_dev->data->mac_addrs == NULL) { 834 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 835 "store MAC addresses", 836 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 837 error = -ENOMEM; 838 goto err_late; 839 } 840 841 /* Copy the permanent MAC address */ 842 ether_addr_copy((struct ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); 843 844 /* initialize the vfta */ 845 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 846 847 /* Now initialize the hardware */ 848 if (igb_hardware_init(hw) != 0) { 849 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 850 rte_free(eth_dev->data->mac_addrs); 851 eth_dev->data->mac_addrs = NULL; 852 error = -ENODEV; 853 goto err_late; 854 } 855 hw->mac.get_link_status = 1; 856 adapter->stopped = 0; 857 858 /* Indicate SOL/IDER usage */ 859 if (e1000_check_reset_block(hw) < 0) { 860 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 861 "SOL/IDER session"); 862 } 863 864 /* initialize PF if max_vfs not zero */ 865 igb_pf_host_init(eth_dev); 866 867 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 868 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 869 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 870 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 871 E1000_WRITE_FLUSH(hw); 872 873 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 874 eth_dev->data->port_id, pci_dev->id.vendor_id, 875 pci_dev->id.device_id); 876 877 rte_intr_callback_register(&pci_dev->intr_handle, 878 eth_igb_interrupt_handler, 879 (void *)eth_dev); 880 881 /* enable uio/vfio intr/eventfd mapping */ 882 rte_intr_enable(&pci_dev->intr_handle); 883 884 /* enable support intr */ 885 igb_intr_enable(eth_dev); 886 887 /* initialize filter info */ 888 memset(filter_info, 0, 889 sizeof(struct e1000_filter_info)); 890 891 TAILQ_INIT(&filter_info->flex_list); 892 TAILQ_INIT(&filter_info->twotuple_list); 893 TAILQ_INIT(&filter_info->fivetuple_list); 894 895 TAILQ_INIT(&igb_filter_ntuple_list); 896 TAILQ_INIT(&igb_filter_ethertype_list); 897 TAILQ_INIT(&igb_filter_syn_list); 898 TAILQ_INIT(&igb_filter_flex_list); 899 TAILQ_INIT(&igb_filter_rss_list); 900 TAILQ_INIT(&igb_flow_list); 901 902 return 0; 903 904 err_late: 905 igb_hw_control_release(hw); 906 907 return error; 908 } 909 910 static int 911 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 912 { 913 struct rte_pci_device *pci_dev; 914 struct rte_intr_handle *intr_handle; 915 struct e1000_hw *hw; 916 struct e1000_adapter *adapter = 917 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 918 struct e1000_filter_info *filter_info = 919 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 920 921 PMD_INIT_FUNC_TRACE(); 922 923 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 924 return -EPERM; 925 926 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 927 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 928 intr_handle = &pci_dev->intr_handle; 929 930 if (adapter->stopped == 0) 931 eth_igb_close(eth_dev); 932 933 eth_dev->dev_ops = NULL; 934 eth_dev->rx_pkt_burst = NULL; 935 eth_dev->tx_pkt_burst = NULL; 936 937 /* Reset any pending lock */ 938 igb_reset_swfw_lock(hw); 939 940 /* uninitialize PF if max_vfs not zero */ 941 igb_pf_host_uninit(eth_dev); 942 943 /* disable uio intr before callback unregister */ 944 rte_intr_disable(intr_handle); 945 rte_intr_callback_unregister(intr_handle, 946 eth_igb_interrupt_handler, eth_dev); 947 948 /* clear the SYN filter info */ 949 filter_info->syn_info = 0; 950 951 /* clear the ethertype filters info */ 952 filter_info->ethertype_mask = 0; 953 memset(filter_info->ethertype_filters, 0, 954 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 955 956 /* clear the rss filter info */ 957 memset(&filter_info->rss_info, 0, 958 sizeof(struct igb_rte_flow_rss_conf)); 959 960 /* remove all ntuple filters of the device */ 961 igb_ntuple_filter_uninit(eth_dev); 962 963 /* remove all flex filters of the device */ 964 igb_flex_filter_uninit(eth_dev); 965 966 /* clear all the filters list */ 967 igb_filterlist_flush(eth_dev); 968 969 return 0; 970 } 971 972 /* 973 * Virtual Function device init 974 */ 975 static int 976 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 977 { 978 struct rte_pci_device *pci_dev; 979 struct rte_intr_handle *intr_handle; 980 struct e1000_adapter *adapter = 981 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 982 struct e1000_hw *hw = 983 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 984 int diag; 985 struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr; 986 987 PMD_INIT_FUNC_TRACE(); 988 989 eth_dev->dev_ops = &igbvf_eth_dev_ops; 990 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 991 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 992 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 993 994 /* for secondary processes, we don't initialise any further as primary 995 * has already done this work. Only check we don't need a different 996 * RX function */ 997 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 998 if (eth_dev->data->scattered_rx) 999 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 1000 return 0; 1001 } 1002 1003 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1004 rte_eth_copy_pci_info(eth_dev, pci_dev); 1005 1006 hw->device_id = pci_dev->id.device_id; 1007 hw->vendor_id = pci_dev->id.vendor_id; 1008 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1009 adapter->stopped = 0; 1010 1011 /* Initialize the shared code (base driver) */ 1012 diag = e1000_setup_init_funcs(hw, TRUE); 1013 if (diag != 0) { 1014 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 1015 diag); 1016 return -EIO; 1017 } 1018 1019 /* init_mailbox_params */ 1020 hw->mbx.ops.init_params(hw); 1021 1022 /* Disable the interrupts for VF */ 1023 igbvf_intr_disable(hw); 1024 1025 diag = hw->mac.ops.reset_hw(hw); 1026 1027 /* Allocate memory for storing MAC addresses */ 1028 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN * 1029 hw->mac.rar_entry_count, 0); 1030 if (eth_dev->data->mac_addrs == NULL) { 1031 PMD_INIT_LOG(ERR, 1032 "Failed to allocate %d bytes needed to store MAC " 1033 "addresses", 1034 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1035 return -ENOMEM; 1036 } 1037 1038 /* Generate a random MAC address, if none was assigned by PF. */ 1039 if (is_zero_ether_addr(perm_addr)) { 1040 eth_random_addr(perm_addr->addr_bytes); 1041 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1042 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1043 "%02x:%02x:%02x:%02x:%02x:%02x", 1044 perm_addr->addr_bytes[0], 1045 perm_addr->addr_bytes[1], 1046 perm_addr->addr_bytes[2], 1047 perm_addr->addr_bytes[3], 1048 perm_addr->addr_bytes[4], 1049 perm_addr->addr_bytes[5]); 1050 } 1051 1052 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1053 if (diag) { 1054 rte_free(eth_dev->data->mac_addrs); 1055 eth_dev->data->mac_addrs = NULL; 1056 return diag; 1057 } 1058 /* Copy the permanent MAC address */ 1059 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1060 ð_dev->data->mac_addrs[0]); 1061 1062 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1063 "mac.type=%s", 1064 eth_dev->data->port_id, pci_dev->id.vendor_id, 1065 pci_dev->id.device_id, "igb_mac_82576_vf"); 1066 1067 intr_handle = &pci_dev->intr_handle; 1068 rte_intr_callback_register(intr_handle, 1069 eth_igbvf_interrupt_handler, eth_dev); 1070 1071 return 0; 1072 } 1073 1074 static int 1075 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1076 { 1077 struct e1000_adapter *adapter = 1078 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1079 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1080 1081 PMD_INIT_FUNC_TRACE(); 1082 1083 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1084 return -EPERM; 1085 1086 if (adapter->stopped == 0) 1087 igbvf_dev_close(eth_dev); 1088 1089 eth_dev->dev_ops = NULL; 1090 eth_dev->rx_pkt_burst = NULL; 1091 eth_dev->tx_pkt_burst = NULL; 1092 1093 /* disable uio intr before callback unregister */ 1094 rte_intr_disable(&pci_dev->intr_handle); 1095 rte_intr_callback_unregister(&pci_dev->intr_handle, 1096 eth_igbvf_interrupt_handler, 1097 (void *)eth_dev); 1098 1099 return 0; 1100 } 1101 1102 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1103 struct rte_pci_device *pci_dev) 1104 { 1105 return rte_eth_dev_pci_generic_probe(pci_dev, 1106 sizeof(struct e1000_adapter), eth_igb_dev_init); 1107 } 1108 1109 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1110 { 1111 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1112 } 1113 1114 static struct rte_pci_driver rte_igb_pmd = { 1115 .id_table = pci_id_igb_map, 1116 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1117 RTE_PCI_DRV_IOVA_AS_VA, 1118 .probe = eth_igb_pci_probe, 1119 .remove = eth_igb_pci_remove, 1120 }; 1121 1122 1123 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1124 struct rte_pci_device *pci_dev) 1125 { 1126 return rte_eth_dev_pci_generic_probe(pci_dev, 1127 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1128 } 1129 1130 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1131 { 1132 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1133 } 1134 1135 /* 1136 * virtual function driver struct 1137 */ 1138 static struct rte_pci_driver rte_igbvf_pmd = { 1139 .id_table = pci_id_igbvf_map, 1140 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1141 .probe = eth_igbvf_pci_probe, 1142 .remove = eth_igbvf_pci_remove, 1143 }; 1144 1145 static void 1146 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1147 { 1148 struct e1000_hw *hw = 1149 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1150 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1151 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1152 rctl |= E1000_RCTL_VFE; 1153 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1154 } 1155 1156 static int 1157 igb_check_mq_mode(struct rte_eth_dev *dev) 1158 { 1159 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1160 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1161 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1162 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1163 1164 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1165 tx_mq_mode == ETH_MQ_TX_DCB || 1166 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1167 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1168 return -EINVAL; 1169 } 1170 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1171 /* Check multi-queue mode. 1172 * To no break software we accept ETH_MQ_RX_NONE as this might 1173 * be used to turn off VLAN filter. 1174 */ 1175 1176 if (rx_mq_mode == ETH_MQ_RX_NONE || 1177 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1178 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1179 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1180 } else { 1181 /* Only support one queue on VFs. 1182 * RSS together with SRIOV is not supported. 1183 */ 1184 PMD_INIT_LOG(ERR, "SRIOV is active," 1185 " wrong mq_mode rx %d.", 1186 rx_mq_mode); 1187 return -EINVAL; 1188 } 1189 /* TX mode is not used here, so mode might be ignored.*/ 1190 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1191 /* SRIOV only works in VMDq enable mode */ 1192 PMD_INIT_LOG(WARNING, "SRIOV is active," 1193 " TX mode %d is not supported. " 1194 " Driver will behave as %d mode.", 1195 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1196 } 1197 1198 /* check valid queue number */ 1199 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1200 PMD_INIT_LOG(ERR, "SRIOV is active," 1201 " only support one queue on VFs."); 1202 return -EINVAL; 1203 } 1204 } else { 1205 /* To no break software that set invalid mode, only display 1206 * warning if invalid mode is used. 1207 */ 1208 if (rx_mq_mode != ETH_MQ_RX_NONE && 1209 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1210 rx_mq_mode != ETH_MQ_RX_RSS) { 1211 /* RSS together with VMDq not supported*/ 1212 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1213 rx_mq_mode); 1214 return -EINVAL; 1215 } 1216 1217 if (tx_mq_mode != ETH_MQ_TX_NONE && 1218 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1219 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1220 " Due to txmode is meaningless in this" 1221 " driver, just ignore.", 1222 tx_mq_mode); 1223 } 1224 } 1225 return 0; 1226 } 1227 1228 static int 1229 eth_igb_configure(struct rte_eth_dev *dev) 1230 { 1231 struct e1000_interrupt *intr = 1232 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1233 int ret; 1234 1235 PMD_INIT_FUNC_TRACE(); 1236 1237 /* multipe queue mode checking */ 1238 ret = igb_check_mq_mode(dev); 1239 if (ret != 0) { 1240 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1241 ret); 1242 return ret; 1243 } 1244 1245 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1246 PMD_INIT_FUNC_TRACE(); 1247 1248 return 0; 1249 } 1250 1251 static void 1252 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1253 bool enable) 1254 { 1255 struct e1000_hw *hw = 1256 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1257 uint32_t tctl, rctl; 1258 1259 tctl = E1000_READ_REG(hw, E1000_TCTL); 1260 rctl = E1000_READ_REG(hw, E1000_RCTL); 1261 1262 if (enable) { 1263 /* enable Tx/Rx */ 1264 tctl |= E1000_TCTL_EN; 1265 rctl |= E1000_RCTL_EN; 1266 } else { 1267 /* disable Tx/Rx */ 1268 tctl &= ~E1000_TCTL_EN; 1269 rctl &= ~E1000_RCTL_EN; 1270 } 1271 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1272 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1273 E1000_WRITE_FLUSH(hw); 1274 } 1275 1276 static int 1277 eth_igb_start(struct rte_eth_dev *dev) 1278 { 1279 struct e1000_hw *hw = 1280 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1281 struct e1000_adapter *adapter = 1282 E1000_DEV_PRIVATE(dev->data->dev_private); 1283 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1284 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1285 int ret, mask; 1286 uint32_t intr_vector = 0; 1287 uint32_t ctrl_ext; 1288 uint32_t *speeds; 1289 int num_speeds; 1290 bool autoneg; 1291 1292 PMD_INIT_FUNC_TRACE(); 1293 1294 /* disable uio/vfio intr/eventfd mapping */ 1295 rte_intr_disable(intr_handle); 1296 1297 /* Power up the phy. Needed to make the link go Up */ 1298 eth_igb_dev_set_link_up(dev); 1299 1300 /* 1301 * Packet Buffer Allocation (PBA) 1302 * Writing PBA sets the receive portion of the buffer 1303 * the remainder is used for the transmit buffer. 1304 */ 1305 if (hw->mac.type == e1000_82575) { 1306 uint32_t pba; 1307 1308 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1309 E1000_WRITE_REG(hw, E1000_PBA, pba); 1310 } 1311 1312 /* Put the address into the Receive Address Array */ 1313 e1000_rar_set(hw, hw->mac.addr, 0); 1314 1315 /* Initialize the hardware */ 1316 if (igb_hardware_init(hw)) { 1317 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1318 return -EIO; 1319 } 1320 adapter->stopped = 0; 1321 1322 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1323 1324 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1325 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1326 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1327 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1328 E1000_WRITE_FLUSH(hw); 1329 1330 /* configure PF module if SRIOV enabled */ 1331 igb_pf_host_configure(dev); 1332 1333 /* check and configure queue intr-vector mapping */ 1334 if ((rte_intr_cap_multiple(intr_handle) || 1335 !RTE_ETH_DEV_SRIOV(dev).active) && 1336 dev->data->dev_conf.intr_conf.rxq != 0) { 1337 intr_vector = dev->data->nb_rx_queues; 1338 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1339 return -1; 1340 } 1341 1342 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1343 intr_handle->intr_vec = 1344 rte_zmalloc("intr_vec", 1345 dev->data->nb_rx_queues * sizeof(int), 0); 1346 if (intr_handle->intr_vec == NULL) { 1347 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1348 " intr_vec", dev->data->nb_rx_queues); 1349 return -ENOMEM; 1350 } 1351 } 1352 1353 /* confiugre msix for rx interrupt */ 1354 eth_igb_configure_msix_intr(dev); 1355 1356 /* Configure for OS presence */ 1357 igb_init_manageability(hw); 1358 1359 eth_igb_tx_init(dev); 1360 1361 /* This can fail when allocating mbufs for descriptor rings */ 1362 ret = eth_igb_rx_init(dev); 1363 if (ret) { 1364 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1365 igb_dev_clear_queues(dev); 1366 return ret; 1367 } 1368 1369 e1000_clear_hw_cntrs_base_generic(hw); 1370 1371 /* 1372 * VLAN Offload Settings 1373 */ 1374 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1375 ETH_VLAN_EXTEND_MASK; 1376 ret = eth_igb_vlan_offload_set(dev, mask); 1377 if (ret) { 1378 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1379 igb_dev_clear_queues(dev); 1380 return ret; 1381 } 1382 1383 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1384 /* Enable VLAN filter since VMDq always use VLAN filter */ 1385 igb_vmdq_vlan_hw_filter_enable(dev); 1386 } 1387 1388 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1389 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1390 (hw->mac.type == e1000_i211)) { 1391 /* Configure EITR with the maximum possible value (0xFFFF) */ 1392 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1393 } 1394 1395 /* Setup link speed and duplex */ 1396 speeds = &dev->data->dev_conf.link_speeds; 1397 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1398 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1399 hw->mac.autoneg = 1; 1400 } else { 1401 num_speeds = 0; 1402 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1403 1404 /* Reset */ 1405 hw->phy.autoneg_advertised = 0; 1406 1407 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1408 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1409 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1410 num_speeds = -1; 1411 goto error_invalid_config; 1412 } 1413 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1414 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1415 num_speeds++; 1416 } 1417 if (*speeds & ETH_LINK_SPEED_10M) { 1418 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1419 num_speeds++; 1420 } 1421 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1422 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1423 num_speeds++; 1424 } 1425 if (*speeds & ETH_LINK_SPEED_100M) { 1426 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1427 num_speeds++; 1428 } 1429 if (*speeds & ETH_LINK_SPEED_1G) { 1430 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1431 num_speeds++; 1432 } 1433 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1434 goto error_invalid_config; 1435 1436 /* Set/reset the mac.autoneg based on the link speed, 1437 * fixed or not 1438 */ 1439 if (!autoneg) { 1440 hw->mac.autoneg = 0; 1441 hw->mac.forced_speed_duplex = 1442 hw->phy.autoneg_advertised; 1443 } else { 1444 hw->mac.autoneg = 1; 1445 } 1446 } 1447 1448 e1000_setup_link(hw); 1449 1450 if (rte_intr_allow_others(intr_handle)) { 1451 /* check if lsc interrupt is enabled */ 1452 if (dev->data->dev_conf.intr_conf.lsc != 0) 1453 eth_igb_lsc_interrupt_setup(dev, TRUE); 1454 else 1455 eth_igb_lsc_interrupt_setup(dev, FALSE); 1456 } else { 1457 rte_intr_callback_unregister(intr_handle, 1458 eth_igb_interrupt_handler, 1459 (void *)dev); 1460 if (dev->data->dev_conf.intr_conf.lsc != 0) 1461 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1462 " no intr multiplex"); 1463 } 1464 1465 /* check if rxq interrupt is enabled */ 1466 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1467 rte_intr_dp_is_en(intr_handle)) 1468 eth_igb_rxq_interrupt_setup(dev); 1469 1470 /* enable uio/vfio intr/eventfd mapping */ 1471 rte_intr_enable(intr_handle); 1472 1473 /* resume enabled intr since hw reset */ 1474 igb_intr_enable(dev); 1475 1476 /* restore all types filter */ 1477 igb_filter_restore(dev); 1478 1479 eth_igb_rxtx_control(dev, true); 1480 eth_igb_link_update(dev, 0); 1481 1482 PMD_INIT_LOG(DEBUG, "<<"); 1483 1484 return 0; 1485 1486 error_invalid_config: 1487 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1488 dev->data->dev_conf.link_speeds, dev->data->port_id); 1489 igb_dev_clear_queues(dev); 1490 return -EINVAL; 1491 } 1492 1493 /********************************************************************* 1494 * 1495 * This routine disables all traffic on the adapter by issuing a 1496 * global reset on the MAC. 1497 * 1498 **********************************************************************/ 1499 static void 1500 eth_igb_stop(struct rte_eth_dev *dev) 1501 { 1502 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1503 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1504 struct rte_eth_link link; 1505 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1506 1507 eth_igb_rxtx_control(dev, false); 1508 1509 igb_intr_disable(dev); 1510 1511 /* disable intr eventfd mapping */ 1512 rte_intr_disable(intr_handle); 1513 1514 igb_pf_reset_hw(hw); 1515 E1000_WRITE_REG(hw, E1000_WUC, 0); 1516 1517 /* Set bit for Go Link disconnect */ 1518 if (hw->mac.type >= e1000_82580) { 1519 uint32_t phpm_reg; 1520 1521 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1522 phpm_reg |= E1000_82580_PM_GO_LINKD; 1523 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1524 } 1525 1526 /* Power down the phy. Needed to make the link go Down */ 1527 eth_igb_dev_set_link_down(dev); 1528 1529 igb_dev_clear_queues(dev); 1530 1531 /* clear the recorded link status */ 1532 memset(&link, 0, sizeof(link)); 1533 rte_eth_linkstatus_set(dev, &link); 1534 1535 if (!rte_intr_allow_others(intr_handle)) 1536 /* resume to the default handler */ 1537 rte_intr_callback_register(intr_handle, 1538 eth_igb_interrupt_handler, 1539 (void *)dev); 1540 1541 /* Clean datapath event and queue/vec mapping */ 1542 rte_intr_efd_disable(intr_handle); 1543 if (intr_handle->intr_vec != NULL) { 1544 rte_free(intr_handle->intr_vec); 1545 intr_handle->intr_vec = NULL; 1546 } 1547 } 1548 1549 static int 1550 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1551 { 1552 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1553 1554 if (hw->phy.media_type == e1000_media_type_copper) 1555 e1000_power_up_phy(hw); 1556 else 1557 e1000_power_up_fiber_serdes_link(hw); 1558 1559 return 0; 1560 } 1561 1562 static int 1563 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1564 { 1565 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1566 1567 if (hw->phy.media_type == e1000_media_type_copper) 1568 e1000_power_down_phy(hw); 1569 else 1570 e1000_shutdown_fiber_serdes_link(hw); 1571 1572 return 0; 1573 } 1574 1575 static void 1576 eth_igb_close(struct rte_eth_dev *dev) 1577 { 1578 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1579 struct e1000_adapter *adapter = 1580 E1000_DEV_PRIVATE(dev->data->dev_private); 1581 struct rte_eth_link link; 1582 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1583 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1584 1585 eth_igb_stop(dev); 1586 adapter->stopped = 1; 1587 1588 e1000_phy_hw_reset(hw); 1589 igb_release_manageability(hw); 1590 igb_hw_control_release(hw); 1591 1592 /* Clear bit for Go Link disconnect */ 1593 if (hw->mac.type >= e1000_82580) { 1594 uint32_t phpm_reg; 1595 1596 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1597 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1598 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1599 } 1600 1601 igb_dev_free_queues(dev); 1602 1603 if (intr_handle->intr_vec) { 1604 rte_free(intr_handle->intr_vec); 1605 intr_handle->intr_vec = NULL; 1606 } 1607 1608 memset(&link, 0, sizeof(link)); 1609 rte_eth_linkstatus_set(dev, &link); 1610 } 1611 1612 /* 1613 * Reset PF device. 1614 */ 1615 static int 1616 eth_igb_reset(struct rte_eth_dev *dev) 1617 { 1618 int ret; 1619 1620 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1621 * its VF to make them align with it. The detailed notification 1622 * mechanism is PMD specific and is currently not implemented. 1623 * To avoid unexpected behavior in VF, currently reset of PF with 1624 * SR-IOV activation is not supported. It might be supported later. 1625 */ 1626 if (dev->data->sriov.active) 1627 return -ENOTSUP; 1628 1629 ret = eth_igb_dev_uninit(dev); 1630 if (ret) 1631 return ret; 1632 1633 ret = eth_igb_dev_init(dev); 1634 1635 return ret; 1636 } 1637 1638 1639 static int 1640 igb_get_rx_buffer_size(struct e1000_hw *hw) 1641 { 1642 uint32_t rx_buf_size; 1643 if (hw->mac.type == e1000_82576) { 1644 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1645 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1646 /* PBS needs to be translated according to a lookup table */ 1647 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1648 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1649 rx_buf_size = (rx_buf_size << 10); 1650 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1651 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1652 } else { 1653 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1654 } 1655 1656 return rx_buf_size; 1657 } 1658 1659 /********************************************************************* 1660 * 1661 * Initialize the hardware 1662 * 1663 **********************************************************************/ 1664 static int 1665 igb_hardware_init(struct e1000_hw *hw) 1666 { 1667 uint32_t rx_buf_size; 1668 int diag; 1669 1670 /* Let the firmware know the OS is in control */ 1671 igb_hw_control_acquire(hw); 1672 1673 /* 1674 * These parameters control the automatic generation (Tx) and 1675 * response (Rx) to Ethernet PAUSE frames. 1676 * - High water mark should allow for at least two standard size (1518) 1677 * frames to be received after sending an XOFF. 1678 * - Low water mark works best when it is very near the high water mark. 1679 * This allows the receiver to restart by sending XON when it has 1680 * drained a bit. Here we use an arbitrary value of 1500 which will 1681 * restart after one full frame is pulled from the buffer. There 1682 * could be several smaller frames in the buffer and if so they will 1683 * not trigger the XON until their total number reduces the buffer 1684 * by 1500. 1685 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1686 */ 1687 rx_buf_size = igb_get_rx_buffer_size(hw); 1688 1689 hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2); 1690 hw->fc.low_water = hw->fc.high_water - 1500; 1691 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1692 hw->fc.send_xon = 1; 1693 1694 /* Set Flow control, use the tunable location if sane */ 1695 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1696 hw->fc.requested_mode = igb_fc_setting; 1697 else 1698 hw->fc.requested_mode = e1000_fc_none; 1699 1700 /* Issue a global reset */ 1701 igb_pf_reset_hw(hw); 1702 E1000_WRITE_REG(hw, E1000_WUC, 0); 1703 1704 diag = e1000_init_hw(hw); 1705 if (diag < 0) 1706 return diag; 1707 1708 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1709 e1000_get_phy_info(hw); 1710 e1000_check_for_link(hw); 1711 1712 return 0; 1713 } 1714 1715 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1716 static void 1717 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1718 { 1719 int pause_frames; 1720 1721 uint64_t old_gprc = stats->gprc; 1722 uint64_t old_gptc = stats->gptc; 1723 uint64_t old_tpr = stats->tpr; 1724 uint64_t old_tpt = stats->tpt; 1725 uint64_t old_rpthc = stats->rpthc; 1726 uint64_t old_hgptc = stats->hgptc; 1727 1728 if(hw->phy.media_type == e1000_media_type_copper || 1729 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1730 stats->symerrs += 1731 E1000_READ_REG(hw,E1000_SYMERRS); 1732 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1733 } 1734 1735 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1736 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1737 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1738 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1739 1740 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1741 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1742 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1743 stats->dc += E1000_READ_REG(hw, E1000_DC); 1744 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1745 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1746 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1747 /* 1748 ** For watchdog management we need to know if we have been 1749 ** paused during the last interval, so capture that here. 1750 */ 1751 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1752 stats->xoffrxc += pause_frames; 1753 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1754 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1755 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1756 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1757 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1758 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1759 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1760 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1761 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1762 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1763 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1764 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1765 1766 /* For the 64-bit byte counters the low dword must be read first. */ 1767 /* Both registers clear on the read of the high dword */ 1768 1769 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1770 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1771 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1772 stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN; 1773 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1774 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1775 stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN; 1776 1777 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1778 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1779 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1780 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1781 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1782 1783 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1784 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1785 1786 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1787 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1788 stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN; 1789 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1790 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1791 stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN; 1792 1793 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1794 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1795 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1796 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1797 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1798 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1799 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1800 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1801 1802 /* Interrupt Counts */ 1803 1804 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1805 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1806 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1807 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1808 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1809 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1810 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1811 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1812 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1813 1814 /* Host to Card Statistics */ 1815 1816 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1817 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1818 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1819 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1820 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1821 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1822 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1823 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1824 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1825 stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN; 1826 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1827 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1828 stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN; 1829 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1830 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1831 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1832 1833 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1834 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1835 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1836 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1837 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1838 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1839 } 1840 1841 static int 1842 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1843 { 1844 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1845 struct e1000_hw_stats *stats = 1846 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1847 1848 igb_read_stats_registers(hw, stats); 1849 1850 if (rte_stats == NULL) 1851 return -EINVAL; 1852 1853 /* Rx Errors */ 1854 rte_stats->imissed = stats->mpc; 1855 rte_stats->ierrors = stats->crcerrs + 1856 stats->rlec + stats->ruc + stats->roc + 1857 stats->rxerrc + stats->algnerrc + stats->cexterr; 1858 1859 /* Tx Errors */ 1860 rte_stats->oerrors = stats->ecol + stats->latecol; 1861 1862 rte_stats->ipackets = stats->gprc; 1863 rte_stats->opackets = stats->gptc; 1864 rte_stats->ibytes = stats->gorc; 1865 rte_stats->obytes = stats->gotc; 1866 return 0; 1867 } 1868 1869 static void 1870 eth_igb_stats_reset(struct rte_eth_dev *dev) 1871 { 1872 struct e1000_hw_stats *hw_stats = 1873 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1874 1875 /* HW registers are cleared on read */ 1876 eth_igb_stats_get(dev, NULL); 1877 1878 /* Reset software totals */ 1879 memset(hw_stats, 0, sizeof(*hw_stats)); 1880 } 1881 1882 static void 1883 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1884 { 1885 struct e1000_hw_stats *stats = 1886 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1887 1888 /* HW registers are cleared on read */ 1889 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1890 1891 /* Reset software totals */ 1892 memset(stats, 0, sizeof(*stats)); 1893 } 1894 1895 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1896 struct rte_eth_xstat_name *xstats_names, 1897 __rte_unused unsigned int size) 1898 { 1899 unsigned i; 1900 1901 if (xstats_names == NULL) 1902 return IGB_NB_XSTATS; 1903 1904 /* Note: limit checked in rte_eth_xstats_names() */ 1905 1906 for (i = 0; i < IGB_NB_XSTATS; i++) { 1907 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), 1908 "%s", rte_igb_stats_strings[i].name); 1909 } 1910 1911 return IGB_NB_XSTATS; 1912 } 1913 1914 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1915 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1916 unsigned int limit) 1917 { 1918 unsigned int i; 1919 1920 if (!ids) { 1921 if (xstats_names == NULL) 1922 return IGB_NB_XSTATS; 1923 1924 for (i = 0; i < IGB_NB_XSTATS; i++) 1925 snprintf(xstats_names[i].name, 1926 sizeof(xstats_names[i].name), 1927 "%s", rte_igb_stats_strings[i].name); 1928 1929 return IGB_NB_XSTATS; 1930 1931 } else { 1932 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1933 1934 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1935 IGB_NB_XSTATS); 1936 1937 for (i = 0; i < limit; i++) { 1938 if (ids[i] >= IGB_NB_XSTATS) { 1939 PMD_INIT_LOG(ERR, "id value isn't valid"); 1940 return -1; 1941 } 1942 strcpy(xstats_names[i].name, 1943 xstats_names_copy[ids[i]].name); 1944 } 1945 return limit; 1946 } 1947 } 1948 1949 static int 1950 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1951 unsigned n) 1952 { 1953 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1954 struct e1000_hw_stats *hw_stats = 1955 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1956 unsigned i; 1957 1958 if (n < IGB_NB_XSTATS) 1959 return IGB_NB_XSTATS; 1960 1961 igb_read_stats_registers(hw, hw_stats); 1962 1963 /* If this is a reset xstats is NULL, and we have cleared the 1964 * registers by reading them. 1965 */ 1966 if (!xstats) 1967 return 0; 1968 1969 /* Extended stats */ 1970 for (i = 0; i < IGB_NB_XSTATS; i++) { 1971 xstats[i].id = i; 1972 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1973 rte_igb_stats_strings[i].offset); 1974 } 1975 1976 return IGB_NB_XSTATS; 1977 } 1978 1979 static int 1980 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1981 uint64_t *values, unsigned int n) 1982 { 1983 unsigned int i; 1984 1985 if (!ids) { 1986 struct e1000_hw *hw = 1987 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1988 struct e1000_hw_stats *hw_stats = 1989 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1990 1991 if (n < IGB_NB_XSTATS) 1992 return IGB_NB_XSTATS; 1993 1994 igb_read_stats_registers(hw, hw_stats); 1995 1996 /* If this is a reset xstats is NULL, and we have cleared the 1997 * registers by reading them. 1998 */ 1999 if (!values) 2000 return 0; 2001 2002 /* Extended stats */ 2003 for (i = 0; i < IGB_NB_XSTATS; i++) 2004 values[i] = *(uint64_t *)(((char *)hw_stats) + 2005 rte_igb_stats_strings[i].offset); 2006 2007 return IGB_NB_XSTATS; 2008 2009 } else { 2010 uint64_t values_copy[IGB_NB_XSTATS]; 2011 2012 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2013 IGB_NB_XSTATS); 2014 2015 for (i = 0; i < n; i++) { 2016 if (ids[i] >= IGB_NB_XSTATS) { 2017 PMD_INIT_LOG(ERR, "id value isn't valid"); 2018 return -1; 2019 } 2020 values[i] = values_copy[ids[i]]; 2021 } 2022 return n; 2023 } 2024 } 2025 2026 static void 2027 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2028 { 2029 /* Good Rx packets, include VF loopback */ 2030 UPDATE_VF_STAT(E1000_VFGPRC, 2031 hw_stats->last_gprc, hw_stats->gprc); 2032 2033 /* Good Rx octets, include VF loopback */ 2034 UPDATE_VF_STAT(E1000_VFGORC, 2035 hw_stats->last_gorc, hw_stats->gorc); 2036 2037 /* Good Tx packets, include VF loopback */ 2038 UPDATE_VF_STAT(E1000_VFGPTC, 2039 hw_stats->last_gptc, hw_stats->gptc); 2040 2041 /* Good Tx octets, include VF loopback */ 2042 UPDATE_VF_STAT(E1000_VFGOTC, 2043 hw_stats->last_gotc, hw_stats->gotc); 2044 2045 /* Rx Multicst packets */ 2046 UPDATE_VF_STAT(E1000_VFMPRC, 2047 hw_stats->last_mprc, hw_stats->mprc); 2048 2049 /* Good Rx loopback packets */ 2050 UPDATE_VF_STAT(E1000_VFGPRLBC, 2051 hw_stats->last_gprlbc, hw_stats->gprlbc); 2052 2053 /* Good Rx loopback octets */ 2054 UPDATE_VF_STAT(E1000_VFGORLBC, 2055 hw_stats->last_gorlbc, hw_stats->gorlbc); 2056 2057 /* Good Tx loopback packets */ 2058 UPDATE_VF_STAT(E1000_VFGPTLBC, 2059 hw_stats->last_gptlbc, hw_stats->gptlbc); 2060 2061 /* Good Tx loopback octets */ 2062 UPDATE_VF_STAT(E1000_VFGOTLBC, 2063 hw_stats->last_gotlbc, hw_stats->gotlbc); 2064 } 2065 2066 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2067 struct rte_eth_xstat_name *xstats_names, 2068 __rte_unused unsigned limit) 2069 { 2070 unsigned i; 2071 2072 if (xstats_names != NULL) 2073 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2074 snprintf(xstats_names[i].name, 2075 sizeof(xstats_names[i].name), "%s", 2076 rte_igbvf_stats_strings[i].name); 2077 } 2078 return IGBVF_NB_XSTATS; 2079 } 2080 2081 static int 2082 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2083 unsigned n) 2084 { 2085 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2086 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2087 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2088 unsigned i; 2089 2090 if (n < IGBVF_NB_XSTATS) 2091 return IGBVF_NB_XSTATS; 2092 2093 igbvf_read_stats_registers(hw, hw_stats); 2094 2095 if (!xstats) 2096 return 0; 2097 2098 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2099 xstats[i].id = i; 2100 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2101 rte_igbvf_stats_strings[i].offset); 2102 } 2103 2104 return IGBVF_NB_XSTATS; 2105 } 2106 2107 static int 2108 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2109 { 2110 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2111 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2112 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2113 2114 igbvf_read_stats_registers(hw, hw_stats); 2115 2116 if (rte_stats == NULL) 2117 return -EINVAL; 2118 2119 rte_stats->ipackets = hw_stats->gprc; 2120 rte_stats->ibytes = hw_stats->gorc; 2121 rte_stats->opackets = hw_stats->gptc; 2122 rte_stats->obytes = hw_stats->gotc; 2123 return 0; 2124 } 2125 2126 static void 2127 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2128 { 2129 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2130 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2131 2132 /* Sync HW register to the last stats */ 2133 eth_igbvf_stats_get(dev, NULL); 2134 2135 /* reset HW current stats*/ 2136 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2137 offsetof(struct e1000_vf_stats, gprc)); 2138 } 2139 2140 static int 2141 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2142 size_t fw_size) 2143 { 2144 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2145 struct e1000_fw_version fw; 2146 int ret; 2147 2148 e1000_get_fw_version(hw, &fw); 2149 2150 switch (hw->mac.type) { 2151 case e1000_i210: 2152 case e1000_i211: 2153 if (!(e1000_get_flash_presence_i210(hw))) { 2154 ret = snprintf(fw_version, fw_size, 2155 "%2d.%2d-%d", 2156 fw.invm_major, fw.invm_minor, 2157 fw.invm_img_type); 2158 break; 2159 } 2160 /* fall through */ 2161 default: 2162 /* if option rom is valid, display its version too */ 2163 if (fw.or_valid) { 2164 ret = snprintf(fw_version, fw_size, 2165 "%d.%d, 0x%08x, %d.%d.%d", 2166 fw.eep_major, fw.eep_minor, fw.etrack_id, 2167 fw.or_major, fw.or_build, fw.or_patch); 2168 /* no option rom */ 2169 } else { 2170 if (fw.etrack_id != 0X0000) { 2171 ret = snprintf(fw_version, fw_size, 2172 "%d.%d, 0x%08x", 2173 fw.eep_major, fw.eep_minor, 2174 fw.etrack_id); 2175 } else { 2176 ret = snprintf(fw_version, fw_size, 2177 "%d.%d.%d", 2178 fw.eep_major, fw.eep_minor, 2179 fw.eep_build); 2180 } 2181 } 2182 break; 2183 } 2184 2185 ret += 1; /* add the size of '\0' */ 2186 if (fw_size < (u32)ret) 2187 return ret; 2188 else 2189 return 0; 2190 } 2191 2192 static void 2193 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2194 { 2195 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2196 2197 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2198 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2199 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2200 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2201 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2202 dev_info->rx_queue_offload_capa; 2203 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2204 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2205 dev_info->tx_queue_offload_capa; 2206 2207 switch (hw->mac.type) { 2208 case e1000_82575: 2209 dev_info->max_rx_queues = 4; 2210 dev_info->max_tx_queues = 4; 2211 dev_info->max_vmdq_pools = 0; 2212 break; 2213 2214 case e1000_82576: 2215 dev_info->max_rx_queues = 16; 2216 dev_info->max_tx_queues = 16; 2217 dev_info->max_vmdq_pools = ETH_8_POOLS; 2218 dev_info->vmdq_queue_num = 16; 2219 break; 2220 2221 case e1000_82580: 2222 dev_info->max_rx_queues = 8; 2223 dev_info->max_tx_queues = 8; 2224 dev_info->max_vmdq_pools = ETH_8_POOLS; 2225 dev_info->vmdq_queue_num = 8; 2226 break; 2227 2228 case e1000_i350: 2229 dev_info->max_rx_queues = 8; 2230 dev_info->max_tx_queues = 8; 2231 dev_info->max_vmdq_pools = ETH_8_POOLS; 2232 dev_info->vmdq_queue_num = 8; 2233 break; 2234 2235 case e1000_i354: 2236 dev_info->max_rx_queues = 8; 2237 dev_info->max_tx_queues = 8; 2238 break; 2239 2240 case e1000_i210: 2241 dev_info->max_rx_queues = 4; 2242 dev_info->max_tx_queues = 4; 2243 dev_info->max_vmdq_pools = 0; 2244 break; 2245 2246 case e1000_i211: 2247 dev_info->max_rx_queues = 2; 2248 dev_info->max_tx_queues = 2; 2249 dev_info->max_vmdq_pools = 0; 2250 break; 2251 2252 default: 2253 /* Should not happen */ 2254 break; 2255 } 2256 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2257 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2258 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2259 2260 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2261 .rx_thresh = { 2262 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2263 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2264 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2265 }, 2266 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2267 .rx_drop_en = 0, 2268 .offloads = 0, 2269 }; 2270 2271 dev_info->default_txconf = (struct rte_eth_txconf) { 2272 .tx_thresh = { 2273 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2274 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2275 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2276 }, 2277 .offloads = 0, 2278 }; 2279 2280 dev_info->rx_desc_lim = rx_desc_lim; 2281 dev_info->tx_desc_lim = tx_desc_lim; 2282 2283 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2284 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2285 ETH_LINK_SPEED_1G; 2286 2287 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2288 dev_info->min_mtu = ETHER_MIN_MTU; 2289 2290 } 2291 2292 static const uint32_t * 2293 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2294 { 2295 static const uint32_t ptypes[] = { 2296 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2297 RTE_PTYPE_L2_ETHER, 2298 RTE_PTYPE_L3_IPV4, 2299 RTE_PTYPE_L3_IPV4_EXT, 2300 RTE_PTYPE_L3_IPV6, 2301 RTE_PTYPE_L3_IPV6_EXT, 2302 RTE_PTYPE_L4_TCP, 2303 RTE_PTYPE_L4_UDP, 2304 RTE_PTYPE_L4_SCTP, 2305 RTE_PTYPE_TUNNEL_IP, 2306 RTE_PTYPE_INNER_L3_IPV6, 2307 RTE_PTYPE_INNER_L3_IPV6_EXT, 2308 RTE_PTYPE_INNER_L4_TCP, 2309 RTE_PTYPE_INNER_L4_UDP, 2310 RTE_PTYPE_UNKNOWN 2311 }; 2312 2313 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2314 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2315 return ptypes; 2316 return NULL; 2317 } 2318 2319 static void 2320 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2321 { 2322 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2323 2324 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2325 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2326 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2327 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2328 DEV_TX_OFFLOAD_IPV4_CKSUM | 2329 DEV_TX_OFFLOAD_UDP_CKSUM | 2330 DEV_TX_OFFLOAD_TCP_CKSUM | 2331 DEV_TX_OFFLOAD_SCTP_CKSUM | 2332 DEV_TX_OFFLOAD_TCP_TSO; 2333 switch (hw->mac.type) { 2334 case e1000_vfadapt: 2335 dev_info->max_rx_queues = 2; 2336 dev_info->max_tx_queues = 2; 2337 break; 2338 case e1000_vfadapt_i350: 2339 dev_info->max_rx_queues = 1; 2340 dev_info->max_tx_queues = 1; 2341 break; 2342 default: 2343 /* Should not happen */ 2344 break; 2345 } 2346 2347 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2348 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2349 dev_info->rx_queue_offload_capa; 2350 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2351 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2352 dev_info->tx_queue_offload_capa; 2353 2354 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2355 .rx_thresh = { 2356 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2357 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2358 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2359 }, 2360 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2361 .rx_drop_en = 0, 2362 .offloads = 0, 2363 }; 2364 2365 dev_info->default_txconf = (struct rte_eth_txconf) { 2366 .tx_thresh = { 2367 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2368 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2369 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2370 }, 2371 .offloads = 0, 2372 }; 2373 2374 dev_info->rx_desc_lim = rx_desc_lim; 2375 dev_info->tx_desc_lim = tx_desc_lim; 2376 } 2377 2378 /* return 0 means link status changed, -1 means not changed */ 2379 static int 2380 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2381 { 2382 struct e1000_hw *hw = 2383 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2384 struct rte_eth_link link; 2385 int link_check, count; 2386 2387 link_check = 0; 2388 hw->mac.get_link_status = 1; 2389 2390 /* possible wait-to-complete in up to 9 seconds */ 2391 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2392 /* Read the real link status */ 2393 switch (hw->phy.media_type) { 2394 case e1000_media_type_copper: 2395 /* Do the work to read phy */ 2396 e1000_check_for_link(hw); 2397 link_check = !hw->mac.get_link_status; 2398 break; 2399 2400 case e1000_media_type_fiber: 2401 e1000_check_for_link(hw); 2402 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2403 E1000_STATUS_LU); 2404 break; 2405 2406 case e1000_media_type_internal_serdes: 2407 e1000_check_for_link(hw); 2408 link_check = hw->mac.serdes_has_link; 2409 break; 2410 2411 /* VF device is type_unknown */ 2412 case e1000_media_type_unknown: 2413 eth_igbvf_link_update(hw); 2414 link_check = !hw->mac.get_link_status; 2415 break; 2416 2417 default: 2418 break; 2419 } 2420 if (link_check || wait_to_complete == 0) 2421 break; 2422 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2423 } 2424 memset(&link, 0, sizeof(link)); 2425 2426 /* Now we check if a transition has happened */ 2427 if (link_check) { 2428 uint16_t duplex, speed; 2429 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2430 link.link_duplex = (duplex == FULL_DUPLEX) ? 2431 ETH_LINK_FULL_DUPLEX : 2432 ETH_LINK_HALF_DUPLEX; 2433 link.link_speed = speed; 2434 link.link_status = ETH_LINK_UP; 2435 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2436 ETH_LINK_SPEED_FIXED); 2437 } else if (!link_check) { 2438 link.link_speed = 0; 2439 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2440 link.link_status = ETH_LINK_DOWN; 2441 link.link_autoneg = ETH_LINK_FIXED; 2442 } 2443 2444 return rte_eth_linkstatus_set(dev, &link); 2445 } 2446 2447 /* 2448 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2449 * For ASF and Pass Through versions of f/w this means 2450 * that the driver is loaded. 2451 */ 2452 static void 2453 igb_hw_control_acquire(struct e1000_hw *hw) 2454 { 2455 uint32_t ctrl_ext; 2456 2457 /* Let firmware know the driver has taken over */ 2458 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2459 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2460 } 2461 2462 /* 2463 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2464 * For ASF and Pass Through versions of f/w this means that the 2465 * driver is no longer loaded. 2466 */ 2467 static void 2468 igb_hw_control_release(struct e1000_hw *hw) 2469 { 2470 uint32_t ctrl_ext; 2471 2472 /* Let firmware taken over control of h/w */ 2473 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2474 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2475 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2476 } 2477 2478 /* 2479 * Bit of a misnomer, what this really means is 2480 * to enable OS management of the system... aka 2481 * to disable special hardware management features. 2482 */ 2483 static void 2484 igb_init_manageability(struct e1000_hw *hw) 2485 { 2486 if (e1000_enable_mng_pass_thru(hw)) { 2487 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2488 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2489 2490 /* disable hardware interception of ARP */ 2491 manc &= ~(E1000_MANC_ARP_EN); 2492 2493 /* enable receiving management packets to the host */ 2494 manc |= E1000_MANC_EN_MNG2HOST; 2495 manc2h |= 1 << 5; /* Mng Port 623 */ 2496 manc2h |= 1 << 6; /* Mng Port 664 */ 2497 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2498 E1000_WRITE_REG(hw, E1000_MANC, manc); 2499 } 2500 } 2501 2502 static void 2503 igb_release_manageability(struct e1000_hw *hw) 2504 { 2505 if (e1000_enable_mng_pass_thru(hw)) { 2506 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2507 2508 manc |= E1000_MANC_ARP_EN; 2509 manc &= ~E1000_MANC_EN_MNG2HOST; 2510 2511 E1000_WRITE_REG(hw, E1000_MANC, manc); 2512 } 2513 } 2514 2515 static void 2516 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2517 { 2518 struct e1000_hw *hw = 2519 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2520 uint32_t rctl; 2521 2522 rctl = E1000_READ_REG(hw, E1000_RCTL); 2523 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2524 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2525 } 2526 2527 static void 2528 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2529 { 2530 struct e1000_hw *hw = 2531 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2532 uint32_t rctl; 2533 2534 rctl = E1000_READ_REG(hw, E1000_RCTL); 2535 rctl &= (~E1000_RCTL_UPE); 2536 if (dev->data->all_multicast == 1) 2537 rctl |= E1000_RCTL_MPE; 2538 else 2539 rctl &= (~E1000_RCTL_MPE); 2540 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2541 } 2542 2543 static void 2544 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2545 { 2546 struct e1000_hw *hw = 2547 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2548 uint32_t rctl; 2549 2550 rctl = E1000_READ_REG(hw, E1000_RCTL); 2551 rctl |= E1000_RCTL_MPE; 2552 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2553 } 2554 2555 static void 2556 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2557 { 2558 struct e1000_hw *hw = 2559 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2560 uint32_t rctl; 2561 2562 if (dev->data->promiscuous == 1) 2563 return; /* must remain in all_multicast mode */ 2564 rctl = E1000_READ_REG(hw, E1000_RCTL); 2565 rctl &= (~E1000_RCTL_MPE); 2566 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2567 } 2568 2569 static int 2570 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2571 { 2572 struct e1000_hw *hw = 2573 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2574 struct e1000_vfta * shadow_vfta = 2575 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2576 uint32_t vfta; 2577 uint32_t vid_idx; 2578 uint32_t vid_bit; 2579 2580 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2581 E1000_VFTA_ENTRY_MASK); 2582 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2583 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2584 if (on) 2585 vfta |= vid_bit; 2586 else 2587 vfta &= ~vid_bit; 2588 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2589 2590 /* update local VFTA copy */ 2591 shadow_vfta->vfta[vid_idx] = vfta; 2592 2593 return 0; 2594 } 2595 2596 static int 2597 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2598 enum rte_vlan_type vlan_type, 2599 uint16_t tpid) 2600 { 2601 struct e1000_hw *hw = 2602 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2603 uint32_t reg, qinq; 2604 2605 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2606 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2607 2608 /* only outer TPID of double VLAN can be configured*/ 2609 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2610 reg = E1000_READ_REG(hw, E1000_VET); 2611 reg = (reg & (~E1000_VET_VET_EXT)) | 2612 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2613 E1000_WRITE_REG(hw, E1000_VET, reg); 2614 2615 return 0; 2616 } 2617 2618 /* all other TPID values are read-only*/ 2619 PMD_DRV_LOG(ERR, "Not supported"); 2620 2621 return -ENOTSUP; 2622 } 2623 2624 static void 2625 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2626 { 2627 struct e1000_hw *hw = 2628 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2629 uint32_t reg; 2630 2631 /* Filter Table Disable */ 2632 reg = E1000_READ_REG(hw, E1000_RCTL); 2633 reg &= ~E1000_RCTL_CFIEN; 2634 reg &= ~E1000_RCTL_VFE; 2635 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2636 } 2637 2638 static void 2639 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2640 { 2641 struct e1000_hw *hw = 2642 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2643 struct e1000_vfta * shadow_vfta = 2644 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2645 uint32_t reg; 2646 int i; 2647 2648 /* Filter Table Enable, CFI not used for packet acceptance */ 2649 reg = E1000_READ_REG(hw, E1000_RCTL); 2650 reg &= ~E1000_RCTL_CFIEN; 2651 reg |= E1000_RCTL_VFE; 2652 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2653 2654 /* restore VFTA table */ 2655 for (i = 0; i < IGB_VFTA_SIZE; i++) 2656 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2657 } 2658 2659 static void 2660 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2661 { 2662 struct e1000_hw *hw = 2663 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2664 uint32_t reg; 2665 2666 /* VLAN Mode Disable */ 2667 reg = E1000_READ_REG(hw, E1000_CTRL); 2668 reg &= ~E1000_CTRL_VME; 2669 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2670 } 2671 2672 static void 2673 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2674 { 2675 struct e1000_hw *hw = 2676 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2677 uint32_t reg; 2678 2679 /* VLAN Mode Enable */ 2680 reg = E1000_READ_REG(hw, E1000_CTRL); 2681 reg |= E1000_CTRL_VME; 2682 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2683 } 2684 2685 static void 2686 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2687 { 2688 struct e1000_hw *hw = 2689 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2690 uint32_t reg; 2691 2692 /* CTRL_EXT: Extended VLAN */ 2693 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2694 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2695 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2696 2697 /* Update maximum packet length */ 2698 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2699 E1000_WRITE_REG(hw, E1000_RLPML, 2700 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2701 VLAN_TAG_SIZE); 2702 } 2703 2704 static void 2705 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2706 { 2707 struct e1000_hw *hw = 2708 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2709 uint32_t reg; 2710 2711 /* CTRL_EXT: Extended VLAN */ 2712 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2713 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2714 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2715 2716 /* Update maximum packet length */ 2717 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2718 E1000_WRITE_REG(hw, E1000_RLPML, 2719 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2720 2 * VLAN_TAG_SIZE); 2721 } 2722 2723 static int 2724 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2725 { 2726 struct rte_eth_rxmode *rxmode; 2727 2728 rxmode = &dev->data->dev_conf.rxmode; 2729 if(mask & ETH_VLAN_STRIP_MASK){ 2730 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2731 igb_vlan_hw_strip_enable(dev); 2732 else 2733 igb_vlan_hw_strip_disable(dev); 2734 } 2735 2736 if(mask & ETH_VLAN_FILTER_MASK){ 2737 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2738 igb_vlan_hw_filter_enable(dev); 2739 else 2740 igb_vlan_hw_filter_disable(dev); 2741 } 2742 2743 if(mask & ETH_VLAN_EXTEND_MASK){ 2744 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2745 igb_vlan_hw_extend_enable(dev); 2746 else 2747 igb_vlan_hw_extend_disable(dev); 2748 } 2749 2750 return 0; 2751 } 2752 2753 2754 /** 2755 * It enables the interrupt mask and then enable the interrupt. 2756 * 2757 * @param dev 2758 * Pointer to struct rte_eth_dev. 2759 * @param on 2760 * Enable or Disable 2761 * 2762 * @return 2763 * - On success, zero. 2764 * - On failure, a negative value. 2765 */ 2766 static int 2767 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2768 { 2769 struct e1000_interrupt *intr = 2770 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2771 2772 if (on) 2773 intr->mask |= E1000_ICR_LSC; 2774 else 2775 intr->mask &= ~E1000_ICR_LSC; 2776 2777 return 0; 2778 } 2779 2780 /* It clears the interrupt causes and enables the interrupt. 2781 * It will be called once only during nic initialized. 2782 * 2783 * @param dev 2784 * Pointer to struct rte_eth_dev. 2785 * 2786 * @return 2787 * - On success, zero. 2788 * - On failure, a negative value. 2789 */ 2790 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2791 { 2792 uint32_t mask, regval; 2793 struct e1000_hw *hw = 2794 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2795 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2796 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2797 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2798 struct rte_eth_dev_info dev_info; 2799 2800 memset(&dev_info, 0, sizeof(dev_info)); 2801 eth_igb_infos_get(dev, &dev_info); 2802 2803 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2804 regval = E1000_READ_REG(hw, E1000_EIMS); 2805 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2806 2807 return 0; 2808 } 2809 2810 /* 2811 * It reads ICR and gets interrupt causes, check it and set a bit flag 2812 * to update link status. 2813 * 2814 * @param dev 2815 * Pointer to struct rte_eth_dev. 2816 * 2817 * @return 2818 * - On success, zero. 2819 * - On failure, a negative value. 2820 */ 2821 static int 2822 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2823 { 2824 uint32_t icr; 2825 struct e1000_hw *hw = 2826 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2827 struct e1000_interrupt *intr = 2828 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2829 2830 igb_intr_disable(dev); 2831 2832 /* read-on-clear nic registers here */ 2833 icr = E1000_READ_REG(hw, E1000_ICR); 2834 2835 intr->flags = 0; 2836 if (icr & E1000_ICR_LSC) { 2837 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2838 } 2839 2840 if (icr & E1000_ICR_VMMB) 2841 intr->flags |= E1000_FLAG_MAILBOX; 2842 2843 return 0; 2844 } 2845 2846 /* 2847 * It executes link_update after knowing an interrupt is prsent. 2848 * 2849 * @param dev 2850 * Pointer to struct rte_eth_dev. 2851 * 2852 * @return 2853 * - On success, zero. 2854 * - On failure, a negative value. 2855 */ 2856 static int 2857 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2858 struct rte_intr_handle *intr_handle) 2859 { 2860 struct e1000_hw *hw = 2861 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2862 struct e1000_interrupt *intr = 2863 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2864 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2865 struct rte_eth_link link; 2866 int ret; 2867 2868 if (intr->flags & E1000_FLAG_MAILBOX) { 2869 igb_pf_mbx_process(dev); 2870 intr->flags &= ~E1000_FLAG_MAILBOX; 2871 } 2872 2873 igb_intr_enable(dev); 2874 rte_intr_enable(intr_handle); 2875 2876 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2877 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2878 2879 /* set get_link_status to check register later */ 2880 hw->mac.get_link_status = 1; 2881 ret = eth_igb_link_update(dev, 0); 2882 2883 /* check if link has changed */ 2884 if (ret < 0) 2885 return 0; 2886 2887 rte_eth_linkstatus_get(dev, &link); 2888 if (link.link_status) { 2889 PMD_INIT_LOG(INFO, 2890 " Port %d: Link Up - speed %u Mbps - %s", 2891 dev->data->port_id, 2892 (unsigned)link.link_speed, 2893 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2894 "full-duplex" : "half-duplex"); 2895 } else { 2896 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2897 dev->data->port_id); 2898 } 2899 2900 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2901 pci_dev->addr.domain, 2902 pci_dev->addr.bus, 2903 pci_dev->addr.devid, 2904 pci_dev->addr.function); 2905 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2906 NULL); 2907 } 2908 2909 return 0; 2910 } 2911 2912 /** 2913 * Interrupt handler which shall be registered at first. 2914 * 2915 * @param handle 2916 * Pointer to interrupt handle. 2917 * @param param 2918 * The address of parameter (struct rte_eth_dev *) regsitered before. 2919 * 2920 * @return 2921 * void 2922 */ 2923 static void 2924 eth_igb_interrupt_handler(void *param) 2925 { 2926 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2927 2928 eth_igb_interrupt_get_status(dev); 2929 eth_igb_interrupt_action(dev, dev->intr_handle); 2930 } 2931 2932 static int 2933 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2934 { 2935 uint32_t eicr; 2936 struct e1000_hw *hw = 2937 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2938 struct e1000_interrupt *intr = 2939 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2940 2941 igbvf_intr_disable(hw); 2942 2943 /* read-on-clear nic registers here */ 2944 eicr = E1000_READ_REG(hw, E1000_EICR); 2945 intr->flags = 0; 2946 2947 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2948 intr->flags |= E1000_FLAG_MAILBOX; 2949 2950 return 0; 2951 } 2952 2953 void igbvf_mbx_process(struct rte_eth_dev *dev) 2954 { 2955 struct e1000_hw *hw = 2956 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2957 struct e1000_mbx_info *mbx = &hw->mbx; 2958 u32 in_msg = 0; 2959 2960 /* peek the message first */ 2961 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2962 2963 /* PF reset VF event */ 2964 if (in_msg == E1000_PF_CONTROL_MSG) { 2965 /* dummy mbx read to ack pf */ 2966 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2967 return; 2968 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2969 NULL); 2970 } 2971 } 2972 2973 static int 2974 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2975 { 2976 struct e1000_interrupt *intr = 2977 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2978 2979 if (intr->flags & E1000_FLAG_MAILBOX) { 2980 igbvf_mbx_process(dev); 2981 intr->flags &= ~E1000_FLAG_MAILBOX; 2982 } 2983 2984 igbvf_intr_enable(dev); 2985 rte_intr_enable(intr_handle); 2986 2987 return 0; 2988 } 2989 2990 static void 2991 eth_igbvf_interrupt_handler(void *param) 2992 { 2993 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2994 2995 eth_igbvf_interrupt_get_status(dev); 2996 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2997 } 2998 2999 static int 3000 eth_igb_led_on(struct rte_eth_dev *dev) 3001 { 3002 struct e1000_hw *hw; 3003 3004 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3005 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3006 } 3007 3008 static int 3009 eth_igb_led_off(struct rte_eth_dev *dev) 3010 { 3011 struct e1000_hw *hw; 3012 3013 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3014 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3015 } 3016 3017 static int 3018 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3019 { 3020 struct e1000_hw *hw; 3021 uint32_t ctrl; 3022 int tx_pause; 3023 int rx_pause; 3024 3025 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3026 fc_conf->pause_time = hw->fc.pause_time; 3027 fc_conf->high_water = hw->fc.high_water; 3028 fc_conf->low_water = hw->fc.low_water; 3029 fc_conf->send_xon = hw->fc.send_xon; 3030 fc_conf->autoneg = hw->mac.autoneg; 3031 3032 /* 3033 * Return rx_pause and tx_pause status according to actual setting of 3034 * the TFCE and RFCE bits in the CTRL register. 3035 */ 3036 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3037 if (ctrl & E1000_CTRL_TFCE) 3038 tx_pause = 1; 3039 else 3040 tx_pause = 0; 3041 3042 if (ctrl & E1000_CTRL_RFCE) 3043 rx_pause = 1; 3044 else 3045 rx_pause = 0; 3046 3047 if (rx_pause && tx_pause) 3048 fc_conf->mode = RTE_FC_FULL; 3049 else if (rx_pause) 3050 fc_conf->mode = RTE_FC_RX_PAUSE; 3051 else if (tx_pause) 3052 fc_conf->mode = RTE_FC_TX_PAUSE; 3053 else 3054 fc_conf->mode = RTE_FC_NONE; 3055 3056 return 0; 3057 } 3058 3059 static int 3060 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3061 { 3062 struct e1000_hw *hw; 3063 int err; 3064 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3065 e1000_fc_none, 3066 e1000_fc_rx_pause, 3067 e1000_fc_tx_pause, 3068 e1000_fc_full 3069 }; 3070 uint32_t rx_buf_size; 3071 uint32_t max_high_water; 3072 uint32_t rctl; 3073 3074 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3075 if (fc_conf->autoneg != hw->mac.autoneg) 3076 return -ENOTSUP; 3077 rx_buf_size = igb_get_rx_buffer_size(hw); 3078 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3079 3080 /* At least reserve one Ethernet frame for watermark */ 3081 max_high_water = rx_buf_size - ETHER_MAX_LEN; 3082 if ((fc_conf->high_water > max_high_water) || 3083 (fc_conf->high_water < fc_conf->low_water)) { 3084 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3085 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3086 return -EINVAL; 3087 } 3088 3089 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3090 hw->fc.pause_time = fc_conf->pause_time; 3091 hw->fc.high_water = fc_conf->high_water; 3092 hw->fc.low_water = fc_conf->low_water; 3093 hw->fc.send_xon = fc_conf->send_xon; 3094 3095 err = e1000_setup_link_generic(hw); 3096 if (err == E1000_SUCCESS) { 3097 3098 /* check if we want to forward MAC frames - driver doesn't have native 3099 * capability to do that, so we'll write the registers ourselves */ 3100 3101 rctl = E1000_READ_REG(hw, E1000_RCTL); 3102 3103 /* set or clear MFLCN.PMCF bit depending on configuration */ 3104 if (fc_conf->mac_ctrl_frame_fwd != 0) 3105 rctl |= E1000_RCTL_PMCF; 3106 else 3107 rctl &= ~E1000_RCTL_PMCF; 3108 3109 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3110 E1000_WRITE_FLUSH(hw); 3111 3112 return 0; 3113 } 3114 3115 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3116 return -EIO; 3117 } 3118 3119 #define E1000_RAH_POOLSEL_SHIFT (18) 3120 static int 3121 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 3122 uint32_t index, uint32_t pool) 3123 { 3124 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3125 uint32_t rah; 3126 3127 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3128 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3129 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3130 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3131 return 0; 3132 } 3133 3134 static void 3135 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3136 { 3137 uint8_t addr[ETHER_ADDR_LEN]; 3138 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3139 3140 memset(addr, 0, sizeof(addr)); 3141 3142 e1000_rar_set(hw, addr, index); 3143 } 3144 3145 static int 3146 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3147 struct ether_addr *addr) 3148 { 3149 eth_igb_rar_clear(dev, 0); 3150 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3151 3152 return 0; 3153 } 3154 /* 3155 * Virtual Function operations 3156 */ 3157 static void 3158 igbvf_intr_disable(struct e1000_hw *hw) 3159 { 3160 PMD_INIT_FUNC_TRACE(); 3161 3162 /* Clear interrupt mask to stop from interrupts being generated */ 3163 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3164 3165 E1000_WRITE_FLUSH(hw); 3166 } 3167 3168 static void 3169 igbvf_stop_adapter(struct rte_eth_dev *dev) 3170 { 3171 u32 reg_val; 3172 u16 i; 3173 struct rte_eth_dev_info dev_info; 3174 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3175 3176 memset(&dev_info, 0, sizeof(dev_info)); 3177 eth_igbvf_infos_get(dev, &dev_info); 3178 3179 /* Clear interrupt mask to stop from interrupts being generated */ 3180 igbvf_intr_disable(hw); 3181 3182 /* Clear any pending interrupts, flush previous writes */ 3183 E1000_READ_REG(hw, E1000_EICR); 3184 3185 /* Disable the transmit unit. Each queue must be disabled. */ 3186 for (i = 0; i < dev_info.max_tx_queues; i++) 3187 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3188 3189 /* Disable the receive unit by stopping each queue */ 3190 for (i = 0; i < dev_info.max_rx_queues; i++) { 3191 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3192 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3193 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3194 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3195 ; 3196 } 3197 3198 /* flush all queues disables */ 3199 E1000_WRITE_FLUSH(hw); 3200 msec_delay(2); 3201 } 3202 3203 static int eth_igbvf_link_update(struct e1000_hw *hw) 3204 { 3205 struct e1000_mbx_info *mbx = &hw->mbx; 3206 struct e1000_mac_info *mac = &hw->mac; 3207 int ret_val = E1000_SUCCESS; 3208 3209 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3210 3211 /* 3212 * We only want to run this if there has been a rst asserted. 3213 * in this case that could mean a link change, device reset, 3214 * or a virtual function reset 3215 */ 3216 3217 /* If we were hit with a reset or timeout drop the link */ 3218 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3219 mac->get_link_status = TRUE; 3220 3221 if (!mac->get_link_status) 3222 goto out; 3223 3224 /* if link status is down no point in checking to see if pf is up */ 3225 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3226 goto out; 3227 3228 /* if we passed all the tests above then the link is up and we no 3229 * longer need to check for link */ 3230 mac->get_link_status = FALSE; 3231 3232 out: 3233 return ret_val; 3234 } 3235 3236 3237 static int 3238 igbvf_dev_configure(struct rte_eth_dev *dev) 3239 { 3240 struct rte_eth_conf* conf = &dev->data->dev_conf; 3241 3242 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3243 dev->data->port_id); 3244 3245 /* 3246 * VF has no ability to enable/disable HW CRC 3247 * Keep the persistent behavior the same as Host PF 3248 */ 3249 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3250 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3251 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3252 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3253 } 3254 #else 3255 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3256 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3257 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3258 } 3259 #endif 3260 3261 return 0; 3262 } 3263 3264 static int 3265 igbvf_dev_start(struct rte_eth_dev *dev) 3266 { 3267 struct e1000_hw *hw = 3268 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3269 struct e1000_adapter *adapter = 3270 E1000_DEV_PRIVATE(dev->data->dev_private); 3271 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3272 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3273 int ret; 3274 uint32_t intr_vector = 0; 3275 3276 PMD_INIT_FUNC_TRACE(); 3277 3278 hw->mac.ops.reset_hw(hw); 3279 adapter->stopped = 0; 3280 3281 /* Set all vfta */ 3282 igbvf_set_vfta_all(dev,1); 3283 3284 eth_igbvf_tx_init(dev); 3285 3286 /* This can fail when allocating mbufs for descriptor rings */ 3287 ret = eth_igbvf_rx_init(dev); 3288 if (ret) { 3289 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3290 igb_dev_clear_queues(dev); 3291 return ret; 3292 } 3293 3294 /* check and configure queue intr-vector mapping */ 3295 if (rte_intr_cap_multiple(intr_handle) && 3296 dev->data->dev_conf.intr_conf.rxq) { 3297 intr_vector = dev->data->nb_rx_queues; 3298 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3299 if (ret) 3300 return ret; 3301 } 3302 3303 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3304 intr_handle->intr_vec = 3305 rte_zmalloc("intr_vec", 3306 dev->data->nb_rx_queues * sizeof(int), 0); 3307 if (!intr_handle->intr_vec) { 3308 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3309 " intr_vec", dev->data->nb_rx_queues); 3310 return -ENOMEM; 3311 } 3312 } 3313 3314 eth_igbvf_configure_msix_intr(dev); 3315 3316 /* enable uio/vfio intr/eventfd mapping */ 3317 rte_intr_enable(intr_handle); 3318 3319 /* resume enabled intr since hw reset */ 3320 igbvf_intr_enable(dev); 3321 3322 return 0; 3323 } 3324 3325 static void 3326 igbvf_dev_stop(struct rte_eth_dev *dev) 3327 { 3328 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3329 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3330 3331 PMD_INIT_FUNC_TRACE(); 3332 3333 igbvf_stop_adapter(dev); 3334 3335 /* 3336 * Clear what we set, but we still keep shadow_vfta to 3337 * restore after device starts 3338 */ 3339 igbvf_set_vfta_all(dev,0); 3340 3341 igb_dev_clear_queues(dev); 3342 3343 /* disable intr eventfd mapping */ 3344 rte_intr_disable(intr_handle); 3345 3346 /* Clean datapath event and queue/vec mapping */ 3347 rte_intr_efd_disable(intr_handle); 3348 if (intr_handle->intr_vec) { 3349 rte_free(intr_handle->intr_vec); 3350 intr_handle->intr_vec = NULL; 3351 } 3352 } 3353 3354 static void 3355 igbvf_dev_close(struct rte_eth_dev *dev) 3356 { 3357 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3358 struct e1000_adapter *adapter = 3359 E1000_DEV_PRIVATE(dev->data->dev_private); 3360 struct ether_addr addr; 3361 3362 PMD_INIT_FUNC_TRACE(); 3363 3364 e1000_reset_hw(hw); 3365 3366 igbvf_dev_stop(dev); 3367 adapter->stopped = 1; 3368 igb_dev_free_queues(dev); 3369 3370 /** 3371 * reprogram the RAR with a zero mac address, 3372 * to ensure that the VF traffic goes to the PF 3373 * after stop, close and detach of the VF. 3374 **/ 3375 3376 memset(&addr, 0, sizeof(addr)); 3377 igbvf_default_mac_addr_set(dev, &addr); 3378 } 3379 3380 static void 3381 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3382 { 3383 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3384 3385 /* Set both unicast and multicast promisc */ 3386 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3387 } 3388 3389 static void 3390 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3391 { 3392 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3393 3394 /* If in allmulticast mode leave multicast promisc */ 3395 if (dev->data->all_multicast == 1) 3396 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3397 else 3398 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3399 } 3400 3401 static void 3402 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3403 { 3404 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3405 3406 /* In promiscuous mode multicast promisc already set */ 3407 if (dev->data->promiscuous == 0) 3408 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3409 } 3410 3411 static void 3412 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3413 { 3414 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3415 3416 /* In promiscuous mode leave multicast promisc enabled */ 3417 if (dev->data->promiscuous == 0) 3418 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3419 } 3420 3421 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3422 { 3423 struct e1000_mbx_info *mbx = &hw->mbx; 3424 uint32_t msgbuf[2]; 3425 s32 err; 3426 3427 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3428 msgbuf[0] = E1000_VF_SET_VLAN; 3429 msgbuf[1] = vid; 3430 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3431 if (on) 3432 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3433 3434 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3435 if (err) 3436 goto mbx_err; 3437 3438 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3439 if (err) 3440 goto mbx_err; 3441 3442 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3443 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3444 err = -EINVAL; 3445 3446 mbx_err: 3447 return err; 3448 } 3449 3450 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3451 { 3452 struct e1000_hw *hw = 3453 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3454 struct e1000_vfta * shadow_vfta = 3455 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3456 int i = 0, j = 0, vfta = 0, mask = 1; 3457 3458 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3459 vfta = shadow_vfta->vfta[i]; 3460 if(vfta){ 3461 mask = 1; 3462 for (j = 0; j < 32; j++){ 3463 if(vfta & mask) 3464 igbvf_set_vfta(hw, 3465 (uint16_t)((i<<5)+j), on); 3466 mask<<=1; 3467 } 3468 } 3469 } 3470 3471 } 3472 3473 static int 3474 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3475 { 3476 struct e1000_hw *hw = 3477 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3478 struct e1000_vfta * shadow_vfta = 3479 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3480 uint32_t vid_idx = 0; 3481 uint32_t vid_bit = 0; 3482 int ret = 0; 3483 3484 PMD_INIT_FUNC_TRACE(); 3485 3486 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3487 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3488 if(ret){ 3489 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3490 return ret; 3491 } 3492 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3493 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3494 3495 /*Save what we set and retore it after device reset*/ 3496 if (on) 3497 shadow_vfta->vfta[vid_idx] |= vid_bit; 3498 else 3499 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3500 3501 return 0; 3502 } 3503 3504 static int 3505 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) 3506 { 3507 struct e1000_hw *hw = 3508 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3509 3510 /* index is not used by rar_set() */ 3511 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3512 return 0; 3513 } 3514 3515 3516 static int 3517 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3518 struct rte_eth_rss_reta_entry64 *reta_conf, 3519 uint16_t reta_size) 3520 { 3521 uint8_t i, j, mask; 3522 uint32_t reta, r; 3523 uint16_t idx, shift; 3524 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3525 3526 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3527 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3528 "(%d) doesn't match the number hardware can supported " 3529 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3530 return -EINVAL; 3531 } 3532 3533 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3534 idx = i / RTE_RETA_GROUP_SIZE; 3535 shift = i % RTE_RETA_GROUP_SIZE; 3536 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3537 IGB_4_BIT_MASK); 3538 if (!mask) 3539 continue; 3540 if (mask == IGB_4_BIT_MASK) 3541 r = 0; 3542 else 3543 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3544 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3545 if (mask & (0x1 << j)) 3546 reta |= reta_conf[idx].reta[shift + j] << 3547 (CHAR_BIT * j); 3548 else 3549 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3550 } 3551 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3552 } 3553 3554 return 0; 3555 } 3556 3557 static int 3558 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3559 struct rte_eth_rss_reta_entry64 *reta_conf, 3560 uint16_t reta_size) 3561 { 3562 uint8_t i, j, mask; 3563 uint32_t reta; 3564 uint16_t idx, shift; 3565 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3566 3567 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3568 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3569 "(%d) doesn't match the number hardware can supported " 3570 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3571 return -EINVAL; 3572 } 3573 3574 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3575 idx = i / RTE_RETA_GROUP_SIZE; 3576 shift = i % RTE_RETA_GROUP_SIZE; 3577 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3578 IGB_4_BIT_MASK); 3579 if (!mask) 3580 continue; 3581 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3582 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3583 if (mask & (0x1 << j)) 3584 reta_conf[idx].reta[shift + j] = 3585 ((reta >> (CHAR_BIT * j)) & 3586 IGB_8_BIT_MASK); 3587 } 3588 } 3589 3590 return 0; 3591 } 3592 3593 int 3594 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3595 struct rte_eth_syn_filter *filter, 3596 bool add) 3597 { 3598 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3599 struct e1000_filter_info *filter_info = 3600 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3601 uint32_t synqf, rfctl; 3602 3603 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3604 return -EINVAL; 3605 3606 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3607 3608 if (add) { 3609 if (synqf & E1000_SYN_FILTER_ENABLE) 3610 return -EINVAL; 3611 3612 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3613 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3614 3615 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3616 if (filter->hig_pri) 3617 rfctl |= E1000_RFCTL_SYNQFP; 3618 else 3619 rfctl &= ~E1000_RFCTL_SYNQFP; 3620 3621 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3622 } else { 3623 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3624 return -ENOENT; 3625 synqf = 0; 3626 } 3627 3628 filter_info->syn_info = synqf; 3629 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3630 E1000_WRITE_FLUSH(hw); 3631 return 0; 3632 } 3633 3634 static int 3635 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3636 struct rte_eth_syn_filter *filter) 3637 { 3638 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3639 uint32_t synqf, rfctl; 3640 3641 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3642 if (synqf & E1000_SYN_FILTER_ENABLE) { 3643 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3644 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3645 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3646 E1000_SYN_FILTER_QUEUE_SHIFT); 3647 return 0; 3648 } 3649 3650 return -ENOENT; 3651 } 3652 3653 static int 3654 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3655 enum rte_filter_op filter_op, 3656 void *arg) 3657 { 3658 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3659 int ret; 3660 3661 MAC_TYPE_FILTER_SUP(hw->mac.type); 3662 3663 if (filter_op == RTE_ETH_FILTER_NOP) 3664 return 0; 3665 3666 if (arg == NULL) { 3667 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3668 filter_op); 3669 return -EINVAL; 3670 } 3671 3672 switch (filter_op) { 3673 case RTE_ETH_FILTER_ADD: 3674 ret = eth_igb_syn_filter_set(dev, 3675 (struct rte_eth_syn_filter *)arg, 3676 TRUE); 3677 break; 3678 case RTE_ETH_FILTER_DELETE: 3679 ret = eth_igb_syn_filter_set(dev, 3680 (struct rte_eth_syn_filter *)arg, 3681 FALSE); 3682 break; 3683 case RTE_ETH_FILTER_GET: 3684 ret = eth_igb_syn_filter_get(dev, 3685 (struct rte_eth_syn_filter *)arg); 3686 break; 3687 default: 3688 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3689 ret = -EINVAL; 3690 break; 3691 } 3692 3693 return ret; 3694 } 3695 3696 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3697 static inline int 3698 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3699 struct e1000_2tuple_filter_info *filter_info) 3700 { 3701 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3702 return -EINVAL; 3703 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3704 return -EINVAL; /* filter index is out of range. */ 3705 if (filter->tcp_flags > TCP_FLAG_ALL) 3706 return -EINVAL; /* flags is invalid. */ 3707 3708 switch (filter->dst_port_mask) { 3709 case UINT16_MAX: 3710 filter_info->dst_port_mask = 0; 3711 filter_info->dst_port = filter->dst_port; 3712 break; 3713 case 0: 3714 filter_info->dst_port_mask = 1; 3715 break; 3716 default: 3717 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3718 return -EINVAL; 3719 } 3720 3721 switch (filter->proto_mask) { 3722 case UINT8_MAX: 3723 filter_info->proto_mask = 0; 3724 filter_info->proto = filter->proto; 3725 break; 3726 case 0: 3727 filter_info->proto_mask = 1; 3728 break; 3729 default: 3730 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3731 return -EINVAL; 3732 } 3733 3734 filter_info->priority = (uint8_t)filter->priority; 3735 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3736 filter_info->tcp_flags = filter->tcp_flags; 3737 else 3738 filter_info->tcp_flags = 0; 3739 3740 return 0; 3741 } 3742 3743 static inline struct e1000_2tuple_filter * 3744 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3745 struct e1000_2tuple_filter_info *key) 3746 { 3747 struct e1000_2tuple_filter *it; 3748 3749 TAILQ_FOREACH(it, filter_list, entries) { 3750 if (memcmp(key, &it->filter_info, 3751 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3752 return it; 3753 } 3754 } 3755 return NULL; 3756 } 3757 3758 /* inject a igb 2tuple filter to HW */ 3759 static inline void 3760 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3761 struct e1000_2tuple_filter *filter) 3762 { 3763 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3764 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3765 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3766 int i; 3767 3768 i = filter->index; 3769 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3770 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3771 imir |= E1000_IMIR_PORT_BP; 3772 else 3773 imir &= ~E1000_IMIR_PORT_BP; 3774 3775 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3776 3777 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3778 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3779 ttqf |= (uint32_t)(filter->filter_info.proto & 3780 E1000_TTQF_PROTOCOL_MASK); 3781 if (filter->filter_info.proto_mask == 0) 3782 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3783 3784 /* tcp flags bits setting. */ 3785 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 3786 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 3787 imir_ext |= E1000_IMIREXT_CTRL_URG; 3788 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 3789 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3790 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 3791 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3792 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 3793 imir_ext |= E1000_IMIREXT_CTRL_RST; 3794 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 3795 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3796 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 3797 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3798 } else { 3799 imir_ext |= E1000_IMIREXT_CTRL_BP; 3800 } 3801 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3802 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3803 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3804 } 3805 3806 /* 3807 * igb_add_2tuple_filter - add a 2tuple filter 3808 * 3809 * @param 3810 * dev: Pointer to struct rte_eth_dev. 3811 * ntuple_filter: ponter to the filter that will be added. 3812 * 3813 * @return 3814 * - On success, zero. 3815 * - On failure, a negative value. 3816 */ 3817 static int 3818 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3819 struct rte_eth_ntuple_filter *ntuple_filter) 3820 { 3821 struct e1000_filter_info *filter_info = 3822 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3823 struct e1000_2tuple_filter *filter; 3824 int i, ret; 3825 3826 filter = rte_zmalloc("e1000_2tuple_filter", 3827 sizeof(struct e1000_2tuple_filter), 0); 3828 if (filter == NULL) 3829 return -ENOMEM; 3830 3831 ret = ntuple_filter_to_2tuple(ntuple_filter, 3832 &filter->filter_info); 3833 if (ret < 0) { 3834 rte_free(filter); 3835 return ret; 3836 } 3837 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3838 &filter->filter_info) != NULL) { 3839 PMD_DRV_LOG(ERR, "filter exists."); 3840 rte_free(filter); 3841 return -EEXIST; 3842 } 3843 filter->queue = ntuple_filter->queue; 3844 3845 /* 3846 * look for an unused 2tuple filter index, 3847 * and insert the filter to list. 3848 */ 3849 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3850 if (!(filter_info->twotuple_mask & (1 << i))) { 3851 filter_info->twotuple_mask |= 1 << i; 3852 filter->index = i; 3853 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3854 filter, 3855 entries); 3856 break; 3857 } 3858 } 3859 if (i >= E1000_MAX_TTQF_FILTERS) { 3860 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3861 rte_free(filter); 3862 return -ENOSYS; 3863 } 3864 3865 igb_inject_2uple_filter(dev, filter); 3866 return 0; 3867 } 3868 3869 int 3870 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3871 struct e1000_2tuple_filter *filter) 3872 { 3873 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3874 struct e1000_filter_info *filter_info = 3875 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3876 3877 filter_info->twotuple_mask &= ~(1 << filter->index); 3878 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3879 rte_free(filter); 3880 3881 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3882 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3883 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3884 return 0; 3885 } 3886 3887 /* 3888 * igb_remove_2tuple_filter - remove a 2tuple filter 3889 * 3890 * @param 3891 * dev: Pointer to struct rte_eth_dev. 3892 * ntuple_filter: ponter to the filter that will be removed. 3893 * 3894 * @return 3895 * - On success, zero. 3896 * - On failure, a negative value. 3897 */ 3898 static int 3899 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3900 struct rte_eth_ntuple_filter *ntuple_filter) 3901 { 3902 struct e1000_filter_info *filter_info = 3903 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3904 struct e1000_2tuple_filter_info filter_2tuple; 3905 struct e1000_2tuple_filter *filter; 3906 int ret; 3907 3908 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3909 ret = ntuple_filter_to_2tuple(ntuple_filter, 3910 &filter_2tuple); 3911 if (ret < 0) 3912 return ret; 3913 3914 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3915 &filter_2tuple); 3916 if (filter == NULL) { 3917 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3918 return -ENOENT; 3919 } 3920 3921 igb_delete_2tuple_filter(dev, filter); 3922 3923 return 0; 3924 } 3925 3926 /* inject a igb flex filter to HW */ 3927 static inline void 3928 igb_inject_flex_filter(struct rte_eth_dev *dev, 3929 struct e1000_flex_filter *filter) 3930 { 3931 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3932 uint32_t wufc, queueing; 3933 uint32_t reg_off; 3934 uint8_t i, j = 0; 3935 3936 wufc = E1000_READ_REG(hw, E1000_WUFC); 3937 if (filter->index < E1000_MAX_FHFT) 3938 reg_off = E1000_FHFT(filter->index); 3939 else 3940 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3941 3942 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3943 (E1000_WUFC_FLX0 << filter->index)); 3944 queueing = filter->filter_info.len | 3945 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3946 (filter->filter_info.priority << 3947 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3948 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3949 queueing); 3950 3951 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3952 E1000_WRITE_REG(hw, reg_off, 3953 filter->filter_info.dwords[j]); 3954 reg_off += sizeof(uint32_t); 3955 E1000_WRITE_REG(hw, reg_off, 3956 filter->filter_info.dwords[++j]); 3957 reg_off += sizeof(uint32_t); 3958 E1000_WRITE_REG(hw, reg_off, 3959 (uint32_t)filter->filter_info.mask[i]); 3960 reg_off += sizeof(uint32_t) * 2; 3961 ++j; 3962 } 3963 } 3964 3965 static inline struct e1000_flex_filter * 3966 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3967 struct e1000_flex_filter_info *key) 3968 { 3969 struct e1000_flex_filter *it; 3970 3971 TAILQ_FOREACH(it, filter_list, entries) { 3972 if (memcmp(key, &it->filter_info, 3973 sizeof(struct e1000_flex_filter_info)) == 0) 3974 return it; 3975 } 3976 3977 return NULL; 3978 } 3979 3980 /* remove a flex byte filter 3981 * @param 3982 * dev: Pointer to struct rte_eth_dev. 3983 * filter: the pointer of the filter will be removed. 3984 */ 3985 void 3986 igb_remove_flex_filter(struct rte_eth_dev *dev, 3987 struct e1000_flex_filter *filter) 3988 { 3989 struct e1000_filter_info *filter_info = 3990 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3991 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3992 uint32_t wufc, i; 3993 uint32_t reg_off; 3994 3995 wufc = E1000_READ_REG(hw, E1000_WUFC); 3996 if (filter->index < E1000_MAX_FHFT) 3997 reg_off = E1000_FHFT(filter->index); 3998 else 3999 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4000 4001 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4002 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4003 4004 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4005 (~(E1000_WUFC_FLX0 << filter->index))); 4006 4007 filter_info->flex_mask &= ~(1 << filter->index); 4008 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4009 rte_free(filter); 4010 } 4011 4012 int 4013 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4014 struct rte_eth_flex_filter *filter, 4015 bool add) 4016 { 4017 struct e1000_filter_info *filter_info = 4018 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4019 struct e1000_flex_filter *flex_filter, *it; 4020 uint32_t mask; 4021 uint8_t shift, i; 4022 4023 flex_filter = rte_zmalloc("e1000_flex_filter", 4024 sizeof(struct e1000_flex_filter), 0); 4025 if (flex_filter == NULL) 4026 return -ENOMEM; 4027 4028 flex_filter->filter_info.len = filter->len; 4029 flex_filter->filter_info.priority = filter->priority; 4030 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4031 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4032 mask = 0; 4033 /* reverse bits in flex filter's mask*/ 4034 for (shift = 0; shift < CHAR_BIT; shift++) { 4035 if (filter->mask[i] & (0x01 << shift)) 4036 mask |= (0x80 >> shift); 4037 } 4038 flex_filter->filter_info.mask[i] = mask; 4039 } 4040 4041 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4042 &flex_filter->filter_info); 4043 if (it == NULL && !add) { 4044 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4045 rte_free(flex_filter); 4046 return -ENOENT; 4047 } 4048 if (it != NULL && add) { 4049 PMD_DRV_LOG(ERR, "filter exists."); 4050 rte_free(flex_filter); 4051 return -EEXIST; 4052 } 4053 4054 if (add) { 4055 flex_filter->queue = filter->queue; 4056 /* 4057 * look for an unused flex filter index 4058 * and insert the filter into the list. 4059 */ 4060 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4061 if (!(filter_info->flex_mask & (1 << i))) { 4062 filter_info->flex_mask |= 1 << i; 4063 flex_filter->index = i; 4064 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4065 flex_filter, 4066 entries); 4067 break; 4068 } 4069 } 4070 if (i >= E1000_MAX_FLEX_FILTERS) { 4071 PMD_DRV_LOG(ERR, "flex filters are full."); 4072 rte_free(flex_filter); 4073 return -ENOSYS; 4074 } 4075 4076 igb_inject_flex_filter(dev, flex_filter); 4077 4078 } else { 4079 igb_remove_flex_filter(dev, it); 4080 rte_free(flex_filter); 4081 } 4082 4083 return 0; 4084 } 4085 4086 static int 4087 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4088 struct rte_eth_flex_filter *filter) 4089 { 4090 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4091 struct e1000_filter_info *filter_info = 4092 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4093 struct e1000_flex_filter flex_filter, *it; 4094 uint32_t wufc, queueing, wufc_en = 0; 4095 4096 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4097 flex_filter.filter_info.len = filter->len; 4098 flex_filter.filter_info.priority = filter->priority; 4099 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4100 memcpy(flex_filter.filter_info.mask, filter->mask, 4101 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4102 4103 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4104 &flex_filter.filter_info); 4105 if (it == NULL) { 4106 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4107 return -ENOENT; 4108 } 4109 4110 wufc = E1000_READ_REG(hw, E1000_WUFC); 4111 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4112 4113 if ((wufc & wufc_en) == wufc_en) { 4114 uint32_t reg_off = 0; 4115 if (it->index < E1000_MAX_FHFT) 4116 reg_off = E1000_FHFT(it->index); 4117 else 4118 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4119 4120 queueing = E1000_READ_REG(hw, 4121 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4122 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4123 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4124 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4125 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4126 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4127 return 0; 4128 } 4129 return -ENOENT; 4130 } 4131 4132 static int 4133 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4134 enum rte_filter_op filter_op, 4135 void *arg) 4136 { 4137 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4138 struct rte_eth_flex_filter *filter; 4139 int ret = 0; 4140 4141 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4142 4143 if (filter_op == RTE_ETH_FILTER_NOP) 4144 return ret; 4145 4146 if (arg == NULL) { 4147 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4148 filter_op); 4149 return -EINVAL; 4150 } 4151 4152 filter = (struct rte_eth_flex_filter *)arg; 4153 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4154 || filter->len % sizeof(uint64_t) != 0) { 4155 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4156 return -EINVAL; 4157 } 4158 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4159 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4160 return -EINVAL; 4161 } 4162 4163 switch (filter_op) { 4164 case RTE_ETH_FILTER_ADD: 4165 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4166 break; 4167 case RTE_ETH_FILTER_DELETE: 4168 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4169 break; 4170 case RTE_ETH_FILTER_GET: 4171 ret = eth_igb_get_flex_filter(dev, filter); 4172 break; 4173 default: 4174 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4175 ret = -EINVAL; 4176 break; 4177 } 4178 4179 return ret; 4180 } 4181 4182 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4183 static inline int 4184 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4185 struct e1000_5tuple_filter_info *filter_info) 4186 { 4187 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4188 return -EINVAL; 4189 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4190 return -EINVAL; /* filter index is out of range. */ 4191 if (filter->tcp_flags > TCP_FLAG_ALL) 4192 return -EINVAL; /* flags is invalid. */ 4193 4194 switch (filter->dst_ip_mask) { 4195 case UINT32_MAX: 4196 filter_info->dst_ip_mask = 0; 4197 filter_info->dst_ip = filter->dst_ip; 4198 break; 4199 case 0: 4200 filter_info->dst_ip_mask = 1; 4201 break; 4202 default: 4203 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4204 return -EINVAL; 4205 } 4206 4207 switch (filter->src_ip_mask) { 4208 case UINT32_MAX: 4209 filter_info->src_ip_mask = 0; 4210 filter_info->src_ip = filter->src_ip; 4211 break; 4212 case 0: 4213 filter_info->src_ip_mask = 1; 4214 break; 4215 default: 4216 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4217 return -EINVAL; 4218 } 4219 4220 switch (filter->dst_port_mask) { 4221 case UINT16_MAX: 4222 filter_info->dst_port_mask = 0; 4223 filter_info->dst_port = filter->dst_port; 4224 break; 4225 case 0: 4226 filter_info->dst_port_mask = 1; 4227 break; 4228 default: 4229 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4230 return -EINVAL; 4231 } 4232 4233 switch (filter->src_port_mask) { 4234 case UINT16_MAX: 4235 filter_info->src_port_mask = 0; 4236 filter_info->src_port = filter->src_port; 4237 break; 4238 case 0: 4239 filter_info->src_port_mask = 1; 4240 break; 4241 default: 4242 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4243 return -EINVAL; 4244 } 4245 4246 switch (filter->proto_mask) { 4247 case UINT8_MAX: 4248 filter_info->proto_mask = 0; 4249 filter_info->proto = filter->proto; 4250 break; 4251 case 0: 4252 filter_info->proto_mask = 1; 4253 break; 4254 default: 4255 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4256 return -EINVAL; 4257 } 4258 4259 filter_info->priority = (uint8_t)filter->priority; 4260 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4261 filter_info->tcp_flags = filter->tcp_flags; 4262 else 4263 filter_info->tcp_flags = 0; 4264 4265 return 0; 4266 } 4267 4268 static inline struct e1000_5tuple_filter * 4269 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4270 struct e1000_5tuple_filter_info *key) 4271 { 4272 struct e1000_5tuple_filter *it; 4273 4274 TAILQ_FOREACH(it, filter_list, entries) { 4275 if (memcmp(key, &it->filter_info, 4276 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4277 return it; 4278 } 4279 } 4280 return NULL; 4281 } 4282 4283 /* inject a igb 5-tuple filter to HW */ 4284 static inline void 4285 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4286 struct e1000_5tuple_filter *filter) 4287 { 4288 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4289 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4290 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4291 uint8_t i; 4292 4293 i = filter->index; 4294 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4295 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4296 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4297 if (filter->filter_info.dst_ip_mask == 0) 4298 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4299 if (filter->filter_info.src_port_mask == 0) 4300 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4301 if (filter->filter_info.proto_mask == 0) 4302 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4303 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4304 E1000_FTQF_QUEUE_MASK; 4305 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4306 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4307 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4308 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4309 4310 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4311 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4312 4313 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4314 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4315 imir |= E1000_IMIR_PORT_BP; 4316 else 4317 imir &= ~E1000_IMIR_PORT_BP; 4318 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4319 4320 /* tcp flags bits setting. */ 4321 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 4322 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 4323 imir_ext |= E1000_IMIREXT_CTRL_URG; 4324 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 4325 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4326 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 4327 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4328 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 4329 imir_ext |= E1000_IMIREXT_CTRL_RST; 4330 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 4331 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4332 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 4333 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4334 } else { 4335 imir_ext |= E1000_IMIREXT_CTRL_BP; 4336 } 4337 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4338 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4339 } 4340 4341 /* 4342 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4343 * 4344 * @param 4345 * dev: Pointer to struct rte_eth_dev. 4346 * ntuple_filter: ponter to the filter that will be added. 4347 * 4348 * @return 4349 * - On success, zero. 4350 * - On failure, a negative value. 4351 */ 4352 static int 4353 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4354 struct rte_eth_ntuple_filter *ntuple_filter) 4355 { 4356 struct e1000_filter_info *filter_info = 4357 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4358 struct e1000_5tuple_filter *filter; 4359 uint8_t i; 4360 int ret; 4361 4362 filter = rte_zmalloc("e1000_5tuple_filter", 4363 sizeof(struct e1000_5tuple_filter), 0); 4364 if (filter == NULL) 4365 return -ENOMEM; 4366 4367 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4368 &filter->filter_info); 4369 if (ret < 0) { 4370 rte_free(filter); 4371 return ret; 4372 } 4373 4374 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4375 &filter->filter_info) != NULL) { 4376 PMD_DRV_LOG(ERR, "filter exists."); 4377 rte_free(filter); 4378 return -EEXIST; 4379 } 4380 filter->queue = ntuple_filter->queue; 4381 4382 /* 4383 * look for an unused 5tuple filter index, 4384 * and insert the filter to list. 4385 */ 4386 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4387 if (!(filter_info->fivetuple_mask & (1 << i))) { 4388 filter_info->fivetuple_mask |= 1 << i; 4389 filter->index = i; 4390 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4391 filter, 4392 entries); 4393 break; 4394 } 4395 } 4396 if (i >= E1000_MAX_FTQF_FILTERS) { 4397 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4398 rte_free(filter); 4399 return -ENOSYS; 4400 } 4401 4402 igb_inject_5tuple_filter_82576(dev, filter); 4403 return 0; 4404 } 4405 4406 int 4407 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4408 struct e1000_5tuple_filter *filter) 4409 { 4410 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4411 struct e1000_filter_info *filter_info = 4412 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4413 4414 filter_info->fivetuple_mask &= ~(1 << filter->index); 4415 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4416 rte_free(filter); 4417 4418 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4419 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4420 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4421 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4422 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4423 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4424 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4425 return 0; 4426 } 4427 4428 /* 4429 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4430 * 4431 * @param 4432 * dev: Pointer to struct rte_eth_dev. 4433 * ntuple_filter: ponter to the filter that will be removed. 4434 * 4435 * @return 4436 * - On success, zero. 4437 * - On failure, a negative value. 4438 */ 4439 static int 4440 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4441 struct rte_eth_ntuple_filter *ntuple_filter) 4442 { 4443 struct e1000_filter_info *filter_info = 4444 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4445 struct e1000_5tuple_filter_info filter_5tuple; 4446 struct e1000_5tuple_filter *filter; 4447 int ret; 4448 4449 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4450 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4451 &filter_5tuple); 4452 if (ret < 0) 4453 return ret; 4454 4455 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4456 &filter_5tuple); 4457 if (filter == NULL) { 4458 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4459 return -ENOENT; 4460 } 4461 4462 igb_delete_5tuple_filter_82576(dev, filter); 4463 4464 return 0; 4465 } 4466 4467 static int 4468 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4469 { 4470 uint32_t rctl; 4471 struct e1000_hw *hw; 4472 struct rte_eth_dev_info dev_info; 4473 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4474 4475 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4476 4477 #ifdef RTE_LIBRTE_82571_SUPPORT 4478 /* XXX: not bigger than max_rx_pktlen */ 4479 if (hw->mac.type == e1000_82571) 4480 return -ENOTSUP; 4481 #endif 4482 eth_igb_infos_get(dev, &dev_info); 4483 4484 /* check that mtu is within the allowed range */ 4485 if ((mtu < ETHER_MIN_MTU) || 4486 (frame_size > dev_info.max_rx_pktlen)) 4487 return -EINVAL; 4488 4489 /* refuse mtu that requires the support of scattered packets when this 4490 * feature has not been enabled before. */ 4491 if (!dev->data->scattered_rx && 4492 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4493 return -EINVAL; 4494 4495 rctl = E1000_READ_REG(hw, E1000_RCTL); 4496 4497 /* switch to jumbo mode if needed */ 4498 if (frame_size > ETHER_MAX_LEN) { 4499 dev->data->dev_conf.rxmode.offloads |= 4500 DEV_RX_OFFLOAD_JUMBO_FRAME; 4501 rctl |= E1000_RCTL_LPE; 4502 } else { 4503 dev->data->dev_conf.rxmode.offloads &= 4504 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4505 rctl &= ~E1000_RCTL_LPE; 4506 } 4507 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4508 4509 /* update max frame size */ 4510 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4511 4512 E1000_WRITE_REG(hw, E1000_RLPML, 4513 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4514 4515 return 0; 4516 } 4517 4518 /* 4519 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4520 * 4521 * @param 4522 * dev: Pointer to struct rte_eth_dev. 4523 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4524 * add: if true, add filter, if false, remove filter 4525 * 4526 * @return 4527 * - On success, zero. 4528 * - On failure, a negative value. 4529 */ 4530 int 4531 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4532 struct rte_eth_ntuple_filter *ntuple_filter, 4533 bool add) 4534 { 4535 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4536 int ret; 4537 4538 switch (ntuple_filter->flags) { 4539 case RTE_5TUPLE_FLAGS: 4540 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4541 if (hw->mac.type != e1000_82576) 4542 return -ENOTSUP; 4543 if (add) 4544 ret = igb_add_5tuple_filter_82576(dev, 4545 ntuple_filter); 4546 else 4547 ret = igb_remove_5tuple_filter_82576(dev, 4548 ntuple_filter); 4549 break; 4550 case RTE_2TUPLE_FLAGS: 4551 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4552 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4553 hw->mac.type != e1000_i210 && 4554 hw->mac.type != e1000_i211) 4555 return -ENOTSUP; 4556 if (add) 4557 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4558 else 4559 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4560 break; 4561 default: 4562 ret = -EINVAL; 4563 break; 4564 } 4565 4566 return ret; 4567 } 4568 4569 /* 4570 * igb_get_ntuple_filter - get a ntuple filter 4571 * 4572 * @param 4573 * dev: Pointer to struct rte_eth_dev. 4574 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4575 * 4576 * @return 4577 * - On success, zero. 4578 * - On failure, a negative value. 4579 */ 4580 static int 4581 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4582 struct rte_eth_ntuple_filter *ntuple_filter) 4583 { 4584 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4585 struct e1000_filter_info *filter_info = 4586 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4587 struct e1000_5tuple_filter_info filter_5tuple; 4588 struct e1000_2tuple_filter_info filter_2tuple; 4589 struct e1000_5tuple_filter *p_5tuple_filter; 4590 struct e1000_2tuple_filter *p_2tuple_filter; 4591 int ret; 4592 4593 switch (ntuple_filter->flags) { 4594 case RTE_5TUPLE_FLAGS: 4595 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4596 if (hw->mac.type != e1000_82576) 4597 return -ENOTSUP; 4598 memset(&filter_5tuple, 4599 0, 4600 sizeof(struct e1000_5tuple_filter_info)); 4601 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4602 &filter_5tuple); 4603 if (ret < 0) 4604 return ret; 4605 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4606 &filter_info->fivetuple_list, 4607 &filter_5tuple); 4608 if (p_5tuple_filter == NULL) { 4609 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4610 return -ENOENT; 4611 } 4612 ntuple_filter->queue = p_5tuple_filter->queue; 4613 break; 4614 case RTE_2TUPLE_FLAGS: 4615 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4616 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4617 return -ENOTSUP; 4618 memset(&filter_2tuple, 4619 0, 4620 sizeof(struct e1000_2tuple_filter_info)); 4621 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4622 if (ret < 0) 4623 return ret; 4624 p_2tuple_filter = igb_2tuple_filter_lookup( 4625 &filter_info->twotuple_list, 4626 &filter_2tuple); 4627 if (p_2tuple_filter == NULL) { 4628 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4629 return -ENOENT; 4630 } 4631 ntuple_filter->queue = p_2tuple_filter->queue; 4632 break; 4633 default: 4634 ret = -EINVAL; 4635 break; 4636 } 4637 4638 return 0; 4639 } 4640 4641 /* 4642 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4643 * @dev: pointer to rte_eth_dev structure 4644 * @filter_op:operation will be taken. 4645 * @arg: a pointer to specific structure corresponding to the filter_op 4646 */ 4647 static int 4648 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4649 enum rte_filter_op filter_op, 4650 void *arg) 4651 { 4652 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4653 int ret; 4654 4655 MAC_TYPE_FILTER_SUP(hw->mac.type); 4656 4657 if (filter_op == RTE_ETH_FILTER_NOP) 4658 return 0; 4659 4660 if (arg == NULL) { 4661 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4662 filter_op); 4663 return -EINVAL; 4664 } 4665 4666 switch (filter_op) { 4667 case RTE_ETH_FILTER_ADD: 4668 ret = igb_add_del_ntuple_filter(dev, 4669 (struct rte_eth_ntuple_filter *)arg, 4670 TRUE); 4671 break; 4672 case RTE_ETH_FILTER_DELETE: 4673 ret = igb_add_del_ntuple_filter(dev, 4674 (struct rte_eth_ntuple_filter *)arg, 4675 FALSE); 4676 break; 4677 case RTE_ETH_FILTER_GET: 4678 ret = igb_get_ntuple_filter(dev, 4679 (struct rte_eth_ntuple_filter *)arg); 4680 break; 4681 default: 4682 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4683 ret = -EINVAL; 4684 break; 4685 } 4686 return ret; 4687 } 4688 4689 static inline int 4690 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4691 uint16_t ethertype) 4692 { 4693 int i; 4694 4695 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4696 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4697 (filter_info->ethertype_mask & (1 << i))) 4698 return i; 4699 } 4700 return -1; 4701 } 4702 4703 static inline int 4704 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4705 uint16_t ethertype, uint32_t etqf) 4706 { 4707 int i; 4708 4709 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4710 if (!(filter_info->ethertype_mask & (1 << i))) { 4711 filter_info->ethertype_mask |= 1 << i; 4712 filter_info->ethertype_filters[i].ethertype = ethertype; 4713 filter_info->ethertype_filters[i].etqf = etqf; 4714 return i; 4715 } 4716 } 4717 return -1; 4718 } 4719 4720 int 4721 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4722 uint8_t idx) 4723 { 4724 if (idx >= E1000_MAX_ETQF_FILTERS) 4725 return -1; 4726 filter_info->ethertype_mask &= ~(1 << idx); 4727 filter_info->ethertype_filters[idx].ethertype = 0; 4728 filter_info->ethertype_filters[idx].etqf = 0; 4729 return idx; 4730 } 4731 4732 4733 int 4734 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4735 struct rte_eth_ethertype_filter *filter, 4736 bool add) 4737 { 4738 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4739 struct e1000_filter_info *filter_info = 4740 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4741 uint32_t etqf = 0; 4742 int ret; 4743 4744 if (filter->ether_type == ETHER_TYPE_IPv4 || 4745 filter->ether_type == ETHER_TYPE_IPv6) { 4746 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4747 " ethertype filter.", filter->ether_type); 4748 return -EINVAL; 4749 } 4750 4751 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4752 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4753 return -EINVAL; 4754 } 4755 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4756 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4757 return -EINVAL; 4758 } 4759 4760 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4761 if (ret >= 0 && add) { 4762 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4763 filter->ether_type); 4764 return -EEXIST; 4765 } 4766 if (ret < 0 && !add) { 4767 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4768 filter->ether_type); 4769 return -ENOENT; 4770 } 4771 4772 if (add) { 4773 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4774 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4775 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4776 ret = igb_ethertype_filter_insert(filter_info, 4777 filter->ether_type, etqf); 4778 if (ret < 0) { 4779 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4780 return -ENOSYS; 4781 } 4782 } else { 4783 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4784 if (ret < 0) 4785 return -ENOSYS; 4786 } 4787 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4788 E1000_WRITE_FLUSH(hw); 4789 4790 return 0; 4791 } 4792 4793 static int 4794 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4795 struct rte_eth_ethertype_filter *filter) 4796 { 4797 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4798 struct e1000_filter_info *filter_info = 4799 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4800 uint32_t etqf; 4801 int ret; 4802 4803 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4804 if (ret < 0) { 4805 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4806 filter->ether_type); 4807 return -ENOENT; 4808 } 4809 4810 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4811 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4812 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4813 filter->flags = 0; 4814 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4815 E1000_ETQF_QUEUE_SHIFT; 4816 return 0; 4817 } 4818 4819 return -ENOENT; 4820 } 4821 4822 /* 4823 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4824 * @dev: pointer to rte_eth_dev structure 4825 * @filter_op:operation will be taken. 4826 * @arg: a pointer to specific structure corresponding to the filter_op 4827 */ 4828 static int 4829 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4830 enum rte_filter_op filter_op, 4831 void *arg) 4832 { 4833 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4834 int ret; 4835 4836 MAC_TYPE_FILTER_SUP(hw->mac.type); 4837 4838 if (filter_op == RTE_ETH_FILTER_NOP) 4839 return 0; 4840 4841 if (arg == NULL) { 4842 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4843 filter_op); 4844 return -EINVAL; 4845 } 4846 4847 switch (filter_op) { 4848 case RTE_ETH_FILTER_ADD: 4849 ret = igb_add_del_ethertype_filter(dev, 4850 (struct rte_eth_ethertype_filter *)arg, 4851 TRUE); 4852 break; 4853 case RTE_ETH_FILTER_DELETE: 4854 ret = igb_add_del_ethertype_filter(dev, 4855 (struct rte_eth_ethertype_filter *)arg, 4856 FALSE); 4857 break; 4858 case RTE_ETH_FILTER_GET: 4859 ret = igb_get_ethertype_filter(dev, 4860 (struct rte_eth_ethertype_filter *)arg); 4861 break; 4862 default: 4863 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4864 ret = -EINVAL; 4865 break; 4866 } 4867 return ret; 4868 } 4869 4870 static int 4871 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4872 enum rte_filter_type filter_type, 4873 enum rte_filter_op filter_op, 4874 void *arg) 4875 { 4876 int ret = 0; 4877 4878 switch (filter_type) { 4879 case RTE_ETH_FILTER_NTUPLE: 4880 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4881 break; 4882 case RTE_ETH_FILTER_ETHERTYPE: 4883 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4884 break; 4885 case RTE_ETH_FILTER_SYN: 4886 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4887 break; 4888 case RTE_ETH_FILTER_FLEXIBLE: 4889 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4890 break; 4891 case RTE_ETH_FILTER_GENERIC: 4892 if (filter_op != RTE_ETH_FILTER_GET) 4893 return -EINVAL; 4894 *(const void **)arg = &igb_flow_ops; 4895 break; 4896 default: 4897 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4898 filter_type); 4899 break; 4900 } 4901 4902 return ret; 4903 } 4904 4905 static int 4906 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4907 struct ether_addr *mc_addr_set, 4908 uint32_t nb_mc_addr) 4909 { 4910 struct e1000_hw *hw; 4911 4912 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4913 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4914 return 0; 4915 } 4916 4917 static uint64_t 4918 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4919 { 4920 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4921 uint64_t systime_cycles; 4922 4923 switch (hw->mac.type) { 4924 case e1000_i210: 4925 case e1000_i211: 4926 /* 4927 * Need to read System Time Residue Register to be able 4928 * to read the other two registers. 4929 */ 4930 E1000_READ_REG(hw, E1000_SYSTIMR); 4931 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4932 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4933 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4934 * NSEC_PER_SEC; 4935 break; 4936 case e1000_82580: 4937 case e1000_i350: 4938 case e1000_i354: 4939 /* 4940 * Need to read System Time Residue Register to be able 4941 * to read the other two registers. 4942 */ 4943 E1000_READ_REG(hw, E1000_SYSTIMR); 4944 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4945 /* Only the 8 LSB are valid. */ 4946 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4947 & 0xff) << 32; 4948 break; 4949 default: 4950 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4951 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4952 << 32; 4953 break; 4954 } 4955 4956 return systime_cycles; 4957 } 4958 4959 static uint64_t 4960 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4961 { 4962 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4963 uint64_t rx_tstamp_cycles; 4964 4965 switch (hw->mac.type) { 4966 case e1000_i210: 4967 case e1000_i211: 4968 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4969 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4970 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4971 * NSEC_PER_SEC; 4972 break; 4973 case e1000_82580: 4974 case e1000_i350: 4975 case e1000_i354: 4976 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4977 /* Only the 8 LSB are valid. */ 4978 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4979 & 0xff) << 32; 4980 break; 4981 default: 4982 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4983 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4984 << 32; 4985 break; 4986 } 4987 4988 return rx_tstamp_cycles; 4989 } 4990 4991 static uint64_t 4992 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4993 { 4994 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4995 uint64_t tx_tstamp_cycles; 4996 4997 switch (hw->mac.type) { 4998 case e1000_i210: 4999 case e1000_i211: 5000 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5001 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5002 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5003 * NSEC_PER_SEC; 5004 break; 5005 case e1000_82580: 5006 case e1000_i350: 5007 case e1000_i354: 5008 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5009 /* Only the 8 LSB are valid. */ 5010 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5011 & 0xff) << 32; 5012 break; 5013 default: 5014 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5015 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5016 << 32; 5017 break; 5018 } 5019 5020 return tx_tstamp_cycles; 5021 } 5022 5023 static void 5024 igb_start_timecounters(struct rte_eth_dev *dev) 5025 { 5026 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5027 struct e1000_adapter *adapter = 5028 (struct e1000_adapter *)dev->data->dev_private; 5029 uint32_t incval = 1; 5030 uint32_t shift = 0; 5031 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5032 5033 switch (hw->mac.type) { 5034 case e1000_82580: 5035 case e1000_i350: 5036 case e1000_i354: 5037 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5038 mask = (1ULL << 40) - 1; 5039 /* fall-through */ 5040 case e1000_i210: 5041 case e1000_i211: 5042 /* 5043 * Start incrementing the register 5044 * used to timestamp PTP packets. 5045 */ 5046 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5047 break; 5048 case e1000_82576: 5049 incval = E1000_INCVALUE_82576; 5050 shift = IGB_82576_TSYNC_SHIFT; 5051 E1000_WRITE_REG(hw, E1000_TIMINCA, 5052 E1000_INCPERIOD_82576 | incval); 5053 break; 5054 default: 5055 /* Not supported */ 5056 return; 5057 } 5058 5059 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5060 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5061 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5062 5063 adapter->systime_tc.cc_mask = mask; 5064 adapter->systime_tc.cc_shift = shift; 5065 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5066 5067 adapter->rx_tstamp_tc.cc_mask = mask; 5068 adapter->rx_tstamp_tc.cc_shift = shift; 5069 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5070 5071 adapter->tx_tstamp_tc.cc_mask = mask; 5072 adapter->tx_tstamp_tc.cc_shift = shift; 5073 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5074 } 5075 5076 static int 5077 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5078 { 5079 struct e1000_adapter *adapter = 5080 (struct e1000_adapter *)dev->data->dev_private; 5081 5082 adapter->systime_tc.nsec += delta; 5083 adapter->rx_tstamp_tc.nsec += delta; 5084 adapter->tx_tstamp_tc.nsec += delta; 5085 5086 return 0; 5087 } 5088 5089 static int 5090 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5091 { 5092 uint64_t ns; 5093 struct e1000_adapter *adapter = 5094 (struct e1000_adapter *)dev->data->dev_private; 5095 5096 ns = rte_timespec_to_ns(ts); 5097 5098 /* Set the timecounters to a new value. */ 5099 adapter->systime_tc.nsec = ns; 5100 adapter->rx_tstamp_tc.nsec = ns; 5101 adapter->tx_tstamp_tc.nsec = ns; 5102 5103 return 0; 5104 } 5105 5106 static int 5107 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5108 { 5109 uint64_t ns, systime_cycles; 5110 struct e1000_adapter *adapter = 5111 (struct e1000_adapter *)dev->data->dev_private; 5112 5113 systime_cycles = igb_read_systime_cyclecounter(dev); 5114 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5115 *ts = rte_ns_to_timespec(ns); 5116 5117 return 0; 5118 } 5119 5120 static int 5121 igb_timesync_enable(struct rte_eth_dev *dev) 5122 { 5123 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5124 uint32_t tsync_ctl; 5125 uint32_t tsauxc; 5126 5127 /* Stop the timesync system time. */ 5128 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5129 /* Reset the timesync system time value. */ 5130 switch (hw->mac.type) { 5131 case e1000_82580: 5132 case e1000_i350: 5133 case e1000_i354: 5134 case e1000_i210: 5135 case e1000_i211: 5136 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5137 /* fall-through */ 5138 case e1000_82576: 5139 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5140 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5141 break; 5142 default: 5143 /* Not supported. */ 5144 return -ENOTSUP; 5145 } 5146 5147 /* Enable system time for it isn't on by default. */ 5148 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5149 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5150 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5151 5152 igb_start_timecounters(dev); 5153 5154 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5155 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5156 (ETHER_TYPE_1588 | 5157 E1000_ETQF_FILTER_ENABLE | 5158 E1000_ETQF_1588)); 5159 5160 /* Enable timestamping of received PTP packets. */ 5161 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5162 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5163 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5164 5165 /* Enable Timestamping of transmitted PTP packets. */ 5166 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5167 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5168 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5169 5170 return 0; 5171 } 5172 5173 static int 5174 igb_timesync_disable(struct rte_eth_dev *dev) 5175 { 5176 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5177 uint32_t tsync_ctl; 5178 5179 /* Disable timestamping of transmitted PTP packets. */ 5180 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5181 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5182 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5183 5184 /* Disable timestamping of received PTP packets. */ 5185 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5186 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5187 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5188 5189 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5190 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5191 5192 /* Stop incrementating the System Time registers. */ 5193 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5194 5195 return 0; 5196 } 5197 5198 static int 5199 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5200 struct timespec *timestamp, 5201 uint32_t flags __rte_unused) 5202 { 5203 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5204 struct e1000_adapter *adapter = 5205 (struct e1000_adapter *)dev->data->dev_private; 5206 uint32_t tsync_rxctl; 5207 uint64_t rx_tstamp_cycles; 5208 uint64_t ns; 5209 5210 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5211 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5212 return -EINVAL; 5213 5214 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5215 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5216 *timestamp = rte_ns_to_timespec(ns); 5217 5218 return 0; 5219 } 5220 5221 static int 5222 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5223 struct timespec *timestamp) 5224 { 5225 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5226 struct e1000_adapter *adapter = 5227 (struct e1000_adapter *)dev->data->dev_private; 5228 uint32_t tsync_txctl; 5229 uint64_t tx_tstamp_cycles; 5230 uint64_t ns; 5231 5232 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5233 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5234 return -EINVAL; 5235 5236 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5237 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5238 *timestamp = rte_ns_to_timespec(ns); 5239 5240 return 0; 5241 } 5242 5243 static int 5244 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5245 { 5246 int count = 0; 5247 int g_ind = 0; 5248 const struct reg_info *reg_group; 5249 5250 while ((reg_group = igb_regs[g_ind++])) 5251 count += igb_reg_group_count(reg_group); 5252 5253 return count; 5254 } 5255 5256 static int 5257 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5258 { 5259 int count = 0; 5260 int g_ind = 0; 5261 const struct reg_info *reg_group; 5262 5263 while ((reg_group = igbvf_regs[g_ind++])) 5264 count += igb_reg_group_count(reg_group); 5265 5266 return count; 5267 } 5268 5269 static int 5270 eth_igb_get_regs(struct rte_eth_dev *dev, 5271 struct rte_dev_reg_info *regs) 5272 { 5273 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5274 uint32_t *data = regs->data; 5275 int g_ind = 0; 5276 int count = 0; 5277 const struct reg_info *reg_group; 5278 5279 if (data == NULL) { 5280 regs->length = eth_igb_get_reg_length(dev); 5281 regs->width = sizeof(uint32_t); 5282 return 0; 5283 } 5284 5285 /* Support only full register dump */ 5286 if ((regs->length == 0) || 5287 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5288 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5289 hw->device_id; 5290 while ((reg_group = igb_regs[g_ind++])) 5291 count += igb_read_regs_group(dev, &data[count], 5292 reg_group); 5293 return 0; 5294 } 5295 5296 return -ENOTSUP; 5297 } 5298 5299 static int 5300 igbvf_get_regs(struct rte_eth_dev *dev, 5301 struct rte_dev_reg_info *regs) 5302 { 5303 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5304 uint32_t *data = regs->data; 5305 int g_ind = 0; 5306 int count = 0; 5307 const struct reg_info *reg_group; 5308 5309 if (data == NULL) { 5310 regs->length = igbvf_get_reg_length(dev); 5311 regs->width = sizeof(uint32_t); 5312 return 0; 5313 } 5314 5315 /* Support only full register dump */ 5316 if ((regs->length == 0) || 5317 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5318 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5319 hw->device_id; 5320 while ((reg_group = igbvf_regs[g_ind++])) 5321 count += igb_read_regs_group(dev, &data[count], 5322 reg_group); 5323 return 0; 5324 } 5325 5326 return -ENOTSUP; 5327 } 5328 5329 static int 5330 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5331 { 5332 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5333 5334 /* Return unit is byte count */ 5335 return hw->nvm.word_size * 2; 5336 } 5337 5338 static int 5339 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5340 struct rte_dev_eeprom_info *in_eeprom) 5341 { 5342 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5343 struct e1000_nvm_info *nvm = &hw->nvm; 5344 uint16_t *data = in_eeprom->data; 5345 int first, length; 5346 5347 first = in_eeprom->offset >> 1; 5348 length = in_eeprom->length >> 1; 5349 if ((first >= hw->nvm.word_size) || 5350 ((first + length) >= hw->nvm.word_size)) 5351 return -EINVAL; 5352 5353 in_eeprom->magic = hw->vendor_id | 5354 ((uint32_t)hw->device_id << 16); 5355 5356 if ((nvm->ops.read) == NULL) 5357 return -ENOTSUP; 5358 5359 return nvm->ops.read(hw, first, length, data); 5360 } 5361 5362 static int 5363 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5364 struct rte_dev_eeprom_info *in_eeprom) 5365 { 5366 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5367 struct e1000_nvm_info *nvm = &hw->nvm; 5368 uint16_t *data = in_eeprom->data; 5369 int first, length; 5370 5371 first = in_eeprom->offset >> 1; 5372 length = in_eeprom->length >> 1; 5373 if ((first >= hw->nvm.word_size) || 5374 ((first + length) >= hw->nvm.word_size)) 5375 return -EINVAL; 5376 5377 in_eeprom->magic = (uint32_t)hw->vendor_id | 5378 ((uint32_t)hw->device_id << 16); 5379 5380 if ((nvm->ops.write) == NULL) 5381 return -ENOTSUP; 5382 return nvm->ops.write(hw, first, length, data); 5383 } 5384 5385 static int 5386 eth_igb_get_module_info(struct rte_eth_dev *dev, 5387 struct rte_eth_dev_module_info *modinfo) 5388 { 5389 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5390 5391 uint32_t status = 0; 5392 uint16_t sff8472_rev, addr_mode; 5393 bool page_swap = false; 5394 5395 if (hw->phy.media_type == e1000_media_type_copper || 5396 hw->phy.media_type == e1000_media_type_unknown) 5397 return -EOPNOTSUPP; 5398 5399 /* Check whether we support SFF-8472 or not */ 5400 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5401 if (status) 5402 return -EIO; 5403 5404 /* addressing mode is not supported */ 5405 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5406 if (status) 5407 return -EIO; 5408 5409 /* addressing mode is not supported */ 5410 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5411 PMD_DRV_LOG(ERR, 5412 "Address change required to access page 0xA2, " 5413 "but not supported. Please report the module " 5414 "type to the driver maintainers.\n"); 5415 page_swap = true; 5416 } 5417 5418 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5419 /* We have an SFP, but it does not support SFF-8472 */ 5420 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5421 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5422 } else { 5423 /* We have an SFP which supports a revision of SFF-8472 */ 5424 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5425 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5426 } 5427 5428 return 0; 5429 } 5430 5431 static int 5432 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5433 struct rte_dev_eeprom_info *info) 5434 { 5435 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5436 5437 uint32_t status = 0; 5438 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5439 u16 first_word, last_word; 5440 int i = 0; 5441 5442 if (info->length == 0) 5443 return -EINVAL; 5444 5445 first_word = info->offset >> 1; 5446 last_word = (info->offset + info->length - 1) >> 1; 5447 5448 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5449 for (i = 0; i < last_word - first_word + 1; i++) { 5450 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5451 &dataword[i]); 5452 if (status) { 5453 /* Error occurred while reading module */ 5454 return -EIO; 5455 } 5456 5457 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5458 } 5459 5460 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5461 5462 return 0; 5463 } 5464 5465 static int 5466 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5467 { 5468 struct e1000_hw *hw = 5469 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5470 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5471 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5472 uint32_t vec = E1000_MISC_VEC_ID; 5473 5474 if (rte_intr_allow_others(intr_handle)) 5475 vec = E1000_RX_VEC_START; 5476 5477 uint32_t mask = 1 << (queue_id + vec); 5478 5479 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5480 E1000_WRITE_FLUSH(hw); 5481 5482 return 0; 5483 } 5484 5485 static int 5486 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5487 { 5488 struct e1000_hw *hw = 5489 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5490 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5491 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5492 uint32_t vec = E1000_MISC_VEC_ID; 5493 5494 if (rte_intr_allow_others(intr_handle)) 5495 vec = E1000_RX_VEC_START; 5496 5497 uint32_t mask = 1 << (queue_id + vec); 5498 uint32_t regval; 5499 5500 regval = E1000_READ_REG(hw, E1000_EIMS); 5501 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5502 E1000_WRITE_FLUSH(hw); 5503 5504 rte_intr_enable(intr_handle); 5505 5506 return 0; 5507 } 5508 5509 static void 5510 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5511 uint8_t index, uint8_t offset) 5512 { 5513 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5514 5515 /* clear bits */ 5516 val &= ~((uint32_t)0xFF << offset); 5517 5518 /* write vector and valid bit */ 5519 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5520 5521 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5522 } 5523 5524 static void 5525 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5526 uint8_t queue, uint8_t msix_vector) 5527 { 5528 uint32_t tmp = 0; 5529 5530 if (hw->mac.type == e1000_82575) { 5531 if (direction == 0) 5532 tmp = E1000_EICR_RX_QUEUE0 << queue; 5533 else if (direction == 1) 5534 tmp = E1000_EICR_TX_QUEUE0 << queue; 5535 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5536 } else if (hw->mac.type == e1000_82576) { 5537 if ((direction == 0) || (direction == 1)) 5538 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5539 ((queue & 0x8) << 1) + 5540 8 * direction); 5541 } else if ((hw->mac.type == e1000_82580) || 5542 (hw->mac.type == e1000_i350) || 5543 (hw->mac.type == e1000_i354) || 5544 (hw->mac.type == e1000_i210) || 5545 (hw->mac.type == e1000_i211)) { 5546 if ((direction == 0) || (direction == 1)) 5547 eth_igb_write_ivar(hw, msix_vector, 5548 queue >> 1, 5549 ((queue & 0x1) << 4) + 5550 8 * direction); 5551 } 5552 } 5553 5554 /* Sets up the hardware to generate MSI-X interrupts properly 5555 * @hw 5556 * board private structure 5557 */ 5558 static void 5559 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5560 { 5561 int queue_id; 5562 uint32_t tmpval, regval, intr_mask; 5563 struct e1000_hw *hw = 5564 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5565 uint32_t vec = E1000_MISC_VEC_ID; 5566 uint32_t base = E1000_MISC_VEC_ID; 5567 uint32_t misc_shift = 0; 5568 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5569 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5570 5571 /* won't configure msix register if no mapping is done 5572 * between intr vector and event fd 5573 */ 5574 if (!rte_intr_dp_is_en(intr_handle)) 5575 return; 5576 5577 if (rte_intr_allow_others(intr_handle)) { 5578 vec = base = E1000_RX_VEC_START; 5579 misc_shift = 1; 5580 } 5581 5582 /* set interrupt vector for other causes */ 5583 if (hw->mac.type == e1000_82575) { 5584 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5585 /* enable MSI-X PBA support */ 5586 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5587 5588 /* Auto-Mask interrupts upon ICR read */ 5589 tmpval |= E1000_CTRL_EXT_EIAME; 5590 tmpval |= E1000_CTRL_EXT_IRCA; 5591 5592 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5593 5594 /* enable msix_other interrupt */ 5595 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5596 regval = E1000_READ_REG(hw, E1000_EIAC); 5597 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5598 regval = E1000_READ_REG(hw, E1000_EIAM); 5599 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5600 } else if ((hw->mac.type == e1000_82576) || 5601 (hw->mac.type == e1000_82580) || 5602 (hw->mac.type == e1000_i350) || 5603 (hw->mac.type == e1000_i354) || 5604 (hw->mac.type == e1000_i210) || 5605 (hw->mac.type == e1000_i211)) { 5606 /* turn on MSI-X capability first */ 5607 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5608 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5609 E1000_GPIE_NSICR); 5610 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5611 misc_shift; 5612 5613 if (dev->data->dev_conf.intr_conf.lsc != 0) 5614 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5615 5616 regval = E1000_READ_REG(hw, E1000_EIAC); 5617 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5618 5619 /* enable msix_other interrupt */ 5620 regval = E1000_READ_REG(hw, E1000_EIMS); 5621 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5622 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5623 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5624 } 5625 5626 /* use EIAM to auto-mask when MSI-X interrupt 5627 * is asserted, this saves a register write for every interrupt 5628 */ 5629 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5630 misc_shift; 5631 5632 if (dev->data->dev_conf.intr_conf.lsc != 0) 5633 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5634 5635 regval = E1000_READ_REG(hw, E1000_EIAM); 5636 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5637 5638 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5639 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5640 intr_handle->intr_vec[queue_id] = vec; 5641 if (vec < base + intr_handle->nb_efd - 1) 5642 vec++; 5643 } 5644 5645 E1000_WRITE_FLUSH(hw); 5646 } 5647 5648 /* restore n-tuple filter */ 5649 static inline void 5650 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5651 { 5652 struct e1000_filter_info *filter_info = 5653 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5654 struct e1000_5tuple_filter *p_5tuple; 5655 struct e1000_2tuple_filter *p_2tuple; 5656 5657 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5658 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5659 } 5660 5661 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5662 igb_inject_2uple_filter(dev, p_2tuple); 5663 } 5664 } 5665 5666 /* restore SYN filter */ 5667 static inline void 5668 igb_syn_filter_restore(struct rte_eth_dev *dev) 5669 { 5670 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5671 struct e1000_filter_info *filter_info = 5672 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5673 uint32_t synqf; 5674 5675 synqf = filter_info->syn_info; 5676 5677 if (synqf & E1000_SYN_FILTER_ENABLE) { 5678 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5679 E1000_WRITE_FLUSH(hw); 5680 } 5681 } 5682 5683 /* restore ethernet type filter */ 5684 static inline void 5685 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5686 { 5687 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5688 struct e1000_filter_info *filter_info = 5689 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5690 int i; 5691 5692 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5693 if (filter_info->ethertype_mask & (1 << i)) { 5694 E1000_WRITE_REG(hw, E1000_ETQF(i), 5695 filter_info->ethertype_filters[i].etqf); 5696 E1000_WRITE_FLUSH(hw); 5697 } 5698 } 5699 } 5700 5701 /* restore flex byte filter */ 5702 static inline void 5703 igb_flex_filter_restore(struct rte_eth_dev *dev) 5704 { 5705 struct e1000_filter_info *filter_info = 5706 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5707 struct e1000_flex_filter *flex_filter; 5708 5709 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5710 igb_inject_flex_filter(dev, flex_filter); 5711 } 5712 } 5713 5714 /* restore rss filter */ 5715 static inline void 5716 igb_rss_filter_restore(struct rte_eth_dev *dev) 5717 { 5718 struct e1000_filter_info *filter_info = 5719 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5720 5721 if (filter_info->rss_info.conf.queue_num) 5722 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5723 } 5724 5725 /* restore all types filter */ 5726 static int 5727 igb_filter_restore(struct rte_eth_dev *dev) 5728 { 5729 igb_ntuple_filter_restore(dev); 5730 igb_ethertype_filter_restore(dev); 5731 igb_syn_filter_restore(dev); 5732 igb_flex_filter_restore(dev); 5733 igb_rss_filter_restore(dev); 5734 5735 return 0; 5736 } 5737 5738 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5739 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5740 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5741 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5742 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5743 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5744 5745 /* see e1000_logs.c */ 5746 RTE_INIT(e1000_init_log) 5747 { 5748 e1000_igb_init_log(); 5749 } 5750