1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static void eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static void eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 100 unsigned int limit); 101 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static void eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static void igbvf_dev_stop(struct rte_eth_dev *dev); 158 static void igbvf_dev_close(struct rte_eth_dev *dev); 159 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 190 struct rte_eth_syn_filter *filter); 191 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 192 enum rte_filter_op filter_op, 193 void *arg); 194 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 195 struct rte_eth_ntuple_filter *ntuple_filter); 196 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 197 struct rte_eth_ntuple_filter *ntuple_filter); 198 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 199 struct rte_eth_flex_filter *filter); 200 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 201 enum rte_filter_op filter_op, 202 void *arg); 203 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *ntuple_filter); 205 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 206 struct rte_eth_ntuple_filter *ntuple_filter); 207 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 208 struct rte_eth_ntuple_filter *filter); 209 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 210 enum rte_filter_op filter_op, 211 void *arg); 212 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 213 enum rte_filter_op filter_op, 214 void *arg); 215 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 216 struct rte_eth_ethertype_filter *filter); 217 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 218 enum rte_filter_type filter_type, 219 enum rte_filter_op filter_op, 220 void *arg); 221 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 222 static int eth_igb_get_regs(struct rte_eth_dev *dev, 223 struct rte_dev_reg_info *regs); 224 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 225 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 226 struct rte_dev_eeprom_info *eeprom); 227 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 228 struct rte_dev_eeprom_info *eeprom); 229 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 230 struct rte_eth_dev_module_info *modinfo); 231 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 232 struct rte_dev_eeprom_info *info); 233 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 234 struct ether_addr *mc_addr_set, 235 uint32_t nb_mc_addr); 236 static int igb_timesync_enable(struct rte_eth_dev *dev); 237 static int igb_timesync_disable(struct rte_eth_dev *dev); 238 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 239 struct timespec *timestamp, 240 uint32_t flags); 241 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 242 struct timespec *timestamp); 243 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 244 static int igb_timesync_read_time(struct rte_eth_dev *dev, 245 struct timespec *timestamp); 246 static int igb_timesync_write_time(struct rte_eth_dev *dev, 247 const struct timespec *timestamp); 248 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 249 uint16_t queue_id); 250 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 251 uint16_t queue_id); 252 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 253 uint8_t queue, uint8_t msix_vector); 254 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 255 uint8_t index, uint8_t offset); 256 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 257 static void eth_igbvf_interrupt_handler(void *param); 258 static void igbvf_mbx_process(struct rte_eth_dev *dev); 259 static int igb_filter_restore(struct rte_eth_dev *dev); 260 261 /* 262 * Define VF Stats MACRO for Non "cleared on read" register 263 */ 264 #define UPDATE_VF_STAT(reg, last, cur) \ 265 { \ 266 u32 latest = E1000_READ_REG(hw, reg); \ 267 cur += (latest - last) & UINT_MAX; \ 268 last = latest; \ 269 } 270 271 #define IGB_FC_PAUSE_TIME 0x0680 272 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 273 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 274 275 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 276 277 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 278 279 /* 280 * The set of PCI devices this driver supports 281 */ 282 static const struct rte_pci_id pci_id_igb_map[] = { 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 291 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 295 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 302 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 320 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 321 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 322 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 323 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 324 { .vendor_id = 0, /* sentinel */ }, 325 }; 326 327 /* 328 * The set of PCI devices this driver supports (for 82576&I350 VF) 329 */ 330 static const struct rte_pci_id pci_id_igbvf_map[] = { 331 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 332 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 333 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 334 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 335 { .vendor_id = 0, /* sentinel */ }, 336 }; 337 338 static const struct rte_eth_desc_lim rx_desc_lim = { 339 .nb_max = E1000_MAX_RING_DESC, 340 .nb_min = E1000_MIN_RING_DESC, 341 .nb_align = IGB_RXD_ALIGN, 342 }; 343 344 static const struct rte_eth_desc_lim tx_desc_lim = { 345 .nb_max = E1000_MAX_RING_DESC, 346 .nb_min = E1000_MIN_RING_DESC, 347 .nb_align = IGB_RXD_ALIGN, 348 .nb_seg_max = IGB_TX_MAX_SEG, 349 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 350 }; 351 352 static const struct eth_dev_ops eth_igb_ops = { 353 .dev_configure = eth_igb_configure, 354 .dev_start = eth_igb_start, 355 .dev_stop = eth_igb_stop, 356 .dev_set_link_up = eth_igb_dev_set_link_up, 357 .dev_set_link_down = eth_igb_dev_set_link_down, 358 .dev_close = eth_igb_close, 359 .dev_reset = eth_igb_reset, 360 .promiscuous_enable = eth_igb_promiscuous_enable, 361 .promiscuous_disable = eth_igb_promiscuous_disable, 362 .allmulticast_enable = eth_igb_allmulticast_enable, 363 .allmulticast_disable = eth_igb_allmulticast_disable, 364 .link_update = eth_igb_link_update, 365 .stats_get = eth_igb_stats_get, 366 .xstats_get = eth_igb_xstats_get, 367 .xstats_get_by_id = eth_igb_xstats_get_by_id, 368 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 369 .xstats_get_names = eth_igb_xstats_get_names, 370 .stats_reset = eth_igb_stats_reset, 371 .xstats_reset = eth_igb_xstats_reset, 372 .fw_version_get = eth_igb_fw_version_get, 373 .dev_infos_get = eth_igb_infos_get, 374 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 375 .mtu_set = eth_igb_mtu_set, 376 .vlan_filter_set = eth_igb_vlan_filter_set, 377 .vlan_tpid_set = eth_igb_vlan_tpid_set, 378 .vlan_offload_set = eth_igb_vlan_offload_set, 379 .rx_queue_setup = eth_igb_rx_queue_setup, 380 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 381 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 382 .rx_queue_release = eth_igb_rx_queue_release, 383 .rx_queue_count = eth_igb_rx_queue_count, 384 .rx_descriptor_done = eth_igb_rx_descriptor_done, 385 .rx_descriptor_status = eth_igb_rx_descriptor_status, 386 .tx_descriptor_status = eth_igb_tx_descriptor_status, 387 .tx_queue_setup = eth_igb_tx_queue_setup, 388 .tx_queue_release = eth_igb_tx_queue_release, 389 .tx_done_cleanup = eth_igb_tx_done_cleanup, 390 .dev_led_on = eth_igb_led_on, 391 .dev_led_off = eth_igb_led_off, 392 .flow_ctrl_get = eth_igb_flow_ctrl_get, 393 .flow_ctrl_set = eth_igb_flow_ctrl_set, 394 .mac_addr_add = eth_igb_rar_set, 395 .mac_addr_remove = eth_igb_rar_clear, 396 .mac_addr_set = eth_igb_default_mac_addr_set, 397 .reta_update = eth_igb_rss_reta_update, 398 .reta_query = eth_igb_rss_reta_query, 399 .rss_hash_update = eth_igb_rss_hash_update, 400 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 401 .filter_ctrl = eth_igb_filter_ctrl, 402 .set_mc_addr_list = eth_igb_set_mc_addr_list, 403 .rxq_info_get = igb_rxq_info_get, 404 .txq_info_get = igb_txq_info_get, 405 .timesync_enable = igb_timesync_enable, 406 .timesync_disable = igb_timesync_disable, 407 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 408 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 409 .get_reg = eth_igb_get_regs, 410 .get_eeprom_length = eth_igb_get_eeprom_length, 411 .get_eeprom = eth_igb_get_eeprom, 412 .set_eeprom = eth_igb_set_eeprom, 413 .get_module_info = eth_igb_get_module_info, 414 .get_module_eeprom = eth_igb_get_module_eeprom, 415 .timesync_adjust_time = igb_timesync_adjust_time, 416 .timesync_read_time = igb_timesync_read_time, 417 .timesync_write_time = igb_timesync_write_time, 418 }; 419 420 /* 421 * dev_ops for virtual function, bare necessities for basic vf 422 * operation have been implemented 423 */ 424 static const struct eth_dev_ops igbvf_eth_dev_ops = { 425 .dev_configure = igbvf_dev_configure, 426 .dev_start = igbvf_dev_start, 427 .dev_stop = igbvf_dev_stop, 428 .dev_close = igbvf_dev_close, 429 .promiscuous_enable = igbvf_promiscuous_enable, 430 .promiscuous_disable = igbvf_promiscuous_disable, 431 .allmulticast_enable = igbvf_allmulticast_enable, 432 .allmulticast_disable = igbvf_allmulticast_disable, 433 .link_update = eth_igb_link_update, 434 .stats_get = eth_igbvf_stats_get, 435 .xstats_get = eth_igbvf_xstats_get, 436 .xstats_get_names = eth_igbvf_xstats_get_names, 437 .stats_reset = eth_igbvf_stats_reset, 438 .xstats_reset = eth_igbvf_stats_reset, 439 .vlan_filter_set = igbvf_vlan_filter_set, 440 .dev_infos_get = eth_igbvf_infos_get, 441 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 442 .rx_queue_setup = eth_igb_rx_queue_setup, 443 .rx_queue_release = eth_igb_rx_queue_release, 444 .rx_descriptor_done = eth_igb_rx_descriptor_done, 445 .rx_descriptor_status = eth_igb_rx_descriptor_status, 446 .tx_descriptor_status = eth_igb_tx_descriptor_status, 447 .tx_queue_setup = eth_igb_tx_queue_setup, 448 .tx_queue_release = eth_igb_tx_queue_release, 449 .set_mc_addr_list = eth_igb_set_mc_addr_list, 450 .rxq_info_get = igb_rxq_info_get, 451 .txq_info_get = igb_txq_info_get, 452 .mac_addr_set = igbvf_default_mac_addr_set, 453 .get_reg = igbvf_get_regs, 454 }; 455 456 /* store statistics names and its offset in stats structure */ 457 struct rte_igb_xstats_name_off { 458 char name[RTE_ETH_XSTATS_NAME_SIZE]; 459 unsigned offset; 460 }; 461 462 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 463 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 464 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 465 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 466 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 467 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 468 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 469 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 470 ecol)}, 471 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 472 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 473 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 474 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 475 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 476 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 477 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 478 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 479 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 480 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 481 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 482 fcruc)}, 483 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 484 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 485 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 486 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 487 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 488 prc1023)}, 489 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 490 prc1522)}, 491 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 492 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 493 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 494 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 495 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 496 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 497 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 498 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 499 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 500 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 501 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 502 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 503 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 504 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 505 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 506 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 507 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 508 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 509 ptc1023)}, 510 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 511 ptc1522)}, 512 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 513 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 514 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 515 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 516 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 517 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 518 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 519 520 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 521 }; 522 523 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 524 sizeof(rte_igb_stats_strings[0])) 525 526 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 527 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 528 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 529 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 530 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 531 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 532 }; 533 534 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 535 sizeof(rte_igbvf_stats_strings[0])) 536 537 538 static inline void 539 igb_intr_enable(struct rte_eth_dev *dev) 540 { 541 struct e1000_interrupt *intr = 542 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 543 struct e1000_hw *hw = 544 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 545 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 546 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 547 548 if (rte_intr_allow_others(intr_handle) && 549 dev->data->dev_conf.intr_conf.lsc != 0) { 550 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 551 } 552 553 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 554 E1000_WRITE_FLUSH(hw); 555 } 556 557 static void 558 igb_intr_disable(struct rte_eth_dev *dev) 559 { 560 struct e1000_hw *hw = 561 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 562 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 563 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 564 565 if (rte_intr_allow_others(intr_handle) && 566 dev->data->dev_conf.intr_conf.lsc != 0) { 567 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 568 } 569 570 E1000_WRITE_REG(hw, E1000_IMC, ~0); 571 E1000_WRITE_FLUSH(hw); 572 } 573 574 static inline void 575 igbvf_intr_enable(struct rte_eth_dev *dev) 576 { 577 struct e1000_hw *hw = 578 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 579 580 /* only for mailbox */ 581 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 582 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 583 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 584 E1000_WRITE_FLUSH(hw); 585 } 586 587 /* only for mailbox now. If RX/TX needed, should extend this function. */ 588 static void 589 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 590 { 591 uint32_t tmp = 0; 592 593 /* mailbox */ 594 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 595 tmp |= E1000_VTIVAR_VALID; 596 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 597 } 598 599 static void 600 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 601 { 602 struct e1000_hw *hw = 603 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 604 605 /* Configure VF other cause ivar */ 606 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 607 } 608 609 static inline int32_t 610 igb_pf_reset_hw(struct e1000_hw *hw) 611 { 612 uint32_t ctrl_ext; 613 int32_t status; 614 615 status = e1000_reset_hw(hw); 616 617 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 618 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 619 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 620 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 621 E1000_WRITE_FLUSH(hw); 622 623 return status; 624 } 625 626 static void 627 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 628 { 629 struct e1000_hw *hw = 630 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 631 632 633 hw->vendor_id = pci_dev->id.vendor_id; 634 hw->device_id = pci_dev->id.device_id; 635 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 636 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 637 638 e1000_set_mac_type(hw); 639 640 /* need to check if it is a vf device below */ 641 } 642 643 static int 644 igb_reset_swfw_lock(struct e1000_hw *hw) 645 { 646 int ret_val; 647 648 /* 649 * Do mac ops initialization manually here, since we will need 650 * some function pointers set by this call. 651 */ 652 ret_val = e1000_init_mac_params(hw); 653 if (ret_val) 654 return ret_val; 655 656 /* 657 * SMBI lock should not fail in this early stage. If this is the case, 658 * it is due to an improper exit of the application. 659 * So force the release of the faulty lock. 660 */ 661 if (e1000_get_hw_semaphore_generic(hw) < 0) { 662 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 663 } 664 e1000_put_hw_semaphore_generic(hw); 665 666 if (hw->mac.ops.acquire_swfw_sync != NULL) { 667 uint16_t mask; 668 669 /* 670 * Phy lock should not fail in this early stage. If this is the case, 671 * it is due to an improper exit of the application. 672 * So force the release of the faulty lock. 673 */ 674 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 675 if (hw->bus.func > E1000_FUNC_1) 676 mask <<= 2; 677 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 678 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 679 hw->bus.func); 680 } 681 hw->mac.ops.release_swfw_sync(hw, mask); 682 683 /* 684 * This one is more tricky since it is common to all ports; but 685 * swfw_sync retries last long enough (1s) to be almost sure that if 686 * lock can not be taken it is due to an improper lock of the 687 * semaphore. 688 */ 689 mask = E1000_SWFW_EEP_SM; 690 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 691 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 692 } 693 hw->mac.ops.release_swfw_sync(hw, mask); 694 } 695 696 return E1000_SUCCESS; 697 } 698 699 /* Remove all ntuple filters of the device */ 700 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 701 { 702 struct e1000_filter_info *filter_info = 703 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 704 struct e1000_5tuple_filter *p_5tuple; 705 struct e1000_2tuple_filter *p_2tuple; 706 707 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 708 TAILQ_REMOVE(&filter_info->fivetuple_list, 709 p_5tuple, entries); 710 rte_free(p_5tuple); 711 } 712 filter_info->fivetuple_mask = 0; 713 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 714 TAILQ_REMOVE(&filter_info->twotuple_list, 715 p_2tuple, entries); 716 rte_free(p_2tuple); 717 } 718 filter_info->twotuple_mask = 0; 719 720 return 0; 721 } 722 723 /* Remove all flex filters of the device */ 724 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 725 { 726 struct e1000_filter_info *filter_info = 727 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 728 struct e1000_flex_filter *p_flex; 729 730 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 731 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 732 rte_free(p_flex); 733 } 734 filter_info->flex_mask = 0; 735 736 return 0; 737 } 738 739 static int 740 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 741 { 742 int error = 0; 743 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 744 struct e1000_hw *hw = 745 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 746 struct e1000_vfta * shadow_vfta = 747 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 748 struct e1000_filter_info *filter_info = 749 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 750 struct e1000_adapter *adapter = 751 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 752 753 uint32_t ctrl_ext; 754 755 eth_dev->dev_ops = ð_igb_ops; 756 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 757 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 758 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 759 760 /* for secondary processes, we don't initialise any further as primary 761 * has already done this work. Only check we don't need a different 762 * RX function */ 763 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 764 if (eth_dev->data->scattered_rx) 765 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 766 return 0; 767 } 768 769 rte_eth_copy_pci_info(eth_dev, pci_dev); 770 771 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 772 773 igb_identify_hardware(eth_dev, pci_dev); 774 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 775 error = -EIO; 776 goto err_late; 777 } 778 779 e1000_get_bus_info(hw); 780 781 /* Reset any pending lock */ 782 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 783 error = -EIO; 784 goto err_late; 785 } 786 787 /* Finish initialization */ 788 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 789 error = -EIO; 790 goto err_late; 791 } 792 793 hw->mac.autoneg = 1; 794 hw->phy.autoneg_wait_to_complete = 0; 795 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 796 797 /* Copper options */ 798 if (hw->phy.media_type == e1000_media_type_copper) { 799 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 800 hw->phy.disable_polarity_correction = 0; 801 hw->phy.ms_type = e1000_ms_hw_default; 802 } 803 804 /* 805 * Start from a known state, this is important in reading the nvm 806 * and mac from that. 807 */ 808 igb_pf_reset_hw(hw); 809 810 /* Make sure we have a good EEPROM before we read from it */ 811 if (e1000_validate_nvm_checksum(hw) < 0) { 812 /* 813 * Some PCI-E parts fail the first check due to 814 * the link being in sleep state, call it again, 815 * if it fails a second time its a real issue. 816 */ 817 if (e1000_validate_nvm_checksum(hw) < 0) { 818 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 819 error = -EIO; 820 goto err_late; 821 } 822 } 823 824 /* Read the permanent MAC address out of the EEPROM */ 825 if (e1000_read_mac_addr(hw) != 0) { 826 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 827 error = -EIO; 828 goto err_late; 829 } 830 831 /* Allocate memory for storing MAC addresses */ 832 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 833 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 834 if (eth_dev->data->mac_addrs == NULL) { 835 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 836 "store MAC addresses", 837 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 838 error = -ENOMEM; 839 goto err_late; 840 } 841 842 /* Copy the permanent MAC address */ 843 ether_addr_copy((struct ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); 844 845 /* initialize the vfta */ 846 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 847 848 /* Now initialize the hardware */ 849 if (igb_hardware_init(hw) != 0) { 850 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 851 rte_free(eth_dev->data->mac_addrs); 852 eth_dev->data->mac_addrs = NULL; 853 error = -ENODEV; 854 goto err_late; 855 } 856 hw->mac.get_link_status = 1; 857 adapter->stopped = 0; 858 859 /* Indicate SOL/IDER usage */ 860 if (e1000_check_reset_block(hw) < 0) { 861 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 862 "SOL/IDER session"); 863 } 864 865 /* initialize PF if max_vfs not zero */ 866 igb_pf_host_init(eth_dev); 867 868 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 869 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 870 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 871 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 872 E1000_WRITE_FLUSH(hw); 873 874 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 875 eth_dev->data->port_id, pci_dev->id.vendor_id, 876 pci_dev->id.device_id); 877 878 rte_intr_callback_register(&pci_dev->intr_handle, 879 eth_igb_interrupt_handler, 880 (void *)eth_dev); 881 882 /* enable uio/vfio intr/eventfd mapping */ 883 rte_intr_enable(&pci_dev->intr_handle); 884 885 /* enable support intr */ 886 igb_intr_enable(eth_dev); 887 888 /* initialize filter info */ 889 memset(filter_info, 0, 890 sizeof(struct e1000_filter_info)); 891 892 TAILQ_INIT(&filter_info->flex_list); 893 TAILQ_INIT(&filter_info->twotuple_list); 894 TAILQ_INIT(&filter_info->fivetuple_list); 895 896 TAILQ_INIT(&igb_filter_ntuple_list); 897 TAILQ_INIT(&igb_filter_ethertype_list); 898 TAILQ_INIT(&igb_filter_syn_list); 899 TAILQ_INIT(&igb_filter_flex_list); 900 TAILQ_INIT(&igb_filter_rss_list); 901 TAILQ_INIT(&igb_flow_list); 902 903 return 0; 904 905 err_late: 906 igb_hw_control_release(hw); 907 908 return error; 909 } 910 911 static int 912 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 913 { 914 struct rte_pci_device *pci_dev; 915 struct rte_intr_handle *intr_handle; 916 struct e1000_hw *hw; 917 struct e1000_adapter *adapter = 918 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 919 struct e1000_filter_info *filter_info = 920 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 921 922 PMD_INIT_FUNC_TRACE(); 923 924 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 925 return -EPERM; 926 927 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 928 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 929 intr_handle = &pci_dev->intr_handle; 930 931 if (adapter->stopped == 0) 932 eth_igb_close(eth_dev); 933 934 eth_dev->dev_ops = NULL; 935 eth_dev->rx_pkt_burst = NULL; 936 eth_dev->tx_pkt_burst = NULL; 937 938 /* Reset any pending lock */ 939 igb_reset_swfw_lock(hw); 940 941 /* uninitialize PF if max_vfs not zero */ 942 igb_pf_host_uninit(eth_dev); 943 944 /* disable uio intr before callback unregister */ 945 rte_intr_disable(intr_handle); 946 rte_intr_callback_unregister(intr_handle, 947 eth_igb_interrupt_handler, eth_dev); 948 949 /* clear the SYN filter info */ 950 filter_info->syn_info = 0; 951 952 /* clear the ethertype filters info */ 953 filter_info->ethertype_mask = 0; 954 memset(filter_info->ethertype_filters, 0, 955 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 956 957 /* clear the rss filter info */ 958 memset(&filter_info->rss_info, 0, 959 sizeof(struct igb_rte_flow_rss_conf)); 960 961 /* remove all ntuple filters of the device */ 962 igb_ntuple_filter_uninit(eth_dev); 963 964 /* remove all flex filters of the device */ 965 igb_flex_filter_uninit(eth_dev); 966 967 /* clear all the filters list */ 968 igb_filterlist_flush(eth_dev); 969 970 return 0; 971 } 972 973 /* 974 * Virtual Function device init 975 */ 976 static int 977 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 978 { 979 struct rte_pci_device *pci_dev; 980 struct rte_intr_handle *intr_handle; 981 struct e1000_adapter *adapter = 982 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 983 struct e1000_hw *hw = 984 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 985 int diag; 986 struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr; 987 988 PMD_INIT_FUNC_TRACE(); 989 990 eth_dev->dev_ops = &igbvf_eth_dev_ops; 991 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 992 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 993 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 994 995 /* for secondary processes, we don't initialise any further as primary 996 * has already done this work. Only check we don't need a different 997 * RX function */ 998 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 999 if (eth_dev->data->scattered_rx) 1000 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 1001 return 0; 1002 } 1003 1004 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1005 rte_eth_copy_pci_info(eth_dev, pci_dev); 1006 1007 hw->device_id = pci_dev->id.device_id; 1008 hw->vendor_id = pci_dev->id.vendor_id; 1009 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1010 adapter->stopped = 0; 1011 1012 /* Initialize the shared code (base driver) */ 1013 diag = e1000_setup_init_funcs(hw, TRUE); 1014 if (diag != 0) { 1015 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 1016 diag); 1017 return -EIO; 1018 } 1019 1020 /* init_mailbox_params */ 1021 hw->mbx.ops.init_params(hw); 1022 1023 /* Disable the interrupts for VF */ 1024 igbvf_intr_disable(hw); 1025 1026 diag = hw->mac.ops.reset_hw(hw); 1027 1028 /* Allocate memory for storing MAC addresses */ 1029 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN * 1030 hw->mac.rar_entry_count, 0); 1031 if (eth_dev->data->mac_addrs == NULL) { 1032 PMD_INIT_LOG(ERR, 1033 "Failed to allocate %d bytes needed to store MAC " 1034 "addresses", 1035 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1036 return -ENOMEM; 1037 } 1038 1039 /* Generate a random MAC address, if none was assigned by PF. */ 1040 if (is_zero_ether_addr(perm_addr)) { 1041 eth_random_addr(perm_addr->addr_bytes); 1042 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1043 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1044 "%02x:%02x:%02x:%02x:%02x:%02x", 1045 perm_addr->addr_bytes[0], 1046 perm_addr->addr_bytes[1], 1047 perm_addr->addr_bytes[2], 1048 perm_addr->addr_bytes[3], 1049 perm_addr->addr_bytes[4], 1050 perm_addr->addr_bytes[5]); 1051 } 1052 1053 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1054 if (diag) { 1055 rte_free(eth_dev->data->mac_addrs); 1056 eth_dev->data->mac_addrs = NULL; 1057 return diag; 1058 } 1059 /* Copy the permanent MAC address */ 1060 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1061 ð_dev->data->mac_addrs[0]); 1062 1063 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1064 "mac.type=%s", 1065 eth_dev->data->port_id, pci_dev->id.vendor_id, 1066 pci_dev->id.device_id, "igb_mac_82576_vf"); 1067 1068 intr_handle = &pci_dev->intr_handle; 1069 rte_intr_callback_register(intr_handle, 1070 eth_igbvf_interrupt_handler, eth_dev); 1071 1072 return 0; 1073 } 1074 1075 static int 1076 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1077 { 1078 struct e1000_adapter *adapter = 1079 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1080 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1081 1082 PMD_INIT_FUNC_TRACE(); 1083 1084 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1085 return -EPERM; 1086 1087 if (adapter->stopped == 0) 1088 igbvf_dev_close(eth_dev); 1089 1090 eth_dev->dev_ops = NULL; 1091 eth_dev->rx_pkt_burst = NULL; 1092 eth_dev->tx_pkt_burst = NULL; 1093 1094 /* disable uio intr before callback unregister */ 1095 rte_intr_disable(&pci_dev->intr_handle); 1096 rte_intr_callback_unregister(&pci_dev->intr_handle, 1097 eth_igbvf_interrupt_handler, 1098 (void *)eth_dev); 1099 1100 return 0; 1101 } 1102 1103 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1104 struct rte_pci_device *pci_dev) 1105 { 1106 return rte_eth_dev_pci_generic_probe(pci_dev, 1107 sizeof(struct e1000_adapter), eth_igb_dev_init); 1108 } 1109 1110 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1111 { 1112 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1113 } 1114 1115 static struct rte_pci_driver rte_igb_pmd = { 1116 .id_table = pci_id_igb_map, 1117 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1118 RTE_PCI_DRV_IOVA_AS_VA, 1119 .probe = eth_igb_pci_probe, 1120 .remove = eth_igb_pci_remove, 1121 }; 1122 1123 1124 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1125 struct rte_pci_device *pci_dev) 1126 { 1127 return rte_eth_dev_pci_generic_probe(pci_dev, 1128 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1129 } 1130 1131 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1132 { 1133 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1134 } 1135 1136 /* 1137 * virtual function driver struct 1138 */ 1139 static struct rte_pci_driver rte_igbvf_pmd = { 1140 .id_table = pci_id_igbvf_map, 1141 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1142 .probe = eth_igbvf_pci_probe, 1143 .remove = eth_igbvf_pci_remove, 1144 }; 1145 1146 static void 1147 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1148 { 1149 struct e1000_hw *hw = 1150 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1151 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1152 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1153 rctl |= E1000_RCTL_VFE; 1154 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1155 } 1156 1157 static int 1158 igb_check_mq_mode(struct rte_eth_dev *dev) 1159 { 1160 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1161 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1162 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1163 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1164 1165 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1166 tx_mq_mode == ETH_MQ_TX_DCB || 1167 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1168 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1169 return -EINVAL; 1170 } 1171 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1172 /* Check multi-queue mode. 1173 * To no break software we accept ETH_MQ_RX_NONE as this might 1174 * be used to turn off VLAN filter. 1175 */ 1176 1177 if (rx_mq_mode == ETH_MQ_RX_NONE || 1178 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1179 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1180 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1181 } else { 1182 /* Only support one queue on VFs. 1183 * RSS together with SRIOV is not supported. 1184 */ 1185 PMD_INIT_LOG(ERR, "SRIOV is active," 1186 " wrong mq_mode rx %d.", 1187 rx_mq_mode); 1188 return -EINVAL; 1189 } 1190 /* TX mode is not used here, so mode might be ignored.*/ 1191 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1192 /* SRIOV only works in VMDq enable mode */ 1193 PMD_INIT_LOG(WARNING, "SRIOV is active," 1194 " TX mode %d is not supported. " 1195 " Driver will behave as %d mode.", 1196 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1197 } 1198 1199 /* check valid queue number */ 1200 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1201 PMD_INIT_LOG(ERR, "SRIOV is active," 1202 " only support one queue on VFs."); 1203 return -EINVAL; 1204 } 1205 } else { 1206 /* To no break software that set invalid mode, only display 1207 * warning if invalid mode is used. 1208 */ 1209 if (rx_mq_mode != ETH_MQ_RX_NONE && 1210 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1211 rx_mq_mode != ETH_MQ_RX_RSS) { 1212 /* RSS together with VMDq not supported*/ 1213 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1214 rx_mq_mode); 1215 return -EINVAL; 1216 } 1217 1218 if (tx_mq_mode != ETH_MQ_TX_NONE && 1219 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1220 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1221 " Due to txmode is meaningless in this" 1222 " driver, just ignore.", 1223 tx_mq_mode); 1224 } 1225 } 1226 return 0; 1227 } 1228 1229 static int 1230 eth_igb_configure(struct rte_eth_dev *dev) 1231 { 1232 struct e1000_interrupt *intr = 1233 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1234 int ret; 1235 1236 PMD_INIT_FUNC_TRACE(); 1237 1238 /* multipe queue mode checking */ 1239 ret = igb_check_mq_mode(dev); 1240 if (ret != 0) { 1241 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1242 ret); 1243 return ret; 1244 } 1245 1246 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1247 PMD_INIT_FUNC_TRACE(); 1248 1249 return 0; 1250 } 1251 1252 static void 1253 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1254 bool enable) 1255 { 1256 struct e1000_hw *hw = 1257 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1258 uint32_t tctl, rctl; 1259 1260 tctl = E1000_READ_REG(hw, E1000_TCTL); 1261 rctl = E1000_READ_REG(hw, E1000_RCTL); 1262 1263 if (enable) { 1264 /* enable Tx/Rx */ 1265 tctl |= E1000_TCTL_EN; 1266 rctl |= E1000_RCTL_EN; 1267 } else { 1268 /* disable Tx/Rx */ 1269 tctl &= ~E1000_TCTL_EN; 1270 rctl &= ~E1000_RCTL_EN; 1271 } 1272 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1273 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1274 E1000_WRITE_FLUSH(hw); 1275 } 1276 1277 static int 1278 eth_igb_start(struct rte_eth_dev *dev) 1279 { 1280 struct e1000_hw *hw = 1281 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1282 struct e1000_adapter *adapter = 1283 E1000_DEV_PRIVATE(dev->data->dev_private); 1284 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1285 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1286 int ret, mask; 1287 uint32_t intr_vector = 0; 1288 uint32_t ctrl_ext; 1289 uint32_t *speeds; 1290 int num_speeds; 1291 bool autoneg; 1292 1293 PMD_INIT_FUNC_TRACE(); 1294 1295 /* disable uio/vfio intr/eventfd mapping */ 1296 rte_intr_disable(intr_handle); 1297 1298 /* Power up the phy. Needed to make the link go Up */ 1299 eth_igb_dev_set_link_up(dev); 1300 1301 /* 1302 * Packet Buffer Allocation (PBA) 1303 * Writing PBA sets the receive portion of the buffer 1304 * the remainder is used for the transmit buffer. 1305 */ 1306 if (hw->mac.type == e1000_82575) { 1307 uint32_t pba; 1308 1309 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1310 E1000_WRITE_REG(hw, E1000_PBA, pba); 1311 } 1312 1313 /* Put the address into the Receive Address Array */ 1314 e1000_rar_set(hw, hw->mac.addr, 0); 1315 1316 /* Initialize the hardware */ 1317 if (igb_hardware_init(hw)) { 1318 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1319 return -EIO; 1320 } 1321 adapter->stopped = 0; 1322 1323 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1324 1325 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1326 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1327 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1328 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1329 E1000_WRITE_FLUSH(hw); 1330 1331 /* configure PF module if SRIOV enabled */ 1332 igb_pf_host_configure(dev); 1333 1334 /* check and configure queue intr-vector mapping */ 1335 if ((rte_intr_cap_multiple(intr_handle) || 1336 !RTE_ETH_DEV_SRIOV(dev).active) && 1337 dev->data->dev_conf.intr_conf.rxq != 0) { 1338 intr_vector = dev->data->nb_rx_queues; 1339 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1340 return -1; 1341 } 1342 1343 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1344 intr_handle->intr_vec = 1345 rte_zmalloc("intr_vec", 1346 dev->data->nb_rx_queues * sizeof(int), 0); 1347 if (intr_handle->intr_vec == NULL) { 1348 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1349 " intr_vec", dev->data->nb_rx_queues); 1350 return -ENOMEM; 1351 } 1352 } 1353 1354 /* confiugre msix for rx interrupt */ 1355 eth_igb_configure_msix_intr(dev); 1356 1357 /* Configure for OS presence */ 1358 igb_init_manageability(hw); 1359 1360 eth_igb_tx_init(dev); 1361 1362 /* This can fail when allocating mbufs for descriptor rings */ 1363 ret = eth_igb_rx_init(dev); 1364 if (ret) { 1365 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1366 igb_dev_clear_queues(dev); 1367 return ret; 1368 } 1369 1370 e1000_clear_hw_cntrs_base_generic(hw); 1371 1372 /* 1373 * VLAN Offload Settings 1374 */ 1375 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1376 ETH_VLAN_EXTEND_MASK; 1377 ret = eth_igb_vlan_offload_set(dev, mask); 1378 if (ret) { 1379 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1380 igb_dev_clear_queues(dev); 1381 return ret; 1382 } 1383 1384 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1385 /* Enable VLAN filter since VMDq always use VLAN filter */ 1386 igb_vmdq_vlan_hw_filter_enable(dev); 1387 } 1388 1389 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1390 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1391 (hw->mac.type == e1000_i211)) { 1392 /* Configure EITR with the maximum possible value (0xFFFF) */ 1393 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1394 } 1395 1396 /* Setup link speed and duplex */ 1397 speeds = &dev->data->dev_conf.link_speeds; 1398 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1399 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1400 hw->mac.autoneg = 1; 1401 } else { 1402 num_speeds = 0; 1403 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1404 1405 /* Reset */ 1406 hw->phy.autoneg_advertised = 0; 1407 1408 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1409 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1410 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1411 num_speeds = -1; 1412 goto error_invalid_config; 1413 } 1414 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1415 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1416 num_speeds++; 1417 } 1418 if (*speeds & ETH_LINK_SPEED_10M) { 1419 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1420 num_speeds++; 1421 } 1422 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1423 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1424 num_speeds++; 1425 } 1426 if (*speeds & ETH_LINK_SPEED_100M) { 1427 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1428 num_speeds++; 1429 } 1430 if (*speeds & ETH_LINK_SPEED_1G) { 1431 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1432 num_speeds++; 1433 } 1434 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1435 goto error_invalid_config; 1436 1437 /* Set/reset the mac.autoneg based on the link speed, 1438 * fixed or not 1439 */ 1440 if (!autoneg) { 1441 hw->mac.autoneg = 0; 1442 hw->mac.forced_speed_duplex = 1443 hw->phy.autoneg_advertised; 1444 } else { 1445 hw->mac.autoneg = 1; 1446 } 1447 } 1448 1449 e1000_setup_link(hw); 1450 1451 if (rte_intr_allow_others(intr_handle)) { 1452 /* check if lsc interrupt is enabled */ 1453 if (dev->data->dev_conf.intr_conf.lsc != 0) 1454 eth_igb_lsc_interrupt_setup(dev, TRUE); 1455 else 1456 eth_igb_lsc_interrupt_setup(dev, FALSE); 1457 } else { 1458 rte_intr_callback_unregister(intr_handle, 1459 eth_igb_interrupt_handler, 1460 (void *)dev); 1461 if (dev->data->dev_conf.intr_conf.lsc != 0) 1462 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1463 " no intr multiplex"); 1464 } 1465 1466 /* check if rxq interrupt is enabled */ 1467 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1468 rte_intr_dp_is_en(intr_handle)) 1469 eth_igb_rxq_interrupt_setup(dev); 1470 1471 /* enable uio/vfio intr/eventfd mapping */ 1472 rte_intr_enable(intr_handle); 1473 1474 /* resume enabled intr since hw reset */ 1475 igb_intr_enable(dev); 1476 1477 /* restore all types filter */ 1478 igb_filter_restore(dev); 1479 1480 eth_igb_rxtx_control(dev, true); 1481 eth_igb_link_update(dev, 0); 1482 1483 PMD_INIT_LOG(DEBUG, "<<"); 1484 1485 return 0; 1486 1487 error_invalid_config: 1488 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1489 dev->data->dev_conf.link_speeds, dev->data->port_id); 1490 igb_dev_clear_queues(dev); 1491 return -EINVAL; 1492 } 1493 1494 /********************************************************************* 1495 * 1496 * This routine disables all traffic on the adapter by issuing a 1497 * global reset on the MAC. 1498 * 1499 **********************************************************************/ 1500 static void 1501 eth_igb_stop(struct rte_eth_dev *dev) 1502 { 1503 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1504 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1505 struct rte_eth_link link; 1506 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1507 1508 eth_igb_rxtx_control(dev, false); 1509 1510 igb_intr_disable(dev); 1511 1512 /* disable intr eventfd mapping */ 1513 rte_intr_disable(intr_handle); 1514 1515 igb_pf_reset_hw(hw); 1516 E1000_WRITE_REG(hw, E1000_WUC, 0); 1517 1518 /* Set bit for Go Link disconnect */ 1519 if (hw->mac.type >= e1000_82580) { 1520 uint32_t phpm_reg; 1521 1522 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1523 phpm_reg |= E1000_82580_PM_GO_LINKD; 1524 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1525 } 1526 1527 /* Power down the phy. Needed to make the link go Down */ 1528 eth_igb_dev_set_link_down(dev); 1529 1530 igb_dev_clear_queues(dev); 1531 1532 /* clear the recorded link status */ 1533 memset(&link, 0, sizeof(link)); 1534 rte_eth_linkstatus_set(dev, &link); 1535 1536 if (!rte_intr_allow_others(intr_handle)) 1537 /* resume to the default handler */ 1538 rte_intr_callback_register(intr_handle, 1539 eth_igb_interrupt_handler, 1540 (void *)dev); 1541 1542 /* Clean datapath event and queue/vec mapping */ 1543 rte_intr_efd_disable(intr_handle); 1544 if (intr_handle->intr_vec != NULL) { 1545 rte_free(intr_handle->intr_vec); 1546 intr_handle->intr_vec = NULL; 1547 } 1548 } 1549 1550 static int 1551 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1552 { 1553 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1554 1555 if (hw->phy.media_type == e1000_media_type_copper) 1556 e1000_power_up_phy(hw); 1557 else 1558 e1000_power_up_fiber_serdes_link(hw); 1559 1560 return 0; 1561 } 1562 1563 static int 1564 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1565 { 1566 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1567 1568 if (hw->phy.media_type == e1000_media_type_copper) 1569 e1000_power_down_phy(hw); 1570 else 1571 e1000_shutdown_fiber_serdes_link(hw); 1572 1573 return 0; 1574 } 1575 1576 static void 1577 eth_igb_close(struct rte_eth_dev *dev) 1578 { 1579 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1580 struct e1000_adapter *adapter = 1581 E1000_DEV_PRIVATE(dev->data->dev_private); 1582 struct rte_eth_link link; 1583 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1584 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1585 1586 eth_igb_stop(dev); 1587 adapter->stopped = 1; 1588 1589 e1000_phy_hw_reset(hw); 1590 igb_release_manageability(hw); 1591 igb_hw_control_release(hw); 1592 1593 /* Clear bit for Go Link disconnect */ 1594 if (hw->mac.type >= e1000_82580) { 1595 uint32_t phpm_reg; 1596 1597 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1598 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1599 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1600 } 1601 1602 igb_dev_free_queues(dev); 1603 1604 if (intr_handle->intr_vec) { 1605 rte_free(intr_handle->intr_vec); 1606 intr_handle->intr_vec = NULL; 1607 } 1608 1609 memset(&link, 0, sizeof(link)); 1610 rte_eth_linkstatus_set(dev, &link); 1611 } 1612 1613 /* 1614 * Reset PF device. 1615 */ 1616 static int 1617 eth_igb_reset(struct rte_eth_dev *dev) 1618 { 1619 int ret; 1620 1621 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1622 * its VF to make them align with it. The detailed notification 1623 * mechanism is PMD specific and is currently not implemented. 1624 * To avoid unexpected behavior in VF, currently reset of PF with 1625 * SR-IOV activation is not supported. It might be supported later. 1626 */ 1627 if (dev->data->sriov.active) 1628 return -ENOTSUP; 1629 1630 ret = eth_igb_dev_uninit(dev); 1631 if (ret) 1632 return ret; 1633 1634 ret = eth_igb_dev_init(dev); 1635 1636 return ret; 1637 } 1638 1639 1640 static int 1641 igb_get_rx_buffer_size(struct e1000_hw *hw) 1642 { 1643 uint32_t rx_buf_size; 1644 if (hw->mac.type == e1000_82576) { 1645 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1646 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1647 /* PBS needs to be translated according to a lookup table */ 1648 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1649 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1650 rx_buf_size = (rx_buf_size << 10); 1651 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1652 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1653 } else { 1654 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1655 } 1656 1657 return rx_buf_size; 1658 } 1659 1660 /********************************************************************* 1661 * 1662 * Initialize the hardware 1663 * 1664 **********************************************************************/ 1665 static int 1666 igb_hardware_init(struct e1000_hw *hw) 1667 { 1668 uint32_t rx_buf_size; 1669 int diag; 1670 1671 /* Let the firmware know the OS is in control */ 1672 igb_hw_control_acquire(hw); 1673 1674 /* 1675 * These parameters control the automatic generation (Tx) and 1676 * response (Rx) to Ethernet PAUSE frames. 1677 * - High water mark should allow for at least two standard size (1518) 1678 * frames to be received after sending an XOFF. 1679 * - Low water mark works best when it is very near the high water mark. 1680 * This allows the receiver to restart by sending XON when it has 1681 * drained a bit. Here we use an arbitrary value of 1500 which will 1682 * restart after one full frame is pulled from the buffer. There 1683 * could be several smaller frames in the buffer and if so they will 1684 * not trigger the XON until their total number reduces the buffer 1685 * by 1500. 1686 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1687 */ 1688 rx_buf_size = igb_get_rx_buffer_size(hw); 1689 1690 hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2); 1691 hw->fc.low_water = hw->fc.high_water - 1500; 1692 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1693 hw->fc.send_xon = 1; 1694 1695 /* Set Flow control, use the tunable location if sane */ 1696 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1697 hw->fc.requested_mode = igb_fc_setting; 1698 else 1699 hw->fc.requested_mode = e1000_fc_none; 1700 1701 /* Issue a global reset */ 1702 igb_pf_reset_hw(hw); 1703 E1000_WRITE_REG(hw, E1000_WUC, 0); 1704 1705 diag = e1000_init_hw(hw); 1706 if (diag < 0) 1707 return diag; 1708 1709 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1710 e1000_get_phy_info(hw); 1711 e1000_check_for_link(hw); 1712 1713 return 0; 1714 } 1715 1716 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1717 static void 1718 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1719 { 1720 int pause_frames; 1721 1722 uint64_t old_gprc = stats->gprc; 1723 uint64_t old_gptc = stats->gptc; 1724 uint64_t old_tpr = stats->tpr; 1725 uint64_t old_tpt = stats->tpt; 1726 uint64_t old_rpthc = stats->rpthc; 1727 uint64_t old_hgptc = stats->hgptc; 1728 1729 if(hw->phy.media_type == e1000_media_type_copper || 1730 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1731 stats->symerrs += 1732 E1000_READ_REG(hw,E1000_SYMERRS); 1733 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1734 } 1735 1736 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1737 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1738 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1739 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1740 1741 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1742 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1743 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1744 stats->dc += E1000_READ_REG(hw, E1000_DC); 1745 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1746 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1747 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1748 /* 1749 ** For watchdog management we need to know if we have been 1750 ** paused during the last interval, so capture that here. 1751 */ 1752 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1753 stats->xoffrxc += pause_frames; 1754 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1755 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1756 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1757 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1758 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1759 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1760 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1761 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1762 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1763 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1764 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1765 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1766 1767 /* For the 64-bit byte counters the low dword must be read first. */ 1768 /* Both registers clear on the read of the high dword */ 1769 1770 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1771 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1772 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1773 stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN; 1774 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1775 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1776 stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN; 1777 1778 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1779 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1780 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1781 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1782 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1783 1784 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1785 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1786 1787 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1788 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1789 stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN; 1790 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1791 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1792 stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN; 1793 1794 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1795 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1796 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1797 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1798 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1799 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1800 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1801 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1802 1803 /* Interrupt Counts */ 1804 1805 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1806 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1807 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1808 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1809 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1810 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1811 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1812 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1813 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1814 1815 /* Host to Card Statistics */ 1816 1817 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1818 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1819 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1820 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1821 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1822 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1823 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1824 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1825 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1826 stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN; 1827 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1828 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1829 stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN; 1830 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1831 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1832 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1833 1834 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1835 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1836 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1837 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1838 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1839 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1840 } 1841 1842 static int 1843 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1844 { 1845 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1846 struct e1000_hw_stats *stats = 1847 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1848 1849 igb_read_stats_registers(hw, stats); 1850 1851 if (rte_stats == NULL) 1852 return -EINVAL; 1853 1854 /* Rx Errors */ 1855 rte_stats->imissed = stats->mpc; 1856 rte_stats->ierrors = stats->crcerrs + 1857 stats->rlec + stats->ruc + stats->roc + 1858 stats->rxerrc + stats->algnerrc + stats->cexterr; 1859 1860 /* Tx Errors */ 1861 rte_stats->oerrors = stats->ecol + stats->latecol; 1862 1863 rte_stats->ipackets = stats->gprc; 1864 rte_stats->opackets = stats->gptc; 1865 rte_stats->ibytes = stats->gorc; 1866 rte_stats->obytes = stats->gotc; 1867 return 0; 1868 } 1869 1870 static void 1871 eth_igb_stats_reset(struct rte_eth_dev *dev) 1872 { 1873 struct e1000_hw_stats *hw_stats = 1874 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1875 1876 /* HW registers are cleared on read */ 1877 eth_igb_stats_get(dev, NULL); 1878 1879 /* Reset software totals */ 1880 memset(hw_stats, 0, sizeof(*hw_stats)); 1881 } 1882 1883 static void 1884 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1885 { 1886 struct e1000_hw_stats *stats = 1887 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1888 1889 /* HW registers are cleared on read */ 1890 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1891 1892 /* Reset software totals */ 1893 memset(stats, 0, sizeof(*stats)); 1894 } 1895 1896 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1897 struct rte_eth_xstat_name *xstats_names, 1898 __rte_unused unsigned int size) 1899 { 1900 unsigned i; 1901 1902 if (xstats_names == NULL) 1903 return IGB_NB_XSTATS; 1904 1905 /* Note: limit checked in rte_eth_xstats_names() */ 1906 1907 for (i = 0; i < IGB_NB_XSTATS; i++) { 1908 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1909 sizeof(xstats_names[i].name)); 1910 } 1911 1912 return IGB_NB_XSTATS; 1913 } 1914 1915 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1916 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1917 unsigned int limit) 1918 { 1919 unsigned int i; 1920 1921 if (!ids) { 1922 if (xstats_names == NULL) 1923 return IGB_NB_XSTATS; 1924 1925 for (i = 0; i < IGB_NB_XSTATS; i++) 1926 strlcpy(xstats_names[i].name, 1927 rte_igb_stats_strings[i].name, 1928 sizeof(xstats_names[i].name)); 1929 1930 return IGB_NB_XSTATS; 1931 1932 } else { 1933 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1934 1935 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1936 IGB_NB_XSTATS); 1937 1938 for (i = 0; i < limit; i++) { 1939 if (ids[i] >= IGB_NB_XSTATS) { 1940 PMD_INIT_LOG(ERR, "id value isn't valid"); 1941 return -1; 1942 } 1943 strcpy(xstats_names[i].name, 1944 xstats_names_copy[ids[i]].name); 1945 } 1946 return limit; 1947 } 1948 } 1949 1950 static int 1951 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1952 unsigned n) 1953 { 1954 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1955 struct e1000_hw_stats *hw_stats = 1956 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1957 unsigned i; 1958 1959 if (n < IGB_NB_XSTATS) 1960 return IGB_NB_XSTATS; 1961 1962 igb_read_stats_registers(hw, hw_stats); 1963 1964 /* If this is a reset xstats is NULL, and we have cleared the 1965 * registers by reading them. 1966 */ 1967 if (!xstats) 1968 return 0; 1969 1970 /* Extended stats */ 1971 for (i = 0; i < IGB_NB_XSTATS; i++) { 1972 xstats[i].id = i; 1973 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1974 rte_igb_stats_strings[i].offset); 1975 } 1976 1977 return IGB_NB_XSTATS; 1978 } 1979 1980 static int 1981 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1982 uint64_t *values, unsigned int n) 1983 { 1984 unsigned int i; 1985 1986 if (!ids) { 1987 struct e1000_hw *hw = 1988 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1989 struct e1000_hw_stats *hw_stats = 1990 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1991 1992 if (n < IGB_NB_XSTATS) 1993 return IGB_NB_XSTATS; 1994 1995 igb_read_stats_registers(hw, hw_stats); 1996 1997 /* If this is a reset xstats is NULL, and we have cleared the 1998 * registers by reading them. 1999 */ 2000 if (!values) 2001 return 0; 2002 2003 /* Extended stats */ 2004 for (i = 0; i < IGB_NB_XSTATS; i++) 2005 values[i] = *(uint64_t *)(((char *)hw_stats) + 2006 rte_igb_stats_strings[i].offset); 2007 2008 return IGB_NB_XSTATS; 2009 2010 } else { 2011 uint64_t values_copy[IGB_NB_XSTATS]; 2012 2013 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2014 IGB_NB_XSTATS); 2015 2016 for (i = 0; i < n; i++) { 2017 if (ids[i] >= IGB_NB_XSTATS) { 2018 PMD_INIT_LOG(ERR, "id value isn't valid"); 2019 return -1; 2020 } 2021 values[i] = values_copy[ids[i]]; 2022 } 2023 return n; 2024 } 2025 } 2026 2027 static void 2028 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2029 { 2030 /* Good Rx packets, include VF loopback */ 2031 UPDATE_VF_STAT(E1000_VFGPRC, 2032 hw_stats->last_gprc, hw_stats->gprc); 2033 2034 /* Good Rx octets, include VF loopback */ 2035 UPDATE_VF_STAT(E1000_VFGORC, 2036 hw_stats->last_gorc, hw_stats->gorc); 2037 2038 /* Good Tx packets, include VF loopback */ 2039 UPDATE_VF_STAT(E1000_VFGPTC, 2040 hw_stats->last_gptc, hw_stats->gptc); 2041 2042 /* Good Tx octets, include VF loopback */ 2043 UPDATE_VF_STAT(E1000_VFGOTC, 2044 hw_stats->last_gotc, hw_stats->gotc); 2045 2046 /* Rx Multicst packets */ 2047 UPDATE_VF_STAT(E1000_VFMPRC, 2048 hw_stats->last_mprc, hw_stats->mprc); 2049 2050 /* Good Rx loopback packets */ 2051 UPDATE_VF_STAT(E1000_VFGPRLBC, 2052 hw_stats->last_gprlbc, hw_stats->gprlbc); 2053 2054 /* Good Rx loopback octets */ 2055 UPDATE_VF_STAT(E1000_VFGORLBC, 2056 hw_stats->last_gorlbc, hw_stats->gorlbc); 2057 2058 /* Good Tx loopback packets */ 2059 UPDATE_VF_STAT(E1000_VFGPTLBC, 2060 hw_stats->last_gptlbc, hw_stats->gptlbc); 2061 2062 /* Good Tx loopback octets */ 2063 UPDATE_VF_STAT(E1000_VFGOTLBC, 2064 hw_stats->last_gotlbc, hw_stats->gotlbc); 2065 } 2066 2067 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2068 struct rte_eth_xstat_name *xstats_names, 2069 __rte_unused unsigned limit) 2070 { 2071 unsigned i; 2072 2073 if (xstats_names != NULL) 2074 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2075 strlcpy(xstats_names[i].name, 2076 rte_igbvf_stats_strings[i].name, 2077 sizeof(xstats_names[i].name)); 2078 } 2079 return IGBVF_NB_XSTATS; 2080 } 2081 2082 static int 2083 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2084 unsigned n) 2085 { 2086 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2087 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2088 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2089 unsigned i; 2090 2091 if (n < IGBVF_NB_XSTATS) 2092 return IGBVF_NB_XSTATS; 2093 2094 igbvf_read_stats_registers(hw, hw_stats); 2095 2096 if (!xstats) 2097 return 0; 2098 2099 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2100 xstats[i].id = i; 2101 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2102 rte_igbvf_stats_strings[i].offset); 2103 } 2104 2105 return IGBVF_NB_XSTATS; 2106 } 2107 2108 static int 2109 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2110 { 2111 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2112 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2113 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2114 2115 igbvf_read_stats_registers(hw, hw_stats); 2116 2117 if (rte_stats == NULL) 2118 return -EINVAL; 2119 2120 rte_stats->ipackets = hw_stats->gprc; 2121 rte_stats->ibytes = hw_stats->gorc; 2122 rte_stats->opackets = hw_stats->gptc; 2123 rte_stats->obytes = hw_stats->gotc; 2124 return 0; 2125 } 2126 2127 static void 2128 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2129 { 2130 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2131 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2132 2133 /* Sync HW register to the last stats */ 2134 eth_igbvf_stats_get(dev, NULL); 2135 2136 /* reset HW current stats*/ 2137 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2138 offsetof(struct e1000_vf_stats, gprc)); 2139 } 2140 2141 static int 2142 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2143 size_t fw_size) 2144 { 2145 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2146 struct e1000_fw_version fw; 2147 int ret; 2148 2149 e1000_get_fw_version(hw, &fw); 2150 2151 switch (hw->mac.type) { 2152 case e1000_i210: 2153 case e1000_i211: 2154 if (!(e1000_get_flash_presence_i210(hw))) { 2155 ret = snprintf(fw_version, fw_size, 2156 "%2d.%2d-%d", 2157 fw.invm_major, fw.invm_minor, 2158 fw.invm_img_type); 2159 break; 2160 } 2161 /* fall through */ 2162 default: 2163 /* if option rom is valid, display its version too */ 2164 if (fw.or_valid) { 2165 ret = snprintf(fw_version, fw_size, 2166 "%d.%d, 0x%08x, %d.%d.%d", 2167 fw.eep_major, fw.eep_minor, fw.etrack_id, 2168 fw.or_major, fw.or_build, fw.or_patch); 2169 /* no option rom */ 2170 } else { 2171 if (fw.etrack_id != 0X0000) { 2172 ret = snprintf(fw_version, fw_size, 2173 "%d.%d, 0x%08x", 2174 fw.eep_major, fw.eep_minor, 2175 fw.etrack_id); 2176 } else { 2177 ret = snprintf(fw_version, fw_size, 2178 "%d.%d.%d", 2179 fw.eep_major, fw.eep_minor, 2180 fw.eep_build); 2181 } 2182 } 2183 break; 2184 } 2185 2186 ret += 1; /* add the size of '\0' */ 2187 if (fw_size < (u32)ret) 2188 return ret; 2189 else 2190 return 0; 2191 } 2192 2193 static void 2194 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2195 { 2196 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2197 2198 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2199 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2200 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2201 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2202 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2203 dev_info->rx_queue_offload_capa; 2204 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2205 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2206 dev_info->tx_queue_offload_capa; 2207 2208 switch (hw->mac.type) { 2209 case e1000_82575: 2210 dev_info->max_rx_queues = 4; 2211 dev_info->max_tx_queues = 4; 2212 dev_info->max_vmdq_pools = 0; 2213 break; 2214 2215 case e1000_82576: 2216 dev_info->max_rx_queues = 16; 2217 dev_info->max_tx_queues = 16; 2218 dev_info->max_vmdq_pools = ETH_8_POOLS; 2219 dev_info->vmdq_queue_num = 16; 2220 break; 2221 2222 case e1000_82580: 2223 dev_info->max_rx_queues = 8; 2224 dev_info->max_tx_queues = 8; 2225 dev_info->max_vmdq_pools = ETH_8_POOLS; 2226 dev_info->vmdq_queue_num = 8; 2227 break; 2228 2229 case e1000_i350: 2230 dev_info->max_rx_queues = 8; 2231 dev_info->max_tx_queues = 8; 2232 dev_info->max_vmdq_pools = ETH_8_POOLS; 2233 dev_info->vmdq_queue_num = 8; 2234 break; 2235 2236 case e1000_i354: 2237 dev_info->max_rx_queues = 8; 2238 dev_info->max_tx_queues = 8; 2239 break; 2240 2241 case e1000_i210: 2242 dev_info->max_rx_queues = 4; 2243 dev_info->max_tx_queues = 4; 2244 dev_info->max_vmdq_pools = 0; 2245 break; 2246 2247 case e1000_i211: 2248 dev_info->max_rx_queues = 2; 2249 dev_info->max_tx_queues = 2; 2250 dev_info->max_vmdq_pools = 0; 2251 break; 2252 2253 default: 2254 /* Should not happen */ 2255 break; 2256 } 2257 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2258 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2259 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2260 2261 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2262 .rx_thresh = { 2263 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2264 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2265 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2266 }, 2267 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2268 .rx_drop_en = 0, 2269 .offloads = 0, 2270 }; 2271 2272 dev_info->default_txconf = (struct rte_eth_txconf) { 2273 .tx_thresh = { 2274 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2275 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2276 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2277 }, 2278 .offloads = 0, 2279 }; 2280 2281 dev_info->rx_desc_lim = rx_desc_lim; 2282 dev_info->tx_desc_lim = tx_desc_lim; 2283 2284 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2285 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2286 ETH_LINK_SPEED_1G; 2287 2288 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2289 dev_info->min_mtu = ETHER_MIN_MTU; 2290 2291 } 2292 2293 static const uint32_t * 2294 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2295 { 2296 static const uint32_t ptypes[] = { 2297 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2298 RTE_PTYPE_L2_ETHER, 2299 RTE_PTYPE_L3_IPV4, 2300 RTE_PTYPE_L3_IPV4_EXT, 2301 RTE_PTYPE_L3_IPV6, 2302 RTE_PTYPE_L3_IPV6_EXT, 2303 RTE_PTYPE_L4_TCP, 2304 RTE_PTYPE_L4_UDP, 2305 RTE_PTYPE_L4_SCTP, 2306 RTE_PTYPE_TUNNEL_IP, 2307 RTE_PTYPE_INNER_L3_IPV6, 2308 RTE_PTYPE_INNER_L3_IPV6_EXT, 2309 RTE_PTYPE_INNER_L4_TCP, 2310 RTE_PTYPE_INNER_L4_UDP, 2311 RTE_PTYPE_UNKNOWN 2312 }; 2313 2314 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2315 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2316 return ptypes; 2317 return NULL; 2318 } 2319 2320 static void 2321 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2322 { 2323 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2324 2325 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2326 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2327 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2328 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2329 DEV_TX_OFFLOAD_IPV4_CKSUM | 2330 DEV_TX_OFFLOAD_UDP_CKSUM | 2331 DEV_TX_OFFLOAD_TCP_CKSUM | 2332 DEV_TX_OFFLOAD_SCTP_CKSUM | 2333 DEV_TX_OFFLOAD_TCP_TSO; 2334 switch (hw->mac.type) { 2335 case e1000_vfadapt: 2336 dev_info->max_rx_queues = 2; 2337 dev_info->max_tx_queues = 2; 2338 break; 2339 case e1000_vfadapt_i350: 2340 dev_info->max_rx_queues = 1; 2341 dev_info->max_tx_queues = 1; 2342 break; 2343 default: 2344 /* Should not happen */ 2345 break; 2346 } 2347 2348 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2349 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2350 dev_info->rx_queue_offload_capa; 2351 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2352 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2353 dev_info->tx_queue_offload_capa; 2354 2355 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2356 .rx_thresh = { 2357 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2358 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2359 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2360 }, 2361 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2362 .rx_drop_en = 0, 2363 .offloads = 0, 2364 }; 2365 2366 dev_info->default_txconf = (struct rte_eth_txconf) { 2367 .tx_thresh = { 2368 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2369 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2370 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2371 }, 2372 .offloads = 0, 2373 }; 2374 2375 dev_info->rx_desc_lim = rx_desc_lim; 2376 dev_info->tx_desc_lim = tx_desc_lim; 2377 } 2378 2379 /* return 0 means link status changed, -1 means not changed */ 2380 static int 2381 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2382 { 2383 struct e1000_hw *hw = 2384 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2385 struct rte_eth_link link; 2386 int link_check, count; 2387 2388 link_check = 0; 2389 hw->mac.get_link_status = 1; 2390 2391 /* possible wait-to-complete in up to 9 seconds */ 2392 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2393 /* Read the real link status */ 2394 switch (hw->phy.media_type) { 2395 case e1000_media_type_copper: 2396 /* Do the work to read phy */ 2397 e1000_check_for_link(hw); 2398 link_check = !hw->mac.get_link_status; 2399 break; 2400 2401 case e1000_media_type_fiber: 2402 e1000_check_for_link(hw); 2403 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2404 E1000_STATUS_LU); 2405 break; 2406 2407 case e1000_media_type_internal_serdes: 2408 e1000_check_for_link(hw); 2409 link_check = hw->mac.serdes_has_link; 2410 break; 2411 2412 /* VF device is type_unknown */ 2413 case e1000_media_type_unknown: 2414 eth_igbvf_link_update(hw); 2415 link_check = !hw->mac.get_link_status; 2416 break; 2417 2418 default: 2419 break; 2420 } 2421 if (link_check || wait_to_complete == 0) 2422 break; 2423 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2424 } 2425 memset(&link, 0, sizeof(link)); 2426 2427 /* Now we check if a transition has happened */ 2428 if (link_check) { 2429 uint16_t duplex, speed; 2430 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2431 link.link_duplex = (duplex == FULL_DUPLEX) ? 2432 ETH_LINK_FULL_DUPLEX : 2433 ETH_LINK_HALF_DUPLEX; 2434 link.link_speed = speed; 2435 link.link_status = ETH_LINK_UP; 2436 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2437 ETH_LINK_SPEED_FIXED); 2438 } else if (!link_check) { 2439 link.link_speed = 0; 2440 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2441 link.link_status = ETH_LINK_DOWN; 2442 link.link_autoneg = ETH_LINK_FIXED; 2443 } 2444 2445 return rte_eth_linkstatus_set(dev, &link); 2446 } 2447 2448 /* 2449 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2450 * For ASF and Pass Through versions of f/w this means 2451 * that the driver is loaded. 2452 */ 2453 static void 2454 igb_hw_control_acquire(struct e1000_hw *hw) 2455 { 2456 uint32_t ctrl_ext; 2457 2458 /* Let firmware know the driver has taken over */ 2459 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2460 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2461 } 2462 2463 /* 2464 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2465 * For ASF and Pass Through versions of f/w this means that the 2466 * driver is no longer loaded. 2467 */ 2468 static void 2469 igb_hw_control_release(struct e1000_hw *hw) 2470 { 2471 uint32_t ctrl_ext; 2472 2473 /* Let firmware taken over control of h/w */ 2474 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2475 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2476 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2477 } 2478 2479 /* 2480 * Bit of a misnomer, what this really means is 2481 * to enable OS management of the system... aka 2482 * to disable special hardware management features. 2483 */ 2484 static void 2485 igb_init_manageability(struct e1000_hw *hw) 2486 { 2487 if (e1000_enable_mng_pass_thru(hw)) { 2488 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2489 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2490 2491 /* disable hardware interception of ARP */ 2492 manc &= ~(E1000_MANC_ARP_EN); 2493 2494 /* enable receiving management packets to the host */ 2495 manc |= E1000_MANC_EN_MNG2HOST; 2496 manc2h |= 1 << 5; /* Mng Port 623 */ 2497 manc2h |= 1 << 6; /* Mng Port 664 */ 2498 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2499 E1000_WRITE_REG(hw, E1000_MANC, manc); 2500 } 2501 } 2502 2503 static void 2504 igb_release_manageability(struct e1000_hw *hw) 2505 { 2506 if (e1000_enable_mng_pass_thru(hw)) { 2507 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2508 2509 manc |= E1000_MANC_ARP_EN; 2510 manc &= ~E1000_MANC_EN_MNG2HOST; 2511 2512 E1000_WRITE_REG(hw, E1000_MANC, manc); 2513 } 2514 } 2515 2516 static void 2517 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2518 { 2519 struct e1000_hw *hw = 2520 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2521 uint32_t rctl; 2522 2523 rctl = E1000_READ_REG(hw, E1000_RCTL); 2524 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2525 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2526 } 2527 2528 static void 2529 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2530 { 2531 struct e1000_hw *hw = 2532 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2533 uint32_t rctl; 2534 2535 rctl = E1000_READ_REG(hw, E1000_RCTL); 2536 rctl &= (~E1000_RCTL_UPE); 2537 if (dev->data->all_multicast == 1) 2538 rctl |= E1000_RCTL_MPE; 2539 else 2540 rctl &= (~E1000_RCTL_MPE); 2541 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2542 } 2543 2544 static void 2545 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2546 { 2547 struct e1000_hw *hw = 2548 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2549 uint32_t rctl; 2550 2551 rctl = E1000_READ_REG(hw, E1000_RCTL); 2552 rctl |= E1000_RCTL_MPE; 2553 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2554 } 2555 2556 static void 2557 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2558 { 2559 struct e1000_hw *hw = 2560 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2561 uint32_t rctl; 2562 2563 if (dev->data->promiscuous == 1) 2564 return; /* must remain in all_multicast mode */ 2565 rctl = E1000_READ_REG(hw, E1000_RCTL); 2566 rctl &= (~E1000_RCTL_MPE); 2567 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2568 } 2569 2570 static int 2571 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2572 { 2573 struct e1000_hw *hw = 2574 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2575 struct e1000_vfta * shadow_vfta = 2576 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2577 uint32_t vfta; 2578 uint32_t vid_idx; 2579 uint32_t vid_bit; 2580 2581 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2582 E1000_VFTA_ENTRY_MASK); 2583 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2584 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2585 if (on) 2586 vfta |= vid_bit; 2587 else 2588 vfta &= ~vid_bit; 2589 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2590 2591 /* update local VFTA copy */ 2592 shadow_vfta->vfta[vid_idx] = vfta; 2593 2594 return 0; 2595 } 2596 2597 static int 2598 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2599 enum rte_vlan_type vlan_type, 2600 uint16_t tpid) 2601 { 2602 struct e1000_hw *hw = 2603 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2604 uint32_t reg, qinq; 2605 2606 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2607 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2608 2609 /* only outer TPID of double VLAN can be configured*/ 2610 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2611 reg = E1000_READ_REG(hw, E1000_VET); 2612 reg = (reg & (~E1000_VET_VET_EXT)) | 2613 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2614 E1000_WRITE_REG(hw, E1000_VET, reg); 2615 2616 return 0; 2617 } 2618 2619 /* all other TPID values are read-only*/ 2620 PMD_DRV_LOG(ERR, "Not supported"); 2621 2622 return -ENOTSUP; 2623 } 2624 2625 static void 2626 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2627 { 2628 struct e1000_hw *hw = 2629 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2630 uint32_t reg; 2631 2632 /* Filter Table Disable */ 2633 reg = E1000_READ_REG(hw, E1000_RCTL); 2634 reg &= ~E1000_RCTL_CFIEN; 2635 reg &= ~E1000_RCTL_VFE; 2636 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2637 } 2638 2639 static void 2640 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2641 { 2642 struct e1000_hw *hw = 2643 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2644 struct e1000_vfta * shadow_vfta = 2645 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2646 uint32_t reg; 2647 int i; 2648 2649 /* Filter Table Enable, CFI not used for packet acceptance */ 2650 reg = E1000_READ_REG(hw, E1000_RCTL); 2651 reg &= ~E1000_RCTL_CFIEN; 2652 reg |= E1000_RCTL_VFE; 2653 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2654 2655 /* restore VFTA table */ 2656 for (i = 0; i < IGB_VFTA_SIZE; i++) 2657 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2658 } 2659 2660 static void 2661 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2662 { 2663 struct e1000_hw *hw = 2664 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2665 uint32_t reg; 2666 2667 /* VLAN Mode Disable */ 2668 reg = E1000_READ_REG(hw, E1000_CTRL); 2669 reg &= ~E1000_CTRL_VME; 2670 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2671 } 2672 2673 static void 2674 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2675 { 2676 struct e1000_hw *hw = 2677 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2678 uint32_t reg; 2679 2680 /* VLAN Mode Enable */ 2681 reg = E1000_READ_REG(hw, E1000_CTRL); 2682 reg |= E1000_CTRL_VME; 2683 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2684 } 2685 2686 static void 2687 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2688 { 2689 struct e1000_hw *hw = 2690 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2691 uint32_t reg; 2692 2693 /* CTRL_EXT: Extended VLAN */ 2694 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2695 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2696 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2697 2698 /* Update maximum packet length */ 2699 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2700 E1000_WRITE_REG(hw, E1000_RLPML, 2701 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2702 VLAN_TAG_SIZE); 2703 } 2704 2705 static void 2706 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2707 { 2708 struct e1000_hw *hw = 2709 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2710 uint32_t reg; 2711 2712 /* CTRL_EXT: Extended VLAN */ 2713 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2714 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2715 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2716 2717 /* Update maximum packet length */ 2718 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2719 E1000_WRITE_REG(hw, E1000_RLPML, 2720 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2721 2 * VLAN_TAG_SIZE); 2722 } 2723 2724 static int 2725 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2726 { 2727 struct rte_eth_rxmode *rxmode; 2728 2729 rxmode = &dev->data->dev_conf.rxmode; 2730 if(mask & ETH_VLAN_STRIP_MASK){ 2731 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2732 igb_vlan_hw_strip_enable(dev); 2733 else 2734 igb_vlan_hw_strip_disable(dev); 2735 } 2736 2737 if(mask & ETH_VLAN_FILTER_MASK){ 2738 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2739 igb_vlan_hw_filter_enable(dev); 2740 else 2741 igb_vlan_hw_filter_disable(dev); 2742 } 2743 2744 if(mask & ETH_VLAN_EXTEND_MASK){ 2745 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2746 igb_vlan_hw_extend_enable(dev); 2747 else 2748 igb_vlan_hw_extend_disable(dev); 2749 } 2750 2751 return 0; 2752 } 2753 2754 2755 /** 2756 * It enables the interrupt mask and then enable the interrupt. 2757 * 2758 * @param dev 2759 * Pointer to struct rte_eth_dev. 2760 * @param on 2761 * Enable or Disable 2762 * 2763 * @return 2764 * - On success, zero. 2765 * - On failure, a negative value. 2766 */ 2767 static int 2768 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2769 { 2770 struct e1000_interrupt *intr = 2771 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2772 2773 if (on) 2774 intr->mask |= E1000_ICR_LSC; 2775 else 2776 intr->mask &= ~E1000_ICR_LSC; 2777 2778 return 0; 2779 } 2780 2781 /* It clears the interrupt causes and enables the interrupt. 2782 * It will be called once only during nic initialized. 2783 * 2784 * @param dev 2785 * Pointer to struct rte_eth_dev. 2786 * 2787 * @return 2788 * - On success, zero. 2789 * - On failure, a negative value. 2790 */ 2791 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2792 { 2793 uint32_t mask, regval; 2794 struct e1000_hw *hw = 2795 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2796 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2797 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2798 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2799 struct rte_eth_dev_info dev_info; 2800 2801 memset(&dev_info, 0, sizeof(dev_info)); 2802 eth_igb_infos_get(dev, &dev_info); 2803 2804 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2805 regval = E1000_READ_REG(hw, E1000_EIMS); 2806 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2807 2808 return 0; 2809 } 2810 2811 /* 2812 * It reads ICR and gets interrupt causes, check it and set a bit flag 2813 * to update link status. 2814 * 2815 * @param dev 2816 * Pointer to struct rte_eth_dev. 2817 * 2818 * @return 2819 * - On success, zero. 2820 * - On failure, a negative value. 2821 */ 2822 static int 2823 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2824 { 2825 uint32_t icr; 2826 struct e1000_hw *hw = 2827 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2828 struct e1000_interrupt *intr = 2829 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2830 2831 igb_intr_disable(dev); 2832 2833 /* read-on-clear nic registers here */ 2834 icr = E1000_READ_REG(hw, E1000_ICR); 2835 2836 intr->flags = 0; 2837 if (icr & E1000_ICR_LSC) { 2838 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2839 } 2840 2841 if (icr & E1000_ICR_VMMB) 2842 intr->flags |= E1000_FLAG_MAILBOX; 2843 2844 return 0; 2845 } 2846 2847 /* 2848 * It executes link_update after knowing an interrupt is prsent. 2849 * 2850 * @param dev 2851 * Pointer to struct rte_eth_dev. 2852 * 2853 * @return 2854 * - On success, zero. 2855 * - On failure, a negative value. 2856 */ 2857 static int 2858 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2859 struct rte_intr_handle *intr_handle) 2860 { 2861 struct e1000_hw *hw = 2862 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2863 struct e1000_interrupt *intr = 2864 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2865 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2866 struct rte_eth_link link; 2867 int ret; 2868 2869 if (intr->flags & E1000_FLAG_MAILBOX) { 2870 igb_pf_mbx_process(dev); 2871 intr->flags &= ~E1000_FLAG_MAILBOX; 2872 } 2873 2874 igb_intr_enable(dev); 2875 rte_intr_enable(intr_handle); 2876 2877 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2878 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2879 2880 /* set get_link_status to check register later */ 2881 hw->mac.get_link_status = 1; 2882 ret = eth_igb_link_update(dev, 0); 2883 2884 /* check if link has changed */ 2885 if (ret < 0) 2886 return 0; 2887 2888 rte_eth_linkstatus_get(dev, &link); 2889 if (link.link_status) { 2890 PMD_INIT_LOG(INFO, 2891 " Port %d: Link Up - speed %u Mbps - %s", 2892 dev->data->port_id, 2893 (unsigned)link.link_speed, 2894 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2895 "full-duplex" : "half-duplex"); 2896 } else { 2897 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2898 dev->data->port_id); 2899 } 2900 2901 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2902 pci_dev->addr.domain, 2903 pci_dev->addr.bus, 2904 pci_dev->addr.devid, 2905 pci_dev->addr.function); 2906 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2907 NULL); 2908 } 2909 2910 return 0; 2911 } 2912 2913 /** 2914 * Interrupt handler which shall be registered at first. 2915 * 2916 * @param handle 2917 * Pointer to interrupt handle. 2918 * @param param 2919 * The address of parameter (struct rte_eth_dev *) regsitered before. 2920 * 2921 * @return 2922 * void 2923 */ 2924 static void 2925 eth_igb_interrupt_handler(void *param) 2926 { 2927 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2928 2929 eth_igb_interrupt_get_status(dev); 2930 eth_igb_interrupt_action(dev, dev->intr_handle); 2931 } 2932 2933 static int 2934 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2935 { 2936 uint32_t eicr; 2937 struct e1000_hw *hw = 2938 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2939 struct e1000_interrupt *intr = 2940 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2941 2942 igbvf_intr_disable(hw); 2943 2944 /* read-on-clear nic registers here */ 2945 eicr = E1000_READ_REG(hw, E1000_EICR); 2946 intr->flags = 0; 2947 2948 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2949 intr->flags |= E1000_FLAG_MAILBOX; 2950 2951 return 0; 2952 } 2953 2954 void igbvf_mbx_process(struct rte_eth_dev *dev) 2955 { 2956 struct e1000_hw *hw = 2957 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2958 struct e1000_mbx_info *mbx = &hw->mbx; 2959 u32 in_msg = 0; 2960 2961 /* peek the message first */ 2962 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2963 2964 /* PF reset VF event */ 2965 if (in_msg == E1000_PF_CONTROL_MSG) { 2966 /* dummy mbx read to ack pf */ 2967 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2968 return; 2969 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2970 NULL); 2971 } 2972 } 2973 2974 static int 2975 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2976 { 2977 struct e1000_interrupt *intr = 2978 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2979 2980 if (intr->flags & E1000_FLAG_MAILBOX) { 2981 igbvf_mbx_process(dev); 2982 intr->flags &= ~E1000_FLAG_MAILBOX; 2983 } 2984 2985 igbvf_intr_enable(dev); 2986 rte_intr_enable(intr_handle); 2987 2988 return 0; 2989 } 2990 2991 static void 2992 eth_igbvf_interrupt_handler(void *param) 2993 { 2994 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2995 2996 eth_igbvf_interrupt_get_status(dev); 2997 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2998 } 2999 3000 static int 3001 eth_igb_led_on(struct rte_eth_dev *dev) 3002 { 3003 struct e1000_hw *hw; 3004 3005 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3006 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3007 } 3008 3009 static int 3010 eth_igb_led_off(struct rte_eth_dev *dev) 3011 { 3012 struct e1000_hw *hw; 3013 3014 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3015 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3016 } 3017 3018 static int 3019 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3020 { 3021 struct e1000_hw *hw; 3022 uint32_t ctrl; 3023 int tx_pause; 3024 int rx_pause; 3025 3026 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3027 fc_conf->pause_time = hw->fc.pause_time; 3028 fc_conf->high_water = hw->fc.high_water; 3029 fc_conf->low_water = hw->fc.low_water; 3030 fc_conf->send_xon = hw->fc.send_xon; 3031 fc_conf->autoneg = hw->mac.autoneg; 3032 3033 /* 3034 * Return rx_pause and tx_pause status according to actual setting of 3035 * the TFCE and RFCE bits in the CTRL register. 3036 */ 3037 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3038 if (ctrl & E1000_CTRL_TFCE) 3039 tx_pause = 1; 3040 else 3041 tx_pause = 0; 3042 3043 if (ctrl & E1000_CTRL_RFCE) 3044 rx_pause = 1; 3045 else 3046 rx_pause = 0; 3047 3048 if (rx_pause && tx_pause) 3049 fc_conf->mode = RTE_FC_FULL; 3050 else if (rx_pause) 3051 fc_conf->mode = RTE_FC_RX_PAUSE; 3052 else if (tx_pause) 3053 fc_conf->mode = RTE_FC_TX_PAUSE; 3054 else 3055 fc_conf->mode = RTE_FC_NONE; 3056 3057 return 0; 3058 } 3059 3060 static int 3061 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3062 { 3063 struct e1000_hw *hw; 3064 int err; 3065 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3066 e1000_fc_none, 3067 e1000_fc_rx_pause, 3068 e1000_fc_tx_pause, 3069 e1000_fc_full 3070 }; 3071 uint32_t rx_buf_size; 3072 uint32_t max_high_water; 3073 uint32_t rctl; 3074 3075 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3076 if (fc_conf->autoneg != hw->mac.autoneg) 3077 return -ENOTSUP; 3078 rx_buf_size = igb_get_rx_buffer_size(hw); 3079 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3080 3081 /* At least reserve one Ethernet frame for watermark */ 3082 max_high_water = rx_buf_size - ETHER_MAX_LEN; 3083 if ((fc_conf->high_water > max_high_water) || 3084 (fc_conf->high_water < fc_conf->low_water)) { 3085 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3086 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3087 return -EINVAL; 3088 } 3089 3090 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3091 hw->fc.pause_time = fc_conf->pause_time; 3092 hw->fc.high_water = fc_conf->high_water; 3093 hw->fc.low_water = fc_conf->low_water; 3094 hw->fc.send_xon = fc_conf->send_xon; 3095 3096 err = e1000_setup_link_generic(hw); 3097 if (err == E1000_SUCCESS) { 3098 3099 /* check if we want to forward MAC frames - driver doesn't have native 3100 * capability to do that, so we'll write the registers ourselves */ 3101 3102 rctl = E1000_READ_REG(hw, E1000_RCTL); 3103 3104 /* set or clear MFLCN.PMCF bit depending on configuration */ 3105 if (fc_conf->mac_ctrl_frame_fwd != 0) 3106 rctl |= E1000_RCTL_PMCF; 3107 else 3108 rctl &= ~E1000_RCTL_PMCF; 3109 3110 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3111 E1000_WRITE_FLUSH(hw); 3112 3113 return 0; 3114 } 3115 3116 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3117 return -EIO; 3118 } 3119 3120 #define E1000_RAH_POOLSEL_SHIFT (18) 3121 static int 3122 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 3123 uint32_t index, uint32_t pool) 3124 { 3125 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3126 uint32_t rah; 3127 3128 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3129 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3130 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3131 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3132 return 0; 3133 } 3134 3135 static void 3136 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3137 { 3138 uint8_t addr[ETHER_ADDR_LEN]; 3139 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3140 3141 memset(addr, 0, sizeof(addr)); 3142 3143 e1000_rar_set(hw, addr, index); 3144 } 3145 3146 static int 3147 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3148 struct ether_addr *addr) 3149 { 3150 eth_igb_rar_clear(dev, 0); 3151 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3152 3153 return 0; 3154 } 3155 /* 3156 * Virtual Function operations 3157 */ 3158 static void 3159 igbvf_intr_disable(struct e1000_hw *hw) 3160 { 3161 PMD_INIT_FUNC_TRACE(); 3162 3163 /* Clear interrupt mask to stop from interrupts being generated */ 3164 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3165 3166 E1000_WRITE_FLUSH(hw); 3167 } 3168 3169 static void 3170 igbvf_stop_adapter(struct rte_eth_dev *dev) 3171 { 3172 u32 reg_val; 3173 u16 i; 3174 struct rte_eth_dev_info dev_info; 3175 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3176 3177 memset(&dev_info, 0, sizeof(dev_info)); 3178 eth_igbvf_infos_get(dev, &dev_info); 3179 3180 /* Clear interrupt mask to stop from interrupts being generated */ 3181 igbvf_intr_disable(hw); 3182 3183 /* Clear any pending interrupts, flush previous writes */ 3184 E1000_READ_REG(hw, E1000_EICR); 3185 3186 /* Disable the transmit unit. Each queue must be disabled. */ 3187 for (i = 0; i < dev_info.max_tx_queues; i++) 3188 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3189 3190 /* Disable the receive unit by stopping each queue */ 3191 for (i = 0; i < dev_info.max_rx_queues; i++) { 3192 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3193 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3194 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3195 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3196 ; 3197 } 3198 3199 /* flush all queues disables */ 3200 E1000_WRITE_FLUSH(hw); 3201 msec_delay(2); 3202 } 3203 3204 static int eth_igbvf_link_update(struct e1000_hw *hw) 3205 { 3206 struct e1000_mbx_info *mbx = &hw->mbx; 3207 struct e1000_mac_info *mac = &hw->mac; 3208 int ret_val = E1000_SUCCESS; 3209 3210 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3211 3212 /* 3213 * We only want to run this if there has been a rst asserted. 3214 * in this case that could mean a link change, device reset, 3215 * or a virtual function reset 3216 */ 3217 3218 /* If we were hit with a reset or timeout drop the link */ 3219 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3220 mac->get_link_status = TRUE; 3221 3222 if (!mac->get_link_status) 3223 goto out; 3224 3225 /* if link status is down no point in checking to see if pf is up */ 3226 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3227 goto out; 3228 3229 /* if we passed all the tests above then the link is up and we no 3230 * longer need to check for link */ 3231 mac->get_link_status = FALSE; 3232 3233 out: 3234 return ret_val; 3235 } 3236 3237 3238 static int 3239 igbvf_dev_configure(struct rte_eth_dev *dev) 3240 { 3241 struct rte_eth_conf* conf = &dev->data->dev_conf; 3242 3243 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3244 dev->data->port_id); 3245 3246 /* 3247 * VF has no ability to enable/disable HW CRC 3248 * Keep the persistent behavior the same as Host PF 3249 */ 3250 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3251 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3252 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3253 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3254 } 3255 #else 3256 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3257 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3258 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3259 } 3260 #endif 3261 3262 return 0; 3263 } 3264 3265 static int 3266 igbvf_dev_start(struct rte_eth_dev *dev) 3267 { 3268 struct e1000_hw *hw = 3269 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3270 struct e1000_adapter *adapter = 3271 E1000_DEV_PRIVATE(dev->data->dev_private); 3272 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3273 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3274 int ret; 3275 uint32_t intr_vector = 0; 3276 3277 PMD_INIT_FUNC_TRACE(); 3278 3279 hw->mac.ops.reset_hw(hw); 3280 adapter->stopped = 0; 3281 3282 /* Set all vfta */ 3283 igbvf_set_vfta_all(dev,1); 3284 3285 eth_igbvf_tx_init(dev); 3286 3287 /* This can fail when allocating mbufs for descriptor rings */ 3288 ret = eth_igbvf_rx_init(dev); 3289 if (ret) { 3290 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3291 igb_dev_clear_queues(dev); 3292 return ret; 3293 } 3294 3295 /* check and configure queue intr-vector mapping */ 3296 if (rte_intr_cap_multiple(intr_handle) && 3297 dev->data->dev_conf.intr_conf.rxq) { 3298 intr_vector = dev->data->nb_rx_queues; 3299 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3300 if (ret) 3301 return ret; 3302 } 3303 3304 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3305 intr_handle->intr_vec = 3306 rte_zmalloc("intr_vec", 3307 dev->data->nb_rx_queues * sizeof(int), 0); 3308 if (!intr_handle->intr_vec) { 3309 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3310 " intr_vec", dev->data->nb_rx_queues); 3311 return -ENOMEM; 3312 } 3313 } 3314 3315 eth_igbvf_configure_msix_intr(dev); 3316 3317 /* enable uio/vfio intr/eventfd mapping */ 3318 rte_intr_enable(intr_handle); 3319 3320 /* resume enabled intr since hw reset */ 3321 igbvf_intr_enable(dev); 3322 3323 return 0; 3324 } 3325 3326 static void 3327 igbvf_dev_stop(struct rte_eth_dev *dev) 3328 { 3329 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3330 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3331 3332 PMD_INIT_FUNC_TRACE(); 3333 3334 igbvf_stop_adapter(dev); 3335 3336 /* 3337 * Clear what we set, but we still keep shadow_vfta to 3338 * restore after device starts 3339 */ 3340 igbvf_set_vfta_all(dev,0); 3341 3342 igb_dev_clear_queues(dev); 3343 3344 /* disable intr eventfd mapping */ 3345 rte_intr_disable(intr_handle); 3346 3347 /* Clean datapath event and queue/vec mapping */ 3348 rte_intr_efd_disable(intr_handle); 3349 if (intr_handle->intr_vec) { 3350 rte_free(intr_handle->intr_vec); 3351 intr_handle->intr_vec = NULL; 3352 } 3353 } 3354 3355 static void 3356 igbvf_dev_close(struct rte_eth_dev *dev) 3357 { 3358 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3359 struct e1000_adapter *adapter = 3360 E1000_DEV_PRIVATE(dev->data->dev_private); 3361 struct ether_addr addr; 3362 3363 PMD_INIT_FUNC_TRACE(); 3364 3365 e1000_reset_hw(hw); 3366 3367 igbvf_dev_stop(dev); 3368 adapter->stopped = 1; 3369 igb_dev_free_queues(dev); 3370 3371 /** 3372 * reprogram the RAR with a zero mac address, 3373 * to ensure that the VF traffic goes to the PF 3374 * after stop, close and detach of the VF. 3375 **/ 3376 3377 memset(&addr, 0, sizeof(addr)); 3378 igbvf_default_mac_addr_set(dev, &addr); 3379 } 3380 3381 static void 3382 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3383 { 3384 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3385 3386 /* Set both unicast and multicast promisc */ 3387 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3388 } 3389 3390 static void 3391 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3392 { 3393 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3394 3395 /* If in allmulticast mode leave multicast promisc */ 3396 if (dev->data->all_multicast == 1) 3397 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3398 else 3399 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3400 } 3401 3402 static void 3403 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3404 { 3405 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3406 3407 /* In promiscuous mode multicast promisc already set */ 3408 if (dev->data->promiscuous == 0) 3409 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3410 } 3411 3412 static void 3413 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3414 { 3415 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3416 3417 /* In promiscuous mode leave multicast promisc enabled */ 3418 if (dev->data->promiscuous == 0) 3419 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3420 } 3421 3422 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3423 { 3424 struct e1000_mbx_info *mbx = &hw->mbx; 3425 uint32_t msgbuf[2]; 3426 s32 err; 3427 3428 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3429 msgbuf[0] = E1000_VF_SET_VLAN; 3430 msgbuf[1] = vid; 3431 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3432 if (on) 3433 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3434 3435 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3436 if (err) 3437 goto mbx_err; 3438 3439 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3440 if (err) 3441 goto mbx_err; 3442 3443 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3444 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3445 err = -EINVAL; 3446 3447 mbx_err: 3448 return err; 3449 } 3450 3451 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3452 { 3453 struct e1000_hw *hw = 3454 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3455 struct e1000_vfta * shadow_vfta = 3456 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3457 int i = 0, j = 0, vfta = 0, mask = 1; 3458 3459 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3460 vfta = shadow_vfta->vfta[i]; 3461 if(vfta){ 3462 mask = 1; 3463 for (j = 0; j < 32; j++){ 3464 if(vfta & mask) 3465 igbvf_set_vfta(hw, 3466 (uint16_t)((i<<5)+j), on); 3467 mask<<=1; 3468 } 3469 } 3470 } 3471 3472 } 3473 3474 static int 3475 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3476 { 3477 struct e1000_hw *hw = 3478 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3479 struct e1000_vfta * shadow_vfta = 3480 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3481 uint32_t vid_idx = 0; 3482 uint32_t vid_bit = 0; 3483 int ret = 0; 3484 3485 PMD_INIT_FUNC_TRACE(); 3486 3487 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3488 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3489 if(ret){ 3490 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3491 return ret; 3492 } 3493 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3494 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3495 3496 /*Save what we set and retore it after device reset*/ 3497 if (on) 3498 shadow_vfta->vfta[vid_idx] |= vid_bit; 3499 else 3500 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3501 3502 return 0; 3503 } 3504 3505 static int 3506 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) 3507 { 3508 struct e1000_hw *hw = 3509 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3510 3511 /* index is not used by rar_set() */ 3512 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3513 return 0; 3514 } 3515 3516 3517 static int 3518 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3519 struct rte_eth_rss_reta_entry64 *reta_conf, 3520 uint16_t reta_size) 3521 { 3522 uint8_t i, j, mask; 3523 uint32_t reta, r; 3524 uint16_t idx, shift; 3525 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3526 3527 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3528 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3529 "(%d) doesn't match the number hardware can supported " 3530 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3531 return -EINVAL; 3532 } 3533 3534 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3535 idx = i / RTE_RETA_GROUP_SIZE; 3536 shift = i % RTE_RETA_GROUP_SIZE; 3537 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3538 IGB_4_BIT_MASK); 3539 if (!mask) 3540 continue; 3541 if (mask == IGB_4_BIT_MASK) 3542 r = 0; 3543 else 3544 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3545 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3546 if (mask & (0x1 << j)) 3547 reta |= reta_conf[idx].reta[shift + j] << 3548 (CHAR_BIT * j); 3549 else 3550 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3551 } 3552 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3553 } 3554 3555 return 0; 3556 } 3557 3558 static int 3559 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3560 struct rte_eth_rss_reta_entry64 *reta_conf, 3561 uint16_t reta_size) 3562 { 3563 uint8_t i, j, mask; 3564 uint32_t reta; 3565 uint16_t idx, shift; 3566 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3567 3568 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3569 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3570 "(%d) doesn't match the number hardware can supported " 3571 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3572 return -EINVAL; 3573 } 3574 3575 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3576 idx = i / RTE_RETA_GROUP_SIZE; 3577 shift = i % RTE_RETA_GROUP_SIZE; 3578 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3579 IGB_4_BIT_MASK); 3580 if (!mask) 3581 continue; 3582 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3583 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3584 if (mask & (0x1 << j)) 3585 reta_conf[idx].reta[shift + j] = 3586 ((reta >> (CHAR_BIT * j)) & 3587 IGB_8_BIT_MASK); 3588 } 3589 } 3590 3591 return 0; 3592 } 3593 3594 int 3595 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3596 struct rte_eth_syn_filter *filter, 3597 bool add) 3598 { 3599 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3600 struct e1000_filter_info *filter_info = 3601 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3602 uint32_t synqf, rfctl; 3603 3604 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3605 return -EINVAL; 3606 3607 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3608 3609 if (add) { 3610 if (synqf & E1000_SYN_FILTER_ENABLE) 3611 return -EINVAL; 3612 3613 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3614 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3615 3616 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3617 if (filter->hig_pri) 3618 rfctl |= E1000_RFCTL_SYNQFP; 3619 else 3620 rfctl &= ~E1000_RFCTL_SYNQFP; 3621 3622 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3623 } else { 3624 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3625 return -ENOENT; 3626 synqf = 0; 3627 } 3628 3629 filter_info->syn_info = synqf; 3630 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3631 E1000_WRITE_FLUSH(hw); 3632 return 0; 3633 } 3634 3635 static int 3636 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3637 struct rte_eth_syn_filter *filter) 3638 { 3639 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3640 uint32_t synqf, rfctl; 3641 3642 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3643 if (synqf & E1000_SYN_FILTER_ENABLE) { 3644 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3645 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3646 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3647 E1000_SYN_FILTER_QUEUE_SHIFT); 3648 return 0; 3649 } 3650 3651 return -ENOENT; 3652 } 3653 3654 static int 3655 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3656 enum rte_filter_op filter_op, 3657 void *arg) 3658 { 3659 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3660 int ret; 3661 3662 MAC_TYPE_FILTER_SUP(hw->mac.type); 3663 3664 if (filter_op == RTE_ETH_FILTER_NOP) 3665 return 0; 3666 3667 if (arg == NULL) { 3668 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3669 filter_op); 3670 return -EINVAL; 3671 } 3672 3673 switch (filter_op) { 3674 case RTE_ETH_FILTER_ADD: 3675 ret = eth_igb_syn_filter_set(dev, 3676 (struct rte_eth_syn_filter *)arg, 3677 TRUE); 3678 break; 3679 case RTE_ETH_FILTER_DELETE: 3680 ret = eth_igb_syn_filter_set(dev, 3681 (struct rte_eth_syn_filter *)arg, 3682 FALSE); 3683 break; 3684 case RTE_ETH_FILTER_GET: 3685 ret = eth_igb_syn_filter_get(dev, 3686 (struct rte_eth_syn_filter *)arg); 3687 break; 3688 default: 3689 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3690 ret = -EINVAL; 3691 break; 3692 } 3693 3694 return ret; 3695 } 3696 3697 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3698 static inline int 3699 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3700 struct e1000_2tuple_filter_info *filter_info) 3701 { 3702 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3703 return -EINVAL; 3704 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3705 return -EINVAL; /* filter index is out of range. */ 3706 if (filter->tcp_flags > TCP_FLAG_ALL) 3707 return -EINVAL; /* flags is invalid. */ 3708 3709 switch (filter->dst_port_mask) { 3710 case UINT16_MAX: 3711 filter_info->dst_port_mask = 0; 3712 filter_info->dst_port = filter->dst_port; 3713 break; 3714 case 0: 3715 filter_info->dst_port_mask = 1; 3716 break; 3717 default: 3718 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3719 return -EINVAL; 3720 } 3721 3722 switch (filter->proto_mask) { 3723 case UINT8_MAX: 3724 filter_info->proto_mask = 0; 3725 filter_info->proto = filter->proto; 3726 break; 3727 case 0: 3728 filter_info->proto_mask = 1; 3729 break; 3730 default: 3731 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3732 return -EINVAL; 3733 } 3734 3735 filter_info->priority = (uint8_t)filter->priority; 3736 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3737 filter_info->tcp_flags = filter->tcp_flags; 3738 else 3739 filter_info->tcp_flags = 0; 3740 3741 return 0; 3742 } 3743 3744 static inline struct e1000_2tuple_filter * 3745 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3746 struct e1000_2tuple_filter_info *key) 3747 { 3748 struct e1000_2tuple_filter *it; 3749 3750 TAILQ_FOREACH(it, filter_list, entries) { 3751 if (memcmp(key, &it->filter_info, 3752 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3753 return it; 3754 } 3755 } 3756 return NULL; 3757 } 3758 3759 /* inject a igb 2tuple filter to HW */ 3760 static inline void 3761 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3762 struct e1000_2tuple_filter *filter) 3763 { 3764 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3765 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3766 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3767 int i; 3768 3769 i = filter->index; 3770 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3771 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3772 imir |= E1000_IMIR_PORT_BP; 3773 else 3774 imir &= ~E1000_IMIR_PORT_BP; 3775 3776 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3777 3778 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3779 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3780 ttqf |= (uint32_t)(filter->filter_info.proto & 3781 E1000_TTQF_PROTOCOL_MASK); 3782 if (filter->filter_info.proto_mask == 0) 3783 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3784 3785 /* tcp flags bits setting. */ 3786 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 3787 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 3788 imir_ext |= E1000_IMIREXT_CTRL_URG; 3789 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 3790 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3791 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 3792 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3793 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 3794 imir_ext |= E1000_IMIREXT_CTRL_RST; 3795 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 3796 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3797 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 3798 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3799 } else { 3800 imir_ext |= E1000_IMIREXT_CTRL_BP; 3801 } 3802 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3803 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3804 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3805 } 3806 3807 /* 3808 * igb_add_2tuple_filter - add a 2tuple filter 3809 * 3810 * @param 3811 * dev: Pointer to struct rte_eth_dev. 3812 * ntuple_filter: ponter to the filter that will be added. 3813 * 3814 * @return 3815 * - On success, zero. 3816 * - On failure, a negative value. 3817 */ 3818 static int 3819 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3820 struct rte_eth_ntuple_filter *ntuple_filter) 3821 { 3822 struct e1000_filter_info *filter_info = 3823 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3824 struct e1000_2tuple_filter *filter; 3825 int i, ret; 3826 3827 filter = rte_zmalloc("e1000_2tuple_filter", 3828 sizeof(struct e1000_2tuple_filter), 0); 3829 if (filter == NULL) 3830 return -ENOMEM; 3831 3832 ret = ntuple_filter_to_2tuple(ntuple_filter, 3833 &filter->filter_info); 3834 if (ret < 0) { 3835 rte_free(filter); 3836 return ret; 3837 } 3838 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3839 &filter->filter_info) != NULL) { 3840 PMD_DRV_LOG(ERR, "filter exists."); 3841 rte_free(filter); 3842 return -EEXIST; 3843 } 3844 filter->queue = ntuple_filter->queue; 3845 3846 /* 3847 * look for an unused 2tuple filter index, 3848 * and insert the filter to list. 3849 */ 3850 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3851 if (!(filter_info->twotuple_mask & (1 << i))) { 3852 filter_info->twotuple_mask |= 1 << i; 3853 filter->index = i; 3854 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3855 filter, 3856 entries); 3857 break; 3858 } 3859 } 3860 if (i >= E1000_MAX_TTQF_FILTERS) { 3861 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3862 rte_free(filter); 3863 return -ENOSYS; 3864 } 3865 3866 igb_inject_2uple_filter(dev, filter); 3867 return 0; 3868 } 3869 3870 int 3871 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3872 struct e1000_2tuple_filter *filter) 3873 { 3874 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3875 struct e1000_filter_info *filter_info = 3876 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3877 3878 filter_info->twotuple_mask &= ~(1 << filter->index); 3879 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3880 rte_free(filter); 3881 3882 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3883 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3884 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3885 return 0; 3886 } 3887 3888 /* 3889 * igb_remove_2tuple_filter - remove a 2tuple filter 3890 * 3891 * @param 3892 * dev: Pointer to struct rte_eth_dev. 3893 * ntuple_filter: ponter to the filter that will be removed. 3894 * 3895 * @return 3896 * - On success, zero. 3897 * - On failure, a negative value. 3898 */ 3899 static int 3900 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3901 struct rte_eth_ntuple_filter *ntuple_filter) 3902 { 3903 struct e1000_filter_info *filter_info = 3904 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3905 struct e1000_2tuple_filter_info filter_2tuple; 3906 struct e1000_2tuple_filter *filter; 3907 int ret; 3908 3909 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3910 ret = ntuple_filter_to_2tuple(ntuple_filter, 3911 &filter_2tuple); 3912 if (ret < 0) 3913 return ret; 3914 3915 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3916 &filter_2tuple); 3917 if (filter == NULL) { 3918 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3919 return -ENOENT; 3920 } 3921 3922 igb_delete_2tuple_filter(dev, filter); 3923 3924 return 0; 3925 } 3926 3927 /* inject a igb flex filter to HW */ 3928 static inline void 3929 igb_inject_flex_filter(struct rte_eth_dev *dev, 3930 struct e1000_flex_filter *filter) 3931 { 3932 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3933 uint32_t wufc, queueing; 3934 uint32_t reg_off; 3935 uint8_t i, j = 0; 3936 3937 wufc = E1000_READ_REG(hw, E1000_WUFC); 3938 if (filter->index < E1000_MAX_FHFT) 3939 reg_off = E1000_FHFT(filter->index); 3940 else 3941 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3942 3943 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3944 (E1000_WUFC_FLX0 << filter->index)); 3945 queueing = filter->filter_info.len | 3946 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3947 (filter->filter_info.priority << 3948 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3949 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3950 queueing); 3951 3952 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3953 E1000_WRITE_REG(hw, reg_off, 3954 filter->filter_info.dwords[j]); 3955 reg_off += sizeof(uint32_t); 3956 E1000_WRITE_REG(hw, reg_off, 3957 filter->filter_info.dwords[++j]); 3958 reg_off += sizeof(uint32_t); 3959 E1000_WRITE_REG(hw, reg_off, 3960 (uint32_t)filter->filter_info.mask[i]); 3961 reg_off += sizeof(uint32_t) * 2; 3962 ++j; 3963 } 3964 } 3965 3966 static inline struct e1000_flex_filter * 3967 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3968 struct e1000_flex_filter_info *key) 3969 { 3970 struct e1000_flex_filter *it; 3971 3972 TAILQ_FOREACH(it, filter_list, entries) { 3973 if (memcmp(key, &it->filter_info, 3974 sizeof(struct e1000_flex_filter_info)) == 0) 3975 return it; 3976 } 3977 3978 return NULL; 3979 } 3980 3981 /* remove a flex byte filter 3982 * @param 3983 * dev: Pointer to struct rte_eth_dev. 3984 * filter: the pointer of the filter will be removed. 3985 */ 3986 void 3987 igb_remove_flex_filter(struct rte_eth_dev *dev, 3988 struct e1000_flex_filter *filter) 3989 { 3990 struct e1000_filter_info *filter_info = 3991 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3992 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3993 uint32_t wufc, i; 3994 uint32_t reg_off; 3995 3996 wufc = E1000_READ_REG(hw, E1000_WUFC); 3997 if (filter->index < E1000_MAX_FHFT) 3998 reg_off = E1000_FHFT(filter->index); 3999 else 4000 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4001 4002 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4003 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4004 4005 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4006 (~(E1000_WUFC_FLX0 << filter->index))); 4007 4008 filter_info->flex_mask &= ~(1 << filter->index); 4009 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4010 rte_free(filter); 4011 } 4012 4013 int 4014 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4015 struct rte_eth_flex_filter *filter, 4016 bool add) 4017 { 4018 struct e1000_filter_info *filter_info = 4019 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4020 struct e1000_flex_filter *flex_filter, *it; 4021 uint32_t mask; 4022 uint8_t shift, i; 4023 4024 flex_filter = rte_zmalloc("e1000_flex_filter", 4025 sizeof(struct e1000_flex_filter), 0); 4026 if (flex_filter == NULL) 4027 return -ENOMEM; 4028 4029 flex_filter->filter_info.len = filter->len; 4030 flex_filter->filter_info.priority = filter->priority; 4031 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4032 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4033 mask = 0; 4034 /* reverse bits in flex filter's mask*/ 4035 for (shift = 0; shift < CHAR_BIT; shift++) { 4036 if (filter->mask[i] & (0x01 << shift)) 4037 mask |= (0x80 >> shift); 4038 } 4039 flex_filter->filter_info.mask[i] = mask; 4040 } 4041 4042 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4043 &flex_filter->filter_info); 4044 if (it == NULL && !add) { 4045 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4046 rte_free(flex_filter); 4047 return -ENOENT; 4048 } 4049 if (it != NULL && add) { 4050 PMD_DRV_LOG(ERR, "filter exists."); 4051 rte_free(flex_filter); 4052 return -EEXIST; 4053 } 4054 4055 if (add) { 4056 flex_filter->queue = filter->queue; 4057 /* 4058 * look for an unused flex filter index 4059 * and insert the filter into the list. 4060 */ 4061 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4062 if (!(filter_info->flex_mask & (1 << i))) { 4063 filter_info->flex_mask |= 1 << i; 4064 flex_filter->index = i; 4065 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4066 flex_filter, 4067 entries); 4068 break; 4069 } 4070 } 4071 if (i >= E1000_MAX_FLEX_FILTERS) { 4072 PMD_DRV_LOG(ERR, "flex filters are full."); 4073 rte_free(flex_filter); 4074 return -ENOSYS; 4075 } 4076 4077 igb_inject_flex_filter(dev, flex_filter); 4078 4079 } else { 4080 igb_remove_flex_filter(dev, it); 4081 rte_free(flex_filter); 4082 } 4083 4084 return 0; 4085 } 4086 4087 static int 4088 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4089 struct rte_eth_flex_filter *filter) 4090 { 4091 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4092 struct e1000_filter_info *filter_info = 4093 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4094 struct e1000_flex_filter flex_filter, *it; 4095 uint32_t wufc, queueing, wufc_en = 0; 4096 4097 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4098 flex_filter.filter_info.len = filter->len; 4099 flex_filter.filter_info.priority = filter->priority; 4100 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4101 memcpy(flex_filter.filter_info.mask, filter->mask, 4102 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4103 4104 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4105 &flex_filter.filter_info); 4106 if (it == NULL) { 4107 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4108 return -ENOENT; 4109 } 4110 4111 wufc = E1000_READ_REG(hw, E1000_WUFC); 4112 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4113 4114 if ((wufc & wufc_en) == wufc_en) { 4115 uint32_t reg_off = 0; 4116 if (it->index < E1000_MAX_FHFT) 4117 reg_off = E1000_FHFT(it->index); 4118 else 4119 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4120 4121 queueing = E1000_READ_REG(hw, 4122 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4123 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4124 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4125 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4126 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4127 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4128 return 0; 4129 } 4130 return -ENOENT; 4131 } 4132 4133 static int 4134 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4135 enum rte_filter_op filter_op, 4136 void *arg) 4137 { 4138 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4139 struct rte_eth_flex_filter *filter; 4140 int ret = 0; 4141 4142 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4143 4144 if (filter_op == RTE_ETH_FILTER_NOP) 4145 return ret; 4146 4147 if (arg == NULL) { 4148 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4149 filter_op); 4150 return -EINVAL; 4151 } 4152 4153 filter = (struct rte_eth_flex_filter *)arg; 4154 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4155 || filter->len % sizeof(uint64_t) != 0) { 4156 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4157 return -EINVAL; 4158 } 4159 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4160 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4161 return -EINVAL; 4162 } 4163 4164 switch (filter_op) { 4165 case RTE_ETH_FILTER_ADD: 4166 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4167 break; 4168 case RTE_ETH_FILTER_DELETE: 4169 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4170 break; 4171 case RTE_ETH_FILTER_GET: 4172 ret = eth_igb_get_flex_filter(dev, filter); 4173 break; 4174 default: 4175 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4176 ret = -EINVAL; 4177 break; 4178 } 4179 4180 return ret; 4181 } 4182 4183 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4184 static inline int 4185 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4186 struct e1000_5tuple_filter_info *filter_info) 4187 { 4188 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4189 return -EINVAL; 4190 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4191 return -EINVAL; /* filter index is out of range. */ 4192 if (filter->tcp_flags > TCP_FLAG_ALL) 4193 return -EINVAL; /* flags is invalid. */ 4194 4195 switch (filter->dst_ip_mask) { 4196 case UINT32_MAX: 4197 filter_info->dst_ip_mask = 0; 4198 filter_info->dst_ip = filter->dst_ip; 4199 break; 4200 case 0: 4201 filter_info->dst_ip_mask = 1; 4202 break; 4203 default: 4204 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4205 return -EINVAL; 4206 } 4207 4208 switch (filter->src_ip_mask) { 4209 case UINT32_MAX: 4210 filter_info->src_ip_mask = 0; 4211 filter_info->src_ip = filter->src_ip; 4212 break; 4213 case 0: 4214 filter_info->src_ip_mask = 1; 4215 break; 4216 default: 4217 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4218 return -EINVAL; 4219 } 4220 4221 switch (filter->dst_port_mask) { 4222 case UINT16_MAX: 4223 filter_info->dst_port_mask = 0; 4224 filter_info->dst_port = filter->dst_port; 4225 break; 4226 case 0: 4227 filter_info->dst_port_mask = 1; 4228 break; 4229 default: 4230 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4231 return -EINVAL; 4232 } 4233 4234 switch (filter->src_port_mask) { 4235 case UINT16_MAX: 4236 filter_info->src_port_mask = 0; 4237 filter_info->src_port = filter->src_port; 4238 break; 4239 case 0: 4240 filter_info->src_port_mask = 1; 4241 break; 4242 default: 4243 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4244 return -EINVAL; 4245 } 4246 4247 switch (filter->proto_mask) { 4248 case UINT8_MAX: 4249 filter_info->proto_mask = 0; 4250 filter_info->proto = filter->proto; 4251 break; 4252 case 0: 4253 filter_info->proto_mask = 1; 4254 break; 4255 default: 4256 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4257 return -EINVAL; 4258 } 4259 4260 filter_info->priority = (uint8_t)filter->priority; 4261 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4262 filter_info->tcp_flags = filter->tcp_flags; 4263 else 4264 filter_info->tcp_flags = 0; 4265 4266 return 0; 4267 } 4268 4269 static inline struct e1000_5tuple_filter * 4270 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4271 struct e1000_5tuple_filter_info *key) 4272 { 4273 struct e1000_5tuple_filter *it; 4274 4275 TAILQ_FOREACH(it, filter_list, entries) { 4276 if (memcmp(key, &it->filter_info, 4277 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4278 return it; 4279 } 4280 } 4281 return NULL; 4282 } 4283 4284 /* inject a igb 5-tuple filter to HW */ 4285 static inline void 4286 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4287 struct e1000_5tuple_filter *filter) 4288 { 4289 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4290 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4291 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4292 uint8_t i; 4293 4294 i = filter->index; 4295 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4296 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4297 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4298 if (filter->filter_info.dst_ip_mask == 0) 4299 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4300 if (filter->filter_info.src_port_mask == 0) 4301 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4302 if (filter->filter_info.proto_mask == 0) 4303 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4304 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4305 E1000_FTQF_QUEUE_MASK; 4306 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4307 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4308 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4309 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4310 4311 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4312 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4313 4314 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4315 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4316 imir |= E1000_IMIR_PORT_BP; 4317 else 4318 imir &= ~E1000_IMIR_PORT_BP; 4319 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4320 4321 /* tcp flags bits setting. */ 4322 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 4323 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 4324 imir_ext |= E1000_IMIREXT_CTRL_URG; 4325 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 4326 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4327 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 4328 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4329 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 4330 imir_ext |= E1000_IMIREXT_CTRL_RST; 4331 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 4332 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4333 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 4334 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4335 } else { 4336 imir_ext |= E1000_IMIREXT_CTRL_BP; 4337 } 4338 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4339 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4340 } 4341 4342 /* 4343 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4344 * 4345 * @param 4346 * dev: Pointer to struct rte_eth_dev. 4347 * ntuple_filter: ponter to the filter that will be added. 4348 * 4349 * @return 4350 * - On success, zero. 4351 * - On failure, a negative value. 4352 */ 4353 static int 4354 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4355 struct rte_eth_ntuple_filter *ntuple_filter) 4356 { 4357 struct e1000_filter_info *filter_info = 4358 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4359 struct e1000_5tuple_filter *filter; 4360 uint8_t i; 4361 int ret; 4362 4363 filter = rte_zmalloc("e1000_5tuple_filter", 4364 sizeof(struct e1000_5tuple_filter), 0); 4365 if (filter == NULL) 4366 return -ENOMEM; 4367 4368 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4369 &filter->filter_info); 4370 if (ret < 0) { 4371 rte_free(filter); 4372 return ret; 4373 } 4374 4375 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4376 &filter->filter_info) != NULL) { 4377 PMD_DRV_LOG(ERR, "filter exists."); 4378 rte_free(filter); 4379 return -EEXIST; 4380 } 4381 filter->queue = ntuple_filter->queue; 4382 4383 /* 4384 * look for an unused 5tuple filter index, 4385 * and insert the filter to list. 4386 */ 4387 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4388 if (!(filter_info->fivetuple_mask & (1 << i))) { 4389 filter_info->fivetuple_mask |= 1 << i; 4390 filter->index = i; 4391 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4392 filter, 4393 entries); 4394 break; 4395 } 4396 } 4397 if (i >= E1000_MAX_FTQF_FILTERS) { 4398 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4399 rte_free(filter); 4400 return -ENOSYS; 4401 } 4402 4403 igb_inject_5tuple_filter_82576(dev, filter); 4404 return 0; 4405 } 4406 4407 int 4408 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4409 struct e1000_5tuple_filter *filter) 4410 { 4411 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4412 struct e1000_filter_info *filter_info = 4413 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4414 4415 filter_info->fivetuple_mask &= ~(1 << filter->index); 4416 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4417 rte_free(filter); 4418 4419 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4420 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4421 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4422 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4423 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4424 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4425 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4426 return 0; 4427 } 4428 4429 /* 4430 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4431 * 4432 * @param 4433 * dev: Pointer to struct rte_eth_dev. 4434 * ntuple_filter: ponter to the filter that will be removed. 4435 * 4436 * @return 4437 * - On success, zero. 4438 * - On failure, a negative value. 4439 */ 4440 static int 4441 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4442 struct rte_eth_ntuple_filter *ntuple_filter) 4443 { 4444 struct e1000_filter_info *filter_info = 4445 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4446 struct e1000_5tuple_filter_info filter_5tuple; 4447 struct e1000_5tuple_filter *filter; 4448 int ret; 4449 4450 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4451 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4452 &filter_5tuple); 4453 if (ret < 0) 4454 return ret; 4455 4456 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4457 &filter_5tuple); 4458 if (filter == NULL) { 4459 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4460 return -ENOENT; 4461 } 4462 4463 igb_delete_5tuple_filter_82576(dev, filter); 4464 4465 return 0; 4466 } 4467 4468 static int 4469 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4470 { 4471 uint32_t rctl; 4472 struct e1000_hw *hw; 4473 struct rte_eth_dev_info dev_info; 4474 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4475 4476 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4477 4478 #ifdef RTE_LIBRTE_82571_SUPPORT 4479 /* XXX: not bigger than max_rx_pktlen */ 4480 if (hw->mac.type == e1000_82571) 4481 return -ENOTSUP; 4482 #endif 4483 eth_igb_infos_get(dev, &dev_info); 4484 4485 /* check that mtu is within the allowed range */ 4486 if ((mtu < ETHER_MIN_MTU) || 4487 (frame_size > dev_info.max_rx_pktlen)) 4488 return -EINVAL; 4489 4490 /* refuse mtu that requires the support of scattered packets when this 4491 * feature has not been enabled before. */ 4492 if (!dev->data->scattered_rx && 4493 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4494 return -EINVAL; 4495 4496 rctl = E1000_READ_REG(hw, E1000_RCTL); 4497 4498 /* switch to jumbo mode if needed */ 4499 if (frame_size > ETHER_MAX_LEN) { 4500 dev->data->dev_conf.rxmode.offloads |= 4501 DEV_RX_OFFLOAD_JUMBO_FRAME; 4502 rctl |= E1000_RCTL_LPE; 4503 } else { 4504 dev->data->dev_conf.rxmode.offloads &= 4505 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4506 rctl &= ~E1000_RCTL_LPE; 4507 } 4508 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4509 4510 /* update max frame size */ 4511 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4512 4513 E1000_WRITE_REG(hw, E1000_RLPML, 4514 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4515 4516 return 0; 4517 } 4518 4519 /* 4520 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4521 * 4522 * @param 4523 * dev: Pointer to struct rte_eth_dev. 4524 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4525 * add: if true, add filter, if false, remove filter 4526 * 4527 * @return 4528 * - On success, zero. 4529 * - On failure, a negative value. 4530 */ 4531 int 4532 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4533 struct rte_eth_ntuple_filter *ntuple_filter, 4534 bool add) 4535 { 4536 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4537 int ret; 4538 4539 switch (ntuple_filter->flags) { 4540 case RTE_5TUPLE_FLAGS: 4541 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4542 if (hw->mac.type != e1000_82576) 4543 return -ENOTSUP; 4544 if (add) 4545 ret = igb_add_5tuple_filter_82576(dev, 4546 ntuple_filter); 4547 else 4548 ret = igb_remove_5tuple_filter_82576(dev, 4549 ntuple_filter); 4550 break; 4551 case RTE_2TUPLE_FLAGS: 4552 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4553 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4554 hw->mac.type != e1000_i210 && 4555 hw->mac.type != e1000_i211) 4556 return -ENOTSUP; 4557 if (add) 4558 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4559 else 4560 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4561 break; 4562 default: 4563 ret = -EINVAL; 4564 break; 4565 } 4566 4567 return ret; 4568 } 4569 4570 /* 4571 * igb_get_ntuple_filter - get a ntuple filter 4572 * 4573 * @param 4574 * dev: Pointer to struct rte_eth_dev. 4575 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4576 * 4577 * @return 4578 * - On success, zero. 4579 * - On failure, a negative value. 4580 */ 4581 static int 4582 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4583 struct rte_eth_ntuple_filter *ntuple_filter) 4584 { 4585 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4586 struct e1000_filter_info *filter_info = 4587 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4588 struct e1000_5tuple_filter_info filter_5tuple; 4589 struct e1000_2tuple_filter_info filter_2tuple; 4590 struct e1000_5tuple_filter *p_5tuple_filter; 4591 struct e1000_2tuple_filter *p_2tuple_filter; 4592 int ret; 4593 4594 switch (ntuple_filter->flags) { 4595 case RTE_5TUPLE_FLAGS: 4596 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4597 if (hw->mac.type != e1000_82576) 4598 return -ENOTSUP; 4599 memset(&filter_5tuple, 4600 0, 4601 sizeof(struct e1000_5tuple_filter_info)); 4602 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4603 &filter_5tuple); 4604 if (ret < 0) 4605 return ret; 4606 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4607 &filter_info->fivetuple_list, 4608 &filter_5tuple); 4609 if (p_5tuple_filter == NULL) { 4610 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4611 return -ENOENT; 4612 } 4613 ntuple_filter->queue = p_5tuple_filter->queue; 4614 break; 4615 case RTE_2TUPLE_FLAGS: 4616 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4617 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4618 return -ENOTSUP; 4619 memset(&filter_2tuple, 4620 0, 4621 sizeof(struct e1000_2tuple_filter_info)); 4622 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4623 if (ret < 0) 4624 return ret; 4625 p_2tuple_filter = igb_2tuple_filter_lookup( 4626 &filter_info->twotuple_list, 4627 &filter_2tuple); 4628 if (p_2tuple_filter == NULL) { 4629 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4630 return -ENOENT; 4631 } 4632 ntuple_filter->queue = p_2tuple_filter->queue; 4633 break; 4634 default: 4635 ret = -EINVAL; 4636 break; 4637 } 4638 4639 return 0; 4640 } 4641 4642 /* 4643 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4644 * @dev: pointer to rte_eth_dev structure 4645 * @filter_op:operation will be taken. 4646 * @arg: a pointer to specific structure corresponding to the filter_op 4647 */ 4648 static int 4649 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4650 enum rte_filter_op filter_op, 4651 void *arg) 4652 { 4653 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4654 int ret; 4655 4656 MAC_TYPE_FILTER_SUP(hw->mac.type); 4657 4658 if (filter_op == RTE_ETH_FILTER_NOP) 4659 return 0; 4660 4661 if (arg == NULL) { 4662 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4663 filter_op); 4664 return -EINVAL; 4665 } 4666 4667 switch (filter_op) { 4668 case RTE_ETH_FILTER_ADD: 4669 ret = igb_add_del_ntuple_filter(dev, 4670 (struct rte_eth_ntuple_filter *)arg, 4671 TRUE); 4672 break; 4673 case RTE_ETH_FILTER_DELETE: 4674 ret = igb_add_del_ntuple_filter(dev, 4675 (struct rte_eth_ntuple_filter *)arg, 4676 FALSE); 4677 break; 4678 case RTE_ETH_FILTER_GET: 4679 ret = igb_get_ntuple_filter(dev, 4680 (struct rte_eth_ntuple_filter *)arg); 4681 break; 4682 default: 4683 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4684 ret = -EINVAL; 4685 break; 4686 } 4687 return ret; 4688 } 4689 4690 static inline int 4691 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4692 uint16_t ethertype) 4693 { 4694 int i; 4695 4696 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4697 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4698 (filter_info->ethertype_mask & (1 << i))) 4699 return i; 4700 } 4701 return -1; 4702 } 4703 4704 static inline int 4705 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4706 uint16_t ethertype, uint32_t etqf) 4707 { 4708 int i; 4709 4710 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4711 if (!(filter_info->ethertype_mask & (1 << i))) { 4712 filter_info->ethertype_mask |= 1 << i; 4713 filter_info->ethertype_filters[i].ethertype = ethertype; 4714 filter_info->ethertype_filters[i].etqf = etqf; 4715 return i; 4716 } 4717 } 4718 return -1; 4719 } 4720 4721 int 4722 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4723 uint8_t idx) 4724 { 4725 if (idx >= E1000_MAX_ETQF_FILTERS) 4726 return -1; 4727 filter_info->ethertype_mask &= ~(1 << idx); 4728 filter_info->ethertype_filters[idx].ethertype = 0; 4729 filter_info->ethertype_filters[idx].etqf = 0; 4730 return idx; 4731 } 4732 4733 4734 int 4735 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4736 struct rte_eth_ethertype_filter *filter, 4737 bool add) 4738 { 4739 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4740 struct e1000_filter_info *filter_info = 4741 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4742 uint32_t etqf = 0; 4743 int ret; 4744 4745 if (filter->ether_type == ETHER_TYPE_IPv4 || 4746 filter->ether_type == ETHER_TYPE_IPv6) { 4747 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4748 " ethertype filter.", filter->ether_type); 4749 return -EINVAL; 4750 } 4751 4752 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4753 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4754 return -EINVAL; 4755 } 4756 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4757 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4758 return -EINVAL; 4759 } 4760 4761 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4762 if (ret >= 0 && add) { 4763 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4764 filter->ether_type); 4765 return -EEXIST; 4766 } 4767 if (ret < 0 && !add) { 4768 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4769 filter->ether_type); 4770 return -ENOENT; 4771 } 4772 4773 if (add) { 4774 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4775 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4776 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4777 ret = igb_ethertype_filter_insert(filter_info, 4778 filter->ether_type, etqf); 4779 if (ret < 0) { 4780 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4781 return -ENOSYS; 4782 } 4783 } else { 4784 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4785 if (ret < 0) 4786 return -ENOSYS; 4787 } 4788 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4789 E1000_WRITE_FLUSH(hw); 4790 4791 return 0; 4792 } 4793 4794 static int 4795 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4796 struct rte_eth_ethertype_filter *filter) 4797 { 4798 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4799 struct e1000_filter_info *filter_info = 4800 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4801 uint32_t etqf; 4802 int ret; 4803 4804 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4805 if (ret < 0) { 4806 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4807 filter->ether_type); 4808 return -ENOENT; 4809 } 4810 4811 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4812 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4813 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4814 filter->flags = 0; 4815 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4816 E1000_ETQF_QUEUE_SHIFT; 4817 return 0; 4818 } 4819 4820 return -ENOENT; 4821 } 4822 4823 /* 4824 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4825 * @dev: pointer to rte_eth_dev structure 4826 * @filter_op:operation will be taken. 4827 * @arg: a pointer to specific structure corresponding to the filter_op 4828 */ 4829 static int 4830 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4831 enum rte_filter_op filter_op, 4832 void *arg) 4833 { 4834 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4835 int ret; 4836 4837 MAC_TYPE_FILTER_SUP(hw->mac.type); 4838 4839 if (filter_op == RTE_ETH_FILTER_NOP) 4840 return 0; 4841 4842 if (arg == NULL) { 4843 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4844 filter_op); 4845 return -EINVAL; 4846 } 4847 4848 switch (filter_op) { 4849 case RTE_ETH_FILTER_ADD: 4850 ret = igb_add_del_ethertype_filter(dev, 4851 (struct rte_eth_ethertype_filter *)arg, 4852 TRUE); 4853 break; 4854 case RTE_ETH_FILTER_DELETE: 4855 ret = igb_add_del_ethertype_filter(dev, 4856 (struct rte_eth_ethertype_filter *)arg, 4857 FALSE); 4858 break; 4859 case RTE_ETH_FILTER_GET: 4860 ret = igb_get_ethertype_filter(dev, 4861 (struct rte_eth_ethertype_filter *)arg); 4862 break; 4863 default: 4864 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4865 ret = -EINVAL; 4866 break; 4867 } 4868 return ret; 4869 } 4870 4871 static int 4872 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4873 enum rte_filter_type filter_type, 4874 enum rte_filter_op filter_op, 4875 void *arg) 4876 { 4877 int ret = 0; 4878 4879 switch (filter_type) { 4880 case RTE_ETH_FILTER_NTUPLE: 4881 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4882 break; 4883 case RTE_ETH_FILTER_ETHERTYPE: 4884 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4885 break; 4886 case RTE_ETH_FILTER_SYN: 4887 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4888 break; 4889 case RTE_ETH_FILTER_FLEXIBLE: 4890 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4891 break; 4892 case RTE_ETH_FILTER_GENERIC: 4893 if (filter_op != RTE_ETH_FILTER_GET) 4894 return -EINVAL; 4895 *(const void **)arg = &igb_flow_ops; 4896 break; 4897 default: 4898 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4899 filter_type); 4900 break; 4901 } 4902 4903 return ret; 4904 } 4905 4906 static int 4907 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4908 struct ether_addr *mc_addr_set, 4909 uint32_t nb_mc_addr) 4910 { 4911 struct e1000_hw *hw; 4912 4913 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4914 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4915 return 0; 4916 } 4917 4918 static uint64_t 4919 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4920 { 4921 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4922 uint64_t systime_cycles; 4923 4924 switch (hw->mac.type) { 4925 case e1000_i210: 4926 case e1000_i211: 4927 /* 4928 * Need to read System Time Residue Register to be able 4929 * to read the other two registers. 4930 */ 4931 E1000_READ_REG(hw, E1000_SYSTIMR); 4932 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4933 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4934 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4935 * NSEC_PER_SEC; 4936 break; 4937 case e1000_82580: 4938 case e1000_i350: 4939 case e1000_i354: 4940 /* 4941 * Need to read System Time Residue Register to be able 4942 * to read the other two registers. 4943 */ 4944 E1000_READ_REG(hw, E1000_SYSTIMR); 4945 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4946 /* Only the 8 LSB are valid. */ 4947 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4948 & 0xff) << 32; 4949 break; 4950 default: 4951 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4952 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4953 << 32; 4954 break; 4955 } 4956 4957 return systime_cycles; 4958 } 4959 4960 static uint64_t 4961 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4962 { 4963 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4964 uint64_t rx_tstamp_cycles; 4965 4966 switch (hw->mac.type) { 4967 case e1000_i210: 4968 case e1000_i211: 4969 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4970 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4971 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4972 * NSEC_PER_SEC; 4973 break; 4974 case e1000_82580: 4975 case e1000_i350: 4976 case e1000_i354: 4977 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4978 /* Only the 8 LSB are valid. */ 4979 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4980 & 0xff) << 32; 4981 break; 4982 default: 4983 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4984 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4985 << 32; 4986 break; 4987 } 4988 4989 return rx_tstamp_cycles; 4990 } 4991 4992 static uint64_t 4993 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4994 { 4995 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4996 uint64_t tx_tstamp_cycles; 4997 4998 switch (hw->mac.type) { 4999 case e1000_i210: 5000 case e1000_i211: 5001 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5002 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5003 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5004 * NSEC_PER_SEC; 5005 break; 5006 case e1000_82580: 5007 case e1000_i350: 5008 case e1000_i354: 5009 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5010 /* Only the 8 LSB are valid. */ 5011 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5012 & 0xff) << 32; 5013 break; 5014 default: 5015 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5016 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5017 << 32; 5018 break; 5019 } 5020 5021 return tx_tstamp_cycles; 5022 } 5023 5024 static void 5025 igb_start_timecounters(struct rte_eth_dev *dev) 5026 { 5027 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5028 struct e1000_adapter *adapter = 5029 (struct e1000_adapter *)dev->data->dev_private; 5030 uint32_t incval = 1; 5031 uint32_t shift = 0; 5032 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5033 5034 switch (hw->mac.type) { 5035 case e1000_82580: 5036 case e1000_i350: 5037 case e1000_i354: 5038 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5039 mask = (1ULL << 40) - 1; 5040 /* fall-through */ 5041 case e1000_i210: 5042 case e1000_i211: 5043 /* 5044 * Start incrementing the register 5045 * used to timestamp PTP packets. 5046 */ 5047 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5048 break; 5049 case e1000_82576: 5050 incval = E1000_INCVALUE_82576; 5051 shift = IGB_82576_TSYNC_SHIFT; 5052 E1000_WRITE_REG(hw, E1000_TIMINCA, 5053 E1000_INCPERIOD_82576 | incval); 5054 break; 5055 default: 5056 /* Not supported */ 5057 return; 5058 } 5059 5060 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5061 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5062 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5063 5064 adapter->systime_tc.cc_mask = mask; 5065 adapter->systime_tc.cc_shift = shift; 5066 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5067 5068 adapter->rx_tstamp_tc.cc_mask = mask; 5069 adapter->rx_tstamp_tc.cc_shift = shift; 5070 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5071 5072 adapter->tx_tstamp_tc.cc_mask = mask; 5073 adapter->tx_tstamp_tc.cc_shift = shift; 5074 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5075 } 5076 5077 static int 5078 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5079 { 5080 struct e1000_adapter *adapter = 5081 (struct e1000_adapter *)dev->data->dev_private; 5082 5083 adapter->systime_tc.nsec += delta; 5084 adapter->rx_tstamp_tc.nsec += delta; 5085 adapter->tx_tstamp_tc.nsec += delta; 5086 5087 return 0; 5088 } 5089 5090 static int 5091 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5092 { 5093 uint64_t ns; 5094 struct e1000_adapter *adapter = 5095 (struct e1000_adapter *)dev->data->dev_private; 5096 5097 ns = rte_timespec_to_ns(ts); 5098 5099 /* Set the timecounters to a new value. */ 5100 adapter->systime_tc.nsec = ns; 5101 adapter->rx_tstamp_tc.nsec = ns; 5102 adapter->tx_tstamp_tc.nsec = ns; 5103 5104 return 0; 5105 } 5106 5107 static int 5108 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5109 { 5110 uint64_t ns, systime_cycles; 5111 struct e1000_adapter *adapter = 5112 (struct e1000_adapter *)dev->data->dev_private; 5113 5114 systime_cycles = igb_read_systime_cyclecounter(dev); 5115 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5116 *ts = rte_ns_to_timespec(ns); 5117 5118 return 0; 5119 } 5120 5121 static int 5122 igb_timesync_enable(struct rte_eth_dev *dev) 5123 { 5124 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5125 uint32_t tsync_ctl; 5126 uint32_t tsauxc; 5127 5128 /* Stop the timesync system time. */ 5129 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5130 /* Reset the timesync system time value. */ 5131 switch (hw->mac.type) { 5132 case e1000_82580: 5133 case e1000_i350: 5134 case e1000_i354: 5135 case e1000_i210: 5136 case e1000_i211: 5137 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5138 /* fall-through */ 5139 case e1000_82576: 5140 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5141 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5142 break; 5143 default: 5144 /* Not supported. */ 5145 return -ENOTSUP; 5146 } 5147 5148 /* Enable system time for it isn't on by default. */ 5149 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5150 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5151 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5152 5153 igb_start_timecounters(dev); 5154 5155 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5156 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5157 (ETHER_TYPE_1588 | 5158 E1000_ETQF_FILTER_ENABLE | 5159 E1000_ETQF_1588)); 5160 5161 /* Enable timestamping of received PTP packets. */ 5162 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5163 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5164 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5165 5166 /* Enable Timestamping of transmitted PTP packets. */ 5167 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5168 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5169 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5170 5171 return 0; 5172 } 5173 5174 static int 5175 igb_timesync_disable(struct rte_eth_dev *dev) 5176 { 5177 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5178 uint32_t tsync_ctl; 5179 5180 /* Disable timestamping of transmitted PTP packets. */ 5181 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5182 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5183 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5184 5185 /* Disable timestamping of received PTP packets. */ 5186 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5187 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5188 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5189 5190 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5191 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5192 5193 /* Stop incrementating the System Time registers. */ 5194 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5195 5196 return 0; 5197 } 5198 5199 static int 5200 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5201 struct timespec *timestamp, 5202 uint32_t flags __rte_unused) 5203 { 5204 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5205 struct e1000_adapter *adapter = 5206 (struct e1000_adapter *)dev->data->dev_private; 5207 uint32_t tsync_rxctl; 5208 uint64_t rx_tstamp_cycles; 5209 uint64_t ns; 5210 5211 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5212 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5213 return -EINVAL; 5214 5215 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5216 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5217 *timestamp = rte_ns_to_timespec(ns); 5218 5219 return 0; 5220 } 5221 5222 static int 5223 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5224 struct timespec *timestamp) 5225 { 5226 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5227 struct e1000_adapter *adapter = 5228 (struct e1000_adapter *)dev->data->dev_private; 5229 uint32_t tsync_txctl; 5230 uint64_t tx_tstamp_cycles; 5231 uint64_t ns; 5232 5233 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5234 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5235 return -EINVAL; 5236 5237 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5238 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5239 *timestamp = rte_ns_to_timespec(ns); 5240 5241 return 0; 5242 } 5243 5244 static int 5245 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5246 { 5247 int count = 0; 5248 int g_ind = 0; 5249 const struct reg_info *reg_group; 5250 5251 while ((reg_group = igb_regs[g_ind++])) 5252 count += igb_reg_group_count(reg_group); 5253 5254 return count; 5255 } 5256 5257 static int 5258 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5259 { 5260 int count = 0; 5261 int g_ind = 0; 5262 const struct reg_info *reg_group; 5263 5264 while ((reg_group = igbvf_regs[g_ind++])) 5265 count += igb_reg_group_count(reg_group); 5266 5267 return count; 5268 } 5269 5270 static int 5271 eth_igb_get_regs(struct rte_eth_dev *dev, 5272 struct rte_dev_reg_info *regs) 5273 { 5274 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5275 uint32_t *data = regs->data; 5276 int g_ind = 0; 5277 int count = 0; 5278 const struct reg_info *reg_group; 5279 5280 if (data == NULL) { 5281 regs->length = eth_igb_get_reg_length(dev); 5282 regs->width = sizeof(uint32_t); 5283 return 0; 5284 } 5285 5286 /* Support only full register dump */ 5287 if ((regs->length == 0) || 5288 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5289 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5290 hw->device_id; 5291 while ((reg_group = igb_regs[g_ind++])) 5292 count += igb_read_regs_group(dev, &data[count], 5293 reg_group); 5294 return 0; 5295 } 5296 5297 return -ENOTSUP; 5298 } 5299 5300 static int 5301 igbvf_get_regs(struct rte_eth_dev *dev, 5302 struct rte_dev_reg_info *regs) 5303 { 5304 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5305 uint32_t *data = regs->data; 5306 int g_ind = 0; 5307 int count = 0; 5308 const struct reg_info *reg_group; 5309 5310 if (data == NULL) { 5311 regs->length = igbvf_get_reg_length(dev); 5312 regs->width = sizeof(uint32_t); 5313 return 0; 5314 } 5315 5316 /* Support only full register dump */ 5317 if ((regs->length == 0) || 5318 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5319 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5320 hw->device_id; 5321 while ((reg_group = igbvf_regs[g_ind++])) 5322 count += igb_read_regs_group(dev, &data[count], 5323 reg_group); 5324 return 0; 5325 } 5326 5327 return -ENOTSUP; 5328 } 5329 5330 static int 5331 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5332 { 5333 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5334 5335 /* Return unit is byte count */ 5336 return hw->nvm.word_size * 2; 5337 } 5338 5339 static int 5340 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5341 struct rte_dev_eeprom_info *in_eeprom) 5342 { 5343 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5344 struct e1000_nvm_info *nvm = &hw->nvm; 5345 uint16_t *data = in_eeprom->data; 5346 int first, length; 5347 5348 first = in_eeprom->offset >> 1; 5349 length = in_eeprom->length >> 1; 5350 if ((first >= hw->nvm.word_size) || 5351 ((first + length) >= hw->nvm.word_size)) 5352 return -EINVAL; 5353 5354 in_eeprom->magic = hw->vendor_id | 5355 ((uint32_t)hw->device_id << 16); 5356 5357 if ((nvm->ops.read) == NULL) 5358 return -ENOTSUP; 5359 5360 return nvm->ops.read(hw, first, length, data); 5361 } 5362 5363 static int 5364 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5365 struct rte_dev_eeprom_info *in_eeprom) 5366 { 5367 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5368 struct e1000_nvm_info *nvm = &hw->nvm; 5369 uint16_t *data = in_eeprom->data; 5370 int first, length; 5371 5372 first = in_eeprom->offset >> 1; 5373 length = in_eeprom->length >> 1; 5374 if ((first >= hw->nvm.word_size) || 5375 ((first + length) >= hw->nvm.word_size)) 5376 return -EINVAL; 5377 5378 in_eeprom->magic = (uint32_t)hw->vendor_id | 5379 ((uint32_t)hw->device_id << 16); 5380 5381 if ((nvm->ops.write) == NULL) 5382 return -ENOTSUP; 5383 return nvm->ops.write(hw, first, length, data); 5384 } 5385 5386 static int 5387 eth_igb_get_module_info(struct rte_eth_dev *dev, 5388 struct rte_eth_dev_module_info *modinfo) 5389 { 5390 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5391 5392 uint32_t status = 0; 5393 uint16_t sff8472_rev, addr_mode; 5394 bool page_swap = false; 5395 5396 if (hw->phy.media_type == e1000_media_type_copper || 5397 hw->phy.media_type == e1000_media_type_unknown) 5398 return -EOPNOTSUPP; 5399 5400 /* Check whether we support SFF-8472 or not */ 5401 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5402 if (status) 5403 return -EIO; 5404 5405 /* addressing mode is not supported */ 5406 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5407 if (status) 5408 return -EIO; 5409 5410 /* addressing mode is not supported */ 5411 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5412 PMD_DRV_LOG(ERR, 5413 "Address change required to access page 0xA2, " 5414 "but not supported. Please report the module " 5415 "type to the driver maintainers.\n"); 5416 page_swap = true; 5417 } 5418 5419 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5420 /* We have an SFP, but it does not support SFF-8472 */ 5421 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5422 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5423 } else { 5424 /* We have an SFP which supports a revision of SFF-8472 */ 5425 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5426 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5427 } 5428 5429 return 0; 5430 } 5431 5432 static int 5433 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5434 struct rte_dev_eeprom_info *info) 5435 { 5436 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5437 5438 uint32_t status = 0; 5439 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5440 u16 first_word, last_word; 5441 int i = 0; 5442 5443 if (info->length == 0) 5444 return -EINVAL; 5445 5446 first_word = info->offset >> 1; 5447 last_word = (info->offset + info->length - 1) >> 1; 5448 5449 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5450 for (i = 0; i < last_word - first_word + 1; i++) { 5451 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5452 &dataword[i]); 5453 if (status) { 5454 /* Error occurred while reading module */ 5455 return -EIO; 5456 } 5457 5458 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5459 } 5460 5461 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5462 5463 return 0; 5464 } 5465 5466 static int 5467 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5468 { 5469 struct e1000_hw *hw = 5470 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5471 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5472 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5473 uint32_t vec = E1000_MISC_VEC_ID; 5474 5475 if (rte_intr_allow_others(intr_handle)) 5476 vec = E1000_RX_VEC_START; 5477 5478 uint32_t mask = 1 << (queue_id + vec); 5479 5480 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5481 E1000_WRITE_FLUSH(hw); 5482 5483 return 0; 5484 } 5485 5486 static int 5487 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5488 { 5489 struct e1000_hw *hw = 5490 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5491 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5492 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5493 uint32_t vec = E1000_MISC_VEC_ID; 5494 5495 if (rte_intr_allow_others(intr_handle)) 5496 vec = E1000_RX_VEC_START; 5497 5498 uint32_t mask = 1 << (queue_id + vec); 5499 uint32_t regval; 5500 5501 regval = E1000_READ_REG(hw, E1000_EIMS); 5502 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5503 E1000_WRITE_FLUSH(hw); 5504 5505 rte_intr_enable(intr_handle); 5506 5507 return 0; 5508 } 5509 5510 static void 5511 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5512 uint8_t index, uint8_t offset) 5513 { 5514 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5515 5516 /* clear bits */ 5517 val &= ~((uint32_t)0xFF << offset); 5518 5519 /* write vector and valid bit */ 5520 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5521 5522 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5523 } 5524 5525 static void 5526 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5527 uint8_t queue, uint8_t msix_vector) 5528 { 5529 uint32_t tmp = 0; 5530 5531 if (hw->mac.type == e1000_82575) { 5532 if (direction == 0) 5533 tmp = E1000_EICR_RX_QUEUE0 << queue; 5534 else if (direction == 1) 5535 tmp = E1000_EICR_TX_QUEUE0 << queue; 5536 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5537 } else if (hw->mac.type == e1000_82576) { 5538 if ((direction == 0) || (direction == 1)) 5539 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5540 ((queue & 0x8) << 1) + 5541 8 * direction); 5542 } else if ((hw->mac.type == e1000_82580) || 5543 (hw->mac.type == e1000_i350) || 5544 (hw->mac.type == e1000_i354) || 5545 (hw->mac.type == e1000_i210) || 5546 (hw->mac.type == e1000_i211)) { 5547 if ((direction == 0) || (direction == 1)) 5548 eth_igb_write_ivar(hw, msix_vector, 5549 queue >> 1, 5550 ((queue & 0x1) << 4) + 5551 8 * direction); 5552 } 5553 } 5554 5555 /* Sets up the hardware to generate MSI-X interrupts properly 5556 * @hw 5557 * board private structure 5558 */ 5559 static void 5560 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5561 { 5562 int queue_id; 5563 uint32_t tmpval, regval, intr_mask; 5564 struct e1000_hw *hw = 5565 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5566 uint32_t vec = E1000_MISC_VEC_ID; 5567 uint32_t base = E1000_MISC_VEC_ID; 5568 uint32_t misc_shift = 0; 5569 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5570 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5571 5572 /* won't configure msix register if no mapping is done 5573 * between intr vector and event fd 5574 */ 5575 if (!rte_intr_dp_is_en(intr_handle)) 5576 return; 5577 5578 if (rte_intr_allow_others(intr_handle)) { 5579 vec = base = E1000_RX_VEC_START; 5580 misc_shift = 1; 5581 } 5582 5583 /* set interrupt vector for other causes */ 5584 if (hw->mac.type == e1000_82575) { 5585 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5586 /* enable MSI-X PBA support */ 5587 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5588 5589 /* Auto-Mask interrupts upon ICR read */ 5590 tmpval |= E1000_CTRL_EXT_EIAME; 5591 tmpval |= E1000_CTRL_EXT_IRCA; 5592 5593 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5594 5595 /* enable msix_other interrupt */ 5596 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5597 regval = E1000_READ_REG(hw, E1000_EIAC); 5598 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5599 regval = E1000_READ_REG(hw, E1000_EIAM); 5600 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5601 } else if ((hw->mac.type == e1000_82576) || 5602 (hw->mac.type == e1000_82580) || 5603 (hw->mac.type == e1000_i350) || 5604 (hw->mac.type == e1000_i354) || 5605 (hw->mac.type == e1000_i210) || 5606 (hw->mac.type == e1000_i211)) { 5607 /* turn on MSI-X capability first */ 5608 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5609 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5610 E1000_GPIE_NSICR); 5611 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5612 misc_shift; 5613 5614 if (dev->data->dev_conf.intr_conf.lsc != 0) 5615 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5616 5617 regval = E1000_READ_REG(hw, E1000_EIAC); 5618 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5619 5620 /* enable msix_other interrupt */ 5621 regval = E1000_READ_REG(hw, E1000_EIMS); 5622 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5623 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5624 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5625 } 5626 5627 /* use EIAM to auto-mask when MSI-X interrupt 5628 * is asserted, this saves a register write for every interrupt 5629 */ 5630 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5631 misc_shift; 5632 5633 if (dev->data->dev_conf.intr_conf.lsc != 0) 5634 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5635 5636 regval = E1000_READ_REG(hw, E1000_EIAM); 5637 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5638 5639 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5640 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5641 intr_handle->intr_vec[queue_id] = vec; 5642 if (vec < base + intr_handle->nb_efd - 1) 5643 vec++; 5644 } 5645 5646 E1000_WRITE_FLUSH(hw); 5647 } 5648 5649 /* restore n-tuple filter */ 5650 static inline void 5651 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5652 { 5653 struct e1000_filter_info *filter_info = 5654 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5655 struct e1000_5tuple_filter *p_5tuple; 5656 struct e1000_2tuple_filter *p_2tuple; 5657 5658 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5659 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5660 } 5661 5662 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5663 igb_inject_2uple_filter(dev, p_2tuple); 5664 } 5665 } 5666 5667 /* restore SYN filter */ 5668 static inline void 5669 igb_syn_filter_restore(struct rte_eth_dev *dev) 5670 { 5671 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5672 struct e1000_filter_info *filter_info = 5673 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5674 uint32_t synqf; 5675 5676 synqf = filter_info->syn_info; 5677 5678 if (synqf & E1000_SYN_FILTER_ENABLE) { 5679 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5680 E1000_WRITE_FLUSH(hw); 5681 } 5682 } 5683 5684 /* restore ethernet type filter */ 5685 static inline void 5686 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5687 { 5688 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5689 struct e1000_filter_info *filter_info = 5690 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5691 int i; 5692 5693 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5694 if (filter_info->ethertype_mask & (1 << i)) { 5695 E1000_WRITE_REG(hw, E1000_ETQF(i), 5696 filter_info->ethertype_filters[i].etqf); 5697 E1000_WRITE_FLUSH(hw); 5698 } 5699 } 5700 } 5701 5702 /* restore flex byte filter */ 5703 static inline void 5704 igb_flex_filter_restore(struct rte_eth_dev *dev) 5705 { 5706 struct e1000_filter_info *filter_info = 5707 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5708 struct e1000_flex_filter *flex_filter; 5709 5710 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5711 igb_inject_flex_filter(dev, flex_filter); 5712 } 5713 } 5714 5715 /* restore rss filter */ 5716 static inline void 5717 igb_rss_filter_restore(struct rte_eth_dev *dev) 5718 { 5719 struct e1000_filter_info *filter_info = 5720 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5721 5722 if (filter_info->rss_info.conf.queue_num) 5723 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5724 } 5725 5726 /* restore all types filter */ 5727 static int 5728 igb_filter_restore(struct rte_eth_dev *dev) 5729 { 5730 igb_ntuple_filter_restore(dev); 5731 igb_ethertype_filter_restore(dev); 5732 igb_syn_filter_restore(dev); 5733 igb_flex_filter_restore(dev); 5734 igb_rss_filter_restore(dev); 5735 5736 return 0; 5737 } 5738 5739 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5740 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5741 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5742 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5743 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5744 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5745 5746 /* see e1000_logs.c */ 5747 RTE_INIT(e1000_init_log) 5748 { 5749 e1000_igb_init_log(); 5750 } 5751