1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_string_fns.h> 12 #include <rte_common.h> 13 #include <rte_interrupts.h> 14 #include <rte_byteorder.h> 15 #include <rte_log.h> 16 #include <rte_debug.h> 17 #include <rte_pci.h> 18 #include <rte_bus_pci.h> 19 #include <rte_ether.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_ethdev_pci.h> 22 #include <rte_memory.h> 23 #include <rte_eal.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 /* Bit shift and mask */ 46 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 47 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 48 #define IGB_8_BIT_WIDTH CHAR_BIT 49 #define IGB_8_BIT_MASK UINT8_MAX 50 51 /* Additional timesync values. */ 52 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 53 #define E1000_ETQF_FILTER_1588 3 54 #define IGB_82576_TSYNC_SHIFT 16 55 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 56 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 57 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 58 59 #define E1000_VTIVAR_MISC 0x01740 60 #define E1000_VTIVAR_MISC_MASK 0xFF 61 #define E1000_VTIVAR_VALID 0x80 62 #define E1000_VTIVAR_MISC_MAILBOX 0 63 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 64 65 /* External VLAN Enable bit mask */ 66 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 67 68 /* External VLAN Ether Type bit mask and shift */ 69 #define E1000_VET_VET_EXT 0xFFFF0000 70 #define E1000_VET_VET_EXT_SHIFT 16 71 72 /* MSI-X other interrupt vector */ 73 #define IGB_MSIX_OTHER_INTR_VEC 0 74 75 static int eth_igb_configure(struct rte_eth_dev *dev); 76 static int eth_igb_start(struct rte_eth_dev *dev); 77 static void eth_igb_stop(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 79 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 80 static void eth_igb_close(struct rte_eth_dev *dev); 81 static int eth_igb_reset(struct rte_eth_dev *dev); 82 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 83 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 84 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 85 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 86 static int eth_igb_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int eth_igb_stats_get(struct rte_eth_dev *dev, 89 struct rte_eth_stats *rte_stats); 90 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 91 struct rte_eth_xstat *xstats, unsigned n); 92 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 93 const uint64_t *ids, 94 uint64_t *values, unsigned int n); 95 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, 97 unsigned int size); 98 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 99 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 100 unsigned int limit); 101 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 102 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 103 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 104 char *fw_version, size_t fw_size); 105 static void eth_igb_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 108 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 109 struct rte_eth_dev_info *dev_info); 110 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 113 struct rte_eth_fc_conf *fc_conf); 114 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 115 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 116 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 117 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 118 struct rte_intr_handle *handle); 119 static void eth_igb_interrupt_handler(void *param); 120 static int igb_hardware_init(struct e1000_hw *hw); 121 static void igb_hw_control_acquire(struct e1000_hw *hw); 122 static void igb_hw_control_release(struct e1000_hw *hw); 123 static void igb_init_manageability(struct e1000_hw *hw); 124 static void igb_release_manageability(struct e1000_hw *hw); 125 126 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 127 128 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 129 uint16_t vlan_id, int on); 130 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 131 enum rte_vlan_type vlan_type, 132 uint16_t tpid_id); 133 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 134 135 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 139 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 140 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 141 142 static int eth_igb_led_on(struct rte_eth_dev *dev); 143 static int eth_igb_led_off(struct rte_eth_dev *dev); 144 145 static void igb_intr_disable(struct rte_eth_dev *dev); 146 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 147 static int eth_igb_rar_set(struct rte_eth_dev *dev, 148 struct rte_ether_addr *mac_addr, 149 uint32_t index, uint32_t pool); 150 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 151 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 152 struct rte_ether_addr *addr); 153 154 static void igbvf_intr_disable(struct e1000_hw *hw); 155 static int igbvf_dev_configure(struct rte_eth_dev *dev); 156 static int igbvf_dev_start(struct rte_eth_dev *dev); 157 static void igbvf_dev_stop(struct rte_eth_dev *dev); 158 static void igbvf_dev_close(struct rte_eth_dev *dev); 159 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 160 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 161 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 162 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 163 static int eth_igbvf_link_update(struct e1000_hw *hw); 164 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 165 struct rte_eth_stats *rte_stats); 166 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 167 struct rte_eth_xstat *xstats, unsigned n); 168 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 169 struct rte_eth_xstat_name *xstats_names, 170 unsigned limit); 171 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 172 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 173 uint16_t vlan_id, int on); 174 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 175 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 176 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 177 struct rte_ether_addr *addr); 178 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 179 static int igbvf_get_regs(struct rte_eth_dev *dev, 180 struct rte_dev_reg_info *regs); 181 182 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 186 struct rte_eth_rss_reta_entry64 *reta_conf, 187 uint16_t reta_size); 188 189 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 190 struct rte_eth_syn_filter *filter); 191 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 192 enum rte_filter_op filter_op, 193 void *arg); 194 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 195 struct rte_eth_ntuple_filter *ntuple_filter); 196 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 197 struct rte_eth_ntuple_filter *ntuple_filter); 198 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 199 struct rte_eth_flex_filter *filter); 200 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 201 enum rte_filter_op filter_op, 202 void *arg); 203 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *ntuple_filter); 205 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 206 struct rte_eth_ntuple_filter *ntuple_filter); 207 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 208 struct rte_eth_ntuple_filter *filter); 209 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 210 enum rte_filter_op filter_op, 211 void *arg); 212 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 213 enum rte_filter_op filter_op, 214 void *arg); 215 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 216 struct rte_eth_ethertype_filter *filter); 217 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 218 enum rte_filter_type filter_type, 219 enum rte_filter_op filter_op, 220 void *arg); 221 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 222 static int eth_igb_get_regs(struct rte_eth_dev *dev, 223 struct rte_dev_reg_info *regs); 224 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 225 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 226 struct rte_dev_eeprom_info *eeprom); 227 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 228 struct rte_dev_eeprom_info *eeprom); 229 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 230 struct rte_eth_dev_module_info *modinfo); 231 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 232 struct rte_dev_eeprom_info *info); 233 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 234 struct rte_ether_addr *mc_addr_set, 235 uint32_t nb_mc_addr); 236 static int igb_timesync_enable(struct rte_eth_dev *dev); 237 static int igb_timesync_disable(struct rte_eth_dev *dev); 238 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 239 struct timespec *timestamp, 240 uint32_t flags); 241 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 242 struct timespec *timestamp); 243 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 244 static int igb_timesync_read_time(struct rte_eth_dev *dev, 245 struct timespec *timestamp); 246 static int igb_timesync_write_time(struct rte_eth_dev *dev, 247 const struct timespec *timestamp); 248 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 249 uint16_t queue_id); 250 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 251 uint16_t queue_id); 252 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 253 uint8_t queue, uint8_t msix_vector); 254 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 255 uint8_t index, uint8_t offset); 256 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 257 static void eth_igbvf_interrupt_handler(void *param); 258 static void igbvf_mbx_process(struct rte_eth_dev *dev); 259 static int igb_filter_restore(struct rte_eth_dev *dev); 260 261 /* 262 * Define VF Stats MACRO for Non "cleared on read" register 263 */ 264 #define UPDATE_VF_STAT(reg, last, cur) \ 265 { \ 266 u32 latest = E1000_READ_REG(hw, reg); \ 267 cur += (latest - last) & UINT_MAX; \ 268 last = latest; \ 269 } 270 271 #define IGB_FC_PAUSE_TIME 0x0680 272 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 273 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 274 275 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 276 277 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 278 279 /* 280 * The set of PCI devices this driver supports 281 */ 282 static const struct rte_pci_id pci_id_igb_map[] = { 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 291 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 295 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 302 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 320 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 321 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 322 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 323 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 324 { .vendor_id = 0, /* sentinel */ }, 325 }; 326 327 /* 328 * The set of PCI devices this driver supports (for 82576&I350 VF) 329 */ 330 static const struct rte_pci_id pci_id_igbvf_map[] = { 331 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 332 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 333 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 334 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 335 { .vendor_id = 0, /* sentinel */ }, 336 }; 337 338 static const struct rte_eth_desc_lim rx_desc_lim = { 339 .nb_max = E1000_MAX_RING_DESC, 340 .nb_min = E1000_MIN_RING_DESC, 341 .nb_align = IGB_RXD_ALIGN, 342 }; 343 344 static const struct rte_eth_desc_lim tx_desc_lim = { 345 .nb_max = E1000_MAX_RING_DESC, 346 .nb_min = E1000_MIN_RING_DESC, 347 .nb_align = IGB_RXD_ALIGN, 348 .nb_seg_max = IGB_TX_MAX_SEG, 349 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 350 }; 351 352 static const struct eth_dev_ops eth_igb_ops = { 353 .dev_configure = eth_igb_configure, 354 .dev_start = eth_igb_start, 355 .dev_stop = eth_igb_stop, 356 .dev_set_link_up = eth_igb_dev_set_link_up, 357 .dev_set_link_down = eth_igb_dev_set_link_down, 358 .dev_close = eth_igb_close, 359 .dev_reset = eth_igb_reset, 360 .promiscuous_enable = eth_igb_promiscuous_enable, 361 .promiscuous_disable = eth_igb_promiscuous_disable, 362 .allmulticast_enable = eth_igb_allmulticast_enable, 363 .allmulticast_disable = eth_igb_allmulticast_disable, 364 .link_update = eth_igb_link_update, 365 .stats_get = eth_igb_stats_get, 366 .xstats_get = eth_igb_xstats_get, 367 .xstats_get_by_id = eth_igb_xstats_get_by_id, 368 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 369 .xstats_get_names = eth_igb_xstats_get_names, 370 .stats_reset = eth_igb_stats_reset, 371 .xstats_reset = eth_igb_xstats_reset, 372 .fw_version_get = eth_igb_fw_version_get, 373 .dev_infos_get = eth_igb_infos_get, 374 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 375 .mtu_set = eth_igb_mtu_set, 376 .vlan_filter_set = eth_igb_vlan_filter_set, 377 .vlan_tpid_set = eth_igb_vlan_tpid_set, 378 .vlan_offload_set = eth_igb_vlan_offload_set, 379 .rx_queue_setup = eth_igb_rx_queue_setup, 380 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 381 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 382 .rx_queue_release = eth_igb_rx_queue_release, 383 .rx_queue_count = eth_igb_rx_queue_count, 384 .rx_descriptor_done = eth_igb_rx_descriptor_done, 385 .rx_descriptor_status = eth_igb_rx_descriptor_status, 386 .tx_descriptor_status = eth_igb_tx_descriptor_status, 387 .tx_queue_setup = eth_igb_tx_queue_setup, 388 .tx_queue_release = eth_igb_tx_queue_release, 389 .tx_done_cleanup = eth_igb_tx_done_cleanup, 390 .dev_led_on = eth_igb_led_on, 391 .dev_led_off = eth_igb_led_off, 392 .flow_ctrl_get = eth_igb_flow_ctrl_get, 393 .flow_ctrl_set = eth_igb_flow_ctrl_set, 394 .mac_addr_add = eth_igb_rar_set, 395 .mac_addr_remove = eth_igb_rar_clear, 396 .mac_addr_set = eth_igb_default_mac_addr_set, 397 .reta_update = eth_igb_rss_reta_update, 398 .reta_query = eth_igb_rss_reta_query, 399 .rss_hash_update = eth_igb_rss_hash_update, 400 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 401 .filter_ctrl = eth_igb_filter_ctrl, 402 .set_mc_addr_list = eth_igb_set_mc_addr_list, 403 .rxq_info_get = igb_rxq_info_get, 404 .txq_info_get = igb_txq_info_get, 405 .timesync_enable = igb_timesync_enable, 406 .timesync_disable = igb_timesync_disable, 407 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 408 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 409 .get_reg = eth_igb_get_regs, 410 .get_eeprom_length = eth_igb_get_eeprom_length, 411 .get_eeprom = eth_igb_get_eeprom, 412 .set_eeprom = eth_igb_set_eeprom, 413 .get_module_info = eth_igb_get_module_info, 414 .get_module_eeprom = eth_igb_get_module_eeprom, 415 .timesync_adjust_time = igb_timesync_adjust_time, 416 .timesync_read_time = igb_timesync_read_time, 417 .timesync_write_time = igb_timesync_write_time, 418 }; 419 420 /* 421 * dev_ops for virtual function, bare necessities for basic vf 422 * operation have been implemented 423 */ 424 static const struct eth_dev_ops igbvf_eth_dev_ops = { 425 .dev_configure = igbvf_dev_configure, 426 .dev_start = igbvf_dev_start, 427 .dev_stop = igbvf_dev_stop, 428 .dev_close = igbvf_dev_close, 429 .promiscuous_enable = igbvf_promiscuous_enable, 430 .promiscuous_disable = igbvf_promiscuous_disable, 431 .allmulticast_enable = igbvf_allmulticast_enable, 432 .allmulticast_disable = igbvf_allmulticast_disable, 433 .link_update = eth_igb_link_update, 434 .stats_get = eth_igbvf_stats_get, 435 .xstats_get = eth_igbvf_xstats_get, 436 .xstats_get_names = eth_igbvf_xstats_get_names, 437 .stats_reset = eth_igbvf_stats_reset, 438 .xstats_reset = eth_igbvf_stats_reset, 439 .vlan_filter_set = igbvf_vlan_filter_set, 440 .dev_infos_get = eth_igbvf_infos_get, 441 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 442 .rx_queue_setup = eth_igb_rx_queue_setup, 443 .rx_queue_release = eth_igb_rx_queue_release, 444 .rx_descriptor_done = eth_igb_rx_descriptor_done, 445 .rx_descriptor_status = eth_igb_rx_descriptor_status, 446 .tx_descriptor_status = eth_igb_tx_descriptor_status, 447 .tx_queue_setup = eth_igb_tx_queue_setup, 448 .tx_queue_release = eth_igb_tx_queue_release, 449 .set_mc_addr_list = eth_igb_set_mc_addr_list, 450 .rxq_info_get = igb_rxq_info_get, 451 .txq_info_get = igb_txq_info_get, 452 .mac_addr_set = igbvf_default_mac_addr_set, 453 .get_reg = igbvf_get_regs, 454 }; 455 456 /* store statistics names and its offset in stats structure */ 457 struct rte_igb_xstats_name_off { 458 char name[RTE_ETH_XSTATS_NAME_SIZE]; 459 unsigned offset; 460 }; 461 462 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 463 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 464 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 465 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 466 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 467 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 468 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 469 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 470 ecol)}, 471 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 472 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 473 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 474 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 475 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 476 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 477 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 478 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 479 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 480 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 481 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 482 fcruc)}, 483 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 484 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 485 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 486 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 487 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 488 prc1023)}, 489 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 490 prc1522)}, 491 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 492 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 493 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 494 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 495 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 496 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 497 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 498 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 499 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 500 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 501 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 502 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 503 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 504 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 505 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 506 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 507 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 508 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 509 ptc1023)}, 510 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 511 ptc1522)}, 512 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 513 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 514 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 515 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 516 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 517 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 518 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 519 520 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 521 }; 522 523 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 524 sizeof(rte_igb_stats_strings[0])) 525 526 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 527 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 528 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 529 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 530 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 531 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 532 }; 533 534 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 535 sizeof(rte_igbvf_stats_strings[0])) 536 537 538 static inline void 539 igb_intr_enable(struct rte_eth_dev *dev) 540 { 541 struct e1000_interrupt *intr = 542 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 543 struct e1000_hw *hw = 544 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 545 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 546 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 547 548 if (rte_intr_allow_others(intr_handle) && 549 dev->data->dev_conf.intr_conf.lsc != 0) { 550 E1000_WRITE_REG(hw, E1000_EIMS, 1 << IGB_MSIX_OTHER_INTR_VEC); 551 } 552 553 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 554 E1000_WRITE_FLUSH(hw); 555 } 556 557 static void 558 igb_intr_disable(struct rte_eth_dev *dev) 559 { 560 struct e1000_hw *hw = 561 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 562 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 563 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 564 565 if (rte_intr_allow_others(intr_handle) && 566 dev->data->dev_conf.intr_conf.lsc != 0) { 567 E1000_WRITE_REG(hw, E1000_EIMC, 1 << IGB_MSIX_OTHER_INTR_VEC); 568 } 569 570 E1000_WRITE_REG(hw, E1000_IMC, ~0); 571 E1000_WRITE_FLUSH(hw); 572 } 573 574 static inline void 575 igbvf_intr_enable(struct rte_eth_dev *dev) 576 { 577 struct e1000_hw *hw = 578 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 579 580 /* only for mailbox */ 581 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 582 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 583 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 584 E1000_WRITE_FLUSH(hw); 585 } 586 587 /* only for mailbox now. If RX/TX needed, should extend this function. */ 588 static void 589 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 590 { 591 uint32_t tmp = 0; 592 593 /* mailbox */ 594 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 595 tmp |= E1000_VTIVAR_VALID; 596 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 597 } 598 599 static void 600 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 601 { 602 struct e1000_hw *hw = 603 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 604 605 /* Configure VF other cause ivar */ 606 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 607 } 608 609 static inline int32_t 610 igb_pf_reset_hw(struct e1000_hw *hw) 611 { 612 uint32_t ctrl_ext; 613 int32_t status; 614 615 status = e1000_reset_hw(hw); 616 617 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 618 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 619 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 620 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 621 E1000_WRITE_FLUSH(hw); 622 623 return status; 624 } 625 626 static void 627 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 628 { 629 struct e1000_hw *hw = 630 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 631 632 633 hw->vendor_id = pci_dev->id.vendor_id; 634 hw->device_id = pci_dev->id.device_id; 635 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 636 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 637 638 e1000_set_mac_type(hw); 639 640 /* need to check if it is a vf device below */ 641 } 642 643 static int 644 igb_reset_swfw_lock(struct e1000_hw *hw) 645 { 646 int ret_val; 647 648 /* 649 * Do mac ops initialization manually here, since we will need 650 * some function pointers set by this call. 651 */ 652 ret_val = e1000_init_mac_params(hw); 653 if (ret_val) 654 return ret_val; 655 656 /* 657 * SMBI lock should not fail in this early stage. If this is the case, 658 * it is due to an improper exit of the application. 659 * So force the release of the faulty lock. 660 */ 661 if (e1000_get_hw_semaphore_generic(hw) < 0) { 662 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 663 } 664 e1000_put_hw_semaphore_generic(hw); 665 666 if (hw->mac.ops.acquire_swfw_sync != NULL) { 667 uint16_t mask; 668 669 /* 670 * Phy lock should not fail in this early stage. If this is the case, 671 * it is due to an improper exit of the application. 672 * So force the release of the faulty lock. 673 */ 674 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 675 if (hw->bus.func > E1000_FUNC_1) 676 mask <<= 2; 677 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 678 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 679 hw->bus.func); 680 } 681 hw->mac.ops.release_swfw_sync(hw, mask); 682 683 /* 684 * This one is more tricky since it is common to all ports; but 685 * swfw_sync retries last long enough (1s) to be almost sure that if 686 * lock can not be taken it is due to an improper lock of the 687 * semaphore. 688 */ 689 mask = E1000_SWFW_EEP_SM; 690 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 691 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 692 } 693 hw->mac.ops.release_swfw_sync(hw, mask); 694 } 695 696 return E1000_SUCCESS; 697 } 698 699 /* Remove all ntuple filters of the device */ 700 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 701 { 702 struct e1000_filter_info *filter_info = 703 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 704 struct e1000_5tuple_filter *p_5tuple; 705 struct e1000_2tuple_filter *p_2tuple; 706 707 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 708 TAILQ_REMOVE(&filter_info->fivetuple_list, 709 p_5tuple, entries); 710 rte_free(p_5tuple); 711 } 712 filter_info->fivetuple_mask = 0; 713 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 714 TAILQ_REMOVE(&filter_info->twotuple_list, 715 p_2tuple, entries); 716 rte_free(p_2tuple); 717 } 718 filter_info->twotuple_mask = 0; 719 720 return 0; 721 } 722 723 /* Remove all flex filters of the device */ 724 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 725 { 726 struct e1000_filter_info *filter_info = 727 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 728 struct e1000_flex_filter *p_flex; 729 730 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 731 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 732 rte_free(p_flex); 733 } 734 filter_info->flex_mask = 0; 735 736 return 0; 737 } 738 739 static int 740 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 741 { 742 int error = 0; 743 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 744 struct e1000_hw *hw = 745 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 746 struct e1000_vfta * shadow_vfta = 747 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 748 struct e1000_filter_info *filter_info = 749 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 750 struct e1000_adapter *adapter = 751 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 752 753 uint32_t ctrl_ext; 754 755 eth_dev->dev_ops = ð_igb_ops; 756 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 757 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 758 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 759 760 /* for secondary processes, we don't initialise any further as primary 761 * has already done this work. Only check we don't need a different 762 * RX function */ 763 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 764 if (eth_dev->data->scattered_rx) 765 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 766 return 0; 767 } 768 769 rte_eth_copy_pci_info(eth_dev, pci_dev); 770 771 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 772 773 igb_identify_hardware(eth_dev, pci_dev); 774 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 775 error = -EIO; 776 goto err_late; 777 } 778 779 e1000_get_bus_info(hw); 780 781 /* Reset any pending lock */ 782 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 783 error = -EIO; 784 goto err_late; 785 } 786 787 /* Finish initialization */ 788 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 789 error = -EIO; 790 goto err_late; 791 } 792 793 hw->mac.autoneg = 1; 794 hw->phy.autoneg_wait_to_complete = 0; 795 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 796 797 /* Copper options */ 798 if (hw->phy.media_type == e1000_media_type_copper) { 799 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 800 hw->phy.disable_polarity_correction = 0; 801 hw->phy.ms_type = e1000_ms_hw_default; 802 } 803 804 /* 805 * Start from a known state, this is important in reading the nvm 806 * and mac from that. 807 */ 808 igb_pf_reset_hw(hw); 809 810 /* Make sure we have a good EEPROM before we read from it */ 811 if (e1000_validate_nvm_checksum(hw) < 0) { 812 /* 813 * Some PCI-E parts fail the first check due to 814 * the link being in sleep state, call it again, 815 * if it fails a second time its a real issue. 816 */ 817 if (e1000_validate_nvm_checksum(hw) < 0) { 818 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 819 error = -EIO; 820 goto err_late; 821 } 822 } 823 824 /* Read the permanent MAC address out of the EEPROM */ 825 if (e1000_read_mac_addr(hw) != 0) { 826 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 827 error = -EIO; 828 goto err_late; 829 } 830 831 /* Allocate memory for storing MAC addresses */ 832 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 833 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 834 if (eth_dev->data->mac_addrs == NULL) { 835 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 836 "store MAC addresses", 837 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 838 error = -ENOMEM; 839 goto err_late; 840 } 841 842 /* Copy the permanent MAC address */ 843 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 844 ð_dev->data->mac_addrs[0]); 845 846 /* initialize the vfta */ 847 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 848 849 /* Now initialize the hardware */ 850 if (igb_hardware_init(hw) != 0) { 851 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 852 rte_free(eth_dev->data->mac_addrs); 853 eth_dev->data->mac_addrs = NULL; 854 error = -ENODEV; 855 goto err_late; 856 } 857 hw->mac.get_link_status = 1; 858 adapter->stopped = 0; 859 860 /* Indicate SOL/IDER usage */ 861 if (e1000_check_reset_block(hw) < 0) { 862 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 863 "SOL/IDER session"); 864 } 865 866 /* initialize PF if max_vfs not zero */ 867 igb_pf_host_init(eth_dev); 868 869 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 870 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 871 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 872 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 873 E1000_WRITE_FLUSH(hw); 874 875 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 876 eth_dev->data->port_id, pci_dev->id.vendor_id, 877 pci_dev->id.device_id); 878 879 rte_intr_callback_register(&pci_dev->intr_handle, 880 eth_igb_interrupt_handler, 881 (void *)eth_dev); 882 883 /* enable uio/vfio intr/eventfd mapping */ 884 rte_intr_enable(&pci_dev->intr_handle); 885 886 /* enable support intr */ 887 igb_intr_enable(eth_dev); 888 889 /* initialize filter info */ 890 memset(filter_info, 0, 891 sizeof(struct e1000_filter_info)); 892 893 TAILQ_INIT(&filter_info->flex_list); 894 TAILQ_INIT(&filter_info->twotuple_list); 895 TAILQ_INIT(&filter_info->fivetuple_list); 896 897 TAILQ_INIT(&igb_filter_ntuple_list); 898 TAILQ_INIT(&igb_filter_ethertype_list); 899 TAILQ_INIT(&igb_filter_syn_list); 900 TAILQ_INIT(&igb_filter_flex_list); 901 TAILQ_INIT(&igb_filter_rss_list); 902 TAILQ_INIT(&igb_flow_list); 903 904 return 0; 905 906 err_late: 907 igb_hw_control_release(hw); 908 909 return error; 910 } 911 912 static int 913 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 914 { 915 struct rte_pci_device *pci_dev; 916 struct rte_intr_handle *intr_handle; 917 struct e1000_hw *hw; 918 struct e1000_adapter *adapter = 919 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 920 struct e1000_filter_info *filter_info = 921 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 922 923 PMD_INIT_FUNC_TRACE(); 924 925 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 926 return -EPERM; 927 928 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 929 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 930 intr_handle = &pci_dev->intr_handle; 931 932 if (adapter->stopped == 0) 933 eth_igb_close(eth_dev); 934 935 eth_dev->dev_ops = NULL; 936 eth_dev->rx_pkt_burst = NULL; 937 eth_dev->tx_pkt_burst = NULL; 938 939 /* Reset any pending lock */ 940 igb_reset_swfw_lock(hw); 941 942 /* uninitialize PF if max_vfs not zero */ 943 igb_pf_host_uninit(eth_dev); 944 945 /* disable uio intr before callback unregister */ 946 rte_intr_disable(intr_handle); 947 rte_intr_callback_unregister(intr_handle, 948 eth_igb_interrupt_handler, eth_dev); 949 950 /* clear the SYN filter info */ 951 filter_info->syn_info = 0; 952 953 /* clear the ethertype filters info */ 954 filter_info->ethertype_mask = 0; 955 memset(filter_info->ethertype_filters, 0, 956 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 957 958 /* clear the rss filter info */ 959 memset(&filter_info->rss_info, 0, 960 sizeof(struct igb_rte_flow_rss_conf)); 961 962 /* remove all ntuple filters of the device */ 963 igb_ntuple_filter_uninit(eth_dev); 964 965 /* remove all flex filters of the device */ 966 igb_flex_filter_uninit(eth_dev); 967 968 /* clear all the filters list */ 969 igb_filterlist_flush(eth_dev); 970 971 return 0; 972 } 973 974 /* 975 * Virtual Function device init 976 */ 977 static int 978 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 979 { 980 struct rte_pci_device *pci_dev; 981 struct rte_intr_handle *intr_handle; 982 struct e1000_adapter *adapter = 983 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 984 struct e1000_hw *hw = 985 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 986 int diag; 987 struct rte_ether_addr *perm_addr = 988 (struct rte_ether_addr *)hw->mac.perm_addr; 989 990 PMD_INIT_FUNC_TRACE(); 991 992 eth_dev->dev_ops = &igbvf_eth_dev_ops; 993 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 994 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 995 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 996 997 /* for secondary processes, we don't initialise any further as primary 998 * has already done this work. Only check we don't need a different 999 * RX function */ 1000 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 1001 if (eth_dev->data->scattered_rx) 1002 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 1003 return 0; 1004 } 1005 1006 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1007 rte_eth_copy_pci_info(eth_dev, pci_dev); 1008 1009 hw->device_id = pci_dev->id.device_id; 1010 hw->vendor_id = pci_dev->id.vendor_id; 1011 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1012 adapter->stopped = 0; 1013 1014 /* Initialize the shared code (base driver) */ 1015 diag = e1000_setup_init_funcs(hw, TRUE); 1016 if (diag != 0) { 1017 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 1018 diag); 1019 return -EIO; 1020 } 1021 1022 /* init_mailbox_params */ 1023 hw->mbx.ops.init_params(hw); 1024 1025 /* Disable the interrupts for VF */ 1026 igbvf_intr_disable(hw); 1027 1028 diag = hw->mac.ops.reset_hw(hw); 1029 1030 /* Allocate memory for storing MAC addresses */ 1031 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", RTE_ETHER_ADDR_LEN * 1032 hw->mac.rar_entry_count, 0); 1033 if (eth_dev->data->mac_addrs == NULL) { 1034 PMD_INIT_LOG(ERR, 1035 "Failed to allocate %d bytes needed to store MAC " 1036 "addresses", 1037 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1038 return -ENOMEM; 1039 } 1040 1041 /* Generate a random MAC address, if none was assigned by PF. */ 1042 if (rte_is_zero_ether_addr(perm_addr)) { 1043 rte_eth_random_addr(perm_addr->addr_bytes); 1044 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1045 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1046 "%02x:%02x:%02x:%02x:%02x:%02x", 1047 perm_addr->addr_bytes[0], 1048 perm_addr->addr_bytes[1], 1049 perm_addr->addr_bytes[2], 1050 perm_addr->addr_bytes[3], 1051 perm_addr->addr_bytes[4], 1052 perm_addr->addr_bytes[5]); 1053 } 1054 1055 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1056 if (diag) { 1057 rte_free(eth_dev->data->mac_addrs); 1058 eth_dev->data->mac_addrs = NULL; 1059 return diag; 1060 } 1061 /* Copy the permanent MAC address */ 1062 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1063 ð_dev->data->mac_addrs[0]); 1064 1065 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1066 "mac.type=%s", 1067 eth_dev->data->port_id, pci_dev->id.vendor_id, 1068 pci_dev->id.device_id, "igb_mac_82576_vf"); 1069 1070 intr_handle = &pci_dev->intr_handle; 1071 rte_intr_callback_register(intr_handle, 1072 eth_igbvf_interrupt_handler, eth_dev); 1073 1074 return 0; 1075 } 1076 1077 static int 1078 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1079 { 1080 struct e1000_adapter *adapter = 1081 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1082 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1083 1084 PMD_INIT_FUNC_TRACE(); 1085 1086 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1087 return -EPERM; 1088 1089 if (adapter->stopped == 0) 1090 igbvf_dev_close(eth_dev); 1091 1092 eth_dev->dev_ops = NULL; 1093 eth_dev->rx_pkt_burst = NULL; 1094 eth_dev->tx_pkt_burst = NULL; 1095 1096 /* disable uio intr before callback unregister */ 1097 rte_intr_disable(&pci_dev->intr_handle); 1098 rte_intr_callback_unregister(&pci_dev->intr_handle, 1099 eth_igbvf_interrupt_handler, 1100 (void *)eth_dev); 1101 1102 return 0; 1103 } 1104 1105 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1106 struct rte_pci_device *pci_dev) 1107 { 1108 return rte_eth_dev_pci_generic_probe(pci_dev, 1109 sizeof(struct e1000_adapter), eth_igb_dev_init); 1110 } 1111 1112 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1113 { 1114 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1115 } 1116 1117 static struct rte_pci_driver rte_igb_pmd = { 1118 .id_table = pci_id_igb_map, 1119 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1120 .probe = eth_igb_pci_probe, 1121 .remove = eth_igb_pci_remove, 1122 }; 1123 1124 1125 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1126 struct rte_pci_device *pci_dev) 1127 { 1128 return rte_eth_dev_pci_generic_probe(pci_dev, 1129 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1130 } 1131 1132 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1133 { 1134 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1135 } 1136 1137 /* 1138 * virtual function driver struct 1139 */ 1140 static struct rte_pci_driver rte_igbvf_pmd = { 1141 .id_table = pci_id_igbvf_map, 1142 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1143 .probe = eth_igbvf_pci_probe, 1144 .remove = eth_igbvf_pci_remove, 1145 }; 1146 1147 static void 1148 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1149 { 1150 struct e1000_hw *hw = 1151 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1152 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1153 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1154 rctl |= E1000_RCTL_VFE; 1155 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1156 } 1157 1158 static int 1159 igb_check_mq_mode(struct rte_eth_dev *dev) 1160 { 1161 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1162 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1163 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1164 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1165 1166 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1167 tx_mq_mode == ETH_MQ_TX_DCB || 1168 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1169 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1170 return -EINVAL; 1171 } 1172 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1173 /* Check multi-queue mode. 1174 * To no break software we accept ETH_MQ_RX_NONE as this might 1175 * be used to turn off VLAN filter. 1176 */ 1177 1178 if (rx_mq_mode == ETH_MQ_RX_NONE || 1179 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1180 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1181 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1182 } else { 1183 /* Only support one queue on VFs. 1184 * RSS together with SRIOV is not supported. 1185 */ 1186 PMD_INIT_LOG(ERR, "SRIOV is active," 1187 " wrong mq_mode rx %d.", 1188 rx_mq_mode); 1189 return -EINVAL; 1190 } 1191 /* TX mode is not used here, so mode might be ignored.*/ 1192 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1193 /* SRIOV only works in VMDq enable mode */ 1194 PMD_INIT_LOG(WARNING, "SRIOV is active," 1195 " TX mode %d is not supported. " 1196 " Driver will behave as %d mode.", 1197 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1198 } 1199 1200 /* check valid queue number */ 1201 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1202 PMD_INIT_LOG(ERR, "SRIOV is active," 1203 " only support one queue on VFs."); 1204 return -EINVAL; 1205 } 1206 } else { 1207 /* To no break software that set invalid mode, only display 1208 * warning if invalid mode is used. 1209 */ 1210 if (rx_mq_mode != ETH_MQ_RX_NONE && 1211 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1212 rx_mq_mode != ETH_MQ_RX_RSS) { 1213 /* RSS together with VMDq not supported*/ 1214 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1215 rx_mq_mode); 1216 return -EINVAL; 1217 } 1218 1219 if (tx_mq_mode != ETH_MQ_TX_NONE && 1220 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1221 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1222 " Due to txmode is meaningless in this" 1223 " driver, just ignore.", 1224 tx_mq_mode); 1225 } 1226 } 1227 return 0; 1228 } 1229 1230 static int 1231 eth_igb_configure(struct rte_eth_dev *dev) 1232 { 1233 struct e1000_interrupt *intr = 1234 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1235 int ret; 1236 1237 PMD_INIT_FUNC_TRACE(); 1238 1239 /* multipe queue mode checking */ 1240 ret = igb_check_mq_mode(dev); 1241 if (ret != 0) { 1242 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1243 ret); 1244 return ret; 1245 } 1246 1247 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1248 PMD_INIT_FUNC_TRACE(); 1249 1250 return 0; 1251 } 1252 1253 static void 1254 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1255 bool enable) 1256 { 1257 struct e1000_hw *hw = 1258 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1259 uint32_t tctl, rctl; 1260 1261 tctl = E1000_READ_REG(hw, E1000_TCTL); 1262 rctl = E1000_READ_REG(hw, E1000_RCTL); 1263 1264 if (enable) { 1265 /* enable Tx/Rx */ 1266 tctl |= E1000_TCTL_EN; 1267 rctl |= E1000_RCTL_EN; 1268 } else { 1269 /* disable Tx/Rx */ 1270 tctl &= ~E1000_TCTL_EN; 1271 rctl &= ~E1000_RCTL_EN; 1272 } 1273 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1274 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1275 E1000_WRITE_FLUSH(hw); 1276 } 1277 1278 static int 1279 eth_igb_start(struct rte_eth_dev *dev) 1280 { 1281 struct e1000_hw *hw = 1282 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1283 struct e1000_adapter *adapter = 1284 E1000_DEV_PRIVATE(dev->data->dev_private); 1285 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1286 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1287 int ret, mask; 1288 uint32_t intr_vector = 0; 1289 uint32_t ctrl_ext; 1290 uint32_t *speeds; 1291 int num_speeds; 1292 bool autoneg; 1293 1294 PMD_INIT_FUNC_TRACE(); 1295 1296 /* disable uio/vfio intr/eventfd mapping */ 1297 rte_intr_disable(intr_handle); 1298 1299 /* Power up the phy. Needed to make the link go Up */ 1300 eth_igb_dev_set_link_up(dev); 1301 1302 /* 1303 * Packet Buffer Allocation (PBA) 1304 * Writing PBA sets the receive portion of the buffer 1305 * the remainder is used for the transmit buffer. 1306 */ 1307 if (hw->mac.type == e1000_82575) { 1308 uint32_t pba; 1309 1310 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1311 E1000_WRITE_REG(hw, E1000_PBA, pba); 1312 } 1313 1314 /* Put the address into the Receive Address Array */ 1315 e1000_rar_set(hw, hw->mac.addr, 0); 1316 1317 /* Initialize the hardware */ 1318 if (igb_hardware_init(hw)) { 1319 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1320 return -EIO; 1321 } 1322 adapter->stopped = 0; 1323 1324 E1000_WRITE_REG(hw, E1000_VET, 1325 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1326 1327 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1328 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1329 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1330 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1331 E1000_WRITE_FLUSH(hw); 1332 1333 /* configure PF module if SRIOV enabled */ 1334 igb_pf_host_configure(dev); 1335 1336 /* check and configure queue intr-vector mapping */ 1337 if ((rte_intr_cap_multiple(intr_handle) || 1338 !RTE_ETH_DEV_SRIOV(dev).active) && 1339 dev->data->dev_conf.intr_conf.rxq != 0) { 1340 intr_vector = dev->data->nb_rx_queues; 1341 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1342 return -1; 1343 } 1344 1345 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1346 intr_handle->intr_vec = 1347 rte_zmalloc("intr_vec", 1348 dev->data->nb_rx_queues * sizeof(int), 0); 1349 if (intr_handle->intr_vec == NULL) { 1350 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1351 " intr_vec", dev->data->nb_rx_queues); 1352 return -ENOMEM; 1353 } 1354 } 1355 1356 /* confiugre msix for rx interrupt */ 1357 eth_igb_configure_msix_intr(dev); 1358 1359 /* Configure for OS presence */ 1360 igb_init_manageability(hw); 1361 1362 eth_igb_tx_init(dev); 1363 1364 /* This can fail when allocating mbufs for descriptor rings */ 1365 ret = eth_igb_rx_init(dev); 1366 if (ret) { 1367 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1368 igb_dev_clear_queues(dev); 1369 return ret; 1370 } 1371 1372 e1000_clear_hw_cntrs_base_generic(hw); 1373 1374 /* 1375 * VLAN Offload Settings 1376 */ 1377 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1378 ETH_VLAN_EXTEND_MASK; 1379 ret = eth_igb_vlan_offload_set(dev, mask); 1380 if (ret) { 1381 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1382 igb_dev_clear_queues(dev); 1383 return ret; 1384 } 1385 1386 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1387 /* Enable VLAN filter since VMDq always use VLAN filter */ 1388 igb_vmdq_vlan_hw_filter_enable(dev); 1389 } 1390 1391 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1392 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1393 (hw->mac.type == e1000_i211)) { 1394 /* Configure EITR with the maximum possible value (0xFFFF) */ 1395 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1396 } 1397 1398 /* Setup link speed and duplex */ 1399 speeds = &dev->data->dev_conf.link_speeds; 1400 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1401 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1402 hw->mac.autoneg = 1; 1403 } else { 1404 num_speeds = 0; 1405 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1406 1407 /* Reset */ 1408 hw->phy.autoneg_advertised = 0; 1409 1410 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1411 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1412 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1413 num_speeds = -1; 1414 goto error_invalid_config; 1415 } 1416 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1417 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1418 num_speeds++; 1419 } 1420 if (*speeds & ETH_LINK_SPEED_10M) { 1421 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1422 num_speeds++; 1423 } 1424 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1425 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1426 num_speeds++; 1427 } 1428 if (*speeds & ETH_LINK_SPEED_100M) { 1429 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1430 num_speeds++; 1431 } 1432 if (*speeds & ETH_LINK_SPEED_1G) { 1433 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1434 num_speeds++; 1435 } 1436 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1437 goto error_invalid_config; 1438 1439 /* Set/reset the mac.autoneg based on the link speed, 1440 * fixed or not 1441 */ 1442 if (!autoneg) { 1443 hw->mac.autoneg = 0; 1444 hw->mac.forced_speed_duplex = 1445 hw->phy.autoneg_advertised; 1446 } else { 1447 hw->mac.autoneg = 1; 1448 } 1449 } 1450 1451 e1000_setup_link(hw); 1452 1453 if (rte_intr_allow_others(intr_handle)) { 1454 /* check if lsc interrupt is enabled */ 1455 if (dev->data->dev_conf.intr_conf.lsc != 0) 1456 eth_igb_lsc_interrupt_setup(dev, TRUE); 1457 else 1458 eth_igb_lsc_interrupt_setup(dev, FALSE); 1459 } else { 1460 rte_intr_callback_unregister(intr_handle, 1461 eth_igb_interrupt_handler, 1462 (void *)dev); 1463 if (dev->data->dev_conf.intr_conf.lsc != 0) 1464 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1465 " no intr multiplex"); 1466 } 1467 1468 /* check if rxq interrupt is enabled */ 1469 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1470 rte_intr_dp_is_en(intr_handle)) 1471 eth_igb_rxq_interrupt_setup(dev); 1472 1473 /* enable uio/vfio intr/eventfd mapping */ 1474 rte_intr_enable(intr_handle); 1475 1476 /* resume enabled intr since hw reset */ 1477 igb_intr_enable(dev); 1478 1479 /* restore all types filter */ 1480 igb_filter_restore(dev); 1481 1482 eth_igb_rxtx_control(dev, true); 1483 eth_igb_link_update(dev, 0); 1484 1485 PMD_INIT_LOG(DEBUG, "<<"); 1486 1487 return 0; 1488 1489 error_invalid_config: 1490 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1491 dev->data->dev_conf.link_speeds, dev->data->port_id); 1492 igb_dev_clear_queues(dev); 1493 return -EINVAL; 1494 } 1495 1496 /********************************************************************* 1497 * 1498 * This routine disables all traffic on the adapter by issuing a 1499 * global reset on the MAC. 1500 * 1501 **********************************************************************/ 1502 static void 1503 eth_igb_stop(struct rte_eth_dev *dev) 1504 { 1505 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1506 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1507 struct rte_eth_link link; 1508 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1509 1510 eth_igb_rxtx_control(dev, false); 1511 1512 igb_intr_disable(dev); 1513 1514 /* disable intr eventfd mapping */ 1515 rte_intr_disable(intr_handle); 1516 1517 igb_pf_reset_hw(hw); 1518 E1000_WRITE_REG(hw, E1000_WUC, 0); 1519 1520 /* Set bit for Go Link disconnect */ 1521 if (hw->mac.type >= e1000_82580) { 1522 uint32_t phpm_reg; 1523 1524 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1525 phpm_reg |= E1000_82580_PM_GO_LINKD; 1526 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1527 } 1528 1529 /* Power down the phy. Needed to make the link go Down */ 1530 eth_igb_dev_set_link_down(dev); 1531 1532 igb_dev_clear_queues(dev); 1533 1534 /* clear the recorded link status */ 1535 memset(&link, 0, sizeof(link)); 1536 rte_eth_linkstatus_set(dev, &link); 1537 1538 if (!rte_intr_allow_others(intr_handle)) 1539 /* resume to the default handler */ 1540 rte_intr_callback_register(intr_handle, 1541 eth_igb_interrupt_handler, 1542 (void *)dev); 1543 1544 /* Clean datapath event and queue/vec mapping */ 1545 rte_intr_efd_disable(intr_handle); 1546 if (intr_handle->intr_vec != NULL) { 1547 rte_free(intr_handle->intr_vec); 1548 intr_handle->intr_vec = NULL; 1549 } 1550 } 1551 1552 static int 1553 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1554 { 1555 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1556 1557 if (hw->phy.media_type == e1000_media_type_copper) 1558 e1000_power_up_phy(hw); 1559 else 1560 e1000_power_up_fiber_serdes_link(hw); 1561 1562 return 0; 1563 } 1564 1565 static int 1566 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1567 { 1568 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1569 1570 if (hw->phy.media_type == e1000_media_type_copper) 1571 e1000_power_down_phy(hw); 1572 else 1573 e1000_shutdown_fiber_serdes_link(hw); 1574 1575 return 0; 1576 } 1577 1578 static void 1579 eth_igb_close(struct rte_eth_dev *dev) 1580 { 1581 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1582 struct e1000_adapter *adapter = 1583 E1000_DEV_PRIVATE(dev->data->dev_private); 1584 struct rte_eth_link link; 1585 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1586 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1587 1588 eth_igb_stop(dev); 1589 adapter->stopped = 1; 1590 1591 e1000_phy_hw_reset(hw); 1592 igb_release_manageability(hw); 1593 igb_hw_control_release(hw); 1594 1595 /* Clear bit for Go Link disconnect */ 1596 if (hw->mac.type >= e1000_82580) { 1597 uint32_t phpm_reg; 1598 1599 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1600 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1601 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1602 } 1603 1604 igb_dev_free_queues(dev); 1605 1606 if (intr_handle->intr_vec) { 1607 rte_free(intr_handle->intr_vec); 1608 intr_handle->intr_vec = NULL; 1609 } 1610 1611 memset(&link, 0, sizeof(link)); 1612 rte_eth_linkstatus_set(dev, &link); 1613 } 1614 1615 /* 1616 * Reset PF device. 1617 */ 1618 static int 1619 eth_igb_reset(struct rte_eth_dev *dev) 1620 { 1621 int ret; 1622 1623 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1624 * its VF to make them align with it. The detailed notification 1625 * mechanism is PMD specific and is currently not implemented. 1626 * To avoid unexpected behavior in VF, currently reset of PF with 1627 * SR-IOV activation is not supported. It might be supported later. 1628 */ 1629 if (dev->data->sriov.active) 1630 return -ENOTSUP; 1631 1632 ret = eth_igb_dev_uninit(dev); 1633 if (ret) 1634 return ret; 1635 1636 ret = eth_igb_dev_init(dev); 1637 1638 return ret; 1639 } 1640 1641 1642 static int 1643 igb_get_rx_buffer_size(struct e1000_hw *hw) 1644 { 1645 uint32_t rx_buf_size; 1646 if (hw->mac.type == e1000_82576) { 1647 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1648 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1649 /* PBS needs to be translated according to a lookup table */ 1650 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1651 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1652 rx_buf_size = (rx_buf_size << 10); 1653 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1654 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1655 } else { 1656 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1657 } 1658 1659 return rx_buf_size; 1660 } 1661 1662 /********************************************************************* 1663 * 1664 * Initialize the hardware 1665 * 1666 **********************************************************************/ 1667 static int 1668 igb_hardware_init(struct e1000_hw *hw) 1669 { 1670 uint32_t rx_buf_size; 1671 int diag; 1672 1673 /* Let the firmware know the OS is in control */ 1674 igb_hw_control_acquire(hw); 1675 1676 /* 1677 * These parameters control the automatic generation (Tx) and 1678 * response (Rx) to Ethernet PAUSE frames. 1679 * - High water mark should allow for at least two standard size (1518) 1680 * frames to be received after sending an XOFF. 1681 * - Low water mark works best when it is very near the high water mark. 1682 * This allows the receiver to restart by sending XON when it has 1683 * drained a bit. Here we use an arbitrary value of 1500 which will 1684 * restart after one full frame is pulled from the buffer. There 1685 * could be several smaller frames in the buffer and if so they will 1686 * not trigger the XON until their total number reduces the buffer 1687 * by 1500. 1688 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1689 */ 1690 rx_buf_size = igb_get_rx_buffer_size(hw); 1691 1692 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2); 1693 hw->fc.low_water = hw->fc.high_water - 1500; 1694 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1695 hw->fc.send_xon = 1; 1696 1697 /* Set Flow control, use the tunable location if sane */ 1698 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1699 hw->fc.requested_mode = igb_fc_setting; 1700 else 1701 hw->fc.requested_mode = e1000_fc_none; 1702 1703 /* Issue a global reset */ 1704 igb_pf_reset_hw(hw); 1705 E1000_WRITE_REG(hw, E1000_WUC, 0); 1706 1707 diag = e1000_init_hw(hw); 1708 if (diag < 0) 1709 return diag; 1710 1711 E1000_WRITE_REG(hw, E1000_VET, 1712 RTE_ETHER_TYPE_VLAN << 16 | RTE_ETHER_TYPE_VLAN); 1713 e1000_get_phy_info(hw); 1714 e1000_check_for_link(hw); 1715 1716 return 0; 1717 } 1718 1719 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1720 static void 1721 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1722 { 1723 int pause_frames; 1724 1725 uint64_t old_gprc = stats->gprc; 1726 uint64_t old_gptc = stats->gptc; 1727 uint64_t old_tpr = stats->tpr; 1728 uint64_t old_tpt = stats->tpt; 1729 uint64_t old_rpthc = stats->rpthc; 1730 uint64_t old_hgptc = stats->hgptc; 1731 1732 if(hw->phy.media_type == e1000_media_type_copper || 1733 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1734 stats->symerrs += 1735 E1000_READ_REG(hw,E1000_SYMERRS); 1736 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1737 } 1738 1739 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1740 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1741 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1742 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1743 1744 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1745 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1746 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1747 stats->dc += E1000_READ_REG(hw, E1000_DC); 1748 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1749 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1750 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1751 /* 1752 ** For watchdog management we need to know if we have been 1753 ** paused during the last interval, so capture that here. 1754 */ 1755 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1756 stats->xoffrxc += pause_frames; 1757 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1758 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1759 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1760 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1761 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1762 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1763 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1764 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1765 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1766 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1767 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1768 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1769 1770 /* For the 64-bit byte counters the low dword must be read first. */ 1771 /* Both registers clear on the read of the high dword */ 1772 1773 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1774 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1775 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1776 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN; 1777 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1778 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1779 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN; 1780 1781 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1782 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1783 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1784 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1785 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1786 1787 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1788 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1789 1790 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1791 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1792 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 1793 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1794 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1795 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN; 1796 1797 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1798 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1799 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1800 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1801 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1802 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1803 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1804 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1805 1806 /* Interrupt Counts */ 1807 1808 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1809 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1810 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1811 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1812 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1813 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1814 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1815 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1816 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1817 1818 /* Host to Card Statistics */ 1819 1820 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1821 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1822 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1823 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1824 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1825 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1826 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1827 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1828 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1829 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN; 1830 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1831 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1832 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN; 1833 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1834 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1835 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1836 1837 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1838 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1839 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1840 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1841 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1842 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1843 } 1844 1845 static int 1846 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1847 { 1848 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1849 struct e1000_hw_stats *stats = 1850 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1851 1852 igb_read_stats_registers(hw, stats); 1853 1854 if (rte_stats == NULL) 1855 return -EINVAL; 1856 1857 /* Rx Errors */ 1858 rte_stats->imissed = stats->mpc; 1859 rte_stats->ierrors = stats->crcerrs + 1860 stats->rlec + stats->ruc + stats->roc + 1861 stats->rxerrc + stats->algnerrc + stats->cexterr; 1862 1863 /* Tx Errors */ 1864 rte_stats->oerrors = stats->ecol + stats->latecol; 1865 1866 rte_stats->ipackets = stats->gprc; 1867 rte_stats->opackets = stats->gptc; 1868 rte_stats->ibytes = stats->gorc; 1869 rte_stats->obytes = stats->gotc; 1870 return 0; 1871 } 1872 1873 static void 1874 eth_igb_stats_reset(struct rte_eth_dev *dev) 1875 { 1876 struct e1000_hw_stats *hw_stats = 1877 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1878 1879 /* HW registers are cleared on read */ 1880 eth_igb_stats_get(dev, NULL); 1881 1882 /* Reset software totals */ 1883 memset(hw_stats, 0, sizeof(*hw_stats)); 1884 } 1885 1886 static void 1887 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1888 { 1889 struct e1000_hw_stats *stats = 1890 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1891 1892 /* HW registers are cleared on read */ 1893 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1894 1895 /* Reset software totals */ 1896 memset(stats, 0, sizeof(*stats)); 1897 } 1898 1899 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1900 struct rte_eth_xstat_name *xstats_names, 1901 __rte_unused unsigned int size) 1902 { 1903 unsigned i; 1904 1905 if (xstats_names == NULL) 1906 return IGB_NB_XSTATS; 1907 1908 /* Note: limit checked in rte_eth_xstats_names() */ 1909 1910 for (i = 0; i < IGB_NB_XSTATS; i++) { 1911 strlcpy(xstats_names[i].name, rte_igb_stats_strings[i].name, 1912 sizeof(xstats_names[i].name)); 1913 } 1914 1915 return IGB_NB_XSTATS; 1916 } 1917 1918 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1919 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1920 unsigned int limit) 1921 { 1922 unsigned int i; 1923 1924 if (!ids) { 1925 if (xstats_names == NULL) 1926 return IGB_NB_XSTATS; 1927 1928 for (i = 0; i < IGB_NB_XSTATS; i++) 1929 strlcpy(xstats_names[i].name, 1930 rte_igb_stats_strings[i].name, 1931 sizeof(xstats_names[i].name)); 1932 1933 return IGB_NB_XSTATS; 1934 1935 } else { 1936 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1937 1938 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1939 IGB_NB_XSTATS); 1940 1941 for (i = 0; i < limit; i++) { 1942 if (ids[i] >= IGB_NB_XSTATS) { 1943 PMD_INIT_LOG(ERR, "id value isn't valid"); 1944 return -1; 1945 } 1946 strcpy(xstats_names[i].name, 1947 xstats_names_copy[ids[i]].name); 1948 } 1949 return limit; 1950 } 1951 } 1952 1953 static int 1954 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1955 unsigned n) 1956 { 1957 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1958 struct e1000_hw_stats *hw_stats = 1959 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1960 unsigned i; 1961 1962 if (n < IGB_NB_XSTATS) 1963 return IGB_NB_XSTATS; 1964 1965 igb_read_stats_registers(hw, hw_stats); 1966 1967 /* If this is a reset xstats is NULL, and we have cleared the 1968 * registers by reading them. 1969 */ 1970 if (!xstats) 1971 return 0; 1972 1973 /* Extended stats */ 1974 for (i = 0; i < IGB_NB_XSTATS; i++) { 1975 xstats[i].id = i; 1976 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1977 rte_igb_stats_strings[i].offset); 1978 } 1979 1980 return IGB_NB_XSTATS; 1981 } 1982 1983 static int 1984 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1985 uint64_t *values, unsigned int n) 1986 { 1987 unsigned int i; 1988 1989 if (!ids) { 1990 struct e1000_hw *hw = 1991 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1992 struct e1000_hw_stats *hw_stats = 1993 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1994 1995 if (n < IGB_NB_XSTATS) 1996 return IGB_NB_XSTATS; 1997 1998 igb_read_stats_registers(hw, hw_stats); 1999 2000 /* If this is a reset xstats is NULL, and we have cleared the 2001 * registers by reading them. 2002 */ 2003 if (!values) 2004 return 0; 2005 2006 /* Extended stats */ 2007 for (i = 0; i < IGB_NB_XSTATS; i++) 2008 values[i] = *(uint64_t *)(((char *)hw_stats) + 2009 rte_igb_stats_strings[i].offset); 2010 2011 return IGB_NB_XSTATS; 2012 2013 } else { 2014 uint64_t values_copy[IGB_NB_XSTATS]; 2015 2016 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2017 IGB_NB_XSTATS); 2018 2019 for (i = 0; i < n; i++) { 2020 if (ids[i] >= IGB_NB_XSTATS) { 2021 PMD_INIT_LOG(ERR, "id value isn't valid"); 2022 return -1; 2023 } 2024 values[i] = values_copy[ids[i]]; 2025 } 2026 return n; 2027 } 2028 } 2029 2030 static void 2031 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2032 { 2033 /* Good Rx packets, include VF loopback */ 2034 UPDATE_VF_STAT(E1000_VFGPRC, 2035 hw_stats->last_gprc, hw_stats->gprc); 2036 2037 /* Good Rx octets, include VF loopback */ 2038 UPDATE_VF_STAT(E1000_VFGORC, 2039 hw_stats->last_gorc, hw_stats->gorc); 2040 2041 /* Good Tx packets, include VF loopback */ 2042 UPDATE_VF_STAT(E1000_VFGPTC, 2043 hw_stats->last_gptc, hw_stats->gptc); 2044 2045 /* Good Tx octets, include VF loopback */ 2046 UPDATE_VF_STAT(E1000_VFGOTC, 2047 hw_stats->last_gotc, hw_stats->gotc); 2048 2049 /* Rx Multicst packets */ 2050 UPDATE_VF_STAT(E1000_VFMPRC, 2051 hw_stats->last_mprc, hw_stats->mprc); 2052 2053 /* Good Rx loopback packets */ 2054 UPDATE_VF_STAT(E1000_VFGPRLBC, 2055 hw_stats->last_gprlbc, hw_stats->gprlbc); 2056 2057 /* Good Rx loopback octets */ 2058 UPDATE_VF_STAT(E1000_VFGORLBC, 2059 hw_stats->last_gorlbc, hw_stats->gorlbc); 2060 2061 /* Good Tx loopback packets */ 2062 UPDATE_VF_STAT(E1000_VFGPTLBC, 2063 hw_stats->last_gptlbc, hw_stats->gptlbc); 2064 2065 /* Good Tx loopback octets */ 2066 UPDATE_VF_STAT(E1000_VFGOTLBC, 2067 hw_stats->last_gotlbc, hw_stats->gotlbc); 2068 } 2069 2070 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2071 struct rte_eth_xstat_name *xstats_names, 2072 __rte_unused unsigned limit) 2073 { 2074 unsigned i; 2075 2076 if (xstats_names != NULL) 2077 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2078 strlcpy(xstats_names[i].name, 2079 rte_igbvf_stats_strings[i].name, 2080 sizeof(xstats_names[i].name)); 2081 } 2082 return IGBVF_NB_XSTATS; 2083 } 2084 2085 static int 2086 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2087 unsigned n) 2088 { 2089 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2090 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2091 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2092 unsigned i; 2093 2094 if (n < IGBVF_NB_XSTATS) 2095 return IGBVF_NB_XSTATS; 2096 2097 igbvf_read_stats_registers(hw, hw_stats); 2098 2099 if (!xstats) 2100 return 0; 2101 2102 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2103 xstats[i].id = i; 2104 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2105 rte_igbvf_stats_strings[i].offset); 2106 } 2107 2108 return IGBVF_NB_XSTATS; 2109 } 2110 2111 static int 2112 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2113 { 2114 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2115 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2116 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2117 2118 igbvf_read_stats_registers(hw, hw_stats); 2119 2120 if (rte_stats == NULL) 2121 return -EINVAL; 2122 2123 rte_stats->ipackets = hw_stats->gprc; 2124 rte_stats->ibytes = hw_stats->gorc; 2125 rte_stats->opackets = hw_stats->gptc; 2126 rte_stats->obytes = hw_stats->gotc; 2127 return 0; 2128 } 2129 2130 static void 2131 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2132 { 2133 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2134 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2135 2136 /* Sync HW register to the last stats */ 2137 eth_igbvf_stats_get(dev, NULL); 2138 2139 /* reset HW current stats*/ 2140 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2141 offsetof(struct e1000_vf_stats, gprc)); 2142 } 2143 2144 static int 2145 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2146 size_t fw_size) 2147 { 2148 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2149 struct e1000_fw_version fw; 2150 int ret; 2151 2152 e1000_get_fw_version(hw, &fw); 2153 2154 switch (hw->mac.type) { 2155 case e1000_i210: 2156 case e1000_i211: 2157 if (!(e1000_get_flash_presence_i210(hw))) { 2158 ret = snprintf(fw_version, fw_size, 2159 "%2d.%2d-%d", 2160 fw.invm_major, fw.invm_minor, 2161 fw.invm_img_type); 2162 break; 2163 } 2164 /* fall through */ 2165 default: 2166 /* if option rom is valid, display its version too */ 2167 if (fw.or_valid) { 2168 ret = snprintf(fw_version, fw_size, 2169 "%d.%d, 0x%08x, %d.%d.%d", 2170 fw.eep_major, fw.eep_minor, fw.etrack_id, 2171 fw.or_major, fw.or_build, fw.or_patch); 2172 /* no option rom */ 2173 } else { 2174 if (fw.etrack_id != 0X0000) { 2175 ret = snprintf(fw_version, fw_size, 2176 "%d.%d, 0x%08x", 2177 fw.eep_major, fw.eep_minor, 2178 fw.etrack_id); 2179 } else { 2180 ret = snprintf(fw_version, fw_size, 2181 "%d.%d.%d", 2182 fw.eep_major, fw.eep_minor, 2183 fw.eep_build); 2184 } 2185 } 2186 break; 2187 } 2188 2189 ret += 1; /* add the size of '\0' */ 2190 if (fw_size < (u32)ret) 2191 return ret; 2192 else 2193 return 0; 2194 } 2195 2196 static void 2197 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2198 { 2199 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2200 2201 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2202 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2203 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2204 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2205 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2206 dev_info->rx_queue_offload_capa; 2207 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2208 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2209 dev_info->tx_queue_offload_capa; 2210 2211 switch (hw->mac.type) { 2212 case e1000_82575: 2213 dev_info->max_rx_queues = 4; 2214 dev_info->max_tx_queues = 4; 2215 dev_info->max_vmdq_pools = 0; 2216 break; 2217 2218 case e1000_82576: 2219 dev_info->max_rx_queues = 16; 2220 dev_info->max_tx_queues = 16; 2221 dev_info->max_vmdq_pools = ETH_8_POOLS; 2222 dev_info->vmdq_queue_num = 16; 2223 break; 2224 2225 case e1000_82580: 2226 dev_info->max_rx_queues = 8; 2227 dev_info->max_tx_queues = 8; 2228 dev_info->max_vmdq_pools = ETH_8_POOLS; 2229 dev_info->vmdq_queue_num = 8; 2230 break; 2231 2232 case e1000_i350: 2233 dev_info->max_rx_queues = 8; 2234 dev_info->max_tx_queues = 8; 2235 dev_info->max_vmdq_pools = ETH_8_POOLS; 2236 dev_info->vmdq_queue_num = 8; 2237 break; 2238 2239 case e1000_i354: 2240 dev_info->max_rx_queues = 8; 2241 dev_info->max_tx_queues = 8; 2242 break; 2243 2244 case e1000_i210: 2245 dev_info->max_rx_queues = 4; 2246 dev_info->max_tx_queues = 4; 2247 dev_info->max_vmdq_pools = 0; 2248 break; 2249 2250 case e1000_i211: 2251 dev_info->max_rx_queues = 2; 2252 dev_info->max_tx_queues = 2; 2253 dev_info->max_vmdq_pools = 0; 2254 break; 2255 2256 default: 2257 /* Should not happen */ 2258 break; 2259 } 2260 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2261 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2262 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2263 2264 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2265 .rx_thresh = { 2266 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2267 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2268 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2269 }, 2270 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2271 .rx_drop_en = 0, 2272 .offloads = 0, 2273 }; 2274 2275 dev_info->default_txconf = (struct rte_eth_txconf) { 2276 .tx_thresh = { 2277 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2278 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2279 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2280 }, 2281 .offloads = 0, 2282 }; 2283 2284 dev_info->rx_desc_lim = rx_desc_lim; 2285 dev_info->tx_desc_lim = tx_desc_lim; 2286 2287 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2288 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2289 ETH_LINK_SPEED_1G; 2290 2291 dev_info->max_mtu = dev_info->max_rx_pktlen - E1000_ETH_OVERHEAD; 2292 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 2293 2294 } 2295 2296 static const uint32_t * 2297 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2298 { 2299 static const uint32_t ptypes[] = { 2300 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2301 RTE_PTYPE_L2_ETHER, 2302 RTE_PTYPE_L3_IPV4, 2303 RTE_PTYPE_L3_IPV4_EXT, 2304 RTE_PTYPE_L3_IPV6, 2305 RTE_PTYPE_L3_IPV6_EXT, 2306 RTE_PTYPE_L4_TCP, 2307 RTE_PTYPE_L4_UDP, 2308 RTE_PTYPE_L4_SCTP, 2309 RTE_PTYPE_TUNNEL_IP, 2310 RTE_PTYPE_INNER_L3_IPV6, 2311 RTE_PTYPE_INNER_L3_IPV6_EXT, 2312 RTE_PTYPE_INNER_L4_TCP, 2313 RTE_PTYPE_INNER_L4_UDP, 2314 RTE_PTYPE_UNKNOWN 2315 }; 2316 2317 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2318 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2319 return ptypes; 2320 return NULL; 2321 } 2322 2323 static void 2324 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2325 { 2326 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2327 2328 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2329 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2330 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2331 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2332 DEV_TX_OFFLOAD_IPV4_CKSUM | 2333 DEV_TX_OFFLOAD_UDP_CKSUM | 2334 DEV_TX_OFFLOAD_TCP_CKSUM | 2335 DEV_TX_OFFLOAD_SCTP_CKSUM | 2336 DEV_TX_OFFLOAD_TCP_TSO; 2337 switch (hw->mac.type) { 2338 case e1000_vfadapt: 2339 dev_info->max_rx_queues = 2; 2340 dev_info->max_tx_queues = 2; 2341 break; 2342 case e1000_vfadapt_i350: 2343 dev_info->max_rx_queues = 1; 2344 dev_info->max_tx_queues = 1; 2345 break; 2346 default: 2347 /* Should not happen */ 2348 break; 2349 } 2350 2351 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2352 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2353 dev_info->rx_queue_offload_capa; 2354 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2355 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2356 dev_info->tx_queue_offload_capa; 2357 2358 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2359 .rx_thresh = { 2360 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2361 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2362 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2363 }, 2364 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2365 .rx_drop_en = 0, 2366 .offloads = 0, 2367 }; 2368 2369 dev_info->default_txconf = (struct rte_eth_txconf) { 2370 .tx_thresh = { 2371 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2372 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2373 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2374 }, 2375 .offloads = 0, 2376 }; 2377 2378 dev_info->rx_desc_lim = rx_desc_lim; 2379 dev_info->tx_desc_lim = tx_desc_lim; 2380 } 2381 2382 /* return 0 means link status changed, -1 means not changed */ 2383 static int 2384 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2385 { 2386 struct e1000_hw *hw = 2387 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2388 struct rte_eth_link link; 2389 int link_check, count; 2390 2391 link_check = 0; 2392 hw->mac.get_link_status = 1; 2393 2394 /* possible wait-to-complete in up to 9 seconds */ 2395 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2396 /* Read the real link status */ 2397 switch (hw->phy.media_type) { 2398 case e1000_media_type_copper: 2399 /* Do the work to read phy */ 2400 e1000_check_for_link(hw); 2401 link_check = !hw->mac.get_link_status; 2402 break; 2403 2404 case e1000_media_type_fiber: 2405 e1000_check_for_link(hw); 2406 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2407 E1000_STATUS_LU); 2408 break; 2409 2410 case e1000_media_type_internal_serdes: 2411 e1000_check_for_link(hw); 2412 link_check = hw->mac.serdes_has_link; 2413 break; 2414 2415 /* VF device is type_unknown */ 2416 case e1000_media_type_unknown: 2417 eth_igbvf_link_update(hw); 2418 link_check = !hw->mac.get_link_status; 2419 break; 2420 2421 default: 2422 break; 2423 } 2424 if (link_check || wait_to_complete == 0) 2425 break; 2426 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2427 } 2428 memset(&link, 0, sizeof(link)); 2429 2430 /* Now we check if a transition has happened */ 2431 if (link_check) { 2432 uint16_t duplex, speed; 2433 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2434 link.link_duplex = (duplex == FULL_DUPLEX) ? 2435 ETH_LINK_FULL_DUPLEX : 2436 ETH_LINK_HALF_DUPLEX; 2437 link.link_speed = speed; 2438 link.link_status = ETH_LINK_UP; 2439 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2440 ETH_LINK_SPEED_FIXED); 2441 } else if (!link_check) { 2442 link.link_speed = 0; 2443 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2444 link.link_status = ETH_LINK_DOWN; 2445 link.link_autoneg = ETH_LINK_FIXED; 2446 } 2447 2448 return rte_eth_linkstatus_set(dev, &link); 2449 } 2450 2451 /* 2452 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2453 * For ASF and Pass Through versions of f/w this means 2454 * that the driver is loaded. 2455 */ 2456 static void 2457 igb_hw_control_acquire(struct e1000_hw *hw) 2458 { 2459 uint32_t ctrl_ext; 2460 2461 /* Let firmware know the driver has taken over */ 2462 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2463 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2464 } 2465 2466 /* 2467 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2468 * For ASF and Pass Through versions of f/w this means that the 2469 * driver is no longer loaded. 2470 */ 2471 static void 2472 igb_hw_control_release(struct e1000_hw *hw) 2473 { 2474 uint32_t ctrl_ext; 2475 2476 /* Let firmware taken over control of h/w */ 2477 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2478 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2479 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2480 } 2481 2482 /* 2483 * Bit of a misnomer, what this really means is 2484 * to enable OS management of the system... aka 2485 * to disable special hardware management features. 2486 */ 2487 static void 2488 igb_init_manageability(struct e1000_hw *hw) 2489 { 2490 if (e1000_enable_mng_pass_thru(hw)) { 2491 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2492 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2493 2494 /* disable hardware interception of ARP */ 2495 manc &= ~(E1000_MANC_ARP_EN); 2496 2497 /* enable receiving management packets to the host */ 2498 manc |= E1000_MANC_EN_MNG2HOST; 2499 manc2h |= 1 << 5; /* Mng Port 623 */ 2500 manc2h |= 1 << 6; /* Mng Port 664 */ 2501 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2502 E1000_WRITE_REG(hw, E1000_MANC, manc); 2503 } 2504 } 2505 2506 static void 2507 igb_release_manageability(struct e1000_hw *hw) 2508 { 2509 if (e1000_enable_mng_pass_thru(hw)) { 2510 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2511 2512 manc |= E1000_MANC_ARP_EN; 2513 manc &= ~E1000_MANC_EN_MNG2HOST; 2514 2515 E1000_WRITE_REG(hw, E1000_MANC, manc); 2516 } 2517 } 2518 2519 static void 2520 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2521 { 2522 struct e1000_hw *hw = 2523 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2524 uint32_t rctl; 2525 2526 rctl = E1000_READ_REG(hw, E1000_RCTL); 2527 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2528 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2529 } 2530 2531 static void 2532 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2533 { 2534 struct e1000_hw *hw = 2535 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2536 uint32_t rctl; 2537 2538 rctl = E1000_READ_REG(hw, E1000_RCTL); 2539 rctl &= (~E1000_RCTL_UPE); 2540 if (dev->data->all_multicast == 1) 2541 rctl |= E1000_RCTL_MPE; 2542 else 2543 rctl &= (~E1000_RCTL_MPE); 2544 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2545 } 2546 2547 static void 2548 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2549 { 2550 struct e1000_hw *hw = 2551 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2552 uint32_t rctl; 2553 2554 rctl = E1000_READ_REG(hw, E1000_RCTL); 2555 rctl |= E1000_RCTL_MPE; 2556 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2557 } 2558 2559 static void 2560 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2561 { 2562 struct e1000_hw *hw = 2563 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2564 uint32_t rctl; 2565 2566 if (dev->data->promiscuous == 1) 2567 return; /* must remain in all_multicast mode */ 2568 rctl = E1000_READ_REG(hw, E1000_RCTL); 2569 rctl &= (~E1000_RCTL_MPE); 2570 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2571 } 2572 2573 static int 2574 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2575 { 2576 struct e1000_hw *hw = 2577 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2578 struct e1000_vfta * shadow_vfta = 2579 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2580 uint32_t vfta; 2581 uint32_t vid_idx; 2582 uint32_t vid_bit; 2583 2584 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2585 E1000_VFTA_ENTRY_MASK); 2586 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2587 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2588 if (on) 2589 vfta |= vid_bit; 2590 else 2591 vfta &= ~vid_bit; 2592 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2593 2594 /* update local VFTA copy */ 2595 shadow_vfta->vfta[vid_idx] = vfta; 2596 2597 return 0; 2598 } 2599 2600 static int 2601 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2602 enum rte_vlan_type vlan_type, 2603 uint16_t tpid) 2604 { 2605 struct e1000_hw *hw = 2606 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2607 uint32_t reg, qinq; 2608 2609 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2610 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2611 2612 /* only outer TPID of double VLAN can be configured*/ 2613 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2614 reg = E1000_READ_REG(hw, E1000_VET); 2615 reg = (reg & (~E1000_VET_VET_EXT)) | 2616 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2617 E1000_WRITE_REG(hw, E1000_VET, reg); 2618 2619 return 0; 2620 } 2621 2622 /* all other TPID values are read-only*/ 2623 PMD_DRV_LOG(ERR, "Not supported"); 2624 2625 return -ENOTSUP; 2626 } 2627 2628 static void 2629 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2630 { 2631 struct e1000_hw *hw = 2632 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2633 uint32_t reg; 2634 2635 /* Filter Table Disable */ 2636 reg = E1000_READ_REG(hw, E1000_RCTL); 2637 reg &= ~E1000_RCTL_CFIEN; 2638 reg &= ~E1000_RCTL_VFE; 2639 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2640 } 2641 2642 static void 2643 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2644 { 2645 struct e1000_hw *hw = 2646 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2647 struct e1000_vfta * shadow_vfta = 2648 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2649 uint32_t reg; 2650 int i; 2651 2652 /* Filter Table Enable, CFI not used for packet acceptance */ 2653 reg = E1000_READ_REG(hw, E1000_RCTL); 2654 reg &= ~E1000_RCTL_CFIEN; 2655 reg |= E1000_RCTL_VFE; 2656 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2657 2658 /* restore VFTA table */ 2659 for (i = 0; i < IGB_VFTA_SIZE; i++) 2660 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2661 } 2662 2663 static void 2664 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2665 { 2666 struct e1000_hw *hw = 2667 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2668 uint32_t reg; 2669 2670 /* VLAN Mode Disable */ 2671 reg = E1000_READ_REG(hw, E1000_CTRL); 2672 reg &= ~E1000_CTRL_VME; 2673 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2674 } 2675 2676 static void 2677 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2678 { 2679 struct e1000_hw *hw = 2680 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2681 uint32_t reg; 2682 2683 /* VLAN Mode Enable */ 2684 reg = E1000_READ_REG(hw, E1000_CTRL); 2685 reg |= E1000_CTRL_VME; 2686 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2687 } 2688 2689 static void 2690 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2691 { 2692 struct e1000_hw *hw = 2693 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2694 uint32_t reg; 2695 2696 /* CTRL_EXT: Extended VLAN */ 2697 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2698 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2699 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2700 2701 /* Update maximum packet length */ 2702 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2703 E1000_WRITE_REG(hw, E1000_RLPML, 2704 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2705 VLAN_TAG_SIZE); 2706 } 2707 2708 static void 2709 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2710 { 2711 struct e1000_hw *hw = 2712 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2713 uint32_t reg; 2714 2715 /* CTRL_EXT: Extended VLAN */ 2716 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2717 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2718 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2719 2720 /* Update maximum packet length */ 2721 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2722 E1000_WRITE_REG(hw, E1000_RLPML, 2723 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2724 2 * VLAN_TAG_SIZE); 2725 } 2726 2727 static int 2728 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2729 { 2730 struct rte_eth_rxmode *rxmode; 2731 2732 rxmode = &dev->data->dev_conf.rxmode; 2733 if(mask & ETH_VLAN_STRIP_MASK){ 2734 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2735 igb_vlan_hw_strip_enable(dev); 2736 else 2737 igb_vlan_hw_strip_disable(dev); 2738 } 2739 2740 if(mask & ETH_VLAN_FILTER_MASK){ 2741 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2742 igb_vlan_hw_filter_enable(dev); 2743 else 2744 igb_vlan_hw_filter_disable(dev); 2745 } 2746 2747 if(mask & ETH_VLAN_EXTEND_MASK){ 2748 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2749 igb_vlan_hw_extend_enable(dev); 2750 else 2751 igb_vlan_hw_extend_disable(dev); 2752 } 2753 2754 return 0; 2755 } 2756 2757 2758 /** 2759 * It enables the interrupt mask and then enable the interrupt. 2760 * 2761 * @param dev 2762 * Pointer to struct rte_eth_dev. 2763 * @param on 2764 * Enable or Disable 2765 * 2766 * @return 2767 * - On success, zero. 2768 * - On failure, a negative value. 2769 */ 2770 static int 2771 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2772 { 2773 struct e1000_interrupt *intr = 2774 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2775 2776 if (on) 2777 intr->mask |= E1000_ICR_LSC; 2778 else 2779 intr->mask &= ~E1000_ICR_LSC; 2780 2781 return 0; 2782 } 2783 2784 /* It clears the interrupt causes and enables the interrupt. 2785 * It will be called once only during nic initialized. 2786 * 2787 * @param dev 2788 * Pointer to struct rte_eth_dev. 2789 * 2790 * @return 2791 * - On success, zero. 2792 * - On failure, a negative value. 2793 */ 2794 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2795 { 2796 uint32_t mask, regval; 2797 struct e1000_hw *hw = 2798 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2799 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2800 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2801 int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0; 2802 struct rte_eth_dev_info dev_info; 2803 2804 memset(&dev_info, 0, sizeof(dev_info)); 2805 eth_igb_infos_get(dev, &dev_info); 2806 2807 mask = (0xFFFFFFFF >> (32 - dev_info.max_rx_queues)) << misc_shift; 2808 regval = E1000_READ_REG(hw, E1000_EIMS); 2809 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2810 2811 return 0; 2812 } 2813 2814 /* 2815 * It reads ICR and gets interrupt causes, check it and set a bit flag 2816 * to update link status. 2817 * 2818 * @param dev 2819 * Pointer to struct rte_eth_dev. 2820 * 2821 * @return 2822 * - On success, zero. 2823 * - On failure, a negative value. 2824 */ 2825 static int 2826 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2827 { 2828 uint32_t icr; 2829 struct e1000_hw *hw = 2830 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2831 struct e1000_interrupt *intr = 2832 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2833 2834 igb_intr_disable(dev); 2835 2836 /* read-on-clear nic registers here */ 2837 icr = E1000_READ_REG(hw, E1000_ICR); 2838 2839 intr->flags = 0; 2840 if (icr & E1000_ICR_LSC) { 2841 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2842 } 2843 2844 if (icr & E1000_ICR_VMMB) 2845 intr->flags |= E1000_FLAG_MAILBOX; 2846 2847 return 0; 2848 } 2849 2850 /* 2851 * It executes link_update after knowing an interrupt is prsent. 2852 * 2853 * @param dev 2854 * Pointer to struct rte_eth_dev. 2855 * 2856 * @return 2857 * - On success, zero. 2858 * - On failure, a negative value. 2859 */ 2860 static int 2861 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2862 struct rte_intr_handle *intr_handle) 2863 { 2864 struct e1000_hw *hw = 2865 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2866 struct e1000_interrupt *intr = 2867 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2868 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2869 struct rte_eth_link link; 2870 int ret; 2871 2872 if (intr->flags & E1000_FLAG_MAILBOX) { 2873 igb_pf_mbx_process(dev); 2874 intr->flags &= ~E1000_FLAG_MAILBOX; 2875 } 2876 2877 igb_intr_enable(dev); 2878 rte_intr_ack(intr_handle); 2879 2880 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2881 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2882 2883 /* set get_link_status to check register later */ 2884 hw->mac.get_link_status = 1; 2885 ret = eth_igb_link_update(dev, 0); 2886 2887 /* check if link has changed */ 2888 if (ret < 0) 2889 return 0; 2890 2891 rte_eth_linkstatus_get(dev, &link); 2892 if (link.link_status) { 2893 PMD_INIT_LOG(INFO, 2894 " Port %d: Link Up - speed %u Mbps - %s", 2895 dev->data->port_id, 2896 (unsigned)link.link_speed, 2897 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2898 "full-duplex" : "half-duplex"); 2899 } else { 2900 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2901 dev->data->port_id); 2902 } 2903 2904 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2905 pci_dev->addr.domain, 2906 pci_dev->addr.bus, 2907 pci_dev->addr.devid, 2908 pci_dev->addr.function); 2909 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2910 NULL); 2911 } 2912 2913 return 0; 2914 } 2915 2916 /** 2917 * Interrupt handler which shall be registered at first. 2918 * 2919 * @param handle 2920 * Pointer to interrupt handle. 2921 * @param param 2922 * The address of parameter (struct rte_eth_dev *) regsitered before. 2923 * 2924 * @return 2925 * void 2926 */ 2927 static void 2928 eth_igb_interrupt_handler(void *param) 2929 { 2930 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2931 2932 eth_igb_interrupt_get_status(dev); 2933 eth_igb_interrupt_action(dev, dev->intr_handle); 2934 } 2935 2936 static int 2937 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2938 { 2939 uint32_t eicr; 2940 struct e1000_hw *hw = 2941 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2942 struct e1000_interrupt *intr = 2943 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2944 2945 igbvf_intr_disable(hw); 2946 2947 /* read-on-clear nic registers here */ 2948 eicr = E1000_READ_REG(hw, E1000_EICR); 2949 intr->flags = 0; 2950 2951 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2952 intr->flags |= E1000_FLAG_MAILBOX; 2953 2954 return 0; 2955 } 2956 2957 void igbvf_mbx_process(struct rte_eth_dev *dev) 2958 { 2959 struct e1000_hw *hw = 2960 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2961 struct e1000_mbx_info *mbx = &hw->mbx; 2962 u32 in_msg = 0; 2963 2964 /* peek the message first */ 2965 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2966 2967 /* PF reset VF event */ 2968 if (in_msg == E1000_PF_CONTROL_MSG) { 2969 /* dummy mbx read to ack pf */ 2970 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2971 return; 2972 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2973 NULL); 2974 } 2975 } 2976 2977 static int 2978 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2979 { 2980 struct e1000_interrupt *intr = 2981 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2982 2983 if (intr->flags & E1000_FLAG_MAILBOX) { 2984 igbvf_mbx_process(dev); 2985 intr->flags &= ~E1000_FLAG_MAILBOX; 2986 } 2987 2988 igbvf_intr_enable(dev); 2989 rte_intr_ack(intr_handle); 2990 2991 return 0; 2992 } 2993 2994 static void 2995 eth_igbvf_interrupt_handler(void *param) 2996 { 2997 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2998 2999 eth_igbvf_interrupt_get_status(dev); 3000 eth_igbvf_interrupt_action(dev, dev->intr_handle); 3001 } 3002 3003 static int 3004 eth_igb_led_on(struct rte_eth_dev *dev) 3005 { 3006 struct e1000_hw *hw; 3007 3008 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3009 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3010 } 3011 3012 static int 3013 eth_igb_led_off(struct rte_eth_dev *dev) 3014 { 3015 struct e1000_hw *hw; 3016 3017 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3018 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3019 } 3020 3021 static int 3022 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3023 { 3024 struct e1000_hw *hw; 3025 uint32_t ctrl; 3026 int tx_pause; 3027 int rx_pause; 3028 3029 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3030 fc_conf->pause_time = hw->fc.pause_time; 3031 fc_conf->high_water = hw->fc.high_water; 3032 fc_conf->low_water = hw->fc.low_water; 3033 fc_conf->send_xon = hw->fc.send_xon; 3034 fc_conf->autoneg = hw->mac.autoneg; 3035 3036 /* 3037 * Return rx_pause and tx_pause status according to actual setting of 3038 * the TFCE and RFCE bits in the CTRL register. 3039 */ 3040 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3041 if (ctrl & E1000_CTRL_TFCE) 3042 tx_pause = 1; 3043 else 3044 tx_pause = 0; 3045 3046 if (ctrl & E1000_CTRL_RFCE) 3047 rx_pause = 1; 3048 else 3049 rx_pause = 0; 3050 3051 if (rx_pause && tx_pause) 3052 fc_conf->mode = RTE_FC_FULL; 3053 else if (rx_pause) 3054 fc_conf->mode = RTE_FC_RX_PAUSE; 3055 else if (tx_pause) 3056 fc_conf->mode = RTE_FC_TX_PAUSE; 3057 else 3058 fc_conf->mode = RTE_FC_NONE; 3059 3060 return 0; 3061 } 3062 3063 static int 3064 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3065 { 3066 struct e1000_hw *hw; 3067 int err; 3068 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3069 e1000_fc_none, 3070 e1000_fc_rx_pause, 3071 e1000_fc_tx_pause, 3072 e1000_fc_full 3073 }; 3074 uint32_t rx_buf_size; 3075 uint32_t max_high_water; 3076 uint32_t rctl; 3077 3078 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3079 if (fc_conf->autoneg != hw->mac.autoneg) 3080 return -ENOTSUP; 3081 rx_buf_size = igb_get_rx_buffer_size(hw); 3082 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3083 3084 /* At least reserve one Ethernet frame for watermark */ 3085 max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN; 3086 if ((fc_conf->high_water > max_high_water) || 3087 (fc_conf->high_water < fc_conf->low_water)) { 3088 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3089 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3090 return -EINVAL; 3091 } 3092 3093 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3094 hw->fc.pause_time = fc_conf->pause_time; 3095 hw->fc.high_water = fc_conf->high_water; 3096 hw->fc.low_water = fc_conf->low_water; 3097 hw->fc.send_xon = fc_conf->send_xon; 3098 3099 err = e1000_setup_link_generic(hw); 3100 if (err == E1000_SUCCESS) { 3101 3102 /* check if we want to forward MAC frames - driver doesn't have native 3103 * capability to do that, so we'll write the registers ourselves */ 3104 3105 rctl = E1000_READ_REG(hw, E1000_RCTL); 3106 3107 /* set or clear MFLCN.PMCF bit depending on configuration */ 3108 if (fc_conf->mac_ctrl_frame_fwd != 0) 3109 rctl |= E1000_RCTL_PMCF; 3110 else 3111 rctl &= ~E1000_RCTL_PMCF; 3112 3113 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3114 E1000_WRITE_FLUSH(hw); 3115 3116 return 0; 3117 } 3118 3119 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3120 return -EIO; 3121 } 3122 3123 #define E1000_RAH_POOLSEL_SHIFT (18) 3124 static int 3125 eth_igb_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 3126 uint32_t index, uint32_t pool) 3127 { 3128 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3129 uint32_t rah; 3130 3131 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3132 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3133 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3134 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3135 return 0; 3136 } 3137 3138 static void 3139 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3140 { 3141 uint8_t addr[RTE_ETHER_ADDR_LEN]; 3142 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3143 3144 memset(addr, 0, sizeof(addr)); 3145 3146 e1000_rar_set(hw, addr, index); 3147 } 3148 3149 static int 3150 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3151 struct rte_ether_addr *addr) 3152 { 3153 eth_igb_rar_clear(dev, 0); 3154 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3155 3156 return 0; 3157 } 3158 /* 3159 * Virtual Function operations 3160 */ 3161 static void 3162 igbvf_intr_disable(struct e1000_hw *hw) 3163 { 3164 PMD_INIT_FUNC_TRACE(); 3165 3166 /* Clear interrupt mask to stop from interrupts being generated */ 3167 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3168 3169 E1000_WRITE_FLUSH(hw); 3170 } 3171 3172 static void 3173 igbvf_stop_adapter(struct rte_eth_dev *dev) 3174 { 3175 u32 reg_val; 3176 u16 i; 3177 struct rte_eth_dev_info dev_info; 3178 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3179 3180 memset(&dev_info, 0, sizeof(dev_info)); 3181 eth_igbvf_infos_get(dev, &dev_info); 3182 3183 /* Clear interrupt mask to stop from interrupts being generated */ 3184 igbvf_intr_disable(hw); 3185 3186 /* Clear any pending interrupts, flush previous writes */ 3187 E1000_READ_REG(hw, E1000_EICR); 3188 3189 /* Disable the transmit unit. Each queue must be disabled. */ 3190 for (i = 0; i < dev_info.max_tx_queues; i++) 3191 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3192 3193 /* Disable the receive unit by stopping each queue */ 3194 for (i = 0; i < dev_info.max_rx_queues; i++) { 3195 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3196 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3197 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3198 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3199 ; 3200 } 3201 3202 /* flush all queues disables */ 3203 E1000_WRITE_FLUSH(hw); 3204 msec_delay(2); 3205 } 3206 3207 static int eth_igbvf_link_update(struct e1000_hw *hw) 3208 { 3209 struct e1000_mbx_info *mbx = &hw->mbx; 3210 struct e1000_mac_info *mac = &hw->mac; 3211 int ret_val = E1000_SUCCESS; 3212 3213 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3214 3215 /* 3216 * We only want to run this if there has been a rst asserted. 3217 * in this case that could mean a link change, device reset, 3218 * or a virtual function reset 3219 */ 3220 3221 /* If we were hit with a reset or timeout drop the link */ 3222 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3223 mac->get_link_status = TRUE; 3224 3225 if (!mac->get_link_status) 3226 goto out; 3227 3228 /* if link status is down no point in checking to see if pf is up */ 3229 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3230 goto out; 3231 3232 /* if we passed all the tests above then the link is up and we no 3233 * longer need to check for link */ 3234 mac->get_link_status = FALSE; 3235 3236 out: 3237 return ret_val; 3238 } 3239 3240 3241 static int 3242 igbvf_dev_configure(struct rte_eth_dev *dev) 3243 { 3244 struct rte_eth_conf* conf = &dev->data->dev_conf; 3245 3246 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3247 dev->data->port_id); 3248 3249 /* 3250 * VF has no ability to enable/disable HW CRC 3251 * Keep the persistent behavior the same as Host PF 3252 */ 3253 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3254 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3255 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3256 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3257 } 3258 #else 3259 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3260 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3261 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3262 } 3263 #endif 3264 3265 return 0; 3266 } 3267 3268 static int 3269 igbvf_dev_start(struct rte_eth_dev *dev) 3270 { 3271 struct e1000_hw *hw = 3272 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3273 struct e1000_adapter *adapter = 3274 E1000_DEV_PRIVATE(dev->data->dev_private); 3275 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3276 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3277 int ret; 3278 uint32_t intr_vector = 0; 3279 3280 PMD_INIT_FUNC_TRACE(); 3281 3282 hw->mac.ops.reset_hw(hw); 3283 adapter->stopped = 0; 3284 3285 /* Set all vfta */ 3286 igbvf_set_vfta_all(dev,1); 3287 3288 eth_igbvf_tx_init(dev); 3289 3290 /* This can fail when allocating mbufs for descriptor rings */ 3291 ret = eth_igbvf_rx_init(dev); 3292 if (ret) { 3293 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3294 igb_dev_clear_queues(dev); 3295 return ret; 3296 } 3297 3298 /* check and configure queue intr-vector mapping */ 3299 if (rte_intr_cap_multiple(intr_handle) && 3300 dev->data->dev_conf.intr_conf.rxq) { 3301 intr_vector = dev->data->nb_rx_queues; 3302 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3303 if (ret) 3304 return ret; 3305 } 3306 3307 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3308 intr_handle->intr_vec = 3309 rte_zmalloc("intr_vec", 3310 dev->data->nb_rx_queues * sizeof(int), 0); 3311 if (!intr_handle->intr_vec) { 3312 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3313 " intr_vec", dev->data->nb_rx_queues); 3314 return -ENOMEM; 3315 } 3316 } 3317 3318 eth_igbvf_configure_msix_intr(dev); 3319 3320 /* enable uio/vfio intr/eventfd mapping */ 3321 rte_intr_enable(intr_handle); 3322 3323 /* resume enabled intr since hw reset */ 3324 igbvf_intr_enable(dev); 3325 3326 return 0; 3327 } 3328 3329 static void 3330 igbvf_dev_stop(struct rte_eth_dev *dev) 3331 { 3332 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3333 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3334 3335 PMD_INIT_FUNC_TRACE(); 3336 3337 igbvf_stop_adapter(dev); 3338 3339 /* 3340 * Clear what we set, but we still keep shadow_vfta to 3341 * restore after device starts 3342 */ 3343 igbvf_set_vfta_all(dev,0); 3344 3345 igb_dev_clear_queues(dev); 3346 3347 /* disable intr eventfd mapping */ 3348 rte_intr_disable(intr_handle); 3349 3350 /* Clean datapath event and queue/vec mapping */ 3351 rte_intr_efd_disable(intr_handle); 3352 if (intr_handle->intr_vec) { 3353 rte_free(intr_handle->intr_vec); 3354 intr_handle->intr_vec = NULL; 3355 } 3356 } 3357 3358 static void 3359 igbvf_dev_close(struct rte_eth_dev *dev) 3360 { 3361 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3362 struct e1000_adapter *adapter = 3363 E1000_DEV_PRIVATE(dev->data->dev_private); 3364 struct rte_ether_addr addr; 3365 3366 PMD_INIT_FUNC_TRACE(); 3367 3368 e1000_reset_hw(hw); 3369 3370 igbvf_dev_stop(dev); 3371 adapter->stopped = 1; 3372 igb_dev_free_queues(dev); 3373 3374 /** 3375 * reprogram the RAR with a zero mac address, 3376 * to ensure that the VF traffic goes to the PF 3377 * after stop, close and detach of the VF. 3378 **/ 3379 3380 memset(&addr, 0, sizeof(addr)); 3381 igbvf_default_mac_addr_set(dev, &addr); 3382 } 3383 3384 static void 3385 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3386 { 3387 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3388 3389 /* Set both unicast and multicast promisc */ 3390 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3391 } 3392 3393 static void 3394 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3395 { 3396 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3397 3398 /* If in allmulticast mode leave multicast promisc */ 3399 if (dev->data->all_multicast == 1) 3400 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3401 else 3402 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3403 } 3404 3405 static void 3406 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3407 { 3408 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3409 3410 /* In promiscuous mode multicast promisc already set */ 3411 if (dev->data->promiscuous == 0) 3412 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3413 } 3414 3415 static void 3416 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3417 { 3418 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3419 3420 /* In promiscuous mode leave multicast promisc enabled */ 3421 if (dev->data->promiscuous == 0) 3422 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3423 } 3424 3425 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3426 { 3427 struct e1000_mbx_info *mbx = &hw->mbx; 3428 uint32_t msgbuf[2]; 3429 s32 err; 3430 3431 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3432 msgbuf[0] = E1000_VF_SET_VLAN; 3433 msgbuf[1] = vid; 3434 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3435 if (on) 3436 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3437 3438 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3439 if (err) 3440 goto mbx_err; 3441 3442 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3443 if (err) 3444 goto mbx_err; 3445 3446 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3447 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3448 err = -EINVAL; 3449 3450 mbx_err: 3451 return err; 3452 } 3453 3454 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3455 { 3456 struct e1000_hw *hw = 3457 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3458 struct e1000_vfta * shadow_vfta = 3459 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3460 int i = 0, j = 0, vfta = 0, mask = 1; 3461 3462 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3463 vfta = shadow_vfta->vfta[i]; 3464 if(vfta){ 3465 mask = 1; 3466 for (j = 0; j < 32; j++){ 3467 if(vfta & mask) 3468 igbvf_set_vfta(hw, 3469 (uint16_t)((i<<5)+j), on); 3470 mask<<=1; 3471 } 3472 } 3473 } 3474 3475 } 3476 3477 static int 3478 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3479 { 3480 struct e1000_hw *hw = 3481 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3482 struct e1000_vfta * shadow_vfta = 3483 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3484 uint32_t vid_idx = 0; 3485 uint32_t vid_bit = 0; 3486 int ret = 0; 3487 3488 PMD_INIT_FUNC_TRACE(); 3489 3490 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3491 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3492 if(ret){ 3493 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3494 return ret; 3495 } 3496 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3497 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3498 3499 /*Save what we set and retore it after device reset*/ 3500 if (on) 3501 shadow_vfta->vfta[vid_idx] |= vid_bit; 3502 else 3503 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3504 3505 return 0; 3506 } 3507 3508 static int 3509 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 3510 { 3511 struct e1000_hw *hw = 3512 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3513 3514 /* index is not used by rar_set() */ 3515 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3516 return 0; 3517 } 3518 3519 3520 static int 3521 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3522 struct rte_eth_rss_reta_entry64 *reta_conf, 3523 uint16_t reta_size) 3524 { 3525 uint8_t i, j, mask; 3526 uint32_t reta, r; 3527 uint16_t idx, shift; 3528 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3529 3530 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3531 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3532 "(%d) doesn't match the number hardware can supported " 3533 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3534 return -EINVAL; 3535 } 3536 3537 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3538 idx = i / RTE_RETA_GROUP_SIZE; 3539 shift = i % RTE_RETA_GROUP_SIZE; 3540 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3541 IGB_4_BIT_MASK); 3542 if (!mask) 3543 continue; 3544 if (mask == IGB_4_BIT_MASK) 3545 r = 0; 3546 else 3547 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3548 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3549 if (mask & (0x1 << j)) 3550 reta |= reta_conf[idx].reta[shift + j] << 3551 (CHAR_BIT * j); 3552 else 3553 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3554 } 3555 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3556 } 3557 3558 return 0; 3559 } 3560 3561 static int 3562 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3563 struct rte_eth_rss_reta_entry64 *reta_conf, 3564 uint16_t reta_size) 3565 { 3566 uint8_t i, j, mask; 3567 uint32_t reta; 3568 uint16_t idx, shift; 3569 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3570 3571 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3572 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3573 "(%d) doesn't match the number hardware can supported " 3574 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3575 return -EINVAL; 3576 } 3577 3578 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3579 idx = i / RTE_RETA_GROUP_SIZE; 3580 shift = i % RTE_RETA_GROUP_SIZE; 3581 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3582 IGB_4_BIT_MASK); 3583 if (!mask) 3584 continue; 3585 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3586 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3587 if (mask & (0x1 << j)) 3588 reta_conf[idx].reta[shift + j] = 3589 ((reta >> (CHAR_BIT * j)) & 3590 IGB_8_BIT_MASK); 3591 } 3592 } 3593 3594 return 0; 3595 } 3596 3597 int 3598 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3599 struct rte_eth_syn_filter *filter, 3600 bool add) 3601 { 3602 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3603 struct e1000_filter_info *filter_info = 3604 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3605 uint32_t synqf, rfctl; 3606 3607 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3608 return -EINVAL; 3609 3610 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3611 3612 if (add) { 3613 if (synqf & E1000_SYN_FILTER_ENABLE) 3614 return -EINVAL; 3615 3616 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3617 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3618 3619 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3620 if (filter->hig_pri) 3621 rfctl |= E1000_RFCTL_SYNQFP; 3622 else 3623 rfctl &= ~E1000_RFCTL_SYNQFP; 3624 3625 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3626 } else { 3627 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3628 return -ENOENT; 3629 synqf = 0; 3630 } 3631 3632 filter_info->syn_info = synqf; 3633 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3634 E1000_WRITE_FLUSH(hw); 3635 return 0; 3636 } 3637 3638 static int 3639 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3640 struct rte_eth_syn_filter *filter) 3641 { 3642 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3643 uint32_t synqf, rfctl; 3644 3645 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3646 if (synqf & E1000_SYN_FILTER_ENABLE) { 3647 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3648 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3649 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3650 E1000_SYN_FILTER_QUEUE_SHIFT); 3651 return 0; 3652 } 3653 3654 return -ENOENT; 3655 } 3656 3657 static int 3658 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3659 enum rte_filter_op filter_op, 3660 void *arg) 3661 { 3662 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3663 int ret; 3664 3665 MAC_TYPE_FILTER_SUP(hw->mac.type); 3666 3667 if (filter_op == RTE_ETH_FILTER_NOP) 3668 return 0; 3669 3670 if (arg == NULL) { 3671 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3672 filter_op); 3673 return -EINVAL; 3674 } 3675 3676 switch (filter_op) { 3677 case RTE_ETH_FILTER_ADD: 3678 ret = eth_igb_syn_filter_set(dev, 3679 (struct rte_eth_syn_filter *)arg, 3680 TRUE); 3681 break; 3682 case RTE_ETH_FILTER_DELETE: 3683 ret = eth_igb_syn_filter_set(dev, 3684 (struct rte_eth_syn_filter *)arg, 3685 FALSE); 3686 break; 3687 case RTE_ETH_FILTER_GET: 3688 ret = eth_igb_syn_filter_get(dev, 3689 (struct rte_eth_syn_filter *)arg); 3690 break; 3691 default: 3692 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3693 ret = -EINVAL; 3694 break; 3695 } 3696 3697 return ret; 3698 } 3699 3700 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3701 static inline int 3702 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3703 struct e1000_2tuple_filter_info *filter_info) 3704 { 3705 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3706 return -EINVAL; 3707 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3708 return -EINVAL; /* filter index is out of range. */ 3709 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 3710 return -EINVAL; /* flags is invalid. */ 3711 3712 switch (filter->dst_port_mask) { 3713 case UINT16_MAX: 3714 filter_info->dst_port_mask = 0; 3715 filter_info->dst_port = filter->dst_port; 3716 break; 3717 case 0: 3718 filter_info->dst_port_mask = 1; 3719 break; 3720 default: 3721 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3722 return -EINVAL; 3723 } 3724 3725 switch (filter->proto_mask) { 3726 case UINT8_MAX: 3727 filter_info->proto_mask = 0; 3728 filter_info->proto = filter->proto; 3729 break; 3730 case 0: 3731 filter_info->proto_mask = 1; 3732 break; 3733 default: 3734 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3735 return -EINVAL; 3736 } 3737 3738 filter_info->priority = (uint8_t)filter->priority; 3739 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3740 filter_info->tcp_flags = filter->tcp_flags; 3741 else 3742 filter_info->tcp_flags = 0; 3743 3744 return 0; 3745 } 3746 3747 static inline struct e1000_2tuple_filter * 3748 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3749 struct e1000_2tuple_filter_info *key) 3750 { 3751 struct e1000_2tuple_filter *it; 3752 3753 TAILQ_FOREACH(it, filter_list, entries) { 3754 if (memcmp(key, &it->filter_info, 3755 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3756 return it; 3757 } 3758 } 3759 return NULL; 3760 } 3761 3762 /* inject a igb 2tuple filter to HW */ 3763 static inline void 3764 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3765 struct e1000_2tuple_filter *filter) 3766 { 3767 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3768 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3769 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3770 int i; 3771 3772 i = filter->index; 3773 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3774 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3775 imir |= E1000_IMIR_PORT_BP; 3776 else 3777 imir &= ~E1000_IMIR_PORT_BP; 3778 3779 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3780 3781 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3782 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3783 ttqf |= (uint32_t)(filter->filter_info.proto & 3784 E1000_TTQF_PROTOCOL_MASK); 3785 if (filter->filter_info.proto_mask == 0) 3786 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3787 3788 /* tcp flags bits setting. */ 3789 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 3790 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 3791 imir_ext |= E1000_IMIREXT_CTRL_URG; 3792 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 3793 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3794 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 3795 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3796 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 3797 imir_ext |= E1000_IMIREXT_CTRL_RST; 3798 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 3799 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3800 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 3801 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3802 } else { 3803 imir_ext |= E1000_IMIREXT_CTRL_BP; 3804 } 3805 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3806 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3807 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3808 } 3809 3810 /* 3811 * igb_add_2tuple_filter - add a 2tuple filter 3812 * 3813 * @param 3814 * dev: Pointer to struct rte_eth_dev. 3815 * ntuple_filter: ponter to the filter that will be added. 3816 * 3817 * @return 3818 * - On success, zero. 3819 * - On failure, a negative value. 3820 */ 3821 static int 3822 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3823 struct rte_eth_ntuple_filter *ntuple_filter) 3824 { 3825 struct e1000_filter_info *filter_info = 3826 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3827 struct e1000_2tuple_filter *filter; 3828 int i, ret; 3829 3830 filter = rte_zmalloc("e1000_2tuple_filter", 3831 sizeof(struct e1000_2tuple_filter), 0); 3832 if (filter == NULL) 3833 return -ENOMEM; 3834 3835 ret = ntuple_filter_to_2tuple(ntuple_filter, 3836 &filter->filter_info); 3837 if (ret < 0) { 3838 rte_free(filter); 3839 return ret; 3840 } 3841 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3842 &filter->filter_info) != NULL) { 3843 PMD_DRV_LOG(ERR, "filter exists."); 3844 rte_free(filter); 3845 return -EEXIST; 3846 } 3847 filter->queue = ntuple_filter->queue; 3848 3849 /* 3850 * look for an unused 2tuple filter index, 3851 * and insert the filter to list. 3852 */ 3853 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3854 if (!(filter_info->twotuple_mask & (1 << i))) { 3855 filter_info->twotuple_mask |= 1 << i; 3856 filter->index = i; 3857 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3858 filter, 3859 entries); 3860 break; 3861 } 3862 } 3863 if (i >= E1000_MAX_TTQF_FILTERS) { 3864 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3865 rte_free(filter); 3866 return -ENOSYS; 3867 } 3868 3869 igb_inject_2uple_filter(dev, filter); 3870 return 0; 3871 } 3872 3873 int 3874 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3875 struct e1000_2tuple_filter *filter) 3876 { 3877 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3878 struct e1000_filter_info *filter_info = 3879 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3880 3881 filter_info->twotuple_mask &= ~(1 << filter->index); 3882 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3883 rte_free(filter); 3884 3885 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3886 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3887 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3888 return 0; 3889 } 3890 3891 /* 3892 * igb_remove_2tuple_filter - remove a 2tuple filter 3893 * 3894 * @param 3895 * dev: Pointer to struct rte_eth_dev. 3896 * ntuple_filter: ponter to the filter that will be removed. 3897 * 3898 * @return 3899 * - On success, zero. 3900 * - On failure, a negative value. 3901 */ 3902 static int 3903 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3904 struct rte_eth_ntuple_filter *ntuple_filter) 3905 { 3906 struct e1000_filter_info *filter_info = 3907 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3908 struct e1000_2tuple_filter_info filter_2tuple; 3909 struct e1000_2tuple_filter *filter; 3910 int ret; 3911 3912 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3913 ret = ntuple_filter_to_2tuple(ntuple_filter, 3914 &filter_2tuple); 3915 if (ret < 0) 3916 return ret; 3917 3918 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3919 &filter_2tuple); 3920 if (filter == NULL) { 3921 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3922 return -ENOENT; 3923 } 3924 3925 igb_delete_2tuple_filter(dev, filter); 3926 3927 return 0; 3928 } 3929 3930 /* inject a igb flex filter to HW */ 3931 static inline void 3932 igb_inject_flex_filter(struct rte_eth_dev *dev, 3933 struct e1000_flex_filter *filter) 3934 { 3935 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3936 uint32_t wufc, queueing; 3937 uint32_t reg_off; 3938 uint8_t i, j = 0; 3939 3940 wufc = E1000_READ_REG(hw, E1000_WUFC); 3941 if (filter->index < E1000_MAX_FHFT) 3942 reg_off = E1000_FHFT(filter->index); 3943 else 3944 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3945 3946 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3947 (E1000_WUFC_FLX0 << filter->index)); 3948 queueing = filter->filter_info.len | 3949 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3950 (filter->filter_info.priority << 3951 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3952 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3953 queueing); 3954 3955 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3956 E1000_WRITE_REG(hw, reg_off, 3957 filter->filter_info.dwords[j]); 3958 reg_off += sizeof(uint32_t); 3959 E1000_WRITE_REG(hw, reg_off, 3960 filter->filter_info.dwords[++j]); 3961 reg_off += sizeof(uint32_t); 3962 E1000_WRITE_REG(hw, reg_off, 3963 (uint32_t)filter->filter_info.mask[i]); 3964 reg_off += sizeof(uint32_t) * 2; 3965 ++j; 3966 } 3967 } 3968 3969 static inline struct e1000_flex_filter * 3970 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3971 struct e1000_flex_filter_info *key) 3972 { 3973 struct e1000_flex_filter *it; 3974 3975 TAILQ_FOREACH(it, filter_list, entries) { 3976 if (memcmp(key, &it->filter_info, 3977 sizeof(struct e1000_flex_filter_info)) == 0) 3978 return it; 3979 } 3980 3981 return NULL; 3982 } 3983 3984 /* remove a flex byte filter 3985 * @param 3986 * dev: Pointer to struct rte_eth_dev. 3987 * filter: the pointer of the filter will be removed. 3988 */ 3989 void 3990 igb_remove_flex_filter(struct rte_eth_dev *dev, 3991 struct e1000_flex_filter *filter) 3992 { 3993 struct e1000_filter_info *filter_info = 3994 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3995 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3996 uint32_t wufc, i; 3997 uint32_t reg_off; 3998 3999 wufc = E1000_READ_REG(hw, E1000_WUFC); 4000 if (filter->index < E1000_MAX_FHFT) 4001 reg_off = E1000_FHFT(filter->index); 4002 else 4003 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4004 4005 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4006 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4007 4008 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4009 (~(E1000_WUFC_FLX0 << filter->index))); 4010 4011 filter_info->flex_mask &= ~(1 << filter->index); 4012 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4013 rte_free(filter); 4014 } 4015 4016 int 4017 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4018 struct rte_eth_flex_filter *filter, 4019 bool add) 4020 { 4021 struct e1000_filter_info *filter_info = 4022 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4023 struct e1000_flex_filter *flex_filter, *it; 4024 uint32_t mask; 4025 uint8_t shift, i; 4026 4027 flex_filter = rte_zmalloc("e1000_flex_filter", 4028 sizeof(struct e1000_flex_filter), 0); 4029 if (flex_filter == NULL) 4030 return -ENOMEM; 4031 4032 flex_filter->filter_info.len = filter->len; 4033 flex_filter->filter_info.priority = filter->priority; 4034 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4035 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4036 mask = 0; 4037 /* reverse bits in flex filter's mask*/ 4038 for (shift = 0; shift < CHAR_BIT; shift++) { 4039 if (filter->mask[i] & (0x01 << shift)) 4040 mask |= (0x80 >> shift); 4041 } 4042 flex_filter->filter_info.mask[i] = mask; 4043 } 4044 4045 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4046 &flex_filter->filter_info); 4047 if (it == NULL && !add) { 4048 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4049 rte_free(flex_filter); 4050 return -ENOENT; 4051 } 4052 if (it != NULL && add) { 4053 PMD_DRV_LOG(ERR, "filter exists."); 4054 rte_free(flex_filter); 4055 return -EEXIST; 4056 } 4057 4058 if (add) { 4059 flex_filter->queue = filter->queue; 4060 /* 4061 * look for an unused flex filter index 4062 * and insert the filter into the list. 4063 */ 4064 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4065 if (!(filter_info->flex_mask & (1 << i))) { 4066 filter_info->flex_mask |= 1 << i; 4067 flex_filter->index = i; 4068 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4069 flex_filter, 4070 entries); 4071 break; 4072 } 4073 } 4074 if (i >= E1000_MAX_FLEX_FILTERS) { 4075 PMD_DRV_LOG(ERR, "flex filters are full."); 4076 rte_free(flex_filter); 4077 return -ENOSYS; 4078 } 4079 4080 igb_inject_flex_filter(dev, flex_filter); 4081 4082 } else { 4083 igb_remove_flex_filter(dev, it); 4084 rte_free(flex_filter); 4085 } 4086 4087 return 0; 4088 } 4089 4090 static int 4091 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4092 struct rte_eth_flex_filter *filter) 4093 { 4094 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4095 struct e1000_filter_info *filter_info = 4096 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4097 struct e1000_flex_filter flex_filter, *it; 4098 uint32_t wufc, queueing, wufc_en = 0; 4099 4100 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4101 flex_filter.filter_info.len = filter->len; 4102 flex_filter.filter_info.priority = filter->priority; 4103 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4104 memcpy(flex_filter.filter_info.mask, filter->mask, 4105 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4106 4107 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4108 &flex_filter.filter_info); 4109 if (it == NULL) { 4110 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4111 return -ENOENT; 4112 } 4113 4114 wufc = E1000_READ_REG(hw, E1000_WUFC); 4115 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4116 4117 if ((wufc & wufc_en) == wufc_en) { 4118 uint32_t reg_off = 0; 4119 if (it->index < E1000_MAX_FHFT) 4120 reg_off = E1000_FHFT(it->index); 4121 else 4122 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4123 4124 queueing = E1000_READ_REG(hw, 4125 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4126 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4127 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4128 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4129 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4130 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4131 return 0; 4132 } 4133 return -ENOENT; 4134 } 4135 4136 static int 4137 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4138 enum rte_filter_op filter_op, 4139 void *arg) 4140 { 4141 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4142 struct rte_eth_flex_filter *filter; 4143 int ret = 0; 4144 4145 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4146 4147 if (filter_op == RTE_ETH_FILTER_NOP) 4148 return ret; 4149 4150 if (arg == NULL) { 4151 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4152 filter_op); 4153 return -EINVAL; 4154 } 4155 4156 filter = (struct rte_eth_flex_filter *)arg; 4157 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4158 || filter->len % sizeof(uint64_t) != 0) { 4159 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4160 return -EINVAL; 4161 } 4162 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4163 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4164 return -EINVAL; 4165 } 4166 4167 switch (filter_op) { 4168 case RTE_ETH_FILTER_ADD: 4169 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4170 break; 4171 case RTE_ETH_FILTER_DELETE: 4172 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4173 break; 4174 case RTE_ETH_FILTER_GET: 4175 ret = eth_igb_get_flex_filter(dev, filter); 4176 break; 4177 default: 4178 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4179 ret = -EINVAL; 4180 break; 4181 } 4182 4183 return ret; 4184 } 4185 4186 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4187 static inline int 4188 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4189 struct e1000_5tuple_filter_info *filter_info) 4190 { 4191 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4192 return -EINVAL; 4193 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4194 return -EINVAL; /* filter index is out of range. */ 4195 if (filter->tcp_flags > RTE_NTUPLE_TCP_FLAGS_MASK) 4196 return -EINVAL; /* flags is invalid. */ 4197 4198 switch (filter->dst_ip_mask) { 4199 case UINT32_MAX: 4200 filter_info->dst_ip_mask = 0; 4201 filter_info->dst_ip = filter->dst_ip; 4202 break; 4203 case 0: 4204 filter_info->dst_ip_mask = 1; 4205 break; 4206 default: 4207 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4208 return -EINVAL; 4209 } 4210 4211 switch (filter->src_ip_mask) { 4212 case UINT32_MAX: 4213 filter_info->src_ip_mask = 0; 4214 filter_info->src_ip = filter->src_ip; 4215 break; 4216 case 0: 4217 filter_info->src_ip_mask = 1; 4218 break; 4219 default: 4220 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4221 return -EINVAL; 4222 } 4223 4224 switch (filter->dst_port_mask) { 4225 case UINT16_MAX: 4226 filter_info->dst_port_mask = 0; 4227 filter_info->dst_port = filter->dst_port; 4228 break; 4229 case 0: 4230 filter_info->dst_port_mask = 1; 4231 break; 4232 default: 4233 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4234 return -EINVAL; 4235 } 4236 4237 switch (filter->src_port_mask) { 4238 case UINT16_MAX: 4239 filter_info->src_port_mask = 0; 4240 filter_info->src_port = filter->src_port; 4241 break; 4242 case 0: 4243 filter_info->src_port_mask = 1; 4244 break; 4245 default: 4246 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4247 return -EINVAL; 4248 } 4249 4250 switch (filter->proto_mask) { 4251 case UINT8_MAX: 4252 filter_info->proto_mask = 0; 4253 filter_info->proto = filter->proto; 4254 break; 4255 case 0: 4256 filter_info->proto_mask = 1; 4257 break; 4258 default: 4259 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4260 return -EINVAL; 4261 } 4262 4263 filter_info->priority = (uint8_t)filter->priority; 4264 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4265 filter_info->tcp_flags = filter->tcp_flags; 4266 else 4267 filter_info->tcp_flags = 0; 4268 4269 return 0; 4270 } 4271 4272 static inline struct e1000_5tuple_filter * 4273 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4274 struct e1000_5tuple_filter_info *key) 4275 { 4276 struct e1000_5tuple_filter *it; 4277 4278 TAILQ_FOREACH(it, filter_list, entries) { 4279 if (memcmp(key, &it->filter_info, 4280 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4281 return it; 4282 } 4283 } 4284 return NULL; 4285 } 4286 4287 /* inject a igb 5-tuple filter to HW */ 4288 static inline void 4289 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4290 struct e1000_5tuple_filter *filter) 4291 { 4292 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4293 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4294 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4295 uint8_t i; 4296 4297 i = filter->index; 4298 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4299 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4300 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4301 if (filter->filter_info.dst_ip_mask == 0) 4302 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4303 if (filter->filter_info.src_port_mask == 0) 4304 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4305 if (filter->filter_info.proto_mask == 0) 4306 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4307 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4308 E1000_FTQF_QUEUE_MASK; 4309 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4310 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4311 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4312 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4313 4314 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4315 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4316 4317 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4318 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4319 imir |= E1000_IMIR_PORT_BP; 4320 else 4321 imir &= ~E1000_IMIR_PORT_BP; 4322 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4323 4324 /* tcp flags bits setting. */ 4325 if (filter->filter_info.tcp_flags & RTE_NTUPLE_TCP_FLAGS_MASK) { 4326 if (filter->filter_info.tcp_flags & RTE_TCP_URG_FLAG) 4327 imir_ext |= E1000_IMIREXT_CTRL_URG; 4328 if (filter->filter_info.tcp_flags & RTE_TCP_ACK_FLAG) 4329 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4330 if (filter->filter_info.tcp_flags & RTE_TCP_PSH_FLAG) 4331 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4332 if (filter->filter_info.tcp_flags & RTE_TCP_RST_FLAG) 4333 imir_ext |= E1000_IMIREXT_CTRL_RST; 4334 if (filter->filter_info.tcp_flags & RTE_TCP_SYN_FLAG) 4335 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4336 if (filter->filter_info.tcp_flags & RTE_TCP_FIN_FLAG) 4337 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4338 } else { 4339 imir_ext |= E1000_IMIREXT_CTRL_BP; 4340 } 4341 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4342 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4343 } 4344 4345 /* 4346 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4347 * 4348 * @param 4349 * dev: Pointer to struct rte_eth_dev. 4350 * ntuple_filter: ponter to the filter that will be added. 4351 * 4352 * @return 4353 * - On success, zero. 4354 * - On failure, a negative value. 4355 */ 4356 static int 4357 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4358 struct rte_eth_ntuple_filter *ntuple_filter) 4359 { 4360 struct e1000_filter_info *filter_info = 4361 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4362 struct e1000_5tuple_filter *filter; 4363 uint8_t i; 4364 int ret; 4365 4366 filter = rte_zmalloc("e1000_5tuple_filter", 4367 sizeof(struct e1000_5tuple_filter), 0); 4368 if (filter == NULL) 4369 return -ENOMEM; 4370 4371 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4372 &filter->filter_info); 4373 if (ret < 0) { 4374 rte_free(filter); 4375 return ret; 4376 } 4377 4378 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4379 &filter->filter_info) != NULL) { 4380 PMD_DRV_LOG(ERR, "filter exists."); 4381 rte_free(filter); 4382 return -EEXIST; 4383 } 4384 filter->queue = ntuple_filter->queue; 4385 4386 /* 4387 * look for an unused 5tuple filter index, 4388 * and insert the filter to list. 4389 */ 4390 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4391 if (!(filter_info->fivetuple_mask & (1 << i))) { 4392 filter_info->fivetuple_mask |= 1 << i; 4393 filter->index = i; 4394 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4395 filter, 4396 entries); 4397 break; 4398 } 4399 } 4400 if (i >= E1000_MAX_FTQF_FILTERS) { 4401 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4402 rte_free(filter); 4403 return -ENOSYS; 4404 } 4405 4406 igb_inject_5tuple_filter_82576(dev, filter); 4407 return 0; 4408 } 4409 4410 int 4411 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4412 struct e1000_5tuple_filter *filter) 4413 { 4414 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4415 struct e1000_filter_info *filter_info = 4416 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4417 4418 filter_info->fivetuple_mask &= ~(1 << filter->index); 4419 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4420 rte_free(filter); 4421 4422 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4423 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4424 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4425 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4426 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4427 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4428 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4429 return 0; 4430 } 4431 4432 /* 4433 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4434 * 4435 * @param 4436 * dev: Pointer to struct rte_eth_dev. 4437 * ntuple_filter: ponter to the filter that will be removed. 4438 * 4439 * @return 4440 * - On success, zero. 4441 * - On failure, a negative value. 4442 */ 4443 static int 4444 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4445 struct rte_eth_ntuple_filter *ntuple_filter) 4446 { 4447 struct e1000_filter_info *filter_info = 4448 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4449 struct e1000_5tuple_filter_info filter_5tuple; 4450 struct e1000_5tuple_filter *filter; 4451 int ret; 4452 4453 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4454 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4455 &filter_5tuple); 4456 if (ret < 0) 4457 return ret; 4458 4459 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4460 &filter_5tuple); 4461 if (filter == NULL) { 4462 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4463 return -ENOENT; 4464 } 4465 4466 igb_delete_5tuple_filter_82576(dev, filter); 4467 4468 return 0; 4469 } 4470 4471 static int 4472 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4473 { 4474 uint32_t rctl; 4475 struct e1000_hw *hw; 4476 struct rte_eth_dev_info dev_info; 4477 uint32_t frame_size = mtu + E1000_ETH_OVERHEAD; 4478 4479 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4480 4481 #ifdef RTE_LIBRTE_82571_SUPPORT 4482 /* XXX: not bigger than max_rx_pktlen */ 4483 if (hw->mac.type == e1000_82571) 4484 return -ENOTSUP; 4485 #endif 4486 eth_igb_infos_get(dev, &dev_info); 4487 4488 /* check that mtu is within the allowed range */ 4489 if (mtu < RTE_ETHER_MIN_MTU || 4490 frame_size > dev_info.max_rx_pktlen) 4491 return -EINVAL; 4492 4493 /* refuse mtu that requires the support of scattered packets when this 4494 * feature has not been enabled before. */ 4495 if (!dev->data->scattered_rx && 4496 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4497 return -EINVAL; 4498 4499 rctl = E1000_READ_REG(hw, E1000_RCTL); 4500 4501 /* switch to jumbo mode if needed */ 4502 if (frame_size > RTE_ETHER_MAX_LEN) { 4503 dev->data->dev_conf.rxmode.offloads |= 4504 DEV_RX_OFFLOAD_JUMBO_FRAME; 4505 rctl |= E1000_RCTL_LPE; 4506 } else { 4507 dev->data->dev_conf.rxmode.offloads &= 4508 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4509 rctl &= ~E1000_RCTL_LPE; 4510 } 4511 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4512 4513 /* update max frame size */ 4514 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4515 4516 E1000_WRITE_REG(hw, E1000_RLPML, 4517 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4518 4519 return 0; 4520 } 4521 4522 /* 4523 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4524 * 4525 * @param 4526 * dev: Pointer to struct rte_eth_dev. 4527 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4528 * add: if true, add filter, if false, remove filter 4529 * 4530 * @return 4531 * - On success, zero. 4532 * - On failure, a negative value. 4533 */ 4534 int 4535 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4536 struct rte_eth_ntuple_filter *ntuple_filter, 4537 bool add) 4538 { 4539 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4540 int ret; 4541 4542 switch (ntuple_filter->flags) { 4543 case RTE_5TUPLE_FLAGS: 4544 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4545 if (hw->mac.type != e1000_82576) 4546 return -ENOTSUP; 4547 if (add) 4548 ret = igb_add_5tuple_filter_82576(dev, 4549 ntuple_filter); 4550 else 4551 ret = igb_remove_5tuple_filter_82576(dev, 4552 ntuple_filter); 4553 break; 4554 case RTE_2TUPLE_FLAGS: 4555 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4556 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4557 hw->mac.type != e1000_i210 && 4558 hw->mac.type != e1000_i211) 4559 return -ENOTSUP; 4560 if (add) 4561 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4562 else 4563 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4564 break; 4565 default: 4566 ret = -EINVAL; 4567 break; 4568 } 4569 4570 return ret; 4571 } 4572 4573 /* 4574 * igb_get_ntuple_filter - get a ntuple filter 4575 * 4576 * @param 4577 * dev: Pointer to struct rte_eth_dev. 4578 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4579 * 4580 * @return 4581 * - On success, zero. 4582 * - On failure, a negative value. 4583 */ 4584 static int 4585 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4586 struct rte_eth_ntuple_filter *ntuple_filter) 4587 { 4588 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4589 struct e1000_filter_info *filter_info = 4590 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4591 struct e1000_5tuple_filter_info filter_5tuple; 4592 struct e1000_2tuple_filter_info filter_2tuple; 4593 struct e1000_5tuple_filter *p_5tuple_filter; 4594 struct e1000_2tuple_filter *p_2tuple_filter; 4595 int ret; 4596 4597 switch (ntuple_filter->flags) { 4598 case RTE_5TUPLE_FLAGS: 4599 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4600 if (hw->mac.type != e1000_82576) 4601 return -ENOTSUP; 4602 memset(&filter_5tuple, 4603 0, 4604 sizeof(struct e1000_5tuple_filter_info)); 4605 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4606 &filter_5tuple); 4607 if (ret < 0) 4608 return ret; 4609 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4610 &filter_info->fivetuple_list, 4611 &filter_5tuple); 4612 if (p_5tuple_filter == NULL) { 4613 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4614 return -ENOENT; 4615 } 4616 ntuple_filter->queue = p_5tuple_filter->queue; 4617 break; 4618 case RTE_2TUPLE_FLAGS: 4619 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4620 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4621 return -ENOTSUP; 4622 memset(&filter_2tuple, 4623 0, 4624 sizeof(struct e1000_2tuple_filter_info)); 4625 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4626 if (ret < 0) 4627 return ret; 4628 p_2tuple_filter = igb_2tuple_filter_lookup( 4629 &filter_info->twotuple_list, 4630 &filter_2tuple); 4631 if (p_2tuple_filter == NULL) { 4632 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4633 return -ENOENT; 4634 } 4635 ntuple_filter->queue = p_2tuple_filter->queue; 4636 break; 4637 default: 4638 ret = -EINVAL; 4639 break; 4640 } 4641 4642 return 0; 4643 } 4644 4645 /* 4646 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4647 * @dev: pointer to rte_eth_dev structure 4648 * @filter_op:operation will be taken. 4649 * @arg: a pointer to specific structure corresponding to the filter_op 4650 */ 4651 static int 4652 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4653 enum rte_filter_op filter_op, 4654 void *arg) 4655 { 4656 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4657 int ret; 4658 4659 MAC_TYPE_FILTER_SUP(hw->mac.type); 4660 4661 if (filter_op == RTE_ETH_FILTER_NOP) 4662 return 0; 4663 4664 if (arg == NULL) { 4665 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4666 filter_op); 4667 return -EINVAL; 4668 } 4669 4670 switch (filter_op) { 4671 case RTE_ETH_FILTER_ADD: 4672 ret = igb_add_del_ntuple_filter(dev, 4673 (struct rte_eth_ntuple_filter *)arg, 4674 TRUE); 4675 break; 4676 case RTE_ETH_FILTER_DELETE: 4677 ret = igb_add_del_ntuple_filter(dev, 4678 (struct rte_eth_ntuple_filter *)arg, 4679 FALSE); 4680 break; 4681 case RTE_ETH_FILTER_GET: 4682 ret = igb_get_ntuple_filter(dev, 4683 (struct rte_eth_ntuple_filter *)arg); 4684 break; 4685 default: 4686 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4687 ret = -EINVAL; 4688 break; 4689 } 4690 return ret; 4691 } 4692 4693 static inline int 4694 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4695 uint16_t ethertype) 4696 { 4697 int i; 4698 4699 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4700 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4701 (filter_info->ethertype_mask & (1 << i))) 4702 return i; 4703 } 4704 return -1; 4705 } 4706 4707 static inline int 4708 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4709 uint16_t ethertype, uint32_t etqf) 4710 { 4711 int i; 4712 4713 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4714 if (!(filter_info->ethertype_mask & (1 << i))) { 4715 filter_info->ethertype_mask |= 1 << i; 4716 filter_info->ethertype_filters[i].ethertype = ethertype; 4717 filter_info->ethertype_filters[i].etqf = etqf; 4718 return i; 4719 } 4720 } 4721 return -1; 4722 } 4723 4724 int 4725 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4726 uint8_t idx) 4727 { 4728 if (idx >= E1000_MAX_ETQF_FILTERS) 4729 return -1; 4730 filter_info->ethertype_mask &= ~(1 << idx); 4731 filter_info->ethertype_filters[idx].ethertype = 0; 4732 filter_info->ethertype_filters[idx].etqf = 0; 4733 return idx; 4734 } 4735 4736 4737 int 4738 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4739 struct rte_eth_ethertype_filter *filter, 4740 bool add) 4741 { 4742 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4743 struct e1000_filter_info *filter_info = 4744 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4745 uint32_t etqf = 0; 4746 int ret; 4747 4748 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 4749 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 4750 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4751 " ethertype filter.", filter->ether_type); 4752 return -EINVAL; 4753 } 4754 4755 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4756 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4757 return -EINVAL; 4758 } 4759 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4760 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4761 return -EINVAL; 4762 } 4763 4764 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4765 if (ret >= 0 && add) { 4766 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4767 filter->ether_type); 4768 return -EEXIST; 4769 } 4770 if (ret < 0 && !add) { 4771 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4772 filter->ether_type); 4773 return -ENOENT; 4774 } 4775 4776 if (add) { 4777 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4778 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4779 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4780 ret = igb_ethertype_filter_insert(filter_info, 4781 filter->ether_type, etqf); 4782 if (ret < 0) { 4783 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4784 return -ENOSYS; 4785 } 4786 } else { 4787 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4788 if (ret < 0) 4789 return -ENOSYS; 4790 } 4791 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4792 E1000_WRITE_FLUSH(hw); 4793 4794 return 0; 4795 } 4796 4797 static int 4798 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4799 struct rte_eth_ethertype_filter *filter) 4800 { 4801 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4802 struct e1000_filter_info *filter_info = 4803 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4804 uint32_t etqf; 4805 int ret; 4806 4807 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4808 if (ret < 0) { 4809 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4810 filter->ether_type); 4811 return -ENOENT; 4812 } 4813 4814 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4815 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4816 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4817 filter->flags = 0; 4818 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4819 E1000_ETQF_QUEUE_SHIFT; 4820 return 0; 4821 } 4822 4823 return -ENOENT; 4824 } 4825 4826 /* 4827 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4828 * @dev: pointer to rte_eth_dev structure 4829 * @filter_op:operation will be taken. 4830 * @arg: a pointer to specific structure corresponding to the filter_op 4831 */ 4832 static int 4833 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4834 enum rte_filter_op filter_op, 4835 void *arg) 4836 { 4837 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4838 int ret; 4839 4840 MAC_TYPE_FILTER_SUP(hw->mac.type); 4841 4842 if (filter_op == RTE_ETH_FILTER_NOP) 4843 return 0; 4844 4845 if (arg == NULL) { 4846 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4847 filter_op); 4848 return -EINVAL; 4849 } 4850 4851 switch (filter_op) { 4852 case RTE_ETH_FILTER_ADD: 4853 ret = igb_add_del_ethertype_filter(dev, 4854 (struct rte_eth_ethertype_filter *)arg, 4855 TRUE); 4856 break; 4857 case RTE_ETH_FILTER_DELETE: 4858 ret = igb_add_del_ethertype_filter(dev, 4859 (struct rte_eth_ethertype_filter *)arg, 4860 FALSE); 4861 break; 4862 case RTE_ETH_FILTER_GET: 4863 ret = igb_get_ethertype_filter(dev, 4864 (struct rte_eth_ethertype_filter *)arg); 4865 break; 4866 default: 4867 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4868 ret = -EINVAL; 4869 break; 4870 } 4871 return ret; 4872 } 4873 4874 static int 4875 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4876 enum rte_filter_type filter_type, 4877 enum rte_filter_op filter_op, 4878 void *arg) 4879 { 4880 int ret = 0; 4881 4882 switch (filter_type) { 4883 case RTE_ETH_FILTER_NTUPLE: 4884 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4885 break; 4886 case RTE_ETH_FILTER_ETHERTYPE: 4887 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4888 break; 4889 case RTE_ETH_FILTER_SYN: 4890 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4891 break; 4892 case RTE_ETH_FILTER_FLEXIBLE: 4893 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4894 break; 4895 case RTE_ETH_FILTER_GENERIC: 4896 if (filter_op != RTE_ETH_FILTER_GET) 4897 return -EINVAL; 4898 *(const void **)arg = &igb_flow_ops; 4899 break; 4900 default: 4901 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4902 filter_type); 4903 break; 4904 } 4905 4906 return ret; 4907 } 4908 4909 static int 4910 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4911 struct rte_ether_addr *mc_addr_set, 4912 uint32_t nb_mc_addr) 4913 { 4914 struct e1000_hw *hw; 4915 4916 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4917 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4918 return 0; 4919 } 4920 4921 static uint64_t 4922 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4923 { 4924 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4925 uint64_t systime_cycles; 4926 4927 switch (hw->mac.type) { 4928 case e1000_i210: 4929 case e1000_i211: 4930 /* 4931 * Need to read System Time Residue Register to be able 4932 * to read the other two registers. 4933 */ 4934 E1000_READ_REG(hw, E1000_SYSTIMR); 4935 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4936 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4937 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4938 * NSEC_PER_SEC; 4939 break; 4940 case e1000_82580: 4941 case e1000_i350: 4942 case e1000_i354: 4943 /* 4944 * Need to read System Time Residue Register to be able 4945 * to read the other two registers. 4946 */ 4947 E1000_READ_REG(hw, E1000_SYSTIMR); 4948 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4949 /* Only the 8 LSB are valid. */ 4950 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4951 & 0xff) << 32; 4952 break; 4953 default: 4954 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4955 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4956 << 32; 4957 break; 4958 } 4959 4960 return systime_cycles; 4961 } 4962 4963 static uint64_t 4964 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4965 { 4966 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4967 uint64_t rx_tstamp_cycles; 4968 4969 switch (hw->mac.type) { 4970 case e1000_i210: 4971 case e1000_i211: 4972 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4973 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4974 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4975 * NSEC_PER_SEC; 4976 break; 4977 case e1000_82580: 4978 case e1000_i350: 4979 case e1000_i354: 4980 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4981 /* Only the 8 LSB are valid. */ 4982 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4983 & 0xff) << 32; 4984 break; 4985 default: 4986 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4987 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4988 << 32; 4989 break; 4990 } 4991 4992 return rx_tstamp_cycles; 4993 } 4994 4995 static uint64_t 4996 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4997 { 4998 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4999 uint64_t tx_tstamp_cycles; 5000 5001 switch (hw->mac.type) { 5002 case e1000_i210: 5003 case e1000_i211: 5004 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5005 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5006 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5007 * NSEC_PER_SEC; 5008 break; 5009 case e1000_82580: 5010 case e1000_i350: 5011 case e1000_i354: 5012 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5013 /* Only the 8 LSB are valid. */ 5014 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5015 & 0xff) << 32; 5016 break; 5017 default: 5018 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5019 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5020 << 32; 5021 break; 5022 } 5023 5024 return tx_tstamp_cycles; 5025 } 5026 5027 static void 5028 igb_start_timecounters(struct rte_eth_dev *dev) 5029 { 5030 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5031 struct e1000_adapter *adapter = dev->data->dev_private; 5032 uint32_t incval = 1; 5033 uint32_t shift = 0; 5034 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5035 5036 switch (hw->mac.type) { 5037 case e1000_82580: 5038 case e1000_i350: 5039 case e1000_i354: 5040 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5041 mask = (1ULL << 40) - 1; 5042 /* fall-through */ 5043 case e1000_i210: 5044 case e1000_i211: 5045 /* 5046 * Start incrementing the register 5047 * used to timestamp PTP packets. 5048 */ 5049 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5050 break; 5051 case e1000_82576: 5052 incval = E1000_INCVALUE_82576; 5053 shift = IGB_82576_TSYNC_SHIFT; 5054 E1000_WRITE_REG(hw, E1000_TIMINCA, 5055 E1000_INCPERIOD_82576 | incval); 5056 break; 5057 default: 5058 /* Not supported */ 5059 return; 5060 } 5061 5062 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5063 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5064 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5065 5066 adapter->systime_tc.cc_mask = mask; 5067 adapter->systime_tc.cc_shift = shift; 5068 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5069 5070 adapter->rx_tstamp_tc.cc_mask = mask; 5071 adapter->rx_tstamp_tc.cc_shift = shift; 5072 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5073 5074 adapter->tx_tstamp_tc.cc_mask = mask; 5075 adapter->tx_tstamp_tc.cc_shift = shift; 5076 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5077 } 5078 5079 static int 5080 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5081 { 5082 struct e1000_adapter *adapter = dev->data->dev_private; 5083 5084 adapter->systime_tc.nsec += delta; 5085 adapter->rx_tstamp_tc.nsec += delta; 5086 adapter->tx_tstamp_tc.nsec += delta; 5087 5088 return 0; 5089 } 5090 5091 static int 5092 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5093 { 5094 uint64_t ns; 5095 struct e1000_adapter *adapter = dev->data->dev_private; 5096 5097 ns = rte_timespec_to_ns(ts); 5098 5099 /* Set the timecounters to a new value. */ 5100 adapter->systime_tc.nsec = ns; 5101 adapter->rx_tstamp_tc.nsec = ns; 5102 adapter->tx_tstamp_tc.nsec = ns; 5103 5104 return 0; 5105 } 5106 5107 static int 5108 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5109 { 5110 uint64_t ns, systime_cycles; 5111 struct e1000_adapter *adapter = dev->data->dev_private; 5112 5113 systime_cycles = igb_read_systime_cyclecounter(dev); 5114 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5115 *ts = rte_ns_to_timespec(ns); 5116 5117 return 0; 5118 } 5119 5120 static int 5121 igb_timesync_enable(struct rte_eth_dev *dev) 5122 { 5123 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5124 uint32_t tsync_ctl; 5125 uint32_t tsauxc; 5126 5127 /* Stop the timesync system time. */ 5128 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5129 /* Reset the timesync system time value. */ 5130 switch (hw->mac.type) { 5131 case e1000_82580: 5132 case e1000_i350: 5133 case e1000_i354: 5134 case e1000_i210: 5135 case e1000_i211: 5136 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5137 /* fall-through */ 5138 case e1000_82576: 5139 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5140 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5141 break; 5142 default: 5143 /* Not supported. */ 5144 return -ENOTSUP; 5145 } 5146 5147 /* Enable system time for it isn't on by default. */ 5148 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5149 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5150 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5151 5152 igb_start_timecounters(dev); 5153 5154 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5155 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5156 (RTE_ETHER_TYPE_1588 | 5157 E1000_ETQF_FILTER_ENABLE | 5158 E1000_ETQF_1588)); 5159 5160 /* Enable timestamping of received PTP packets. */ 5161 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5162 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5163 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5164 5165 /* Enable Timestamping of transmitted PTP packets. */ 5166 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5167 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5168 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5169 5170 return 0; 5171 } 5172 5173 static int 5174 igb_timesync_disable(struct rte_eth_dev *dev) 5175 { 5176 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5177 uint32_t tsync_ctl; 5178 5179 /* Disable timestamping of transmitted PTP packets. */ 5180 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5181 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5182 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5183 5184 /* Disable timestamping of received PTP packets. */ 5185 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5186 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5187 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5188 5189 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5190 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5191 5192 /* Stop incrementating the System Time registers. */ 5193 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5194 5195 return 0; 5196 } 5197 5198 static int 5199 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5200 struct timespec *timestamp, 5201 uint32_t flags __rte_unused) 5202 { 5203 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5204 struct e1000_adapter *adapter = dev->data->dev_private; 5205 uint32_t tsync_rxctl; 5206 uint64_t rx_tstamp_cycles; 5207 uint64_t ns; 5208 5209 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5210 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5211 return -EINVAL; 5212 5213 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5214 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5215 *timestamp = rte_ns_to_timespec(ns); 5216 5217 return 0; 5218 } 5219 5220 static int 5221 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5222 struct timespec *timestamp) 5223 { 5224 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5225 struct e1000_adapter *adapter = dev->data->dev_private; 5226 uint32_t tsync_txctl; 5227 uint64_t tx_tstamp_cycles; 5228 uint64_t ns; 5229 5230 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5231 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5232 return -EINVAL; 5233 5234 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5235 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5236 *timestamp = rte_ns_to_timespec(ns); 5237 5238 return 0; 5239 } 5240 5241 static int 5242 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5243 { 5244 int count = 0; 5245 int g_ind = 0; 5246 const struct reg_info *reg_group; 5247 5248 while ((reg_group = igb_regs[g_ind++])) 5249 count += igb_reg_group_count(reg_group); 5250 5251 return count; 5252 } 5253 5254 static int 5255 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5256 { 5257 int count = 0; 5258 int g_ind = 0; 5259 const struct reg_info *reg_group; 5260 5261 while ((reg_group = igbvf_regs[g_ind++])) 5262 count += igb_reg_group_count(reg_group); 5263 5264 return count; 5265 } 5266 5267 static int 5268 eth_igb_get_regs(struct rte_eth_dev *dev, 5269 struct rte_dev_reg_info *regs) 5270 { 5271 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5272 uint32_t *data = regs->data; 5273 int g_ind = 0; 5274 int count = 0; 5275 const struct reg_info *reg_group; 5276 5277 if (data == NULL) { 5278 regs->length = eth_igb_get_reg_length(dev); 5279 regs->width = sizeof(uint32_t); 5280 return 0; 5281 } 5282 5283 /* Support only full register dump */ 5284 if ((regs->length == 0) || 5285 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5286 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5287 hw->device_id; 5288 while ((reg_group = igb_regs[g_ind++])) 5289 count += igb_read_regs_group(dev, &data[count], 5290 reg_group); 5291 return 0; 5292 } 5293 5294 return -ENOTSUP; 5295 } 5296 5297 static int 5298 igbvf_get_regs(struct rte_eth_dev *dev, 5299 struct rte_dev_reg_info *regs) 5300 { 5301 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5302 uint32_t *data = regs->data; 5303 int g_ind = 0; 5304 int count = 0; 5305 const struct reg_info *reg_group; 5306 5307 if (data == NULL) { 5308 regs->length = igbvf_get_reg_length(dev); 5309 regs->width = sizeof(uint32_t); 5310 return 0; 5311 } 5312 5313 /* Support only full register dump */ 5314 if ((regs->length == 0) || 5315 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5316 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5317 hw->device_id; 5318 while ((reg_group = igbvf_regs[g_ind++])) 5319 count += igb_read_regs_group(dev, &data[count], 5320 reg_group); 5321 return 0; 5322 } 5323 5324 return -ENOTSUP; 5325 } 5326 5327 static int 5328 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5329 { 5330 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5331 5332 /* Return unit is byte count */ 5333 return hw->nvm.word_size * 2; 5334 } 5335 5336 static int 5337 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5338 struct rte_dev_eeprom_info *in_eeprom) 5339 { 5340 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5341 struct e1000_nvm_info *nvm = &hw->nvm; 5342 uint16_t *data = in_eeprom->data; 5343 int first, length; 5344 5345 first = in_eeprom->offset >> 1; 5346 length = in_eeprom->length >> 1; 5347 if ((first >= hw->nvm.word_size) || 5348 ((first + length) >= hw->nvm.word_size)) 5349 return -EINVAL; 5350 5351 in_eeprom->magic = hw->vendor_id | 5352 ((uint32_t)hw->device_id << 16); 5353 5354 if ((nvm->ops.read) == NULL) 5355 return -ENOTSUP; 5356 5357 return nvm->ops.read(hw, first, length, data); 5358 } 5359 5360 static int 5361 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5362 struct rte_dev_eeprom_info *in_eeprom) 5363 { 5364 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5365 struct e1000_nvm_info *nvm = &hw->nvm; 5366 uint16_t *data = in_eeprom->data; 5367 int first, length; 5368 5369 first = in_eeprom->offset >> 1; 5370 length = in_eeprom->length >> 1; 5371 if ((first >= hw->nvm.word_size) || 5372 ((first + length) >= hw->nvm.word_size)) 5373 return -EINVAL; 5374 5375 in_eeprom->magic = (uint32_t)hw->vendor_id | 5376 ((uint32_t)hw->device_id << 16); 5377 5378 if ((nvm->ops.write) == NULL) 5379 return -ENOTSUP; 5380 return nvm->ops.write(hw, first, length, data); 5381 } 5382 5383 static int 5384 eth_igb_get_module_info(struct rte_eth_dev *dev, 5385 struct rte_eth_dev_module_info *modinfo) 5386 { 5387 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5388 5389 uint32_t status = 0; 5390 uint16_t sff8472_rev, addr_mode; 5391 bool page_swap = false; 5392 5393 if (hw->phy.media_type == e1000_media_type_copper || 5394 hw->phy.media_type == e1000_media_type_unknown) 5395 return -EOPNOTSUPP; 5396 5397 /* Check whether we support SFF-8472 or not */ 5398 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5399 if (status) 5400 return -EIO; 5401 5402 /* addressing mode is not supported */ 5403 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5404 if (status) 5405 return -EIO; 5406 5407 /* addressing mode is not supported */ 5408 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5409 PMD_DRV_LOG(ERR, 5410 "Address change required to access page 0xA2, " 5411 "but not supported. Please report the module " 5412 "type to the driver maintainers.\n"); 5413 page_swap = true; 5414 } 5415 5416 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5417 /* We have an SFP, but it does not support SFF-8472 */ 5418 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5419 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5420 } else { 5421 /* We have an SFP which supports a revision of SFF-8472 */ 5422 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5423 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5424 } 5425 5426 return 0; 5427 } 5428 5429 static int 5430 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5431 struct rte_dev_eeprom_info *info) 5432 { 5433 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5434 5435 uint32_t status = 0; 5436 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5437 u16 first_word, last_word; 5438 int i = 0; 5439 5440 if (info->length == 0) 5441 return -EINVAL; 5442 5443 first_word = info->offset >> 1; 5444 last_word = (info->offset + info->length - 1) >> 1; 5445 5446 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5447 for (i = 0; i < last_word - first_word + 1; i++) { 5448 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5449 &dataword[i]); 5450 if (status) { 5451 /* Error occurred while reading module */ 5452 return -EIO; 5453 } 5454 5455 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5456 } 5457 5458 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5459 5460 return 0; 5461 } 5462 5463 static int 5464 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5465 { 5466 struct e1000_hw *hw = 5467 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5468 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5469 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5470 uint32_t vec = E1000_MISC_VEC_ID; 5471 5472 if (rte_intr_allow_others(intr_handle)) 5473 vec = E1000_RX_VEC_START; 5474 5475 uint32_t mask = 1 << (queue_id + vec); 5476 5477 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5478 E1000_WRITE_FLUSH(hw); 5479 5480 return 0; 5481 } 5482 5483 static int 5484 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5485 { 5486 struct e1000_hw *hw = 5487 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5488 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5489 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5490 uint32_t vec = E1000_MISC_VEC_ID; 5491 5492 if (rte_intr_allow_others(intr_handle)) 5493 vec = E1000_RX_VEC_START; 5494 5495 uint32_t mask = 1 << (queue_id + vec); 5496 uint32_t regval; 5497 5498 regval = E1000_READ_REG(hw, E1000_EIMS); 5499 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5500 E1000_WRITE_FLUSH(hw); 5501 5502 rte_intr_ack(intr_handle); 5503 5504 return 0; 5505 } 5506 5507 static void 5508 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5509 uint8_t index, uint8_t offset) 5510 { 5511 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5512 5513 /* clear bits */ 5514 val &= ~((uint32_t)0xFF << offset); 5515 5516 /* write vector and valid bit */ 5517 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5518 5519 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5520 } 5521 5522 static void 5523 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5524 uint8_t queue, uint8_t msix_vector) 5525 { 5526 uint32_t tmp = 0; 5527 5528 if (hw->mac.type == e1000_82575) { 5529 if (direction == 0) 5530 tmp = E1000_EICR_RX_QUEUE0 << queue; 5531 else if (direction == 1) 5532 tmp = E1000_EICR_TX_QUEUE0 << queue; 5533 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5534 } else if (hw->mac.type == e1000_82576) { 5535 if ((direction == 0) || (direction == 1)) 5536 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5537 ((queue & 0x8) << 1) + 5538 8 * direction); 5539 } else if ((hw->mac.type == e1000_82580) || 5540 (hw->mac.type == e1000_i350) || 5541 (hw->mac.type == e1000_i354) || 5542 (hw->mac.type == e1000_i210) || 5543 (hw->mac.type == e1000_i211)) { 5544 if ((direction == 0) || (direction == 1)) 5545 eth_igb_write_ivar(hw, msix_vector, 5546 queue >> 1, 5547 ((queue & 0x1) << 4) + 5548 8 * direction); 5549 } 5550 } 5551 5552 /* Sets up the hardware to generate MSI-X interrupts properly 5553 * @hw 5554 * board private structure 5555 */ 5556 static void 5557 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5558 { 5559 int queue_id; 5560 uint32_t tmpval, regval, intr_mask; 5561 struct e1000_hw *hw = 5562 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5563 uint32_t vec = E1000_MISC_VEC_ID; 5564 uint32_t base = E1000_MISC_VEC_ID; 5565 uint32_t misc_shift = 0; 5566 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5567 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5568 5569 /* won't configure msix register if no mapping is done 5570 * between intr vector and event fd 5571 */ 5572 if (!rte_intr_dp_is_en(intr_handle)) 5573 return; 5574 5575 if (rte_intr_allow_others(intr_handle)) { 5576 vec = base = E1000_RX_VEC_START; 5577 misc_shift = 1; 5578 } 5579 5580 /* set interrupt vector for other causes */ 5581 if (hw->mac.type == e1000_82575) { 5582 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5583 /* enable MSI-X PBA support */ 5584 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5585 5586 /* Auto-Mask interrupts upon ICR read */ 5587 tmpval |= E1000_CTRL_EXT_EIAME; 5588 tmpval |= E1000_CTRL_EXT_IRCA; 5589 5590 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5591 5592 /* enable msix_other interrupt */ 5593 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5594 regval = E1000_READ_REG(hw, E1000_EIAC); 5595 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5596 regval = E1000_READ_REG(hw, E1000_EIAM); 5597 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5598 } else if ((hw->mac.type == e1000_82576) || 5599 (hw->mac.type == e1000_82580) || 5600 (hw->mac.type == e1000_i350) || 5601 (hw->mac.type == e1000_i354) || 5602 (hw->mac.type == e1000_i210) || 5603 (hw->mac.type == e1000_i211)) { 5604 /* turn on MSI-X capability first */ 5605 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5606 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5607 E1000_GPIE_NSICR); 5608 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5609 misc_shift; 5610 5611 if (dev->data->dev_conf.intr_conf.lsc != 0) 5612 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5613 5614 regval = E1000_READ_REG(hw, E1000_EIAC); 5615 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5616 5617 /* enable msix_other interrupt */ 5618 regval = E1000_READ_REG(hw, E1000_EIMS); 5619 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5620 tmpval = (IGB_MSIX_OTHER_INTR_VEC | E1000_IVAR_VALID) << 8; 5621 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5622 } 5623 5624 /* use EIAM to auto-mask when MSI-X interrupt 5625 * is asserted, this saves a register write for every interrupt 5626 */ 5627 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5628 misc_shift; 5629 5630 if (dev->data->dev_conf.intr_conf.lsc != 0) 5631 intr_mask |= (1 << IGB_MSIX_OTHER_INTR_VEC); 5632 5633 regval = E1000_READ_REG(hw, E1000_EIAM); 5634 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5635 5636 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5637 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5638 intr_handle->intr_vec[queue_id] = vec; 5639 if (vec < base + intr_handle->nb_efd - 1) 5640 vec++; 5641 } 5642 5643 E1000_WRITE_FLUSH(hw); 5644 } 5645 5646 /* restore n-tuple filter */ 5647 static inline void 5648 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5649 { 5650 struct e1000_filter_info *filter_info = 5651 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5652 struct e1000_5tuple_filter *p_5tuple; 5653 struct e1000_2tuple_filter *p_2tuple; 5654 5655 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5656 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5657 } 5658 5659 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5660 igb_inject_2uple_filter(dev, p_2tuple); 5661 } 5662 } 5663 5664 /* restore SYN filter */ 5665 static inline void 5666 igb_syn_filter_restore(struct rte_eth_dev *dev) 5667 { 5668 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5669 struct e1000_filter_info *filter_info = 5670 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5671 uint32_t synqf; 5672 5673 synqf = filter_info->syn_info; 5674 5675 if (synqf & E1000_SYN_FILTER_ENABLE) { 5676 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5677 E1000_WRITE_FLUSH(hw); 5678 } 5679 } 5680 5681 /* restore ethernet type filter */ 5682 static inline void 5683 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5684 { 5685 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5686 struct e1000_filter_info *filter_info = 5687 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5688 int i; 5689 5690 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5691 if (filter_info->ethertype_mask & (1 << i)) { 5692 E1000_WRITE_REG(hw, E1000_ETQF(i), 5693 filter_info->ethertype_filters[i].etqf); 5694 E1000_WRITE_FLUSH(hw); 5695 } 5696 } 5697 } 5698 5699 /* restore flex byte filter */ 5700 static inline void 5701 igb_flex_filter_restore(struct rte_eth_dev *dev) 5702 { 5703 struct e1000_filter_info *filter_info = 5704 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5705 struct e1000_flex_filter *flex_filter; 5706 5707 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5708 igb_inject_flex_filter(dev, flex_filter); 5709 } 5710 } 5711 5712 /* restore rss filter */ 5713 static inline void 5714 igb_rss_filter_restore(struct rte_eth_dev *dev) 5715 { 5716 struct e1000_filter_info *filter_info = 5717 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5718 5719 if (filter_info->rss_info.conf.queue_num) 5720 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5721 } 5722 5723 /* restore all types filter */ 5724 static int 5725 igb_filter_restore(struct rte_eth_dev *dev) 5726 { 5727 igb_ntuple_filter_restore(dev); 5728 igb_ethertype_filter_restore(dev); 5729 igb_syn_filter_restore(dev); 5730 igb_flex_filter_restore(dev); 5731 igb_rss_filter_restore(dev); 5732 5733 return 0; 5734 } 5735 5736 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5737 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5738 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5739 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5740 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5741 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5742 5743 /* see e1000_logs.c */ 5744 RTE_INIT(e1000_init_log) 5745 { 5746 e1000_igb_init_log(); 5747 } 5748