1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_common.h> 12 #include <rte_interrupts.h> 13 #include <rte_byteorder.h> 14 #include <rte_log.h> 15 #include <rte_debug.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_ether.h> 19 #include <rte_ethdev_driver.h> 20 #include <rte_ethdev_pci.h> 21 #include <rte_memory.h> 22 #include <rte_eal.h> 23 #include <rte_malloc.h> 24 #include <rte_dev.h> 25 26 #include "e1000_logs.h" 27 #include "base/e1000_api.h" 28 #include "e1000_ethdev.h" 29 #include "igb_regs.h" 30 31 /* 32 * Default values for port configuration 33 */ 34 #define IGB_DEFAULT_RX_FREE_THRESH 32 35 36 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 37 #define IGB_DEFAULT_RX_HTHRESH 8 38 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 39 40 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 41 #define IGB_DEFAULT_TX_HTHRESH 1 42 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 43 44 /* Bit shift and mask */ 45 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 46 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 47 #define IGB_8_BIT_WIDTH CHAR_BIT 48 #define IGB_8_BIT_MASK UINT8_MAX 49 50 /* Additional timesync values. */ 51 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 52 #define E1000_ETQF_FILTER_1588 3 53 #define IGB_82576_TSYNC_SHIFT 16 54 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 55 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 56 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 57 58 #define E1000_VTIVAR_MISC 0x01740 59 #define E1000_VTIVAR_MISC_MASK 0xFF 60 #define E1000_VTIVAR_VALID 0x80 61 #define E1000_VTIVAR_MISC_MAILBOX 0 62 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 63 64 /* External VLAN Enable bit mask */ 65 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 66 67 /* External VLAN Ether Type bit mask and shift */ 68 #define E1000_VET_VET_EXT 0xFFFF0000 69 #define E1000_VET_VET_EXT_SHIFT 16 70 71 static int eth_igb_configure(struct rte_eth_dev *dev); 72 static int eth_igb_start(struct rte_eth_dev *dev); 73 static void eth_igb_stop(struct rte_eth_dev *dev); 74 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 75 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 76 static void eth_igb_close(struct rte_eth_dev *dev); 77 static int eth_igb_reset(struct rte_eth_dev *dev); 78 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 79 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 80 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 81 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 82 static int eth_igb_link_update(struct rte_eth_dev *dev, 83 int wait_to_complete); 84 static int eth_igb_stats_get(struct rte_eth_dev *dev, 85 struct rte_eth_stats *rte_stats); 86 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 87 struct rte_eth_xstat *xstats, unsigned n); 88 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 89 const uint64_t *ids, 90 uint64_t *values, unsigned int n); 91 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 92 struct rte_eth_xstat_name *xstats_names, 93 unsigned int size); 94 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 95 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 96 unsigned int limit); 97 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 98 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 99 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 100 char *fw_version, size_t fw_size); 101 static void eth_igb_infos_get(struct rte_eth_dev *dev, 102 struct rte_eth_dev_info *dev_info); 103 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 104 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 105 struct rte_eth_dev_info *dev_info); 106 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 107 struct rte_eth_fc_conf *fc_conf); 108 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 109 struct rte_eth_fc_conf *fc_conf); 110 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 111 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 112 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 113 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 114 struct rte_intr_handle *handle); 115 static void eth_igb_interrupt_handler(void *param); 116 static int igb_hardware_init(struct e1000_hw *hw); 117 static void igb_hw_control_acquire(struct e1000_hw *hw); 118 static void igb_hw_control_release(struct e1000_hw *hw); 119 static void igb_init_manageability(struct e1000_hw *hw); 120 static void igb_release_manageability(struct e1000_hw *hw); 121 122 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 123 124 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 125 uint16_t vlan_id, int on); 126 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 127 enum rte_vlan_type vlan_type, 128 uint16_t tpid_id); 129 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 130 131 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 132 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 133 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 134 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 135 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 137 138 static int eth_igb_led_on(struct rte_eth_dev *dev); 139 static int eth_igb_led_off(struct rte_eth_dev *dev); 140 141 static void igb_intr_disable(struct e1000_hw *hw); 142 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 143 static int eth_igb_rar_set(struct rte_eth_dev *dev, 144 struct ether_addr *mac_addr, 145 uint32_t index, uint32_t pool); 146 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 147 static int eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 148 struct ether_addr *addr); 149 150 static void igbvf_intr_disable(struct e1000_hw *hw); 151 static int igbvf_dev_configure(struct rte_eth_dev *dev); 152 static int igbvf_dev_start(struct rte_eth_dev *dev); 153 static void igbvf_dev_stop(struct rte_eth_dev *dev); 154 static void igbvf_dev_close(struct rte_eth_dev *dev); 155 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 156 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 157 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 158 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 159 static int eth_igbvf_link_update(struct e1000_hw *hw); 160 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 161 struct rte_eth_stats *rte_stats); 162 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 163 struct rte_eth_xstat *xstats, unsigned n); 164 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 165 struct rte_eth_xstat_name *xstats_names, 166 unsigned limit); 167 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 168 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 169 uint16_t vlan_id, int on); 170 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 171 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 172 static int igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 173 struct ether_addr *addr); 174 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 175 static int igbvf_get_regs(struct rte_eth_dev *dev, 176 struct rte_dev_reg_info *regs); 177 178 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 179 struct rte_eth_rss_reta_entry64 *reta_conf, 180 uint16_t reta_size); 181 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 182 struct rte_eth_rss_reta_entry64 *reta_conf, 183 uint16_t reta_size); 184 185 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 186 struct rte_eth_syn_filter *filter); 187 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 188 enum rte_filter_op filter_op, 189 void *arg); 190 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 191 struct rte_eth_ntuple_filter *ntuple_filter); 192 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 193 struct rte_eth_ntuple_filter *ntuple_filter); 194 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 195 struct rte_eth_flex_filter *filter); 196 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 197 enum rte_filter_op filter_op, 198 void *arg); 199 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 200 struct rte_eth_ntuple_filter *ntuple_filter); 201 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 202 struct rte_eth_ntuple_filter *ntuple_filter); 203 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *filter); 205 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 206 enum rte_filter_op filter_op, 207 void *arg); 208 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 209 enum rte_filter_op filter_op, 210 void *arg); 211 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 212 struct rte_eth_ethertype_filter *filter); 213 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 214 enum rte_filter_type filter_type, 215 enum rte_filter_op filter_op, 216 void *arg); 217 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 218 static int eth_igb_get_regs(struct rte_eth_dev *dev, 219 struct rte_dev_reg_info *regs); 220 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 221 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 222 struct rte_dev_eeprom_info *eeprom); 223 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 224 struct rte_dev_eeprom_info *eeprom); 225 static int eth_igb_get_module_info(struct rte_eth_dev *dev, 226 struct rte_eth_dev_module_info *modinfo); 227 static int eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 228 struct rte_dev_eeprom_info *info); 229 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 230 struct ether_addr *mc_addr_set, 231 uint32_t nb_mc_addr); 232 static int igb_timesync_enable(struct rte_eth_dev *dev); 233 static int igb_timesync_disable(struct rte_eth_dev *dev); 234 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 235 struct timespec *timestamp, 236 uint32_t flags); 237 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 238 struct timespec *timestamp); 239 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 240 static int igb_timesync_read_time(struct rte_eth_dev *dev, 241 struct timespec *timestamp); 242 static int igb_timesync_write_time(struct rte_eth_dev *dev, 243 const struct timespec *timestamp); 244 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 245 uint16_t queue_id); 246 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 247 uint16_t queue_id); 248 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 249 uint8_t queue, uint8_t msix_vector); 250 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 251 uint8_t index, uint8_t offset); 252 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 253 static void eth_igbvf_interrupt_handler(void *param); 254 static void igbvf_mbx_process(struct rte_eth_dev *dev); 255 static int igb_filter_restore(struct rte_eth_dev *dev); 256 257 /* 258 * Define VF Stats MACRO for Non "cleared on read" register 259 */ 260 #define UPDATE_VF_STAT(reg, last, cur) \ 261 { \ 262 u32 latest = E1000_READ_REG(hw, reg); \ 263 cur += (latest - last) & UINT_MAX; \ 264 last = latest; \ 265 } 266 267 #define IGB_FC_PAUSE_TIME 0x0680 268 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 269 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 270 271 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 272 273 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 274 275 /* 276 * The set of PCI devices this driver supports 277 */ 278 static const struct rte_pci_id pci_id_igb_map[] = { 279 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 280 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 281 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 287 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 291 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 295 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 298 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 302 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 318 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 319 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 320 { .vendor_id = 0, /* sentinel */ }, 321 }; 322 323 /* 324 * The set of PCI devices this driver supports (for 82576&I350 VF) 325 */ 326 static const struct rte_pci_id pci_id_igbvf_map[] = { 327 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 328 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 329 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 330 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 331 { .vendor_id = 0, /* sentinel */ }, 332 }; 333 334 static const struct rte_eth_desc_lim rx_desc_lim = { 335 .nb_max = E1000_MAX_RING_DESC, 336 .nb_min = E1000_MIN_RING_DESC, 337 .nb_align = IGB_RXD_ALIGN, 338 }; 339 340 static const struct rte_eth_desc_lim tx_desc_lim = { 341 .nb_max = E1000_MAX_RING_DESC, 342 .nb_min = E1000_MIN_RING_DESC, 343 .nb_align = IGB_RXD_ALIGN, 344 .nb_seg_max = IGB_TX_MAX_SEG, 345 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 346 }; 347 348 static const struct eth_dev_ops eth_igb_ops = { 349 .dev_configure = eth_igb_configure, 350 .dev_start = eth_igb_start, 351 .dev_stop = eth_igb_stop, 352 .dev_set_link_up = eth_igb_dev_set_link_up, 353 .dev_set_link_down = eth_igb_dev_set_link_down, 354 .dev_close = eth_igb_close, 355 .dev_reset = eth_igb_reset, 356 .promiscuous_enable = eth_igb_promiscuous_enable, 357 .promiscuous_disable = eth_igb_promiscuous_disable, 358 .allmulticast_enable = eth_igb_allmulticast_enable, 359 .allmulticast_disable = eth_igb_allmulticast_disable, 360 .link_update = eth_igb_link_update, 361 .stats_get = eth_igb_stats_get, 362 .xstats_get = eth_igb_xstats_get, 363 .xstats_get_by_id = eth_igb_xstats_get_by_id, 364 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 365 .xstats_get_names = eth_igb_xstats_get_names, 366 .stats_reset = eth_igb_stats_reset, 367 .xstats_reset = eth_igb_xstats_reset, 368 .fw_version_get = eth_igb_fw_version_get, 369 .dev_infos_get = eth_igb_infos_get, 370 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 371 .mtu_set = eth_igb_mtu_set, 372 .vlan_filter_set = eth_igb_vlan_filter_set, 373 .vlan_tpid_set = eth_igb_vlan_tpid_set, 374 .vlan_offload_set = eth_igb_vlan_offload_set, 375 .rx_queue_setup = eth_igb_rx_queue_setup, 376 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 377 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 378 .rx_queue_release = eth_igb_rx_queue_release, 379 .rx_queue_count = eth_igb_rx_queue_count, 380 .rx_descriptor_done = eth_igb_rx_descriptor_done, 381 .rx_descriptor_status = eth_igb_rx_descriptor_status, 382 .tx_descriptor_status = eth_igb_tx_descriptor_status, 383 .tx_queue_setup = eth_igb_tx_queue_setup, 384 .tx_queue_release = eth_igb_tx_queue_release, 385 .tx_done_cleanup = eth_igb_tx_done_cleanup, 386 .dev_led_on = eth_igb_led_on, 387 .dev_led_off = eth_igb_led_off, 388 .flow_ctrl_get = eth_igb_flow_ctrl_get, 389 .flow_ctrl_set = eth_igb_flow_ctrl_set, 390 .mac_addr_add = eth_igb_rar_set, 391 .mac_addr_remove = eth_igb_rar_clear, 392 .mac_addr_set = eth_igb_default_mac_addr_set, 393 .reta_update = eth_igb_rss_reta_update, 394 .reta_query = eth_igb_rss_reta_query, 395 .rss_hash_update = eth_igb_rss_hash_update, 396 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 397 .filter_ctrl = eth_igb_filter_ctrl, 398 .set_mc_addr_list = eth_igb_set_mc_addr_list, 399 .rxq_info_get = igb_rxq_info_get, 400 .txq_info_get = igb_txq_info_get, 401 .timesync_enable = igb_timesync_enable, 402 .timesync_disable = igb_timesync_disable, 403 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 404 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 405 .get_reg = eth_igb_get_regs, 406 .get_eeprom_length = eth_igb_get_eeprom_length, 407 .get_eeprom = eth_igb_get_eeprom, 408 .set_eeprom = eth_igb_set_eeprom, 409 .get_module_info = eth_igb_get_module_info, 410 .get_module_eeprom = eth_igb_get_module_eeprom, 411 .timesync_adjust_time = igb_timesync_adjust_time, 412 .timesync_read_time = igb_timesync_read_time, 413 .timesync_write_time = igb_timesync_write_time, 414 }; 415 416 /* 417 * dev_ops for virtual function, bare necessities for basic vf 418 * operation have been implemented 419 */ 420 static const struct eth_dev_ops igbvf_eth_dev_ops = { 421 .dev_configure = igbvf_dev_configure, 422 .dev_start = igbvf_dev_start, 423 .dev_stop = igbvf_dev_stop, 424 .dev_close = igbvf_dev_close, 425 .promiscuous_enable = igbvf_promiscuous_enable, 426 .promiscuous_disable = igbvf_promiscuous_disable, 427 .allmulticast_enable = igbvf_allmulticast_enable, 428 .allmulticast_disable = igbvf_allmulticast_disable, 429 .link_update = eth_igb_link_update, 430 .stats_get = eth_igbvf_stats_get, 431 .xstats_get = eth_igbvf_xstats_get, 432 .xstats_get_names = eth_igbvf_xstats_get_names, 433 .stats_reset = eth_igbvf_stats_reset, 434 .xstats_reset = eth_igbvf_stats_reset, 435 .vlan_filter_set = igbvf_vlan_filter_set, 436 .dev_infos_get = eth_igbvf_infos_get, 437 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 438 .rx_queue_setup = eth_igb_rx_queue_setup, 439 .rx_queue_release = eth_igb_rx_queue_release, 440 .rx_descriptor_done = eth_igb_rx_descriptor_done, 441 .rx_descriptor_status = eth_igb_rx_descriptor_status, 442 .tx_descriptor_status = eth_igb_tx_descriptor_status, 443 .tx_queue_setup = eth_igb_tx_queue_setup, 444 .tx_queue_release = eth_igb_tx_queue_release, 445 .set_mc_addr_list = eth_igb_set_mc_addr_list, 446 .rxq_info_get = igb_rxq_info_get, 447 .txq_info_get = igb_txq_info_get, 448 .mac_addr_set = igbvf_default_mac_addr_set, 449 .get_reg = igbvf_get_regs, 450 }; 451 452 /* store statistics names and its offset in stats structure */ 453 struct rte_igb_xstats_name_off { 454 char name[RTE_ETH_XSTATS_NAME_SIZE]; 455 unsigned offset; 456 }; 457 458 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 459 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 460 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 461 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 462 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 463 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 464 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 465 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 466 ecol)}, 467 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 468 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 469 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 470 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 471 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 472 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 473 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 474 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 475 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 476 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 477 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 478 fcruc)}, 479 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 480 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 481 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 482 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 483 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 484 prc1023)}, 485 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 486 prc1522)}, 487 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 488 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 489 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 490 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 491 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 492 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 493 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 494 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 495 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 496 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 497 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 498 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 499 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 500 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 501 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 502 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 503 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 504 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 505 ptc1023)}, 506 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 507 ptc1522)}, 508 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 509 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 510 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 511 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 512 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 513 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 514 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 515 516 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 517 }; 518 519 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 520 sizeof(rte_igb_stats_strings[0])) 521 522 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 523 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 524 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 525 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 526 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 527 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 528 }; 529 530 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 531 sizeof(rte_igbvf_stats_strings[0])) 532 533 534 static inline void 535 igb_intr_enable(struct rte_eth_dev *dev) 536 { 537 struct e1000_interrupt *intr = 538 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 539 struct e1000_hw *hw = 540 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 541 542 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 543 E1000_WRITE_FLUSH(hw); 544 } 545 546 static void 547 igb_intr_disable(struct e1000_hw *hw) 548 { 549 E1000_WRITE_REG(hw, E1000_IMC, ~0); 550 E1000_WRITE_FLUSH(hw); 551 } 552 553 static inline void 554 igbvf_intr_enable(struct rte_eth_dev *dev) 555 { 556 struct e1000_hw *hw = 557 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 558 559 /* only for mailbox */ 560 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 561 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 562 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 563 E1000_WRITE_FLUSH(hw); 564 } 565 566 /* only for mailbox now. If RX/TX needed, should extend this function. */ 567 static void 568 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 569 { 570 uint32_t tmp = 0; 571 572 /* mailbox */ 573 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 574 tmp |= E1000_VTIVAR_VALID; 575 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 576 } 577 578 static void 579 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 580 { 581 struct e1000_hw *hw = 582 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 583 584 /* Configure VF other cause ivar */ 585 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 586 } 587 588 static inline int32_t 589 igb_pf_reset_hw(struct e1000_hw *hw) 590 { 591 uint32_t ctrl_ext; 592 int32_t status; 593 594 status = e1000_reset_hw(hw); 595 596 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 597 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 598 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 599 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 600 E1000_WRITE_FLUSH(hw); 601 602 return status; 603 } 604 605 static void 606 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 607 { 608 struct e1000_hw *hw = 609 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 610 611 612 hw->vendor_id = pci_dev->id.vendor_id; 613 hw->device_id = pci_dev->id.device_id; 614 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 615 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 616 617 e1000_set_mac_type(hw); 618 619 /* need to check if it is a vf device below */ 620 } 621 622 static int 623 igb_reset_swfw_lock(struct e1000_hw *hw) 624 { 625 int ret_val; 626 627 /* 628 * Do mac ops initialization manually here, since we will need 629 * some function pointers set by this call. 630 */ 631 ret_val = e1000_init_mac_params(hw); 632 if (ret_val) 633 return ret_val; 634 635 /* 636 * SMBI lock should not fail in this early stage. If this is the case, 637 * it is due to an improper exit of the application. 638 * So force the release of the faulty lock. 639 */ 640 if (e1000_get_hw_semaphore_generic(hw) < 0) { 641 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 642 } 643 e1000_put_hw_semaphore_generic(hw); 644 645 if (hw->mac.ops.acquire_swfw_sync != NULL) { 646 uint16_t mask; 647 648 /* 649 * Phy lock should not fail in this early stage. If this is the case, 650 * it is due to an improper exit of the application. 651 * So force the release of the faulty lock. 652 */ 653 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 654 if (hw->bus.func > E1000_FUNC_1) 655 mask <<= 2; 656 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 657 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 658 hw->bus.func); 659 } 660 hw->mac.ops.release_swfw_sync(hw, mask); 661 662 /* 663 * This one is more tricky since it is common to all ports; but 664 * swfw_sync retries last long enough (1s) to be almost sure that if 665 * lock can not be taken it is due to an improper lock of the 666 * semaphore. 667 */ 668 mask = E1000_SWFW_EEP_SM; 669 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 670 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 671 } 672 hw->mac.ops.release_swfw_sync(hw, mask); 673 } 674 675 return E1000_SUCCESS; 676 } 677 678 /* Remove all ntuple filters of the device */ 679 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 680 { 681 struct e1000_filter_info *filter_info = 682 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 683 struct e1000_5tuple_filter *p_5tuple; 684 struct e1000_2tuple_filter *p_2tuple; 685 686 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 687 TAILQ_REMOVE(&filter_info->fivetuple_list, 688 p_5tuple, entries); 689 rte_free(p_5tuple); 690 } 691 filter_info->fivetuple_mask = 0; 692 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 693 TAILQ_REMOVE(&filter_info->twotuple_list, 694 p_2tuple, entries); 695 rte_free(p_2tuple); 696 } 697 filter_info->twotuple_mask = 0; 698 699 return 0; 700 } 701 702 /* Remove all flex filters of the device */ 703 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 704 { 705 struct e1000_filter_info *filter_info = 706 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 707 struct e1000_flex_filter *p_flex; 708 709 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 710 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 711 rte_free(p_flex); 712 } 713 filter_info->flex_mask = 0; 714 715 return 0; 716 } 717 718 static int 719 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 720 { 721 int error = 0; 722 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 723 struct e1000_hw *hw = 724 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 725 struct e1000_vfta * shadow_vfta = 726 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 727 struct e1000_filter_info *filter_info = 728 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 729 struct e1000_adapter *adapter = 730 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 731 732 uint32_t ctrl_ext; 733 734 eth_dev->dev_ops = ð_igb_ops; 735 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 736 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 737 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 738 739 /* for secondary processes, we don't initialise any further as primary 740 * has already done this work. Only check we don't need a different 741 * RX function */ 742 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 743 if (eth_dev->data->scattered_rx) 744 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 745 return 0; 746 } 747 748 rte_eth_copy_pci_info(eth_dev, pci_dev); 749 750 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 751 752 igb_identify_hardware(eth_dev, pci_dev); 753 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 754 error = -EIO; 755 goto err_late; 756 } 757 758 e1000_get_bus_info(hw); 759 760 /* Reset any pending lock */ 761 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 762 error = -EIO; 763 goto err_late; 764 } 765 766 /* Finish initialization */ 767 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 768 error = -EIO; 769 goto err_late; 770 } 771 772 hw->mac.autoneg = 1; 773 hw->phy.autoneg_wait_to_complete = 0; 774 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 775 776 /* Copper options */ 777 if (hw->phy.media_type == e1000_media_type_copper) { 778 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 779 hw->phy.disable_polarity_correction = 0; 780 hw->phy.ms_type = e1000_ms_hw_default; 781 } 782 783 /* 784 * Start from a known state, this is important in reading the nvm 785 * and mac from that. 786 */ 787 igb_pf_reset_hw(hw); 788 789 /* Make sure we have a good EEPROM before we read from it */ 790 if (e1000_validate_nvm_checksum(hw) < 0) { 791 /* 792 * Some PCI-E parts fail the first check due to 793 * the link being in sleep state, call it again, 794 * if it fails a second time its a real issue. 795 */ 796 if (e1000_validate_nvm_checksum(hw) < 0) { 797 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 798 error = -EIO; 799 goto err_late; 800 } 801 } 802 803 /* Read the permanent MAC address out of the EEPROM */ 804 if (e1000_read_mac_addr(hw) != 0) { 805 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 806 error = -EIO; 807 goto err_late; 808 } 809 810 /* Allocate memory for storing MAC addresses */ 811 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 812 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 813 if (eth_dev->data->mac_addrs == NULL) { 814 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 815 "store MAC addresses", 816 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 817 error = -ENOMEM; 818 goto err_late; 819 } 820 821 /* Copy the permanent MAC address */ 822 ether_addr_copy((struct ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); 823 824 /* initialize the vfta */ 825 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 826 827 /* Now initialize the hardware */ 828 if (igb_hardware_init(hw) != 0) { 829 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 830 rte_free(eth_dev->data->mac_addrs); 831 eth_dev->data->mac_addrs = NULL; 832 error = -ENODEV; 833 goto err_late; 834 } 835 hw->mac.get_link_status = 1; 836 adapter->stopped = 0; 837 838 /* Indicate SOL/IDER usage */ 839 if (e1000_check_reset_block(hw) < 0) { 840 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 841 "SOL/IDER session"); 842 } 843 844 /* initialize PF if max_vfs not zero */ 845 igb_pf_host_init(eth_dev); 846 847 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 848 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 849 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 850 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 851 E1000_WRITE_FLUSH(hw); 852 853 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 854 eth_dev->data->port_id, pci_dev->id.vendor_id, 855 pci_dev->id.device_id); 856 857 rte_intr_callback_register(&pci_dev->intr_handle, 858 eth_igb_interrupt_handler, 859 (void *)eth_dev); 860 861 /* enable uio/vfio intr/eventfd mapping */ 862 rte_intr_enable(&pci_dev->intr_handle); 863 864 /* enable support intr */ 865 igb_intr_enable(eth_dev); 866 867 /* initialize filter info */ 868 memset(filter_info, 0, 869 sizeof(struct e1000_filter_info)); 870 871 TAILQ_INIT(&filter_info->flex_list); 872 TAILQ_INIT(&filter_info->twotuple_list); 873 TAILQ_INIT(&filter_info->fivetuple_list); 874 875 TAILQ_INIT(&igb_filter_ntuple_list); 876 TAILQ_INIT(&igb_filter_ethertype_list); 877 TAILQ_INIT(&igb_filter_syn_list); 878 TAILQ_INIT(&igb_filter_flex_list); 879 TAILQ_INIT(&igb_filter_rss_list); 880 TAILQ_INIT(&igb_flow_list); 881 882 return 0; 883 884 err_late: 885 igb_hw_control_release(hw); 886 887 return error; 888 } 889 890 static int 891 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 892 { 893 struct rte_pci_device *pci_dev; 894 struct rte_intr_handle *intr_handle; 895 struct e1000_hw *hw; 896 struct e1000_adapter *adapter = 897 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 898 struct e1000_filter_info *filter_info = 899 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 900 901 PMD_INIT_FUNC_TRACE(); 902 903 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 904 return -EPERM; 905 906 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 907 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 908 intr_handle = &pci_dev->intr_handle; 909 910 if (adapter->stopped == 0) 911 eth_igb_close(eth_dev); 912 913 eth_dev->dev_ops = NULL; 914 eth_dev->rx_pkt_burst = NULL; 915 eth_dev->tx_pkt_burst = NULL; 916 917 /* Reset any pending lock */ 918 igb_reset_swfw_lock(hw); 919 920 rte_free(eth_dev->data->mac_addrs); 921 eth_dev->data->mac_addrs = NULL; 922 923 /* uninitialize PF if max_vfs not zero */ 924 igb_pf_host_uninit(eth_dev); 925 926 /* disable uio intr before callback unregister */ 927 rte_intr_disable(intr_handle); 928 rte_intr_callback_unregister(intr_handle, 929 eth_igb_interrupt_handler, eth_dev); 930 931 /* clear the SYN filter info */ 932 filter_info->syn_info = 0; 933 934 /* clear the ethertype filters info */ 935 filter_info->ethertype_mask = 0; 936 memset(filter_info->ethertype_filters, 0, 937 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 938 939 /* clear the rss filter info */ 940 memset(&filter_info->rss_info, 0, 941 sizeof(struct igb_rte_flow_rss_conf)); 942 943 /* remove all ntuple filters of the device */ 944 igb_ntuple_filter_uninit(eth_dev); 945 946 /* remove all flex filters of the device */ 947 igb_flex_filter_uninit(eth_dev); 948 949 /* clear all the filters list */ 950 igb_filterlist_flush(eth_dev); 951 952 return 0; 953 } 954 955 /* 956 * Virtual Function device init 957 */ 958 static int 959 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 960 { 961 struct rte_pci_device *pci_dev; 962 struct rte_intr_handle *intr_handle; 963 struct e1000_adapter *adapter = 964 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 965 struct e1000_hw *hw = 966 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 967 int diag; 968 struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr; 969 970 PMD_INIT_FUNC_TRACE(); 971 972 eth_dev->dev_ops = &igbvf_eth_dev_ops; 973 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 974 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 975 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 976 977 /* for secondary processes, we don't initialise any further as primary 978 * has already done this work. Only check we don't need a different 979 * RX function */ 980 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 981 if (eth_dev->data->scattered_rx) 982 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 983 return 0; 984 } 985 986 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 987 rte_eth_copy_pci_info(eth_dev, pci_dev); 988 989 hw->device_id = pci_dev->id.device_id; 990 hw->vendor_id = pci_dev->id.vendor_id; 991 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 992 adapter->stopped = 0; 993 994 /* Initialize the shared code (base driver) */ 995 diag = e1000_setup_init_funcs(hw, TRUE); 996 if (diag != 0) { 997 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 998 diag); 999 return -EIO; 1000 } 1001 1002 /* init_mailbox_params */ 1003 hw->mbx.ops.init_params(hw); 1004 1005 /* Disable the interrupts for VF */ 1006 igbvf_intr_disable(hw); 1007 1008 diag = hw->mac.ops.reset_hw(hw); 1009 1010 /* Allocate memory for storing MAC addresses */ 1011 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN * 1012 hw->mac.rar_entry_count, 0); 1013 if (eth_dev->data->mac_addrs == NULL) { 1014 PMD_INIT_LOG(ERR, 1015 "Failed to allocate %d bytes needed to store MAC " 1016 "addresses", 1017 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1018 return -ENOMEM; 1019 } 1020 1021 /* Generate a random MAC address, if none was assigned by PF. */ 1022 if (is_zero_ether_addr(perm_addr)) { 1023 eth_random_addr(perm_addr->addr_bytes); 1024 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1025 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1026 "%02x:%02x:%02x:%02x:%02x:%02x", 1027 perm_addr->addr_bytes[0], 1028 perm_addr->addr_bytes[1], 1029 perm_addr->addr_bytes[2], 1030 perm_addr->addr_bytes[3], 1031 perm_addr->addr_bytes[4], 1032 perm_addr->addr_bytes[5]); 1033 } 1034 1035 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1036 if (diag) { 1037 rte_free(eth_dev->data->mac_addrs); 1038 eth_dev->data->mac_addrs = NULL; 1039 return diag; 1040 } 1041 /* Copy the permanent MAC address */ 1042 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1043 ð_dev->data->mac_addrs[0]); 1044 1045 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1046 "mac.type=%s", 1047 eth_dev->data->port_id, pci_dev->id.vendor_id, 1048 pci_dev->id.device_id, "igb_mac_82576_vf"); 1049 1050 intr_handle = &pci_dev->intr_handle; 1051 rte_intr_callback_register(intr_handle, 1052 eth_igbvf_interrupt_handler, eth_dev); 1053 1054 return 0; 1055 } 1056 1057 static int 1058 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1059 { 1060 struct e1000_adapter *adapter = 1061 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1062 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1063 1064 PMD_INIT_FUNC_TRACE(); 1065 1066 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1067 return -EPERM; 1068 1069 if (adapter->stopped == 0) 1070 igbvf_dev_close(eth_dev); 1071 1072 eth_dev->dev_ops = NULL; 1073 eth_dev->rx_pkt_burst = NULL; 1074 eth_dev->tx_pkt_burst = NULL; 1075 1076 rte_free(eth_dev->data->mac_addrs); 1077 eth_dev->data->mac_addrs = NULL; 1078 1079 /* disable uio intr before callback unregister */ 1080 rte_intr_disable(&pci_dev->intr_handle); 1081 rte_intr_callback_unregister(&pci_dev->intr_handle, 1082 eth_igbvf_interrupt_handler, 1083 (void *)eth_dev); 1084 1085 return 0; 1086 } 1087 1088 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1089 struct rte_pci_device *pci_dev) 1090 { 1091 return rte_eth_dev_pci_generic_probe(pci_dev, 1092 sizeof(struct e1000_adapter), eth_igb_dev_init); 1093 } 1094 1095 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1096 { 1097 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1098 } 1099 1100 static struct rte_pci_driver rte_igb_pmd = { 1101 .id_table = pci_id_igb_map, 1102 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1103 RTE_PCI_DRV_IOVA_AS_VA, 1104 .probe = eth_igb_pci_probe, 1105 .remove = eth_igb_pci_remove, 1106 }; 1107 1108 1109 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1110 struct rte_pci_device *pci_dev) 1111 { 1112 return rte_eth_dev_pci_generic_probe(pci_dev, 1113 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1114 } 1115 1116 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1117 { 1118 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1119 } 1120 1121 /* 1122 * virtual function driver struct 1123 */ 1124 static struct rte_pci_driver rte_igbvf_pmd = { 1125 .id_table = pci_id_igbvf_map, 1126 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1127 .probe = eth_igbvf_pci_probe, 1128 .remove = eth_igbvf_pci_remove, 1129 }; 1130 1131 static void 1132 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1133 { 1134 struct e1000_hw *hw = 1135 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1136 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1137 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1138 rctl |= E1000_RCTL_VFE; 1139 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1140 } 1141 1142 static int 1143 igb_check_mq_mode(struct rte_eth_dev *dev) 1144 { 1145 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1146 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1147 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1148 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1149 1150 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1151 tx_mq_mode == ETH_MQ_TX_DCB || 1152 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1153 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1154 return -EINVAL; 1155 } 1156 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1157 /* Check multi-queue mode. 1158 * To no break software we accept ETH_MQ_RX_NONE as this might 1159 * be used to turn off VLAN filter. 1160 */ 1161 1162 if (rx_mq_mode == ETH_MQ_RX_NONE || 1163 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1164 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1165 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1166 } else { 1167 /* Only support one queue on VFs. 1168 * RSS together with SRIOV is not supported. 1169 */ 1170 PMD_INIT_LOG(ERR, "SRIOV is active," 1171 " wrong mq_mode rx %d.", 1172 rx_mq_mode); 1173 return -EINVAL; 1174 } 1175 /* TX mode is not used here, so mode might be ignored.*/ 1176 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1177 /* SRIOV only works in VMDq enable mode */ 1178 PMD_INIT_LOG(WARNING, "SRIOV is active," 1179 " TX mode %d is not supported. " 1180 " Driver will behave as %d mode.", 1181 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1182 } 1183 1184 /* check valid queue number */ 1185 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1186 PMD_INIT_LOG(ERR, "SRIOV is active," 1187 " only support one queue on VFs."); 1188 return -EINVAL; 1189 } 1190 } else { 1191 /* To no break software that set invalid mode, only display 1192 * warning if invalid mode is used. 1193 */ 1194 if (rx_mq_mode != ETH_MQ_RX_NONE && 1195 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1196 rx_mq_mode != ETH_MQ_RX_RSS) { 1197 /* RSS together with VMDq not supported*/ 1198 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1199 rx_mq_mode); 1200 return -EINVAL; 1201 } 1202 1203 if (tx_mq_mode != ETH_MQ_TX_NONE && 1204 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1205 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1206 " Due to txmode is meaningless in this" 1207 " driver, just ignore.", 1208 tx_mq_mode); 1209 } 1210 } 1211 return 0; 1212 } 1213 1214 static int 1215 eth_igb_configure(struct rte_eth_dev *dev) 1216 { 1217 struct e1000_interrupt *intr = 1218 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1219 int ret; 1220 1221 PMD_INIT_FUNC_TRACE(); 1222 1223 /* multipe queue mode checking */ 1224 ret = igb_check_mq_mode(dev); 1225 if (ret != 0) { 1226 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1227 ret); 1228 return ret; 1229 } 1230 1231 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1232 PMD_INIT_FUNC_TRACE(); 1233 1234 return 0; 1235 } 1236 1237 static void 1238 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1239 bool enable) 1240 { 1241 struct e1000_hw *hw = 1242 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1243 uint32_t tctl, rctl; 1244 1245 tctl = E1000_READ_REG(hw, E1000_TCTL); 1246 rctl = E1000_READ_REG(hw, E1000_RCTL); 1247 1248 if (enable) { 1249 /* enable Tx/Rx */ 1250 tctl |= E1000_TCTL_EN; 1251 rctl |= E1000_RCTL_EN; 1252 } else { 1253 /* disable Tx/Rx */ 1254 tctl &= ~E1000_TCTL_EN; 1255 rctl &= ~E1000_RCTL_EN; 1256 } 1257 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1258 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1259 E1000_WRITE_FLUSH(hw); 1260 } 1261 1262 static int 1263 eth_igb_start(struct rte_eth_dev *dev) 1264 { 1265 struct e1000_hw *hw = 1266 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1267 struct e1000_adapter *adapter = 1268 E1000_DEV_PRIVATE(dev->data->dev_private); 1269 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1270 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1271 int ret, mask; 1272 uint32_t intr_vector = 0; 1273 uint32_t ctrl_ext; 1274 uint32_t *speeds; 1275 int num_speeds; 1276 bool autoneg; 1277 1278 PMD_INIT_FUNC_TRACE(); 1279 1280 /* disable uio/vfio intr/eventfd mapping */ 1281 rte_intr_disable(intr_handle); 1282 1283 /* Power up the phy. Needed to make the link go Up */ 1284 eth_igb_dev_set_link_up(dev); 1285 1286 /* 1287 * Packet Buffer Allocation (PBA) 1288 * Writing PBA sets the receive portion of the buffer 1289 * the remainder is used for the transmit buffer. 1290 */ 1291 if (hw->mac.type == e1000_82575) { 1292 uint32_t pba; 1293 1294 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1295 E1000_WRITE_REG(hw, E1000_PBA, pba); 1296 } 1297 1298 /* Put the address into the Receive Address Array */ 1299 e1000_rar_set(hw, hw->mac.addr, 0); 1300 1301 /* Initialize the hardware */ 1302 if (igb_hardware_init(hw)) { 1303 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1304 return -EIO; 1305 } 1306 adapter->stopped = 0; 1307 1308 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1309 1310 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1311 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1312 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1313 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1314 E1000_WRITE_FLUSH(hw); 1315 1316 /* configure PF module if SRIOV enabled */ 1317 igb_pf_host_configure(dev); 1318 1319 /* check and configure queue intr-vector mapping */ 1320 if ((rte_intr_cap_multiple(intr_handle) || 1321 !RTE_ETH_DEV_SRIOV(dev).active) && 1322 dev->data->dev_conf.intr_conf.rxq != 0) { 1323 intr_vector = dev->data->nb_rx_queues; 1324 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1325 return -1; 1326 } 1327 1328 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1329 intr_handle->intr_vec = 1330 rte_zmalloc("intr_vec", 1331 dev->data->nb_rx_queues * sizeof(int), 0); 1332 if (intr_handle->intr_vec == NULL) { 1333 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1334 " intr_vec", dev->data->nb_rx_queues); 1335 return -ENOMEM; 1336 } 1337 } 1338 1339 /* confiugre msix for rx interrupt */ 1340 eth_igb_configure_msix_intr(dev); 1341 1342 /* Configure for OS presence */ 1343 igb_init_manageability(hw); 1344 1345 eth_igb_tx_init(dev); 1346 1347 /* This can fail when allocating mbufs for descriptor rings */ 1348 ret = eth_igb_rx_init(dev); 1349 if (ret) { 1350 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1351 igb_dev_clear_queues(dev); 1352 return ret; 1353 } 1354 1355 e1000_clear_hw_cntrs_base_generic(hw); 1356 1357 /* 1358 * VLAN Offload Settings 1359 */ 1360 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1361 ETH_VLAN_EXTEND_MASK; 1362 ret = eth_igb_vlan_offload_set(dev, mask); 1363 if (ret) { 1364 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1365 igb_dev_clear_queues(dev); 1366 return ret; 1367 } 1368 1369 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1370 /* Enable VLAN filter since VMDq always use VLAN filter */ 1371 igb_vmdq_vlan_hw_filter_enable(dev); 1372 } 1373 1374 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1375 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1376 (hw->mac.type == e1000_i211)) { 1377 /* Configure EITR with the maximum possible value (0xFFFF) */ 1378 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1379 } 1380 1381 /* Setup link speed and duplex */ 1382 speeds = &dev->data->dev_conf.link_speeds; 1383 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1384 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1385 hw->mac.autoneg = 1; 1386 } else { 1387 num_speeds = 0; 1388 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1389 1390 /* Reset */ 1391 hw->phy.autoneg_advertised = 0; 1392 1393 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1394 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1395 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1396 num_speeds = -1; 1397 goto error_invalid_config; 1398 } 1399 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1400 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1401 num_speeds++; 1402 } 1403 if (*speeds & ETH_LINK_SPEED_10M) { 1404 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1405 num_speeds++; 1406 } 1407 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1408 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1409 num_speeds++; 1410 } 1411 if (*speeds & ETH_LINK_SPEED_100M) { 1412 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1413 num_speeds++; 1414 } 1415 if (*speeds & ETH_LINK_SPEED_1G) { 1416 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1417 num_speeds++; 1418 } 1419 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1420 goto error_invalid_config; 1421 1422 /* Set/reset the mac.autoneg based on the link speed, 1423 * fixed or not 1424 */ 1425 if (!autoneg) { 1426 hw->mac.autoneg = 0; 1427 hw->mac.forced_speed_duplex = 1428 hw->phy.autoneg_advertised; 1429 } else { 1430 hw->mac.autoneg = 1; 1431 } 1432 } 1433 1434 e1000_setup_link(hw); 1435 1436 if (rte_intr_allow_others(intr_handle)) { 1437 /* check if lsc interrupt is enabled */ 1438 if (dev->data->dev_conf.intr_conf.lsc != 0) 1439 eth_igb_lsc_interrupt_setup(dev, TRUE); 1440 else 1441 eth_igb_lsc_interrupt_setup(dev, FALSE); 1442 } else { 1443 rte_intr_callback_unregister(intr_handle, 1444 eth_igb_interrupt_handler, 1445 (void *)dev); 1446 if (dev->data->dev_conf.intr_conf.lsc != 0) 1447 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1448 " no intr multiplex"); 1449 } 1450 1451 /* check if rxq interrupt is enabled */ 1452 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1453 rte_intr_dp_is_en(intr_handle)) 1454 eth_igb_rxq_interrupt_setup(dev); 1455 1456 /* enable uio/vfio intr/eventfd mapping */ 1457 rte_intr_enable(intr_handle); 1458 1459 /* resume enabled intr since hw reset */ 1460 igb_intr_enable(dev); 1461 1462 /* restore all types filter */ 1463 igb_filter_restore(dev); 1464 1465 eth_igb_rxtx_control(dev, true); 1466 eth_igb_link_update(dev, 0); 1467 1468 PMD_INIT_LOG(DEBUG, "<<"); 1469 1470 return 0; 1471 1472 error_invalid_config: 1473 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1474 dev->data->dev_conf.link_speeds, dev->data->port_id); 1475 igb_dev_clear_queues(dev); 1476 return -EINVAL; 1477 } 1478 1479 /********************************************************************* 1480 * 1481 * This routine disables all traffic on the adapter by issuing a 1482 * global reset on the MAC. 1483 * 1484 **********************************************************************/ 1485 static void 1486 eth_igb_stop(struct rte_eth_dev *dev) 1487 { 1488 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1489 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1490 struct rte_eth_link link; 1491 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1492 1493 eth_igb_rxtx_control(dev, false); 1494 1495 igb_intr_disable(hw); 1496 1497 /* disable intr eventfd mapping */ 1498 rte_intr_disable(intr_handle); 1499 1500 igb_pf_reset_hw(hw); 1501 E1000_WRITE_REG(hw, E1000_WUC, 0); 1502 1503 /* Set bit for Go Link disconnect */ 1504 if (hw->mac.type >= e1000_82580) { 1505 uint32_t phpm_reg; 1506 1507 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1508 phpm_reg |= E1000_82580_PM_GO_LINKD; 1509 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1510 } 1511 1512 /* Power down the phy. Needed to make the link go Down */ 1513 eth_igb_dev_set_link_down(dev); 1514 1515 igb_dev_clear_queues(dev); 1516 1517 /* clear the recorded link status */ 1518 memset(&link, 0, sizeof(link)); 1519 rte_eth_linkstatus_set(dev, &link); 1520 1521 if (!rte_intr_allow_others(intr_handle)) 1522 /* resume to the default handler */ 1523 rte_intr_callback_register(intr_handle, 1524 eth_igb_interrupt_handler, 1525 (void *)dev); 1526 1527 /* Clean datapath event and queue/vec mapping */ 1528 rte_intr_efd_disable(intr_handle); 1529 if (intr_handle->intr_vec != NULL) { 1530 rte_free(intr_handle->intr_vec); 1531 intr_handle->intr_vec = NULL; 1532 } 1533 } 1534 1535 static int 1536 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1537 { 1538 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1539 1540 if (hw->phy.media_type == e1000_media_type_copper) 1541 e1000_power_up_phy(hw); 1542 else 1543 e1000_power_up_fiber_serdes_link(hw); 1544 1545 return 0; 1546 } 1547 1548 static int 1549 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1550 { 1551 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1552 1553 if (hw->phy.media_type == e1000_media_type_copper) 1554 e1000_power_down_phy(hw); 1555 else 1556 e1000_shutdown_fiber_serdes_link(hw); 1557 1558 return 0; 1559 } 1560 1561 static void 1562 eth_igb_close(struct rte_eth_dev *dev) 1563 { 1564 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1565 struct e1000_adapter *adapter = 1566 E1000_DEV_PRIVATE(dev->data->dev_private); 1567 struct rte_eth_link link; 1568 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1569 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1570 1571 eth_igb_stop(dev); 1572 adapter->stopped = 1; 1573 1574 e1000_phy_hw_reset(hw); 1575 igb_release_manageability(hw); 1576 igb_hw_control_release(hw); 1577 1578 /* Clear bit for Go Link disconnect */ 1579 if (hw->mac.type >= e1000_82580) { 1580 uint32_t phpm_reg; 1581 1582 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1583 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1584 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1585 } 1586 1587 igb_dev_free_queues(dev); 1588 1589 if (intr_handle->intr_vec) { 1590 rte_free(intr_handle->intr_vec); 1591 intr_handle->intr_vec = NULL; 1592 } 1593 1594 memset(&link, 0, sizeof(link)); 1595 rte_eth_linkstatus_set(dev, &link); 1596 } 1597 1598 /* 1599 * Reset PF device. 1600 */ 1601 static int 1602 eth_igb_reset(struct rte_eth_dev *dev) 1603 { 1604 int ret; 1605 1606 /* When a DPDK PMD PF begin to reset PF port, it should notify all 1607 * its VF to make them align with it. The detailed notification 1608 * mechanism is PMD specific and is currently not implemented. 1609 * To avoid unexpected behavior in VF, currently reset of PF with 1610 * SR-IOV activation is not supported. It might be supported later. 1611 */ 1612 if (dev->data->sriov.active) 1613 return -ENOTSUP; 1614 1615 ret = eth_igb_dev_uninit(dev); 1616 if (ret) 1617 return ret; 1618 1619 ret = eth_igb_dev_init(dev); 1620 1621 return ret; 1622 } 1623 1624 1625 static int 1626 igb_get_rx_buffer_size(struct e1000_hw *hw) 1627 { 1628 uint32_t rx_buf_size; 1629 if (hw->mac.type == e1000_82576) { 1630 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1631 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1632 /* PBS needs to be translated according to a lookup table */ 1633 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1634 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1635 rx_buf_size = (rx_buf_size << 10); 1636 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1637 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1638 } else { 1639 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1640 } 1641 1642 return rx_buf_size; 1643 } 1644 1645 /********************************************************************* 1646 * 1647 * Initialize the hardware 1648 * 1649 **********************************************************************/ 1650 static int 1651 igb_hardware_init(struct e1000_hw *hw) 1652 { 1653 uint32_t rx_buf_size; 1654 int diag; 1655 1656 /* Let the firmware know the OS is in control */ 1657 igb_hw_control_acquire(hw); 1658 1659 /* 1660 * These parameters control the automatic generation (Tx) and 1661 * response (Rx) to Ethernet PAUSE frames. 1662 * - High water mark should allow for at least two standard size (1518) 1663 * frames to be received after sending an XOFF. 1664 * - Low water mark works best when it is very near the high water mark. 1665 * This allows the receiver to restart by sending XON when it has 1666 * drained a bit. Here we use an arbitrary value of 1500 which will 1667 * restart after one full frame is pulled from the buffer. There 1668 * could be several smaller frames in the buffer and if so they will 1669 * not trigger the XON until their total number reduces the buffer 1670 * by 1500. 1671 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1672 */ 1673 rx_buf_size = igb_get_rx_buffer_size(hw); 1674 1675 hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2); 1676 hw->fc.low_water = hw->fc.high_water - 1500; 1677 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1678 hw->fc.send_xon = 1; 1679 1680 /* Set Flow control, use the tunable location if sane */ 1681 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1682 hw->fc.requested_mode = igb_fc_setting; 1683 else 1684 hw->fc.requested_mode = e1000_fc_none; 1685 1686 /* Issue a global reset */ 1687 igb_pf_reset_hw(hw); 1688 E1000_WRITE_REG(hw, E1000_WUC, 0); 1689 1690 diag = e1000_init_hw(hw); 1691 if (diag < 0) 1692 return diag; 1693 1694 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1695 e1000_get_phy_info(hw); 1696 e1000_check_for_link(hw); 1697 1698 return 0; 1699 } 1700 1701 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1702 static void 1703 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1704 { 1705 int pause_frames; 1706 1707 uint64_t old_gprc = stats->gprc; 1708 uint64_t old_gptc = stats->gptc; 1709 uint64_t old_tpr = stats->tpr; 1710 uint64_t old_tpt = stats->tpt; 1711 uint64_t old_rpthc = stats->rpthc; 1712 uint64_t old_hgptc = stats->hgptc; 1713 1714 if(hw->phy.media_type == e1000_media_type_copper || 1715 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1716 stats->symerrs += 1717 E1000_READ_REG(hw,E1000_SYMERRS); 1718 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1719 } 1720 1721 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1722 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1723 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1724 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1725 1726 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1727 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1728 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1729 stats->dc += E1000_READ_REG(hw, E1000_DC); 1730 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1731 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1732 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1733 /* 1734 ** For watchdog management we need to know if we have been 1735 ** paused during the last interval, so capture that here. 1736 */ 1737 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1738 stats->xoffrxc += pause_frames; 1739 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1740 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1741 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1742 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1743 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1744 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1745 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1746 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1747 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1748 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1749 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1750 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1751 1752 /* For the 64-bit byte counters the low dword must be read first. */ 1753 /* Both registers clear on the read of the high dword */ 1754 1755 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1756 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1757 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1758 stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN; 1759 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1760 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1761 stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN; 1762 1763 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1764 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1765 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1766 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1767 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1768 1769 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1770 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1771 1772 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1773 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1774 stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN; 1775 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1776 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1777 stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN; 1778 1779 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1780 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1781 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1782 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1783 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1784 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1785 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1786 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1787 1788 /* Interrupt Counts */ 1789 1790 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1791 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1792 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1793 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1794 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1795 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1796 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1797 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1798 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1799 1800 /* Host to Card Statistics */ 1801 1802 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1803 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1804 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1805 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1806 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1807 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1808 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1809 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1810 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1811 stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN; 1812 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1813 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1814 stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN; 1815 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1816 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1817 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1818 1819 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1820 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1821 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1822 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1823 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1824 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1825 } 1826 1827 static int 1828 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1829 { 1830 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1831 struct e1000_hw_stats *stats = 1832 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1833 1834 igb_read_stats_registers(hw, stats); 1835 1836 if (rte_stats == NULL) 1837 return -EINVAL; 1838 1839 /* Rx Errors */ 1840 rte_stats->imissed = stats->mpc; 1841 rte_stats->ierrors = stats->crcerrs + 1842 stats->rlec + stats->ruc + stats->roc + 1843 stats->rxerrc + stats->algnerrc + stats->cexterr; 1844 1845 /* Tx Errors */ 1846 rte_stats->oerrors = stats->ecol + stats->latecol; 1847 1848 rte_stats->ipackets = stats->gprc; 1849 rte_stats->opackets = stats->gptc; 1850 rte_stats->ibytes = stats->gorc; 1851 rte_stats->obytes = stats->gotc; 1852 return 0; 1853 } 1854 1855 static void 1856 eth_igb_stats_reset(struct rte_eth_dev *dev) 1857 { 1858 struct e1000_hw_stats *hw_stats = 1859 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1860 1861 /* HW registers are cleared on read */ 1862 eth_igb_stats_get(dev, NULL); 1863 1864 /* Reset software totals */ 1865 memset(hw_stats, 0, sizeof(*hw_stats)); 1866 } 1867 1868 static void 1869 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1870 { 1871 struct e1000_hw_stats *stats = 1872 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1873 1874 /* HW registers are cleared on read */ 1875 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1876 1877 /* Reset software totals */ 1878 memset(stats, 0, sizeof(*stats)); 1879 } 1880 1881 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1882 struct rte_eth_xstat_name *xstats_names, 1883 __rte_unused unsigned int size) 1884 { 1885 unsigned i; 1886 1887 if (xstats_names == NULL) 1888 return IGB_NB_XSTATS; 1889 1890 /* Note: limit checked in rte_eth_xstats_names() */ 1891 1892 for (i = 0; i < IGB_NB_XSTATS; i++) { 1893 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), 1894 "%s", rte_igb_stats_strings[i].name); 1895 } 1896 1897 return IGB_NB_XSTATS; 1898 } 1899 1900 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1901 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1902 unsigned int limit) 1903 { 1904 unsigned int i; 1905 1906 if (!ids) { 1907 if (xstats_names == NULL) 1908 return IGB_NB_XSTATS; 1909 1910 for (i = 0; i < IGB_NB_XSTATS; i++) 1911 snprintf(xstats_names[i].name, 1912 sizeof(xstats_names[i].name), 1913 "%s", rte_igb_stats_strings[i].name); 1914 1915 return IGB_NB_XSTATS; 1916 1917 } else { 1918 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1919 1920 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1921 IGB_NB_XSTATS); 1922 1923 for (i = 0; i < limit; i++) { 1924 if (ids[i] >= IGB_NB_XSTATS) { 1925 PMD_INIT_LOG(ERR, "id value isn't valid"); 1926 return -1; 1927 } 1928 strcpy(xstats_names[i].name, 1929 xstats_names_copy[ids[i]].name); 1930 } 1931 return limit; 1932 } 1933 } 1934 1935 static int 1936 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1937 unsigned n) 1938 { 1939 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1940 struct e1000_hw_stats *hw_stats = 1941 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1942 unsigned i; 1943 1944 if (n < IGB_NB_XSTATS) 1945 return IGB_NB_XSTATS; 1946 1947 igb_read_stats_registers(hw, hw_stats); 1948 1949 /* If this is a reset xstats is NULL, and we have cleared the 1950 * registers by reading them. 1951 */ 1952 if (!xstats) 1953 return 0; 1954 1955 /* Extended stats */ 1956 for (i = 0; i < IGB_NB_XSTATS; i++) { 1957 xstats[i].id = i; 1958 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1959 rte_igb_stats_strings[i].offset); 1960 } 1961 1962 return IGB_NB_XSTATS; 1963 } 1964 1965 static int 1966 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1967 uint64_t *values, unsigned int n) 1968 { 1969 unsigned int i; 1970 1971 if (!ids) { 1972 struct e1000_hw *hw = 1973 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1974 struct e1000_hw_stats *hw_stats = 1975 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1976 1977 if (n < IGB_NB_XSTATS) 1978 return IGB_NB_XSTATS; 1979 1980 igb_read_stats_registers(hw, hw_stats); 1981 1982 /* If this is a reset xstats is NULL, and we have cleared the 1983 * registers by reading them. 1984 */ 1985 if (!values) 1986 return 0; 1987 1988 /* Extended stats */ 1989 for (i = 0; i < IGB_NB_XSTATS; i++) 1990 values[i] = *(uint64_t *)(((char *)hw_stats) + 1991 rte_igb_stats_strings[i].offset); 1992 1993 return IGB_NB_XSTATS; 1994 1995 } else { 1996 uint64_t values_copy[IGB_NB_XSTATS]; 1997 1998 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 1999 IGB_NB_XSTATS); 2000 2001 for (i = 0; i < n; i++) { 2002 if (ids[i] >= IGB_NB_XSTATS) { 2003 PMD_INIT_LOG(ERR, "id value isn't valid"); 2004 return -1; 2005 } 2006 values[i] = values_copy[ids[i]]; 2007 } 2008 return n; 2009 } 2010 } 2011 2012 static void 2013 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2014 { 2015 /* Good Rx packets, include VF loopback */ 2016 UPDATE_VF_STAT(E1000_VFGPRC, 2017 hw_stats->last_gprc, hw_stats->gprc); 2018 2019 /* Good Rx octets, include VF loopback */ 2020 UPDATE_VF_STAT(E1000_VFGORC, 2021 hw_stats->last_gorc, hw_stats->gorc); 2022 2023 /* Good Tx packets, include VF loopback */ 2024 UPDATE_VF_STAT(E1000_VFGPTC, 2025 hw_stats->last_gptc, hw_stats->gptc); 2026 2027 /* Good Tx octets, include VF loopback */ 2028 UPDATE_VF_STAT(E1000_VFGOTC, 2029 hw_stats->last_gotc, hw_stats->gotc); 2030 2031 /* Rx Multicst packets */ 2032 UPDATE_VF_STAT(E1000_VFMPRC, 2033 hw_stats->last_mprc, hw_stats->mprc); 2034 2035 /* Good Rx loopback packets */ 2036 UPDATE_VF_STAT(E1000_VFGPRLBC, 2037 hw_stats->last_gprlbc, hw_stats->gprlbc); 2038 2039 /* Good Rx loopback octets */ 2040 UPDATE_VF_STAT(E1000_VFGORLBC, 2041 hw_stats->last_gorlbc, hw_stats->gorlbc); 2042 2043 /* Good Tx loopback packets */ 2044 UPDATE_VF_STAT(E1000_VFGPTLBC, 2045 hw_stats->last_gptlbc, hw_stats->gptlbc); 2046 2047 /* Good Tx loopback octets */ 2048 UPDATE_VF_STAT(E1000_VFGOTLBC, 2049 hw_stats->last_gotlbc, hw_stats->gotlbc); 2050 } 2051 2052 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2053 struct rte_eth_xstat_name *xstats_names, 2054 __rte_unused unsigned limit) 2055 { 2056 unsigned i; 2057 2058 if (xstats_names != NULL) 2059 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2060 snprintf(xstats_names[i].name, 2061 sizeof(xstats_names[i].name), "%s", 2062 rte_igbvf_stats_strings[i].name); 2063 } 2064 return IGBVF_NB_XSTATS; 2065 } 2066 2067 static int 2068 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2069 unsigned n) 2070 { 2071 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2072 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2073 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2074 unsigned i; 2075 2076 if (n < IGBVF_NB_XSTATS) 2077 return IGBVF_NB_XSTATS; 2078 2079 igbvf_read_stats_registers(hw, hw_stats); 2080 2081 if (!xstats) 2082 return 0; 2083 2084 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2085 xstats[i].id = i; 2086 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2087 rte_igbvf_stats_strings[i].offset); 2088 } 2089 2090 return IGBVF_NB_XSTATS; 2091 } 2092 2093 static int 2094 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2095 { 2096 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2097 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2098 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2099 2100 igbvf_read_stats_registers(hw, hw_stats); 2101 2102 if (rte_stats == NULL) 2103 return -EINVAL; 2104 2105 rte_stats->ipackets = hw_stats->gprc; 2106 rte_stats->ibytes = hw_stats->gorc; 2107 rte_stats->opackets = hw_stats->gptc; 2108 rte_stats->obytes = hw_stats->gotc; 2109 return 0; 2110 } 2111 2112 static void 2113 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2114 { 2115 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2116 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2117 2118 /* Sync HW register to the last stats */ 2119 eth_igbvf_stats_get(dev, NULL); 2120 2121 /* reset HW current stats*/ 2122 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2123 offsetof(struct e1000_vf_stats, gprc)); 2124 } 2125 2126 static int 2127 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2128 size_t fw_size) 2129 { 2130 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2131 struct e1000_fw_version fw; 2132 int ret; 2133 2134 e1000_get_fw_version(hw, &fw); 2135 2136 switch (hw->mac.type) { 2137 case e1000_i210: 2138 case e1000_i211: 2139 if (!(e1000_get_flash_presence_i210(hw))) { 2140 ret = snprintf(fw_version, fw_size, 2141 "%2d.%2d-%d", 2142 fw.invm_major, fw.invm_minor, 2143 fw.invm_img_type); 2144 break; 2145 } 2146 /* fall through */ 2147 default: 2148 /* if option rom is valid, display its version too */ 2149 if (fw.or_valid) { 2150 ret = snprintf(fw_version, fw_size, 2151 "%d.%d, 0x%08x, %d.%d.%d", 2152 fw.eep_major, fw.eep_minor, fw.etrack_id, 2153 fw.or_major, fw.or_build, fw.or_patch); 2154 /* no option rom */ 2155 } else { 2156 if (fw.etrack_id != 0X0000) { 2157 ret = snprintf(fw_version, fw_size, 2158 "%d.%d, 0x%08x", 2159 fw.eep_major, fw.eep_minor, 2160 fw.etrack_id); 2161 } else { 2162 ret = snprintf(fw_version, fw_size, 2163 "%d.%d.%d", 2164 fw.eep_major, fw.eep_minor, 2165 fw.eep_build); 2166 } 2167 } 2168 break; 2169 } 2170 2171 ret += 1; /* add the size of '\0' */ 2172 if (fw_size < (u32)ret) 2173 return ret; 2174 else 2175 return 0; 2176 } 2177 2178 static void 2179 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2180 { 2181 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2182 2183 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2184 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2185 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2186 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2187 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2188 dev_info->rx_queue_offload_capa; 2189 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2190 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2191 dev_info->tx_queue_offload_capa; 2192 2193 switch (hw->mac.type) { 2194 case e1000_82575: 2195 dev_info->max_rx_queues = 4; 2196 dev_info->max_tx_queues = 4; 2197 dev_info->max_vmdq_pools = 0; 2198 break; 2199 2200 case e1000_82576: 2201 dev_info->max_rx_queues = 16; 2202 dev_info->max_tx_queues = 16; 2203 dev_info->max_vmdq_pools = ETH_8_POOLS; 2204 dev_info->vmdq_queue_num = 16; 2205 break; 2206 2207 case e1000_82580: 2208 dev_info->max_rx_queues = 8; 2209 dev_info->max_tx_queues = 8; 2210 dev_info->max_vmdq_pools = ETH_8_POOLS; 2211 dev_info->vmdq_queue_num = 8; 2212 break; 2213 2214 case e1000_i350: 2215 dev_info->max_rx_queues = 8; 2216 dev_info->max_tx_queues = 8; 2217 dev_info->max_vmdq_pools = ETH_8_POOLS; 2218 dev_info->vmdq_queue_num = 8; 2219 break; 2220 2221 case e1000_i354: 2222 dev_info->max_rx_queues = 8; 2223 dev_info->max_tx_queues = 8; 2224 break; 2225 2226 case e1000_i210: 2227 dev_info->max_rx_queues = 4; 2228 dev_info->max_tx_queues = 4; 2229 dev_info->max_vmdq_pools = 0; 2230 break; 2231 2232 case e1000_i211: 2233 dev_info->max_rx_queues = 2; 2234 dev_info->max_tx_queues = 2; 2235 dev_info->max_vmdq_pools = 0; 2236 break; 2237 2238 default: 2239 /* Should not happen */ 2240 break; 2241 } 2242 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2243 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2244 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2245 2246 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2247 .rx_thresh = { 2248 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2249 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2250 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2251 }, 2252 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2253 .rx_drop_en = 0, 2254 .offloads = 0, 2255 }; 2256 2257 dev_info->default_txconf = (struct rte_eth_txconf) { 2258 .tx_thresh = { 2259 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2260 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2261 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2262 }, 2263 .offloads = 0, 2264 }; 2265 2266 dev_info->rx_desc_lim = rx_desc_lim; 2267 dev_info->tx_desc_lim = tx_desc_lim; 2268 2269 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2270 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2271 ETH_LINK_SPEED_1G; 2272 } 2273 2274 static const uint32_t * 2275 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2276 { 2277 static const uint32_t ptypes[] = { 2278 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2279 RTE_PTYPE_L2_ETHER, 2280 RTE_PTYPE_L3_IPV4, 2281 RTE_PTYPE_L3_IPV4_EXT, 2282 RTE_PTYPE_L3_IPV6, 2283 RTE_PTYPE_L3_IPV6_EXT, 2284 RTE_PTYPE_L4_TCP, 2285 RTE_PTYPE_L4_UDP, 2286 RTE_PTYPE_L4_SCTP, 2287 RTE_PTYPE_TUNNEL_IP, 2288 RTE_PTYPE_INNER_L3_IPV6, 2289 RTE_PTYPE_INNER_L3_IPV6_EXT, 2290 RTE_PTYPE_INNER_L4_TCP, 2291 RTE_PTYPE_INNER_L4_UDP, 2292 RTE_PTYPE_UNKNOWN 2293 }; 2294 2295 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2296 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2297 return ptypes; 2298 return NULL; 2299 } 2300 2301 static void 2302 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2303 { 2304 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2305 2306 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2307 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2308 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2309 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2310 DEV_TX_OFFLOAD_IPV4_CKSUM | 2311 DEV_TX_OFFLOAD_UDP_CKSUM | 2312 DEV_TX_OFFLOAD_TCP_CKSUM | 2313 DEV_TX_OFFLOAD_SCTP_CKSUM | 2314 DEV_TX_OFFLOAD_TCP_TSO; 2315 switch (hw->mac.type) { 2316 case e1000_vfadapt: 2317 dev_info->max_rx_queues = 2; 2318 dev_info->max_tx_queues = 2; 2319 break; 2320 case e1000_vfadapt_i350: 2321 dev_info->max_rx_queues = 1; 2322 dev_info->max_tx_queues = 1; 2323 break; 2324 default: 2325 /* Should not happen */ 2326 break; 2327 } 2328 2329 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2330 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2331 dev_info->rx_queue_offload_capa; 2332 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2333 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2334 dev_info->tx_queue_offload_capa; 2335 2336 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2337 .rx_thresh = { 2338 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2339 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2340 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2341 }, 2342 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2343 .rx_drop_en = 0, 2344 .offloads = 0, 2345 }; 2346 2347 dev_info->default_txconf = (struct rte_eth_txconf) { 2348 .tx_thresh = { 2349 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2350 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2351 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2352 }, 2353 .offloads = 0, 2354 }; 2355 2356 dev_info->rx_desc_lim = rx_desc_lim; 2357 dev_info->tx_desc_lim = tx_desc_lim; 2358 } 2359 2360 /* return 0 means link status changed, -1 means not changed */ 2361 static int 2362 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2363 { 2364 struct e1000_hw *hw = 2365 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2366 struct rte_eth_link link; 2367 int link_check, count; 2368 2369 link_check = 0; 2370 hw->mac.get_link_status = 1; 2371 2372 /* possible wait-to-complete in up to 9 seconds */ 2373 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2374 /* Read the real link status */ 2375 switch (hw->phy.media_type) { 2376 case e1000_media_type_copper: 2377 /* Do the work to read phy */ 2378 e1000_check_for_link(hw); 2379 link_check = !hw->mac.get_link_status; 2380 break; 2381 2382 case e1000_media_type_fiber: 2383 e1000_check_for_link(hw); 2384 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2385 E1000_STATUS_LU); 2386 break; 2387 2388 case e1000_media_type_internal_serdes: 2389 e1000_check_for_link(hw); 2390 link_check = hw->mac.serdes_has_link; 2391 break; 2392 2393 /* VF device is type_unknown */ 2394 case e1000_media_type_unknown: 2395 eth_igbvf_link_update(hw); 2396 link_check = !hw->mac.get_link_status; 2397 break; 2398 2399 default: 2400 break; 2401 } 2402 if (link_check || wait_to_complete == 0) 2403 break; 2404 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2405 } 2406 memset(&link, 0, sizeof(link)); 2407 2408 /* Now we check if a transition has happened */ 2409 if (link_check) { 2410 uint16_t duplex, speed; 2411 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2412 link.link_duplex = (duplex == FULL_DUPLEX) ? 2413 ETH_LINK_FULL_DUPLEX : 2414 ETH_LINK_HALF_DUPLEX; 2415 link.link_speed = speed; 2416 link.link_status = ETH_LINK_UP; 2417 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2418 ETH_LINK_SPEED_FIXED); 2419 } else if (!link_check) { 2420 link.link_speed = 0; 2421 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2422 link.link_status = ETH_LINK_DOWN; 2423 link.link_autoneg = ETH_LINK_FIXED; 2424 } 2425 2426 return rte_eth_linkstatus_set(dev, &link); 2427 } 2428 2429 /* 2430 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2431 * For ASF and Pass Through versions of f/w this means 2432 * that the driver is loaded. 2433 */ 2434 static void 2435 igb_hw_control_acquire(struct e1000_hw *hw) 2436 { 2437 uint32_t ctrl_ext; 2438 2439 /* Let firmware know the driver has taken over */ 2440 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2441 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2442 } 2443 2444 /* 2445 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2446 * For ASF and Pass Through versions of f/w this means that the 2447 * driver is no longer loaded. 2448 */ 2449 static void 2450 igb_hw_control_release(struct e1000_hw *hw) 2451 { 2452 uint32_t ctrl_ext; 2453 2454 /* Let firmware taken over control of h/w */ 2455 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2456 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2457 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2458 } 2459 2460 /* 2461 * Bit of a misnomer, what this really means is 2462 * to enable OS management of the system... aka 2463 * to disable special hardware management features. 2464 */ 2465 static void 2466 igb_init_manageability(struct e1000_hw *hw) 2467 { 2468 if (e1000_enable_mng_pass_thru(hw)) { 2469 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2470 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2471 2472 /* disable hardware interception of ARP */ 2473 manc &= ~(E1000_MANC_ARP_EN); 2474 2475 /* enable receiving management packets to the host */ 2476 manc |= E1000_MANC_EN_MNG2HOST; 2477 manc2h |= 1 << 5; /* Mng Port 623 */ 2478 manc2h |= 1 << 6; /* Mng Port 664 */ 2479 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2480 E1000_WRITE_REG(hw, E1000_MANC, manc); 2481 } 2482 } 2483 2484 static void 2485 igb_release_manageability(struct e1000_hw *hw) 2486 { 2487 if (e1000_enable_mng_pass_thru(hw)) { 2488 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2489 2490 manc |= E1000_MANC_ARP_EN; 2491 manc &= ~E1000_MANC_EN_MNG2HOST; 2492 2493 E1000_WRITE_REG(hw, E1000_MANC, manc); 2494 } 2495 } 2496 2497 static void 2498 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2499 { 2500 struct e1000_hw *hw = 2501 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2502 uint32_t rctl; 2503 2504 rctl = E1000_READ_REG(hw, E1000_RCTL); 2505 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2506 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2507 } 2508 2509 static void 2510 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2511 { 2512 struct e1000_hw *hw = 2513 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2514 uint32_t rctl; 2515 2516 rctl = E1000_READ_REG(hw, E1000_RCTL); 2517 rctl &= (~E1000_RCTL_UPE); 2518 if (dev->data->all_multicast == 1) 2519 rctl |= E1000_RCTL_MPE; 2520 else 2521 rctl &= (~E1000_RCTL_MPE); 2522 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2523 } 2524 2525 static void 2526 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2527 { 2528 struct e1000_hw *hw = 2529 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2530 uint32_t rctl; 2531 2532 rctl = E1000_READ_REG(hw, E1000_RCTL); 2533 rctl |= E1000_RCTL_MPE; 2534 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2535 } 2536 2537 static void 2538 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2539 { 2540 struct e1000_hw *hw = 2541 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2542 uint32_t rctl; 2543 2544 if (dev->data->promiscuous == 1) 2545 return; /* must remain in all_multicast mode */ 2546 rctl = E1000_READ_REG(hw, E1000_RCTL); 2547 rctl &= (~E1000_RCTL_MPE); 2548 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2549 } 2550 2551 static int 2552 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2553 { 2554 struct e1000_hw *hw = 2555 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2556 struct e1000_vfta * shadow_vfta = 2557 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2558 uint32_t vfta; 2559 uint32_t vid_idx; 2560 uint32_t vid_bit; 2561 2562 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2563 E1000_VFTA_ENTRY_MASK); 2564 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2565 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2566 if (on) 2567 vfta |= vid_bit; 2568 else 2569 vfta &= ~vid_bit; 2570 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2571 2572 /* update local VFTA copy */ 2573 shadow_vfta->vfta[vid_idx] = vfta; 2574 2575 return 0; 2576 } 2577 2578 static int 2579 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2580 enum rte_vlan_type vlan_type, 2581 uint16_t tpid) 2582 { 2583 struct e1000_hw *hw = 2584 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2585 uint32_t reg, qinq; 2586 2587 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2588 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2589 2590 /* only outer TPID of double VLAN can be configured*/ 2591 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2592 reg = E1000_READ_REG(hw, E1000_VET); 2593 reg = (reg & (~E1000_VET_VET_EXT)) | 2594 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2595 E1000_WRITE_REG(hw, E1000_VET, reg); 2596 2597 return 0; 2598 } 2599 2600 /* all other TPID values are read-only*/ 2601 PMD_DRV_LOG(ERR, "Not supported"); 2602 2603 return -ENOTSUP; 2604 } 2605 2606 static void 2607 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2608 { 2609 struct e1000_hw *hw = 2610 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2611 uint32_t reg; 2612 2613 /* Filter Table Disable */ 2614 reg = E1000_READ_REG(hw, E1000_RCTL); 2615 reg &= ~E1000_RCTL_CFIEN; 2616 reg &= ~E1000_RCTL_VFE; 2617 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2618 } 2619 2620 static void 2621 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2622 { 2623 struct e1000_hw *hw = 2624 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2625 struct e1000_vfta * shadow_vfta = 2626 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2627 uint32_t reg; 2628 int i; 2629 2630 /* Filter Table Enable, CFI not used for packet acceptance */ 2631 reg = E1000_READ_REG(hw, E1000_RCTL); 2632 reg &= ~E1000_RCTL_CFIEN; 2633 reg |= E1000_RCTL_VFE; 2634 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2635 2636 /* restore VFTA table */ 2637 for (i = 0; i < IGB_VFTA_SIZE; i++) 2638 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2639 } 2640 2641 static void 2642 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2643 { 2644 struct e1000_hw *hw = 2645 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2646 uint32_t reg; 2647 2648 /* VLAN Mode Disable */ 2649 reg = E1000_READ_REG(hw, E1000_CTRL); 2650 reg &= ~E1000_CTRL_VME; 2651 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2652 } 2653 2654 static void 2655 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2656 { 2657 struct e1000_hw *hw = 2658 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2659 uint32_t reg; 2660 2661 /* VLAN Mode Enable */ 2662 reg = E1000_READ_REG(hw, E1000_CTRL); 2663 reg |= E1000_CTRL_VME; 2664 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2665 } 2666 2667 static void 2668 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2669 { 2670 struct e1000_hw *hw = 2671 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2672 uint32_t reg; 2673 2674 /* CTRL_EXT: Extended VLAN */ 2675 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2676 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2677 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2678 2679 /* Update maximum packet length */ 2680 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2681 E1000_WRITE_REG(hw, E1000_RLPML, 2682 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2683 VLAN_TAG_SIZE); 2684 } 2685 2686 static void 2687 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2688 { 2689 struct e1000_hw *hw = 2690 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2691 uint32_t reg; 2692 2693 /* CTRL_EXT: Extended VLAN */ 2694 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2695 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2696 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2697 2698 /* Update maximum packet length */ 2699 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2700 E1000_WRITE_REG(hw, E1000_RLPML, 2701 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2702 2 * VLAN_TAG_SIZE); 2703 } 2704 2705 static int 2706 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2707 { 2708 struct rte_eth_rxmode *rxmode; 2709 2710 rxmode = &dev->data->dev_conf.rxmode; 2711 if(mask & ETH_VLAN_STRIP_MASK){ 2712 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2713 igb_vlan_hw_strip_enable(dev); 2714 else 2715 igb_vlan_hw_strip_disable(dev); 2716 } 2717 2718 if(mask & ETH_VLAN_FILTER_MASK){ 2719 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2720 igb_vlan_hw_filter_enable(dev); 2721 else 2722 igb_vlan_hw_filter_disable(dev); 2723 } 2724 2725 if(mask & ETH_VLAN_EXTEND_MASK){ 2726 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2727 igb_vlan_hw_extend_enable(dev); 2728 else 2729 igb_vlan_hw_extend_disable(dev); 2730 } 2731 2732 return 0; 2733 } 2734 2735 2736 /** 2737 * It enables the interrupt mask and then enable the interrupt. 2738 * 2739 * @param dev 2740 * Pointer to struct rte_eth_dev. 2741 * @param on 2742 * Enable or Disable 2743 * 2744 * @return 2745 * - On success, zero. 2746 * - On failure, a negative value. 2747 */ 2748 static int 2749 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2750 { 2751 struct e1000_interrupt *intr = 2752 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2753 2754 if (on) 2755 intr->mask |= E1000_ICR_LSC; 2756 else 2757 intr->mask &= ~E1000_ICR_LSC; 2758 2759 return 0; 2760 } 2761 2762 /* It clears the interrupt causes and enables the interrupt. 2763 * It will be called once only during nic initialized. 2764 * 2765 * @param dev 2766 * Pointer to struct rte_eth_dev. 2767 * 2768 * @return 2769 * - On success, zero. 2770 * - On failure, a negative value. 2771 */ 2772 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2773 { 2774 uint32_t mask, regval; 2775 struct e1000_hw *hw = 2776 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2777 struct rte_eth_dev_info dev_info; 2778 2779 memset(&dev_info, 0, sizeof(dev_info)); 2780 eth_igb_infos_get(dev, &dev_info); 2781 2782 mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues); 2783 regval = E1000_READ_REG(hw, E1000_EIMS); 2784 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2785 2786 return 0; 2787 } 2788 2789 /* 2790 * It reads ICR and gets interrupt causes, check it and set a bit flag 2791 * to update link status. 2792 * 2793 * @param dev 2794 * Pointer to struct rte_eth_dev. 2795 * 2796 * @return 2797 * - On success, zero. 2798 * - On failure, a negative value. 2799 */ 2800 static int 2801 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2802 { 2803 uint32_t icr; 2804 struct e1000_hw *hw = 2805 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2806 struct e1000_interrupt *intr = 2807 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2808 2809 igb_intr_disable(hw); 2810 2811 /* read-on-clear nic registers here */ 2812 icr = E1000_READ_REG(hw, E1000_ICR); 2813 2814 intr->flags = 0; 2815 if (icr & E1000_ICR_LSC) { 2816 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2817 } 2818 2819 if (icr & E1000_ICR_VMMB) 2820 intr->flags |= E1000_FLAG_MAILBOX; 2821 2822 return 0; 2823 } 2824 2825 /* 2826 * It executes link_update after knowing an interrupt is prsent. 2827 * 2828 * @param dev 2829 * Pointer to struct rte_eth_dev. 2830 * 2831 * @return 2832 * - On success, zero. 2833 * - On failure, a negative value. 2834 */ 2835 static int 2836 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2837 struct rte_intr_handle *intr_handle) 2838 { 2839 struct e1000_hw *hw = 2840 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2841 struct e1000_interrupt *intr = 2842 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2843 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2844 struct rte_eth_link link; 2845 int ret; 2846 2847 if (intr->flags & E1000_FLAG_MAILBOX) { 2848 igb_pf_mbx_process(dev); 2849 intr->flags &= ~E1000_FLAG_MAILBOX; 2850 } 2851 2852 igb_intr_enable(dev); 2853 rte_intr_enable(intr_handle); 2854 2855 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2856 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2857 2858 /* set get_link_status to check register later */ 2859 hw->mac.get_link_status = 1; 2860 ret = eth_igb_link_update(dev, 0); 2861 2862 /* check if link has changed */ 2863 if (ret < 0) 2864 return 0; 2865 2866 rte_eth_linkstatus_get(dev, &link); 2867 if (link.link_status) { 2868 PMD_INIT_LOG(INFO, 2869 " Port %d: Link Up - speed %u Mbps - %s", 2870 dev->data->port_id, 2871 (unsigned)link.link_speed, 2872 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2873 "full-duplex" : "half-duplex"); 2874 } else { 2875 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2876 dev->data->port_id); 2877 } 2878 2879 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2880 pci_dev->addr.domain, 2881 pci_dev->addr.bus, 2882 pci_dev->addr.devid, 2883 pci_dev->addr.function); 2884 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2885 NULL); 2886 } 2887 2888 return 0; 2889 } 2890 2891 /** 2892 * Interrupt handler which shall be registered at first. 2893 * 2894 * @param handle 2895 * Pointer to interrupt handle. 2896 * @param param 2897 * The address of parameter (struct rte_eth_dev *) regsitered before. 2898 * 2899 * @return 2900 * void 2901 */ 2902 static void 2903 eth_igb_interrupt_handler(void *param) 2904 { 2905 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2906 2907 eth_igb_interrupt_get_status(dev); 2908 eth_igb_interrupt_action(dev, dev->intr_handle); 2909 } 2910 2911 static int 2912 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2913 { 2914 uint32_t eicr; 2915 struct e1000_hw *hw = 2916 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2917 struct e1000_interrupt *intr = 2918 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2919 2920 igbvf_intr_disable(hw); 2921 2922 /* read-on-clear nic registers here */ 2923 eicr = E1000_READ_REG(hw, E1000_EICR); 2924 intr->flags = 0; 2925 2926 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2927 intr->flags |= E1000_FLAG_MAILBOX; 2928 2929 return 0; 2930 } 2931 2932 void igbvf_mbx_process(struct rte_eth_dev *dev) 2933 { 2934 struct e1000_hw *hw = 2935 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2936 struct e1000_mbx_info *mbx = &hw->mbx; 2937 u32 in_msg = 0; 2938 2939 /* peek the message first */ 2940 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2941 2942 /* PF reset VF event */ 2943 if (in_msg == E1000_PF_CONTROL_MSG) { 2944 /* dummy mbx read to ack pf */ 2945 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2946 return; 2947 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2948 NULL); 2949 } 2950 } 2951 2952 static int 2953 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2954 { 2955 struct e1000_interrupt *intr = 2956 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2957 2958 if (intr->flags & E1000_FLAG_MAILBOX) { 2959 igbvf_mbx_process(dev); 2960 intr->flags &= ~E1000_FLAG_MAILBOX; 2961 } 2962 2963 igbvf_intr_enable(dev); 2964 rte_intr_enable(intr_handle); 2965 2966 return 0; 2967 } 2968 2969 static void 2970 eth_igbvf_interrupt_handler(void *param) 2971 { 2972 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2973 2974 eth_igbvf_interrupt_get_status(dev); 2975 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2976 } 2977 2978 static int 2979 eth_igb_led_on(struct rte_eth_dev *dev) 2980 { 2981 struct e1000_hw *hw; 2982 2983 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2984 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2985 } 2986 2987 static int 2988 eth_igb_led_off(struct rte_eth_dev *dev) 2989 { 2990 struct e1000_hw *hw; 2991 2992 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2993 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2994 } 2995 2996 static int 2997 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 2998 { 2999 struct e1000_hw *hw; 3000 uint32_t ctrl; 3001 int tx_pause; 3002 int rx_pause; 3003 3004 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3005 fc_conf->pause_time = hw->fc.pause_time; 3006 fc_conf->high_water = hw->fc.high_water; 3007 fc_conf->low_water = hw->fc.low_water; 3008 fc_conf->send_xon = hw->fc.send_xon; 3009 fc_conf->autoneg = hw->mac.autoneg; 3010 3011 /* 3012 * Return rx_pause and tx_pause status according to actual setting of 3013 * the TFCE and RFCE bits in the CTRL register. 3014 */ 3015 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3016 if (ctrl & E1000_CTRL_TFCE) 3017 tx_pause = 1; 3018 else 3019 tx_pause = 0; 3020 3021 if (ctrl & E1000_CTRL_RFCE) 3022 rx_pause = 1; 3023 else 3024 rx_pause = 0; 3025 3026 if (rx_pause && tx_pause) 3027 fc_conf->mode = RTE_FC_FULL; 3028 else if (rx_pause) 3029 fc_conf->mode = RTE_FC_RX_PAUSE; 3030 else if (tx_pause) 3031 fc_conf->mode = RTE_FC_TX_PAUSE; 3032 else 3033 fc_conf->mode = RTE_FC_NONE; 3034 3035 return 0; 3036 } 3037 3038 static int 3039 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3040 { 3041 struct e1000_hw *hw; 3042 int err; 3043 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3044 e1000_fc_none, 3045 e1000_fc_rx_pause, 3046 e1000_fc_tx_pause, 3047 e1000_fc_full 3048 }; 3049 uint32_t rx_buf_size; 3050 uint32_t max_high_water; 3051 uint32_t rctl; 3052 3053 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3054 if (fc_conf->autoneg != hw->mac.autoneg) 3055 return -ENOTSUP; 3056 rx_buf_size = igb_get_rx_buffer_size(hw); 3057 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3058 3059 /* At least reserve one Ethernet frame for watermark */ 3060 max_high_water = rx_buf_size - ETHER_MAX_LEN; 3061 if ((fc_conf->high_water > max_high_water) || 3062 (fc_conf->high_water < fc_conf->low_water)) { 3063 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3064 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3065 return -EINVAL; 3066 } 3067 3068 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3069 hw->fc.pause_time = fc_conf->pause_time; 3070 hw->fc.high_water = fc_conf->high_water; 3071 hw->fc.low_water = fc_conf->low_water; 3072 hw->fc.send_xon = fc_conf->send_xon; 3073 3074 err = e1000_setup_link_generic(hw); 3075 if (err == E1000_SUCCESS) { 3076 3077 /* check if we want to forward MAC frames - driver doesn't have native 3078 * capability to do that, so we'll write the registers ourselves */ 3079 3080 rctl = E1000_READ_REG(hw, E1000_RCTL); 3081 3082 /* set or clear MFLCN.PMCF bit depending on configuration */ 3083 if (fc_conf->mac_ctrl_frame_fwd != 0) 3084 rctl |= E1000_RCTL_PMCF; 3085 else 3086 rctl &= ~E1000_RCTL_PMCF; 3087 3088 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3089 E1000_WRITE_FLUSH(hw); 3090 3091 return 0; 3092 } 3093 3094 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3095 return -EIO; 3096 } 3097 3098 #define E1000_RAH_POOLSEL_SHIFT (18) 3099 static int 3100 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 3101 uint32_t index, uint32_t pool) 3102 { 3103 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3104 uint32_t rah; 3105 3106 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3107 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3108 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3109 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3110 return 0; 3111 } 3112 3113 static void 3114 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3115 { 3116 uint8_t addr[ETHER_ADDR_LEN]; 3117 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3118 3119 memset(addr, 0, sizeof(addr)); 3120 3121 e1000_rar_set(hw, addr, index); 3122 } 3123 3124 static int 3125 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3126 struct ether_addr *addr) 3127 { 3128 eth_igb_rar_clear(dev, 0); 3129 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3130 3131 return 0; 3132 } 3133 /* 3134 * Virtual Function operations 3135 */ 3136 static void 3137 igbvf_intr_disable(struct e1000_hw *hw) 3138 { 3139 PMD_INIT_FUNC_TRACE(); 3140 3141 /* Clear interrupt mask to stop from interrupts being generated */ 3142 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3143 3144 E1000_WRITE_FLUSH(hw); 3145 } 3146 3147 static void 3148 igbvf_stop_adapter(struct rte_eth_dev *dev) 3149 { 3150 u32 reg_val; 3151 u16 i; 3152 struct rte_eth_dev_info dev_info; 3153 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3154 3155 memset(&dev_info, 0, sizeof(dev_info)); 3156 eth_igbvf_infos_get(dev, &dev_info); 3157 3158 /* Clear interrupt mask to stop from interrupts being generated */ 3159 igbvf_intr_disable(hw); 3160 3161 /* Clear any pending interrupts, flush previous writes */ 3162 E1000_READ_REG(hw, E1000_EICR); 3163 3164 /* Disable the transmit unit. Each queue must be disabled. */ 3165 for (i = 0; i < dev_info.max_tx_queues; i++) 3166 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3167 3168 /* Disable the receive unit by stopping each queue */ 3169 for (i = 0; i < dev_info.max_rx_queues; i++) { 3170 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3171 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3172 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3173 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3174 ; 3175 } 3176 3177 /* flush all queues disables */ 3178 E1000_WRITE_FLUSH(hw); 3179 msec_delay(2); 3180 } 3181 3182 static int eth_igbvf_link_update(struct e1000_hw *hw) 3183 { 3184 struct e1000_mbx_info *mbx = &hw->mbx; 3185 struct e1000_mac_info *mac = &hw->mac; 3186 int ret_val = E1000_SUCCESS; 3187 3188 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3189 3190 /* 3191 * We only want to run this if there has been a rst asserted. 3192 * in this case that could mean a link change, device reset, 3193 * or a virtual function reset 3194 */ 3195 3196 /* If we were hit with a reset or timeout drop the link */ 3197 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3198 mac->get_link_status = TRUE; 3199 3200 if (!mac->get_link_status) 3201 goto out; 3202 3203 /* if link status is down no point in checking to see if pf is up */ 3204 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3205 goto out; 3206 3207 /* if we passed all the tests above then the link is up and we no 3208 * longer need to check for link */ 3209 mac->get_link_status = FALSE; 3210 3211 out: 3212 return ret_val; 3213 } 3214 3215 3216 static int 3217 igbvf_dev_configure(struct rte_eth_dev *dev) 3218 { 3219 struct rte_eth_conf* conf = &dev->data->dev_conf; 3220 3221 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3222 dev->data->port_id); 3223 3224 /* 3225 * VF has no ability to enable/disable HW CRC 3226 * Keep the persistent behavior the same as Host PF 3227 */ 3228 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3229 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 3230 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3231 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 3232 } 3233 #else 3234 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 3235 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3236 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 3237 } 3238 #endif 3239 3240 return 0; 3241 } 3242 3243 static int 3244 igbvf_dev_start(struct rte_eth_dev *dev) 3245 { 3246 struct e1000_hw *hw = 3247 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3248 struct e1000_adapter *adapter = 3249 E1000_DEV_PRIVATE(dev->data->dev_private); 3250 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3251 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3252 int ret; 3253 uint32_t intr_vector = 0; 3254 3255 PMD_INIT_FUNC_TRACE(); 3256 3257 hw->mac.ops.reset_hw(hw); 3258 adapter->stopped = 0; 3259 3260 /* Set all vfta */ 3261 igbvf_set_vfta_all(dev,1); 3262 3263 eth_igbvf_tx_init(dev); 3264 3265 /* This can fail when allocating mbufs for descriptor rings */ 3266 ret = eth_igbvf_rx_init(dev); 3267 if (ret) { 3268 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3269 igb_dev_clear_queues(dev); 3270 return ret; 3271 } 3272 3273 /* check and configure queue intr-vector mapping */ 3274 if (rte_intr_cap_multiple(intr_handle) && 3275 dev->data->dev_conf.intr_conf.rxq) { 3276 intr_vector = dev->data->nb_rx_queues; 3277 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3278 if (ret) 3279 return ret; 3280 } 3281 3282 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3283 intr_handle->intr_vec = 3284 rte_zmalloc("intr_vec", 3285 dev->data->nb_rx_queues * sizeof(int), 0); 3286 if (!intr_handle->intr_vec) { 3287 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3288 " intr_vec", dev->data->nb_rx_queues); 3289 return -ENOMEM; 3290 } 3291 } 3292 3293 eth_igbvf_configure_msix_intr(dev); 3294 3295 /* enable uio/vfio intr/eventfd mapping */ 3296 rte_intr_enable(intr_handle); 3297 3298 /* resume enabled intr since hw reset */ 3299 igbvf_intr_enable(dev); 3300 3301 return 0; 3302 } 3303 3304 static void 3305 igbvf_dev_stop(struct rte_eth_dev *dev) 3306 { 3307 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3308 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3309 3310 PMD_INIT_FUNC_TRACE(); 3311 3312 igbvf_stop_adapter(dev); 3313 3314 /* 3315 * Clear what we set, but we still keep shadow_vfta to 3316 * restore after device starts 3317 */ 3318 igbvf_set_vfta_all(dev,0); 3319 3320 igb_dev_clear_queues(dev); 3321 3322 /* disable intr eventfd mapping */ 3323 rte_intr_disable(intr_handle); 3324 3325 /* Clean datapath event and queue/vec mapping */ 3326 rte_intr_efd_disable(intr_handle); 3327 if (intr_handle->intr_vec) { 3328 rte_free(intr_handle->intr_vec); 3329 intr_handle->intr_vec = NULL; 3330 } 3331 } 3332 3333 static void 3334 igbvf_dev_close(struct rte_eth_dev *dev) 3335 { 3336 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3337 struct e1000_adapter *adapter = 3338 E1000_DEV_PRIVATE(dev->data->dev_private); 3339 struct ether_addr addr; 3340 3341 PMD_INIT_FUNC_TRACE(); 3342 3343 e1000_reset_hw(hw); 3344 3345 igbvf_dev_stop(dev); 3346 adapter->stopped = 1; 3347 igb_dev_free_queues(dev); 3348 3349 /** 3350 * reprogram the RAR with a zero mac address, 3351 * to ensure that the VF traffic goes to the PF 3352 * after stop, close and detach of the VF. 3353 **/ 3354 3355 memset(&addr, 0, sizeof(addr)); 3356 igbvf_default_mac_addr_set(dev, &addr); 3357 } 3358 3359 static void 3360 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3361 { 3362 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3363 3364 /* Set both unicast and multicast promisc */ 3365 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3366 } 3367 3368 static void 3369 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3370 { 3371 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3372 3373 /* If in allmulticast mode leave multicast promisc */ 3374 if (dev->data->all_multicast == 1) 3375 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3376 else 3377 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3378 } 3379 3380 static void 3381 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3382 { 3383 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3384 3385 /* In promiscuous mode multicast promisc already set */ 3386 if (dev->data->promiscuous == 0) 3387 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3388 } 3389 3390 static void 3391 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3392 { 3393 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3394 3395 /* In promiscuous mode leave multicast promisc enabled */ 3396 if (dev->data->promiscuous == 0) 3397 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3398 } 3399 3400 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3401 { 3402 struct e1000_mbx_info *mbx = &hw->mbx; 3403 uint32_t msgbuf[2]; 3404 s32 err; 3405 3406 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3407 msgbuf[0] = E1000_VF_SET_VLAN; 3408 msgbuf[1] = vid; 3409 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3410 if (on) 3411 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3412 3413 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3414 if (err) 3415 goto mbx_err; 3416 3417 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3418 if (err) 3419 goto mbx_err; 3420 3421 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3422 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3423 err = -EINVAL; 3424 3425 mbx_err: 3426 return err; 3427 } 3428 3429 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3430 { 3431 struct e1000_hw *hw = 3432 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3433 struct e1000_vfta * shadow_vfta = 3434 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3435 int i = 0, j = 0, vfta = 0, mask = 1; 3436 3437 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3438 vfta = shadow_vfta->vfta[i]; 3439 if(vfta){ 3440 mask = 1; 3441 for (j = 0; j < 32; j++){ 3442 if(vfta & mask) 3443 igbvf_set_vfta(hw, 3444 (uint16_t)((i<<5)+j), on); 3445 mask<<=1; 3446 } 3447 } 3448 } 3449 3450 } 3451 3452 static int 3453 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3454 { 3455 struct e1000_hw *hw = 3456 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3457 struct e1000_vfta * shadow_vfta = 3458 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3459 uint32_t vid_idx = 0; 3460 uint32_t vid_bit = 0; 3461 int ret = 0; 3462 3463 PMD_INIT_FUNC_TRACE(); 3464 3465 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3466 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3467 if(ret){ 3468 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3469 return ret; 3470 } 3471 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3472 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3473 3474 /*Save what we set and retore it after device reset*/ 3475 if (on) 3476 shadow_vfta->vfta[vid_idx] |= vid_bit; 3477 else 3478 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3479 3480 return 0; 3481 } 3482 3483 static int 3484 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) 3485 { 3486 struct e1000_hw *hw = 3487 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3488 3489 /* index is not used by rar_set() */ 3490 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3491 return 0; 3492 } 3493 3494 3495 static int 3496 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3497 struct rte_eth_rss_reta_entry64 *reta_conf, 3498 uint16_t reta_size) 3499 { 3500 uint8_t i, j, mask; 3501 uint32_t reta, r; 3502 uint16_t idx, shift; 3503 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3504 3505 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3506 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3507 "(%d) doesn't match the number hardware can supported " 3508 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3509 return -EINVAL; 3510 } 3511 3512 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3513 idx = i / RTE_RETA_GROUP_SIZE; 3514 shift = i % RTE_RETA_GROUP_SIZE; 3515 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3516 IGB_4_BIT_MASK); 3517 if (!mask) 3518 continue; 3519 if (mask == IGB_4_BIT_MASK) 3520 r = 0; 3521 else 3522 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3523 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3524 if (mask & (0x1 << j)) 3525 reta |= reta_conf[idx].reta[shift + j] << 3526 (CHAR_BIT * j); 3527 else 3528 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3529 } 3530 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3531 } 3532 3533 return 0; 3534 } 3535 3536 static int 3537 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3538 struct rte_eth_rss_reta_entry64 *reta_conf, 3539 uint16_t reta_size) 3540 { 3541 uint8_t i, j, mask; 3542 uint32_t reta; 3543 uint16_t idx, shift; 3544 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3545 3546 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3547 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3548 "(%d) doesn't match the number hardware can supported " 3549 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3550 return -EINVAL; 3551 } 3552 3553 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3554 idx = i / RTE_RETA_GROUP_SIZE; 3555 shift = i % RTE_RETA_GROUP_SIZE; 3556 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3557 IGB_4_BIT_MASK); 3558 if (!mask) 3559 continue; 3560 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3561 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3562 if (mask & (0x1 << j)) 3563 reta_conf[idx].reta[shift + j] = 3564 ((reta >> (CHAR_BIT * j)) & 3565 IGB_8_BIT_MASK); 3566 } 3567 } 3568 3569 return 0; 3570 } 3571 3572 int 3573 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3574 struct rte_eth_syn_filter *filter, 3575 bool add) 3576 { 3577 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3578 struct e1000_filter_info *filter_info = 3579 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3580 uint32_t synqf, rfctl; 3581 3582 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3583 return -EINVAL; 3584 3585 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3586 3587 if (add) { 3588 if (synqf & E1000_SYN_FILTER_ENABLE) 3589 return -EINVAL; 3590 3591 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3592 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3593 3594 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3595 if (filter->hig_pri) 3596 rfctl |= E1000_RFCTL_SYNQFP; 3597 else 3598 rfctl &= ~E1000_RFCTL_SYNQFP; 3599 3600 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3601 } else { 3602 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3603 return -ENOENT; 3604 synqf = 0; 3605 } 3606 3607 filter_info->syn_info = synqf; 3608 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3609 E1000_WRITE_FLUSH(hw); 3610 return 0; 3611 } 3612 3613 static int 3614 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3615 struct rte_eth_syn_filter *filter) 3616 { 3617 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3618 uint32_t synqf, rfctl; 3619 3620 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3621 if (synqf & E1000_SYN_FILTER_ENABLE) { 3622 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3623 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3624 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3625 E1000_SYN_FILTER_QUEUE_SHIFT); 3626 return 0; 3627 } 3628 3629 return -ENOENT; 3630 } 3631 3632 static int 3633 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3634 enum rte_filter_op filter_op, 3635 void *arg) 3636 { 3637 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3638 int ret; 3639 3640 MAC_TYPE_FILTER_SUP(hw->mac.type); 3641 3642 if (filter_op == RTE_ETH_FILTER_NOP) 3643 return 0; 3644 3645 if (arg == NULL) { 3646 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3647 filter_op); 3648 return -EINVAL; 3649 } 3650 3651 switch (filter_op) { 3652 case RTE_ETH_FILTER_ADD: 3653 ret = eth_igb_syn_filter_set(dev, 3654 (struct rte_eth_syn_filter *)arg, 3655 TRUE); 3656 break; 3657 case RTE_ETH_FILTER_DELETE: 3658 ret = eth_igb_syn_filter_set(dev, 3659 (struct rte_eth_syn_filter *)arg, 3660 FALSE); 3661 break; 3662 case RTE_ETH_FILTER_GET: 3663 ret = eth_igb_syn_filter_get(dev, 3664 (struct rte_eth_syn_filter *)arg); 3665 break; 3666 default: 3667 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3668 ret = -EINVAL; 3669 break; 3670 } 3671 3672 return ret; 3673 } 3674 3675 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3676 static inline int 3677 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3678 struct e1000_2tuple_filter_info *filter_info) 3679 { 3680 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3681 return -EINVAL; 3682 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3683 return -EINVAL; /* filter index is out of range. */ 3684 if (filter->tcp_flags > TCP_FLAG_ALL) 3685 return -EINVAL; /* flags is invalid. */ 3686 3687 switch (filter->dst_port_mask) { 3688 case UINT16_MAX: 3689 filter_info->dst_port_mask = 0; 3690 filter_info->dst_port = filter->dst_port; 3691 break; 3692 case 0: 3693 filter_info->dst_port_mask = 1; 3694 break; 3695 default: 3696 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3697 return -EINVAL; 3698 } 3699 3700 switch (filter->proto_mask) { 3701 case UINT8_MAX: 3702 filter_info->proto_mask = 0; 3703 filter_info->proto = filter->proto; 3704 break; 3705 case 0: 3706 filter_info->proto_mask = 1; 3707 break; 3708 default: 3709 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3710 return -EINVAL; 3711 } 3712 3713 filter_info->priority = (uint8_t)filter->priority; 3714 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3715 filter_info->tcp_flags = filter->tcp_flags; 3716 else 3717 filter_info->tcp_flags = 0; 3718 3719 return 0; 3720 } 3721 3722 static inline struct e1000_2tuple_filter * 3723 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3724 struct e1000_2tuple_filter_info *key) 3725 { 3726 struct e1000_2tuple_filter *it; 3727 3728 TAILQ_FOREACH(it, filter_list, entries) { 3729 if (memcmp(key, &it->filter_info, 3730 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3731 return it; 3732 } 3733 } 3734 return NULL; 3735 } 3736 3737 /* inject a igb 2tuple filter to HW */ 3738 static inline void 3739 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3740 struct e1000_2tuple_filter *filter) 3741 { 3742 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3743 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3744 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3745 int i; 3746 3747 i = filter->index; 3748 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3749 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3750 imir |= E1000_IMIR_PORT_BP; 3751 else 3752 imir &= ~E1000_IMIR_PORT_BP; 3753 3754 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3755 3756 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3757 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3758 ttqf |= (uint32_t)(filter->filter_info.proto & 3759 E1000_TTQF_PROTOCOL_MASK); 3760 if (filter->filter_info.proto_mask == 0) 3761 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3762 3763 /* tcp flags bits setting. */ 3764 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 3765 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 3766 imir_ext |= E1000_IMIREXT_CTRL_URG; 3767 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 3768 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3769 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 3770 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3771 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 3772 imir_ext |= E1000_IMIREXT_CTRL_RST; 3773 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 3774 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3775 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 3776 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3777 } else { 3778 imir_ext |= E1000_IMIREXT_CTRL_BP; 3779 } 3780 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3781 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3782 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3783 } 3784 3785 /* 3786 * igb_add_2tuple_filter - add a 2tuple filter 3787 * 3788 * @param 3789 * dev: Pointer to struct rte_eth_dev. 3790 * ntuple_filter: ponter to the filter that will be added. 3791 * 3792 * @return 3793 * - On success, zero. 3794 * - On failure, a negative value. 3795 */ 3796 static int 3797 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3798 struct rte_eth_ntuple_filter *ntuple_filter) 3799 { 3800 struct e1000_filter_info *filter_info = 3801 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3802 struct e1000_2tuple_filter *filter; 3803 int i, ret; 3804 3805 filter = rte_zmalloc("e1000_2tuple_filter", 3806 sizeof(struct e1000_2tuple_filter), 0); 3807 if (filter == NULL) 3808 return -ENOMEM; 3809 3810 ret = ntuple_filter_to_2tuple(ntuple_filter, 3811 &filter->filter_info); 3812 if (ret < 0) { 3813 rte_free(filter); 3814 return ret; 3815 } 3816 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3817 &filter->filter_info) != NULL) { 3818 PMD_DRV_LOG(ERR, "filter exists."); 3819 rte_free(filter); 3820 return -EEXIST; 3821 } 3822 filter->queue = ntuple_filter->queue; 3823 3824 /* 3825 * look for an unused 2tuple filter index, 3826 * and insert the filter to list. 3827 */ 3828 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3829 if (!(filter_info->twotuple_mask & (1 << i))) { 3830 filter_info->twotuple_mask |= 1 << i; 3831 filter->index = i; 3832 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3833 filter, 3834 entries); 3835 break; 3836 } 3837 } 3838 if (i >= E1000_MAX_TTQF_FILTERS) { 3839 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3840 rte_free(filter); 3841 return -ENOSYS; 3842 } 3843 3844 igb_inject_2uple_filter(dev, filter); 3845 return 0; 3846 } 3847 3848 int 3849 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3850 struct e1000_2tuple_filter *filter) 3851 { 3852 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3853 struct e1000_filter_info *filter_info = 3854 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3855 3856 filter_info->twotuple_mask &= ~(1 << filter->index); 3857 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3858 rte_free(filter); 3859 3860 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3861 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3862 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3863 return 0; 3864 } 3865 3866 /* 3867 * igb_remove_2tuple_filter - remove a 2tuple filter 3868 * 3869 * @param 3870 * dev: Pointer to struct rte_eth_dev. 3871 * ntuple_filter: ponter to the filter that will be removed. 3872 * 3873 * @return 3874 * - On success, zero. 3875 * - On failure, a negative value. 3876 */ 3877 static int 3878 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3879 struct rte_eth_ntuple_filter *ntuple_filter) 3880 { 3881 struct e1000_filter_info *filter_info = 3882 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3883 struct e1000_2tuple_filter_info filter_2tuple; 3884 struct e1000_2tuple_filter *filter; 3885 int ret; 3886 3887 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3888 ret = ntuple_filter_to_2tuple(ntuple_filter, 3889 &filter_2tuple); 3890 if (ret < 0) 3891 return ret; 3892 3893 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3894 &filter_2tuple); 3895 if (filter == NULL) { 3896 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3897 return -ENOENT; 3898 } 3899 3900 igb_delete_2tuple_filter(dev, filter); 3901 3902 return 0; 3903 } 3904 3905 /* inject a igb flex filter to HW */ 3906 static inline void 3907 igb_inject_flex_filter(struct rte_eth_dev *dev, 3908 struct e1000_flex_filter *filter) 3909 { 3910 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3911 uint32_t wufc, queueing; 3912 uint32_t reg_off; 3913 uint8_t i, j = 0; 3914 3915 wufc = E1000_READ_REG(hw, E1000_WUFC); 3916 if (filter->index < E1000_MAX_FHFT) 3917 reg_off = E1000_FHFT(filter->index); 3918 else 3919 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3920 3921 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3922 (E1000_WUFC_FLX0 << filter->index)); 3923 queueing = filter->filter_info.len | 3924 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3925 (filter->filter_info.priority << 3926 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3927 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3928 queueing); 3929 3930 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3931 E1000_WRITE_REG(hw, reg_off, 3932 filter->filter_info.dwords[j]); 3933 reg_off += sizeof(uint32_t); 3934 E1000_WRITE_REG(hw, reg_off, 3935 filter->filter_info.dwords[++j]); 3936 reg_off += sizeof(uint32_t); 3937 E1000_WRITE_REG(hw, reg_off, 3938 (uint32_t)filter->filter_info.mask[i]); 3939 reg_off += sizeof(uint32_t) * 2; 3940 ++j; 3941 } 3942 } 3943 3944 static inline struct e1000_flex_filter * 3945 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3946 struct e1000_flex_filter_info *key) 3947 { 3948 struct e1000_flex_filter *it; 3949 3950 TAILQ_FOREACH(it, filter_list, entries) { 3951 if (memcmp(key, &it->filter_info, 3952 sizeof(struct e1000_flex_filter_info)) == 0) 3953 return it; 3954 } 3955 3956 return NULL; 3957 } 3958 3959 /* remove a flex byte filter 3960 * @param 3961 * dev: Pointer to struct rte_eth_dev. 3962 * filter: the pointer of the filter will be removed. 3963 */ 3964 void 3965 igb_remove_flex_filter(struct rte_eth_dev *dev, 3966 struct e1000_flex_filter *filter) 3967 { 3968 struct e1000_filter_info *filter_info = 3969 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3970 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3971 uint32_t wufc, i; 3972 uint32_t reg_off; 3973 3974 wufc = E1000_READ_REG(hw, E1000_WUFC); 3975 if (filter->index < E1000_MAX_FHFT) 3976 reg_off = E1000_FHFT(filter->index); 3977 else 3978 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3979 3980 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 3981 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 3982 3983 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 3984 (~(E1000_WUFC_FLX0 << filter->index))); 3985 3986 filter_info->flex_mask &= ~(1 << filter->index); 3987 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 3988 rte_free(filter); 3989 } 3990 3991 int 3992 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 3993 struct rte_eth_flex_filter *filter, 3994 bool add) 3995 { 3996 struct e1000_filter_info *filter_info = 3997 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3998 struct e1000_flex_filter *flex_filter, *it; 3999 uint32_t mask; 4000 uint8_t shift, i; 4001 4002 flex_filter = rte_zmalloc("e1000_flex_filter", 4003 sizeof(struct e1000_flex_filter), 0); 4004 if (flex_filter == NULL) 4005 return -ENOMEM; 4006 4007 flex_filter->filter_info.len = filter->len; 4008 flex_filter->filter_info.priority = filter->priority; 4009 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4010 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4011 mask = 0; 4012 /* reverse bits in flex filter's mask*/ 4013 for (shift = 0; shift < CHAR_BIT; shift++) { 4014 if (filter->mask[i] & (0x01 << shift)) 4015 mask |= (0x80 >> shift); 4016 } 4017 flex_filter->filter_info.mask[i] = mask; 4018 } 4019 4020 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4021 &flex_filter->filter_info); 4022 if (it == NULL && !add) { 4023 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4024 rte_free(flex_filter); 4025 return -ENOENT; 4026 } 4027 if (it != NULL && add) { 4028 PMD_DRV_LOG(ERR, "filter exists."); 4029 rte_free(flex_filter); 4030 return -EEXIST; 4031 } 4032 4033 if (add) { 4034 flex_filter->queue = filter->queue; 4035 /* 4036 * look for an unused flex filter index 4037 * and insert the filter into the list. 4038 */ 4039 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4040 if (!(filter_info->flex_mask & (1 << i))) { 4041 filter_info->flex_mask |= 1 << i; 4042 flex_filter->index = i; 4043 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4044 flex_filter, 4045 entries); 4046 break; 4047 } 4048 } 4049 if (i >= E1000_MAX_FLEX_FILTERS) { 4050 PMD_DRV_LOG(ERR, "flex filters are full."); 4051 rte_free(flex_filter); 4052 return -ENOSYS; 4053 } 4054 4055 igb_inject_flex_filter(dev, flex_filter); 4056 4057 } else { 4058 igb_remove_flex_filter(dev, it); 4059 rte_free(flex_filter); 4060 } 4061 4062 return 0; 4063 } 4064 4065 static int 4066 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4067 struct rte_eth_flex_filter *filter) 4068 { 4069 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4070 struct e1000_filter_info *filter_info = 4071 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4072 struct e1000_flex_filter flex_filter, *it; 4073 uint32_t wufc, queueing, wufc_en = 0; 4074 4075 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4076 flex_filter.filter_info.len = filter->len; 4077 flex_filter.filter_info.priority = filter->priority; 4078 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4079 memcpy(flex_filter.filter_info.mask, filter->mask, 4080 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4081 4082 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4083 &flex_filter.filter_info); 4084 if (it == NULL) { 4085 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4086 return -ENOENT; 4087 } 4088 4089 wufc = E1000_READ_REG(hw, E1000_WUFC); 4090 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4091 4092 if ((wufc & wufc_en) == wufc_en) { 4093 uint32_t reg_off = 0; 4094 if (it->index < E1000_MAX_FHFT) 4095 reg_off = E1000_FHFT(it->index); 4096 else 4097 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4098 4099 queueing = E1000_READ_REG(hw, 4100 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4101 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4102 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4103 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4104 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4105 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4106 return 0; 4107 } 4108 return -ENOENT; 4109 } 4110 4111 static int 4112 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4113 enum rte_filter_op filter_op, 4114 void *arg) 4115 { 4116 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4117 struct rte_eth_flex_filter *filter; 4118 int ret = 0; 4119 4120 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4121 4122 if (filter_op == RTE_ETH_FILTER_NOP) 4123 return ret; 4124 4125 if (arg == NULL) { 4126 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4127 filter_op); 4128 return -EINVAL; 4129 } 4130 4131 filter = (struct rte_eth_flex_filter *)arg; 4132 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4133 || filter->len % sizeof(uint64_t) != 0) { 4134 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4135 return -EINVAL; 4136 } 4137 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4138 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4139 return -EINVAL; 4140 } 4141 4142 switch (filter_op) { 4143 case RTE_ETH_FILTER_ADD: 4144 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4145 break; 4146 case RTE_ETH_FILTER_DELETE: 4147 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4148 break; 4149 case RTE_ETH_FILTER_GET: 4150 ret = eth_igb_get_flex_filter(dev, filter); 4151 break; 4152 default: 4153 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4154 ret = -EINVAL; 4155 break; 4156 } 4157 4158 return ret; 4159 } 4160 4161 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4162 static inline int 4163 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4164 struct e1000_5tuple_filter_info *filter_info) 4165 { 4166 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4167 return -EINVAL; 4168 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4169 return -EINVAL; /* filter index is out of range. */ 4170 if (filter->tcp_flags > TCP_FLAG_ALL) 4171 return -EINVAL; /* flags is invalid. */ 4172 4173 switch (filter->dst_ip_mask) { 4174 case UINT32_MAX: 4175 filter_info->dst_ip_mask = 0; 4176 filter_info->dst_ip = filter->dst_ip; 4177 break; 4178 case 0: 4179 filter_info->dst_ip_mask = 1; 4180 break; 4181 default: 4182 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4183 return -EINVAL; 4184 } 4185 4186 switch (filter->src_ip_mask) { 4187 case UINT32_MAX: 4188 filter_info->src_ip_mask = 0; 4189 filter_info->src_ip = filter->src_ip; 4190 break; 4191 case 0: 4192 filter_info->src_ip_mask = 1; 4193 break; 4194 default: 4195 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4196 return -EINVAL; 4197 } 4198 4199 switch (filter->dst_port_mask) { 4200 case UINT16_MAX: 4201 filter_info->dst_port_mask = 0; 4202 filter_info->dst_port = filter->dst_port; 4203 break; 4204 case 0: 4205 filter_info->dst_port_mask = 1; 4206 break; 4207 default: 4208 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4209 return -EINVAL; 4210 } 4211 4212 switch (filter->src_port_mask) { 4213 case UINT16_MAX: 4214 filter_info->src_port_mask = 0; 4215 filter_info->src_port = filter->src_port; 4216 break; 4217 case 0: 4218 filter_info->src_port_mask = 1; 4219 break; 4220 default: 4221 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4222 return -EINVAL; 4223 } 4224 4225 switch (filter->proto_mask) { 4226 case UINT8_MAX: 4227 filter_info->proto_mask = 0; 4228 filter_info->proto = filter->proto; 4229 break; 4230 case 0: 4231 filter_info->proto_mask = 1; 4232 break; 4233 default: 4234 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4235 return -EINVAL; 4236 } 4237 4238 filter_info->priority = (uint8_t)filter->priority; 4239 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4240 filter_info->tcp_flags = filter->tcp_flags; 4241 else 4242 filter_info->tcp_flags = 0; 4243 4244 return 0; 4245 } 4246 4247 static inline struct e1000_5tuple_filter * 4248 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4249 struct e1000_5tuple_filter_info *key) 4250 { 4251 struct e1000_5tuple_filter *it; 4252 4253 TAILQ_FOREACH(it, filter_list, entries) { 4254 if (memcmp(key, &it->filter_info, 4255 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4256 return it; 4257 } 4258 } 4259 return NULL; 4260 } 4261 4262 /* inject a igb 5-tuple filter to HW */ 4263 static inline void 4264 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4265 struct e1000_5tuple_filter *filter) 4266 { 4267 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4268 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4269 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4270 uint8_t i; 4271 4272 i = filter->index; 4273 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4274 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4275 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4276 if (filter->filter_info.dst_ip_mask == 0) 4277 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4278 if (filter->filter_info.src_port_mask == 0) 4279 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4280 if (filter->filter_info.proto_mask == 0) 4281 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4282 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4283 E1000_FTQF_QUEUE_MASK; 4284 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4285 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4286 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4287 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4288 4289 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4290 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4291 4292 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4293 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4294 imir |= E1000_IMIR_PORT_BP; 4295 else 4296 imir &= ~E1000_IMIR_PORT_BP; 4297 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4298 4299 /* tcp flags bits setting. */ 4300 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 4301 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 4302 imir_ext |= E1000_IMIREXT_CTRL_URG; 4303 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 4304 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4305 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 4306 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4307 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 4308 imir_ext |= E1000_IMIREXT_CTRL_RST; 4309 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 4310 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4311 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 4312 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4313 } else { 4314 imir_ext |= E1000_IMIREXT_CTRL_BP; 4315 } 4316 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4317 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4318 } 4319 4320 /* 4321 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4322 * 4323 * @param 4324 * dev: Pointer to struct rte_eth_dev. 4325 * ntuple_filter: ponter to the filter that will be added. 4326 * 4327 * @return 4328 * - On success, zero. 4329 * - On failure, a negative value. 4330 */ 4331 static int 4332 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4333 struct rte_eth_ntuple_filter *ntuple_filter) 4334 { 4335 struct e1000_filter_info *filter_info = 4336 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4337 struct e1000_5tuple_filter *filter; 4338 uint8_t i; 4339 int ret; 4340 4341 filter = rte_zmalloc("e1000_5tuple_filter", 4342 sizeof(struct e1000_5tuple_filter), 0); 4343 if (filter == NULL) 4344 return -ENOMEM; 4345 4346 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4347 &filter->filter_info); 4348 if (ret < 0) { 4349 rte_free(filter); 4350 return ret; 4351 } 4352 4353 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4354 &filter->filter_info) != NULL) { 4355 PMD_DRV_LOG(ERR, "filter exists."); 4356 rte_free(filter); 4357 return -EEXIST; 4358 } 4359 filter->queue = ntuple_filter->queue; 4360 4361 /* 4362 * look for an unused 5tuple filter index, 4363 * and insert the filter to list. 4364 */ 4365 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4366 if (!(filter_info->fivetuple_mask & (1 << i))) { 4367 filter_info->fivetuple_mask |= 1 << i; 4368 filter->index = i; 4369 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4370 filter, 4371 entries); 4372 break; 4373 } 4374 } 4375 if (i >= E1000_MAX_FTQF_FILTERS) { 4376 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4377 rte_free(filter); 4378 return -ENOSYS; 4379 } 4380 4381 igb_inject_5tuple_filter_82576(dev, filter); 4382 return 0; 4383 } 4384 4385 int 4386 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4387 struct e1000_5tuple_filter *filter) 4388 { 4389 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4390 struct e1000_filter_info *filter_info = 4391 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4392 4393 filter_info->fivetuple_mask &= ~(1 << filter->index); 4394 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4395 rte_free(filter); 4396 4397 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4398 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4399 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4400 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4401 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4402 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4403 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4404 return 0; 4405 } 4406 4407 /* 4408 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4409 * 4410 * @param 4411 * dev: Pointer to struct rte_eth_dev. 4412 * ntuple_filter: ponter to the filter that will be removed. 4413 * 4414 * @return 4415 * - On success, zero. 4416 * - On failure, a negative value. 4417 */ 4418 static int 4419 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4420 struct rte_eth_ntuple_filter *ntuple_filter) 4421 { 4422 struct e1000_filter_info *filter_info = 4423 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4424 struct e1000_5tuple_filter_info filter_5tuple; 4425 struct e1000_5tuple_filter *filter; 4426 int ret; 4427 4428 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4429 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4430 &filter_5tuple); 4431 if (ret < 0) 4432 return ret; 4433 4434 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4435 &filter_5tuple); 4436 if (filter == NULL) { 4437 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4438 return -ENOENT; 4439 } 4440 4441 igb_delete_5tuple_filter_82576(dev, filter); 4442 4443 return 0; 4444 } 4445 4446 static int 4447 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4448 { 4449 uint32_t rctl; 4450 struct e1000_hw *hw; 4451 struct rte_eth_dev_info dev_info; 4452 uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4453 VLAN_TAG_SIZE); 4454 4455 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4456 4457 #ifdef RTE_LIBRTE_82571_SUPPORT 4458 /* XXX: not bigger than max_rx_pktlen */ 4459 if (hw->mac.type == e1000_82571) 4460 return -ENOTSUP; 4461 #endif 4462 eth_igb_infos_get(dev, &dev_info); 4463 4464 /* check that mtu is within the allowed range */ 4465 if ((mtu < ETHER_MIN_MTU) || 4466 (frame_size > dev_info.max_rx_pktlen)) 4467 return -EINVAL; 4468 4469 /* refuse mtu that requires the support of scattered packets when this 4470 * feature has not been enabled before. */ 4471 if (!dev->data->scattered_rx && 4472 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4473 return -EINVAL; 4474 4475 rctl = E1000_READ_REG(hw, E1000_RCTL); 4476 4477 /* switch to jumbo mode if needed */ 4478 if (frame_size > ETHER_MAX_LEN) { 4479 dev->data->dev_conf.rxmode.offloads |= 4480 DEV_RX_OFFLOAD_JUMBO_FRAME; 4481 rctl |= E1000_RCTL_LPE; 4482 } else { 4483 dev->data->dev_conf.rxmode.offloads &= 4484 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4485 rctl &= ~E1000_RCTL_LPE; 4486 } 4487 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4488 4489 /* update max frame size */ 4490 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4491 4492 E1000_WRITE_REG(hw, E1000_RLPML, 4493 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4494 4495 return 0; 4496 } 4497 4498 /* 4499 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4500 * 4501 * @param 4502 * dev: Pointer to struct rte_eth_dev. 4503 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4504 * add: if true, add filter, if false, remove filter 4505 * 4506 * @return 4507 * - On success, zero. 4508 * - On failure, a negative value. 4509 */ 4510 int 4511 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4512 struct rte_eth_ntuple_filter *ntuple_filter, 4513 bool add) 4514 { 4515 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4516 int ret; 4517 4518 switch (ntuple_filter->flags) { 4519 case RTE_5TUPLE_FLAGS: 4520 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4521 if (hw->mac.type != e1000_82576) 4522 return -ENOTSUP; 4523 if (add) 4524 ret = igb_add_5tuple_filter_82576(dev, 4525 ntuple_filter); 4526 else 4527 ret = igb_remove_5tuple_filter_82576(dev, 4528 ntuple_filter); 4529 break; 4530 case RTE_2TUPLE_FLAGS: 4531 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4532 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4533 hw->mac.type != e1000_i210 && 4534 hw->mac.type != e1000_i211) 4535 return -ENOTSUP; 4536 if (add) 4537 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4538 else 4539 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4540 break; 4541 default: 4542 ret = -EINVAL; 4543 break; 4544 } 4545 4546 return ret; 4547 } 4548 4549 /* 4550 * igb_get_ntuple_filter - get a ntuple filter 4551 * 4552 * @param 4553 * dev: Pointer to struct rte_eth_dev. 4554 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4555 * 4556 * @return 4557 * - On success, zero. 4558 * - On failure, a negative value. 4559 */ 4560 static int 4561 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4562 struct rte_eth_ntuple_filter *ntuple_filter) 4563 { 4564 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4565 struct e1000_filter_info *filter_info = 4566 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4567 struct e1000_5tuple_filter_info filter_5tuple; 4568 struct e1000_2tuple_filter_info filter_2tuple; 4569 struct e1000_5tuple_filter *p_5tuple_filter; 4570 struct e1000_2tuple_filter *p_2tuple_filter; 4571 int ret; 4572 4573 switch (ntuple_filter->flags) { 4574 case RTE_5TUPLE_FLAGS: 4575 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4576 if (hw->mac.type != e1000_82576) 4577 return -ENOTSUP; 4578 memset(&filter_5tuple, 4579 0, 4580 sizeof(struct e1000_5tuple_filter_info)); 4581 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4582 &filter_5tuple); 4583 if (ret < 0) 4584 return ret; 4585 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4586 &filter_info->fivetuple_list, 4587 &filter_5tuple); 4588 if (p_5tuple_filter == NULL) { 4589 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4590 return -ENOENT; 4591 } 4592 ntuple_filter->queue = p_5tuple_filter->queue; 4593 break; 4594 case RTE_2TUPLE_FLAGS: 4595 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4596 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4597 return -ENOTSUP; 4598 memset(&filter_2tuple, 4599 0, 4600 sizeof(struct e1000_2tuple_filter_info)); 4601 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4602 if (ret < 0) 4603 return ret; 4604 p_2tuple_filter = igb_2tuple_filter_lookup( 4605 &filter_info->twotuple_list, 4606 &filter_2tuple); 4607 if (p_2tuple_filter == NULL) { 4608 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4609 return -ENOENT; 4610 } 4611 ntuple_filter->queue = p_2tuple_filter->queue; 4612 break; 4613 default: 4614 ret = -EINVAL; 4615 break; 4616 } 4617 4618 return 0; 4619 } 4620 4621 /* 4622 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4623 * @dev: pointer to rte_eth_dev structure 4624 * @filter_op:operation will be taken. 4625 * @arg: a pointer to specific structure corresponding to the filter_op 4626 */ 4627 static int 4628 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4629 enum rte_filter_op filter_op, 4630 void *arg) 4631 { 4632 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4633 int ret; 4634 4635 MAC_TYPE_FILTER_SUP(hw->mac.type); 4636 4637 if (filter_op == RTE_ETH_FILTER_NOP) 4638 return 0; 4639 4640 if (arg == NULL) { 4641 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4642 filter_op); 4643 return -EINVAL; 4644 } 4645 4646 switch (filter_op) { 4647 case RTE_ETH_FILTER_ADD: 4648 ret = igb_add_del_ntuple_filter(dev, 4649 (struct rte_eth_ntuple_filter *)arg, 4650 TRUE); 4651 break; 4652 case RTE_ETH_FILTER_DELETE: 4653 ret = igb_add_del_ntuple_filter(dev, 4654 (struct rte_eth_ntuple_filter *)arg, 4655 FALSE); 4656 break; 4657 case RTE_ETH_FILTER_GET: 4658 ret = igb_get_ntuple_filter(dev, 4659 (struct rte_eth_ntuple_filter *)arg); 4660 break; 4661 default: 4662 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4663 ret = -EINVAL; 4664 break; 4665 } 4666 return ret; 4667 } 4668 4669 static inline int 4670 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4671 uint16_t ethertype) 4672 { 4673 int i; 4674 4675 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4676 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4677 (filter_info->ethertype_mask & (1 << i))) 4678 return i; 4679 } 4680 return -1; 4681 } 4682 4683 static inline int 4684 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4685 uint16_t ethertype, uint32_t etqf) 4686 { 4687 int i; 4688 4689 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4690 if (!(filter_info->ethertype_mask & (1 << i))) { 4691 filter_info->ethertype_mask |= 1 << i; 4692 filter_info->ethertype_filters[i].ethertype = ethertype; 4693 filter_info->ethertype_filters[i].etqf = etqf; 4694 return i; 4695 } 4696 } 4697 return -1; 4698 } 4699 4700 int 4701 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4702 uint8_t idx) 4703 { 4704 if (idx >= E1000_MAX_ETQF_FILTERS) 4705 return -1; 4706 filter_info->ethertype_mask &= ~(1 << idx); 4707 filter_info->ethertype_filters[idx].ethertype = 0; 4708 filter_info->ethertype_filters[idx].etqf = 0; 4709 return idx; 4710 } 4711 4712 4713 int 4714 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4715 struct rte_eth_ethertype_filter *filter, 4716 bool add) 4717 { 4718 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4719 struct e1000_filter_info *filter_info = 4720 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4721 uint32_t etqf = 0; 4722 int ret; 4723 4724 if (filter->ether_type == ETHER_TYPE_IPv4 || 4725 filter->ether_type == ETHER_TYPE_IPv6) { 4726 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4727 " ethertype filter.", filter->ether_type); 4728 return -EINVAL; 4729 } 4730 4731 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4732 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4733 return -EINVAL; 4734 } 4735 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4736 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4737 return -EINVAL; 4738 } 4739 4740 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4741 if (ret >= 0 && add) { 4742 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4743 filter->ether_type); 4744 return -EEXIST; 4745 } 4746 if (ret < 0 && !add) { 4747 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4748 filter->ether_type); 4749 return -ENOENT; 4750 } 4751 4752 if (add) { 4753 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4754 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4755 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4756 ret = igb_ethertype_filter_insert(filter_info, 4757 filter->ether_type, etqf); 4758 if (ret < 0) { 4759 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4760 return -ENOSYS; 4761 } 4762 } else { 4763 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4764 if (ret < 0) 4765 return -ENOSYS; 4766 } 4767 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4768 E1000_WRITE_FLUSH(hw); 4769 4770 return 0; 4771 } 4772 4773 static int 4774 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4775 struct rte_eth_ethertype_filter *filter) 4776 { 4777 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4778 struct e1000_filter_info *filter_info = 4779 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4780 uint32_t etqf; 4781 int ret; 4782 4783 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4784 if (ret < 0) { 4785 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4786 filter->ether_type); 4787 return -ENOENT; 4788 } 4789 4790 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4791 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4792 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4793 filter->flags = 0; 4794 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4795 E1000_ETQF_QUEUE_SHIFT; 4796 return 0; 4797 } 4798 4799 return -ENOENT; 4800 } 4801 4802 /* 4803 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4804 * @dev: pointer to rte_eth_dev structure 4805 * @filter_op:operation will be taken. 4806 * @arg: a pointer to specific structure corresponding to the filter_op 4807 */ 4808 static int 4809 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4810 enum rte_filter_op filter_op, 4811 void *arg) 4812 { 4813 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4814 int ret; 4815 4816 MAC_TYPE_FILTER_SUP(hw->mac.type); 4817 4818 if (filter_op == RTE_ETH_FILTER_NOP) 4819 return 0; 4820 4821 if (arg == NULL) { 4822 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4823 filter_op); 4824 return -EINVAL; 4825 } 4826 4827 switch (filter_op) { 4828 case RTE_ETH_FILTER_ADD: 4829 ret = igb_add_del_ethertype_filter(dev, 4830 (struct rte_eth_ethertype_filter *)arg, 4831 TRUE); 4832 break; 4833 case RTE_ETH_FILTER_DELETE: 4834 ret = igb_add_del_ethertype_filter(dev, 4835 (struct rte_eth_ethertype_filter *)arg, 4836 FALSE); 4837 break; 4838 case RTE_ETH_FILTER_GET: 4839 ret = igb_get_ethertype_filter(dev, 4840 (struct rte_eth_ethertype_filter *)arg); 4841 break; 4842 default: 4843 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4844 ret = -EINVAL; 4845 break; 4846 } 4847 return ret; 4848 } 4849 4850 static int 4851 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4852 enum rte_filter_type filter_type, 4853 enum rte_filter_op filter_op, 4854 void *arg) 4855 { 4856 int ret = 0; 4857 4858 switch (filter_type) { 4859 case RTE_ETH_FILTER_NTUPLE: 4860 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4861 break; 4862 case RTE_ETH_FILTER_ETHERTYPE: 4863 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4864 break; 4865 case RTE_ETH_FILTER_SYN: 4866 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4867 break; 4868 case RTE_ETH_FILTER_FLEXIBLE: 4869 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4870 break; 4871 case RTE_ETH_FILTER_GENERIC: 4872 if (filter_op != RTE_ETH_FILTER_GET) 4873 return -EINVAL; 4874 *(const void **)arg = &igb_flow_ops; 4875 break; 4876 default: 4877 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4878 filter_type); 4879 break; 4880 } 4881 4882 return ret; 4883 } 4884 4885 static int 4886 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4887 struct ether_addr *mc_addr_set, 4888 uint32_t nb_mc_addr) 4889 { 4890 struct e1000_hw *hw; 4891 4892 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4893 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4894 return 0; 4895 } 4896 4897 static uint64_t 4898 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4899 { 4900 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4901 uint64_t systime_cycles; 4902 4903 switch (hw->mac.type) { 4904 case e1000_i210: 4905 case e1000_i211: 4906 /* 4907 * Need to read System Time Residue Register to be able 4908 * to read the other two registers. 4909 */ 4910 E1000_READ_REG(hw, E1000_SYSTIMR); 4911 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4912 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4913 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4914 * NSEC_PER_SEC; 4915 break; 4916 case e1000_82580: 4917 case e1000_i350: 4918 case e1000_i354: 4919 /* 4920 * Need to read System Time Residue Register to be able 4921 * to read the other two registers. 4922 */ 4923 E1000_READ_REG(hw, E1000_SYSTIMR); 4924 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4925 /* Only the 8 LSB are valid. */ 4926 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4927 & 0xff) << 32; 4928 break; 4929 default: 4930 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4931 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4932 << 32; 4933 break; 4934 } 4935 4936 return systime_cycles; 4937 } 4938 4939 static uint64_t 4940 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4941 { 4942 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4943 uint64_t rx_tstamp_cycles; 4944 4945 switch (hw->mac.type) { 4946 case e1000_i210: 4947 case e1000_i211: 4948 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4949 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4950 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4951 * NSEC_PER_SEC; 4952 break; 4953 case e1000_82580: 4954 case e1000_i350: 4955 case e1000_i354: 4956 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4957 /* Only the 8 LSB are valid. */ 4958 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4959 & 0xff) << 32; 4960 break; 4961 default: 4962 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4963 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4964 << 32; 4965 break; 4966 } 4967 4968 return rx_tstamp_cycles; 4969 } 4970 4971 static uint64_t 4972 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4973 { 4974 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4975 uint64_t tx_tstamp_cycles; 4976 4977 switch (hw->mac.type) { 4978 case e1000_i210: 4979 case e1000_i211: 4980 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4981 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4982 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4983 * NSEC_PER_SEC; 4984 break; 4985 case e1000_82580: 4986 case e1000_i350: 4987 case e1000_i354: 4988 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4989 /* Only the 8 LSB are valid. */ 4990 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 4991 & 0xff) << 32; 4992 break; 4993 default: 4994 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4995 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4996 << 32; 4997 break; 4998 } 4999 5000 return tx_tstamp_cycles; 5001 } 5002 5003 static void 5004 igb_start_timecounters(struct rte_eth_dev *dev) 5005 { 5006 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5007 struct e1000_adapter *adapter = 5008 (struct e1000_adapter *)dev->data->dev_private; 5009 uint32_t incval = 1; 5010 uint32_t shift = 0; 5011 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5012 5013 switch (hw->mac.type) { 5014 case e1000_82580: 5015 case e1000_i350: 5016 case e1000_i354: 5017 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5018 mask = (1ULL << 40) - 1; 5019 /* fall-through */ 5020 case e1000_i210: 5021 case e1000_i211: 5022 /* 5023 * Start incrementing the register 5024 * used to timestamp PTP packets. 5025 */ 5026 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5027 break; 5028 case e1000_82576: 5029 incval = E1000_INCVALUE_82576; 5030 shift = IGB_82576_TSYNC_SHIFT; 5031 E1000_WRITE_REG(hw, E1000_TIMINCA, 5032 E1000_INCPERIOD_82576 | incval); 5033 break; 5034 default: 5035 /* Not supported */ 5036 return; 5037 } 5038 5039 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5040 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5041 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5042 5043 adapter->systime_tc.cc_mask = mask; 5044 adapter->systime_tc.cc_shift = shift; 5045 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5046 5047 adapter->rx_tstamp_tc.cc_mask = mask; 5048 adapter->rx_tstamp_tc.cc_shift = shift; 5049 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5050 5051 adapter->tx_tstamp_tc.cc_mask = mask; 5052 adapter->tx_tstamp_tc.cc_shift = shift; 5053 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5054 } 5055 5056 static int 5057 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5058 { 5059 struct e1000_adapter *adapter = 5060 (struct e1000_adapter *)dev->data->dev_private; 5061 5062 adapter->systime_tc.nsec += delta; 5063 adapter->rx_tstamp_tc.nsec += delta; 5064 adapter->tx_tstamp_tc.nsec += delta; 5065 5066 return 0; 5067 } 5068 5069 static int 5070 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5071 { 5072 uint64_t ns; 5073 struct e1000_adapter *adapter = 5074 (struct e1000_adapter *)dev->data->dev_private; 5075 5076 ns = rte_timespec_to_ns(ts); 5077 5078 /* Set the timecounters to a new value. */ 5079 adapter->systime_tc.nsec = ns; 5080 adapter->rx_tstamp_tc.nsec = ns; 5081 adapter->tx_tstamp_tc.nsec = ns; 5082 5083 return 0; 5084 } 5085 5086 static int 5087 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5088 { 5089 uint64_t ns, systime_cycles; 5090 struct e1000_adapter *adapter = 5091 (struct e1000_adapter *)dev->data->dev_private; 5092 5093 systime_cycles = igb_read_systime_cyclecounter(dev); 5094 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5095 *ts = rte_ns_to_timespec(ns); 5096 5097 return 0; 5098 } 5099 5100 static int 5101 igb_timesync_enable(struct rte_eth_dev *dev) 5102 { 5103 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5104 uint32_t tsync_ctl; 5105 uint32_t tsauxc; 5106 5107 /* Stop the timesync system time. */ 5108 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5109 /* Reset the timesync system time value. */ 5110 switch (hw->mac.type) { 5111 case e1000_82580: 5112 case e1000_i350: 5113 case e1000_i354: 5114 case e1000_i210: 5115 case e1000_i211: 5116 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5117 /* fall-through */ 5118 case e1000_82576: 5119 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5120 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5121 break; 5122 default: 5123 /* Not supported. */ 5124 return -ENOTSUP; 5125 } 5126 5127 /* Enable system time for it isn't on by default. */ 5128 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5129 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5130 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5131 5132 igb_start_timecounters(dev); 5133 5134 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5135 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5136 (ETHER_TYPE_1588 | 5137 E1000_ETQF_FILTER_ENABLE | 5138 E1000_ETQF_1588)); 5139 5140 /* Enable timestamping of received PTP packets. */ 5141 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5142 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5143 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5144 5145 /* Enable Timestamping of transmitted PTP packets. */ 5146 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5147 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5148 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5149 5150 return 0; 5151 } 5152 5153 static int 5154 igb_timesync_disable(struct rte_eth_dev *dev) 5155 { 5156 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5157 uint32_t tsync_ctl; 5158 5159 /* Disable timestamping of transmitted PTP packets. */ 5160 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5161 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5162 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5163 5164 /* Disable timestamping of received PTP packets. */ 5165 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5166 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5167 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5168 5169 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5170 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5171 5172 /* Stop incrementating the System Time registers. */ 5173 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5174 5175 return 0; 5176 } 5177 5178 static int 5179 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5180 struct timespec *timestamp, 5181 uint32_t flags __rte_unused) 5182 { 5183 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5184 struct e1000_adapter *adapter = 5185 (struct e1000_adapter *)dev->data->dev_private; 5186 uint32_t tsync_rxctl; 5187 uint64_t rx_tstamp_cycles; 5188 uint64_t ns; 5189 5190 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5191 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5192 return -EINVAL; 5193 5194 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5195 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5196 *timestamp = rte_ns_to_timespec(ns); 5197 5198 return 0; 5199 } 5200 5201 static int 5202 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5203 struct timespec *timestamp) 5204 { 5205 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5206 struct e1000_adapter *adapter = 5207 (struct e1000_adapter *)dev->data->dev_private; 5208 uint32_t tsync_txctl; 5209 uint64_t tx_tstamp_cycles; 5210 uint64_t ns; 5211 5212 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5213 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5214 return -EINVAL; 5215 5216 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5217 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5218 *timestamp = rte_ns_to_timespec(ns); 5219 5220 return 0; 5221 } 5222 5223 static int 5224 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5225 { 5226 int count = 0; 5227 int g_ind = 0; 5228 const struct reg_info *reg_group; 5229 5230 while ((reg_group = igb_regs[g_ind++])) 5231 count += igb_reg_group_count(reg_group); 5232 5233 return count; 5234 } 5235 5236 static int 5237 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5238 { 5239 int count = 0; 5240 int g_ind = 0; 5241 const struct reg_info *reg_group; 5242 5243 while ((reg_group = igbvf_regs[g_ind++])) 5244 count += igb_reg_group_count(reg_group); 5245 5246 return count; 5247 } 5248 5249 static int 5250 eth_igb_get_regs(struct rte_eth_dev *dev, 5251 struct rte_dev_reg_info *regs) 5252 { 5253 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5254 uint32_t *data = regs->data; 5255 int g_ind = 0; 5256 int count = 0; 5257 const struct reg_info *reg_group; 5258 5259 if (data == NULL) { 5260 regs->length = eth_igb_get_reg_length(dev); 5261 regs->width = sizeof(uint32_t); 5262 return 0; 5263 } 5264 5265 /* Support only full register dump */ 5266 if ((regs->length == 0) || 5267 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5268 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5269 hw->device_id; 5270 while ((reg_group = igb_regs[g_ind++])) 5271 count += igb_read_regs_group(dev, &data[count], 5272 reg_group); 5273 return 0; 5274 } 5275 5276 return -ENOTSUP; 5277 } 5278 5279 static int 5280 igbvf_get_regs(struct rte_eth_dev *dev, 5281 struct rte_dev_reg_info *regs) 5282 { 5283 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5284 uint32_t *data = regs->data; 5285 int g_ind = 0; 5286 int count = 0; 5287 const struct reg_info *reg_group; 5288 5289 if (data == NULL) { 5290 regs->length = igbvf_get_reg_length(dev); 5291 regs->width = sizeof(uint32_t); 5292 return 0; 5293 } 5294 5295 /* Support only full register dump */ 5296 if ((regs->length == 0) || 5297 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5298 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5299 hw->device_id; 5300 while ((reg_group = igbvf_regs[g_ind++])) 5301 count += igb_read_regs_group(dev, &data[count], 5302 reg_group); 5303 return 0; 5304 } 5305 5306 return -ENOTSUP; 5307 } 5308 5309 static int 5310 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5311 { 5312 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5313 5314 /* Return unit is byte count */ 5315 return hw->nvm.word_size * 2; 5316 } 5317 5318 static int 5319 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5320 struct rte_dev_eeprom_info *in_eeprom) 5321 { 5322 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5323 struct e1000_nvm_info *nvm = &hw->nvm; 5324 uint16_t *data = in_eeprom->data; 5325 int first, length; 5326 5327 first = in_eeprom->offset >> 1; 5328 length = in_eeprom->length >> 1; 5329 if ((first >= hw->nvm.word_size) || 5330 ((first + length) >= hw->nvm.word_size)) 5331 return -EINVAL; 5332 5333 in_eeprom->magic = hw->vendor_id | 5334 ((uint32_t)hw->device_id << 16); 5335 5336 if ((nvm->ops.read) == NULL) 5337 return -ENOTSUP; 5338 5339 return nvm->ops.read(hw, first, length, data); 5340 } 5341 5342 static int 5343 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5344 struct rte_dev_eeprom_info *in_eeprom) 5345 { 5346 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5347 struct e1000_nvm_info *nvm = &hw->nvm; 5348 uint16_t *data = in_eeprom->data; 5349 int first, length; 5350 5351 first = in_eeprom->offset >> 1; 5352 length = in_eeprom->length >> 1; 5353 if ((first >= hw->nvm.word_size) || 5354 ((first + length) >= hw->nvm.word_size)) 5355 return -EINVAL; 5356 5357 in_eeprom->magic = (uint32_t)hw->vendor_id | 5358 ((uint32_t)hw->device_id << 16); 5359 5360 if ((nvm->ops.write) == NULL) 5361 return -ENOTSUP; 5362 return nvm->ops.write(hw, first, length, data); 5363 } 5364 5365 static int 5366 eth_igb_get_module_info(struct rte_eth_dev *dev, 5367 struct rte_eth_dev_module_info *modinfo) 5368 { 5369 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5370 5371 uint32_t status = 0; 5372 uint16_t sff8472_rev, addr_mode; 5373 bool page_swap = false; 5374 5375 if (hw->phy.media_type == e1000_media_type_copper || 5376 hw->phy.media_type == e1000_media_type_unknown) 5377 return -EOPNOTSUPP; 5378 5379 /* Check whether we support SFF-8472 or not */ 5380 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_COMP, &sff8472_rev); 5381 if (status) 5382 return -EIO; 5383 5384 /* addressing mode is not supported */ 5385 status = e1000_read_phy_reg_i2c(hw, IGB_SFF_8472_SWAP, &addr_mode); 5386 if (status) 5387 return -EIO; 5388 5389 /* addressing mode is not supported */ 5390 if ((addr_mode & 0xFF) & IGB_SFF_ADDRESSING_MODE) { 5391 PMD_DRV_LOG(ERR, 5392 "Address change required to access page 0xA2, " 5393 "but not supported. Please report the module " 5394 "type to the driver maintainers.\n"); 5395 page_swap = true; 5396 } 5397 5398 if ((sff8472_rev & 0xFF) == IGB_SFF_8472_UNSUP || page_swap) { 5399 /* We have an SFP, but it does not support SFF-8472 */ 5400 modinfo->type = RTE_ETH_MODULE_SFF_8079; 5401 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 5402 } else { 5403 /* We have an SFP which supports a revision of SFF-8472 */ 5404 modinfo->type = RTE_ETH_MODULE_SFF_8472; 5405 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 5406 } 5407 5408 return 0; 5409 } 5410 5411 static int 5412 eth_igb_get_module_eeprom(struct rte_eth_dev *dev, 5413 struct rte_dev_eeprom_info *info) 5414 { 5415 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5416 5417 uint32_t status = 0; 5418 uint16_t dataword[RTE_ETH_MODULE_SFF_8472_LEN / 2 + 1]; 5419 u16 first_word, last_word; 5420 int i = 0; 5421 5422 if (info->length == 0) 5423 return -EINVAL; 5424 5425 first_word = info->offset >> 1; 5426 last_word = (info->offset + info->length - 1) >> 1; 5427 5428 /* Read EEPROM block, SFF-8079/SFF-8472, word at a time */ 5429 for (i = 0; i < last_word - first_word + 1; i++) { 5430 status = e1000_read_phy_reg_i2c(hw, (first_word + i) * 2, 5431 &dataword[i]); 5432 if (status) { 5433 /* Error occurred while reading module */ 5434 return -EIO; 5435 } 5436 5437 dataword[i] = rte_be_to_cpu_16(dataword[i]); 5438 } 5439 5440 memcpy(info->data, (u8 *)dataword + (info->offset & 1), info->length); 5441 5442 return 0; 5443 } 5444 5445 static int 5446 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5447 { 5448 struct e1000_hw *hw = 5449 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5450 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5451 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5452 uint32_t vec = E1000_MISC_VEC_ID; 5453 5454 if (rte_intr_allow_others(intr_handle)) 5455 vec = E1000_RX_VEC_START; 5456 5457 uint32_t mask = 1 << (queue_id + vec); 5458 5459 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5460 E1000_WRITE_FLUSH(hw); 5461 5462 return 0; 5463 } 5464 5465 static int 5466 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5467 { 5468 struct e1000_hw *hw = 5469 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5470 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5471 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5472 uint32_t vec = E1000_MISC_VEC_ID; 5473 5474 if (rte_intr_allow_others(intr_handle)) 5475 vec = E1000_RX_VEC_START; 5476 5477 uint32_t mask = 1 << (queue_id + vec); 5478 uint32_t regval; 5479 5480 regval = E1000_READ_REG(hw, E1000_EIMS); 5481 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5482 E1000_WRITE_FLUSH(hw); 5483 5484 rte_intr_enable(intr_handle); 5485 5486 return 0; 5487 } 5488 5489 static void 5490 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5491 uint8_t index, uint8_t offset) 5492 { 5493 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5494 5495 /* clear bits */ 5496 val &= ~((uint32_t)0xFF << offset); 5497 5498 /* write vector and valid bit */ 5499 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5500 5501 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5502 } 5503 5504 static void 5505 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5506 uint8_t queue, uint8_t msix_vector) 5507 { 5508 uint32_t tmp = 0; 5509 5510 if (hw->mac.type == e1000_82575) { 5511 if (direction == 0) 5512 tmp = E1000_EICR_RX_QUEUE0 << queue; 5513 else if (direction == 1) 5514 tmp = E1000_EICR_TX_QUEUE0 << queue; 5515 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5516 } else if (hw->mac.type == e1000_82576) { 5517 if ((direction == 0) || (direction == 1)) 5518 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5519 ((queue & 0x8) << 1) + 5520 8 * direction); 5521 } else if ((hw->mac.type == e1000_82580) || 5522 (hw->mac.type == e1000_i350) || 5523 (hw->mac.type == e1000_i354) || 5524 (hw->mac.type == e1000_i210) || 5525 (hw->mac.type == e1000_i211)) { 5526 if ((direction == 0) || (direction == 1)) 5527 eth_igb_write_ivar(hw, msix_vector, 5528 queue >> 1, 5529 ((queue & 0x1) << 4) + 5530 8 * direction); 5531 } 5532 } 5533 5534 /* Sets up the hardware to generate MSI-X interrupts properly 5535 * @hw 5536 * board private structure 5537 */ 5538 static void 5539 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5540 { 5541 int queue_id; 5542 uint32_t tmpval, regval, intr_mask; 5543 struct e1000_hw *hw = 5544 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5545 uint32_t vec = E1000_MISC_VEC_ID; 5546 uint32_t base = E1000_MISC_VEC_ID; 5547 uint32_t misc_shift = 0; 5548 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5549 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5550 5551 /* won't configure msix register if no mapping is done 5552 * between intr vector and event fd 5553 */ 5554 if (!rte_intr_dp_is_en(intr_handle)) 5555 return; 5556 5557 if (rte_intr_allow_others(intr_handle)) { 5558 vec = base = E1000_RX_VEC_START; 5559 misc_shift = 1; 5560 } 5561 5562 /* set interrupt vector for other causes */ 5563 if (hw->mac.type == e1000_82575) { 5564 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5565 /* enable MSI-X PBA support */ 5566 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5567 5568 /* Auto-Mask interrupts upon ICR read */ 5569 tmpval |= E1000_CTRL_EXT_EIAME; 5570 tmpval |= E1000_CTRL_EXT_IRCA; 5571 5572 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5573 5574 /* enable msix_other interrupt */ 5575 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5576 regval = E1000_READ_REG(hw, E1000_EIAC); 5577 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5578 regval = E1000_READ_REG(hw, E1000_EIAM); 5579 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5580 } else if ((hw->mac.type == e1000_82576) || 5581 (hw->mac.type == e1000_82580) || 5582 (hw->mac.type == e1000_i350) || 5583 (hw->mac.type == e1000_i354) || 5584 (hw->mac.type == e1000_i210) || 5585 (hw->mac.type == e1000_i211)) { 5586 /* turn on MSI-X capability first */ 5587 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5588 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5589 E1000_GPIE_NSICR); 5590 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5591 misc_shift; 5592 regval = E1000_READ_REG(hw, E1000_EIAC); 5593 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5594 5595 /* enable msix_other interrupt */ 5596 regval = E1000_READ_REG(hw, E1000_EIMS); 5597 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5598 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8; 5599 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5600 } 5601 5602 /* use EIAM to auto-mask when MSI-X interrupt 5603 * is asserted, this saves a register write for every interrupt 5604 */ 5605 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5606 misc_shift; 5607 regval = E1000_READ_REG(hw, E1000_EIAM); 5608 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5609 5610 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5611 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5612 intr_handle->intr_vec[queue_id] = vec; 5613 if (vec < base + intr_handle->nb_efd - 1) 5614 vec++; 5615 } 5616 5617 E1000_WRITE_FLUSH(hw); 5618 } 5619 5620 /* restore n-tuple filter */ 5621 static inline void 5622 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5623 { 5624 struct e1000_filter_info *filter_info = 5625 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5626 struct e1000_5tuple_filter *p_5tuple; 5627 struct e1000_2tuple_filter *p_2tuple; 5628 5629 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5630 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5631 } 5632 5633 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5634 igb_inject_2uple_filter(dev, p_2tuple); 5635 } 5636 } 5637 5638 /* restore SYN filter */ 5639 static inline void 5640 igb_syn_filter_restore(struct rte_eth_dev *dev) 5641 { 5642 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5643 struct e1000_filter_info *filter_info = 5644 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5645 uint32_t synqf; 5646 5647 synqf = filter_info->syn_info; 5648 5649 if (synqf & E1000_SYN_FILTER_ENABLE) { 5650 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5651 E1000_WRITE_FLUSH(hw); 5652 } 5653 } 5654 5655 /* restore ethernet type filter */ 5656 static inline void 5657 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5658 { 5659 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5660 struct e1000_filter_info *filter_info = 5661 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5662 int i; 5663 5664 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5665 if (filter_info->ethertype_mask & (1 << i)) { 5666 E1000_WRITE_REG(hw, E1000_ETQF(i), 5667 filter_info->ethertype_filters[i].etqf); 5668 E1000_WRITE_FLUSH(hw); 5669 } 5670 } 5671 } 5672 5673 /* restore flex byte filter */ 5674 static inline void 5675 igb_flex_filter_restore(struct rte_eth_dev *dev) 5676 { 5677 struct e1000_filter_info *filter_info = 5678 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5679 struct e1000_flex_filter *flex_filter; 5680 5681 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5682 igb_inject_flex_filter(dev, flex_filter); 5683 } 5684 } 5685 5686 /* restore rss filter */ 5687 static inline void 5688 igb_rss_filter_restore(struct rte_eth_dev *dev) 5689 { 5690 struct e1000_filter_info *filter_info = 5691 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5692 5693 if (filter_info->rss_info.conf.queue_num) 5694 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5695 } 5696 5697 /* restore all types filter */ 5698 static int 5699 igb_filter_restore(struct rte_eth_dev *dev) 5700 { 5701 igb_ntuple_filter_restore(dev); 5702 igb_ethertype_filter_restore(dev); 5703 igb_syn_filter_restore(dev); 5704 igb_flex_filter_restore(dev); 5705 igb_rss_filter_restore(dev); 5706 5707 return 0; 5708 } 5709 5710 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5711 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5712 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5713 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5714 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5715 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5716 5717 /* see e1000_logs.c */ 5718 RTE_INIT(e1000_init_log) 5719 { 5720 e1000_igb_init_log(); 5721 } 5722