1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_common.h> 12 #include <rte_interrupts.h> 13 #include <rte_byteorder.h> 14 #include <rte_log.h> 15 #include <rte_debug.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_ether.h> 19 #include <rte_ethdev.h> 20 #include <rte_ethdev_pci.h> 21 #include <rte_memory.h> 22 #include <rte_eal.h> 23 #include <rte_atomic.h> 24 #include <rte_malloc.h> 25 #include <rte_dev.h> 26 27 #include "e1000_logs.h" 28 #include "base/e1000_api.h" 29 #include "e1000_ethdev.h" 30 #include "igb_regs.h" 31 32 /* 33 * Default values for port configuration 34 */ 35 #define IGB_DEFAULT_RX_FREE_THRESH 32 36 37 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 38 #define IGB_DEFAULT_RX_HTHRESH 8 39 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 40 41 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 42 #define IGB_DEFAULT_TX_HTHRESH 1 43 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 44 45 #define IGB_HKEY_MAX_INDEX 10 46 47 /* Bit shift and mask */ 48 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 49 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 50 #define IGB_8_BIT_WIDTH CHAR_BIT 51 #define IGB_8_BIT_MASK UINT8_MAX 52 53 /* Additional timesync values. */ 54 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 55 #define E1000_ETQF_FILTER_1588 3 56 #define IGB_82576_TSYNC_SHIFT 16 57 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 58 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 59 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 60 61 #define E1000_VTIVAR_MISC 0x01740 62 #define E1000_VTIVAR_MISC_MASK 0xFF 63 #define E1000_VTIVAR_VALID 0x80 64 #define E1000_VTIVAR_MISC_MAILBOX 0 65 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 66 67 /* External VLAN Enable bit mask */ 68 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 69 70 /* External VLAN Ether Type bit mask and shift */ 71 #define E1000_VET_VET_EXT 0xFFFF0000 72 #define E1000_VET_VET_EXT_SHIFT 16 73 74 static int eth_igb_configure(struct rte_eth_dev *dev); 75 static int eth_igb_start(struct rte_eth_dev *dev); 76 static void eth_igb_stop(struct rte_eth_dev *dev); 77 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 78 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 79 static void eth_igb_close(struct rte_eth_dev *dev); 80 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 81 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 82 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 83 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 84 static int eth_igb_link_update(struct rte_eth_dev *dev, 85 int wait_to_complete); 86 static int eth_igb_stats_get(struct rte_eth_dev *dev, 87 struct rte_eth_stats *rte_stats); 88 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 89 struct rte_eth_xstat *xstats, unsigned n); 90 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 91 const uint64_t *ids, 92 uint64_t *values, unsigned int n); 93 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 94 struct rte_eth_xstat_name *xstats_names, 95 unsigned int size); 96 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 97 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 98 unsigned int limit); 99 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 100 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 101 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 102 char *fw_version, size_t fw_size); 103 static void eth_igb_infos_get(struct rte_eth_dev *dev, 104 struct rte_eth_dev_info *dev_info); 105 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 106 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 107 struct rte_eth_dev_info *dev_info); 108 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 109 struct rte_eth_fc_conf *fc_conf); 110 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 111 struct rte_eth_fc_conf *fc_conf); 112 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 113 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 114 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 115 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 116 struct rte_intr_handle *handle); 117 static void eth_igb_interrupt_handler(void *param); 118 static int igb_hardware_init(struct e1000_hw *hw); 119 static void igb_hw_control_acquire(struct e1000_hw *hw); 120 static void igb_hw_control_release(struct e1000_hw *hw); 121 static void igb_init_manageability(struct e1000_hw *hw); 122 static void igb_release_manageability(struct e1000_hw *hw); 123 124 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 125 126 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 127 uint16_t vlan_id, int on); 128 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 129 enum rte_vlan_type vlan_type, 130 uint16_t tpid_id); 131 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 132 133 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 134 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 135 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 138 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 139 140 static int eth_igb_led_on(struct rte_eth_dev *dev); 141 static int eth_igb_led_off(struct rte_eth_dev *dev); 142 143 static void igb_intr_disable(struct e1000_hw *hw); 144 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 145 static int eth_igb_rar_set(struct rte_eth_dev *dev, 146 struct ether_addr *mac_addr, 147 uint32_t index, uint32_t pool); 148 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 149 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 150 struct ether_addr *addr); 151 152 static void igbvf_intr_disable(struct e1000_hw *hw); 153 static int igbvf_dev_configure(struct rte_eth_dev *dev); 154 static int igbvf_dev_start(struct rte_eth_dev *dev); 155 static void igbvf_dev_stop(struct rte_eth_dev *dev); 156 static void igbvf_dev_close(struct rte_eth_dev *dev); 157 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 158 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 159 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 160 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 161 static int eth_igbvf_link_update(struct e1000_hw *hw); 162 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 163 struct rte_eth_stats *rte_stats); 164 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 165 struct rte_eth_xstat *xstats, unsigned n); 166 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 167 struct rte_eth_xstat_name *xstats_names, 168 unsigned limit); 169 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 170 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 171 uint16_t vlan_id, int on); 172 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 173 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 174 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 175 struct ether_addr *addr); 176 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 177 static int igbvf_get_regs(struct rte_eth_dev *dev, 178 struct rte_dev_reg_info *regs); 179 180 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 181 struct rte_eth_rss_reta_entry64 *reta_conf, 182 uint16_t reta_size); 183 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 184 struct rte_eth_rss_reta_entry64 *reta_conf, 185 uint16_t reta_size); 186 187 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 188 struct rte_eth_syn_filter *filter); 189 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 190 enum rte_filter_op filter_op, 191 void *arg); 192 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 193 struct rte_eth_ntuple_filter *ntuple_filter); 194 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 195 struct rte_eth_ntuple_filter *ntuple_filter); 196 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 197 struct rte_eth_flex_filter *filter); 198 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 199 enum rte_filter_op filter_op, 200 void *arg); 201 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 202 struct rte_eth_ntuple_filter *ntuple_filter); 203 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 204 struct rte_eth_ntuple_filter *ntuple_filter); 205 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 206 struct rte_eth_ntuple_filter *filter); 207 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 208 enum rte_filter_op filter_op, 209 void *arg); 210 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 211 enum rte_filter_op filter_op, 212 void *arg); 213 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 214 struct rte_eth_ethertype_filter *filter); 215 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 216 enum rte_filter_type filter_type, 217 enum rte_filter_op filter_op, 218 void *arg); 219 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 220 static int eth_igb_get_regs(struct rte_eth_dev *dev, 221 struct rte_dev_reg_info *regs); 222 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 223 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 224 struct rte_dev_eeprom_info *eeprom); 225 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 226 struct rte_dev_eeprom_info *eeprom); 227 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 228 struct ether_addr *mc_addr_set, 229 uint32_t nb_mc_addr); 230 static int igb_timesync_enable(struct rte_eth_dev *dev); 231 static int igb_timesync_disable(struct rte_eth_dev *dev); 232 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 233 struct timespec *timestamp, 234 uint32_t flags); 235 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 236 struct timespec *timestamp); 237 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 238 static int igb_timesync_read_time(struct rte_eth_dev *dev, 239 struct timespec *timestamp); 240 static int igb_timesync_write_time(struct rte_eth_dev *dev, 241 const struct timespec *timestamp); 242 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 243 uint16_t queue_id); 244 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 245 uint16_t queue_id); 246 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 247 uint8_t queue, uint8_t msix_vector); 248 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 249 uint8_t index, uint8_t offset); 250 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 251 static void eth_igbvf_interrupt_handler(void *param); 252 static void igbvf_mbx_process(struct rte_eth_dev *dev); 253 static int igb_filter_restore(struct rte_eth_dev *dev); 254 255 /* 256 * Define VF Stats MACRO for Non "cleared on read" register 257 */ 258 #define UPDATE_VF_STAT(reg, last, cur) \ 259 { \ 260 u32 latest = E1000_READ_REG(hw, reg); \ 261 cur += (latest - last) & UINT_MAX; \ 262 last = latest; \ 263 } 264 265 #define IGB_FC_PAUSE_TIME 0x0680 266 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 267 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 268 269 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 270 271 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 272 273 /* 274 * The set of PCI devices this driver supports 275 */ 276 static const struct rte_pci_id pci_id_igb_map[] = { 277 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 278 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 279 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 280 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 281 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 284 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 285 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 288 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 289 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 291 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 295 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 296 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 302 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 317 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 318 { .vendor_id = 0, /* sentinel */ }, 319 }; 320 321 /* 322 * The set of PCI devices this driver supports (for 82576&I350 VF) 323 */ 324 static const struct rte_pci_id pci_id_igbvf_map[] = { 325 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 326 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 327 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 328 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 329 { .vendor_id = 0, /* sentinel */ }, 330 }; 331 332 static const struct rte_eth_desc_lim rx_desc_lim = { 333 .nb_max = E1000_MAX_RING_DESC, 334 .nb_min = E1000_MIN_RING_DESC, 335 .nb_align = IGB_RXD_ALIGN, 336 }; 337 338 static const struct rte_eth_desc_lim tx_desc_lim = { 339 .nb_max = E1000_MAX_RING_DESC, 340 .nb_min = E1000_MIN_RING_DESC, 341 .nb_align = IGB_RXD_ALIGN, 342 .nb_seg_max = IGB_TX_MAX_SEG, 343 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 344 }; 345 346 static const struct eth_dev_ops eth_igb_ops = { 347 .dev_configure = eth_igb_configure, 348 .dev_start = eth_igb_start, 349 .dev_stop = eth_igb_stop, 350 .dev_set_link_up = eth_igb_dev_set_link_up, 351 .dev_set_link_down = eth_igb_dev_set_link_down, 352 .dev_close = eth_igb_close, 353 .promiscuous_enable = eth_igb_promiscuous_enable, 354 .promiscuous_disable = eth_igb_promiscuous_disable, 355 .allmulticast_enable = eth_igb_allmulticast_enable, 356 .allmulticast_disable = eth_igb_allmulticast_disable, 357 .link_update = eth_igb_link_update, 358 .stats_get = eth_igb_stats_get, 359 .xstats_get = eth_igb_xstats_get, 360 .xstats_get_by_id = eth_igb_xstats_get_by_id, 361 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 362 .xstats_get_names = eth_igb_xstats_get_names, 363 .stats_reset = eth_igb_stats_reset, 364 .xstats_reset = eth_igb_xstats_reset, 365 .fw_version_get = eth_igb_fw_version_get, 366 .dev_infos_get = eth_igb_infos_get, 367 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 368 .mtu_set = eth_igb_mtu_set, 369 .vlan_filter_set = eth_igb_vlan_filter_set, 370 .vlan_tpid_set = eth_igb_vlan_tpid_set, 371 .vlan_offload_set = eth_igb_vlan_offload_set, 372 .rx_queue_setup = eth_igb_rx_queue_setup, 373 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 374 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 375 .rx_queue_release = eth_igb_rx_queue_release, 376 .rx_queue_count = eth_igb_rx_queue_count, 377 .rx_descriptor_done = eth_igb_rx_descriptor_done, 378 .rx_descriptor_status = eth_igb_rx_descriptor_status, 379 .tx_descriptor_status = eth_igb_tx_descriptor_status, 380 .tx_queue_setup = eth_igb_tx_queue_setup, 381 .tx_queue_release = eth_igb_tx_queue_release, 382 .tx_done_cleanup = eth_igb_tx_done_cleanup, 383 .dev_led_on = eth_igb_led_on, 384 .dev_led_off = eth_igb_led_off, 385 .flow_ctrl_get = eth_igb_flow_ctrl_get, 386 .flow_ctrl_set = eth_igb_flow_ctrl_set, 387 .mac_addr_add = eth_igb_rar_set, 388 .mac_addr_remove = eth_igb_rar_clear, 389 .mac_addr_set = eth_igb_default_mac_addr_set, 390 .reta_update = eth_igb_rss_reta_update, 391 .reta_query = eth_igb_rss_reta_query, 392 .rss_hash_update = eth_igb_rss_hash_update, 393 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 394 .filter_ctrl = eth_igb_filter_ctrl, 395 .set_mc_addr_list = eth_igb_set_mc_addr_list, 396 .rxq_info_get = igb_rxq_info_get, 397 .txq_info_get = igb_txq_info_get, 398 .timesync_enable = igb_timesync_enable, 399 .timesync_disable = igb_timesync_disable, 400 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 401 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 402 .get_reg = eth_igb_get_regs, 403 .get_eeprom_length = eth_igb_get_eeprom_length, 404 .get_eeprom = eth_igb_get_eeprom, 405 .set_eeprom = eth_igb_set_eeprom, 406 .timesync_adjust_time = igb_timesync_adjust_time, 407 .timesync_read_time = igb_timesync_read_time, 408 .timesync_write_time = igb_timesync_write_time, 409 }; 410 411 /* 412 * dev_ops for virtual function, bare necessities for basic vf 413 * operation have been implemented 414 */ 415 static const struct eth_dev_ops igbvf_eth_dev_ops = { 416 .dev_configure = igbvf_dev_configure, 417 .dev_start = igbvf_dev_start, 418 .dev_stop = igbvf_dev_stop, 419 .dev_close = igbvf_dev_close, 420 .promiscuous_enable = igbvf_promiscuous_enable, 421 .promiscuous_disable = igbvf_promiscuous_disable, 422 .allmulticast_enable = igbvf_allmulticast_enable, 423 .allmulticast_disable = igbvf_allmulticast_disable, 424 .link_update = eth_igb_link_update, 425 .stats_get = eth_igbvf_stats_get, 426 .xstats_get = eth_igbvf_xstats_get, 427 .xstats_get_names = eth_igbvf_xstats_get_names, 428 .stats_reset = eth_igbvf_stats_reset, 429 .xstats_reset = eth_igbvf_stats_reset, 430 .vlan_filter_set = igbvf_vlan_filter_set, 431 .dev_infos_get = eth_igbvf_infos_get, 432 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 433 .rx_queue_setup = eth_igb_rx_queue_setup, 434 .rx_queue_release = eth_igb_rx_queue_release, 435 .tx_queue_setup = eth_igb_tx_queue_setup, 436 .tx_queue_release = eth_igb_tx_queue_release, 437 .set_mc_addr_list = eth_igb_set_mc_addr_list, 438 .rxq_info_get = igb_rxq_info_get, 439 .txq_info_get = igb_txq_info_get, 440 .mac_addr_set = igbvf_default_mac_addr_set, 441 .get_reg = igbvf_get_regs, 442 }; 443 444 /* store statistics names and its offset in stats structure */ 445 struct rte_igb_xstats_name_off { 446 char name[RTE_ETH_XSTATS_NAME_SIZE]; 447 unsigned offset; 448 }; 449 450 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 451 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 452 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 453 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 454 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 455 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 456 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 457 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 458 ecol)}, 459 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 460 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 461 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 462 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 463 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 464 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 465 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 466 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 467 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 468 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 469 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 470 fcruc)}, 471 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 472 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 473 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 474 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 475 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 476 prc1023)}, 477 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 478 prc1522)}, 479 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 480 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 481 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 482 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 483 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 484 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 485 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 486 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 487 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 488 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 489 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 490 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 491 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 492 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 493 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 494 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 495 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 496 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 497 ptc1023)}, 498 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 499 ptc1522)}, 500 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 501 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 502 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 503 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 504 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 505 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 506 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 507 508 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 509 }; 510 511 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 512 sizeof(rte_igb_stats_strings[0])) 513 514 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 515 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 516 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 517 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 518 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 519 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 520 }; 521 522 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 523 sizeof(rte_igbvf_stats_strings[0])) 524 525 /** 526 * Atomically reads the link status information from global 527 * structure rte_eth_dev. 528 * 529 * @param dev 530 * - Pointer to the structure rte_eth_dev to read from. 531 * - Pointer to the buffer to be saved with the link status. 532 * 533 * @return 534 * - On success, zero. 535 * - On failure, negative value. 536 */ 537 static inline int 538 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev, 539 struct rte_eth_link *link) 540 { 541 struct rte_eth_link *dst = link; 542 struct rte_eth_link *src = &(dev->data->dev_link); 543 544 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 545 *(uint64_t *)src) == 0) 546 return -1; 547 548 return 0; 549 } 550 551 /** 552 * Atomically writes the link status information into global 553 * structure rte_eth_dev. 554 * 555 * @param dev 556 * - Pointer to the structure rte_eth_dev to read from. 557 * - Pointer to the buffer to be saved with the link status. 558 * 559 * @return 560 * - On success, zero. 561 * - On failure, negative value. 562 */ 563 static inline int 564 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev, 565 struct rte_eth_link *link) 566 { 567 struct rte_eth_link *dst = &(dev->data->dev_link); 568 struct rte_eth_link *src = link; 569 570 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 571 *(uint64_t *)src) == 0) 572 return -1; 573 574 return 0; 575 } 576 577 static inline void 578 igb_intr_enable(struct rte_eth_dev *dev) 579 { 580 struct e1000_interrupt *intr = 581 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 582 struct e1000_hw *hw = 583 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 584 585 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 586 E1000_WRITE_FLUSH(hw); 587 } 588 589 static void 590 igb_intr_disable(struct e1000_hw *hw) 591 { 592 E1000_WRITE_REG(hw, E1000_IMC, ~0); 593 E1000_WRITE_FLUSH(hw); 594 } 595 596 static inline void 597 igbvf_intr_enable(struct rte_eth_dev *dev) 598 { 599 struct e1000_hw *hw = 600 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 601 602 /* only for mailbox */ 603 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 604 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 605 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 606 E1000_WRITE_FLUSH(hw); 607 } 608 609 /* only for mailbox now. If RX/TX needed, should extend this function. */ 610 static void 611 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 612 { 613 uint32_t tmp = 0; 614 615 /* mailbox */ 616 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 617 tmp |= E1000_VTIVAR_VALID; 618 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 619 } 620 621 static void 622 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 623 { 624 struct e1000_hw *hw = 625 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 626 627 /* Configure VF other cause ivar */ 628 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 629 } 630 631 static inline int32_t 632 igb_pf_reset_hw(struct e1000_hw *hw) 633 { 634 uint32_t ctrl_ext; 635 int32_t status; 636 637 status = e1000_reset_hw(hw); 638 639 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 640 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 641 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 642 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 643 E1000_WRITE_FLUSH(hw); 644 645 return status; 646 } 647 648 static void 649 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 650 { 651 struct e1000_hw *hw = 652 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 653 654 655 hw->vendor_id = pci_dev->id.vendor_id; 656 hw->device_id = pci_dev->id.device_id; 657 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 658 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 659 660 e1000_set_mac_type(hw); 661 662 /* need to check if it is a vf device below */ 663 } 664 665 static int 666 igb_reset_swfw_lock(struct e1000_hw *hw) 667 { 668 int ret_val; 669 670 /* 671 * Do mac ops initialization manually here, since we will need 672 * some function pointers set by this call. 673 */ 674 ret_val = e1000_init_mac_params(hw); 675 if (ret_val) 676 return ret_val; 677 678 /* 679 * SMBI lock should not fail in this early stage. If this is the case, 680 * it is due to an improper exit of the application. 681 * So force the release of the faulty lock. 682 */ 683 if (e1000_get_hw_semaphore_generic(hw) < 0) { 684 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 685 } 686 e1000_put_hw_semaphore_generic(hw); 687 688 if (hw->mac.ops.acquire_swfw_sync != NULL) { 689 uint16_t mask; 690 691 /* 692 * Phy lock should not fail in this early stage. If this is the case, 693 * it is due to an improper exit of the application. 694 * So force the release of the faulty lock. 695 */ 696 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 697 if (hw->bus.func > E1000_FUNC_1) 698 mask <<= 2; 699 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 700 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 701 hw->bus.func); 702 } 703 hw->mac.ops.release_swfw_sync(hw, mask); 704 705 /* 706 * This one is more tricky since it is common to all ports; but 707 * swfw_sync retries last long enough (1s) to be almost sure that if 708 * lock can not be taken it is due to an improper lock of the 709 * semaphore. 710 */ 711 mask = E1000_SWFW_EEP_SM; 712 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 713 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 714 } 715 hw->mac.ops.release_swfw_sync(hw, mask); 716 } 717 718 return E1000_SUCCESS; 719 } 720 721 /* Remove all ntuple filters of the device */ 722 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 723 { 724 struct e1000_filter_info *filter_info = 725 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 726 struct e1000_5tuple_filter *p_5tuple; 727 struct e1000_2tuple_filter *p_2tuple; 728 729 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 730 TAILQ_REMOVE(&filter_info->fivetuple_list, 731 p_5tuple, entries); 732 rte_free(p_5tuple); 733 } 734 filter_info->fivetuple_mask = 0; 735 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 736 TAILQ_REMOVE(&filter_info->twotuple_list, 737 p_2tuple, entries); 738 rte_free(p_2tuple); 739 } 740 filter_info->twotuple_mask = 0; 741 742 return 0; 743 } 744 745 /* Remove all flex filters of the device */ 746 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 747 { 748 struct e1000_filter_info *filter_info = 749 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 750 struct e1000_flex_filter *p_flex; 751 752 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 753 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 754 rte_free(p_flex); 755 } 756 filter_info->flex_mask = 0; 757 758 return 0; 759 } 760 761 static int 762 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 763 { 764 int error = 0; 765 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 766 struct e1000_hw *hw = 767 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 768 struct e1000_vfta * shadow_vfta = 769 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 770 struct e1000_filter_info *filter_info = 771 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 772 struct e1000_adapter *adapter = 773 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 774 775 uint32_t ctrl_ext; 776 777 eth_dev->dev_ops = ð_igb_ops; 778 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 779 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 780 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 781 782 /* for secondary processes, we don't initialise any further as primary 783 * has already done this work. Only check we don't need a different 784 * RX function */ 785 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 786 if (eth_dev->data->scattered_rx) 787 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 788 return 0; 789 } 790 791 rte_eth_copy_pci_info(eth_dev, pci_dev); 792 793 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 794 795 igb_identify_hardware(eth_dev, pci_dev); 796 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 797 error = -EIO; 798 goto err_late; 799 } 800 801 e1000_get_bus_info(hw); 802 803 /* Reset any pending lock */ 804 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 805 error = -EIO; 806 goto err_late; 807 } 808 809 /* Finish initialization */ 810 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 811 error = -EIO; 812 goto err_late; 813 } 814 815 hw->mac.autoneg = 1; 816 hw->phy.autoneg_wait_to_complete = 0; 817 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 818 819 /* Copper options */ 820 if (hw->phy.media_type == e1000_media_type_copper) { 821 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 822 hw->phy.disable_polarity_correction = 0; 823 hw->phy.ms_type = e1000_ms_hw_default; 824 } 825 826 /* 827 * Start from a known state, this is important in reading the nvm 828 * and mac from that. 829 */ 830 igb_pf_reset_hw(hw); 831 832 /* Make sure we have a good EEPROM before we read from it */ 833 if (e1000_validate_nvm_checksum(hw) < 0) { 834 /* 835 * Some PCI-E parts fail the first check due to 836 * the link being in sleep state, call it again, 837 * if it fails a second time its a real issue. 838 */ 839 if (e1000_validate_nvm_checksum(hw) < 0) { 840 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 841 error = -EIO; 842 goto err_late; 843 } 844 } 845 846 /* Read the permanent MAC address out of the EEPROM */ 847 if (e1000_read_mac_addr(hw) != 0) { 848 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 849 error = -EIO; 850 goto err_late; 851 } 852 853 /* Allocate memory for storing MAC addresses */ 854 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 855 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 856 if (eth_dev->data->mac_addrs == NULL) { 857 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 858 "store MAC addresses", 859 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 860 error = -ENOMEM; 861 goto err_late; 862 } 863 864 /* Copy the permanent MAC address */ 865 ether_addr_copy((struct ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); 866 867 /* initialize the vfta */ 868 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 869 870 /* Now initialize the hardware */ 871 if (igb_hardware_init(hw) != 0) { 872 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 873 rte_free(eth_dev->data->mac_addrs); 874 eth_dev->data->mac_addrs = NULL; 875 error = -ENODEV; 876 goto err_late; 877 } 878 hw->mac.get_link_status = 1; 879 adapter->stopped = 0; 880 881 /* Indicate SOL/IDER usage */ 882 if (e1000_check_reset_block(hw) < 0) { 883 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 884 "SOL/IDER session"); 885 } 886 887 /* initialize PF if max_vfs not zero */ 888 igb_pf_host_init(eth_dev); 889 890 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 891 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 892 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 893 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 894 E1000_WRITE_FLUSH(hw); 895 896 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 897 eth_dev->data->port_id, pci_dev->id.vendor_id, 898 pci_dev->id.device_id); 899 900 rte_intr_callback_register(&pci_dev->intr_handle, 901 eth_igb_interrupt_handler, 902 (void *)eth_dev); 903 904 /* enable uio/vfio intr/eventfd mapping */ 905 rte_intr_enable(&pci_dev->intr_handle); 906 907 /* enable support intr */ 908 igb_intr_enable(eth_dev); 909 910 /* initialize filter info */ 911 memset(filter_info, 0, 912 sizeof(struct e1000_filter_info)); 913 914 TAILQ_INIT(&filter_info->flex_list); 915 TAILQ_INIT(&filter_info->twotuple_list); 916 TAILQ_INIT(&filter_info->fivetuple_list); 917 918 TAILQ_INIT(&igb_filter_ntuple_list); 919 TAILQ_INIT(&igb_filter_ethertype_list); 920 TAILQ_INIT(&igb_filter_syn_list); 921 TAILQ_INIT(&igb_filter_flex_list); 922 TAILQ_INIT(&igb_filter_rss_list); 923 TAILQ_INIT(&igb_flow_list); 924 925 return 0; 926 927 err_late: 928 igb_hw_control_release(hw); 929 930 return error; 931 } 932 933 static int 934 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 935 { 936 struct rte_pci_device *pci_dev; 937 struct rte_intr_handle *intr_handle; 938 struct e1000_hw *hw; 939 struct e1000_adapter *adapter = 940 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 941 struct e1000_filter_info *filter_info = 942 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 943 944 PMD_INIT_FUNC_TRACE(); 945 946 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 947 return -EPERM; 948 949 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 950 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 951 intr_handle = &pci_dev->intr_handle; 952 953 if (adapter->stopped == 0) 954 eth_igb_close(eth_dev); 955 956 eth_dev->dev_ops = NULL; 957 eth_dev->rx_pkt_burst = NULL; 958 eth_dev->tx_pkt_burst = NULL; 959 960 /* Reset any pending lock */ 961 igb_reset_swfw_lock(hw); 962 963 rte_free(eth_dev->data->mac_addrs); 964 eth_dev->data->mac_addrs = NULL; 965 966 /* uninitialize PF if max_vfs not zero */ 967 igb_pf_host_uninit(eth_dev); 968 969 /* disable uio intr before callback unregister */ 970 rte_intr_disable(intr_handle); 971 rte_intr_callback_unregister(intr_handle, 972 eth_igb_interrupt_handler, eth_dev); 973 974 /* clear the SYN filter info */ 975 filter_info->syn_info = 0; 976 977 /* clear the ethertype filters info */ 978 filter_info->ethertype_mask = 0; 979 memset(filter_info->ethertype_filters, 0, 980 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 981 982 /* clear the rss filter info */ 983 memset(&filter_info->rss_info, 0, 984 sizeof(struct igb_rte_flow_rss_conf)); 985 986 /* remove all ntuple filters of the device */ 987 igb_ntuple_filter_uninit(eth_dev); 988 989 /* remove all flex filters of the device */ 990 igb_flex_filter_uninit(eth_dev); 991 992 /* clear all the filters list */ 993 igb_filterlist_flush(eth_dev); 994 995 return 0; 996 } 997 998 /* 999 * Virtual Function device init 1000 */ 1001 static int 1002 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 1003 { 1004 struct rte_pci_device *pci_dev; 1005 struct rte_intr_handle *intr_handle; 1006 struct e1000_adapter *adapter = 1007 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1008 struct e1000_hw *hw = 1009 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1010 int diag; 1011 struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr; 1012 1013 PMD_INIT_FUNC_TRACE(); 1014 1015 eth_dev->dev_ops = &igbvf_eth_dev_ops; 1016 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 1017 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 1018 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 1019 1020 /* for secondary processes, we don't initialise any further as primary 1021 * has already done this work. Only check we don't need a different 1022 * RX function */ 1023 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 1024 if (eth_dev->data->scattered_rx) 1025 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 1026 return 0; 1027 } 1028 1029 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1030 rte_eth_copy_pci_info(eth_dev, pci_dev); 1031 1032 hw->device_id = pci_dev->id.device_id; 1033 hw->vendor_id = pci_dev->id.vendor_id; 1034 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1035 adapter->stopped = 0; 1036 1037 /* Initialize the shared code (base driver) */ 1038 diag = e1000_setup_init_funcs(hw, TRUE); 1039 if (diag != 0) { 1040 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 1041 diag); 1042 return -EIO; 1043 } 1044 1045 /* init_mailbox_params */ 1046 hw->mbx.ops.init_params(hw); 1047 1048 /* Disable the interrupts for VF */ 1049 igbvf_intr_disable(hw); 1050 1051 diag = hw->mac.ops.reset_hw(hw); 1052 1053 /* Allocate memory for storing MAC addresses */ 1054 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN * 1055 hw->mac.rar_entry_count, 0); 1056 if (eth_dev->data->mac_addrs == NULL) { 1057 PMD_INIT_LOG(ERR, 1058 "Failed to allocate %d bytes needed to store MAC " 1059 "addresses", 1060 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1061 return -ENOMEM; 1062 } 1063 1064 /* Generate a random MAC address, if none was assigned by PF. */ 1065 if (is_zero_ether_addr(perm_addr)) { 1066 eth_random_addr(perm_addr->addr_bytes); 1067 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1068 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1069 "%02x:%02x:%02x:%02x:%02x:%02x", 1070 perm_addr->addr_bytes[0], 1071 perm_addr->addr_bytes[1], 1072 perm_addr->addr_bytes[2], 1073 perm_addr->addr_bytes[3], 1074 perm_addr->addr_bytes[4], 1075 perm_addr->addr_bytes[5]); 1076 } 1077 1078 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1079 if (diag) { 1080 rte_free(eth_dev->data->mac_addrs); 1081 eth_dev->data->mac_addrs = NULL; 1082 return diag; 1083 } 1084 /* Copy the permanent MAC address */ 1085 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1086 ð_dev->data->mac_addrs[0]); 1087 1088 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1089 "mac.type=%s", 1090 eth_dev->data->port_id, pci_dev->id.vendor_id, 1091 pci_dev->id.device_id, "igb_mac_82576_vf"); 1092 1093 intr_handle = &pci_dev->intr_handle; 1094 rte_intr_callback_register(intr_handle, 1095 eth_igbvf_interrupt_handler, eth_dev); 1096 1097 return 0; 1098 } 1099 1100 static int 1101 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1102 { 1103 struct e1000_adapter *adapter = 1104 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1105 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1106 1107 PMD_INIT_FUNC_TRACE(); 1108 1109 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1110 return -EPERM; 1111 1112 if (adapter->stopped == 0) 1113 igbvf_dev_close(eth_dev); 1114 1115 eth_dev->dev_ops = NULL; 1116 eth_dev->rx_pkt_burst = NULL; 1117 eth_dev->tx_pkt_burst = NULL; 1118 1119 rte_free(eth_dev->data->mac_addrs); 1120 eth_dev->data->mac_addrs = NULL; 1121 1122 /* disable uio intr before callback unregister */ 1123 rte_intr_disable(&pci_dev->intr_handle); 1124 rte_intr_callback_unregister(&pci_dev->intr_handle, 1125 eth_igbvf_interrupt_handler, 1126 (void *)eth_dev); 1127 1128 return 0; 1129 } 1130 1131 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1132 struct rte_pci_device *pci_dev) 1133 { 1134 return rte_eth_dev_pci_generic_probe(pci_dev, 1135 sizeof(struct e1000_adapter), eth_igb_dev_init); 1136 } 1137 1138 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1139 { 1140 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1141 } 1142 1143 static struct rte_pci_driver rte_igb_pmd = { 1144 .id_table = pci_id_igb_map, 1145 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1146 RTE_PCI_DRV_IOVA_AS_VA, 1147 .probe = eth_igb_pci_probe, 1148 .remove = eth_igb_pci_remove, 1149 }; 1150 1151 1152 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1153 struct rte_pci_device *pci_dev) 1154 { 1155 return rte_eth_dev_pci_generic_probe(pci_dev, 1156 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1157 } 1158 1159 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1160 { 1161 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1162 } 1163 1164 /* 1165 * virtual function driver struct 1166 */ 1167 static struct rte_pci_driver rte_igbvf_pmd = { 1168 .id_table = pci_id_igbvf_map, 1169 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1170 .probe = eth_igbvf_pci_probe, 1171 .remove = eth_igbvf_pci_remove, 1172 }; 1173 1174 static void 1175 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1176 { 1177 struct e1000_hw *hw = 1178 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1179 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1180 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1181 rctl |= E1000_RCTL_VFE; 1182 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1183 } 1184 1185 static int 1186 igb_check_mq_mode(struct rte_eth_dev *dev) 1187 { 1188 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1189 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1190 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1191 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1192 1193 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1194 tx_mq_mode == ETH_MQ_TX_DCB || 1195 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1196 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1197 return -EINVAL; 1198 } 1199 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1200 /* Check multi-queue mode. 1201 * To no break software we accept ETH_MQ_RX_NONE as this might 1202 * be used to turn off VLAN filter. 1203 */ 1204 1205 if (rx_mq_mode == ETH_MQ_RX_NONE || 1206 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1207 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1208 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1209 } else { 1210 /* Only support one queue on VFs. 1211 * RSS together with SRIOV is not supported. 1212 */ 1213 PMD_INIT_LOG(ERR, "SRIOV is active," 1214 " wrong mq_mode rx %d.", 1215 rx_mq_mode); 1216 return -EINVAL; 1217 } 1218 /* TX mode is not used here, so mode might be ignored.*/ 1219 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1220 /* SRIOV only works in VMDq enable mode */ 1221 PMD_INIT_LOG(WARNING, "SRIOV is active," 1222 " TX mode %d is not supported. " 1223 " Driver will behave as %d mode.", 1224 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1225 } 1226 1227 /* check valid queue number */ 1228 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1229 PMD_INIT_LOG(ERR, "SRIOV is active," 1230 " only support one queue on VFs."); 1231 return -EINVAL; 1232 } 1233 } else { 1234 /* To no break software that set invalid mode, only display 1235 * warning if invalid mode is used. 1236 */ 1237 if (rx_mq_mode != ETH_MQ_RX_NONE && 1238 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1239 rx_mq_mode != ETH_MQ_RX_RSS) { 1240 /* RSS together with VMDq not supported*/ 1241 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1242 rx_mq_mode); 1243 return -EINVAL; 1244 } 1245 1246 if (tx_mq_mode != ETH_MQ_TX_NONE && 1247 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1248 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1249 " Due to txmode is meaningless in this" 1250 " driver, just ignore.", 1251 tx_mq_mode); 1252 } 1253 } 1254 return 0; 1255 } 1256 1257 static int 1258 eth_igb_configure(struct rte_eth_dev *dev) 1259 { 1260 struct e1000_interrupt *intr = 1261 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1262 int ret; 1263 1264 PMD_INIT_FUNC_TRACE(); 1265 1266 /* multipe queue mode checking */ 1267 ret = igb_check_mq_mode(dev); 1268 if (ret != 0) { 1269 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1270 ret); 1271 return ret; 1272 } 1273 1274 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1275 PMD_INIT_FUNC_TRACE(); 1276 1277 return 0; 1278 } 1279 1280 static void 1281 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1282 bool enable) 1283 { 1284 struct e1000_hw *hw = 1285 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1286 uint32_t tctl, rctl; 1287 1288 tctl = E1000_READ_REG(hw, E1000_TCTL); 1289 rctl = E1000_READ_REG(hw, E1000_RCTL); 1290 1291 if (enable) { 1292 /* enable Tx/Rx */ 1293 tctl |= E1000_TCTL_EN; 1294 rctl |= E1000_RCTL_EN; 1295 } else { 1296 /* disable Tx/Rx */ 1297 tctl &= ~E1000_TCTL_EN; 1298 rctl &= ~E1000_RCTL_EN; 1299 } 1300 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1301 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1302 E1000_WRITE_FLUSH(hw); 1303 } 1304 1305 static int 1306 eth_igb_start(struct rte_eth_dev *dev) 1307 { 1308 struct e1000_hw *hw = 1309 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1310 struct e1000_adapter *adapter = 1311 E1000_DEV_PRIVATE(dev->data->dev_private); 1312 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1313 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1314 int ret, mask; 1315 uint32_t intr_vector = 0; 1316 uint32_t ctrl_ext; 1317 uint32_t *speeds; 1318 int num_speeds; 1319 bool autoneg; 1320 1321 PMD_INIT_FUNC_TRACE(); 1322 1323 /* disable uio/vfio intr/eventfd mapping */ 1324 rte_intr_disable(intr_handle); 1325 1326 /* Power up the phy. Needed to make the link go Up */ 1327 eth_igb_dev_set_link_up(dev); 1328 1329 /* 1330 * Packet Buffer Allocation (PBA) 1331 * Writing PBA sets the receive portion of the buffer 1332 * the remainder is used for the transmit buffer. 1333 */ 1334 if (hw->mac.type == e1000_82575) { 1335 uint32_t pba; 1336 1337 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1338 E1000_WRITE_REG(hw, E1000_PBA, pba); 1339 } 1340 1341 /* Put the address into the Receive Address Array */ 1342 e1000_rar_set(hw, hw->mac.addr, 0); 1343 1344 /* Initialize the hardware */ 1345 if (igb_hardware_init(hw)) { 1346 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1347 return -EIO; 1348 } 1349 adapter->stopped = 0; 1350 1351 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1352 1353 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1354 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1355 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1356 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1357 E1000_WRITE_FLUSH(hw); 1358 1359 /* configure PF module if SRIOV enabled */ 1360 igb_pf_host_configure(dev); 1361 1362 /* check and configure queue intr-vector mapping */ 1363 if ((rte_intr_cap_multiple(intr_handle) || 1364 !RTE_ETH_DEV_SRIOV(dev).active) && 1365 dev->data->dev_conf.intr_conf.rxq != 0) { 1366 intr_vector = dev->data->nb_rx_queues; 1367 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1368 return -1; 1369 } 1370 1371 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1372 intr_handle->intr_vec = 1373 rte_zmalloc("intr_vec", 1374 dev->data->nb_rx_queues * sizeof(int), 0); 1375 if (intr_handle->intr_vec == NULL) { 1376 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1377 " intr_vec", dev->data->nb_rx_queues); 1378 return -ENOMEM; 1379 } 1380 } 1381 1382 /* confiugre msix for rx interrupt */ 1383 eth_igb_configure_msix_intr(dev); 1384 1385 /* Configure for OS presence */ 1386 igb_init_manageability(hw); 1387 1388 eth_igb_tx_init(dev); 1389 1390 /* This can fail when allocating mbufs for descriptor rings */ 1391 ret = eth_igb_rx_init(dev); 1392 if (ret) { 1393 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1394 igb_dev_clear_queues(dev); 1395 return ret; 1396 } 1397 1398 e1000_clear_hw_cntrs_base_generic(hw); 1399 1400 /* 1401 * VLAN Offload Settings 1402 */ 1403 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1404 ETH_VLAN_EXTEND_MASK; 1405 ret = eth_igb_vlan_offload_set(dev, mask); 1406 if (ret) { 1407 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1408 igb_dev_clear_queues(dev); 1409 return ret; 1410 } 1411 1412 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1413 /* Enable VLAN filter since VMDq always use VLAN filter */ 1414 igb_vmdq_vlan_hw_filter_enable(dev); 1415 } 1416 1417 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1418 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1419 (hw->mac.type == e1000_i211)) { 1420 /* Configure EITR with the maximum possible value (0xFFFF) */ 1421 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1422 } 1423 1424 /* Setup link speed and duplex */ 1425 speeds = &dev->data->dev_conf.link_speeds; 1426 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1427 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1428 hw->mac.autoneg = 1; 1429 } else { 1430 num_speeds = 0; 1431 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1432 1433 /* Reset */ 1434 hw->phy.autoneg_advertised = 0; 1435 1436 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1437 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1438 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1439 num_speeds = -1; 1440 goto error_invalid_config; 1441 } 1442 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1443 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1444 num_speeds++; 1445 } 1446 if (*speeds & ETH_LINK_SPEED_10M) { 1447 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1448 num_speeds++; 1449 } 1450 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1451 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1452 num_speeds++; 1453 } 1454 if (*speeds & ETH_LINK_SPEED_100M) { 1455 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1456 num_speeds++; 1457 } 1458 if (*speeds & ETH_LINK_SPEED_1G) { 1459 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1460 num_speeds++; 1461 } 1462 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1463 goto error_invalid_config; 1464 1465 /* Set/reset the mac.autoneg based on the link speed, 1466 * fixed or not 1467 */ 1468 if (!autoneg) { 1469 hw->mac.autoneg = 0; 1470 hw->mac.forced_speed_duplex = 1471 hw->phy.autoneg_advertised; 1472 } else { 1473 hw->mac.autoneg = 1; 1474 } 1475 } 1476 1477 e1000_setup_link(hw); 1478 1479 if (rte_intr_allow_others(intr_handle)) { 1480 /* check if lsc interrupt is enabled */ 1481 if (dev->data->dev_conf.intr_conf.lsc != 0) 1482 eth_igb_lsc_interrupt_setup(dev, TRUE); 1483 else 1484 eth_igb_lsc_interrupt_setup(dev, FALSE); 1485 } else { 1486 rte_intr_callback_unregister(intr_handle, 1487 eth_igb_interrupt_handler, 1488 (void *)dev); 1489 if (dev->data->dev_conf.intr_conf.lsc != 0) 1490 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1491 " no intr multiplex"); 1492 } 1493 1494 /* check if rxq interrupt is enabled */ 1495 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1496 rte_intr_dp_is_en(intr_handle)) 1497 eth_igb_rxq_interrupt_setup(dev); 1498 1499 /* enable uio/vfio intr/eventfd mapping */ 1500 rte_intr_enable(intr_handle); 1501 1502 /* resume enabled intr since hw reset */ 1503 igb_intr_enable(dev); 1504 1505 /* restore all types filter */ 1506 igb_filter_restore(dev); 1507 1508 eth_igb_rxtx_control(dev, true); 1509 eth_igb_link_update(dev, 0); 1510 1511 PMD_INIT_LOG(DEBUG, "<<"); 1512 1513 return 0; 1514 1515 error_invalid_config: 1516 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1517 dev->data->dev_conf.link_speeds, dev->data->port_id); 1518 igb_dev_clear_queues(dev); 1519 return -EINVAL; 1520 } 1521 1522 /********************************************************************* 1523 * 1524 * This routine disables all traffic on the adapter by issuing a 1525 * global reset on the MAC. 1526 * 1527 **********************************************************************/ 1528 static void 1529 eth_igb_stop(struct rte_eth_dev *dev) 1530 { 1531 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1532 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1533 struct rte_eth_link link; 1534 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1535 1536 eth_igb_rxtx_control(dev, false); 1537 1538 igb_intr_disable(hw); 1539 1540 /* disable intr eventfd mapping */ 1541 rte_intr_disable(intr_handle); 1542 1543 igb_pf_reset_hw(hw); 1544 E1000_WRITE_REG(hw, E1000_WUC, 0); 1545 1546 /* Set bit for Go Link disconnect */ 1547 if (hw->mac.type >= e1000_82580) { 1548 uint32_t phpm_reg; 1549 1550 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1551 phpm_reg |= E1000_82580_PM_GO_LINKD; 1552 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1553 } 1554 1555 /* Power down the phy. Needed to make the link go Down */ 1556 eth_igb_dev_set_link_down(dev); 1557 1558 igb_dev_clear_queues(dev); 1559 1560 /* clear the recorded link status */ 1561 memset(&link, 0, sizeof(link)); 1562 rte_igb_dev_atomic_write_link_status(dev, &link); 1563 1564 if (!rte_intr_allow_others(intr_handle)) 1565 /* resume to the default handler */ 1566 rte_intr_callback_register(intr_handle, 1567 eth_igb_interrupt_handler, 1568 (void *)dev); 1569 1570 /* Clean datapath event and queue/vec mapping */ 1571 rte_intr_efd_disable(intr_handle); 1572 if (intr_handle->intr_vec != NULL) { 1573 rte_free(intr_handle->intr_vec); 1574 intr_handle->intr_vec = NULL; 1575 } 1576 } 1577 1578 static int 1579 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1580 { 1581 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1582 1583 if (hw->phy.media_type == e1000_media_type_copper) 1584 e1000_power_up_phy(hw); 1585 else 1586 e1000_power_up_fiber_serdes_link(hw); 1587 1588 return 0; 1589 } 1590 1591 static int 1592 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1593 { 1594 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1595 1596 if (hw->phy.media_type == e1000_media_type_copper) 1597 e1000_power_down_phy(hw); 1598 else 1599 e1000_shutdown_fiber_serdes_link(hw); 1600 1601 return 0; 1602 } 1603 1604 static void 1605 eth_igb_close(struct rte_eth_dev *dev) 1606 { 1607 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1608 struct e1000_adapter *adapter = 1609 E1000_DEV_PRIVATE(dev->data->dev_private); 1610 struct rte_eth_link link; 1611 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1612 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1613 1614 eth_igb_stop(dev); 1615 adapter->stopped = 1; 1616 1617 e1000_phy_hw_reset(hw); 1618 igb_release_manageability(hw); 1619 igb_hw_control_release(hw); 1620 1621 /* Clear bit for Go Link disconnect */ 1622 if (hw->mac.type >= e1000_82580) { 1623 uint32_t phpm_reg; 1624 1625 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1626 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1627 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1628 } 1629 1630 igb_dev_free_queues(dev); 1631 1632 if (intr_handle->intr_vec) { 1633 rte_free(intr_handle->intr_vec); 1634 intr_handle->intr_vec = NULL; 1635 } 1636 1637 memset(&link, 0, sizeof(link)); 1638 rte_igb_dev_atomic_write_link_status(dev, &link); 1639 } 1640 1641 static int 1642 igb_get_rx_buffer_size(struct e1000_hw *hw) 1643 { 1644 uint32_t rx_buf_size; 1645 if (hw->mac.type == e1000_82576) { 1646 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1647 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1648 /* PBS needs to be translated according to a lookup table */ 1649 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1650 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1651 rx_buf_size = (rx_buf_size << 10); 1652 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1653 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1654 } else { 1655 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1656 } 1657 1658 return rx_buf_size; 1659 } 1660 1661 /********************************************************************* 1662 * 1663 * Initialize the hardware 1664 * 1665 **********************************************************************/ 1666 static int 1667 igb_hardware_init(struct e1000_hw *hw) 1668 { 1669 uint32_t rx_buf_size; 1670 int diag; 1671 1672 /* Let the firmware know the OS is in control */ 1673 igb_hw_control_acquire(hw); 1674 1675 /* 1676 * These parameters control the automatic generation (Tx) and 1677 * response (Rx) to Ethernet PAUSE frames. 1678 * - High water mark should allow for at least two standard size (1518) 1679 * frames to be received after sending an XOFF. 1680 * - Low water mark works best when it is very near the high water mark. 1681 * This allows the receiver to restart by sending XON when it has 1682 * drained a bit. Here we use an arbitrary value of 1500 which will 1683 * restart after one full frame is pulled from the buffer. There 1684 * could be several smaller frames in the buffer and if so they will 1685 * not trigger the XON until their total number reduces the buffer 1686 * by 1500. 1687 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1688 */ 1689 rx_buf_size = igb_get_rx_buffer_size(hw); 1690 1691 hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2); 1692 hw->fc.low_water = hw->fc.high_water - 1500; 1693 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1694 hw->fc.send_xon = 1; 1695 1696 /* Set Flow control, use the tunable location if sane */ 1697 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1698 hw->fc.requested_mode = igb_fc_setting; 1699 else 1700 hw->fc.requested_mode = e1000_fc_none; 1701 1702 /* Issue a global reset */ 1703 igb_pf_reset_hw(hw); 1704 E1000_WRITE_REG(hw, E1000_WUC, 0); 1705 1706 diag = e1000_init_hw(hw); 1707 if (diag < 0) 1708 return diag; 1709 1710 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1711 e1000_get_phy_info(hw); 1712 e1000_check_for_link(hw); 1713 1714 return 0; 1715 } 1716 1717 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1718 static void 1719 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1720 { 1721 int pause_frames; 1722 1723 uint64_t old_gprc = stats->gprc; 1724 uint64_t old_gptc = stats->gptc; 1725 uint64_t old_tpr = stats->tpr; 1726 uint64_t old_tpt = stats->tpt; 1727 uint64_t old_rpthc = stats->rpthc; 1728 uint64_t old_hgptc = stats->hgptc; 1729 1730 if(hw->phy.media_type == e1000_media_type_copper || 1731 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1732 stats->symerrs += 1733 E1000_READ_REG(hw,E1000_SYMERRS); 1734 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1735 } 1736 1737 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1738 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1739 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1740 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1741 1742 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1743 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1744 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1745 stats->dc += E1000_READ_REG(hw, E1000_DC); 1746 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1747 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1748 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1749 /* 1750 ** For watchdog management we need to know if we have been 1751 ** paused during the last interval, so capture that here. 1752 */ 1753 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1754 stats->xoffrxc += pause_frames; 1755 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1756 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1757 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1758 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1759 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1760 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1761 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1762 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1763 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1764 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1765 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1766 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1767 1768 /* For the 64-bit byte counters the low dword must be read first. */ 1769 /* Both registers clear on the read of the high dword */ 1770 1771 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1772 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1773 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1774 stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN; 1775 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1776 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1777 stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN; 1778 1779 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1780 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1781 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1782 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1783 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1784 1785 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1786 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1787 1788 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1789 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1790 stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN; 1791 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1792 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1793 stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN; 1794 1795 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1796 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1797 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1798 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1799 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1800 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1801 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1802 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1803 1804 /* Interrupt Counts */ 1805 1806 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1807 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1808 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1809 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1810 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1811 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1812 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1813 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1814 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1815 1816 /* Host to Card Statistics */ 1817 1818 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1819 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1820 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1821 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1822 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1823 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1824 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1825 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1826 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1827 stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN; 1828 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1829 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1830 stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN; 1831 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1832 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1833 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1834 1835 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1836 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1837 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1838 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1839 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1840 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1841 } 1842 1843 static int 1844 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1845 { 1846 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1847 struct e1000_hw_stats *stats = 1848 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1849 1850 igb_read_stats_registers(hw, stats); 1851 1852 if (rte_stats == NULL) 1853 return -EINVAL; 1854 1855 /* Rx Errors */ 1856 rte_stats->imissed = stats->mpc; 1857 rte_stats->ierrors = stats->crcerrs + 1858 stats->rlec + stats->ruc + stats->roc + 1859 stats->rxerrc + stats->algnerrc + stats->cexterr; 1860 1861 /* Tx Errors */ 1862 rte_stats->oerrors = stats->ecol + stats->latecol; 1863 1864 rte_stats->ipackets = stats->gprc; 1865 rte_stats->opackets = stats->gptc; 1866 rte_stats->ibytes = stats->gorc; 1867 rte_stats->obytes = stats->gotc; 1868 return 0; 1869 } 1870 1871 static void 1872 eth_igb_stats_reset(struct rte_eth_dev *dev) 1873 { 1874 struct e1000_hw_stats *hw_stats = 1875 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1876 1877 /* HW registers are cleared on read */ 1878 eth_igb_stats_get(dev, NULL); 1879 1880 /* Reset software totals */ 1881 memset(hw_stats, 0, sizeof(*hw_stats)); 1882 } 1883 1884 static void 1885 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1886 { 1887 struct e1000_hw_stats *stats = 1888 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1889 1890 /* HW registers are cleared on read */ 1891 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1892 1893 /* Reset software totals */ 1894 memset(stats, 0, sizeof(*stats)); 1895 } 1896 1897 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1898 struct rte_eth_xstat_name *xstats_names, 1899 __rte_unused unsigned int size) 1900 { 1901 unsigned i; 1902 1903 if (xstats_names == NULL) 1904 return IGB_NB_XSTATS; 1905 1906 /* Note: limit checked in rte_eth_xstats_names() */ 1907 1908 for (i = 0; i < IGB_NB_XSTATS; i++) { 1909 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), 1910 "%s", rte_igb_stats_strings[i].name); 1911 } 1912 1913 return IGB_NB_XSTATS; 1914 } 1915 1916 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1917 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1918 unsigned int limit) 1919 { 1920 unsigned int i; 1921 1922 if (!ids) { 1923 if (xstats_names == NULL) 1924 return IGB_NB_XSTATS; 1925 1926 for (i = 0; i < IGB_NB_XSTATS; i++) 1927 snprintf(xstats_names[i].name, 1928 sizeof(xstats_names[i].name), 1929 "%s", rte_igb_stats_strings[i].name); 1930 1931 return IGB_NB_XSTATS; 1932 1933 } else { 1934 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1935 1936 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1937 IGB_NB_XSTATS); 1938 1939 for (i = 0; i < limit; i++) { 1940 if (ids[i] >= IGB_NB_XSTATS) { 1941 PMD_INIT_LOG(ERR, "id value isn't valid"); 1942 return -1; 1943 } 1944 strcpy(xstats_names[i].name, 1945 xstats_names_copy[ids[i]].name); 1946 } 1947 return limit; 1948 } 1949 } 1950 1951 static int 1952 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1953 unsigned n) 1954 { 1955 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1956 struct e1000_hw_stats *hw_stats = 1957 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1958 unsigned i; 1959 1960 if (n < IGB_NB_XSTATS) 1961 return IGB_NB_XSTATS; 1962 1963 igb_read_stats_registers(hw, hw_stats); 1964 1965 /* If this is a reset xstats is NULL, and we have cleared the 1966 * registers by reading them. 1967 */ 1968 if (!xstats) 1969 return 0; 1970 1971 /* Extended stats */ 1972 for (i = 0; i < IGB_NB_XSTATS; i++) { 1973 xstats[i].id = i; 1974 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1975 rte_igb_stats_strings[i].offset); 1976 } 1977 1978 return IGB_NB_XSTATS; 1979 } 1980 1981 static int 1982 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1983 uint64_t *values, unsigned int n) 1984 { 1985 unsigned int i; 1986 1987 if (!ids) { 1988 struct e1000_hw *hw = 1989 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1990 struct e1000_hw_stats *hw_stats = 1991 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1992 1993 if (n < IGB_NB_XSTATS) 1994 return IGB_NB_XSTATS; 1995 1996 igb_read_stats_registers(hw, hw_stats); 1997 1998 /* If this is a reset xstats is NULL, and we have cleared the 1999 * registers by reading them. 2000 */ 2001 if (!values) 2002 return 0; 2003 2004 /* Extended stats */ 2005 for (i = 0; i < IGB_NB_XSTATS; i++) 2006 values[i] = *(uint64_t *)(((char *)hw_stats) + 2007 rte_igb_stats_strings[i].offset); 2008 2009 return IGB_NB_XSTATS; 2010 2011 } else { 2012 uint64_t values_copy[IGB_NB_XSTATS]; 2013 2014 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 2015 IGB_NB_XSTATS); 2016 2017 for (i = 0; i < n; i++) { 2018 if (ids[i] >= IGB_NB_XSTATS) { 2019 PMD_INIT_LOG(ERR, "id value isn't valid"); 2020 return -1; 2021 } 2022 values[i] = values_copy[ids[i]]; 2023 } 2024 return n; 2025 } 2026 } 2027 2028 static void 2029 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 2030 { 2031 /* Good Rx packets, include VF loopback */ 2032 UPDATE_VF_STAT(E1000_VFGPRC, 2033 hw_stats->last_gprc, hw_stats->gprc); 2034 2035 /* Good Rx octets, include VF loopback */ 2036 UPDATE_VF_STAT(E1000_VFGORC, 2037 hw_stats->last_gorc, hw_stats->gorc); 2038 2039 /* Good Tx packets, include VF loopback */ 2040 UPDATE_VF_STAT(E1000_VFGPTC, 2041 hw_stats->last_gptc, hw_stats->gptc); 2042 2043 /* Good Tx octets, include VF loopback */ 2044 UPDATE_VF_STAT(E1000_VFGOTC, 2045 hw_stats->last_gotc, hw_stats->gotc); 2046 2047 /* Rx Multicst packets */ 2048 UPDATE_VF_STAT(E1000_VFMPRC, 2049 hw_stats->last_mprc, hw_stats->mprc); 2050 2051 /* Good Rx loopback packets */ 2052 UPDATE_VF_STAT(E1000_VFGPRLBC, 2053 hw_stats->last_gprlbc, hw_stats->gprlbc); 2054 2055 /* Good Rx loopback octets */ 2056 UPDATE_VF_STAT(E1000_VFGORLBC, 2057 hw_stats->last_gorlbc, hw_stats->gorlbc); 2058 2059 /* Good Tx loopback packets */ 2060 UPDATE_VF_STAT(E1000_VFGPTLBC, 2061 hw_stats->last_gptlbc, hw_stats->gptlbc); 2062 2063 /* Good Tx loopback octets */ 2064 UPDATE_VF_STAT(E1000_VFGOTLBC, 2065 hw_stats->last_gotlbc, hw_stats->gotlbc); 2066 } 2067 2068 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2069 struct rte_eth_xstat_name *xstats_names, 2070 __rte_unused unsigned limit) 2071 { 2072 unsigned i; 2073 2074 if (xstats_names != NULL) 2075 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2076 snprintf(xstats_names[i].name, 2077 sizeof(xstats_names[i].name), "%s", 2078 rte_igbvf_stats_strings[i].name); 2079 } 2080 return IGBVF_NB_XSTATS; 2081 } 2082 2083 static int 2084 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2085 unsigned n) 2086 { 2087 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2088 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2089 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2090 unsigned i; 2091 2092 if (n < IGBVF_NB_XSTATS) 2093 return IGBVF_NB_XSTATS; 2094 2095 igbvf_read_stats_registers(hw, hw_stats); 2096 2097 if (!xstats) 2098 return 0; 2099 2100 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2101 xstats[i].id = i; 2102 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2103 rte_igbvf_stats_strings[i].offset); 2104 } 2105 2106 return IGBVF_NB_XSTATS; 2107 } 2108 2109 static int 2110 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2111 { 2112 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2113 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2114 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2115 2116 igbvf_read_stats_registers(hw, hw_stats); 2117 2118 if (rte_stats == NULL) 2119 return -EINVAL; 2120 2121 rte_stats->ipackets = hw_stats->gprc; 2122 rte_stats->ibytes = hw_stats->gorc; 2123 rte_stats->opackets = hw_stats->gptc; 2124 rte_stats->obytes = hw_stats->gotc; 2125 return 0; 2126 } 2127 2128 static void 2129 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2130 { 2131 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2132 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2133 2134 /* Sync HW register to the last stats */ 2135 eth_igbvf_stats_get(dev, NULL); 2136 2137 /* reset HW current stats*/ 2138 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2139 offsetof(struct e1000_vf_stats, gprc)); 2140 } 2141 2142 static int 2143 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2144 size_t fw_size) 2145 { 2146 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2147 struct e1000_fw_version fw; 2148 int ret; 2149 2150 e1000_get_fw_version(hw, &fw); 2151 2152 switch (hw->mac.type) { 2153 case e1000_i210: 2154 case e1000_i211: 2155 if (!(e1000_get_flash_presence_i210(hw))) { 2156 ret = snprintf(fw_version, fw_size, 2157 "%2d.%2d-%d", 2158 fw.invm_major, fw.invm_minor, 2159 fw.invm_img_type); 2160 break; 2161 } 2162 /* fall through */ 2163 default: 2164 /* if option rom is valid, display its version too */ 2165 if (fw.or_valid) { 2166 ret = snprintf(fw_version, fw_size, 2167 "%d.%d, 0x%08x, %d.%d.%d", 2168 fw.eep_major, fw.eep_minor, fw.etrack_id, 2169 fw.or_major, fw.or_build, fw.or_patch); 2170 /* no option rom */ 2171 } else { 2172 if (fw.etrack_id != 0X0000) { 2173 ret = snprintf(fw_version, fw_size, 2174 "%d.%d, 0x%08x", 2175 fw.eep_major, fw.eep_minor, 2176 fw.etrack_id); 2177 } else { 2178 ret = snprintf(fw_version, fw_size, 2179 "%d.%d.%d", 2180 fw.eep_major, fw.eep_minor, 2181 fw.eep_build); 2182 } 2183 } 2184 break; 2185 } 2186 2187 ret += 1; /* add the size of '\0' */ 2188 if (fw_size < (u32)ret) 2189 return ret; 2190 else 2191 return 0; 2192 } 2193 2194 static void 2195 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2196 { 2197 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2198 2199 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2200 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2201 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2202 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2203 dev_info->rx_offload_capa = 2204 DEV_RX_OFFLOAD_VLAN_STRIP | 2205 DEV_RX_OFFLOAD_IPV4_CKSUM | 2206 DEV_RX_OFFLOAD_UDP_CKSUM | 2207 DEV_RX_OFFLOAD_TCP_CKSUM; 2208 dev_info->tx_offload_capa = 2209 DEV_TX_OFFLOAD_VLAN_INSERT | 2210 DEV_TX_OFFLOAD_IPV4_CKSUM | 2211 DEV_TX_OFFLOAD_UDP_CKSUM | 2212 DEV_TX_OFFLOAD_TCP_CKSUM | 2213 DEV_TX_OFFLOAD_SCTP_CKSUM | 2214 DEV_TX_OFFLOAD_TCP_TSO; 2215 2216 switch (hw->mac.type) { 2217 case e1000_82575: 2218 dev_info->max_rx_queues = 4; 2219 dev_info->max_tx_queues = 4; 2220 dev_info->max_vmdq_pools = 0; 2221 break; 2222 2223 case e1000_82576: 2224 dev_info->max_rx_queues = 16; 2225 dev_info->max_tx_queues = 16; 2226 dev_info->max_vmdq_pools = ETH_8_POOLS; 2227 dev_info->vmdq_queue_num = 16; 2228 break; 2229 2230 case e1000_82580: 2231 dev_info->max_rx_queues = 8; 2232 dev_info->max_tx_queues = 8; 2233 dev_info->max_vmdq_pools = ETH_8_POOLS; 2234 dev_info->vmdq_queue_num = 8; 2235 break; 2236 2237 case e1000_i350: 2238 dev_info->max_rx_queues = 8; 2239 dev_info->max_tx_queues = 8; 2240 dev_info->max_vmdq_pools = ETH_8_POOLS; 2241 dev_info->vmdq_queue_num = 8; 2242 break; 2243 2244 case e1000_i354: 2245 dev_info->max_rx_queues = 8; 2246 dev_info->max_tx_queues = 8; 2247 break; 2248 2249 case e1000_i210: 2250 dev_info->max_rx_queues = 4; 2251 dev_info->max_tx_queues = 4; 2252 dev_info->max_vmdq_pools = 0; 2253 break; 2254 2255 case e1000_i211: 2256 dev_info->max_rx_queues = 2; 2257 dev_info->max_tx_queues = 2; 2258 dev_info->max_vmdq_pools = 0; 2259 break; 2260 2261 default: 2262 /* Should not happen */ 2263 break; 2264 } 2265 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2266 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2267 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2268 2269 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2270 .rx_thresh = { 2271 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2272 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2273 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2274 }, 2275 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2276 .rx_drop_en = 0, 2277 }; 2278 2279 dev_info->default_txconf = (struct rte_eth_txconf) { 2280 .tx_thresh = { 2281 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2282 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2283 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2284 }, 2285 .txq_flags = 0, 2286 }; 2287 2288 dev_info->rx_desc_lim = rx_desc_lim; 2289 dev_info->tx_desc_lim = tx_desc_lim; 2290 2291 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2292 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2293 ETH_LINK_SPEED_1G; 2294 } 2295 2296 static const uint32_t * 2297 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2298 { 2299 static const uint32_t ptypes[] = { 2300 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2301 RTE_PTYPE_L2_ETHER, 2302 RTE_PTYPE_L3_IPV4, 2303 RTE_PTYPE_L3_IPV4_EXT, 2304 RTE_PTYPE_L3_IPV6, 2305 RTE_PTYPE_L3_IPV6_EXT, 2306 RTE_PTYPE_L4_TCP, 2307 RTE_PTYPE_L4_UDP, 2308 RTE_PTYPE_L4_SCTP, 2309 RTE_PTYPE_TUNNEL_IP, 2310 RTE_PTYPE_INNER_L3_IPV6, 2311 RTE_PTYPE_INNER_L3_IPV6_EXT, 2312 RTE_PTYPE_INNER_L4_TCP, 2313 RTE_PTYPE_INNER_L4_UDP, 2314 RTE_PTYPE_UNKNOWN 2315 }; 2316 2317 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2318 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2319 return ptypes; 2320 return NULL; 2321 } 2322 2323 static void 2324 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2325 { 2326 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2327 2328 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2329 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2330 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2331 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2332 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | 2333 DEV_RX_OFFLOAD_IPV4_CKSUM | 2334 DEV_RX_OFFLOAD_UDP_CKSUM | 2335 DEV_RX_OFFLOAD_TCP_CKSUM; 2336 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2337 DEV_TX_OFFLOAD_IPV4_CKSUM | 2338 DEV_TX_OFFLOAD_UDP_CKSUM | 2339 DEV_TX_OFFLOAD_TCP_CKSUM | 2340 DEV_TX_OFFLOAD_SCTP_CKSUM | 2341 DEV_TX_OFFLOAD_TCP_TSO; 2342 switch (hw->mac.type) { 2343 case e1000_vfadapt: 2344 dev_info->max_rx_queues = 2; 2345 dev_info->max_tx_queues = 2; 2346 break; 2347 case e1000_vfadapt_i350: 2348 dev_info->max_rx_queues = 1; 2349 dev_info->max_tx_queues = 1; 2350 break; 2351 default: 2352 /* Should not happen */ 2353 break; 2354 } 2355 2356 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2357 .rx_thresh = { 2358 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2359 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2360 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2361 }, 2362 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2363 .rx_drop_en = 0, 2364 }; 2365 2366 dev_info->default_txconf = (struct rte_eth_txconf) { 2367 .tx_thresh = { 2368 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2369 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2370 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2371 }, 2372 .txq_flags = 0, 2373 }; 2374 2375 dev_info->rx_desc_lim = rx_desc_lim; 2376 dev_info->tx_desc_lim = tx_desc_lim; 2377 } 2378 2379 /* return 0 means link status changed, -1 means not changed */ 2380 static int 2381 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2382 { 2383 struct e1000_hw *hw = 2384 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2385 struct rte_eth_link link, old; 2386 int link_check, count; 2387 2388 link_check = 0; 2389 hw->mac.get_link_status = 1; 2390 2391 /* possible wait-to-complete in up to 9 seconds */ 2392 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2393 /* Read the real link status */ 2394 switch (hw->phy.media_type) { 2395 case e1000_media_type_copper: 2396 /* Do the work to read phy */ 2397 e1000_check_for_link(hw); 2398 link_check = !hw->mac.get_link_status; 2399 break; 2400 2401 case e1000_media_type_fiber: 2402 e1000_check_for_link(hw); 2403 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2404 E1000_STATUS_LU); 2405 break; 2406 2407 case e1000_media_type_internal_serdes: 2408 e1000_check_for_link(hw); 2409 link_check = hw->mac.serdes_has_link; 2410 break; 2411 2412 /* VF device is type_unknown */ 2413 case e1000_media_type_unknown: 2414 eth_igbvf_link_update(hw); 2415 link_check = !hw->mac.get_link_status; 2416 break; 2417 2418 default: 2419 break; 2420 } 2421 if (link_check || wait_to_complete == 0) 2422 break; 2423 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2424 } 2425 memset(&link, 0, sizeof(link)); 2426 rte_igb_dev_atomic_read_link_status(dev, &link); 2427 old = link; 2428 2429 /* Now we check if a transition has happened */ 2430 if (link_check) { 2431 uint16_t duplex, speed; 2432 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2433 link.link_duplex = (duplex == FULL_DUPLEX) ? 2434 ETH_LINK_FULL_DUPLEX : 2435 ETH_LINK_HALF_DUPLEX; 2436 link.link_speed = speed; 2437 link.link_status = ETH_LINK_UP; 2438 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2439 ETH_LINK_SPEED_FIXED); 2440 } else if (!link_check) { 2441 link.link_speed = 0; 2442 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2443 link.link_status = ETH_LINK_DOWN; 2444 link.link_autoneg = ETH_LINK_FIXED; 2445 } 2446 rte_igb_dev_atomic_write_link_status(dev, &link); 2447 2448 /* not changed */ 2449 if (old.link_status == link.link_status) 2450 return -1; 2451 2452 /* changed */ 2453 return 0; 2454 } 2455 2456 /* 2457 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2458 * For ASF and Pass Through versions of f/w this means 2459 * that the driver is loaded. 2460 */ 2461 static void 2462 igb_hw_control_acquire(struct e1000_hw *hw) 2463 { 2464 uint32_t ctrl_ext; 2465 2466 /* Let firmware know the driver has taken over */ 2467 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2468 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2469 } 2470 2471 /* 2472 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2473 * For ASF and Pass Through versions of f/w this means that the 2474 * driver is no longer loaded. 2475 */ 2476 static void 2477 igb_hw_control_release(struct e1000_hw *hw) 2478 { 2479 uint32_t ctrl_ext; 2480 2481 /* Let firmware taken over control of h/w */ 2482 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2483 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2484 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2485 } 2486 2487 /* 2488 * Bit of a misnomer, what this really means is 2489 * to enable OS management of the system... aka 2490 * to disable special hardware management features. 2491 */ 2492 static void 2493 igb_init_manageability(struct e1000_hw *hw) 2494 { 2495 if (e1000_enable_mng_pass_thru(hw)) { 2496 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2497 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2498 2499 /* disable hardware interception of ARP */ 2500 manc &= ~(E1000_MANC_ARP_EN); 2501 2502 /* enable receiving management packets to the host */ 2503 manc |= E1000_MANC_EN_MNG2HOST; 2504 manc2h |= 1 << 5; /* Mng Port 623 */ 2505 manc2h |= 1 << 6; /* Mng Port 664 */ 2506 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2507 E1000_WRITE_REG(hw, E1000_MANC, manc); 2508 } 2509 } 2510 2511 static void 2512 igb_release_manageability(struct e1000_hw *hw) 2513 { 2514 if (e1000_enable_mng_pass_thru(hw)) { 2515 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2516 2517 manc |= E1000_MANC_ARP_EN; 2518 manc &= ~E1000_MANC_EN_MNG2HOST; 2519 2520 E1000_WRITE_REG(hw, E1000_MANC, manc); 2521 } 2522 } 2523 2524 static void 2525 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2526 { 2527 struct e1000_hw *hw = 2528 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2529 uint32_t rctl; 2530 2531 rctl = E1000_READ_REG(hw, E1000_RCTL); 2532 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2533 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2534 } 2535 2536 static void 2537 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2538 { 2539 struct e1000_hw *hw = 2540 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2541 uint32_t rctl; 2542 2543 rctl = E1000_READ_REG(hw, E1000_RCTL); 2544 rctl &= (~E1000_RCTL_UPE); 2545 if (dev->data->all_multicast == 1) 2546 rctl |= E1000_RCTL_MPE; 2547 else 2548 rctl &= (~E1000_RCTL_MPE); 2549 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2550 } 2551 2552 static void 2553 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2554 { 2555 struct e1000_hw *hw = 2556 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2557 uint32_t rctl; 2558 2559 rctl = E1000_READ_REG(hw, E1000_RCTL); 2560 rctl |= E1000_RCTL_MPE; 2561 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2562 } 2563 2564 static void 2565 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2566 { 2567 struct e1000_hw *hw = 2568 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2569 uint32_t rctl; 2570 2571 if (dev->data->promiscuous == 1) 2572 return; /* must remain in all_multicast mode */ 2573 rctl = E1000_READ_REG(hw, E1000_RCTL); 2574 rctl &= (~E1000_RCTL_MPE); 2575 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2576 } 2577 2578 static int 2579 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2580 { 2581 struct e1000_hw *hw = 2582 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2583 struct e1000_vfta * shadow_vfta = 2584 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2585 uint32_t vfta; 2586 uint32_t vid_idx; 2587 uint32_t vid_bit; 2588 2589 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2590 E1000_VFTA_ENTRY_MASK); 2591 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2592 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2593 if (on) 2594 vfta |= vid_bit; 2595 else 2596 vfta &= ~vid_bit; 2597 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2598 2599 /* update local VFTA copy */ 2600 shadow_vfta->vfta[vid_idx] = vfta; 2601 2602 return 0; 2603 } 2604 2605 static int 2606 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2607 enum rte_vlan_type vlan_type, 2608 uint16_t tpid) 2609 { 2610 struct e1000_hw *hw = 2611 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2612 uint32_t reg, qinq; 2613 2614 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2615 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2616 2617 /* only outer TPID of double VLAN can be configured*/ 2618 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2619 reg = E1000_READ_REG(hw, E1000_VET); 2620 reg = (reg & (~E1000_VET_VET_EXT)) | 2621 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2622 E1000_WRITE_REG(hw, E1000_VET, reg); 2623 2624 return 0; 2625 } 2626 2627 /* all other TPID values are read-only*/ 2628 PMD_DRV_LOG(ERR, "Not supported"); 2629 2630 return -ENOTSUP; 2631 } 2632 2633 static void 2634 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2635 { 2636 struct e1000_hw *hw = 2637 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2638 uint32_t reg; 2639 2640 /* Filter Table Disable */ 2641 reg = E1000_READ_REG(hw, E1000_RCTL); 2642 reg &= ~E1000_RCTL_CFIEN; 2643 reg &= ~E1000_RCTL_VFE; 2644 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2645 } 2646 2647 static void 2648 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2649 { 2650 struct e1000_hw *hw = 2651 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2652 struct e1000_vfta * shadow_vfta = 2653 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2654 uint32_t reg; 2655 int i; 2656 2657 /* Filter Table Enable, CFI not used for packet acceptance */ 2658 reg = E1000_READ_REG(hw, E1000_RCTL); 2659 reg &= ~E1000_RCTL_CFIEN; 2660 reg |= E1000_RCTL_VFE; 2661 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2662 2663 /* restore VFTA table */ 2664 for (i = 0; i < IGB_VFTA_SIZE; i++) 2665 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2666 } 2667 2668 static void 2669 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2670 { 2671 struct e1000_hw *hw = 2672 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2673 uint32_t reg; 2674 2675 /* VLAN Mode Disable */ 2676 reg = E1000_READ_REG(hw, E1000_CTRL); 2677 reg &= ~E1000_CTRL_VME; 2678 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2679 } 2680 2681 static void 2682 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2683 { 2684 struct e1000_hw *hw = 2685 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2686 uint32_t reg; 2687 2688 /* VLAN Mode Enable */ 2689 reg = E1000_READ_REG(hw, E1000_CTRL); 2690 reg |= E1000_CTRL_VME; 2691 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2692 } 2693 2694 static void 2695 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2696 { 2697 struct e1000_hw *hw = 2698 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2699 uint32_t reg; 2700 2701 /* CTRL_EXT: Extended VLAN */ 2702 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2703 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2704 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2705 2706 /* Update maximum packet length */ 2707 if (dev->data->dev_conf.rxmode.jumbo_frame == 1) 2708 E1000_WRITE_REG(hw, E1000_RLPML, 2709 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2710 VLAN_TAG_SIZE); 2711 } 2712 2713 static void 2714 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2715 { 2716 struct e1000_hw *hw = 2717 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2718 uint32_t reg; 2719 2720 /* CTRL_EXT: Extended VLAN */ 2721 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2722 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2723 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2724 2725 /* Update maximum packet length */ 2726 if (dev->data->dev_conf.rxmode.jumbo_frame == 1) 2727 E1000_WRITE_REG(hw, E1000_RLPML, 2728 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2729 2 * VLAN_TAG_SIZE); 2730 } 2731 2732 static int 2733 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2734 { 2735 if(mask & ETH_VLAN_STRIP_MASK){ 2736 if (dev->data->dev_conf.rxmode.hw_vlan_strip) 2737 igb_vlan_hw_strip_enable(dev); 2738 else 2739 igb_vlan_hw_strip_disable(dev); 2740 } 2741 2742 if(mask & ETH_VLAN_FILTER_MASK){ 2743 if (dev->data->dev_conf.rxmode.hw_vlan_filter) 2744 igb_vlan_hw_filter_enable(dev); 2745 else 2746 igb_vlan_hw_filter_disable(dev); 2747 } 2748 2749 if(mask & ETH_VLAN_EXTEND_MASK){ 2750 if (dev->data->dev_conf.rxmode.hw_vlan_extend) 2751 igb_vlan_hw_extend_enable(dev); 2752 else 2753 igb_vlan_hw_extend_disable(dev); 2754 } 2755 2756 return 0; 2757 } 2758 2759 2760 /** 2761 * It enables the interrupt mask and then enable the interrupt. 2762 * 2763 * @param dev 2764 * Pointer to struct rte_eth_dev. 2765 * @param on 2766 * Enable or Disable 2767 * 2768 * @return 2769 * - On success, zero. 2770 * - On failure, a negative value. 2771 */ 2772 static int 2773 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2774 { 2775 struct e1000_interrupt *intr = 2776 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2777 2778 if (on) 2779 intr->mask |= E1000_ICR_LSC; 2780 else 2781 intr->mask &= ~E1000_ICR_LSC; 2782 2783 return 0; 2784 } 2785 2786 /* It clears the interrupt causes and enables the interrupt. 2787 * It will be called once only during nic initialized. 2788 * 2789 * @param dev 2790 * Pointer to struct rte_eth_dev. 2791 * 2792 * @return 2793 * - On success, zero. 2794 * - On failure, a negative value. 2795 */ 2796 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2797 { 2798 uint32_t mask, regval; 2799 struct e1000_hw *hw = 2800 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2801 struct rte_eth_dev_info dev_info; 2802 2803 memset(&dev_info, 0, sizeof(dev_info)); 2804 eth_igb_infos_get(dev, &dev_info); 2805 2806 mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues); 2807 regval = E1000_READ_REG(hw, E1000_EIMS); 2808 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2809 2810 return 0; 2811 } 2812 2813 /* 2814 * It reads ICR and gets interrupt causes, check it and set a bit flag 2815 * to update link status. 2816 * 2817 * @param dev 2818 * Pointer to struct rte_eth_dev. 2819 * 2820 * @return 2821 * - On success, zero. 2822 * - On failure, a negative value. 2823 */ 2824 static int 2825 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2826 { 2827 uint32_t icr; 2828 struct e1000_hw *hw = 2829 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2830 struct e1000_interrupt *intr = 2831 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2832 2833 igb_intr_disable(hw); 2834 2835 /* read-on-clear nic registers here */ 2836 icr = E1000_READ_REG(hw, E1000_ICR); 2837 2838 intr->flags = 0; 2839 if (icr & E1000_ICR_LSC) { 2840 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2841 } 2842 2843 if (icr & E1000_ICR_VMMB) 2844 intr->flags |= E1000_FLAG_MAILBOX; 2845 2846 return 0; 2847 } 2848 2849 /* 2850 * It executes link_update after knowing an interrupt is prsent. 2851 * 2852 * @param dev 2853 * Pointer to struct rte_eth_dev. 2854 * 2855 * @return 2856 * - On success, zero. 2857 * - On failure, a negative value. 2858 */ 2859 static int 2860 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2861 struct rte_intr_handle *intr_handle) 2862 { 2863 struct e1000_hw *hw = 2864 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2865 struct e1000_interrupt *intr = 2866 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2867 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2868 struct rte_eth_link link; 2869 int ret; 2870 2871 if (intr->flags & E1000_FLAG_MAILBOX) { 2872 igb_pf_mbx_process(dev); 2873 intr->flags &= ~E1000_FLAG_MAILBOX; 2874 } 2875 2876 igb_intr_enable(dev); 2877 rte_intr_enable(intr_handle); 2878 2879 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2880 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2881 2882 /* set get_link_status to check register later */ 2883 hw->mac.get_link_status = 1; 2884 ret = eth_igb_link_update(dev, 0); 2885 2886 /* check if link has changed */ 2887 if (ret < 0) 2888 return 0; 2889 2890 memset(&link, 0, sizeof(link)); 2891 rte_igb_dev_atomic_read_link_status(dev, &link); 2892 if (link.link_status) { 2893 PMD_INIT_LOG(INFO, 2894 " Port %d: Link Up - speed %u Mbps - %s", 2895 dev->data->port_id, 2896 (unsigned)link.link_speed, 2897 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2898 "full-duplex" : "half-duplex"); 2899 } else { 2900 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2901 dev->data->port_id); 2902 } 2903 2904 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2905 pci_dev->addr.domain, 2906 pci_dev->addr.bus, 2907 pci_dev->addr.devid, 2908 pci_dev->addr.function); 2909 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2910 NULL); 2911 } 2912 2913 return 0; 2914 } 2915 2916 /** 2917 * Interrupt handler which shall be registered at first. 2918 * 2919 * @param handle 2920 * Pointer to interrupt handle. 2921 * @param param 2922 * The address of parameter (struct rte_eth_dev *) regsitered before. 2923 * 2924 * @return 2925 * void 2926 */ 2927 static void 2928 eth_igb_interrupt_handler(void *param) 2929 { 2930 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2931 2932 eth_igb_interrupt_get_status(dev); 2933 eth_igb_interrupt_action(dev, dev->intr_handle); 2934 } 2935 2936 static int 2937 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2938 { 2939 uint32_t eicr; 2940 struct e1000_hw *hw = 2941 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2942 struct e1000_interrupt *intr = 2943 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2944 2945 igbvf_intr_disable(hw); 2946 2947 /* read-on-clear nic registers here */ 2948 eicr = E1000_READ_REG(hw, E1000_EICR); 2949 intr->flags = 0; 2950 2951 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2952 intr->flags |= E1000_FLAG_MAILBOX; 2953 2954 return 0; 2955 } 2956 2957 void igbvf_mbx_process(struct rte_eth_dev *dev) 2958 { 2959 struct e1000_hw *hw = 2960 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2961 struct e1000_mbx_info *mbx = &hw->mbx; 2962 u32 in_msg = 0; 2963 2964 /* peek the message first */ 2965 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2966 2967 /* PF reset VF event */ 2968 if (in_msg == E1000_PF_CONTROL_MSG) { 2969 /* dummy mbx read to ack pf */ 2970 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2971 return; 2972 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2973 NULL); 2974 } 2975 } 2976 2977 static int 2978 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2979 { 2980 struct e1000_interrupt *intr = 2981 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2982 2983 if (intr->flags & E1000_FLAG_MAILBOX) { 2984 igbvf_mbx_process(dev); 2985 intr->flags &= ~E1000_FLAG_MAILBOX; 2986 } 2987 2988 igbvf_intr_enable(dev); 2989 rte_intr_enable(intr_handle); 2990 2991 return 0; 2992 } 2993 2994 static void 2995 eth_igbvf_interrupt_handler(void *param) 2996 { 2997 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2998 2999 eth_igbvf_interrupt_get_status(dev); 3000 eth_igbvf_interrupt_action(dev, dev->intr_handle); 3001 } 3002 3003 static int 3004 eth_igb_led_on(struct rte_eth_dev *dev) 3005 { 3006 struct e1000_hw *hw; 3007 3008 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3009 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3010 } 3011 3012 static int 3013 eth_igb_led_off(struct rte_eth_dev *dev) 3014 { 3015 struct e1000_hw *hw; 3016 3017 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3018 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 3019 } 3020 3021 static int 3022 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3023 { 3024 struct e1000_hw *hw; 3025 uint32_t ctrl; 3026 int tx_pause; 3027 int rx_pause; 3028 3029 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3030 fc_conf->pause_time = hw->fc.pause_time; 3031 fc_conf->high_water = hw->fc.high_water; 3032 fc_conf->low_water = hw->fc.low_water; 3033 fc_conf->send_xon = hw->fc.send_xon; 3034 fc_conf->autoneg = hw->mac.autoneg; 3035 3036 /* 3037 * Return rx_pause and tx_pause status according to actual setting of 3038 * the TFCE and RFCE bits in the CTRL register. 3039 */ 3040 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3041 if (ctrl & E1000_CTRL_TFCE) 3042 tx_pause = 1; 3043 else 3044 tx_pause = 0; 3045 3046 if (ctrl & E1000_CTRL_RFCE) 3047 rx_pause = 1; 3048 else 3049 rx_pause = 0; 3050 3051 if (rx_pause && tx_pause) 3052 fc_conf->mode = RTE_FC_FULL; 3053 else if (rx_pause) 3054 fc_conf->mode = RTE_FC_RX_PAUSE; 3055 else if (tx_pause) 3056 fc_conf->mode = RTE_FC_TX_PAUSE; 3057 else 3058 fc_conf->mode = RTE_FC_NONE; 3059 3060 return 0; 3061 } 3062 3063 static int 3064 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3065 { 3066 struct e1000_hw *hw; 3067 int err; 3068 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3069 e1000_fc_none, 3070 e1000_fc_rx_pause, 3071 e1000_fc_tx_pause, 3072 e1000_fc_full 3073 }; 3074 uint32_t rx_buf_size; 3075 uint32_t max_high_water; 3076 uint32_t rctl; 3077 3078 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3079 if (fc_conf->autoneg != hw->mac.autoneg) 3080 return -ENOTSUP; 3081 rx_buf_size = igb_get_rx_buffer_size(hw); 3082 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3083 3084 /* At least reserve one Ethernet frame for watermark */ 3085 max_high_water = rx_buf_size - ETHER_MAX_LEN; 3086 if ((fc_conf->high_water > max_high_water) || 3087 (fc_conf->high_water < fc_conf->low_water)) { 3088 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3089 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3090 return -EINVAL; 3091 } 3092 3093 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3094 hw->fc.pause_time = fc_conf->pause_time; 3095 hw->fc.high_water = fc_conf->high_water; 3096 hw->fc.low_water = fc_conf->low_water; 3097 hw->fc.send_xon = fc_conf->send_xon; 3098 3099 err = e1000_setup_link_generic(hw); 3100 if (err == E1000_SUCCESS) { 3101 3102 /* check if we want to forward MAC frames - driver doesn't have native 3103 * capability to do that, so we'll write the registers ourselves */ 3104 3105 rctl = E1000_READ_REG(hw, E1000_RCTL); 3106 3107 /* set or clear MFLCN.PMCF bit depending on configuration */ 3108 if (fc_conf->mac_ctrl_frame_fwd != 0) 3109 rctl |= E1000_RCTL_PMCF; 3110 else 3111 rctl &= ~E1000_RCTL_PMCF; 3112 3113 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3114 E1000_WRITE_FLUSH(hw); 3115 3116 return 0; 3117 } 3118 3119 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3120 return -EIO; 3121 } 3122 3123 #define E1000_RAH_POOLSEL_SHIFT (18) 3124 static int 3125 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 3126 uint32_t index, uint32_t pool) 3127 { 3128 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3129 uint32_t rah; 3130 3131 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3132 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3133 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3134 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3135 return 0; 3136 } 3137 3138 static void 3139 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3140 { 3141 uint8_t addr[ETHER_ADDR_LEN]; 3142 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3143 3144 memset(addr, 0, sizeof(addr)); 3145 3146 e1000_rar_set(hw, addr, index); 3147 } 3148 3149 static void 3150 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3151 struct ether_addr *addr) 3152 { 3153 eth_igb_rar_clear(dev, 0); 3154 3155 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3156 } 3157 /* 3158 * Virtual Function operations 3159 */ 3160 static void 3161 igbvf_intr_disable(struct e1000_hw *hw) 3162 { 3163 PMD_INIT_FUNC_TRACE(); 3164 3165 /* Clear interrupt mask to stop from interrupts being generated */ 3166 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3167 3168 E1000_WRITE_FLUSH(hw); 3169 } 3170 3171 static void 3172 igbvf_stop_adapter(struct rte_eth_dev *dev) 3173 { 3174 u32 reg_val; 3175 u16 i; 3176 struct rte_eth_dev_info dev_info; 3177 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3178 3179 memset(&dev_info, 0, sizeof(dev_info)); 3180 eth_igbvf_infos_get(dev, &dev_info); 3181 3182 /* Clear interrupt mask to stop from interrupts being generated */ 3183 igbvf_intr_disable(hw); 3184 3185 /* Clear any pending interrupts, flush previous writes */ 3186 E1000_READ_REG(hw, E1000_EICR); 3187 3188 /* Disable the transmit unit. Each queue must be disabled. */ 3189 for (i = 0; i < dev_info.max_tx_queues; i++) 3190 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3191 3192 /* Disable the receive unit by stopping each queue */ 3193 for (i = 0; i < dev_info.max_rx_queues; i++) { 3194 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3195 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3196 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3197 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3198 ; 3199 } 3200 3201 /* flush all queues disables */ 3202 E1000_WRITE_FLUSH(hw); 3203 msec_delay(2); 3204 } 3205 3206 static int eth_igbvf_link_update(struct e1000_hw *hw) 3207 { 3208 struct e1000_mbx_info *mbx = &hw->mbx; 3209 struct e1000_mac_info *mac = &hw->mac; 3210 int ret_val = E1000_SUCCESS; 3211 3212 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3213 3214 /* 3215 * We only want to run this if there has been a rst asserted. 3216 * in this case that could mean a link change, device reset, 3217 * or a virtual function reset 3218 */ 3219 3220 /* If we were hit with a reset or timeout drop the link */ 3221 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3222 mac->get_link_status = TRUE; 3223 3224 if (!mac->get_link_status) 3225 goto out; 3226 3227 /* if link status is down no point in checking to see if pf is up */ 3228 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3229 goto out; 3230 3231 /* if we passed all the tests above then the link is up and we no 3232 * longer need to check for link */ 3233 mac->get_link_status = FALSE; 3234 3235 out: 3236 return ret_val; 3237 } 3238 3239 3240 static int 3241 igbvf_dev_configure(struct rte_eth_dev *dev) 3242 { 3243 struct rte_eth_conf* conf = &dev->data->dev_conf; 3244 3245 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3246 dev->data->port_id); 3247 3248 /* 3249 * VF has no ability to enable/disable HW CRC 3250 * Keep the persistent behavior the same as Host PF 3251 */ 3252 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3253 if (!conf->rxmode.hw_strip_crc) { 3254 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3255 conf->rxmode.hw_strip_crc = 1; 3256 } 3257 #else 3258 if (conf->rxmode.hw_strip_crc) { 3259 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3260 conf->rxmode.hw_strip_crc = 0; 3261 } 3262 #endif 3263 3264 return 0; 3265 } 3266 3267 static int 3268 igbvf_dev_start(struct rte_eth_dev *dev) 3269 { 3270 struct e1000_hw *hw = 3271 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3272 struct e1000_adapter *adapter = 3273 E1000_DEV_PRIVATE(dev->data->dev_private); 3274 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3275 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3276 int ret; 3277 uint32_t intr_vector = 0; 3278 3279 PMD_INIT_FUNC_TRACE(); 3280 3281 hw->mac.ops.reset_hw(hw); 3282 adapter->stopped = 0; 3283 3284 /* Set all vfta */ 3285 igbvf_set_vfta_all(dev,1); 3286 3287 eth_igbvf_tx_init(dev); 3288 3289 /* This can fail when allocating mbufs for descriptor rings */ 3290 ret = eth_igbvf_rx_init(dev); 3291 if (ret) { 3292 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3293 igb_dev_clear_queues(dev); 3294 return ret; 3295 } 3296 3297 /* check and configure queue intr-vector mapping */ 3298 if (dev->data->dev_conf.intr_conf.rxq != 0) { 3299 intr_vector = dev->data->nb_rx_queues; 3300 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3301 if (ret) 3302 return ret; 3303 } 3304 3305 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3306 intr_handle->intr_vec = 3307 rte_zmalloc("intr_vec", 3308 dev->data->nb_rx_queues * sizeof(int), 0); 3309 if (!intr_handle->intr_vec) { 3310 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3311 " intr_vec", dev->data->nb_rx_queues); 3312 return -ENOMEM; 3313 } 3314 } 3315 3316 eth_igbvf_configure_msix_intr(dev); 3317 3318 /* enable uio/vfio intr/eventfd mapping */ 3319 rte_intr_enable(intr_handle); 3320 3321 /* resume enabled intr since hw reset */ 3322 igbvf_intr_enable(dev); 3323 3324 return 0; 3325 } 3326 3327 static void 3328 igbvf_dev_stop(struct rte_eth_dev *dev) 3329 { 3330 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3331 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3332 3333 PMD_INIT_FUNC_TRACE(); 3334 3335 igbvf_stop_adapter(dev); 3336 3337 /* 3338 * Clear what we set, but we still keep shadow_vfta to 3339 * restore after device starts 3340 */ 3341 igbvf_set_vfta_all(dev,0); 3342 3343 igb_dev_clear_queues(dev); 3344 3345 /* disable intr eventfd mapping */ 3346 rte_intr_disable(intr_handle); 3347 3348 /* Clean datapath event and queue/vec mapping */ 3349 rte_intr_efd_disable(intr_handle); 3350 if (intr_handle->intr_vec) { 3351 rte_free(intr_handle->intr_vec); 3352 intr_handle->intr_vec = NULL; 3353 } 3354 } 3355 3356 static void 3357 igbvf_dev_close(struct rte_eth_dev *dev) 3358 { 3359 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3360 struct e1000_adapter *adapter = 3361 E1000_DEV_PRIVATE(dev->data->dev_private); 3362 struct ether_addr addr; 3363 3364 PMD_INIT_FUNC_TRACE(); 3365 3366 e1000_reset_hw(hw); 3367 3368 igbvf_dev_stop(dev); 3369 adapter->stopped = 1; 3370 igb_dev_free_queues(dev); 3371 3372 /** 3373 * reprogram the RAR with a zero mac address, 3374 * to ensure that the VF traffic goes to the PF 3375 * after stop, close and detach of the VF. 3376 **/ 3377 3378 memset(&addr, 0, sizeof(addr)); 3379 igbvf_default_mac_addr_set(dev, &addr); 3380 } 3381 3382 static void 3383 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3384 { 3385 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3386 3387 /* Set both unicast and multicast promisc */ 3388 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3389 } 3390 3391 static void 3392 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3393 { 3394 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3395 3396 /* If in allmulticast mode leave multicast promisc */ 3397 if (dev->data->all_multicast == 1) 3398 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3399 else 3400 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3401 } 3402 3403 static void 3404 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3405 { 3406 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3407 3408 /* In promiscuous mode multicast promisc already set */ 3409 if (dev->data->promiscuous == 0) 3410 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3411 } 3412 3413 static void 3414 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3415 { 3416 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3417 3418 /* In promiscuous mode leave multicast promisc enabled */ 3419 if (dev->data->promiscuous == 0) 3420 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3421 } 3422 3423 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3424 { 3425 struct e1000_mbx_info *mbx = &hw->mbx; 3426 uint32_t msgbuf[2]; 3427 s32 err; 3428 3429 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3430 msgbuf[0] = E1000_VF_SET_VLAN; 3431 msgbuf[1] = vid; 3432 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3433 if (on) 3434 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3435 3436 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3437 if (err) 3438 goto mbx_err; 3439 3440 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3441 if (err) 3442 goto mbx_err; 3443 3444 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3445 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3446 err = -EINVAL; 3447 3448 mbx_err: 3449 return err; 3450 } 3451 3452 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3453 { 3454 struct e1000_hw *hw = 3455 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3456 struct e1000_vfta * shadow_vfta = 3457 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3458 int i = 0, j = 0, vfta = 0, mask = 1; 3459 3460 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3461 vfta = shadow_vfta->vfta[i]; 3462 if(vfta){ 3463 mask = 1; 3464 for (j = 0; j < 32; j++){ 3465 if(vfta & mask) 3466 igbvf_set_vfta(hw, 3467 (uint16_t)((i<<5)+j), on); 3468 mask<<=1; 3469 } 3470 } 3471 } 3472 3473 } 3474 3475 static int 3476 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3477 { 3478 struct e1000_hw *hw = 3479 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3480 struct e1000_vfta * shadow_vfta = 3481 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3482 uint32_t vid_idx = 0; 3483 uint32_t vid_bit = 0; 3484 int ret = 0; 3485 3486 PMD_INIT_FUNC_TRACE(); 3487 3488 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3489 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3490 if(ret){ 3491 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3492 return ret; 3493 } 3494 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3495 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3496 3497 /*Save what we set and retore it after device reset*/ 3498 if (on) 3499 shadow_vfta->vfta[vid_idx] |= vid_bit; 3500 else 3501 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3502 3503 return 0; 3504 } 3505 3506 static void 3507 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) 3508 { 3509 struct e1000_hw *hw = 3510 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3511 3512 /* index is not used by rar_set() */ 3513 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3514 } 3515 3516 3517 static int 3518 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3519 struct rte_eth_rss_reta_entry64 *reta_conf, 3520 uint16_t reta_size) 3521 { 3522 uint8_t i, j, mask; 3523 uint32_t reta, r; 3524 uint16_t idx, shift; 3525 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3526 3527 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3528 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3529 "(%d) doesn't match the number hardware can supported " 3530 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3531 return -EINVAL; 3532 } 3533 3534 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3535 idx = i / RTE_RETA_GROUP_SIZE; 3536 shift = i % RTE_RETA_GROUP_SIZE; 3537 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3538 IGB_4_BIT_MASK); 3539 if (!mask) 3540 continue; 3541 if (mask == IGB_4_BIT_MASK) 3542 r = 0; 3543 else 3544 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3545 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3546 if (mask & (0x1 << j)) 3547 reta |= reta_conf[idx].reta[shift + j] << 3548 (CHAR_BIT * j); 3549 else 3550 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3551 } 3552 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3553 } 3554 3555 return 0; 3556 } 3557 3558 static int 3559 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3560 struct rte_eth_rss_reta_entry64 *reta_conf, 3561 uint16_t reta_size) 3562 { 3563 uint8_t i, j, mask; 3564 uint32_t reta; 3565 uint16_t idx, shift; 3566 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3567 3568 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3569 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3570 "(%d) doesn't match the number hardware can supported " 3571 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3572 return -EINVAL; 3573 } 3574 3575 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3576 idx = i / RTE_RETA_GROUP_SIZE; 3577 shift = i % RTE_RETA_GROUP_SIZE; 3578 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3579 IGB_4_BIT_MASK); 3580 if (!mask) 3581 continue; 3582 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3583 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3584 if (mask & (0x1 << j)) 3585 reta_conf[idx].reta[shift + j] = 3586 ((reta >> (CHAR_BIT * j)) & 3587 IGB_8_BIT_MASK); 3588 } 3589 } 3590 3591 return 0; 3592 } 3593 3594 int 3595 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3596 struct rte_eth_syn_filter *filter, 3597 bool add) 3598 { 3599 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3600 struct e1000_filter_info *filter_info = 3601 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3602 uint32_t synqf, rfctl; 3603 3604 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3605 return -EINVAL; 3606 3607 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3608 3609 if (add) { 3610 if (synqf & E1000_SYN_FILTER_ENABLE) 3611 return -EINVAL; 3612 3613 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3614 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3615 3616 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3617 if (filter->hig_pri) 3618 rfctl |= E1000_RFCTL_SYNQFP; 3619 else 3620 rfctl &= ~E1000_RFCTL_SYNQFP; 3621 3622 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3623 } else { 3624 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3625 return -ENOENT; 3626 synqf = 0; 3627 } 3628 3629 filter_info->syn_info = synqf; 3630 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3631 E1000_WRITE_FLUSH(hw); 3632 return 0; 3633 } 3634 3635 static int 3636 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3637 struct rte_eth_syn_filter *filter) 3638 { 3639 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3640 uint32_t synqf, rfctl; 3641 3642 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3643 if (synqf & E1000_SYN_FILTER_ENABLE) { 3644 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3645 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3646 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3647 E1000_SYN_FILTER_QUEUE_SHIFT); 3648 return 0; 3649 } 3650 3651 return -ENOENT; 3652 } 3653 3654 static int 3655 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3656 enum rte_filter_op filter_op, 3657 void *arg) 3658 { 3659 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3660 int ret; 3661 3662 MAC_TYPE_FILTER_SUP(hw->mac.type); 3663 3664 if (filter_op == RTE_ETH_FILTER_NOP) 3665 return 0; 3666 3667 if (arg == NULL) { 3668 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3669 filter_op); 3670 return -EINVAL; 3671 } 3672 3673 switch (filter_op) { 3674 case RTE_ETH_FILTER_ADD: 3675 ret = eth_igb_syn_filter_set(dev, 3676 (struct rte_eth_syn_filter *)arg, 3677 TRUE); 3678 break; 3679 case RTE_ETH_FILTER_DELETE: 3680 ret = eth_igb_syn_filter_set(dev, 3681 (struct rte_eth_syn_filter *)arg, 3682 FALSE); 3683 break; 3684 case RTE_ETH_FILTER_GET: 3685 ret = eth_igb_syn_filter_get(dev, 3686 (struct rte_eth_syn_filter *)arg); 3687 break; 3688 default: 3689 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3690 ret = -EINVAL; 3691 break; 3692 } 3693 3694 return ret; 3695 } 3696 3697 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3698 static inline int 3699 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3700 struct e1000_2tuple_filter_info *filter_info) 3701 { 3702 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3703 return -EINVAL; 3704 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3705 return -EINVAL; /* filter index is out of range. */ 3706 if (filter->tcp_flags > TCP_FLAG_ALL) 3707 return -EINVAL; /* flags is invalid. */ 3708 3709 switch (filter->dst_port_mask) { 3710 case UINT16_MAX: 3711 filter_info->dst_port_mask = 0; 3712 filter_info->dst_port = filter->dst_port; 3713 break; 3714 case 0: 3715 filter_info->dst_port_mask = 1; 3716 break; 3717 default: 3718 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3719 return -EINVAL; 3720 } 3721 3722 switch (filter->proto_mask) { 3723 case UINT8_MAX: 3724 filter_info->proto_mask = 0; 3725 filter_info->proto = filter->proto; 3726 break; 3727 case 0: 3728 filter_info->proto_mask = 1; 3729 break; 3730 default: 3731 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3732 return -EINVAL; 3733 } 3734 3735 filter_info->priority = (uint8_t)filter->priority; 3736 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3737 filter_info->tcp_flags = filter->tcp_flags; 3738 else 3739 filter_info->tcp_flags = 0; 3740 3741 return 0; 3742 } 3743 3744 static inline struct e1000_2tuple_filter * 3745 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3746 struct e1000_2tuple_filter_info *key) 3747 { 3748 struct e1000_2tuple_filter *it; 3749 3750 TAILQ_FOREACH(it, filter_list, entries) { 3751 if (memcmp(key, &it->filter_info, 3752 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3753 return it; 3754 } 3755 } 3756 return NULL; 3757 } 3758 3759 /* inject a igb 2tuple filter to HW */ 3760 static inline void 3761 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3762 struct e1000_2tuple_filter *filter) 3763 { 3764 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3765 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3766 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3767 int i; 3768 3769 i = filter->index; 3770 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3771 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3772 imir |= E1000_IMIR_PORT_BP; 3773 else 3774 imir &= ~E1000_IMIR_PORT_BP; 3775 3776 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3777 3778 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3779 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3780 ttqf |= (uint32_t)(filter->filter_info.proto & 3781 E1000_TTQF_PROTOCOL_MASK); 3782 if (filter->filter_info.proto_mask == 0) 3783 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3784 3785 /* tcp flags bits setting. */ 3786 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 3787 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 3788 imir_ext |= E1000_IMIREXT_CTRL_URG; 3789 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 3790 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3791 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 3792 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3793 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 3794 imir_ext |= E1000_IMIREXT_CTRL_RST; 3795 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 3796 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3797 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 3798 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3799 } else { 3800 imir_ext |= E1000_IMIREXT_CTRL_BP; 3801 } 3802 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3803 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3804 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3805 } 3806 3807 /* 3808 * igb_add_2tuple_filter - add a 2tuple filter 3809 * 3810 * @param 3811 * dev: Pointer to struct rte_eth_dev. 3812 * ntuple_filter: ponter to the filter that will be added. 3813 * 3814 * @return 3815 * - On success, zero. 3816 * - On failure, a negative value. 3817 */ 3818 static int 3819 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3820 struct rte_eth_ntuple_filter *ntuple_filter) 3821 { 3822 struct e1000_filter_info *filter_info = 3823 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3824 struct e1000_2tuple_filter *filter; 3825 int i, ret; 3826 3827 filter = rte_zmalloc("e1000_2tuple_filter", 3828 sizeof(struct e1000_2tuple_filter), 0); 3829 if (filter == NULL) 3830 return -ENOMEM; 3831 3832 ret = ntuple_filter_to_2tuple(ntuple_filter, 3833 &filter->filter_info); 3834 if (ret < 0) { 3835 rte_free(filter); 3836 return ret; 3837 } 3838 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3839 &filter->filter_info) != NULL) { 3840 PMD_DRV_LOG(ERR, "filter exists."); 3841 rte_free(filter); 3842 return -EEXIST; 3843 } 3844 filter->queue = ntuple_filter->queue; 3845 3846 /* 3847 * look for an unused 2tuple filter index, 3848 * and insert the filter to list. 3849 */ 3850 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3851 if (!(filter_info->twotuple_mask & (1 << i))) { 3852 filter_info->twotuple_mask |= 1 << i; 3853 filter->index = i; 3854 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3855 filter, 3856 entries); 3857 break; 3858 } 3859 } 3860 if (i >= E1000_MAX_TTQF_FILTERS) { 3861 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3862 rte_free(filter); 3863 return -ENOSYS; 3864 } 3865 3866 igb_inject_2uple_filter(dev, filter); 3867 return 0; 3868 } 3869 3870 int 3871 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3872 struct e1000_2tuple_filter *filter) 3873 { 3874 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3875 struct e1000_filter_info *filter_info = 3876 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3877 3878 filter_info->twotuple_mask &= ~(1 << filter->index); 3879 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3880 rte_free(filter); 3881 3882 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3883 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3884 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3885 return 0; 3886 } 3887 3888 /* 3889 * igb_remove_2tuple_filter - remove a 2tuple filter 3890 * 3891 * @param 3892 * dev: Pointer to struct rte_eth_dev. 3893 * ntuple_filter: ponter to the filter that will be removed. 3894 * 3895 * @return 3896 * - On success, zero. 3897 * - On failure, a negative value. 3898 */ 3899 static int 3900 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3901 struct rte_eth_ntuple_filter *ntuple_filter) 3902 { 3903 struct e1000_filter_info *filter_info = 3904 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3905 struct e1000_2tuple_filter_info filter_2tuple; 3906 struct e1000_2tuple_filter *filter; 3907 int ret; 3908 3909 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3910 ret = ntuple_filter_to_2tuple(ntuple_filter, 3911 &filter_2tuple); 3912 if (ret < 0) 3913 return ret; 3914 3915 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3916 &filter_2tuple); 3917 if (filter == NULL) { 3918 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3919 return -ENOENT; 3920 } 3921 3922 igb_delete_2tuple_filter(dev, filter); 3923 3924 return 0; 3925 } 3926 3927 /* inject a igb flex filter to HW */ 3928 static inline void 3929 igb_inject_flex_filter(struct rte_eth_dev *dev, 3930 struct e1000_flex_filter *filter) 3931 { 3932 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3933 uint32_t wufc, queueing; 3934 uint32_t reg_off; 3935 uint8_t i, j = 0; 3936 3937 wufc = E1000_READ_REG(hw, E1000_WUFC); 3938 if (filter->index < E1000_MAX_FHFT) 3939 reg_off = E1000_FHFT(filter->index); 3940 else 3941 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3942 3943 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3944 (E1000_WUFC_FLX0 << filter->index)); 3945 queueing = filter->filter_info.len | 3946 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3947 (filter->filter_info.priority << 3948 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3949 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3950 queueing); 3951 3952 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3953 E1000_WRITE_REG(hw, reg_off, 3954 filter->filter_info.dwords[j]); 3955 reg_off += sizeof(uint32_t); 3956 E1000_WRITE_REG(hw, reg_off, 3957 filter->filter_info.dwords[++j]); 3958 reg_off += sizeof(uint32_t); 3959 E1000_WRITE_REG(hw, reg_off, 3960 (uint32_t)filter->filter_info.mask[i]); 3961 reg_off += sizeof(uint32_t) * 2; 3962 ++j; 3963 } 3964 } 3965 3966 static inline struct e1000_flex_filter * 3967 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3968 struct e1000_flex_filter_info *key) 3969 { 3970 struct e1000_flex_filter *it; 3971 3972 TAILQ_FOREACH(it, filter_list, entries) { 3973 if (memcmp(key, &it->filter_info, 3974 sizeof(struct e1000_flex_filter_info)) == 0) 3975 return it; 3976 } 3977 3978 return NULL; 3979 } 3980 3981 /* remove a flex byte filter 3982 * @param 3983 * dev: Pointer to struct rte_eth_dev. 3984 * filter: the pointer of the filter will be removed. 3985 */ 3986 void 3987 igb_remove_flex_filter(struct rte_eth_dev *dev, 3988 struct e1000_flex_filter *filter) 3989 { 3990 struct e1000_filter_info *filter_info = 3991 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3992 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3993 uint32_t wufc, i; 3994 uint32_t reg_off; 3995 3996 wufc = E1000_READ_REG(hw, E1000_WUFC); 3997 if (filter->index < E1000_MAX_FHFT) 3998 reg_off = E1000_FHFT(filter->index); 3999 else 4000 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 4001 4002 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 4003 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 4004 4005 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 4006 (~(E1000_WUFC_FLX0 << filter->index))); 4007 4008 filter_info->flex_mask &= ~(1 << filter->index); 4009 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 4010 rte_free(filter); 4011 } 4012 4013 int 4014 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 4015 struct rte_eth_flex_filter *filter, 4016 bool add) 4017 { 4018 struct e1000_filter_info *filter_info = 4019 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4020 struct e1000_flex_filter *flex_filter, *it; 4021 uint32_t mask; 4022 uint8_t shift, i; 4023 4024 flex_filter = rte_zmalloc("e1000_flex_filter", 4025 sizeof(struct e1000_flex_filter), 0); 4026 if (flex_filter == NULL) 4027 return -ENOMEM; 4028 4029 flex_filter->filter_info.len = filter->len; 4030 flex_filter->filter_info.priority = filter->priority; 4031 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 4032 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 4033 mask = 0; 4034 /* reverse bits in flex filter's mask*/ 4035 for (shift = 0; shift < CHAR_BIT; shift++) { 4036 if (filter->mask[i] & (0x01 << shift)) 4037 mask |= (0x80 >> shift); 4038 } 4039 flex_filter->filter_info.mask[i] = mask; 4040 } 4041 4042 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4043 &flex_filter->filter_info); 4044 if (it == NULL && !add) { 4045 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4046 rte_free(flex_filter); 4047 return -ENOENT; 4048 } 4049 if (it != NULL && add) { 4050 PMD_DRV_LOG(ERR, "filter exists."); 4051 rte_free(flex_filter); 4052 return -EEXIST; 4053 } 4054 4055 if (add) { 4056 flex_filter->queue = filter->queue; 4057 /* 4058 * look for an unused flex filter index 4059 * and insert the filter into the list. 4060 */ 4061 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4062 if (!(filter_info->flex_mask & (1 << i))) { 4063 filter_info->flex_mask |= 1 << i; 4064 flex_filter->index = i; 4065 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4066 flex_filter, 4067 entries); 4068 break; 4069 } 4070 } 4071 if (i >= E1000_MAX_FLEX_FILTERS) { 4072 PMD_DRV_LOG(ERR, "flex filters are full."); 4073 rte_free(flex_filter); 4074 return -ENOSYS; 4075 } 4076 4077 igb_inject_flex_filter(dev, flex_filter); 4078 4079 } else { 4080 igb_remove_flex_filter(dev, it); 4081 rte_free(flex_filter); 4082 } 4083 4084 return 0; 4085 } 4086 4087 static int 4088 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4089 struct rte_eth_flex_filter *filter) 4090 { 4091 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4092 struct e1000_filter_info *filter_info = 4093 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4094 struct e1000_flex_filter flex_filter, *it; 4095 uint32_t wufc, queueing, wufc_en = 0; 4096 4097 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4098 flex_filter.filter_info.len = filter->len; 4099 flex_filter.filter_info.priority = filter->priority; 4100 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4101 memcpy(flex_filter.filter_info.mask, filter->mask, 4102 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4103 4104 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4105 &flex_filter.filter_info); 4106 if (it == NULL) { 4107 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4108 return -ENOENT; 4109 } 4110 4111 wufc = E1000_READ_REG(hw, E1000_WUFC); 4112 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4113 4114 if ((wufc & wufc_en) == wufc_en) { 4115 uint32_t reg_off = 0; 4116 if (it->index < E1000_MAX_FHFT) 4117 reg_off = E1000_FHFT(it->index); 4118 else 4119 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4120 4121 queueing = E1000_READ_REG(hw, 4122 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4123 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4124 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4125 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4126 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4127 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4128 return 0; 4129 } 4130 return -ENOENT; 4131 } 4132 4133 static int 4134 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4135 enum rte_filter_op filter_op, 4136 void *arg) 4137 { 4138 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4139 struct rte_eth_flex_filter *filter; 4140 int ret = 0; 4141 4142 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4143 4144 if (filter_op == RTE_ETH_FILTER_NOP) 4145 return ret; 4146 4147 if (arg == NULL) { 4148 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4149 filter_op); 4150 return -EINVAL; 4151 } 4152 4153 filter = (struct rte_eth_flex_filter *)arg; 4154 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4155 || filter->len % sizeof(uint64_t) != 0) { 4156 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4157 return -EINVAL; 4158 } 4159 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4160 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4161 return -EINVAL; 4162 } 4163 4164 switch (filter_op) { 4165 case RTE_ETH_FILTER_ADD: 4166 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4167 break; 4168 case RTE_ETH_FILTER_DELETE: 4169 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4170 break; 4171 case RTE_ETH_FILTER_GET: 4172 ret = eth_igb_get_flex_filter(dev, filter); 4173 break; 4174 default: 4175 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4176 ret = -EINVAL; 4177 break; 4178 } 4179 4180 return ret; 4181 } 4182 4183 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4184 static inline int 4185 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4186 struct e1000_5tuple_filter_info *filter_info) 4187 { 4188 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4189 return -EINVAL; 4190 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4191 return -EINVAL; /* filter index is out of range. */ 4192 if (filter->tcp_flags > TCP_FLAG_ALL) 4193 return -EINVAL; /* flags is invalid. */ 4194 4195 switch (filter->dst_ip_mask) { 4196 case UINT32_MAX: 4197 filter_info->dst_ip_mask = 0; 4198 filter_info->dst_ip = filter->dst_ip; 4199 break; 4200 case 0: 4201 filter_info->dst_ip_mask = 1; 4202 break; 4203 default: 4204 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4205 return -EINVAL; 4206 } 4207 4208 switch (filter->src_ip_mask) { 4209 case UINT32_MAX: 4210 filter_info->src_ip_mask = 0; 4211 filter_info->src_ip = filter->src_ip; 4212 break; 4213 case 0: 4214 filter_info->src_ip_mask = 1; 4215 break; 4216 default: 4217 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4218 return -EINVAL; 4219 } 4220 4221 switch (filter->dst_port_mask) { 4222 case UINT16_MAX: 4223 filter_info->dst_port_mask = 0; 4224 filter_info->dst_port = filter->dst_port; 4225 break; 4226 case 0: 4227 filter_info->dst_port_mask = 1; 4228 break; 4229 default: 4230 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4231 return -EINVAL; 4232 } 4233 4234 switch (filter->src_port_mask) { 4235 case UINT16_MAX: 4236 filter_info->src_port_mask = 0; 4237 filter_info->src_port = filter->src_port; 4238 break; 4239 case 0: 4240 filter_info->src_port_mask = 1; 4241 break; 4242 default: 4243 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4244 return -EINVAL; 4245 } 4246 4247 switch (filter->proto_mask) { 4248 case UINT8_MAX: 4249 filter_info->proto_mask = 0; 4250 filter_info->proto = filter->proto; 4251 break; 4252 case 0: 4253 filter_info->proto_mask = 1; 4254 break; 4255 default: 4256 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4257 return -EINVAL; 4258 } 4259 4260 filter_info->priority = (uint8_t)filter->priority; 4261 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4262 filter_info->tcp_flags = filter->tcp_flags; 4263 else 4264 filter_info->tcp_flags = 0; 4265 4266 return 0; 4267 } 4268 4269 static inline struct e1000_5tuple_filter * 4270 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4271 struct e1000_5tuple_filter_info *key) 4272 { 4273 struct e1000_5tuple_filter *it; 4274 4275 TAILQ_FOREACH(it, filter_list, entries) { 4276 if (memcmp(key, &it->filter_info, 4277 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4278 return it; 4279 } 4280 } 4281 return NULL; 4282 } 4283 4284 /* inject a igb 5-tuple filter to HW */ 4285 static inline void 4286 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4287 struct e1000_5tuple_filter *filter) 4288 { 4289 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4290 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4291 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4292 uint8_t i; 4293 4294 i = filter->index; 4295 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4296 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4297 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4298 if (filter->filter_info.dst_ip_mask == 0) 4299 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4300 if (filter->filter_info.src_port_mask == 0) 4301 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4302 if (filter->filter_info.proto_mask == 0) 4303 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4304 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4305 E1000_FTQF_QUEUE_MASK; 4306 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4307 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4308 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4309 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4310 4311 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4312 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4313 4314 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4315 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4316 imir |= E1000_IMIR_PORT_BP; 4317 else 4318 imir &= ~E1000_IMIR_PORT_BP; 4319 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4320 4321 /* tcp flags bits setting. */ 4322 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 4323 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 4324 imir_ext |= E1000_IMIREXT_CTRL_URG; 4325 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 4326 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4327 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 4328 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4329 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 4330 imir_ext |= E1000_IMIREXT_CTRL_RST; 4331 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 4332 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4333 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 4334 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4335 } else { 4336 imir_ext |= E1000_IMIREXT_CTRL_BP; 4337 } 4338 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4339 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4340 } 4341 4342 /* 4343 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4344 * 4345 * @param 4346 * dev: Pointer to struct rte_eth_dev. 4347 * ntuple_filter: ponter to the filter that will be added. 4348 * 4349 * @return 4350 * - On success, zero. 4351 * - On failure, a negative value. 4352 */ 4353 static int 4354 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4355 struct rte_eth_ntuple_filter *ntuple_filter) 4356 { 4357 struct e1000_filter_info *filter_info = 4358 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4359 struct e1000_5tuple_filter *filter; 4360 uint8_t i; 4361 int ret; 4362 4363 filter = rte_zmalloc("e1000_5tuple_filter", 4364 sizeof(struct e1000_5tuple_filter), 0); 4365 if (filter == NULL) 4366 return -ENOMEM; 4367 4368 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4369 &filter->filter_info); 4370 if (ret < 0) { 4371 rte_free(filter); 4372 return ret; 4373 } 4374 4375 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4376 &filter->filter_info) != NULL) { 4377 PMD_DRV_LOG(ERR, "filter exists."); 4378 rte_free(filter); 4379 return -EEXIST; 4380 } 4381 filter->queue = ntuple_filter->queue; 4382 4383 /* 4384 * look for an unused 5tuple filter index, 4385 * and insert the filter to list. 4386 */ 4387 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4388 if (!(filter_info->fivetuple_mask & (1 << i))) { 4389 filter_info->fivetuple_mask |= 1 << i; 4390 filter->index = i; 4391 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4392 filter, 4393 entries); 4394 break; 4395 } 4396 } 4397 if (i >= E1000_MAX_FTQF_FILTERS) { 4398 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4399 rte_free(filter); 4400 return -ENOSYS; 4401 } 4402 4403 igb_inject_5tuple_filter_82576(dev, filter); 4404 return 0; 4405 } 4406 4407 int 4408 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4409 struct e1000_5tuple_filter *filter) 4410 { 4411 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4412 struct e1000_filter_info *filter_info = 4413 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4414 4415 filter_info->fivetuple_mask &= ~(1 << filter->index); 4416 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4417 rte_free(filter); 4418 4419 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4420 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4421 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4422 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4423 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4424 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4425 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4426 return 0; 4427 } 4428 4429 /* 4430 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4431 * 4432 * @param 4433 * dev: Pointer to struct rte_eth_dev. 4434 * ntuple_filter: ponter to the filter that will be removed. 4435 * 4436 * @return 4437 * - On success, zero. 4438 * - On failure, a negative value. 4439 */ 4440 static int 4441 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4442 struct rte_eth_ntuple_filter *ntuple_filter) 4443 { 4444 struct e1000_filter_info *filter_info = 4445 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4446 struct e1000_5tuple_filter_info filter_5tuple; 4447 struct e1000_5tuple_filter *filter; 4448 int ret; 4449 4450 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4451 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4452 &filter_5tuple); 4453 if (ret < 0) 4454 return ret; 4455 4456 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4457 &filter_5tuple); 4458 if (filter == NULL) { 4459 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4460 return -ENOENT; 4461 } 4462 4463 igb_delete_5tuple_filter_82576(dev, filter); 4464 4465 return 0; 4466 } 4467 4468 static int 4469 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4470 { 4471 uint32_t rctl; 4472 struct e1000_hw *hw; 4473 struct rte_eth_dev_info dev_info; 4474 uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4475 VLAN_TAG_SIZE); 4476 4477 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4478 4479 #ifdef RTE_LIBRTE_82571_SUPPORT 4480 /* XXX: not bigger than max_rx_pktlen */ 4481 if (hw->mac.type == e1000_82571) 4482 return -ENOTSUP; 4483 #endif 4484 eth_igb_infos_get(dev, &dev_info); 4485 4486 /* check that mtu is within the allowed range */ 4487 if ((mtu < ETHER_MIN_MTU) || 4488 (frame_size > dev_info.max_rx_pktlen)) 4489 return -EINVAL; 4490 4491 /* refuse mtu that requires the support of scattered packets when this 4492 * feature has not been enabled before. */ 4493 if (!dev->data->scattered_rx && 4494 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4495 return -EINVAL; 4496 4497 rctl = E1000_READ_REG(hw, E1000_RCTL); 4498 4499 /* switch to jumbo mode if needed */ 4500 if (frame_size > ETHER_MAX_LEN) { 4501 dev->data->dev_conf.rxmode.jumbo_frame = 1; 4502 rctl |= E1000_RCTL_LPE; 4503 } else { 4504 dev->data->dev_conf.rxmode.jumbo_frame = 0; 4505 rctl &= ~E1000_RCTL_LPE; 4506 } 4507 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4508 4509 /* update max frame size */ 4510 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4511 4512 E1000_WRITE_REG(hw, E1000_RLPML, 4513 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4514 4515 return 0; 4516 } 4517 4518 /* 4519 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4520 * 4521 * @param 4522 * dev: Pointer to struct rte_eth_dev. 4523 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4524 * add: if true, add filter, if false, remove filter 4525 * 4526 * @return 4527 * - On success, zero. 4528 * - On failure, a negative value. 4529 */ 4530 int 4531 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4532 struct rte_eth_ntuple_filter *ntuple_filter, 4533 bool add) 4534 { 4535 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4536 int ret; 4537 4538 switch (ntuple_filter->flags) { 4539 case RTE_5TUPLE_FLAGS: 4540 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4541 if (hw->mac.type != e1000_82576) 4542 return -ENOTSUP; 4543 if (add) 4544 ret = igb_add_5tuple_filter_82576(dev, 4545 ntuple_filter); 4546 else 4547 ret = igb_remove_5tuple_filter_82576(dev, 4548 ntuple_filter); 4549 break; 4550 case RTE_2TUPLE_FLAGS: 4551 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4552 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4553 hw->mac.type != e1000_i210 && 4554 hw->mac.type != e1000_i211) 4555 return -ENOTSUP; 4556 if (add) 4557 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4558 else 4559 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4560 break; 4561 default: 4562 ret = -EINVAL; 4563 break; 4564 } 4565 4566 return ret; 4567 } 4568 4569 /* 4570 * igb_get_ntuple_filter - get a ntuple filter 4571 * 4572 * @param 4573 * dev: Pointer to struct rte_eth_dev. 4574 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4575 * 4576 * @return 4577 * - On success, zero. 4578 * - On failure, a negative value. 4579 */ 4580 static int 4581 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4582 struct rte_eth_ntuple_filter *ntuple_filter) 4583 { 4584 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4585 struct e1000_filter_info *filter_info = 4586 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4587 struct e1000_5tuple_filter_info filter_5tuple; 4588 struct e1000_2tuple_filter_info filter_2tuple; 4589 struct e1000_5tuple_filter *p_5tuple_filter; 4590 struct e1000_2tuple_filter *p_2tuple_filter; 4591 int ret; 4592 4593 switch (ntuple_filter->flags) { 4594 case RTE_5TUPLE_FLAGS: 4595 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4596 if (hw->mac.type != e1000_82576) 4597 return -ENOTSUP; 4598 memset(&filter_5tuple, 4599 0, 4600 sizeof(struct e1000_5tuple_filter_info)); 4601 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4602 &filter_5tuple); 4603 if (ret < 0) 4604 return ret; 4605 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4606 &filter_info->fivetuple_list, 4607 &filter_5tuple); 4608 if (p_5tuple_filter == NULL) { 4609 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4610 return -ENOENT; 4611 } 4612 ntuple_filter->queue = p_5tuple_filter->queue; 4613 break; 4614 case RTE_2TUPLE_FLAGS: 4615 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4616 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4617 return -ENOTSUP; 4618 memset(&filter_2tuple, 4619 0, 4620 sizeof(struct e1000_2tuple_filter_info)); 4621 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4622 if (ret < 0) 4623 return ret; 4624 p_2tuple_filter = igb_2tuple_filter_lookup( 4625 &filter_info->twotuple_list, 4626 &filter_2tuple); 4627 if (p_2tuple_filter == NULL) { 4628 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4629 return -ENOENT; 4630 } 4631 ntuple_filter->queue = p_2tuple_filter->queue; 4632 break; 4633 default: 4634 ret = -EINVAL; 4635 break; 4636 } 4637 4638 return 0; 4639 } 4640 4641 /* 4642 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4643 * @dev: pointer to rte_eth_dev structure 4644 * @filter_op:operation will be taken. 4645 * @arg: a pointer to specific structure corresponding to the filter_op 4646 */ 4647 static int 4648 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4649 enum rte_filter_op filter_op, 4650 void *arg) 4651 { 4652 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4653 int ret; 4654 4655 MAC_TYPE_FILTER_SUP(hw->mac.type); 4656 4657 if (filter_op == RTE_ETH_FILTER_NOP) 4658 return 0; 4659 4660 if (arg == NULL) { 4661 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4662 filter_op); 4663 return -EINVAL; 4664 } 4665 4666 switch (filter_op) { 4667 case RTE_ETH_FILTER_ADD: 4668 ret = igb_add_del_ntuple_filter(dev, 4669 (struct rte_eth_ntuple_filter *)arg, 4670 TRUE); 4671 break; 4672 case RTE_ETH_FILTER_DELETE: 4673 ret = igb_add_del_ntuple_filter(dev, 4674 (struct rte_eth_ntuple_filter *)arg, 4675 FALSE); 4676 break; 4677 case RTE_ETH_FILTER_GET: 4678 ret = igb_get_ntuple_filter(dev, 4679 (struct rte_eth_ntuple_filter *)arg); 4680 break; 4681 default: 4682 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4683 ret = -EINVAL; 4684 break; 4685 } 4686 return ret; 4687 } 4688 4689 static inline int 4690 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4691 uint16_t ethertype) 4692 { 4693 int i; 4694 4695 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4696 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4697 (filter_info->ethertype_mask & (1 << i))) 4698 return i; 4699 } 4700 return -1; 4701 } 4702 4703 static inline int 4704 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4705 uint16_t ethertype, uint32_t etqf) 4706 { 4707 int i; 4708 4709 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4710 if (!(filter_info->ethertype_mask & (1 << i))) { 4711 filter_info->ethertype_mask |= 1 << i; 4712 filter_info->ethertype_filters[i].ethertype = ethertype; 4713 filter_info->ethertype_filters[i].etqf = etqf; 4714 return i; 4715 } 4716 } 4717 return -1; 4718 } 4719 4720 int 4721 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4722 uint8_t idx) 4723 { 4724 if (idx >= E1000_MAX_ETQF_FILTERS) 4725 return -1; 4726 filter_info->ethertype_mask &= ~(1 << idx); 4727 filter_info->ethertype_filters[idx].ethertype = 0; 4728 filter_info->ethertype_filters[idx].etqf = 0; 4729 return idx; 4730 } 4731 4732 4733 int 4734 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4735 struct rte_eth_ethertype_filter *filter, 4736 bool add) 4737 { 4738 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4739 struct e1000_filter_info *filter_info = 4740 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4741 uint32_t etqf = 0; 4742 int ret; 4743 4744 if (filter->ether_type == ETHER_TYPE_IPv4 || 4745 filter->ether_type == ETHER_TYPE_IPv6) { 4746 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4747 " ethertype filter.", filter->ether_type); 4748 return -EINVAL; 4749 } 4750 4751 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4752 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4753 return -EINVAL; 4754 } 4755 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4756 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4757 return -EINVAL; 4758 } 4759 4760 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4761 if (ret >= 0 && add) { 4762 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4763 filter->ether_type); 4764 return -EEXIST; 4765 } 4766 if (ret < 0 && !add) { 4767 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4768 filter->ether_type); 4769 return -ENOENT; 4770 } 4771 4772 if (add) { 4773 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4774 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4775 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4776 ret = igb_ethertype_filter_insert(filter_info, 4777 filter->ether_type, etqf); 4778 if (ret < 0) { 4779 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4780 return -ENOSYS; 4781 } 4782 } else { 4783 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4784 if (ret < 0) 4785 return -ENOSYS; 4786 } 4787 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4788 E1000_WRITE_FLUSH(hw); 4789 4790 return 0; 4791 } 4792 4793 static int 4794 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4795 struct rte_eth_ethertype_filter *filter) 4796 { 4797 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4798 struct e1000_filter_info *filter_info = 4799 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4800 uint32_t etqf; 4801 int ret; 4802 4803 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4804 if (ret < 0) { 4805 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4806 filter->ether_type); 4807 return -ENOENT; 4808 } 4809 4810 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4811 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4812 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4813 filter->flags = 0; 4814 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4815 E1000_ETQF_QUEUE_SHIFT; 4816 return 0; 4817 } 4818 4819 return -ENOENT; 4820 } 4821 4822 /* 4823 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4824 * @dev: pointer to rte_eth_dev structure 4825 * @filter_op:operation will be taken. 4826 * @arg: a pointer to specific structure corresponding to the filter_op 4827 */ 4828 static int 4829 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4830 enum rte_filter_op filter_op, 4831 void *arg) 4832 { 4833 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4834 int ret; 4835 4836 MAC_TYPE_FILTER_SUP(hw->mac.type); 4837 4838 if (filter_op == RTE_ETH_FILTER_NOP) 4839 return 0; 4840 4841 if (arg == NULL) { 4842 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4843 filter_op); 4844 return -EINVAL; 4845 } 4846 4847 switch (filter_op) { 4848 case RTE_ETH_FILTER_ADD: 4849 ret = igb_add_del_ethertype_filter(dev, 4850 (struct rte_eth_ethertype_filter *)arg, 4851 TRUE); 4852 break; 4853 case RTE_ETH_FILTER_DELETE: 4854 ret = igb_add_del_ethertype_filter(dev, 4855 (struct rte_eth_ethertype_filter *)arg, 4856 FALSE); 4857 break; 4858 case RTE_ETH_FILTER_GET: 4859 ret = igb_get_ethertype_filter(dev, 4860 (struct rte_eth_ethertype_filter *)arg); 4861 break; 4862 default: 4863 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4864 ret = -EINVAL; 4865 break; 4866 } 4867 return ret; 4868 } 4869 4870 static int 4871 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4872 enum rte_filter_type filter_type, 4873 enum rte_filter_op filter_op, 4874 void *arg) 4875 { 4876 int ret = 0; 4877 4878 switch (filter_type) { 4879 case RTE_ETH_FILTER_NTUPLE: 4880 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4881 break; 4882 case RTE_ETH_FILTER_ETHERTYPE: 4883 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4884 break; 4885 case RTE_ETH_FILTER_SYN: 4886 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4887 break; 4888 case RTE_ETH_FILTER_FLEXIBLE: 4889 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4890 break; 4891 case RTE_ETH_FILTER_GENERIC: 4892 if (filter_op != RTE_ETH_FILTER_GET) 4893 return -EINVAL; 4894 *(const void **)arg = &igb_flow_ops; 4895 break; 4896 default: 4897 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4898 filter_type); 4899 break; 4900 } 4901 4902 return ret; 4903 } 4904 4905 static int 4906 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4907 struct ether_addr *mc_addr_set, 4908 uint32_t nb_mc_addr) 4909 { 4910 struct e1000_hw *hw; 4911 4912 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4913 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4914 return 0; 4915 } 4916 4917 static uint64_t 4918 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4919 { 4920 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4921 uint64_t systime_cycles; 4922 4923 switch (hw->mac.type) { 4924 case e1000_i210: 4925 case e1000_i211: 4926 /* 4927 * Need to read System Time Residue Register to be able 4928 * to read the other two registers. 4929 */ 4930 E1000_READ_REG(hw, E1000_SYSTIMR); 4931 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4932 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4933 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4934 * NSEC_PER_SEC; 4935 break; 4936 case e1000_82580: 4937 case e1000_i350: 4938 case e1000_i354: 4939 /* 4940 * Need to read System Time Residue Register to be able 4941 * to read the other two registers. 4942 */ 4943 E1000_READ_REG(hw, E1000_SYSTIMR); 4944 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4945 /* Only the 8 LSB are valid. */ 4946 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4947 & 0xff) << 32; 4948 break; 4949 default: 4950 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4951 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4952 << 32; 4953 break; 4954 } 4955 4956 return systime_cycles; 4957 } 4958 4959 static uint64_t 4960 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4961 { 4962 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4963 uint64_t rx_tstamp_cycles; 4964 4965 switch (hw->mac.type) { 4966 case e1000_i210: 4967 case e1000_i211: 4968 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4969 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4970 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4971 * NSEC_PER_SEC; 4972 break; 4973 case e1000_82580: 4974 case e1000_i350: 4975 case e1000_i354: 4976 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4977 /* Only the 8 LSB are valid. */ 4978 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4979 & 0xff) << 32; 4980 break; 4981 default: 4982 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4983 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4984 << 32; 4985 break; 4986 } 4987 4988 return rx_tstamp_cycles; 4989 } 4990 4991 static uint64_t 4992 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4993 { 4994 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4995 uint64_t tx_tstamp_cycles; 4996 4997 switch (hw->mac.type) { 4998 case e1000_i210: 4999 case e1000_i211: 5000 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 5001 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5002 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5003 * NSEC_PER_SEC; 5004 break; 5005 case e1000_82580: 5006 case e1000_i350: 5007 case e1000_i354: 5008 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5009 /* Only the 8 LSB are valid. */ 5010 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 5011 & 0xff) << 32; 5012 break; 5013 default: 5014 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 5015 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 5016 << 32; 5017 break; 5018 } 5019 5020 return tx_tstamp_cycles; 5021 } 5022 5023 static void 5024 igb_start_timecounters(struct rte_eth_dev *dev) 5025 { 5026 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5027 struct e1000_adapter *adapter = 5028 (struct e1000_adapter *)dev->data->dev_private; 5029 uint32_t incval = 1; 5030 uint32_t shift = 0; 5031 uint64_t mask = E1000_CYCLECOUNTER_MASK; 5032 5033 switch (hw->mac.type) { 5034 case e1000_82580: 5035 case e1000_i350: 5036 case e1000_i354: 5037 /* 32 LSB bits + 8 MSB bits = 40 bits */ 5038 mask = (1ULL << 40) - 1; 5039 /* fall-through */ 5040 case e1000_i210: 5041 case e1000_i211: 5042 /* 5043 * Start incrementing the register 5044 * used to timestamp PTP packets. 5045 */ 5046 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 5047 break; 5048 case e1000_82576: 5049 incval = E1000_INCVALUE_82576; 5050 shift = IGB_82576_TSYNC_SHIFT; 5051 E1000_WRITE_REG(hw, E1000_TIMINCA, 5052 E1000_INCPERIOD_82576 | incval); 5053 break; 5054 default: 5055 /* Not supported */ 5056 return; 5057 } 5058 5059 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5060 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5061 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5062 5063 adapter->systime_tc.cc_mask = mask; 5064 adapter->systime_tc.cc_shift = shift; 5065 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5066 5067 adapter->rx_tstamp_tc.cc_mask = mask; 5068 adapter->rx_tstamp_tc.cc_shift = shift; 5069 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5070 5071 adapter->tx_tstamp_tc.cc_mask = mask; 5072 adapter->tx_tstamp_tc.cc_shift = shift; 5073 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5074 } 5075 5076 static int 5077 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5078 { 5079 struct e1000_adapter *adapter = 5080 (struct e1000_adapter *)dev->data->dev_private; 5081 5082 adapter->systime_tc.nsec += delta; 5083 adapter->rx_tstamp_tc.nsec += delta; 5084 adapter->tx_tstamp_tc.nsec += delta; 5085 5086 return 0; 5087 } 5088 5089 static int 5090 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5091 { 5092 uint64_t ns; 5093 struct e1000_adapter *adapter = 5094 (struct e1000_adapter *)dev->data->dev_private; 5095 5096 ns = rte_timespec_to_ns(ts); 5097 5098 /* Set the timecounters to a new value. */ 5099 adapter->systime_tc.nsec = ns; 5100 adapter->rx_tstamp_tc.nsec = ns; 5101 adapter->tx_tstamp_tc.nsec = ns; 5102 5103 return 0; 5104 } 5105 5106 static int 5107 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5108 { 5109 uint64_t ns, systime_cycles; 5110 struct e1000_adapter *adapter = 5111 (struct e1000_adapter *)dev->data->dev_private; 5112 5113 systime_cycles = igb_read_systime_cyclecounter(dev); 5114 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5115 *ts = rte_ns_to_timespec(ns); 5116 5117 return 0; 5118 } 5119 5120 static int 5121 igb_timesync_enable(struct rte_eth_dev *dev) 5122 { 5123 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5124 uint32_t tsync_ctl; 5125 uint32_t tsauxc; 5126 5127 /* Stop the timesync system time. */ 5128 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5129 /* Reset the timesync system time value. */ 5130 switch (hw->mac.type) { 5131 case e1000_82580: 5132 case e1000_i350: 5133 case e1000_i354: 5134 case e1000_i210: 5135 case e1000_i211: 5136 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5137 /* fall-through */ 5138 case e1000_82576: 5139 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5140 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5141 break; 5142 default: 5143 /* Not supported. */ 5144 return -ENOTSUP; 5145 } 5146 5147 /* Enable system time for it isn't on by default. */ 5148 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5149 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5150 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5151 5152 igb_start_timecounters(dev); 5153 5154 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5155 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5156 (ETHER_TYPE_1588 | 5157 E1000_ETQF_FILTER_ENABLE | 5158 E1000_ETQF_1588)); 5159 5160 /* Enable timestamping of received PTP packets. */ 5161 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5162 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5163 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5164 5165 /* Enable Timestamping of transmitted PTP packets. */ 5166 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5167 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5168 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5169 5170 return 0; 5171 } 5172 5173 static int 5174 igb_timesync_disable(struct rte_eth_dev *dev) 5175 { 5176 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5177 uint32_t tsync_ctl; 5178 5179 /* Disable timestamping of transmitted PTP packets. */ 5180 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5181 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5182 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5183 5184 /* Disable timestamping of received PTP packets. */ 5185 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5186 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5187 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5188 5189 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5190 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5191 5192 /* Stop incrementating the System Time registers. */ 5193 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5194 5195 return 0; 5196 } 5197 5198 static int 5199 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5200 struct timespec *timestamp, 5201 uint32_t flags __rte_unused) 5202 { 5203 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5204 struct e1000_adapter *adapter = 5205 (struct e1000_adapter *)dev->data->dev_private; 5206 uint32_t tsync_rxctl; 5207 uint64_t rx_tstamp_cycles; 5208 uint64_t ns; 5209 5210 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5211 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5212 return -EINVAL; 5213 5214 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5215 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5216 *timestamp = rte_ns_to_timespec(ns); 5217 5218 return 0; 5219 } 5220 5221 static int 5222 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5223 struct timespec *timestamp) 5224 { 5225 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5226 struct e1000_adapter *adapter = 5227 (struct e1000_adapter *)dev->data->dev_private; 5228 uint32_t tsync_txctl; 5229 uint64_t tx_tstamp_cycles; 5230 uint64_t ns; 5231 5232 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5233 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5234 return -EINVAL; 5235 5236 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5237 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5238 *timestamp = rte_ns_to_timespec(ns); 5239 5240 return 0; 5241 } 5242 5243 static int 5244 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5245 { 5246 int count = 0; 5247 int g_ind = 0; 5248 const struct reg_info *reg_group; 5249 5250 while ((reg_group = igb_regs[g_ind++])) 5251 count += igb_reg_group_count(reg_group); 5252 5253 return count; 5254 } 5255 5256 static int 5257 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5258 { 5259 int count = 0; 5260 int g_ind = 0; 5261 const struct reg_info *reg_group; 5262 5263 while ((reg_group = igbvf_regs[g_ind++])) 5264 count += igb_reg_group_count(reg_group); 5265 5266 return count; 5267 } 5268 5269 static int 5270 eth_igb_get_regs(struct rte_eth_dev *dev, 5271 struct rte_dev_reg_info *regs) 5272 { 5273 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5274 uint32_t *data = regs->data; 5275 int g_ind = 0; 5276 int count = 0; 5277 const struct reg_info *reg_group; 5278 5279 if (data == NULL) { 5280 regs->length = eth_igb_get_reg_length(dev); 5281 regs->width = sizeof(uint32_t); 5282 return 0; 5283 } 5284 5285 /* Support only full register dump */ 5286 if ((regs->length == 0) || 5287 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5288 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5289 hw->device_id; 5290 while ((reg_group = igb_regs[g_ind++])) 5291 count += igb_read_regs_group(dev, &data[count], 5292 reg_group); 5293 return 0; 5294 } 5295 5296 return -ENOTSUP; 5297 } 5298 5299 static int 5300 igbvf_get_regs(struct rte_eth_dev *dev, 5301 struct rte_dev_reg_info *regs) 5302 { 5303 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5304 uint32_t *data = regs->data; 5305 int g_ind = 0; 5306 int count = 0; 5307 const struct reg_info *reg_group; 5308 5309 if (data == NULL) { 5310 regs->length = igbvf_get_reg_length(dev); 5311 regs->width = sizeof(uint32_t); 5312 return 0; 5313 } 5314 5315 /* Support only full register dump */ 5316 if ((regs->length == 0) || 5317 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5318 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5319 hw->device_id; 5320 while ((reg_group = igbvf_regs[g_ind++])) 5321 count += igb_read_regs_group(dev, &data[count], 5322 reg_group); 5323 return 0; 5324 } 5325 5326 return -ENOTSUP; 5327 } 5328 5329 static int 5330 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5331 { 5332 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5333 5334 /* Return unit is byte count */ 5335 return hw->nvm.word_size * 2; 5336 } 5337 5338 static int 5339 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5340 struct rte_dev_eeprom_info *in_eeprom) 5341 { 5342 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5343 struct e1000_nvm_info *nvm = &hw->nvm; 5344 uint16_t *data = in_eeprom->data; 5345 int first, length; 5346 5347 first = in_eeprom->offset >> 1; 5348 length = in_eeprom->length >> 1; 5349 if ((first >= hw->nvm.word_size) || 5350 ((first + length) >= hw->nvm.word_size)) 5351 return -EINVAL; 5352 5353 in_eeprom->magic = hw->vendor_id | 5354 ((uint32_t)hw->device_id << 16); 5355 5356 if ((nvm->ops.read) == NULL) 5357 return -ENOTSUP; 5358 5359 return nvm->ops.read(hw, first, length, data); 5360 } 5361 5362 static int 5363 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5364 struct rte_dev_eeprom_info *in_eeprom) 5365 { 5366 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5367 struct e1000_nvm_info *nvm = &hw->nvm; 5368 uint16_t *data = in_eeprom->data; 5369 int first, length; 5370 5371 first = in_eeprom->offset >> 1; 5372 length = in_eeprom->length >> 1; 5373 if ((first >= hw->nvm.word_size) || 5374 ((first + length) >= hw->nvm.word_size)) 5375 return -EINVAL; 5376 5377 in_eeprom->magic = (uint32_t)hw->vendor_id | 5378 ((uint32_t)hw->device_id << 16); 5379 5380 if ((nvm->ops.write) == NULL) 5381 return -ENOTSUP; 5382 return nvm->ops.write(hw, first, length, data); 5383 } 5384 5385 static int 5386 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5387 { 5388 struct e1000_hw *hw = 5389 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5390 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5391 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5392 uint32_t vec = E1000_MISC_VEC_ID; 5393 5394 if (rte_intr_allow_others(intr_handle)) 5395 vec = E1000_RX_VEC_START; 5396 5397 uint32_t mask = 1 << (queue_id + vec); 5398 5399 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5400 E1000_WRITE_FLUSH(hw); 5401 5402 return 0; 5403 } 5404 5405 static int 5406 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5407 { 5408 struct e1000_hw *hw = 5409 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5410 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5411 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5412 uint32_t vec = E1000_MISC_VEC_ID; 5413 5414 if (rte_intr_allow_others(intr_handle)) 5415 vec = E1000_RX_VEC_START; 5416 5417 uint32_t mask = 1 << (queue_id + vec); 5418 uint32_t regval; 5419 5420 regval = E1000_READ_REG(hw, E1000_EIMS); 5421 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5422 E1000_WRITE_FLUSH(hw); 5423 5424 rte_intr_enable(intr_handle); 5425 5426 return 0; 5427 } 5428 5429 static void 5430 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5431 uint8_t index, uint8_t offset) 5432 { 5433 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5434 5435 /* clear bits */ 5436 val &= ~((uint32_t)0xFF << offset); 5437 5438 /* write vector and valid bit */ 5439 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5440 5441 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5442 } 5443 5444 static void 5445 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5446 uint8_t queue, uint8_t msix_vector) 5447 { 5448 uint32_t tmp = 0; 5449 5450 if (hw->mac.type == e1000_82575) { 5451 if (direction == 0) 5452 tmp = E1000_EICR_RX_QUEUE0 << queue; 5453 else if (direction == 1) 5454 tmp = E1000_EICR_TX_QUEUE0 << queue; 5455 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5456 } else if (hw->mac.type == e1000_82576) { 5457 if ((direction == 0) || (direction == 1)) 5458 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5459 ((queue & 0x8) << 1) + 5460 8 * direction); 5461 } else if ((hw->mac.type == e1000_82580) || 5462 (hw->mac.type == e1000_i350) || 5463 (hw->mac.type == e1000_i354) || 5464 (hw->mac.type == e1000_i210) || 5465 (hw->mac.type == e1000_i211)) { 5466 if ((direction == 0) || (direction == 1)) 5467 eth_igb_write_ivar(hw, msix_vector, 5468 queue >> 1, 5469 ((queue & 0x1) << 4) + 5470 8 * direction); 5471 } 5472 } 5473 5474 /* Sets up the hardware to generate MSI-X interrupts properly 5475 * @hw 5476 * board private structure 5477 */ 5478 static void 5479 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5480 { 5481 int queue_id; 5482 uint32_t tmpval, regval, intr_mask; 5483 struct e1000_hw *hw = 5484 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5485 uint32_t vec = E1000_MISC_VEC_ID; 5486 uint32_t base = E1000_MISC_VEC_ID; 5487 uint32_t misc_shift = 0; 5488 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5489 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5490 5491 /* won't configure msix register if no mapping is done 5492 * between intr vector and event fd 5493 */ 5494 if (!rte_intr_dp_is_en(intr_handle)) 5495 return; 5496 5497 if (rte_intr_allow_others(intr_handle)) { 5498 vec = base = E1000_RX_VEC_START; 5499 misc_shift = 1; 5500 } 5501 5502 /* set interrupt vector for other causes */ 5503 if (hw->mac.type == e1000_82575) { 5504 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5505 /* enable MSI-X PBA support */ 5506 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5507 5508 /* Auto-Mask interrupts upon ICR read */ 5509 tmpval |= E1000_CTRL_EXT_EIAME; 5510 tmpval |= E1000_CTRL_EXT_IRCA; 5511 5512 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5513 5514 /* enable msix_other interrupt */ 5515 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5516 regval = E1000_READ_REG(hw, E1000_EIAC); 5517 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5518 regval = E1000_READ_REG(hw, E1000_EIAM); 5519 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5520 } else if ((hw->mac.type == e1000_82576) || 5521 (hw->mac.type == e1000_82580) || 5522 (hw->mac.type == e1000_i350) || 5523 (hw->mac.type == e1000_i354) || 5524 (hw->mac.type == e1000_i210) || 5525 (hw->mac.type == e1000_i211)) { 5526 /* turn on MSI-X capability first */ 5527 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5528 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5529 E1000_GPIE_NSICR); 5530 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5531 misc_shift; 5532 regval = E1000_READ_REG(hw, E1000_EIAC); 5533 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5534 5535 /* enable msix_other interrupt */ 5536 regval = E1000_READ_REG(hw, E1000_EIMS); 5537 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5538 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8; 5539 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5540 } 5541 5542 /* use EIAM to auto-mask when MSI-X interrupt 5543 * is asserted, this saves a register write for every interrupt 5544 */ 5545 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5546 misc_shift; 5547 regval = E1000_READ_REG(hw, E1000_EIAM); 5548 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5549 5550 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5551 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5552 intr_handle->intr_vec[queue_id] = vec; 5553 if (vec < base + intr_handle->nb_efd - 1) 5554 vec++; 5555 } 5556 5557 E1000_WRITE_FLUSH(hw); 5558 } 5559 5560 /* restore n-tuple filter */ 5561 static inline void 5562 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5563 { 5564 struct e1000_filter_info *filter_info = 5565 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5566 struct e1000_5tuple_filter *p_5tuple; 5567 struct e1000_2tuple_filter *p_2tuple; 5568 5569 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5570 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5571 } 5572 5573 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5574 igb_inject_2uple_filter(dev, p_2tuple); 5575 } 5576 } 5577 5578 /* restore SYN filter */ 5579 static inline void 5580 igb_syn_filter_restore(struct rte_eth_dev *dev) 5581 { 5582 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5583 struct e1000_filter_info *filter_info = 5584 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5585 uint32_t synqf; 5586 5587 synqf = filter_info->syn_info; 5588 5589 if (synqf & E1000_SYN_FILTER_ENABLE) { 5590 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5591 E1000_WRITE_FLUSH(hw); 5592 } 5593 } 5594 5595 /* restore ethernet type filter */ 5596 static inline void 5597 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5598 { 5599 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5600 struct e1000_filter_info *filter_info = 5601 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5602 int i; 5603 5604 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5605 if (filter_info->ethertype_mask & (1 << i)) { 5606 E1000_WRITE_REG(hw, E1000_ETQF(i), 5607 filter_info->ethertype_filters[i].etqf); 5608 E1000_WRITE_FLUSH(hw); 5609 } 5610 } 5611 } 5612 5613 /* restore flex byte filter */ 5614 static inline void 5615 igb_flex_filter_restore(struct rte_eth_dev *dev) 5616 { 5617 struct e1000_filter_info *filter_info = 5618 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5619 struct e1000_flex_filter *flex_filter; 5620 5621 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5622 igb_inject_flex_filter(dev, flex_filter); 5623 } 5624 } 5625 5626 /* restore rss filter */ 5627 static inline void 5628 igb_rss_filter_restore(struct rte_eth_dev *dev) 5629 { 5630 struct e1000_filter_info *filter_info = 5631 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5632 5633 if (filter_info->rss_info.num) 5634 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5635 } 5636 5637 /* restore all types filter */ 5638 static int 5639 igb_filter_restore(struct rte_eth_dev *dev) 5640 { 5641 igb_ntuple_filter_restore(dev); 5642 igb_ethertype_filter_restore(dev); 5643 igb_syn_filter_restore(dev); 5644 igb_flex_filter_restore(dev); 5645 igb_rss_filter_restore(dev); 5646 5647 return 0; 5648 } 5649 5650 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5651 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5652 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5653 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5654 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5655 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5656