1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <stdarg.h> 10 11 #include <rte_common.h> 12 #include <rte_interrupts.h> 13 #include <rte_byteorder.h> 14 #include <rte_log.h> 15 #include <rte_debug.h> 16 #include <rte_pci.h> 17 #include <rte_bus_pci.h> 18 #include <rte_ether.h> 19 #include <rte_ethdev_driver.h> 20 #include <rte_ethdev_pci.h> 21 #include <rte_memory.h> 22 #include <rte_eal.h> 23 #include <rte_malloc.h> 24 #include <rte_dev.h> 25 26 #include "e1000_logs.h" 27 #include "base/e1000_api.h" 28 #include "e1000_ethdev.h" 29 #include "igb_regs.h" 30 31 /* 32 * Default values for port configuration 33 */ 34 #define IGB_DEFAULT_RX_FREE_THRESH 32 35 36 #define IGB_DEFAULT_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : 8) 37 #define IGB_DEFAULT_RX_HTHRESH 8 38 #define IGB_DEFAULT_RX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 4) 39 40 #define IGB_DEFAULT_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 41 #define IGB_DEFAULT_TX_HTHRESH 1 42 #define IGB_DEFAULT_TX_WTHRESH ((hw->mac.type == e1000_82576) ? 1 : 16) 43 44 #define IGB_HKEY_MAX_INDEX 10 45 46 /* Bit shift and mask */ 47 #define IGB_4_BIT_WIDTH (CHAR_BIT / 2) 48 #define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) 49 #define IGB_8_BIT_WIDTH CHAR_BIT 50 #define IGB_8_BIT_MASK UINT8_MAX 51 52 /* Additional timesync values. */ 53 #define E1000_CYCLECOUNTER_MASK 0xffffffffffffffffULL 54 #define E1000_ETQF_FILTER_1588 3 55 #define IGB_82576_TSYNC_SHIFT 16 56 #define E1000_INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) 57 #define E1000_INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) 58 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000 59 60 #define E1000_VTIVAR_MISC 0x01740 61 #define E1000_VTIVAR_MISC_MASK 0xFF 62 #define E1000_VTIVAR_VALID 0x80 63 #define E1000_VTIVAR_MISC_MAILBOX 0 64 #define E1000_VTIVAR_MISC_INTR_MASK 0x3 65 66 /* External VLAN Enable bit mask */ 67 #define E1000_CTRL_EXT_EXT_VLAN (1 << 26) 68 69 /* External VLAN Ether Type bit mask and shift */ 70 #define E1000_VET_VET_EXT 0xFFFF0000 71 #define E1000_VET_VET_EXT_SHIFT 16 72 73 static int eth_igb_configure(struct rte_eth_dev *dev); 74 static int eth_igb_start(struct rte_eth_dev *dev); 75 static void eth_igb_stop(struct rte_eth_dev *dev); 76 static int eth_igb_dev_set_link_up(struct rte_eth_dev *dev); 77 static int eth_igb_dev_set_link_down(struct rte_eth_dev *dev); 78 static void eth_igb_close(struct rte_eth_dev *dev); 79 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev); 80 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev); 81 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); 82 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); 83 static int eth_igb_link_update(struct rte_eth_dev *dev, 84 int wait_to_complete); 85 static int eth_igb_stats_get(struct rte_eth_dev *dev, 86 struct rte_eth_stats *rte_stats); 87 static int eth_igb_xstats_get(struct rte_eth_dev *dev, 88 struct rte_eth_xstat *xstats, unsigned n); 89 static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, 90 const uint64_t *ids, 91 uint64_t *values, unsigned int n); 92 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, 93 struct rte_eth_xstat_name *xstats_names, 94 unsigned int size); 95 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 96 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 97 unsigned int limit); 98 static void eth_igb_stats_reset(struct rte_eth_dev *dev); 99 static void eth_igb_xstats_reset(struct rte_eth_dev *dev); 100 static int eth_igb_fw_version_get(struct rte_eth_dev *dev, 101 char *fw_version, size_t fw_size); 102 static void eth_igb_infos_get(struct rte_eth_dev *dev, 103 struct rte_eth_dev_info *dev_info); 104 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); 105 static void eth_igbvf_infos_get(struct rte_eth_dev *dev, 106 struct rte_eth_dev_info *dev_info); 107 static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, 108 struct rte_eth_fc_conf *fc_conf); 109 static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, 110 struct rte_eth_fc_conf *fc_conf); 111 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 112 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); 113 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); 114 static int eth_igb_interrupt_action(struct rte_eth_dev *dev, 115 struct rte_intr_handle *handle); 116 static void eth_igb_interrupt_handler(void *param); 117 static int igb_hardware_init(struct e1000_hw *hw); 118 static void igb_hw_control_acquire(struct e1000_hw *hw); 119 static void igb_hw_control_release(struct e1000_hw *hw); 120 static void igb_init_manageability(struct e1000_hw *hw); 121 static void igb_release_manageability(struct e1000_hw *hw); 122 123 static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 124 125 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, 126 uint16_t vlan_id, int on); 127 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 128 enum rte_vlan_type vlan_type, 129 uint16_t tpid_id); 130 static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); 131 132 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); 133 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); 134 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev); 135 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev); 136 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev); 137 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev); 138 139 static int eth_igb_led_on(struct rte_eth_dev *dev); 140 static int eth_igb_led_off(struct rte_eth_dev *dev); 141 142 static void igb_intr_disable(struct e1000_hw *hw); 143 static int igb_get_rx_buffer_size(struct e1000_hw *hw); 144 static int eth_igb_rar_set(struct rte_eth_dev *dev, 145 struct ether_addr *mac_addr, 146 uint32_t index, uint32_t pool); 147 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); 148 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 149 struct ether_addr *addr); 150 151 static void igbvf_intr_disable(struct e1000_hw *hw); 152 static int igbvf_dev_configure(struct rte_eth_dev *dev); 153 static int igbvf_dev_start(struct rte_eth_dev *dev); 154 static void igbvf_dev_stop(struct rte_eth_dev *dev); 155 static void igbvf_dev_close(struct rte_eth_dev *dev); 156 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev); 157 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); 158 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); 159 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); 160 static int eth_igbvf_link_update(struct e1000_hw *hw); 161 static int eth_igbvf_stats_get(struct rte_eth_dev *dev, 162 struct rte_eth_stats *rte_stats); 163 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, 164 struct rte_eth_xstat *xstats, unsigned n); 165 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev, 166 struct rte_eth_xstat_name *xstats_names, 167 unsigned limit); 168 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); 169 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, 170 uint16_t vlan_id, int on); 171 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); 172 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); 173 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev, 174 struct ether_addr *addr); 175 static int igbvf_get_reg_length(struct rte_eth_dev *dev); 176 static int igbvf_get_regs(struct rte_eth_dev *dev, 177 struct rte_dev_reg_info *regs); 178 179 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, 180 struct rte_eth_rss_reta_entry64 *reta_conf, 181 uint16_t reta_size); 182 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, 183 struct rte_eth_rss_reta_entry64 *reta_conf, 184 uint16_t reta_size); 185 186 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, 187 struct rte_eth_syn_filter *filter); 188 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 189 enum rte_filter_op filter_op, 190 void *arg); 191 static int igb_add_2tuple_filter(struct rte_eth_dev *dev, 192 struct rte_eth_ntuple_filter *ntuple_filter); 193 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, 194 struct rte_eth_ntuple_filter *ntuple_filter); 195 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, 196 struct rte_eth_flex_filter *filter); 197 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 198 enum rte_filter_op filter_op, 199 void *arg); 200 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 201 struct rte_eth_ntuple_filter *ntuple_filter); 202 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 203 struct rte_eth_ntuple_filter *ntuple_filter); 204 static int igb_get_ntuple_filter(struct rte_eth_dev *dev, 205 struct rte_eth_ntuple_filter *filter); 206 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, 207 enum rte_filter_op filter_op, 208 void *arg); 209 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, 210 enum rte_filter_op filter_op, 211 void *arg); 212 static int igb_get_ethertype_filter(struct rte_eth_dev *dev, 213 struct rte_eth_ethertype_filter *filter); 214 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, 215 enum rte_filter_type filter_type, 216 enum rte_filter_op filter_op, 217 void *arg); 218 static int eth_igb_get_reg_length(struct rte_eth_dev *dev); 219 static int eth_igb_get_regs(struct rte_eth_dev *dev, 220 struct rte_dev_reg_info *regs); 221 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev); 222 static int eth_igb_get_eeprom(struct rte_eth_dev *dev, 223 struct rte_dev_eeprom_info *eeprom); 224 static int eth_igb_set_eeprom(struct rte_eth_dev *dev, 225 struct rte_dev_eeprom_info *eeprom); 226 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 227 struct ether_addr *mc_addr_set, 228 uint32_t nb_mc_addr); 229 static int igb_timesync_enable(struct rte_eth_dev *dev); 230 static int igb_timesync_disable(struct rte_eth_dev *dev); 231 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 232 struct timespec *timestamp, 233 uint32_t flags); 234 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 235 struct timespec *timestamp); 236 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 237 static int igb_timesync_read_time(struct rte_eth_dev *dev, 238 struct timespec *timestamp); 239 static int igb_timesync_write_time(struct rte_eth_dev *dev, 240 const struct timespec *timestamp); 241 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, 242 uint16_t queue_id); 243 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, 244 uint16_t queue_id); 245 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 246 uint8_t queue, uint8_t msix_vector); 247 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 248 uint8_t index, uint8_t offset); 249 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); 250 static void eth_igbvf_interrupt_handler(void *param); 251 static void igbvf_mbx_process(struct rte_eth_dev *dev); 252 static int igb_filter_restore(struct rte_eth_dev *dev); 253 254 /* 255 * Define VF Stats MACRO for Non "cleared on read" register 256 */ 257 #define UPDATE_VF_STAT(reg, last, cur) \ 258 { \ 259 u32 latest = E1000_READ_REG(hw, reg); \ 260 cur += (latest - last) & UINT_MAX; \ 261 last = latest; \ 262 } 263 264 #define IGB_FC_PAUSE_TIME 0x0680 265 #define IGB_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */ 266 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */ 267 268 #define IGBVF_PMD_NAME "rte_igbvf_pmd" /* PMD name */ 269 270 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; 271 272 /* 273 * The set of PCI devices this driver supports 274 */ 275 static const struct rte_pci_id pci_id_igb_map[] = { 276 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, 277 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, 278 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, 279 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, 280 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, 281 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, 282 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, 283 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, 284 285 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, 286 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, 287 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, 288 289 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, 290 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, 291 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, 292 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, 293 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, 294 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, 295 296 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, 297 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, 298 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, 299 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, 300 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, 301 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, 302 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, 303 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, 304 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, 305 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, 306 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, 307 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, 308 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, 309 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, 310 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 311 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, 312 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 313 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, 314 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, 315 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, 316 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, 317 { .vendor_id = 0, /* sentinel */ }, 318 }; 319 320 /* 321 * The set of PCI devices this driver supports (for 82576&I350 VF) 322 */ 323 static const struct rte_pci_id pci_id_igbvf_map[] = { 324 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, 325 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, 326 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, 327 { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, 328 { .vendor_id = 0, /* sentinel */ }, 329 }; 330 331 static const struct rte_eth_desc_lim rx_desc_lim = { 332 .nb_max = E1000_MAX_RING_DESC, 333 .nb_min = E1000_MIN_RING_DESC, 334 .nb_align = IGB_RXD_ALIGN, 335 }; 336 337 static const struct rte_eth_desc_lim tx_desc_lim = { 338 .nb_max = E1000_MAX_RING_DESC, 339 .nb_min = E1000_MIN_RING_DESC, 340 .nb_align = IGB_RXD_ALIGN, 341 .nb_seg_max = IGB_TX_MAX_SEG, 342 .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, 343 }; 344 345 static const struct eth_dev_ops eth_igb_ops = { 346 .dev_configure = eth_igb_configure, 347 .dev_start = eth_igb_start, 348 .dev_stop = eth_igb_stop, 349 .dev_set_link_up = eth_igb_dev_set_link_up, 350 .dev_set_link_down = eth_igb_dev_set_link_down, 351 .dev_close = eth_igb_close, 352 .promiscuous_enable = eth_igb_promiscuous_enable, 353 .promiscuous_disable = eth_igb_promiscuous_disable, 354 .allmulticast_enable = eth_igb_allmulticast_enable, 355 .allmulticast_disable = eth_igb_allmulticast_disable, 356 .link_update = eth_igb_link_update, 357 .stats_get = eth_igb_stats_get, 358 .xstats_get = eth_igb_xstats_get, 359 .xstats_get_by_id = eth_igb_xstats_get_by_id, 360 .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, 361 .xstats_get_names = eth_igb_xstats_get_names, 362 .stats_reset = eth_igb_stats_reset, 363 .xstats_reset = eth_igb_xstats_reset, 364 .fw_version_get = eth_igb_fw_version_get, 365 .dev_infos_get = eth_igb_infos_get, 366 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 367 .mtu_set = eth_igb_mtu_set, 368 .vlan_filter_set = eth_igb_vlan_filter_set, 369 .vlan_tpid_set = eth_igb_vlan_tpid_set, 370 .vlan_offload_set = eth_igb_vlan_offload_set, 371 .rx_queue_setup = eth_igb_rx_queue_setup, 372 .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable, 373 .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable, 374 .rx_queue_release = eth_igb_rx_queue_release, 375 .rx_queue_count = eth_igb_rx_queue_count, 376 .rx_descriptor_done = eth_igb_rx_descriptor_done, 377 .rx_descriptor_status = eth_igb_rx_descriptor_status, 378 .tx_descriptor_status = eth_igb_tx_descriptor_status, 379 .tx_queue_setup = eth_igb_tx_queue_setup, 380 .tx_queue_release = eth_igb_tx_queue_release, 381 .tx_done_cleanup = eth_igb_tx_done_cleanup, 382 .dev_led_on = eth_igb_led_on, 383 .dev_led_off = eth_igb_led_off, 384 .flow_ctrl_get = eth_igb_flow_ctrl_get, 385 .flow_ctrl_set = eth_igb_flow_ctrl_set, 386 .mac_addr_add = eth_igb_rar_set, 387 .mac_addr_remove = eth_igb_rar_clear, 388 .mac_addr_set = eth_igb_default_mac_addr_set, 389 .reta_update = eth_igb_rss_reta_update, 390 .reta_query = eth_igb_rss_reta_query, 391 .rss_hash_update = eth_igb_rss_hash_update, 392 .rss_hash_conf_get = eth_igb_rss_hash_conf_get, 393 .filter_ctrl = eth_igb_filter_ctrl, 394 .set_mc_addr_list = eth_igb_set_mc_addr_list, 395 .rxq_info_get = igb_rxq_info_get, 396 .txq_info_get = igb_txq_info_get, 397 .timesync_enable = igb_timesync_enable, 398 .timesync_disable = igb_timesync_disable, 399 .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp, 400 .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp, 401 .get_reg = eth_igb_get_regs, 402 .get_eeprom_length = eth_igb_get_eeprom_length, 403 .get_eeprom = eth_igb_get_eeprom, 404 .set_eeprom = eth_igb_set_eeprom, 405 .timesync_adjust_time = igb_timesync_adjust_time, 406 .timesync_read_time = igb_timesync_read_time, 407 .timesync_write_time = igb_timesync_write_time, 408 }; 409 410 /* 411 * dev_ops for virtual function, bare necessities for basic vf 412 * operation have been implemented 413 */ 414 static const struct eth_dev_ops igbvf_eth_dev_ops = { 415 .dev_configure = igbvf_dev_configure, 416 .dev_start = igbvf_dev_start, 417 .dev_stop = igbvf_dev_stop, 418 .dev_close = igbvf_dev_close, 419 .promiscuous_enable = igbvf_promiscuous_enable, 420 .promiscuous_disable = igbvf_promiscuous_disable, 421 .allmulticast_enable = igbvf_allmulticast_enable, 422 .allmulticast_disable = igbvf_allmulticast_disable, 423 .link_update = eth_igb_link_update, 424 .stats_get = eth_igbvf_stats_get, 425 .xstats_get = eth_igbvf_xstats_get, 426 .xstats_get_names = eth_igbvf_xstats_get_names, 427 .stats_reset = eth_igbvf_stats_reset, 428 .xstats_reset = eth_igbvf_stats_reset, 429 .vlan_filter_set = igbvf_vlan_filter_set, 430 .dev_infos_get = eth_igbvf_infos_get, 431 .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, 432 .rx_queue_setup = eth_igb_rx_queue_setup, 433 .rx_queue_release = eth_igb_rx_queue_release, 434 .tx_queue_setup = eth_igb_tx_queue_setup, 435 .tx_queue_release = eth_igb_tx_queue_release, 436 .set_mc_addr_list = eth_igb_set_mc_addr_list, 437 .rxq_info_get = igb_rxq_info_get, 438 .txq_info_get = igb_txq_info_get, 439 .mac_addr_set = igbvf_default_mac_addr_set, 440 .get_reg = igbvf_get_regs, 441 }; 442 443 /* store statistics names and its offset in stats structure */ 444 struct rte_igb_xstats_name_off { 445 char name[RTE_ETH_XSTATS_NAME_SIZE]; 446 unsigned offset; 447 }; 448 449 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = { 450 {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)}, 451 {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)}, 452 {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)}, 453 {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)}, 454 {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)}, 455 {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)}, 456 {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats, 457 ecol)}, 458 {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)}, 459 {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)}, 460 {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)}, 461 {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)}, 462 {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)}, 463 {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)}, 464 {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)}, 465 {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)}, 466 {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)}, 467 {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)}, 468 {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats, 469 fcruc)}, 470 {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)}, 471 {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)}, 472 {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)}, 473 {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)}, 474 {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 475 prc1023)}, 476 {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats, 477 prc1522)}, 478 {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)}, 479 {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)}, 480 {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)}, 481 {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)}, 482 {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)}, 483 {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)}, 484 {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)}, 485 {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)}, 486 {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)}, 487 {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)}, 488 {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)}, 489 {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)}, 490 {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)}, 491 {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)}, 492 {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)}, 493 {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)}, 494 {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)}, 495 {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats, 496 ptc1023)}, 497 {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats, 498 ptc1522)}, 499 {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)}, 500 {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)}, 501 {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)}, 502 {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)}, 503 {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)}, 504 {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)}, 505 {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)}, 506 507 {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)}, 508 }; 509 510 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \ 511 sizeof(rte_igb_stats_strings[0])) 512 513 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = { 514 {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)}, 515 {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)}, 516 {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)}, 517 {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)}, 518 {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)}, 519 }; 520 521 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \ 522 sizeof(rte_igbvf_stats_strings[0])) 523 524 525 static inline void 526 igb_intr_enable(struct rte_eth_dev *dev) 527 { 528 struct e1000_interrupt *intr = 529 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 530 struct e1000_hw *hw = 531 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 532 533 E1000_WRITE_REG(hw, E1000_IMS, intr->mask); 534 E1000_WRITE_FLUSH(hw); 535 } 536 537 static void 538 igb_intr_disable(struct e1000_hw *hw) 539 { 540 E1000_WRITE_REG(hw, E1000_IMC, ~0); 541 E1000_WRITE_FLUSH(hw); 542 } 543 544 static inline void 545 igbvf_intr_enable(struct rte_eth_dev *dev) 546 { 547 struct e1000_hw *hw = 548 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 549 550 /* only for mailbox */ 551 E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX); 552 E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX); 553 E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX); 554 E1000_WRITE_FLUSH(hw); 555 } 556 557 /* only for mailbox now. If RX/TX needed, should extend this function. */ 558 static void 559 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector) 560 { 561 uint32_t tmp = 0; 562 563 /* mailbox */ 564 tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK); 565 tmp |= E1000_VTIVAR_VALID; 566 E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp); 567 } 568 569 static void 570 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev) 571 { 572 struct e1000_hw *hw = 573 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 574 575 /* Configure VF other cause ivar */ 576 igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX); 577 } 578 579 static inline int32_t 580 igb_pf_reset_hw(struct e1000_hw *hw) 581 { 582 uint32_t ctrl_ext; 583 int32_t status; 584 585 status = e1000_reset_hw(hw); 586 587 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 588 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 589 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 590 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 591 E1000_WRITE_FLUSH(hw); 592 593 return status; 594 } 595 596 static void 597 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) 598 { 599 struct e1000_hw *hw = 600 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 601 602 603 hw->vendor_id = pci_dev->id.vendor_id; 604 hw->device_id = pci_dev->id.device_id; 605 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 606 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 607 608 e1000_set_mac_type(hw); 609 610 /* need to check if it is a vf device below */ 611 } 612 613 static int 614 igb_reset_swfw_lock(struct e1000_hw *hw) 615 { 616 int ret_val; 617 618 /* 619 * Do mac ops initialization manually here, since we will need 620 * some function pointers set by this call. 621 */ 622 ret_val = e1000_init_mac_params(hw); 623 if (ret_val) 624 return ret_val; 625 626 /* 627 * SMBI lock should not fail in this early stage. If this is the case, 628 * it is due to an improper exit of the application. 629 * So force the release of the faulty lock. 630 */ 631 if (e1000_get_hw_semaphore_generic(hw) < 0) { 632 PMD_DRV_LOG(DEBUG, "SMBI lock released"); 633 } 634 e1000_put_hw_semaphore_generic(hw); 635 636 if (hw->mac.ops.acquire_swfw_sync != NULL) { 637 uint16_t mask; 638 639 /* 640 * Phy lock should not fail in this early stage. If this is the case, 641 * it is due to an improper exit of the application. 642 * So force the release of the faulty lock. 643 */ 644 mask = E1000_SWFW_PHY0_SM << hw->bus.func; 645 if (hw->bus.func > E1000_FUNC_1) 646 mask <<= 2; 647 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 648 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", 649 hw->bus.func); 650 } 651 hw->mac.ops.release_swfw_sync(hw, mask); 652 653 /* 654 * This one is more tricky since it is common to all ports; but 655 * swfw_sync retries last long enough (1s) to be almost sure that if 656 * lock can not be taken it is due to an improper lock of the 657 * semaphore. 658 */ 659 mask = E1000_SWFW_EEP_SM; 660 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { 661 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 662 } 663 hw->mac.ops.release_swfw_sync(hw, mask); 664 } 665 666 return E1000_SUCCESS; 667 } 668 669 /* Remove all ntuple filters of the device */ 670 static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 671 { 672 struct e1000_filter_info *filter_info = 673 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 674 struct e1000_5tuple_filter *p_5tuple; 675 struct e1000_2tuple_filter *p_2tuple; 676 677 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 678 TAILQ_REMOVE(&filter_info->fivetuple_list, 679 p_5tuple, entries); 680 rte_free(p_5tuple); 681 } 682 filter_info->fivetuple_mask = 0; 683 while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { 684 TAILQ_REMOVE(&filter_info->twotuple_list, 685 p_2tuple, entries); 686 rte_free(p_2tuple); 687 } 688 filter_info->twotuple_mask = 0; 689 690 return 0; 691 } 692 693 /* Remove all flex filters of the device */ 694 static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) 695 { 696 struct e1000_filter_info *filter_info = 697 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 698 struct e1000_flex_filter *p_flex; 699 700 while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { 701 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); 702 rte_free(p_flex); 703 } 704 filter_info->flex_mask = 0; 705 706 return 0; 707 } 708 709 static int 710 eth_igb_dev_init(struct rte_eth_dev *eth_dev) 711 { 712 int error = 0; 713 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 714 struct e1000_hw *hw = 715 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 716 struct e1000_vfta * shadow_vfta = 717 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 718 struct e1000_filter_info *filter_info = 719 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 720 struct e1000_adapter *adapter = 721 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 722 723 uint32_t ctrl_ext; 724 725 eth_dev->dev_ops = ð_igb_ops; 726 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 727 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 728 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 729 730 /* for secondary processes, we don't initialise any further as primary 731 * has already done this work. Only check we don't need a different 732 * RX function */ 733 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 734 if (eth_dev->data->scattered_rx) 735 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 736 return 0; 737 } 738 739 rte_eth_copy_pci_info(eth_dev, pci_dev); 740 741 hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; 742 743 igb_identify_hardware(eth_dev, pci_dev); 744 if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { 745 error = -EIO; 746 goto err_late; 747 } 748 749 e1000_get_bus_info(hw); 750 751 /* Reset any pending lock */ 752 if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { 753 error = -EIO; 754 goto err_late; 755 } 756 757 /* Finish initialization */ 758 if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { 759 error = -EIO; 760 goto err_late; 761 } 762 763 hw->mac.autoneg = 1; 764 hw->phy.autoneg_wait_to_complete = 0; 765 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 766 767 /* Copper options */ 768 if (hw->phy.media_type == e1000_media_type_copper) { 769 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 770 hw->phy.disable_polarity_correction = 0; 771 hw->phy.ms_type = e1000_ms_hw_default; 772 } 773 774 /* 775 * Start from a known state, this is important in reading the nvm 776 * and mac from that. 777 */ 778 igb_pf_reset_hw(hw); 779 780 /* Make sure we have a good EEPROM before we read from it */ 781 if (e1000_validate_nvm_checksum(hw) < 0) { 782 /* 783 * Some PCI-E parts fail the first check due to 784 * the link being in sleep state, call it again, 785 * if it fails a second time its a real issue. 786 */ 787 if (e1000_validate_nvm_checksum(hw) < 0) { 788 PMD_INIT_LOG(ERR, "EEPROM checksum invalid"); 789 error = -EIO; 790 goto err_late; 791 } 792 } 793 794 /* Read the permanent MAC address out of the EEPROM */ 795 if (e1000_read_mac_addr(hw) != 0) { 796 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address"); 797 error = -EIO; 798 goto err_late; 799 } 800 801 /* Allocate memory for storing MAC addresses */ 802 eth_dev->data->mac_addrs = rte_zmalloc("e1000", 803 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0); 804 if (eth_dev->data->mac_addrs == NULL) { 805 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to " 806 "store MAC addresses", 807 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 808 error = -ENOMEM; 809 goto err_late; 810 } 811 812 /* Copy the permanent MAC address */ 813 ether_addr_copy((struct ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); 814 815 /* initialize the vfta */ 816 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 817 818 /* Now initialize the hardware */ 819 if (igb_hardware_init(hw) != 0) { 820 PMD_INIT_LOG(ERR, "Hardware initialization failed"); 821 rte_free(eth_dev->data->mac_addrs); 822 eth_dev->data->mac_addrs = NULL; 823 error = -ENODEV; 824 goto err_late; 825 } 826 hw->mac.get_link_status = 1; 827 adapter->stopped = 0; 828 829 /* Indicate SOL/IDER usage */ 830 if (e1000_check_reset_block(hw) < 0) { 831 PMD_INIT_LOG(ERR, "PHY reset is blocked due to" 832 "SOL/IDER session"); 833 } 834 835 /* initialize PF if max_vfs not zero */ 836 igb_pf_host_init(eth_dev); 837 838 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 839 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 840 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 841 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 842 E1000_WRITE_FLUSH(hw); 843 844 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x", 845 eth_dev->data->port_id, pci_dev->id.vendor_id, 846 pci_dev->id.device_id); 847 848 rte_intr_callback_register(&pci_dev->intr_handle, 849 eth_igb_interrupt_handler, 850 (void *)eth_dev); 851 852 /* enable uio/vfio intr/eventfd mapping */ 853 rte_intr_enable(&pci_dev->intr_handle); 854 855 /* enable support intr */ 856 igb_intr_enable(eth_dev); 857 858 /* initialize filter info */ 859 memset(filter_info, 0, 860 sizeof(struct e1000_filter_info)); 861 862 TAILQ_INIT(&filter_info->flex_list); 863 TAILQ_INIT(&filter_info->twotuple_list); 864 TAILQ_INIT(&filter_info->fivetuple_list); 865 866 TAILQ_INIT(&igb_filter_ntuple_list); 867 TAILQ_INIT(&igb_filter_ethertype_list); 868 TAILQ_INIT(&igb_filter_syn_list); 869 TAILQ_INIT(&igb_filter_flex_list); 870 TAILQ_INIT(&igb_filter_rss_list); 871 TAILQ_INIT(&igb_flow_list); 872 873 return 0; 874 875 err_late: 876 igb_hw_control_release(hw); 877 878 return error; 879 } 880 881 static int 882 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) 883 { 884 struct rte_pci_device *pci_dev; 885 struct rte_intr_handle *intr_handle; 886 struct e1000_hw *hw; 887 struct e1000_adapter *adapter = 888 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 889 struct e1000_filter_info *filter_info = 890 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 891 892 PMD_INIT_FUNC_TRACE(); 893 894 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 895 return -EPERM; 896 897 hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 898 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 899 intr_handle = &pci_dev->intr_handle; 900 901 if (adapter->stopped == 0) 902 eth_igb_close(eth_dev); 903 904 eth_dev->dev_ops = NULL; 905 eth_dev->rx_pkt_burst = NULL; 906 eth_dev->tx_pkt_burst = NULL; 907 908 /* Reset any pending lock */ 909 igb_reset_swfw_lock(hw); 910 911 rte_free(eth_dev->data->mac_addrs); 912 eth_dev->data->mac_addrs = NULL; 913 914 /* uninitialize PF if max_vfs not zero */ 915 igb_pf_host_uninit(eth_dev); 916 917 /* disable uio intr before callback unregister */ 918 rte_intr_disable(intr_handle); 919 rte_intr_callback_unregister(intr_handle, 920 eth_igb_interrupt_handler, eth_dev); 921 922 /* clear the SYN filter info */ 923 filter_info->syn_info = 0; 924 925 /* clear the ethertype filters info */ 926 filter_info->ethertype_mask = 0; 927 memset(filter_info->ethertype_filters, 0, 928 E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); 929 930 /* clear the rss filter info */ 931 memset(&filter_info->rss_info, 0, 932 sizeof(struct igb_rte_flow_rss_conf)); 933 934 /* remove all ntuple filters of the device */ 935 igb_ntuple_filter_uninit(eth_dev); 936 937 /* remove all flex filters of the device */ 938 igb_flex_filter_uninit(eth_dev); 939 940 /* clear all the filters list */ 941 igb_filterlist_flush(eth_dev); 942 943 return 0; 944 } 945 946 /* 947 * Virtual Function device init 948 */ 949 static int 950 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) 951 { 952 struct rte_pci_device *pci_dev; 953 struct rte_intr_handle *intr_handle; 954 struct e1000_adapter *adapter = 955 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 956 struct e1000_hw *hw = 957 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 958 int diag; 959 struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr; 960 961 PMD_INIT_FUNC_TRACE(); 962 963 eth_dev->dev_ops = &igbvf_eth_dev_ops; 964 eth_dev->rx_pkt_burst = ð_igb_recv_pkts; 965 eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; 966 eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; 967 968 /* for secondary processes, we don't initialise any further as primary 969 * has already done this work. Only check we don't need a different 970 * RX function */ 971 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 972 if (eth_dev->data->scattered_rx) 973 eth_dev->rx_pkt_burst = ð_igb_recv_scattered_pkts; 974 return 0; 975 } 976 977 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 978 rte_eth_copy_pci_info(eth_dev, pci_dev); 979 980 hw->device_id = pci_dev->id.device_id; 981 hw->vendor_id = pci_dev->id.vendor_id; 982 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 983 adapter->stopped = 0; 984 985 /* Initialize the shared code (base driver) */ 986 diag = e1000_setup_init_funcs(hw, TRUE); 987 if (diag != 0) { 988 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", 989 diag); 990 return -EIO; 991 } 992 993 /* init_mailbox_params */ 994 hw->mbx.ops.init_params(hw); 995 996 /* Disable the interrupts for VF */ 997 igbvf_intr_disable(hw); 998 999 diag = hw->mac.ops.reset_hw(hw); 1000 1001 /* Allocate memory for storing MAC addresses */ 1002 eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN * 1003 hw->mac.rar_entry_count, 0); 1004 if (eth_dev->data->mac_addrs == NULL) { 1005 PMD_INIT_LOG(ERR, 1006 "Failed to allocate %d bytes needed to store MAC " 1007 "addresses", 1008 ETHER_ADDR_LEN * hw->mac.rar_entry_count); 1009 return -ENOMEM; 1010 } 1011 1012 /* Generate a random MAC address, if none was assigned by PF. */ 1013 if (is_zero_ether_addr(perm_addr)) { 1014 eth_random_addr(perm_addr->addr_bytes); 1015 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1016 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1017 "%02x:%02x:%02x:%02x:%02x:%02x", 1018 perm_addr->addr_bytes[0], 1019 perm_addr->addr_bytes[1], 1020 perm_addr->addr_bytes[2], 1021 perm_addr->addr_bytes[3], 1022 perm_addr->addr_bytes[4], 1023 perm_addr->addr_bytes[5]); 1024 } 1025 1026 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); 1027 if (diag) { 1028 rte_free(eth_dev->data->mac_addrs); 1029 eth_dev->data->mac_addrs = NULL; 1030 return diag; 1031 } 1032 /* Copy the permanent MAC address */ 1033 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1034 ð_dev->data->mac_addrs[0]); 1035 1036 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " 1037 "mac.type=%s", 1038 eth_dev->data->port_id, pci_dev->id.vendor_id, 1039 pci_dev->id.device_id, "igb_mac_82576_vf"); 1040 1041 intr_handle = &pci_dev->intr_handle; 1042 rte_intr_callback_register(intr_handle, 1043 eth_igbvf_interrupt_handler, eth_dev); 1044 1045 return 0; 1046 } 1047 1048 static int 1049 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) 1050 { 1051 struct e1000_adapter *adapter = 1052 E1000_DEV_PRIVATE(eth_dev->data->dev_private); 1053 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1054 1055 PMD_INIT_FUNC_TRACE(); 1056 1057 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1058 return -EPERM; 1059 1060 if (adapter->stopped == 0) 1061 igbvf_dev_close(eth_dev); 1062 1063 eth_dev->dev_ops = NULL; 1064 eth_dev->rx_pkt_burst = NULL; 1065 eth_dev->tx_pkt_burst = NULL; 1066 1067 rte_free(eth_dev->data->mac_addrs); 1068 eth_dev->data->mac_addrs = NULL; 1069 1070 /* disable uio intr before callback unregister */ 1071 rte_intr_disable(&pci_dev->intr_handle); 1072 rte_intr_callback_unregister(&pci_dev->intr_handle, 1073 eth_igbvf_interrupt_handler, 1074 (void *)eth_dev); 1075 1076 return 0; 1077 } 1078 1079 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1080 struct rte_pci_device *pci_dev) 1081 { 1082 return rte_eth_dev_pci_generic_probe(pci_dev, 1083 sizeof(struct e1000_adapter), eth_igb_dev_init); 1084 } 1085 1086 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) 1087 { 1088 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); 1089 } 1090 1091 static struct rte_pci_driver rte_igb_pmd = { 1092 .id_table = pci_id_igb_map, 1093 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1094 RTE_PCI_DRV_IOVA_AS_VA, 1095 .probe = eth_igb_pci_probe, 1096 .remove = eth_igb_pci_remove, 1097 }; 1098 1099 1100 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1101 struct rte_pci_device *pci_dev) 1102 { 1103 return rte_eth_dev_pci_generic_probe(pci_dev, 1104 sizeof(struct e1000_adapter), eth_igbvf_dev_init); 1105 } 1106 1107 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) 1108 { 1109 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); 1110 } 1111 1112 /* 1113 * virtual function driver struct 1114 */ 1115 static struct rte_pci_driver rte_igbvf_pmd = { 1116 .id_table = pci_id_igbvf_map, 1117 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1118 .probe = eth_igbvf_pci_probe, 1119 .remove = eth_igbvf_pci_remove, 1120 }; 1121 1122 static void 1123 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1124 { 1125 struct e1000_hw *hw = 1126 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1127 /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ 1128 uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); 1129 rctl |= E1000_RCTL_VFE; 1130 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1131 } 1132 1133 static int 1134 igb_check_mq_mode(struct rte_eth_dev *dev) 1135 { 1136 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 1137 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; 1138 uint16_t nb_rx_q = dev->data->nb_rx_queues; 1139 uint16_t nb_tx_q = dev->data->nb_tx_queues; 1140 1141 if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || 1142 tx_mq_mode == ETH_MQ_TX_DCB || 1143 tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) { 1144 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 1145 return -EINVAL; 1146 } 1147 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 1148 /* Check multi-queue mode. 1149 * To no break software we accept ETH_MQ_RX_NONE as this might 1150 * be used to turn off VLAN filter. 1151 */ 1152 1153 if (rx_mq_mode == ETH_MQ_RX_NONE || 1154 rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1155 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 1156 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; 1157 } else { 1158 /* Only support one queue on VFs. 1159 * RSS together with SRIOV is not supported. 1160 */ 1161 PMD_INIT_LOG(ERR, "SRIOV is active," 1162 " wrong mq_mode rx %d.", 1163 rx_mq_mode); 1164 return -EINVAL; 1165 } 1166 /* TX mode is not used here, so mode might be ignored.*/ 1167 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1168 /* SRIOV only works in VMDq enable mode */ 1169 PMD_INIT_LOG(WARNING, "SRIOV is active," 1170 " TX mode %d is not supported. " 1171 " Driver will behave as %d mode.", 1172 tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY); 1173 } 1174 1175 /* check valid queue number */ 1176 if ((nb_rx_q > 1) || (nb_tx_q > 1)) { 1177 PMD_INIT_LOG(ERR, "SRIOV is active," 1178 " only support one queue on VFs."); 1179 return -EINVAL; 1180 } 1181 } else { 1182 /* To no break software that set invalid mode, only display 1183 * warning if invalid mode is used. 1184 */ 1185 if (rx_mq_mode != ETH_MQ_RX_NONE && 1186 rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY && 1187 rx_mq_mode != ETH_MQ_RX_RSS) { 1188 /* RSS together with VMDq not supported*/ 1189 PMD_INIT_LOG(ERR, "RX mode %d is not supported.", 1190 rx_mq_mode); 1191 return -EINVAL; 1192 } 1193 1194 if (tx_mq_mode != ETH_MQ_TX_NONE && 1195 tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) { 1196 PMD_INIT_LOG(WARNING, "TX mode %d is not supported." 1197 " Due to txmode is meaningless in this" 1198 " driver, just ignore.", 1199 tx_mq_mode); 1200 } 1201 } 1202 return 0; 1203 } 1204 1205 static int 1206 eth_igb_configure(struct rte_eth_dev *dev) 1207 { 1208 struct e1000_interrupt *intr = 1209 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 1210 int ret; 1211 1212 PMD_INIT_FUNC_TRACE(); 1213 1214 /* multipe queue mode checking */ 1215 ret = igb_check_mq_mode(dev); 1216 if (ret != 0) { 1217 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.", 1218 ret); 1219 return ret; 1220 } 1221 1222 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 1223 PMD_INIT_FUNC_TRACE(); 1224 1225 return 0; 1226 } 1227 1228 static void 1229 eth_igb_rxtx_control(struct rte_eth_dev *dev, 1230 bool enable) 1231 { 1232 struct e1000_hw *hw = 1233 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1234 uint32_t tctl, rctl; 1235 1236 tctl = E1000_READ_REG(hw, E1000_TCTL); 1237 rctl = E1000_READ_REG(hw, E1000_RCTL); 1238 1239 if (enable) { 1240 /* enable Tx/Rx */ 1241 tctl |= E1000_TCTL_EN; 1242 rctl |= E1000_RCTL_EN; 1243 } else { 1244 /* disable Tx/Rx */ 1245 tctl &= ~E1000_TCTL_EN; 1246 rctl &= ~E1000_RCTL_EN; 1247 } 1248 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 1249 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 1250 E1000_WRITE_FLUSH(hw); 1251 } 1252 1253 static int 1254 eth_igb_start(struct rte_eth_dev *dev) 1255 { 1256 struct e1000_hw *hw = 1257 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1258 struct e1000_adapter *adapter = 1259 E1000_DEV_PRIVATE(dev->data->dev_private); 1260 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1261 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1262 int ret, mask; 1263 uint32_t intr_vector = 0; 1264 uint32_t ctrl_ext; 1265 uint32_t *speeds; 1266 int num_speeds; 1267 bool autoneg; 1268 1269 PMD_INIT_FUNC_TRACE(); 1270 1271 /* disable uio/vfio intr/eventfd mapping */ 1272 rte_intr_disable(intr_handle); 1273 1274 /* Power up the phy. Needed to make the link go Up */ 1275 eth_igb_dev_set_link_up(dev); 1276 1277 /* 1278 * Packet Buffer Allocation (PBA) 1279 * Writing PBA sets the receive portion of the buffer 1280 * the remainder is used for the transmit buffer. 1281 */ 1282 if (hw->mac.type == e1000_82575) { 1283 uint32_t pba; 1284 1285 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1286 E1000_WRITE_REG(hw, E1000_PBA, pba); 1287 } 1288 1289 /* Put the address into the Receive Address Array */ 1290 e1000_rar_set(hw, hw->mac.addr, 0); 1291 1292 /* Initialize the hardware */ 1293 if (igb_hardware_init(hw)) { 1294 PMD_INIT_LOG(ERR, "Unable to initialize the hardware"); 1295 return -EIO; 1296 } 1297 adapter->stopped = 0; 1298 1299 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1300 1301 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 1302 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1303 ctrl_ext |= E1000_CTRL_EXT_PFRSTD; 1304 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 1305 E1000_WRITE_FLUSH(hw); 1306 1307 /* configure PF module if SRIOV enabled */ 1308 igb_pf_host_configure(dev); 1309 1310 /* check and configure queue intr-vector mapping */ 1311 if ((rte_intr_cap_multiple(intr_handle) || 1312 !RTE_ETH_DEV_SRIOV(dev).active) && 1313 dev->data->dev_conf.intr_conf.rxq != 0) { 1314 intr_vector = dev->data->nb_rx_queues; 1315 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1316 return -1; 1317 } 1318 1319 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1320 intr_handle->intr_vec = 1321 rte_zmalloc("intr_vec", 1322 dev->data->nb_rx_queues * sizeof(int), 0); 1323 if (intr_handle->intr_vec == NULL) { 1324 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1325 " intr_vec", dev->data->nb_rx_queues); 1326 return -ENOMEM; 1327 } 1328 } 1329 1330 /* confiugre msix for rx interrupt */ 1331 eth_igb_configure_msix_intr(dev); 1332 1333 /* Configure for OS presence */ 1334 igb_init_manageability(hw); 1335 1336 eth_igb_tx_init(dev); 1337 1338 /* This can fail when allocating mbufs for descriptor rings */ 1339 ret = eth_igb_rx_init(dev); 1340 if (ret) { 1341 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 1342 igb_dev_clear_queues(dev); 1343 return ret; 1344 } 1345 1346 e1000_clear_hw_cntrs_base_generic(hw); 1347 1348 /* 1349 * VLAN Offload Settings 1350 */ 1351 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ 1352 ETH_VLAN_EXTEND_MASK; 1353 ret = eth_igb_vlan_offload_set(dev, mask); 1354 if (ret) { 1355 PMD_INIT_LOG(ERR, "Unable to set vlan offload"); 1356 igb_dev_clear_queues(dev); 1357 return ret; 1358 } 1359 1360 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 1361 /* Enable VLAN filter since VMDq always use VLAN filter */ 1362 igb_vmdq_vlan_hw_filter_enable(dev); 1363 } 1364 1365 if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || 1366 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || 1367 (hw->mac.type == e1000_i211)) { 1368 /* Configure EITR with the maximum possible value (0xFFFF) */ 1369 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF); 1370 } 1371 1372 /* Setup link speed and duplex */ 1373 speeds = &dev->data->dev_conf.link_speeds; 1374 if (*speeds == ETH_LINK_SPEED_AUTONEG) { 1375 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; 1376 hw->mac.autoneg = 1; 1377 } else { 1378 num_speeds = 0; 1379 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; 1380 1381 /* Reset */ 1382 hw->phy.autoneg_advertised = 0; 1383 1384 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 1385 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 1386 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) { 1387 num_speeds = -1; 1388 goto error_invalid_config; 1389 } 1390 if (*speeds & ETH_LINK_SPEED_10M_HD) { 1391 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF; 1392 num_speeds++; 1393 } 1394 if (*speeds & ETH_LINK_SPEED_10M) { 1395 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL; 1396 num_speeds++; 1397 } 1398 if (*speeds & ETH_LINK_SPEED_100M_HD) { 1399 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF; 1400 num_speeds++; 1401 } 1402 if (*speeds & ETH_LINK_SPEED_100M) { 1403 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL; 1404 num_speeds++; 1405 } 1406 if (*speeds & ETH_LINK_SPEED_1G) { 1407 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL; 1408 num_speeds++; 1409 } 1410 if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) 1411 goto error_invalid_config; 1412 1413 /* Set/reset the mac.autoneg based on the link speed, 1414 * fixed or not 1415 */ 1416 if (!autoneg) { 1417 hw->mac.autoneg = 0; 1418 hw->mac.forced_speed_duplex = 1419 hw->phy.autoneg_advertised; 1420 } else { 1421 hw->mac.autoneg = 1; 1422 } 1423 } 1424 1425 e1000_setup_link(hw); 1426 1427 if (rte_intr_allow_others(intr_handle)) { 1428 /* check if lsc interrupt is enabled */ 1429 if (dev->data->dev_conf.intr_conf.lsc != 0) 1430 eth_igb_lsc_interrupt_setup(dev, TRUE); 1431 else 1432 eth_igb_lsc_interrupt_setup(dev, FALSE); 1433 } else { 1434 rte_intr_callback_unregister(intr_handle, 1435 eth_igb_interrupt_handler, 1436 (void *)dev); 1437 if (dev->data->dev_conf.intr_conf.lsc != 0) 1438 PMD_INIT_LOG(INFO, "lsc won't enable because of" 1439 " no intr multiplex"); 1440 } 1441 1442 /* check if rxq interrupt is enabled */ 1443 if (dev->data->dev_conf.intr_conf.rxq != 0 && 1444 rte_intr_dp_is_en(intr_handle)) 1445 eth_igb_rxq_interrupt_setup(dev); 1446 1447 /* enable uio/vfio intr/eventfd mapping */ 1448 rte_intr_enable(intr_handle); 1449 1450 /* resume enabled intr since hw reset */ 1451 igb_intr_enable(dev); 1452 1453 /* restore all types filter */ 1454 igb_filter_restore(dev); 1455 1456 eth_igb_rxtx_control(dev, true); 1457 eth_igb_link_update(dev, 0); 1458 1459 PMD_INIT_LOG(DEBUG, "<<"); 1460 1461 return 0; 1462 1463 error_invalid_config: 1464 PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u", 1465 dev->data->dev_conf.link_speeds, dev->data->port_id); 1466 igb_dev_clear_queues(dev); 1467 return -EINVAL; 1468 } 1469 1470 /********************************************************************* 1471 * 1472 * This routine disables all traffic on the adapter by issuing a 1473 * global reset on the MAC. 1474 * 1475 **********************************************************************/ 1476 static void 1477 eth_igb_stop(struct rte_eth_dev *dev) 1478 { 1479 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1480 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1481 struct rte_eth_link link; 1482 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1483 1484 eth_igb_rxtx_control(dev, false); 1485 1486 igb_intr_disable(hw); 1487 1488 /* disable intr eventfd mapping */ 1489 rte_intr_disable(intr_handle); 1490 1491 igb_pf_reset_hw(hw); 1492 E1000_WRITE_REG(hw, E1000_WUC, 0); 1493 1494 /* Set bit for Go Link disconnect */ 1495 if (hw->mac.type >= e1000_82580) { 1496 uint32_t phpm_reg; 1497 1498 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1499 phpm_reg |= E1000_82580_PM_GO_LINKD; 1500 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1501 } 1502 1503 /* Power down the phy. Needed to make the link go Down */ 1504 eth_igb_dev_set_link_down(dev); 1505 1506 igb_dev_clear_queues(dev); 1507 1508 /* clear the recorded link status */ 1509 memset(&link, 0, sizeof(link)); 1510 rte_eth_linkstatus_set(dev, &link); 1511 1512 if (!rte_intr_allow_others(intr_handle)) 1513 /* resume to the default handler */ 1514 rte_intr_callback_register(intr_handle, 1515 eth_igb_interrupt_handler, 1516 (void *)dev); 1517 1518 /* Clean datapath event and queue/vec mapping */ 1519 rte_intr_efd_disable(intr_handle); 1520 if (intr_handle->intr_vec != NULL) { 1521 rte_free(intr_handle->intr_vec); 1522 intr_handle->intr_vec = NULL; 1523 } 1524 } 1525 1526 static int 1527 eth_igb_dev_set_link_up(struct rte_eth_dev *dev) 1528 { 1529 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1530 1531 if (hw->phy.media_type == e1000_media_type_copper) 1532 e1000_power_up_phy(hw); 1533 else 1534 e1000_power_up_fiber_serdes_link(hw); 1535 1536 return 0; 1537 } 1538 1539 static int 1540 eth_igb_dev_set_link_down(struct rte_eth_dev *dev) 1541 { 1542 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1543 1544 if (hw->phy.media_type == e1000_media_type_copper) 1545 e1000_power_down_phy(hw); 1546 else 1547 e1000_shutdown_fiber_serdes_link(hw); 1548 1549 return 0; 1550 } 1551 1552 static void 1553 eth_igb_close(struct rte_eth_dev *dev) 1554 { 1555 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1556 struct e1000_adapter *adapter = 1557 E1000_DEV_PRIVATE(dev->data->dev_private); 1558 struct rte_eth_link link; 1559 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1560 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1561 1562 eth_igb_stop(dev); 1563 adapter->stopped = 1; 1564 1565 e1000_phy_hw_reset(hw); 1566 igb_release_manageability(hw); 1567 igb_hw_control_release(hw); 1568 1569 /* Clear bit for Go Link disconnect */ 1570 if (hw->mac.type >= e1000_82580) { 1571 uint32_t phpm_reg; 1572 1573 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT); 1574 phpm_reg &= ~E1000_82580_PM_GO_LINKD; 1575 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg); 1576 } 1577 1578 igb_dev_free_queues(dev); 1579 1580 if (intr_handle->intr_vec) { 1581 rte_free(intr_handle->intr_vec); 1582 intr_handle->intr_vec = NULL; 1583 } 1584 1585 memset(&link, 0, sizeof(link)); 1586 rte_eth_linkstatus_set(dev, &link); 1587 } 1588 1589 static int 1590 igb_get_rx_buffer_size(struct e1000_hw *hw) 1591 { 1592 uint32_t rx_buf_size; 1593 if (hw->mac.type == e1000_82576) { 1594 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10; 1595 } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) { 1596 /* PBS needs to be translated according to a lookup table */ 1597 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); 1598 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); 1599 rx_buf_size = (rx_buf_size << 10); 1600 } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { 1601 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; 1602 } else { 1603 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; 1604 } 1605 1606 return rx_buf_size; 1607 } 1608 1609 /********************************************************************* 1610 * 1611 * Initialize the hardware 1612 * 1613 **********************************************************************/ 1614 static int 1615 igb_hardware_init(struct e1000_hw *hw) 1616 { 1617 uint32_t rx_buf_size; 1618 int diag; 1619 1620 /* Let the firmware know the OS is in control */ 1621 igb_hw_control_acquire(hw); 1622 1623 /* 1624 * These parameters control the automatic generation (Tx) and 1625 * response (Rx) to Ethernet PAUSE frames. 1626 * - High water mark should allow for at least two standard size (1518) 1627 * frames to be received after sending an XOFF. 1628 * - Low water mark works best when it is very near the high water mark. 1629 * This allows the receiver to restart by sending XON when it has 1630 * drained a bit. Here we use an arbitrary value of 1500 which will 1631 * restart after one full frame is pulled from the buffer. There 1632 * could be several smaller frames in the buffer and if so they will 1633 * not trigger the XON until their total number reduces the buffer 1634 * by 1500. 1635 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1636 */ 1637 rx_buf_size = igb_get_rx_buffer_size(hw); 1638 1639 hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2); 1640 hw->fc.low_water = hw->fc.high_water - 1500; 1641 hw->fc.pause_time = IGB_FC_PAUSE_TIME; 1642 hw->fc.send_xon = 1; 1643 1644 /* Set Flow control, use the tunable location if sane */ 1645 if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4)) 1646 hw->fc.requested_mode = igb_fc_setting; 1647 else 1648 hw->fc.requested_mode = e1000_fc_none; 1649 1650 /* Issue a global reset */ 1651 igb_pf_reset_hw(hw); 1652 E1000_WRITE_REG(hw, E1000_WUC, 0); 1653 1654 diag = e1000_init_hw(hw); 1655 if (diag < 0) 1656 return diag; 1657 1658 E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); 1659 e1000_get_phy_info(hw); 1660 e1000_check_for_link(hw); 1661 1662 return 0; 1663 } 1664 1665 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */ 1666 static void 1667 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) 1668 { 1669 int pause_frames; 1670 1671 uint64_t old_gprc = stats->gprc; 1672 uint64_t old_gptc = stats->gptc; 1673 uint64_t old_tpr = stats->tpr; 1674 uint64_t old_tpt = stats->tpt; 1675 uint64_t old_rpthc = stats->rpthc; 1676 uint64_t old_hgptc = stats->hgptc; 1677 1678 if(hw->phy.media_type == e1000_media_type_copper || 1679 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 1680 stats->symerrs += 1681 E1000_READ_REG(hw,E1000_SYMERRS); 1682 stats->sec += E1000_READ_REG(hw, E1000_SEC); 1683 } 1684 1685 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 1686 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 1687 stats->scc += E1000_READ_REG(hw, E1000_SCC); 1688 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 1689 1690 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 1691 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 1692 stats->colc += E1000_READ_REG(hw, E1000_COLC); 1693 stats->dc += E1000_READ_REG(hw, E1000_DC); 1694 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 1695 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 1696 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 1697 /* 1698 ** For watchdog management we need to know if we have been 1699 ** paused during the last interval, so capture that here. 1700 */ 1701 pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC); 1702 stats->xoffrxc += pause_frames; 1703 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 1704 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 1705 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 1706 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 1707 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 1708 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 1709 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 1710 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 1711 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 1712 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 1713 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 1714 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 1715 1716 /* For the 64-bit byte counters the low dword must be read first. */ 1717 /* Both registers clear on the read of the high dword */ 1718 1719 /* Workaround CRC bytes included in size, take away 4 bytes/packet */ 1720 stats->gorc += E1000_READ_REG(hw, E1000_GORCL); 1721 stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32); 1722 stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN; 1723 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL); 1724 stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32); 1725 stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN; 1726 1727 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 1728 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 1729 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 1730 stats->roc += E1000_READ_REG(hw, E1000_ROC); 1731 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 1732 1733 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 1734 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 1735 1736 stats->tor += E1000_READ_REG(hw, E1000_TORL); 1737 stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32); 1738 stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN; 1739 stats->tot += E1000_READ_REG(hw, E1000_TOTL); 1740 stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32); 1741 stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN; 1742 1743 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 1744 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 1745 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 1746 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 1747 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 1748 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 1749 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 1750 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 1751 1752 /* Interrupt Counts */ 1753 1754 stats->iac += E1000_READ_REG(hw, E1000_IAC); 1755 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 1756 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 1757 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 1758 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 1759 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 1760 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 1761 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 1762 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 1763 1764 /* Host to Card Statistics */ 1765 1766 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 1767 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 1768 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 1769 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 1770 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 1771 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 1772 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 1773 stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL); 1774 stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32); 1775 stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN; 1776 stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL); 1777 stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32); 1778 stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN; 1779 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 1780 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 1781 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 1782 1783 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 1784 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 1785 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 1786 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 1787 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 1788 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 1789 } 1790 1791 static int 1792 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 1793 { 1794 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1795 struct e1000_hw_stats *stats = 1796 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1797 1798 igb_read_stats_registers(hw, stats); 1799 1800 if (rte_stats == NULL) 1801 return -EINVAL; 1802 1803 /* Rx Errors */ 1804 rte_stats->imissed = stats->mpc; 1805 rte_stats->ierrors = stats->crcerrs + 1806 stats->rlec + stats->ruc + stats->roc + 1807 stats->rxerrc + stats->algnerrc + stats->cexterr; 1808 1809 /* Tx Errors */ 1810 rte_stats->oerrors = stats->ecol + stats->latecol; 1811 1812 rte_stats->ipackets = stats->gprc; 1813 rte_stats->opackets = stats->gptc; 1814 rte_stats->ibytes = stats->gorc; 1815 rte_stats->obytes = stats->gotc; 1816 return 0; 1817 } 1818 1819 static void 1820 eth_igb_stats_reset(struct rte_eth_dev *dev) 1821 { 1822 struct e1000_hw_stats *hw_stats = 1823 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1824 1825 /* HW registers are cleared on read */ 1826 eth_igb_stats_get(dev, NULL); 1827 1828 /* Reset software totals */ 1829 memset(hw_stats, 0, sizeof(*hw_stats)); 1830 } 1831 1832 static void 1833 eth_igb_xstats_reset(struct rte_eth_dev *dev) 1834 { 1835 struct e1000_hw_stats *stats = 1836 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1837 1838 /* HW registers are cleared on read */ 1839 eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS); 1840 1841 /* Reset software totals */ 1842 memset(stats, 0, sizeof(*stats)); 1843 } 1844 1845 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1846 struct rte_eth_xstat_name *xstats_names, 1847 __rte_unused unsigned int size) 1848 { 1849 unsigned i; 1850 1851 if (xstats_names == NULL) 1852 return IGB_NB_XSTATS; 1853 1854 /* Note: limit checked in rte_eth_xstats_names() */ 1855 1856 for (i = 0; i < IGB_NB_XSTATS; i++) { 1857 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), 1858 "%s", rte_igb_stats_strings[i].name); 1859 } 1860 1861 return IGB_NB_XSTATS; 1862 } 1863 1864 static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, 1865 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, 1866 unsigned int limit) 1867 { 1868 unsigned int i; 1869 1870 if (!ids) { 1871 if (xstats_names == NULL) 1872 return IGB_NB_XSTATS; 1873 1874 for (i = 0; i < IGB_NB_XSTATS; i++) 1875 snprintf(xstats_names[i].name, 1876 sizeof(xstats_names[i].name), 1877 "%s", rte_igb_stats_strings[i].name); 1878 1879 return IGB_NB_XSTATS; 1880 1881 } else { 1882 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; 1883 1884 eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 1885 IGB_NB_XSTATS); 1886 1887 for (i = 0; i < limit; i++) { 1888 if (ids[i] >= IGB_NB_XSTATS) { 1889 PMD_INIT_LOG(ERR, "id value isn't valid"); 1890 return -1; 1891 } 1892 strcpy(xstats_names[i].name, 1893 xstats_names_copy[ids[i]].name); 1894 } 1895 return limit; 1896 } 1897 } 1898 1899 static int 1900 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1901 unsigned n) 1902 { 1903 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1904 struct e1000_hw_stats *hw_stats = 1905 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1906 unsigned i; 1907 1908 if (n < IGB_NB_XSTATS) 1909 return IGB_NB_XSTATS; 1910 1911 igb_read_stats_registers(hw, hw_stats); 1912 1913 /* If this is a reset xstats is NULL, and we have cleared the 1914 * registers by reading them. 1915 */ 1916 if (!xstats) 1917 return 0; 1918 1919 /* Extended stats */ 1920 for (i = 0; i < IGB_NB_XSTATS; i++) { 1921 xstats[i].id = i; 1922 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 1923 rte_igb_stats_strings[i].offset); 1924 } 1925 1926 return IGB_NB_XSTATS; 1927 } 1928 1929 static int 1930 eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 1931 uint64_t *values, unsigned int n) 1932 { 1933 unsigned int i; 1934 1935 if (!ids) { 1936 struct e1000_hw *hw = 1937 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1938 struct e1000_hw_stats *hw_stats = 1939 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1940 1941 if (n < IGB_NB_XSTATS) 1942 return IGB_NB_XSTATS; 1943 1944 igb_read_stats_registers(hw, hw_stats); 1945 1946 /* If this is a reset xstats is NULL, and we have cleared the 1947 * registers by reading them. 1948 */ 1949 if (!values) 1950 return 0; 1951 1952 /* Extended stats */ 1953 for (i = 0; i < IGB_NB_XSTATS; i++) 1954 values[i] = *(uint64_t *)(((char *)hw_stats) + 1955 rte_igb_stats_strings[i].offset); 1956 1957 return IGB_NB_XSTATS; 1958 1959 } else { 1960 uint64_t values_copy[IGB_NB_XSTATS]; 1961 1962 eth_igb_xstats_get_by_id(dev, NULL, values_copy, 1963 IGB_NB_XSTATS); 1964 1965 for (i = 0; i < n; i++) { 1966 if (ids[i] >= IGB_NB_XSTATS) { 1967 PMD_INIT_LOG(ERR, "id value isn't valid"); 1968 return -1; 1969 } 1970 values[i] = values_copy[ids[i]]; 1971 } 1972 return n; 1973 } 1974 } 1975 1976 static void 1977 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) 1978 { 1979 /* Good Rx packets, include VF loopback */ 1980 UPDATE_VF_STAT(E1000_VFGPRC, 1981 hw_stats->last_gprc, hw_stats->gprc); 1982 1983 /* Good Rx octets, include VF loopback */ 1984 UPDATE_VF_STAT(E1000_VFGORC, 1985 hw_stats->last_gorc, hw_stats->gorc); 1986 1987 /* Good Tx packets, include VF loopback */ 1988 UPDATE_VF_STAT(E1000_VFGPTC, 1989 hw_stats->last_gptc, hw_stats->gptc); 1990 1991 /* Good Tx octets, include VF loopback */ 1992 UPDATE_VF_STAT(E1000_VFGOTC, 1993 hw_stats->last_gotc, hw_stats->gotc); 1994 1995 /* Rx Multicst packets */ 1996 UPDATE_VF_STAT(E1000_VFMPRC, 1997 hw_stats->last_mprc, hw_stats->mprc); 1998 1999 /* Good Rx loopback packets */ 2000 UPDATE_VF_STAT(E1000_VFGPRLBC, 2001 hw_stats->last_gprlbc, hw_stats->gprlbc); 2002 2003 /* Good Rx loopback octets */ 2004 UPDATE_VF_STAT(E1000_VFGORLBC, 2005 hw_stats->last_gorlbc, hw_stats->gorlbc); 2006 2007 /* Good Tx loopback packets */ 2008 UPDATE_VF_STAT(E1000_VFGPTLBC, 2009 hw_stats->last_gptlbc, hw_stats->gptlbc); 2010 2011 /* Good Tx loopback octets */ 2012 UPDATE_VF_STAT(E1000_VFGOTLBC, 2013 hw_stats->last_gotlbc, hw_stats->gotlbc); 2014 } 2015 2016 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 2017 struct rte_eth_xstat_name *xstats_names, 2018 __rte_unused unsigned limit) 2019 { 2020 unsigned i; 2021 2022 if (xstats_names != NULL) 2023 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2024 snprintf(xstats_names[i].name, 2025 sizeof(xstats_names[i].name), "%s", 2026 rte_igbvf_stats_strings[i].name); 2027 } 2028 return IGBVF_NB_XSTATS; 2029 } 2030 2031 static int 2032 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 2033 unsigned n) 2034 { 2035 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2036 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2037 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2038 unsigned i; 2039 2040 if (n < IGBVF_NB_XSTATS) 2041 return IGBVF_NB_XSTATS; 2042 2043 igbvf_read_stats_registers(hw, hw_stats); 2044 2045 if (!xstats) 2046 return 0; 2047 2048 for (i = 0; i < IGBVF_NB_XSTATS; i++) { 2049 xstats[i].id = i; 2050 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 2051 rte_igbvf_stats_strings[i].offset); 2052 } 2053 2054 return IGBVF_NB_XSTATS; 2055 } 2056 2057 static int 2058 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) 2059 { 2060 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2061 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *) 2062 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2063 2064 igbvf_read_stats_registers(hw, hw_stats); 2065 2066 if (rte_stats == NULL) 2067 return -EINVAL; 2068 2069 rte_stats->ipackets = hw_stats->gprc; 2070 rte_stats->ibytes = hw_stats->gorc; 2071 rte_stats->opackets = hw_stats->gptc; 2072 rte_stats->obytes = hw_stats->gotc; 2073 return 0; 2074 } 2075 2076 static void 2077 eth_igbvf_stats_reset(struct rte_eth_dev *dev) 2078 { 2079 struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*) 2080 E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 2081 2082 /* Sync HW register to the last stats */ 2083 eth_igbvf_stats_get(dev, NULL); 2084 2085 /* reset HW current stats*/ 2086 memset(&hw_stats->gprc, 0, sizeof(*hw_stats) - 2087 offsetof(struct e1000_vf_stats, gprc)); 2088 } 2089 2090 static int 2091 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 2092 size_t fw_size) 2093 { 2094 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2095 struct e1000_fw_version fw; 2096 int ret; 2097 2098 e1000_get_fw_version(hw, &fw); 2099 2100 switch (hw->mac.type) { 2101 case e1000_i210: 2102 case e1000_i211: 2103 if (!(e1000_get_flash_presence_i210(hw))) { 2104 ret = snprintf(fw_version, fw_size, 2105 "%2d.%2d-%d", 2106 fw.invm_major, fw.invm_minor, 2107 fw.invm_img_type); 2108 break; 2109 } 2110 /* fall through */ 2111 default: 2112 /* if option rom is valid, display its version too */ 2113 if (fw.or_valid) { 2114 ret = snprintf(fw_version, fw_size, 2115 "%d.%d, 0x%08x, %d.%d.%d", 2116 fw.eep_major, fw.eep_minor, fw.etrack_id, 2117 fw.or_major, fw.or_build, fw.or_patch); 2118 /* no option rom */ 2119 } else { 2120 if (fw.etrack_id != 0X0000) { 2121 ret = snprintf(fw_version, fw_size, 2122 "%d.%d, 0x%08x", 2123 fw.eep_major, fw.eep_minor, 2124 fw.etrack_id); 2125 } else { 2126 ret = snprintf(fw_version, fw_size, 2127 "%d.%d.%d", 2128 fw.eep_major, fw.eep_minor, 2129 fw.eep_build); 2130 } 2131 } 2132 break; 2133 } 2134 2135 ret += 1; /* add the size of '\0' */ 2136 if (fw_size < (u32)ret) 2137 return ret; 2138 else 2139 return 0; 2140 } 2141 2142 static void 2143 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2144 { 2145 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2146 2147 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2148 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2149 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2150 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2151 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2152 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2153 dev_info->rx_queue_offload_capa; 2154 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2155 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2156 dev_info->tx_queue_offload_capa; 2157 2158 switch (hw->mac.type) { 2159 case e1000_82575: 2160 dev_info->max_rx_queues = 4; 2161 dev_info->max_tx_queues = 4; 2162 dev_info->max_vmdq_pools = 0; 2163 break; 2164 2165 case e1000_82576: 2166 dev_info->max_rx_queues = 16; 2167 dev_info->max_tx_queues = 16; 2168 dev_info->max_vmdq_pools = ETH_8_POOLS; 2169 dev_info->vmdq_queue_num = 16; 2170 break; 2171 2172 case e1000_82580: 2173 dev_info->max_rx_queues = 8; 2174 dev_info->max_tx_queues = 8; 2175 dev_info->max_vmdq_pools = ETH_8_POOLS; 2176 dev_info->vmdq_queue_num = 8; 2177 break; 2178 2179 case e1000_i350: 2180 dev_info->max_rx_queues = 8; 2181 dev_info->max_tx_queues = 8; 2182 dev_info->max_vmdq_pools = ETH_8_POOLS; 2183 dev_info->vmdq_queue_num = 8; 2184 break; 2185 2186 case e1000_i354: 2187 dev_info->max_rx_queues = 8; 2188 dev_info->max_tx_queues = 8; 2189 break; 2190 2191 case e1000_i210: 2192 dev_info->max_rx_queues = 4; 2193 dev_info->max_tx_queues = 4; 2194 dev_info->max_vmdq_pools = 0; 2195 break; 2196 2197 case e1000_i211: 2198 dev_info->max_rx_queues = 2; 2199 dev_info->max_tx_queues = 2; 2200 dev_info->max_vmdq_pools = 0; 2201 break; 2202 2203 default: 2204 /* Should not happen */ 2205 break; 2206 } 2207 dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t); 2208 dev_info->reta_size = ETH_RSS_RETA_SIZE_128; 2209 dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; 2210 2211 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2212 .rx_thresh = { 2213 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2214 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2215 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2216 }, 2217 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2218 .rx_drop_en = 0, 2219 .offloads = 0, 2220 }; 2221 2222 dev_info->default_txconf = (struct rte_eth_txconf) { 2223 .tx_thresh = { 2224 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2225 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2226 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2227 }, 2228 .txq_flags = 0, 2229 .offloads = 0, 2230 }; 2231 2232 dev_info->rx_desc_lim = rx_desc_lim; 2233 dev_info->tx_desc_lim = tx_desc_lim; 2234 2235 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | 2236 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M | 2237 ETH_LINK_SPEED_1G; 2238 } 2239 2240 static const uint32_t * 2241 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev) 2242 { 2243 static const uint32_t ptypes[] = { 2244 /* refers to igb_rxd_pkt_info_to_pkt_type() */ 2245 RTE_PTYPE_L2_ETHER, 2246 RTE_PTYPE_L3_IPV4, 2247 RTE_PTYPE_L3_IPV4_EXT, 2248 RTE_PTYPE_L3_IPV6, 2249 RTE_PTYPE_L3_IPV6_EXT, 2250 RTE_PTYPE_L4_TCP, 2251 RTE_PTYPE_L4_UDP, 2252 RTE_PTYPE_L4_SCTP, 2253 RTE_PTYPE_TUNNEL_IP, 2254 RTE_PTYPE_INNER_L3_IPV6, 2255 RTE_PTYPE_INNER_L3_IPV6_EXT, 2256 RTE_PTYPE_INNER_L4_TCP, 2257 RTE_PTYPE_INNER_L4_UDP, 2258 RTE_PTYPE_UNKNOWN 2259 }; 2260 2261 if (dev->rx_pkt_burst == eth_igb_recv_pkts || 2262 dev->rx_pkt_burst == eth_igb_recv_scattered_pkts) 2263 return ptypes; 2264 return NULL; 2265 } 2266 2267 static void 2268 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2269 { 2270 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2271 2272 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2273 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ 2274 dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ 2275 dev_info->max_mac_addrs = hw->mac.rar_entry_count; 2276 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 2277 DEV_TX_OFFLOAD_IPV4_CKSUM | 2278 DEV_TX_OFFLOAD_UDP_CKSUM | 2279 DEV_TX_OFFLOAD_TCP_CKSUM | 2280 DEV_TX_OFFLOAD_SCTP_CKSUM | 2281 DEV_TX_OFFLOAD_TCP_TSO; 2282 switch (hw->mac.type) { 2283 case e1000_vfadapt: 2284 dev_info->max_rx_queues = 2; 2285 dev_info->max_tx_queues = 2; 2286 break; 2287 case e1000_vfadapt_i350: 2288 dev_info->max_rx_queues = 1; 2289 dev_info->max_tx_queues = 1; 2290 break; 2291 default: 2292 /* Should not happen */ 2293 break; 2294 } 2295 2296 dev_info->rx_queue_offload_capa = igb_get_rx_queue_offloads_capa(dev); 2297 dev_info->rx_offload_capa = igb_get_rx_port_offloads_capa(dev) | 2298 dev_info->rx_queue_offload_capa; 2299 dev_info->tx_queue_offload_capa = igb_get_tx_queue_offloads_capa(dev); 2300 dev_info->tx_offload_capa = igb_get_tx_port_offloads_capa(dev) | 2301 dev_info->tx_queue_offload_capa; 2302 2303 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2304 .rx_thresh = { 2305 .pthresh = IGB_DEFAULT_RX_PTHRESH, 2306 .hthresh = IGB_DEFAULT_RX_HTHRESH, 2307 .wthresh = IGB_DEFAULT_RX_WTHRESH, 2308 }, 2309 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, 2310 .rx_drop_en = 0, 2311 .offloads = 0, 2312 }; 2313 2314 dev_info->default_txconf = (struct rte_eth_txconf) { 2315 .tx_thresh = { 2316 .pthresh = IGB_DEFAULT_TX_PTHRESH, 2317 .hthresh = IGB_DEFAULT_TX_HTHRESH, 2318 .wthresh = IGB_DEFAULT_TX_WTHRESH, 2319 }, 2320 .txq_flags = 0, 2321 .offloads = 0, 2322 }; 2323 2324 dev_info->rx_desc_lim = rx_desc_lim; 2325 dev_info->tx_desc_lim = tx_desc_lim; 2326 } 2327 2328 /* return 0 means link status changed, -1 means not changed */ 2329 static int 2330 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) 2331 { 2332 struct e1000_hw *hw = 2333 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2334 struct rte_eth_link link; 2335 int link_check, count; 2336 2337 link_check = 0; 2338 hw->mac.get_link_status = 1; 2339 2340 /* possible wait-to-complete in up to 9 seconds */ 2341 for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) { 2342 /* Read the real link status */ 2343 switch (hw->phy.media_type) { 2344 case e1000_media_type_copper: 2345 /* Do the work to read phy */ 2346 e1000_check_for_link(hw); 2347 link_check = !hw->mac.get_link_status; 2348 break; 2349 2350 case e1000_media_type_fiber: 2351 e1000_check_for_link(hw); 2352 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2353 E1000_STATUS_LU); 2354 break; 2355 2356 case e1000_media_type_internal_serdes: 2357 e1000_check_for_link(hw); 2358 link_check = hw->mac.serdes_has_link; 2359 break; 2360 2361 /* VF device is type_unknown */ 2362 case e1000_media_type_unknown: 2363 eth_igbvf_link_update(hw); 2364 link_check = !hw->mac.get_link_status; 2365 break; 2366 2367 default: 2368 break; 2369 } 2370 if (link_check || wait_to_complete == 0) 2371 break; 2372 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL); 2373 } 2374 memset(&link, 0, sizeof(link)); 2375 2376 /* Now we check if a transition has happened */ 2377 if (link_check) { 2378 uint16_t duplex, speed; 2379 hw->mac.ops.get_link_up_info(hw, &speed, &duplex); 2380 link.link_duplex = (duplex == FULL_DUPLEX) ? 2381 ETH_LINK_FULL_DUPLEX : 2382 ETH_LINK_HALF_DUPLEX; 2383 link.link_speed = speed; 2384 link.link_status = ETH_LINK_UP; 2385 link.link_autoneg = !(dev->data->dev_conf.link_speeds & 2386 ETH_LINK_SPEED_FIXED); 2387 } else if (!link_check) { 2388 link.link_speed = 0; 2389 link.link_duplex = ETH_LINK_HALF_DUPLEX; 2390 link.link_status = ETH_LINK_DOWN; 2391 link.link_autoneg = ETH_LINK_FIXED; 2392 } 2393 2394 return rte_eth_linkstatus_set(dev, &link); 2395 } 2396 2397 /* 2398 * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit. 2399 * For ASF and Pass Through versions of f/w this means 2400 * that the driver is loaded. 2401 */ 2402 static void 2403 igb_hw_control_acquire(struct e1000_hw *hw) 2404 { 2405 uint32_t ctrl_ext; 2406 2407 /* Let firmware know the driver has taken over */ 2408 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2409 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2410 } 2411 2412 /* 2413 * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit. 2414 * For ASF and Pass Through versions of f/w this means that the 2415 * driver is no longer loaded. 2416 */ 2417 static void 2418 igb_hw_control_release(struct e1000_hw *hw) 2419 { 2420 uint32_t ctrl_ext; 2421 2422 /* Let firmware taken over control of h/w */ 2423 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 2424 E1000_WRITE_REG(hw, E1000_CTRL_EXT, 2425 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2426 } 2427 2428 /* 2429 * Bit of a misnomer, what this really means is 2430 * to enable OS management of the system... aka 2431 * to disable special hardware management features. 2432 */ 2433 static void 2434 igb_init_manageability(struct e1000_hw *hw) 2435 { 2436 if (e1000_enable_mng_pass_thru(hw)) { 2437 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H); 2438 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2439 2440 /* disable hardware interception of ARP */ 2441 manc &= ~(E1000_MANC_ARP_EN); 2442 2443 /* enable receiving management packets to the host */ 2444 manc |= E1000_MANC_EN_MNG2HOST; 2445 manc2h |= 1 << 5; /* Mng Port 623 */ 2446 manc2h |= 1 << 6; /* Mng Port 664 */ 2447 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h); 2448 E1000_WRITE_REG(hw, E1000_MANC, manc); 2449 } 2450 } 2451 2452 static void 2453 igb_release_manageability(struct e1000_hw *hw) 2454 { 2455 if (e1000_enable_mng_pass_thru(hw)) { 2456 uint32_t manc = E1000_READ_REG(hw, E1000_MANC); 2457 2458 manc |= E1000_MANC_ARP_EN; 2459 manc &= ~E1000_MANC_EN_MNG2HOST; 2460 2461 E1000_WRITE_REG(hw, E1000_MANC, manc); 2462 } 2463 } 2464 2465 static void 2466 eth_igb_promiscuous_enable(struct rte_eth_dev *dev) 2467 { 2468 struct e1000_hw *hw = 2469 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2470 uint32_t rctl; 2471 2472 rctl = E1000_READ_REG(hw, E1000_RCTL); 2473 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2474 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2475 } 2476 2477 static void 2478 eth_igb_promiscuous_disable(struct rte_eth_dev *dev) 2479 { 2480 struct e1000_hw *hw = 2481 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2482 uint32_t rctl; 2483 2484 rctl = E1000_READ_REG(hw, E1000_RCTL); 2485 rctl &= (~E1000_RCTL_UPE); 2486 if (dev->data->all_multicast == 1) 2487 rctl |= E1000_RCTL_MPE; 2488 else 2489 rctl &= (~E1000_RCTL_MPE); 2490 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2491 } 2492 2493 static void 2494 eth_igb_allmulticast_enable(struct rte_eth_dev *dev) 2495 { 2496 struct e1000_hw *hw = 2497 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2498 uint32_t rctl; 2499 2500 rctl = E1000_READ_REG(hw, E1000_RCTL); 2501 rctl |= E1000_RCTL_MPE; 2502 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2503 } 2504 2505 static void 2506 eth_igb_allmulticast_disable(struct rte_eth_dev *dev) 2507 { 2508 struct e1000_hw *hw = 2509 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2510 uint32_t rctl; 2511 2512 if (dev->data->promiscuous == 1) 2513 return; /* must remain in all_multicast mode */ 2514 rctl = E1000_READ_REG(hw, E1000_RCTL); 2515 rctl &= (~E1000_RCTL_MPE); 2516 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2517 } 2518 2519 static int 2520 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2521 { 2522 struct e1000_hw *hw = 2523 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2524 struct e1000_vfta * shadow_vfta = 2525 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2526 uint32_t vfta; 2527 uint32_t vid_idx; 2528 uint32_t vid_bit; 2529 2530 vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) & 2531 E1000_VFTA_ENTRY_MASK); 2532 vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK)); 2533 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx); 2534 if (on) 2535 vfta |= vid_bit; 2536 else 2537 vfta &= ~vid_bit; 2538 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta); 2539 2540 /* update local VFTA copy */ 2541 shadow_vfta->vfta[vid_idx] = vfta; 2542 2543 return 0; 2544 } 2545 2546 static int 2547 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, 2548 enum rte_vlan_type vlan_type, 2549 uint16_t tpid) 2550 { 2551 struct e1000_hw *hw = 2552 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2553 uint32_t reg, qinq; 2554 2555 qinq = E1000_READ_REG(hw, E1000_CTRL_EXT); 2556 qinq &= E1000_CTRL_EXT_EXT_VLAN; 2557 2558 /* only outer TPID of double VLAN can be configured*/ 2559 if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) { 2560 reg = E1000_READ_REG(hw, E1000_VET); 2561 reg = (reg & (~E1000_VET_VET_EXT)) | 2562 ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT); 2563 E1000_WRITE_REG(hw, E1000_VET, reg); 2564 2565 return 0; 2566 } 2567 2568 /* all other TPID values are read-only*/ 2569 PMD_DRV_LOG(ERR, "Not supported"); 2570 2571 return -ENOTSUP; 2572 } 2573 2574 static void 2575 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev) 2576 { 2577 struct e1000_hw *hw = 2578 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2579 uint32_t reg; 2580 2581 /* Filter Table Disable */ 2582 reg = E1000_READ_REG(hw, E1000_RCTL); 2583 reg &= ~E1000_RCTL_CFIEN; 2584 reg &= ~E1000_RCTL_VFE; 2585 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2586 } 2587 2588 static void 2589 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2590 { 2591 struct e1000_hw *hw = 2592 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2593 struct e1000_vfta * shadow_vfta = 2594 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 2595 uint32_t reg; 2596 int i; 2597 2598 /* Filter Table Enable, CFI not used for packet acceptance */ 2599 reg = E1000_READ_REG(hw, E1000_RCTL); 2600 reg &= ~E1000_RCTL_CFIEN; 2601 reg |= E1000_RCTL_VFE; 2602 E1000_WRITE_REG(hw, E1000_RCTL, reg); 2603 2604 /* restore VFTA table */ 2605 for (i = 0; i < IGB_VFTA_SIZE; i++) 2606 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]); 2607 } 2608 2609 static void 2610 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev) 2611 { 2612 struct e1000_hw *hw = 2613 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2614 uint32_t reg; 2615 2616 /* VLAN Mode Disable */ 2617 reg = E1000_READ_REG(hw, E1000_CTRL); 2618 reg &= ~E1000_CTRL_VME; 2619 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2620 } 2621 2622 static void 2623 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev) 2624 { 2625 struct e1000_hw *hw = 2626 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2627 uint32_t reg; 2628 2629 /* VLAN Mode Enable */ 2630 reg = E1000_READ_REG(hw, E1000_CTRL); 2631 reg |= E1000_CTRL_VME; 2632 E1000_WRITE_REG(hw, E1000_CTRL, reg); 2633 } 2634 2635 static void 2636 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2637 { 2638 struct e1000_hw *hw = 2639 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2640 uint32_t reg; 2641 2642 /* CTRL_EXT: Extended VLAN */ 2643 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2644 reg &= ~E1000_CTRL_EXT_EXTEND_VLAN; 2645 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2646 2647 /* Update maximum packet length */ 2648 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2649 E1000_WRITE_REG(hw, E1000_RLPML, 2650 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2651 VLAN_TAG_SIZE); 2652 } 2653 2654 static void 2655 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2656 { 2657 struct e1000_hw *hw = 2658 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2659 uint32_t reg; 2660 2661 /* CTRL_EXT: Extended VLAN */ 2662 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 2663 reg |= E1000_CTRL_EXT_EXTEND_VLAN; 2664 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 2665 2666 /* Update maximum packet length */ 2667 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2668 E1000_WRITE_REG(hw, E1000_RLPML, 2669 dev->data->dev_conf.rxmode.max_rx_pkt_len + 2670 2 * VLAN_TAG_SIZE); 2671 } 2672 2673 static int 2674 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2675 { 2676 struct rte_eth_rxmode *rxmode; 2677 2678 rxmode = &dev->data->dev_conf.rxmode; 2679 if(mask & ETH_VLAN_STRIP_MASK){ 2680 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2681 igb_vlan_hw_strip_enable(dev); 2682 else 2683 igb_vlan_hw_strip_disable(dev); 2684 } 2685 2686 if(mask & ETH_VLAN_FILTER_MASK){ 2687 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2688 igb_vlan_hw_filter_enable(dev); 2689 else 2690 igb_vlan_hw_filter_disable(dev); 2691 } 2692 2693 if(mask & ETH_VLAN_EXTEND_MASK){ 2694 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2695 igb_vlan_hw_extend_enable(dev); 2696 else 2697 igb_vlan_hw_extend_disable(dev); 2698 } 2699 2700 return 0; 2701 } 2702 2703 2704 /** 2705 * It enables the interrupt mask and then enable the interrupt. 2706 * 2707 * @param dev 2708 * Pointer to struct rte_eth_dev. 2709 * @param on 2710 * Enable or Disable 2711 * 2712 * @return 2713 * - On success, zero. 2714 * - On failure, a negative value. 2715 */ 2716 static int 2717 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 2718 { 2719 struct e1000_interrupt *intr = 2720 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2721 2722 if (on) 2723 intr->mask |= E1000_ICR_LSC; 2724 else 2725 intr->mask &= ~E1000_ICR_LSC; 2726 2727 return 0; 2728 } 2729 2730 /* It clears the interrupt causes and enables the interrupt. 2731 * It will be called once only during nic initialized. 2732 * 2733 * @param dev 2734 * Pointer to struct rte_eth_dev. 2735 * 2736 * @return 2737 * - On success, zero. 2738 * - On failure, a negative value. 2739 */ 2740 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev) 2741 { 2742 uint32_t mask, regval; 2743 struct e1000_hw *hw = 2744 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2745 struct rte_eth_dev_info dev_info; 2746 2747 memset(&dev_info, 0, sizeof(dev_info)); 2748 eth_igb_infos_get(dev, &dev_info); 2749 2750 mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues); 2751 regval = E1000_READ_REG(hw, E1000_EIMS); 2752 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 2753 2754 return 0; 2755 } 2756 2757 /* 2758 * It reads ICR and gets interrupt causes, check it and set a bit flag 2759 * to update link status. 2760 * 2761 * @param dev 2762 * Pointer to struct rte_eth_dev. 2763 * 2764 * @return 2765 * - On success, zero. 2766 * - On failure, a negative value. 2767 */ 2768 static int 2769 eth_igb_interrupt_get_status(struct rte_eth_dev *dev) 2770 { 2771 uint32_t icr; 2772 struct e1000_hw *hw = 2773 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2774 struct e1000_interrupt *intr = 2775 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2776 2777 igb_intr_disable(hw); 2778 2779 /* read-on-clear nic registers here */ 2780 icr = E1000_READ_REG(hw, E1000_ICR); 2781 2782 intr->flags = 0; 2783 if (icr & E1000_ICR_LSC) { 2784 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; 2785 } 2786 2787 if (icr & E1000_ICR_VMMB) 2788 intr->flags |= E1000_FLAG_MAILBOX; 2789 2790 return 0; 2791 } 2792 2793 /* 2794 * It executes link_update after knowing an interrupt is prsent. 2795 * 2796 * @param dev 2797 * Pointer to struct rte_eth_dev. 2798 * 2799 * @return 2800 * - On success, zero. 2801 * - On failure, a negative value. 2802 */ 2803 static int 2804 eth_igb_interrupt_action(struct rte_eth_dev *dev, 2805 struct rte_intr_handle *intr_handle) 2806 { 2807 struct e1000_hw *hw = 2808 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2809 struct e1000_interrupt *intr = 2810 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2811 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2812 struct rte_eth_link link; 2813 int ret; 2814 2815 if (intr->flags & E1000_FLAG_MAILBOX) { 2816 igb_pf_mbx_process(dev); 2817 intr->flags &= ~E1000_FLAG_MAILBOX; 2818 } 2819 2820 igb_intr_enable(dev); 2821 rte_intr_enable(intr_handle); 2822 2823 if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { 2824 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; 2825 2826 /* set get_link_status to check register later */ 2827 hw->mac.get_link_status = 1; 2828 ret = eth_igb_link_update(dev, 0); 2829 2830 /* check if link has changed */ 2831 if (ret < 0) 2832 return 0; 2833 2834 rte_eth_linkstatus_get(dev, &link); 2835 if (link.link_status) { 2836 PMD_INIT_LOG(INFO, 2837 " Port %d: Link Up - speed %u Mbps - %s", 2838 dev->data->port_id, 2839 (unsigned)link.link_speed, 2840 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 2841 "full-duplex" : "half-duplex"); 2842 } else { 2843 PMD_INIT_LOG(INFO, " Port %d: Link Down", 2844 dev->data->port_id); 2845 } 2846 2847 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", 2848 pci_dev->addr.domain, 2849 pci_dev->addr.bus, 2850 pci_dev->addr.devid, 2851 pci_dev->addr.function); 2852 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 2853 NULL); 2854 } 2855 2856 return 0; 2857 } 2858 2859 /** 2860 * Interrupt handler which shall be registered at first. 2861 * 2862 * @param handle 2863 * Pointer to interrupt handle. 2864 * @param param 2865 * The address of parameter (struct rte_eth_dev *) regsitered before. 2866 * 2867 * @return 2868 * void 2869 */ 2870 static void 2871 eth_igb_interrupt_handler(void *param) 2872 { 2873 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2874 2875 eth_igb_interrupt_get_status(dev); 2876 eth_igb_interrupt_action(dev, dev->intr_handle); 2877 } 2878 2879 static int 2880 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev) 2881 { 2882 uint32_t eicr; 2883 struct e1000_hw *hw = 2884 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2885 struct e1000_interrupt *intr = 2886 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2887 2888 igbvf_intr_disable(hw); 2889 2890 /* read-on-clear nic registers here */ 2891 eicr = E1000_READ_REG(hw, E1000_EICR); 2892 intr->flags = 0; 2893 2894 if (eicr == E1000_VTIVAR_MISC_MAILBOX) 2895 intr->flags |= E1000_FLAG_MAILBOX; 2896 2897 return 0; 2898 } 2899 2900 void igbvf_mbx_process(struct rte_eth_dev *dev) 2901 { 2902 struct e1000_hw *hw = 2903 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2904 struct e1000_mbx_info *mbx = &hw->mbx; 2905 u32 in_msg = 0; 2906 2907 /* peek the message first */ 2908 in_msg = E1000_READ_REG(hw, E1000_VMBMEM(0)); 2909 2910 /* PF reset VF event */ 2911 if (in_msg == E1000_PF_CONTROL_MSG) { 2912 /* dummy mbx read to ack pf */ 2913 if (mbx->ops.read(hw, &in_msg, 1, 0)) 2914 return; 2915 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 2916 NULL); 2917 } 2918 } 2919 2920 static int 2921 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) 2922 { 2923 struct e1000_interrupt *intr = 2924 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2925 2926 if (intr->flags & E1000_FLAG_MAILBOX) { 2927 igbvf_mbx_process(dev); 2928 intr->flags &= ~E1000_FLAG_MAILBOX; 2929 } 2930 2931 igbvf_intr_enable(dev); 2932 rte_intr_enable(intr_handle); 2933 2934 return 0; 2935 } 2936 2937 static void 2938 eth_igbvf_interrupt_handler(void *param) 2939 { 2940 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2941 2942 eth_igbvf_interrupt_get_status(dev); 2943 eth_igbvf_interrupt_action(dev, dev->intr_handle); 2944 } 2945 2946 static int 2947 eth_igb_led_on(struct rte_eth_dev *dev) 2948 { 2949 struct e1000_hw *hw; 2950 2951 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2952 return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2953 } 2954 2955 static int 2956 eth_igb_led_off(struct rte_eth_dev *dev) 2957 { 2958 struct e1000_hw *hw; 2959 2960 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2961 return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP; 2962 } 2963 2964 static int 2965 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 2966 { 2967 struct e1000_hw *hw; 2968 uint32_t ctrl; 2969 int tx_pause; 2970 int rx_pause; 2971 2972 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2973 fc_conf->pause_time = hw->fc.pause_time; 2974 fc_conf->high_water = hw->fc.high_water; 2975 fc_conf->low_water = hw->fc.low_water; 2976 fc_conf->send_xon = hw->fc.send_xon; 2977 fc_conf->autoneg = hw->mac.autoneg; 2978 2979 /* 2980 * Return rx_pause and tx_pause status according to actual setting of 2981 * the TFCE and RFCE bits in the CTRL register. 2982 */ 2983 ctrl = E1000_READ_REG(hw, E1000_CTRL); 2984 if (ctrl & E1000_CTRL_TFCE) 2985 tx_pause = 1; 2986 else 2987 tx_pause = 0; 2988 2989 if (ctrl & E1000_CTRL_RFCE) 2990 rx_pause = 1; 2991 else 2992 rx_pause = 0; 2993 2994 if (rx_pause && tx_pause) 2995 fc_conf->mode = RTE_FC_FULL; 2996 else if (rx_pause) 2997 fc_conf->mode = RTE_FC_RX_PAUSE; 2998 else if (tx_pause) 2999 fc_conf->mode = RTE_FC_TX_PAUSE; 3000 else 3001 fc_conf->mode = RTE_FC_NONE; 3002 3003 return 0; 3004 } 3005 3006 static int 3007 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 3008 { 3009 struct e1000_hw *hw; 3010 int err; 3011 enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = { 3012 e1000_fc_none, 3013 e1000_fc_rx_pause, 3014 e1000_fc_tx_pause, 3015 e1000_fc_full 3016 }; 3017 uint32_t rx_buf_size; 3018 uint32_t max_high_water; 3019 uint32_t rctl; 3020 3021 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3022 if (fc_conf->autoneg != hw->mac.autoneg) 3023 return -ENOTSUP; 3024 rx_buf_size = igb_get_rx_buffer_size(hw); 3025 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 3026 3027 /* At least reserve one Ethernet frame for watermark */ 3028 max_high_water = rx_buf_size - ETHER_MAX_LEN; 3029 if ((fc_conf->high_water > max_high_water) || 3030 (fc_conf->high_water < fc_conf->low_water)) { 3031 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); 3032 PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); 3033 return -EINVAL; 3034 } 3035 3036 hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode]; 3037 hw->fc.pause_time = fc_conf->pause_time; 3038 hw->fc.high_water = fc_conf->high_water; 3039 hw->fc.low_water = fc_conf->low_water; 3040 hw->fc.send_xon = fc_conf->send_xon; 3041 3042 err = e1000_setup_link_generic(hw); 3043 if (err == E1000_SUCCESS) { 3044 3045 /* check if we want to forward MAC frames - driver doesn't have native 3046 * capability to do that, so we'll write the registers ourselves */ 3047 3048 rctl = E1000_READ_REG(hw, E1000_RCTL); 3049 3050 /* set or clear MFLCN.PMCF bit depending on configuration */ 3051 if (fc_conf->mac_ctrl_frame_fwd != 0) 3052 rctl |= E1000_RCTL_PMCF; 3053 else 3054 rctl &= ~E1000_RCTL_PMCF; 3055 3056 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3057 E1000_WRITE_FLUSH(hw); 3058 3059 return 0; 3060 } 3061 3062 PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); 3063 return -EIO; 3064 } 3065 3066 #define E1000_RAH_POOLSEL_SHIFT (18) 3067 static int 3068 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 3069 uint32_t index, uint32_t pool) 3070 { 3071 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3072 uint32_t rah; 3073 3074 e1000_rar_set(hw, mac_addr->addr_bytes, index); 3075 rah = E1000_READ_REG(hw, E1000_RAH(index)); 3076 rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); 3077 E1000_WRITE_REG(hw, E1000_RAH(index), rah); 3078 return 0; 3079 } 3080 3081 static void 3082 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) 3083 { 3084 uint8_t addr[ETHER_ADDR_LEN]; 3085 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3086 3087 memset(addr, 0, sizeof(addr)); 3088 3089 e1000_rar_set(hw, addr, index); 3090 } 3091 3092 static void 3093 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, 3094 struct ether_addr *addr) 3095 { 3096 eth_igb_rar_clear(dev, 0); 3097 3098 eth_igb_rar_set(dev, (void *)addr, 0, 0); 3099 } 3100 /* 3101 * Virtual Function operations 3102 */ 3103 static void 3104 igbvf_intr_disable(struct e1000_hw *hw) 3105 { 3106 PMD_INIT_FUNC_TRACE(); 3107 3108 /* Clear interrupt mask to stop from interrupts being generated */ 3109 E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); 3110 3111 E1000_WRITE_FLUSH(hw); 3112 } 3113 3114 static void 3115 igbvf_stop_adapter(struct rte_eth_dev *dev) 3116 { 3117 u32 reg_val; 3118 u16 i; 3119 struct rte_eth_dev_info dev_info; 3120 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3121 3122 memset(&dev_info, 0, sizeof(dev_info)); 3123 eth_igbvf_infos_get(dev, &dev_info); 3124 3125 /* Clear interrupt mask to stop from interrupts being generated */ 3126 igbvf_intr_disable(hw); 3127 3128 /* Clear any pending interrupts, flush previous writes */ 3129 E1000_READ_REG(hw, E1000_EICR); 3130 3131 /* Disable the transmit unit. Each queue must be disabled. */ 3132 for (i = 0; i < dev_info.max_tx_queues; i++) 3133 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH); 3134 3135 /* Disable the receive unit by stopping each queue */ 3136 for (i = 0; i < dev_info.max_rx_queues; i++) { 3137 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3138 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE; 3139 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val); 3140 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE) 3141 ; 3142 } 3143 3144 /* flush all queues disables */ 3145 E1000_WRITE_FLUSH(hw); 3146 msec_delay(2); 3147 } 3148 3149 static int eth_igbvf_link_update(struct e1000_hw *hw) 3150 { 3151 struct e1000_mbx_info *mbx = &hw->mbx; 3152 struct e1000_mac_info *mac = &hw->mac; 3153 int ret_val = E1000_SUCCESS; 3154 3155 PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf"); 3156 3157 /* 3158 * We only want to run this if there has been a rst asserted. 3159 * in this case that could mean a link change, device reset, 3160 * or a virtual function reset 3161 */ 3162 3163 /* If we were hit with a reset or timeout drop the link */ 3164 if (!e1000_check_for_rst(hw, 0) || !mbx->timeout) 3165 mac->get_link_status = TRUE; 3166 3167 if (!mac->get_link_status) 3168 goto out; 3169 3170 /* if link status is down no point in checking to see if pf is up */ 3171 if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) 3172 goto out; 3173 3174 /* if we passed all the tests above then the link is up and we no 3175 * longer need to check for link */ 3176 mac->get_link_status = FALSE; 3177 3178 out: 3179 return ret_val; 3180 } 3181 3182 3183 static int 3184 igbvf_dev_configure(struct rte_eth_dev *dev) 3185 { 3186 struct rte_eth_conf* conf = &dev->data->dev_conf; 3187 3188 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 3189 dev->data->port_id); 3190 3191 /* 3192 * VF has no ability to enable/disable HW CRC 3193 * Keep the persistent behavior the same as Host PF 3194 */ 3195 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC 3196 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)) { 3197 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 3198 conf->rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP; 3199 } 3200 #else 3201 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) { 3202 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 3203 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP; 3204 } 3205 #endif 3206 3207 return 0; 3208 } 3209 3210 static int 3211 igbvf_dev_start(struct rte_eth_dev *dev) 3212 { 3213 struct e1000_hw *hw = 3214 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3215 struct e1000_adapter *adapter = 3216 E1000_DEV_PRIVATE(dev->data->dev_private); 3217 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3218 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3219 int ret; 3220 uint32_t intr_vector = 0; 3221 3222 PMD_INIT_FUNC_TRACE(); 3223 3224 hw->mac.ops.reset_hw(hw); 3225 adapter->stopped = 0; 3226 3227 /* Set all vfta */ 3228 igbvf_set_vfta_all(dev,1); 3229 3230 eth_igbvf_tx_init(dev); 3231 3232 /* This can fail when allocating mbufs for descriptor rings */ 3233 ret = eth_igbvf_rx_init(dev); 3234 if (ret) { 3235 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 3236 igb_dev_clear_queues(dev); 3237 return ret; 3238 } 3239 3240 /* check and configure queue intr-vector mapping */ 3241 if (rte_intr_cap_multiple(intr_handle) && 3242 dev->data->dev_conf.intr_conf.rxq) { 3243 intr_vector = dev->data->nb_rx_queues; 3244 ret = rte_intr_efd_enable(intr_handle, intr_vector); 3245 if (ret) 3246 return ret; 3247 } 3248 3249 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 3250 intr_handle->intr_vec = 3251 rte_zmalloc("intr_vec", 3252 dev->data->nb_rx_queues * sizeof(int), 0); 3253 if (!intr_handle->intr_vec) { 3254 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 3255 " intr_vec", dev->data->nb_rx_queues); 3256 return -ENOMEM; 3257 } 3258 } 3259 3260 eth_igbvf_configure_msix_intr(dev); 3261 3262 /* enable uio/vfio intr/eventfd mapping */ 3263 rte_intr_enable(intr_handle); 3264 3265 /* resume enabled intr since hw reset */ 3266 igbvf_intr_enable(dev); 3267 3268 return 0; 3269 } 3270 3271 static void 3272 igbvf_dev_stop(struct rte_eth_dev *dev) 3273 { 3274 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3275 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 3276 3277 PMD_INIT_FUNC_TRACE(); 3278 3279 igbvf_stop_adapter(dev); 3280 3281 /* 3282 * Clear what we set, but we still keep shadow_vfta to 3283 * restore after device starts 3284 */ 3285 igbvf_set_vfta_all(dev,0); 3286 3287 igb_dev_clear_queues(dev); 3288 3289 /* disable intr eventfd mapping */ 3290 rte_intr_disable(intr_handle); 3291 3292 /* Clean datapath event and queue/vec mapping */ 3293 rte_intr_efd_disable(intr_handle); 3294 if (intr_handle->intr_vec) { 3295 rte_free(intr_handle->intr_vec); 3296 intr_handle->intr_vec = NULL; 3297 } 3298 } 3299 3300 static void 3301 igbvf_dev_close(struct rte_eth_dev *dev) 3302 { 3303 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3304 struct e1000_adapter *adapter = 3305 E1000_DEV_PRIVATE(dev->data->dev_private); 3306 struct ether_addr addr; 3307 3308 PMD_INIT_FUNC_TRACE(); 3309 3310 e1000_reset_hw(hw); 3311 3312 igbvf_dev_stop(dev); 3313 adapter->stopped = 1; 3314 igb_dev_free_queues(dev); 3315 3316 /** 3317 * reprogram the RAR with a zero mac address, 3318 * to ensure that the VF traffic goes to the PF 3319 * after stop, close and detach of the VF. 3320 **/ 3321 3322 memset(&addr, 0, sizeof(addr)); 3323 igbvf_default_mac_addr_set(dev, &addr); 3324 } 3325 3326 static void 3327 igbvf_promiscuous_enable(struct rte_eth_dev *dev) 3328 { 3329 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3330 3331 /* Set both unicast and multicast promisc */ 3332 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 3333 } 3334 3335 static void 3336 igbvf_promiscuous_disable(struct rte_eth_dev *dev) 3337 { 3338 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3339 3340 /* If in allmulticast mode leave multicast promisc */ 3341 if (dev->data->all_multicast == 1) 3342 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3343 else 3344 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3345 } 3346 3347 static void 3348 igbvf_allmulticast_enable(struct rte_eth_dev *dev) 3349 { 3350 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3351 3352 /* In promiscuous mode multicast promisc already set */ 3353 if (dev->data->promiscuous == 0) 3354 e1000_promisc_set_vf(hw, e1000_promisc_multicast); 3355 } 3356 3357 static void 3358 igbvf_allmulticast_disable(struct rte_eth_dev *dev) 3359 { 3360 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3361 3362 /* In promiscuous mode leave multicast promisc enabled */ 3363 if (dev->data->promiscuous == 0) 3364 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 3365 } 3366 3367 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) 3368 { 3369 struct e1000_mbx_info *mbx = &hw->mbx; 3370 uint32_t msgbuf[2]; 3371 s32 err; 3372 3373 /* After set vlan, vlan strip will also be enabled in igb driver*/ 3374 msgbuf[0] = E1000_VF_SET_VLAN; 3375 msgbuf[1] = vid; 3376 /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ 3377 if (on) 3378 msgbuf[0] |= E1000_VF_SET_VLAN_ADD; 3379 3380 err = mbx->ops.write_posted(hw, msgbuf, 2, 0); 3381 if (err) 3382 goto mbx_err; 3383 3384 err = mbx->ops.read_posted(hw, msgbuf, 2, 0); 3385 if (err) 3386 goto mbx_err; 3387 3388 msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS; 3389 if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK)) 3390 err = -EINVAL; 3391 3392 mbx_err: 3393 return err; 3394 } 3395 3396 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) 3397 { 3398 struct e1000_hw *hw = 3399 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3400 struct e1000_vfta * shadow_vfta = 3401 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3402 int i = 0, j = 0, vfta = 0, mask = 1; 3403 3404 for (i = 0; i < IGB_VFTA_SIZE; i++){ 3405 vfta = shadow_vfta->vfta[i]; 3406 if(vfta){ 3407 mask = 1; 3408 for (j = 0; j < 32; j++){ 3409 if(vfta & mask) 3410 igbvf_set_vfta(hw, 3411 (uint16_t)((i<<5)+j), on); 3412 mask<<=1; 3413 } 3414 } 3415 } 3416 3417 } 3418 3419 static int 3420 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 3421 { 3422 struct e1000_hw *hw = 3423 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3424 struct e1000_vfta * shadow_vfta = 3425 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 3426 uint32_t vid_idx = 0; 3427 uint32_t vid_bit = 0; 3428 int ret = 0; 3429 3430 PMD_INIT_FUNC_TRACE(); 3431 3432 /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ 3433 ret = igbvf_set_vfta(hw, vlan_id, !!on); 3434 if(ret){ 3435 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 3436 return ret; 3437 } 3438 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 3439 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 3440 3441 /*Save what we set and retore it after device reset*/ 3442 if (on) 3443 shadow_vfta->vfta[vid_idx] |= vid_bit; 3444 else 3445 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 3446 3447 return 0; 3448 } 3449 3450 static void 3451 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr) 3452 { 3453 struct e1000_hw *hw = 3454 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3455 3456 /* index is not used by rar_set() */ 3457 hw->mac.ops.rar_set(hw, (void *)addr, 0); 3458 } 3459 3460 3461 static int 3462 eth_igb_rss_reta_update(struct rte_eth_dev *dev, 3463 struct rte_eth_rss_reta_entry64 *reta_conf, 3464 uint16_t reta_size) 3465 { 3466 uint8_t i, j, mask; 3467 uint32_t reta, r; 3468 uint16_t idx, shift; 3469 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3470 3471 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3472 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3473 "(%d) doesn't match the number hardware can supported " 3474 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3475 return -EINVAL; 3476 } 3477 3478 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3479 idx = i / RTE_RETA_GROUP_SIZE; 3480 shift = i % RTE_RETA_GROUP_SIZE; 3481 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3482 IGB_4_BIT_MASK); 3483 if (!mask) 3484 continue; 3485 if (mask == IGB_4_BIT_MASK) 3486 r = 0; 3487 else 3488 r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3489 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { 3490 if (mask & (0x1 << j)) 3491 reta |= reta_conf[idx].reta[shift + j] << 3492 (CHAR_BIT * j); 3493 else 3494 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); 3495 } 3496 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); 3497 } 3498 3499 return 0; 3500 } 3501 3502 static int 3503 eth_igb_rss_reta_query(struct rte_eth_dev *dev, 3504 struct rte_eth_rss_reta_entry64 *reta_conf, 3505 uint16_t reta_size) 3506 { 3507 uint8_t i, j, mask; 3508 uint32_t reta; 3509 uint16_t idx, shift; 3510 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3511 3512 if (reta_size != ETH_RSS_RETA_SIZE_128) { 3513 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 3514 "(%d) doesn't match the number hardware can supported " 3515 "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); 3516 return -EINVAL; 3517 } 3518 3519 for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { 3520 idx = i / RTE_RETA_GROUP_SIZE; 3521 shift = i % RTE_RETA_GROUP_SIZE; 3522 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 3523 IGB_4_BIT_MASK); 3524 if (!mask) 3525 continue; 3526 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); 3527 for (j = 0; j < IGB_4_BIT_WIDTH; j++) { 3528 if (mask & (0x1 << j)) 3529 reta_conf[idx].reta[shift + j] = 3530 ((reta >> (CHAR_BIT * j)) & 3531 IGB_8_BIT_MASK); 3532 } 3533 } 3534 3535 return 0; 3536 } 3537 3538 int 3539 eth_igb_syn_filter_set(struct rte_eth_dev *dev, 3540 struct rte_eth_syn_filter *filter, 3541 bool add) 3542 { 3543 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3544 struct e1000_filter_info *filter_info = 3545 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3546 uint32_t synqf, rfctl; 3547 3548 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3549 return -EINVAL; 3550 3551 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3552 3553 if (add) { 3554 if (synqf & E1000_SYN_FILTER_ENABLE) 3555 return -EINVAL; 3556 3557 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & 3558 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); 3559 3560 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3561 if (filter->hig_pri) 3562 rfctl |= E1000_RFCTL_SYNQFP; 3563 else 3564 rfctl &= ~E1000_RFCTL_SYNQFP; 3565 3566 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); 3567 } else { 3568 if (!(synqf & E1000_SYN_FILTER_ENABLE)) 3569 return -ENOENT; 3570 synqf = 0; 3571 } 3572 3573 filter_info->syn_info = synqf; 3574 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 3575 E1000_WRITE_FLUSH(hw); 3576 return 0; 3577 } 3578 3579 static int 3580 eth_igb_syn_filter_get(struct rte_eth_dev *dev, 3581 struct rte_eth_syn_filter *filter) 3582 { 3583 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3584 uint32_t synqf, rfctl; 3585 3586 synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); 3587 if (synqf & E1000_SYN_FILTER_ENABLE) { 3588 rfctl = E1000_READ_REG(hw, E1000_RFCTL); 3589 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; 3590 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> 3591 E1000_SYN_FILTER_QUEUE_SHIFT); 3592 return 0; 3593 } 3594 3595 return -ENOENT; 3596 } 3597 3598 static int 3599 eth_igb_syn_filter_handle(struct rte_eth_dev *dev, 3600 enum rte_filter_op filter_op, 3601 void *arg) 3602 { 3603 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3604 int ret; 3605 3606 MAC_TYPE_FILTER_SUP(hw->mac.type); 3607 3608 if (filter_op == RTE_ETH_FILTER_NOP) 3609 return 0; 3610 3611 if (arg == NULL) { 3612 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 3613 filter_op); 3614 return -EINVAL; 3615 } 3616 3617 switch (filter_op) { 3618 case RTE_ETH_FILTER_ADD: 3619 ret = eth_igb_syn_filter_set(dev, 3620 (struct rte_eth_syn_filter *)arg, 3621 TRUE); 3622 break; 3623 case RTE_ETH_FILTER_DELETE: 3624 ret = eth_igb_syn_filter_set(dev, 3625 (struct rte_eth_syn_filter *)arg, 3626 FALSE); 3627 break; 3628 case RTE_ETH_FILTER_GET: 3629 ret = eth_igb_syn_filter_get(dev, 3630 (struct rte_eth_syn_filter *)arg); 3631 break; 3632 default: 3633 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 3634 ret = -EINVAL; 3635 break; 3636 } 3637 3638 return ret; 3639 } 3640 3641 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ 3642 static inline int 3643 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, 3644 struct e1000_2tuple_filter_info *filter_info) 3645 { 3646 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) 3647 return -EINVAL; 3648 if (filter->priority > E1000_2TUPLE_MAX_PRI) 3649 return -EINVAL; /* filter index is out of range. */ 3650 if (filter->tcp_flags > TCP_FLAG_ALL) 3651 return -EINVAL; /* flags is invalid. */ 3652 3653 switch (filter->dst_port_mask) { 3654 case UINT16_MAX: 3655 filter_info->dst_port_mask = 0; 3656 filter_info->dst_port = filter->dst_port; 3657 break; 3658 case 0: 3659 filter_info->dst_port_mask = 1; 3660 break; 3661 default: 3662 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 3663 return -EINVAL; 3664 } 3665 3666 switch (filter->proto_mask) { 3667 case UINT8_MAX: 3668 filter_info->proto_mask = 0; 3669 filter_info->proto = filter->proto; 3670 break; 3671 case 0: 3672 filter_info->proto_mask = 1; 3673 break; 3674 default: 3675 PMD_DRV_LOG(ERR, "invalid protocol mask."); 3676 return -EINVAL; 3677 } 3678 3679 filter_info->priority = (uint8_t)filter->priority; 3680 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 3681 filter_info->tcp_flags = filter->tcp_flags; 3682 else 3683 filter_info->tcp_flags = 0; 3684 3685 return 0; 3686 } 3687 3688 static inline struct e1000_2tuple_filter * 3689 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, 3690 struct e1000_2tuple_filter_info *key) 3691 { 3692 struct e1000_2tuple_filter *it; 3693 3694 TAILQ_FOREACH(it, filter_list, entries) { 3695 if (memcmp(key, &it->filter_info, 3696 sizeof(struct e1000_2tuple_filter_info)) == 0) { 3697 return it; 3698 } 3699 } 3700 return NULL; 3701 } 3702 3703 /* inject a igb 2tuple filter to HW */ 3704 static inline void 3705 igb_inject_2uple_filter(struct rte_eth_dev *dev, 3706 struct e1000_2tuple_filter *filter) 3707 { 3708 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3709 uint32_t ttqf = E1000_TTQF_DISABLE_MASK; 3710 uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; 3711 int i; 3712 3713 i = filter->index; 3714 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 3715 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 3716 imir |= E1000_IMIR_PORT_BP; 3717 else 3718 imir &= ~E1000_IMIR_PORT_BP; 3719 3720 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 3721 3722 ttqf |= E1000_TTQF_QUEUE_ENABLE; 3723 ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); 3724 ttqf |= (uint32_t)(filter->filter_info.proto & 3725 E1000_TTQF_PROTOCOL_MASK); 3726 if (filter->filter_info.proto_mask == 0) 3727 ttqf &= ~E1000_TTQF_MASK_ENABLE; 3728 3729 /* tcp flags bits setting. */ 3730 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 3731 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 3732 imir_ext |= E1000_IMIREXT_CTRL_URG; 3733 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 3734 imir_ext |= E1000_IMIREXT_CTRL_ACK; 3735 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 3736 imir_ext |= E1000_IMIREXT_CTRL_PSH; 3737 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 3738 imir_ext |= E1000_IMIREXT_CTRL_RST; 3739 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 3740 imir_ext |= E1000_IMIREXT_CTRL_SYN; 3741 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 3742 imir_ext |= E1000_IMIREXT_CTRL_FIN; 3743 } else { 3744 imir_ext |= E1000_IMIREXT_CTRL_BP; 3745 } 3746 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 3747 E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); 3748 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 3749 } 3750 3751 /* 3752 * igb_add_2tuple_filter - add a 2tuple filter 3753 * 3754 * @param 3755 * dev: Pointer to struct rte_eth_dev. 3756 * ntuple_filter: ponter to the filter that will be added. 3757 * 3758 * @return 3759 * - On success, zero. 3760 * - On failure, a negative value. 3761 */ 3762 static int 3763 igb_add_2tuple_filter(struct rte_eth_dev *dev, 3764 struct rte_eth_ntuple_filter *ntuple_filter) 3765 { 3766 struct e1000_filter_info *filter_info = 3767 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3768 struct e1000_2tuple_filter *filter; 3769 int i, ret; 3770 3771 filter = rte_zmalloc("e1000_2tuple_filter", 3772 sizeof(struct e1000_2tuple_filter), 0); 3773 if (filter == NULL) 3774 return -ENOMEM; 3775 3776 ret = ntuple_filter_to_2tuple(ntuple_filter, 3777 &filter->filter_info); 3778 if (ret < 0) { 3779 rte_free(filter); 3780 return ret; 3781 } 3782 if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3783 &filter->filter_info) != NULL) { 3784 PMD_DRV_LOG(ERR, "filter exists."); 3785 rte_free(filter); 3786 return -EEXIST; 3787 } 3788 filter->queue = ntuple_filter->queue; 3789 3790 /* 3791 * look for an unused 2tuple filter index, 3792 * and insert the filter to list. 3793 */ 3794 for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { 3795 if (!(filter_info->twotuple_mask & (1 << i))) { 3796 filter_info->twotuple_mask |= 1 << i; 3797 filter->index = i; 3798 TAILQ_INSERT_TAIL(&filter_info->twotuple_list, 3799 filter, 3800 entries); 3801 break; 3802 } 3803 } 3804 if (i >= E1000_MAX_TTQF_FILTERS) { 3805 PMD_DRV_LOG(ERR, "2tuple filters are full."); 3806 rte_free(filter); 3807 return -ENOSYS; 3808 } 3809 3810 igb_inject_2uple_filter(dev, filter); 3811 return 0; 3812 } 3813 3814 int 3815 igb_delete_2tuple_filter(struct rte_eth_dev *dev, 3816 struct e1000_2tuple_filter *filter) 3817 { 3818 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3819 struct e1000_filter_info *filter_info = 3820 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3821 3822 filter_info->twotuple_mask &= ~(1 << filter->index); 3823 TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); 3824 rte_free(filter); 3825 3826 E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); 3827 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 3828 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 3829 return 0; 3830 } 3831 3832 /* 3833 * igb_remove_2tuple_filter - remove a 2tuple filter 3834 * 3835 * @param 3836 * dev: Pointer to struct rte_eth_dev. 3837 * ntuple_filter: ponter to the filter that will be removed. 3838 * 3839 * @return 3840 * - On success, zero. 3841 * - On failure, a negative value. 3842 */ 3843 static int 3844 igb_remove_2tuple_filter(struct rte_eth_dev *dev, 3845 struct rte_eth_ntuple_filter *ntuple_filter) 3846 { 3847 struct e1000_filter_info *filter_info = 3848 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3849 struct e1000_2tuple_filter_info filter_2tuple; 3850 struct e1000_2tuple_filter *filter; 3851 int ret; 3852 3853 memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); 3854 ret = ntuple_filter_to_2tuple(ntuple_filter, 3855 &filter_2tuple); 3856 if (ret < 0) 3857 return ret; 3858 3859 filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, 3860 &filter_2tuple); 3861 if (filter == NULL) { 3862 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3863 return -ENOENT; 3864 } 3865 3866 igb_delete_2tuple_filter(dev, filter); 3867 3868 return 0; 3869 } 3870 3871 /* inject a igb flex filter to HW */ 3872 static inline void 3873 igb_inject_flex_filter(struct rte_eth_dev *dev, 3874 struct e1000_flex_filter *filter) 3875 { 3876 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3877 uint32_t wufc, queueing; 3878 uint32_t reg_off; 3879 uint8_t i, j = 0; 3880 3881 wufc = E1000_READ_REG(hw, E1000_WUFC); 3882 if (filter->index < E1000_MAX_FHFT) 3883 reg_off = E1000_FHFT(filter->index); 3884 else 3885 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3886 3887 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | 3888 (E1000_WUFC_FLX0 << filter->index)); 3889 queueing = filter->filter_info.len | 3890 (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | 3891 (filter->filter_info.priority << 3892 E1000_FHFT_QUEUEING_PRIO_SHIFT); 3893 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, 3894 queueing); 3895 3896 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { 3897 E1000_WRITE_REG(hw, reg_off, 3898 filter->filter_info.dwords[j]); 3899 reg_off += sizeof(uint32_t); 3900 E1000_WRITE_REG(hw, reg_off, 3901 filter->filter_info.dwords[++j]); 3902 reg_off += sizeof(uint32_t); 3903 E1000_WRITE_REG(hw, reg_off, 3904 (uint32_t)filter->filter_info.mask[i]); 3905 reg_off += sizeof(uint32_t) * 2; 3906 ++j; 3907 } 3908 } 3909 3910 static inline struct e1000_flex_filter * 3911 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, 3912 struct e1000_flex_filter_info *key) 3913 { 3914 struct e1000_flex_filter *it; 3915 3916 TAILQ_FOREACH(it, filter_list, entries) { 3917 if (memcmp(key, &it->filter_info, 3918 sizeof(struct e1000_flex_filter_info)) == 0) 3919 return it; 3920 } 3921 3922 return NULL; 3923 } 3924 3925 /* remove a flex byte filter 3926 * @param 3927 * dev: Pointer to struct rte_eth_dev. 3928 * filter: the pointer of the filter will be removed. 3929 */ 3930 void 3931 igb_remove_flex_filter(struct rte_eth_dev *dev, 3932 struct e1000_flex_filter *filter) 3933 { 3934 struct e1000_filter_info *filter_info = 3935 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3936 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3937 uint32_t wufc, i; 3938 uint32_t reg_off; 3939 3940 wufc = E1000_READ_REG(hw, E1000_WUFC); 3941 if (filter->index < E1000_MAX_FHFT) 3942 reg_off = E1000_FHFT(filter->index); 3943 else 3944 reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); 3945 3946 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) 3947 E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); 3948 3949 E1000_WRITE_REG(hw, E1000_WUFC, wufc & 3950 (~(E1000_WUFC_FLX0 << filter->index))); 3951 3952 filter_info->flex_mask &= ~(1 << filter->index); 3953 TAILQ_REMOVE(&filter_info->flex_list, filter, entries); 3954 rte_free(filter); 3955 } 3956 3957 int 3958 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, 3959 struct rte_eth_flex_filter *filter, 3960 bool add) 3961 { 3962 struct e1000_filter_info *filter_info = 3963 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 3964 struct e1000_flex_filter *flex_filter, *it; 3965 uint32_t mask; 3966 uint8_t shift, i; 3967 3968 flex_filter = rte_zmalloc("e1000_flex_filter", 3969 sizeof(struct e1000_flex_filter), 0); 3970 if (flex_filter == NULL) 3971 return -ENOMEM; 3972 3973 flex_filter->filter_info.len = filter->len; 3974 flex_filter->filter_info.priority = filter->priority; 3975 memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); 3976 for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { 3977 mask = 0; 3978 /* reverse bits in flex filter's mask*/ 3979 for (shift = 0; shift < CHAR_BIT; shift++) { 3980 if (filter->mask[i] & (0x01 << shift)) 3981 mask |= (0x80 >> shift); 3982 } 3983 flex_filter->filter_info.mask[i] = mask; 3984 } 3985 3986 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 3987 &flex_filter->filter_info); 3988 if (it == NULL && !add) { 3989 PMD_DRV_LOG(ERR, "filter doesn't exist."); 3990 rte_free(flex_filter); 3991 return -ENOENT; 3992 } 3993 if (it != NULL && add) { 3994 PMD_DRV_LOG(ERR, "filter exists."); 3995 rte_free(flex_filter); 3996 return -EEXIST; 3997 } 3998 3999 if (add) { 4000 flex_filter->queue = filter->queue; 4001 /* 4002 * look for an unused flex filter index 4003 * and insert the filter into the list. 4004 */ 4005 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { 4006 if (!(filter_info->flex_mask & (1 << i))) { 4007 filter_info->flex_mask |= 1 << i; 4008 flex_filter->index = i; 4009 TAILQ_INSERT_TAIL(&filter_info->flex_list, 4010 flex_filter, 4011 entries); 4012 break; 4013 } 4014 } 4015 if (i >= E1000_MAX_FLEX_FILTERS) { 4016 PMD_DRV_LOG(ERR, "flex filters are full."); 4017 rte_free(flex_filter); 4018 return -ENOSYS; 4019 } 4020 4021 igb_inject_flex_filter(dev, flex_filter); 4022 4023 } else { 4024 igb_remove_flex_filter(dev, it); 4025 rte_free(flex_filter); 4026 } 4027 4028 return 0; 4029 } 4030 4031 static int 4032 eth_igb_get_flex_filter(struct rte_eth_dev *dev, 4033 struct rte_eth_flex_filter *filter) 4034 { 4035 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4036 struct e1000_filter_info *filter_info = 4037 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4038 struct e1000_flex_filter flex_filter, *it; 4039 uint32_t wufc, queueing, wufc_en = 0; 4040 4041 memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); 4042 flex_filter.filter_info.len = filter->len; 4043 flex_filter.filter_info.priority = filter->priority; 4044 memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); 4045 memcpy(flex_filter.filter_info.mask, filter->mask, 4046 RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); 4047 4048 it = eth_igb_flex_filter_lookup(&filter_info->flex_list, 4049 &flex_filter.filter_info); 4050 if (it == NULL) { 4051 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4052 return -ENOENT; 4053 } 4054 4055 wufc = E1000_READ_REG(hw, E1000_WUFC); 4056 wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); 4057 4058 if ((wufc & wufc_en) == wufc_en) { 4059 uint32_t reg_off = 0; 4060 if (it->index < E1000_MAX_FHFT) 4061 reg_off = E1000_FHFT(it->index); 4062 else 4063 reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); 4064 4065 queueing = E1000_READ_REG(hw, 4066 reg_off + E1000_FHFT_QUEUEING_OFFSET); 4067 filter->len = queueing & E1000_FHFT_QUEUEING_LEN; 4068 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> 4069 E1000_FHFT_QUEUEING_PRIO_SHIFT; 4070 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> 4071 E1000_FHFT_QUEUEING_QUEUE_SHIFT; 4072 return 0; 4073 } 4074 return -ENOENT; 4075 } 4076 4077 static int 4078 eth_igb_flex_filter_handle(struct rte_eth_dev *dev, 4079 enum rte_filter_op filter_op, 4080 void *arg) 4081 { 4082 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4083 struct rte_eth_flex_filter *filter; 4084 int ret = 0; 4085 4086 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 4087 4088 if (filter_op == RTE_ETH_FILTER_NOP) 4089 return ret; 4090 4091 if (arg == NULL) { 4092 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 4093 filter_op); 4094 return -EINVAL; 4095 } 4096 4097 filter = (struct rte_eth_flex_filter *)arg; 4098 if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN 4099 || filter->len % sizeof(uint64_t) != 0) { 4100 PMD_DRV_LOG(ERR, "filter's length is out of range"); 4101 return -EINVAL; 4102 } 4103 if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { 4104 PMD_DRV_LOG(ERR, "filter's priority is out of range"); 4105 return -EINVAL; 4106 } 4107 4108 switch (filter_op) { 4109 case RTE_ETH_FILTER_ADD: 4110 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); 4111 break; 4112 case RTE_ETH_FILTER_DELETE: 4113 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); 4114 break; 4115 case RTE_ETH_FILTER_GET: 4116 ret = eth_igb_get_flex_filter(dev, filter); 4117 break; 4118 default: 4119 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 4120 ret = -EINVAL; 4121 break; 4122 } 4123 4124 return ret; 4125 } 4126 4127 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ 4128 static inline int 4129 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, 4130 struct e1000_5tuple_filter_info *filter_info) 4131 { 4132 if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) 4133 return -EINVAL; 4134 if (filter->priority > E1000_2TUPLE_MAX_PRI) 4135 return -EINVAL; /* filter index is out of range. */ 4136 if (filter->tcp_flags > TCP_FLAG_ALL) 4137 return -EINVAL; /* flags is invalid. */ 4138 4139 switch (filter->dst_ip_mask) { 4140 case UINT32_MAX: 4141 filter_info->dst_ip_mask = 0; 4142 filter_info->dst_ip = filter->dst_ip; 4143 break; 4144 case 0: 4145 filter_info->dst_ip_mask = 1; 4146 break; 4147 default: 4148 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 4149 return -EINVAL; 4150 } 4151 4152 switch (filter->src_ip_mask) { 4153 case UINT32_MAX: 4154 filter_info->src_ip_mask = 0; 4155 filter_info->src_ip = filter->src_ip; 4156 break; 4157 case 0: 4158 filter_info->src_ip_mask = 1; 4159 break; 4160 default: 4161 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 4162 return -EINVAL; 4163 } 4164 4165 switch (filter->dst_port_mask) { 4166 case UINT16_MAX: 4167 filter_info->dst_port_mask = 0; 4168 filter_info->dst_port = filter->dst_port; 4169 break; 4170 case 0: 4171 filter_info->dst_port_mask = 1; 4172 break; 4173 default: 4174 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 4175 return -EINVAL; 4176 } 4177 4178 switch (filter->src_port_mask) { 4179 case UINT16_MAX: 4180 filter_info->src_port_mask = 0; 4181 filter_info->src_port = filter->src_port; 4182 break; 4183 case 0: 4184 filter_info->src_port_mask = 1; 4185 break; 4186 default: 4187 PMD_DRV_LOG(ERR, "invalid src_port mask."); 4188 return -EINVAL; 4189 } 4190 4191 switch (filter->proto_mask) { 4192 case UINT8_MAX: 4193 filter_info->proto_mask = 0; 4194 filter_info->proto = filter->proto; 4195 break; 4196 case 0: 4197 filter_info->proto_mask = 1; 4198 break; 4199 default: 4200 PMD_DRV_LOG(ERR, "invalid protocol mask."); 4201 return -EINVAL; 4202 } 4203 4204 filter_info->priority = (uint8_t)filter->priority; 4205 if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) 4206 filter_info->tcp_flags = filter->tcp_flags; 4207 else 4208 filter_info->tcp_flags = 0; 4209 4210 return 0; 4211 } 4212 4213 static inline struct e1000_5tuple_filter * 4214 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, 4215 struct e1000_5tuple_filter_info *key) 4216 { 4217 struct e1000_5tuple_filter *it; 4218 4219 TAILQ_FOREACH(it, filter_list, entries) { 4220 if (memcmp(key, &it->filter_info, 4221 sizeof(struct e1000_5tuple_filter_info)) == 0) { 4222 return it; 4223 } 4224 } 4225 return NULL; 4226 } 4227 4228 /* inject a igb 5-tuple filter to HW */ 4229 static inline void 4230 igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, 4231 struct e1000_5tuple_filter *filter) 4232 { 4233 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4234 uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; 4235 uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; 4236 uint8_t i; 4237 4238 i = filter->index; 4239 ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; 4240 if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ 4241 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; 4242 if (filter->filter_info.dst_ip_mask == 0) 4243 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; 4244 if (filter->filter_info.src_port_mask == 0) 4245 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 4246 if (filter->filter_info.proto_mask == 0) 4247 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; 4248 ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & 4249 E1000_FTQF_QUEUE_MASK; 4250 ftqf |= E1000_FTQF_QUEUE_ENABLE; 4251 E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); 4252 E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); 4253 E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); 4254 4255 spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; 4256 E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); 4257 4258 imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); 4259 if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ 4260 imir |= E1000_IMIR_PORT_BP; 4261 else 4262 imir &= ~E1000_IMIR_PORT_BP; 4263 imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; 4264 4265 /* tcp flags bits setting. */ 4266 if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { 4267 if (filter->filter_info.tcp_flags & TCP_URG_FLAG) 4268 imir_ext |= E1000_IMIREXT_CTRL_URG; 4269 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) 4270 imir_ext |= E1000_IMIREXT_CTRL_ACK; 4271 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) 4272 imir_ext |= E1000_IMIREXT_CTRL_PSH; 4273 if (filter->filter_info.tcp_flags & TCP_RST_FLAG) 4274 imir_ext |= E1000_IMIREXT_CTRL_RST; 4275 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) 4276 imir_ext |= E1000_IMIREXT_CTRL_SYN; 4277 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) 4278 imir_ext |= E1000_IMIREXT_CTRL_FIN; 4279 } else { 4280 imir_ext |= E1000_IMIREXT_CTRL_BP; 4281 } 4282 E1000_WRITE_REG(hw, E1000_IMIR(i), imir); 4283 E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); 4284 } 4285 4286 /* 4287 * igb_add_5tuple_filter_82576 - add a 5tuple filter 4288 * 4289 * @param 4290 * dev: Pointer to struct rte_eth_dev. 4291 * ntuple_filter: ponter to the filter that will be added. 4292 * 4293 * @return 4294 * - On success, zero. 4295 * - On failure, a negative value. 4296 */ 4297 static int 4298 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, 4299 struct rte_eth_ntuple_filter *ntuple_filter) 4300 { 4301 struct e1000_filter_info *filter_info = 4302 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4303 struct e1000_5tuple_filter *filter; 4304 uint8_t i; 4305 int ret; 4306 4307 filter = rte_zmalloc("e1000_5tuple_filter", 4308 sizeof(struct e1000_5tuple_filter), 0); 4309 if (filter == NULL) 4310 return -ENOMEM; 4311 4312 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4313 &filter->filter_info); 4314 if (ret < 0) { 4315 rte_free(filter); 4316 return ret; 4317 } 4318 4319 if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4320 &filter->filter_info) != NULL) { 4321 PMD_DRV_LOG(ERR, "filter exists."); 4322 rte_free(filter); 4323 return -EEXIST; 4324 } 4325 filter->queue = ntuple_filter->queue; 4326 4327 /* 4328 * look for an unused 5tuple filter index, 4329 * and insert the filter to list. 4330 */ 4331 for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { 4332 if (!(filter_info->fivetuple_mask & (1 << i))) { 4333 filter_info->fivetuple_mask |= 1 << i; 4334 filter->index = i; 4335 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 4336 filter, 4337 entries); 4338 break; 4339 } 4340 } 4341 if (i >= E1000_MAX_FTQF_FILTERS) { 4342 PMD_DRV_LOG(ERR, "5tuple filters are full."); 4343 rte_free(filter); 4344 return -ENOSYS; 4345 } 4346 4347 igb_inject_5tuple_filter_82576(dev, filter); 4348 return 0; 4349 } 4350 4351 int 4352 igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, 4353 struct e1000_5tuple_filter *filter) 4354 { 4355 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4356 struct e1000_filter_info *filter_info = 4357 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4358 4359 filter_info->fivetuple_mask &= ~(1 << filter->index); 4360 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 4361 rte_free(filter); 4362 4363 E1000_WRITE_REG(hw, E1000_FTQF(filter->index), 4364 E1000_FTQF_VF_BP | E1000_FTQF_MASK); 4365 E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); 4366 E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); 4367 E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); 4368 E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); 4369 E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); 4370 return 0; 4371 } 4372 4373 /* 4374 * igb_remove_5tuple_filter_82576 - remove a 5tuple filter 4375 * 4376 * @param 4377 * dev: Pointer to struct rte_eth_dev. 4378 * ntuple_filter: ponter to the filter that will be removed. 4379 * 4380 * @return 4381 * - On success, zero. 4382 * - On failure, a negative value. 4383 */ 4384 static int 4385 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, 4386 struct rte_eth_ntuple_filter *ntuple_filter) 4387 { 4388 struct e1000_filter_info *filter_info = 4389 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4390 struct e1000_5tuple_filter_info filter_5tuple; 4391 struct e1000_5tuple_filter *filter; 4392 int ret; 4393 4394 memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); 4395 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4396 &filter_5tuple); 4397 if (ret < 0) 4398 return ret; 4399 4400 filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, 4401 &filter_5tuple); 4402 if (filter == NULL) { 4403 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4404 return -ENOENT; 4405 } 4406 4407 igb_delete_5tuple_filter_82576(dev, filter); 4408 4409 return 0; 4410 } 4411 4412 static int 4413 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4414 { 4415 uint32_t rctl; 4416 struct e1000_hw *hw; 4417 struct rte_eth_dev_info dev_info; 4418 uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4419 VLAN_TAG_SIZE); 4420 4421 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4422 4423 #ifdef RTE_LIBRTE_82571_SUPPORT 4424 /* XXX: not bigger than max_rx_pktlen */ 4425 if (hw->mac.type == e1000_82571) 4426 return -ENOTSUP; 4427 #endif 4428 eth_igb_infos_get(dev, &dev_info); 4429 4430 /* check that mtu is within the allowed range */ 4431 if ((mtu < ETHER_MIN_MTU) || 4432 (frame_size > dev_info.max_rx_pktlen)) 4433 return -EINVAL; 4434 4435 /* refuse mtu that requires the support of scattered packets when this 4436 * feature has not been enabled before. */ 4437 if (!dev->data->scattered_rx && 4438 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) 4439 return -EINVAL; 4440 4441 rctl = E1000_READ_REG(hw, E1000_RCTL); 4442 4443 /* switch to jumbo mode if needed */ 4444 if (frame_size > ETHER_MAX_LEN) { 4445 dev->data->dev_conf.rxmode.offloads |= 4446 DEV_RX_OFFLOAD_JUMBO_FRAME; 4447 rctl |= E1000_RCTL_LPE; 4448 } else { 4449 dev->data->dev_conf.rxmode.offloads &= 4450 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 4451 rctl &= ~E1000_RCTL_LPE; 4452 } 4453 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4454 4455 /* update max frame size */ 4456 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4457 4458 E1000_WRITE_REG(hw, E1000_RLPML, 4459 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4460 4461 return 0; 4462 } 4463 4464 /* 4465 * igb_add_del_ntuple_filter - add or delete a ntuple filter 4466 * 4467 * @param 4468 * dev: Pointer to struct rte_eth_dev. 4469 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4470 * add: if true, add filter, if false, remove filter 4471 * 4472 * @return 4473 * - On success, zero. 4474 * - On failure, a negative value. 4475 */ 4476 int 4477 igb_add_del_ntuple_filter(struct rte_eth_dev *dev, 4478 struct rte_eth_ntuple_filter *ntuple_filter, 4479 bool add) 4480 { 4481 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4482 int ret; 4483 4484 switch (ntuple_filter->flags) { 4485 case RTE_5TUPLE_FLAGS: 4486 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4487 if (hw->mac.type != e1000_82576) 4488 return -ENOTSUP; 4489 if (add) 4490 ret = igb_add_5tuple_filter_82576(dev, 4491 ntuple_filter); 4492 else 4493 ret = igb_remove_5tuple_filter_82576(dev, 4494 ntuple_filter); 4495 break; 4496 case RTE_2TUPLE_FLAGS: 4497 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4498 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && 4499 hw->mac.type != e1000_i210 && 4500 hw->mac.type != e1000_i211) 4501 return -ENOTSUP; 4502 if (add) 4503 ret = igb_add_2tuple_filter(dev, ntuple_filter); 4504 else 4505 ret = igb_remove_2tuple_filter(dev, ntuple_filter); 4506 break; 4507 default: 4508 ret = -EINVAL; 4509 break; 4510 } 4511 4512 return ret; 4513 } 4514 4515 /* 4516 * igb_get_ntuple_filter - get a ntuple filter 4517 * 4518 * @param 4519 * dev: Pointer to struct rte_eth_dev. 4520 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 4521 * 4522 * @return 4523 * - On success, zero. 4524 * - On failure, a negative value. 4525 */ 4526 static int 4527 igb_get_ntuple_filter(struct rte_eth_dev *dev, 4528 struct rte_eth_ntuple_filter *ntuple_filter) 4529 { 4530 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4531 struct e1000_filter_info *filter_info = 4532 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4533 struct e1000_5tuple_filter_info filter_5tuple; 4534 struct e1000_2tuple_filter_info filter_2tuple; 4535 struct e1000_5tuple_filter *p_5tuple_filter; 4536 struct e1000_2tuple_filter *p_2tuple_filter; 4537 int ret; 4538 4539 switch (ntuple_filter->flags) { 4540 case RTE_5TUPLE_FLAGS: 4541 case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4542 if (hw->mac.type != e1000_82576) 4543 return -ENOTSUP; 4544 memset(&filter_5tuple, 4545 0, 4546 sizeof(struct e1000_5tuple_filter_info)); 4547 ret = ntuple_filter_to_5tuple_82576(ntuple_filter, 4548 &filter_5tuple); 4549 if (ret < 0) 4550 return ret; 4551 p_5tuple_filter = igb_5tuple_filter_lookup_82576( 4552 &filter_info->fivetuple_list, 4553 &filter_5tuple); 4554 if (p_5tuple_filter == NULL) { 4555 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4556 return -ENOENT; 4557 } 4558 ntuple_filter->queue = p_5tuple_filter->queue; 4559 break; 4560 case RTE_2TUPLE_FLAGS: 4561 case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): 4562 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) 4563 return -ENOTSUP; 4564 memset(&filter_2tuple, 4565 0, 4566 sizeof(struct e1000_2tuple_filter_info)); 4567 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); 4568 if (ret < 0) 4569 return ret; 4570 p_2tuple_filter = igb_2tuple_filter_lookup( 4571 &filter_info->twotuple_list, 4572 &filter_2tuple); 4573 if (p_2tuple_filter == NULL) { 4574 PMD_DRV_LOG(ERR, "filter doesn't exist."); 4575 return -ENOENT; 4576 } 4577 ntuple_filter->queue = p_2tuple_filter->queue; 4578 break; 4579 default: 4580 ret = -EINVAL; 4581 break; 4582 } 4583 4584 return 0; 4585 } 4586 4587 /* 4588 * igb_ntuple_filter_handle - Handle operations for ntuple filter. 4589 * @dev: pointer to rte_eth_dev structure 4590 * @filter_op:operation will be taken. 4591 * @arg: a pointer to specific structure corresponding to the filter_op 4592 */ 4593 static int 4594 igb_ntuple_filter_handle(struct rte_eth_dev *dev, 4595 enum rte_filter_op filter_op, 4596 void *arg) 4597 { 4598 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4599 int ret; 4600 4601 MAC_TYPE_FILTER_SUP(hw->mac.type); 4602 4603 if (filter_op == RTE_ETH_FILTER_NOP) 4604 return 0; 4605 4606 if (arg == NULL) { 4607 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4608 filter_op); 4609 return -EINVAL; 4610 } 4611 4612 switch (filter_op) { 4613 case RTE_ETH_FILTER_ADD: 4614 ret = igb_add_del_ntuple_filter(dev, 4615 (struct rte_eth_ntuple_filter *)arg, 4616 TRUE); 4617 break; 4618 case RTE_ETH_FILTER_DELETE: 4619 ret = igb_add_del_ntuple_filter(dev, 4620 (struct rte_eth_ntuple_filter *)arg, 4621 FALSE); 4622 break; 4623 case RTE_ETH_FILTER_GET: 4624 ret = igb_get_ntuple_filter(dev, 4625 (struct rte_eth_ntuple_filter *)arg); 4626 break; 4627 default: 4628 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4629 ret = -EINVAL; 4630 break; 4631 } 4632 return ret; 4633 } 4634 4635 static inline int 4636 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, 4637 uint16_t ethertype) 4638 { 4639 int i; 4640 4641 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4642 if (filter_info->ethertype_filters[i].ethertype == ethertype && 4643 (filter_info->ethertype_mask & (1 << i))) 4644 return i; 4645 } 4646 return -1; 4647 } 4648 4649 static inline int 4650 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, 4651 uint16_t ethertype, uint32_t etqf) 4652 { 4653 int i; 4654 4655 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 4656 if (!(filter_info->ethertype_mask & (1 << i))) { 4657 filter_info->ethertype_mask |= 1 << i; 4658 filter_info->ethertype_filters[i].ethertype = ethertype; 4659 filter_info->ethertype_filters[i].etqf = etqf; 4660 return i; 4661 } 4662 } 4663 return -1; 4664 } 4665 4666 int 4667 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, 4668 uint8_t idx) 4669 { 4670 if (idx >= E1000_MAX_ETQF_FILTERS) 4671 return -1; 4672 filter_info->ethertype_mask &= ~(1 << idx); 4673 filter_info->ethertype_filters[idx].ethertype = 0; 4674 filter_info->ethertype_filters[idx].etqf = 0; 4675 return idx; 4676 } 4677 4678 4679 int 4680 igb_add_del_ethertype_filter(struct rte_eth_dev *dev, 4681 struct rte_eth_ethertype_filter *filter, 4682 bool add) 4683 { 4684 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4685 struct e1000_filter_info *filter_info = 4686 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4687 uint32_t etqf = 0; 4688 int ret; 4689 4690 if (filter->ether_type == ETHER_TYPE_IPv4 || 4691 filter->ether_type == ETHER_TYPE_IPv6) { 4692 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 4693 " ethertype filter.", filter->ether_type); 4694 return -EINVAL; 4695 } 4696 4697 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 4698 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 4699 return -EINVAL; 4700 } 4701 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 4702 PMD_DRV_LOG(ERR, "drop option is unsupported."); 4703 return -EINVAL; 4704 } 4705 4706 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4707 if (ret >= 0 && add) { 4708 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 4709 filter->ether_type); 4710 return -EEXIST; 4711 } 4712 if (ret < 0 && !add) { 4713 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4714 filter->ether_type); 4715 return -ENOENT; 4716 } 4717 4718 if (add) { 4719 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; 4720 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); 4721 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; 4722 ret = igb_ethertype_filter_insert(filter_info, 4723 filter->ether_type, etqf); 4724 if (ret < 0) { 4725 PMD_DRV_LOG(ERR, "ethertype filters are full."); 4726 return -ENOSYS; 4727 } 4728 } else { 4729 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); 4730 if (ret < 0) 4731 return -ENOSYS; 4732 } 4733 E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); 4734 E1000_WRITE_FLUSH(hw); 4735 4736 return 0; 4737 } 4738 4739 static int 4740 igb_get_ethertype_filter(struct rte_eth_dev *dev, 4741 struct rte_eth_ethertype_filter *filter) 4742 { 4743 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4744 struct e1000_filter_info *filter_info = 4745 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 4746 uint32_t etqf; 4747 int ret; 4748 4749 ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); 4750 if (ret < 0) { 4751 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 4752 filter->ether_type); 4753 return -ENOENT; 4754 } 4755 4756 etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); 4757 if (etqf & E1000_ETQF_FILTER_ENABLE) { 4758 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; 4759 filter->flags = 0; 4760 filter->queue = (etqf & E1000_ETQF_QUEUE) >> 4761 E1000_ETQF_QUEUE_SHIFT; 4762 return 0; 4763 } 4764 4765 return -ENOENT; 4766 } 4767 4768 /* 4769 * igb_ethertype_filter_handle - Handle operations for ethertype filter. 4770 * @dev: pointer to rte_eth_dev structure 4771 * @filter_op:operation will be taken. 4772 * @arg: a pointer to specific structure corresponding to the filter_op 4773 */ 4774 static int 4775 igb_ethertype_filter_handle(struct rte_eth_dev *dev, 4776 enum rte_filter_op filter_op, 4777 void *arg) 4778 { 4779 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4780 int ret; 4781 4782 MAC_TYPE_FILTER_SUP(hw->mac.type); 4783 4784 if (filter_op == RTE_ETH_FILTER_NOP) 4785 return 0; 4786 4787 if (arg == NULL) { 4788 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 4789 filter_op); 4790 return -EINVAL; 4791 } 4792 4793 switch (filter_op) { 4794 case RTE_ETH_FILTER_ADD: 4795 ret = igb_add_del_ethertype_filter(dev, 4796 (struct rte_eth_ethertype_filter *)arg, 4797 TRUE); 4798 break; 4799 case RTE_ETH_FILTER_DELETE: 4800 ret = igb_add_del_ethertype_filter(dev, 4801 (struct rte_eth_ethertype_filter *)arg, 4802 FALSE); 4803 break; 4804 case RTE_ETH_FILTER_GET: 4805 ret = igb_get_ethertype_filter(dev, 4806 (struct rte_eth_ethertype_filter *)arg); 4807 break; 4808 default: 4809 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 4810 ret = -EINVAL; 4811 break; 4812 } 4813 return ret; 4814 } 4815 4816 static int 4817 eth_igb_filter_ctrl(struct rte_eth_dev *dev, 4818 enum rte_filter_type filter_type, 4819 enum rte_filter_op filter_op, 4820 void *arg) 4821 { 4822 int ret = 0; 4823 4824 switch (filter_type) { 4825 case RTE_ETH_FILTER_NTUPLE: 4826 ret = igb_ntuple_filter_handle(dev, filter_op, arg); 4827 break; 4828 case RTE_ETH_FILTER_ETHERTYPE: 4829 ret = igb_ethertype_filter_handle(dev, filter_op, arg); 4830 break; 4831 case RTE_ETH_FILTER_SYN: 4832 ret = eth_igb_syn_filter_handle(dev, filter_op, arg); 4833 break; 4834 case RTE_ETH_FILTER_FLEXIBLE: 4835 ret = eth_igb_flex_filter_handle(dev, filter_op, arg); 4836 break; 4837 case RTE_ETH_FILTER_GENERIC: 4838 if (filter_op != RTE_ETH_FILTER_GET) 4839 return -EINVAL; 4840 *(const void **)arg = &igb_flow_ops; 4841 break; 4842 default: 4843 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 4844 filter_type); 4845 break; 4846 } 4847 4848 return ret; 4849 } 4850 4851 static int 4852 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev, 4853 struct ether_addr *mc_addr_set, 4854 uint32_t nb_mc_addr) 4855 { 4856 struct e1000_hw *hw; 4857 4858 hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4859 e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr); 4860 return 0; 4861 } 4862 4863 static uint64_t 4864 igb_read_systime_cyclecounter(struct rte_eth_dev *dev) 4865 { 4866 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4867 uint64_t systime_cycles; 4868 4869 switch (hw->mac.type) { 4870 case e1000_i210: 4871 case e1000_i211: 4872 /* 4873 * Need to read System Time Residue Register to be able 4874 * to read the other two registers. 4875 */ 4876 E1000_READ_REG(hw, E1000_SYSTIMR); 4877 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 4878 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4879 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4880 * NSEC_PER_SEC; 4881 break; 4882 case e1000_82580: 4883 case e1000_i350: 4884 case e1000_i354: 4885 /* 4886 * Need to read System Time Residue Register to be able 4887 * to read the other two registers. 4888 */ 4889 E1000_READ_REG(hw, E1000_SYSTIMR); 4890 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4891 /* Only the 8 LSB are valid. */ 4892 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH) 4893 & 0xff) << 32; 4894 break; 4895 default: 4896 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML); 4897 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH) 4898 << 32; 4899 break; 4900 } 4901 4902 return systime_cycles; 4903 } 4904 4905 static uint64_t 4906 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4907 { 4908 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4909 uint64_t rx_tstamp_cycles; 4910 4911 switch (hw->mac.type) { 4912 case e1000_i210: 4913 case e1000_i211: 4914 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4915 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4916 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4917 * NSEC_PER_SEC; 4918 break; 4919 case e1000_82580: 4920 case e1000_i350: 4921 case e1000_i354: 4922 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4923 /* Only the 8 LSB are valid. */ 4924 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH) 4925 & 0xff) << 32; 4926 break; 4927 default: 4928 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL); 4929 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH) 4930 << 32; 4931 break; 4932 } 4933 4934 return rx_tstamp_cycles; 4935 } 4936 4937 static uint64_t 4938 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 4939 { 4940 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4941 uint64_t tx_tstamp_cycles; 4942 4943 switch (hw->mac.type) { 4944 case e1000_i210: 4945 case e1000_i211: 4946 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 4947 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4948 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4949 * NSEC_PER_SEC; 4950 break; 4951 case e1000_82580: 4952 case e1000_i350: 4953 case e1000_i354: 4954 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4955 /* Only the 8 LSB are valid. */ 4956 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH) 4957 & 0xff) << 32; 4958 break; 4959 default: 4960 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL); 4961 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH) 4962 << 32; 4963 break; 4964 } 4965 4966 return tx_tstamp_cycles; 4967 } 4968 4969 static void 4970 igb_start_timecounters(struct rte_eth_dev *dev) 4971 { 4972 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4973 struct e1000_adapter *adapter = 4974 (struct e1000_adapter *)dev->data->dev_private; 4975 uint32_t incval = 1; 4976 uint32_t shift = 0; 4977 uint64_t mask = E1000_CYCLECOUNTER_MASK; 4978 4979 switch (hw->mac.type) { 4980 case e1000_82580: 4981 case e1000_i350: 4982 case e1000_i354: 4983 /* 32 LSB bits + 8 MSB bits = 40 bits */ 4984 mask = (1ULL << 40) - 1; 4985 /* fall-through */ 4986 case e1000_i210: 4987 case e1000_i211: 4988 /* 4989 * Start incrementing the register 4990 * used to timestamp PTP packets. 4991 */ 4992 E1000_WRITE_REG(hw, E1000_TIMINCA, incval); 4993 break; 4994 case e1000_82576: 4995 incval = E1000_INCVALUE_82576; 4996 shift = IGB_82576_TSYNC_SHIFT; 4997 E1000_WRITE_REG(hw, E1000_TIMINCA, 4998 E1000_INCPERIOD_82576 | incval); 4999 break; 5000 default: 5001 /* Not supported */ 5002 return; 5003 } 5004 5005 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 5006 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5007 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 5008 5009 adapter->systime_tc.cc_mask = mask; 5010 adapter->systime_tc.cc_shift = shift; 5011 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 5012 5013 adapter->rx_tstamp_tc.cc_mask = mask; 5014 adapter->rx_tstamp_tc.cc_shift = shift; 5015 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5016 5017 adapter->tx_tstamp_tc.cc_mask = mask; 5018 adapter->tx_tstamp_tc.cc_shift = shift; 5019 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 5020 } 5021 5022 static int 5023 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 5024 { 5025 struct e1000_adapter *adapter = 5026 (struct e1000_adapter *)dev->data->dev_private; 5027 5028 adapter->systime_tc.nsec += delta; 5029 adapter->rx_tstamp_tc.nsec += delta; 5030 adapter->tx_tstamp_tc.nsec += delta; 5031 5032 return 0; 5033 } 5034 5035 static int 5036 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 5037 { 5038 uint64_t ns; 5039 struct e1000_adapter *adapter = 5040 (struct e1000_adapter *)dev->data->dev_private; 5041 5042 ns = rte_timespec_to_ns(ts); 5043 5044 /* Set the timecounters to a new value. */ 5045 adapter->systime_tc.nsec = ns; 5046 adapter->rx_tstamp_tc.nsec = ns; 5047 adapter->tx_tstamp_tc.nsec = ns; 5048 5049 return 0; 5050 } 5051 5052 static int 5053 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 5054 { 5055 uint64_t ns, systime_cycles; 5056 struct e1000_adapter *adapter = 5057 (struct e1000_adapter *)dev->data->dev_private; 5058 5059 systime_cycles = igb_read_systime_cyclecounter(dev); 5060 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 5061 *ts = rte_ns_to_timespec(ns); 5062 5063 return 0; 5064 } 5065 5066 static int 5067 igb_timesync_enable(struct rte_eth_dev *dev) 5068 { 5069 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5070 uint32_t tsync_ctl; 5071 uint32_t tsauxc; 5072 5073 /* Stop the timesync system time. */ 5074 E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0); 5075 /* Reset the timesync system time value. */ 5076 switch (hw->mac.type) { 5077 case e1000_82580: 5078 case e1000_i350: 5079 case e1000_i354: 5080 case e1000_i210: 5081 case e1000_i211: 5082 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0); 5083 /* fall-through */ 5084 case e1000_82576: 5085 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0); 5086 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0); 5087 break; 5088 default: 5089 /* Not supported. */ 5090 return -ENOTSUP; 5091 } 5092 5093 /* Enable system time for it isn't on by default. */ 5094 tsauxc = E1000_READ_REG(hw, E1000_TSAUXC); 5095 tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME; 5096 E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc); 5097 5098 igb_start_timecounters(dev); 5099 5100 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5101 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 5102 (ETHER_TYPE_1588 | 5103 E1000_ETQF_FILTER_ENABLE | 5104 E1000_ETQF_1588)); 5105 5106 /* Enable timestamping of received PTP packets. */ 5107 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5108 tsync_ctl |= E1000_TSYNCRXCTL_ENABLED; 5109 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5110 5111 /* Enable Timestamping of transmitted PTP packets. */ 5112 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5113 tsync_ctl |= E1000_TSYNCTXCTL_ENABLED; 5114 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5115 5116 return 0; 5117 } 5118 5119 static int 5120 igb_timesync_disable(struct rte_eth_dev *dev) 5121 { 5122 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5123 uint32_t tsync_ctl; 5124 5125 /* Disable timestamping of transmitted PTP packets. */ 5126 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5127 tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED; 5128 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl); 5129 5130 /* Disable timestamping of received PTP packets. */ 5131 tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5132 tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED; 5133 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl); 5134 5135 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 5136 E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0); 5137 5138 /* Stop incrementating the System Time registers. */ 5139 E1000_WRITE_REG(hw, E1000_TIMINCA, 0); 5140 5141 return 0; 5142 } 5143 5144 static int 5145 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 5146 struct timespec *timestamp, 5147 uint32_t flags __rte_unused) 5148 { 5149 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5150 struct e1000_adapter *adapter = 5151 (struct e1000_adapter *)dev->data->dev_private; 5152 uint32_t tsync_rxctl; 5153 uint64_t rx_tstamp_cycles; 5154 uint64_t ns; 5155 5156 tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); 5157 if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0) 5158 return -EINVAL; 5159 5160 rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev); 5161 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 5162 *timestamp = rte_ns_to_timespec(ns); 5163 5164 return 0; 5165 } 5166 5167 static int 5168 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 5169 struct timespec *timestamp) 5170 { 5171 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5172 struct e1000_adapter *adapter = 5173 (struct e1000_adapter *)dev->data->dev_private; 5174 uint32_t tsync_txctl; 5175 uint64_t tx_tstamp_cycles; 5176 uint64_t ns; 5177 5178 tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); 5179 if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0) 5180 return -EINVAL; 5181 5182 tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev); 5183 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 5184 *timestamp = rte_ns_to_timespec(ns); 5185 5186 return 0; 5187 } 5188 5189 static int 5190 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5191 { 5192 int count = 0; 5193 int g_ind = 0; 5194 const struct reg_info *reg_group; 5195 5196 while ((reg_group = igb_regs[g_ind++])) 5197 count += igb_reg_group_count(reg_group); 5198 5199 return count; 5200 } 5201 5202 static int 5203 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 5204 { 5205 int count = 0; 5206 int g_ind = 0; 5207 const struct reg_info *reg_group; 5208 5209 while ((reg_group = igbvf_regs[g_ind++])) 5210 count += igb_reg_group_count(reg_group); 5211 5212 return count; 5213 } 5214 5215 static int 5216 eth_igb_get_regs(struct rte_eth_dev *dev, 5217 struct rte_dev_reg_info *regs) 5218 { 5219 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5220 uint32_t *data = regs->data; 5221 int g_ind = 0; 5222 int count = 0; 5223 const struct reg_info *reg_group; 5224 5225 if (data == NULL) { 5226 regs->length = eth_igb_get_reg_length(dev); 5227 regs->width = sizeof(uint32_t); 5228 return 0; 5229 } 5230 5231 /* Support only full register dump */ 5232 if ((regs->length == 0) || 5233 (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) { 5234 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5235 hw->device_id; 5236 while ((reg_group = igb_regs[g_ind++])) 5237 count += igb_read_regs_group(dev, &data[count], 5238 reg_group); 5239 return 0; 5240 } 5241 5242 return -ENOTSUP; 5243 } 5244 5245 static int 5246 igbvf_get_regs(struct rte_eth_dev *dev, 5247 struct rte_dev_reg_info *regs) 5248 { 5249 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5250 uint32_t *data = regs->data; 5251 int g_ind = 0; 5252 int count = 0; 5253 const struct reg_info *reg_group; 5254 5255 if (data == NULL) { 5256 regs->length = igbvf_get_reg_length(dev); 5257 regs->width = sizeof(uint32_t); 5258 return 0; 5259 } 5260 5261 /* Support only full register dump */ 5262 if ((regs->length == 0) || 5263 (regs->length == (uint32_t)igbvf_get_reg_length(dev))) { 5264 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 5265 hw->device_id; 5266 while ((reg_group = igbvf_regs[g_ind++])) 5267 count += igb_read_regs_group(dev, &data[count], 5268 reg_group); 5269 return 0; 5270 } 5271 5272 return -ENOTSUP; 5273 } 5274 5275 static int 5276 eth_igb_get_eeprom_length(struct rte_eth_dev *dev) 5277 { 5278 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5279 5280 /* Return unit is byte count */ 5281 return hw->nvm.word_size * 2; 5282 } 5283 5284 static int 5285 eth_igb_get_eeprom(struct rte_eth_dev *dev, 5286 struct rte_dev_eeprom_info *in_eeprom) 5287 { 5288 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5289 struct e1000_nvm_info *nvm = &hw->nvm; 5290 uint16_t *data = in_eeprom->data; 5291 int first, length; 5292 5293 first = in_eeprom->offset >> 1; 5294 length = in_eeprom->length >> 1; 5295 if ((first >= hw->nvm.word_size) || 5296 ((first + length) >= hw->nvm.word_size)) 5297 return -EINVAL; 5298 5299 in_eeprom->magic = hw->vendor_id | 5300 ((uint32_t)hw->device_id << 16); 5301 5302 if ((nvm->ops.read) == NULL) 5303 return -ENOTSUP; 5304 5305 return nvm->ops.read(hw, first, length, data); 5306 } 5307 5308 static int 5309 eth_igb_set_eeprom(struct rte_eth_dev *dev, 5310 struct rte_dev_eeprom_info *in_eeprom) 5311 { 5312 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5313 struct e1000_nvm_info *nvm = &hw->nvm; 5314 uint16_t *data = in_eeprom->data; 5315 int first, length; 5316 5317 first = in_eeprom->offset >> 1; 5318 length = in_eeprom->length >> 1; 5319 if ((first >= hw->nvm.word_size) || 5320 ((first + length) >= hw->nvm.word_size)) 5321 return -EINVAL; 5322 5323 in_eeprom->magic = (uint32_t)hw->vendor_id | 5324 ((uint32_t)hw->device_id << 16); 5325 5326 if ((nvm->ops.write) == NULL) 5327 return -ENOTSUP; 5328 return nvm->ops.write(hw, first, length, data); 5329 } 5330 5331 static int 5332 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5333 { 5334 struct e1000_hw *hw = 5335 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5336 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5337 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5338 uint32_t vec = E1000_MISC_VEC_ID; 5339 5340 if (rte_intr_allow_others(intr_handle)) 5341 vec = E1000_RX_VEC_START; 5342 5343 uint32_t mask = 1 << (queue_id + vec); 5344 5345 E1000_WRITE_REG(hw, E1000_EIMC, mask); 5346 E1000_WRITE_FLUSH(hw); 5347 5348 return 0; 5349 } 5350 5351 static int 5352 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5353 { 5354 struct e1000_hw *hw = 5355 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5356 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5357 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5358 uint32_t vec = E1000_MISC_VEC_ID; 5359 5360 if (rte_intr_allow_others(intr_handle)) 5361 vec = E1000_RX_VEC_START; 5362 5363 uint32_t mask = 1 << (queue_id + vec); 5364 uint32_t regval; 5365 5366 regval = E1000_READ_REG(hw, E1000_EIMS); 5367 E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); 5368 E1000_WRITE_FLUSH(hw); 5369 5370 rte_intr_enable(intr_handle); 5371 5372 return 0; 5373 } 5374 5375 static void 5376 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, 5377 uint8_t index, uint8_t offset) 5378 { 5379 uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 5380 5381 /* clear bits */ 5382 val &= ~((uint32_t)0xFF << offset); 5383 5384 /* write vector and valid bit */ 5385 val |= (msix_vector | E1000_IVAR_VALID) << offset; 5386 5387 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val); 5388 } 5389 5390 static void 5391 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, 5392 uint8_t queue, uint8_t msix_vector) 5393 { 5394 uint32_t tmp = 0; 5395 5396 if (hw->mac.type == e1000_82575) { 5397 if (direction == 0) 5398 tmp = E1000_EICR_RX_QUEUE0 << queue; 5399 else if (direction == 1) 5400 tmp = E1000_EICR_TX_QUEUE0 << queue; 5401 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp); 5402 } else if (hw->mac.type == e1000_82576) { 5403 if ((direction == 0) || (direction == 1)) 5404 eth_igb_write_ivar(hw, msix_vector, queue & 0x7, 5405 ((queue & 0x8) << 1) + 5406 8 * direction); 5407 } else if ((hw->mac.type == e1000_82580) || 5408 (hw->mac.type == e1000_i350) || 5409 (hw->mac.type == e1000_i354) || 5410 (hw->mac.type == e1000_i210) || 5411 (hw->mac.type == e1000_i211)) { 5412 if ((direction == 0) || (direction == 1)) 5413 eth_igb_write_ivar(hw, msix_vector, 5414 queue >> 1, 5415 ((queue & 0x1) << 4) + 5416 8 * direction); 5417 } 5418 } 5419 5420 /* Sets up the hardware to generate MSI-X interrupts properly 5421 * @hw 5422 * board private structure 5423 */ 5424 static void 5425 eth_igb_configure_msix_intr(struct rte_eth_dev *dev) 5426 { 5427 int queue_id; 5428 uint32_t tmpval, regval, intr_mask; 5429 struct e1000_hw *hw = 5430 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5431 uint32_t vec = E1000_MISC_VEC_ID; 5432 uint32_t base = E1000_MISC_VEC_ID; 5433 uint32_t misc_shift = 0; 5434 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5435 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5436 5437 /* won't configure msix register if no mapping is done 5438 * between intr vector and event fd 5439 */ 5440 if (!rte_intr_dp_is_en(intr_handle)) 5441 return; 5442 5443 if (rte_intr_allow_others(intr_handle)) { 5444 vec = base = E1000_RX_VEC_START; 5445 misc_shift = 1; 5446 } 5447 5448 /* set interrupt vector for other causes */ 5449 if (hw->mac.type == e1000_82575) { 5450 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT); 5451 /* enable MSI-X PBA support */ 5452 tmpval |= E1000_CTRL_EXT_PBA_CLR; 5453 5454 /* Auto-Mask interrupts upon ICR read */ 5455 tmpval |= E1000_CTRL_EXT_EIAME; 5456 tmpval |= E1000_CTRL_EXT_IRCA; 5457 5458 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval); 5459 5460 /* enable msix_other interrupt */ 5461 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER); 5462 regval = E1000_READ_REG(hw, E1000_EIAC); 5463 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER); 5464 regval = E1000_READ_REG(hw, E1000_EIAM); 5465 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER); 5466 } else if ((hw->mac.type == e1000_82576) || 5467 (hw->mac.type == e1000_82580) || 5468 (hw->mac.type == e1000_i350) || 5469 (hw->mac.type == e1000_i354) || 5470 (hw->mac.type == e1000_i210) || 5471 (hw->mac.type == e1000_i211)) { 5472 /* turn on MSI-X capability first */ 5473 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | 5474 E1000_GPIE_PBA | E1000_GPIE_EIAME | 5475 E1000_GPIE_NSICR); 5476 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5477 misc_shift; 5478 regval = E1000_READ_REG(hw, E1000_EIAC); 5479 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask); 5480 5481 /* enable msix_other interrupt */ 5482 regval = E1000_READ_REG(hw, E1000_EIMS); 5483 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask); 5484 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8; 5485 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval); 5486 } 5487 5488 /* use EIAM to auto-mask when MSI-X interrupt 5489 * is asserted, this saves a register write for every interrupt 5490 */ 5491 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << 5492 misc_shift; 5493 regval = E1000_READ_REG(hw, E1000_EIAM); 5494 E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask); 5495 5496 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) { 5497 eth_igb_assign_msix_vector(hw, 0, queue_id, vec); 5498 intr_handle->intr_vec[queue_id] = vec; 5499 if (vec < base + intr_handle->nb_efd - 1) 5500 vec++; 5501 } 5502 5503 E1000_WRITE_FLUSH(hw); 5504 } 5505 5506 /* restore n-tuple filter */ 5507 static inline void 5508 igb_ntuple_filter_restore(struct rte_eth_dev *dev) 5509 { 5510 struct e1000_filter_info *filter_info = 5511 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5512 struct e1000_5tuple_filter *p_5tuple; 5513 struct e1000_2tuple_filter *p_2tuple; 5514 5515 TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { 5516 igb_inject_5tuple_filter_82576(dev, p_5tuple); 5517 } 5518 5519 TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { 5520 igb_inject_2uple_filter(dev, p_2tuple); 5521 } 5522 } 5523 5524 /* restore SYN filter */ 5525 static inline void 5526 igb_syn_filter_restore(struct rte_eth_dev *dev) 5527 { 5528 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5529 struct e1000_filter_info *filter_info = 5530 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5531 uint32_t synqf; 5532 5533 synqf = filter_info->syn_info; 5534 5535 if (synqf & E1000_SYN_FILTER_ENABLE) { 5536 E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); 5537 E1000_WRITE_FLUSH(hw); 5538 } 5539 } 5540 5541 /* restore ethernet type filter */ 5542 static inline void 5543 igb_ethertype_filter_restore(struct rte_eth_dev *dev) 5544 { 5545 struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5546 struct e1000_filter_info *filter_info = 5547 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5548 int i; 5549 5550 for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { 5551 if (filter_info->ethertype_mask & (1 << i)) { 5552 E1000_WRITE_REG(hw, E1000_ETQF(i), 5553 filter_info->ethertype_filters[i].etqf); 5554 E1000_WRITE_FLUSH(hw); 5555 } 5556 } 5557 } 5558 5559 /* restore flex byte filter */ 5560 static inline void 5561 igb_flex_filter_restore(struct rte_eth_dev *dev) 5562 { 5563 struct e1000_filter_info *filter_info = 5564 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5565 struct e1000_flex_filter *flex_filter; 5566 5567 TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { 5568 igb_inject_flex_filter(dev, flex_filter); 5569 } 5570 } 5571 5572 /* restore rss filter */ 5573 static inline void 5574 igb_rss_filter_restore(struct rte_eth_dev *dev) 5575 { 5576 struct e1000_filter_info *filter_info = 5577 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5578 5579 if (filter_info->rss_info.num) 5580 igb_config_rss_filter(dev, &filter_info->rss_info, TRUE); 5581 } 5582 5583 /* restore all types filter */ 5584 static int 5585 igb_filter_restore(struct rte_eth_dev *dev) 5586 { 5587 igb_ntuple_filter_restore(dev); 5588 igb_ethertype_filter_restore(dev); 5589 igb_syn_filter_restore(dev); 5590 igb_flex_filter_restore(dev); 5591 igb_rss_filter_restore(dev); 5592 5593 return 0; 5594 } 5595 5596 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); 5597 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); 5598 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); 5599 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); 5600 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); 5601 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); 5602