1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <stdio.h> 36 #include <errno.h> 37 #include <stdint.h> 38 #include <string.h> 39 #include <unistd.h> 40 #include <stdarg.h> 41 #include <inttypes.h> 42 #include <netinet/in.h> 43 #include <rte_byteorder.h> 44 #include <rte_common.h> 45 #include <rte_cycles.h> 46 47 #include <rte_interrupts.h> 48 #include <rte_log.h> 49 #include <rte_debug.h> 50 #include <rte_pci.h> 51 #include <rte_bus_pci.h> 52 #include <rte_atomic.h> 53 #include <rte_branch_prediction.h> 54 #include <rte_memory.h> 55 #include <rte_eal.h> 56 #include <rte_alarm.h> 57 #include <rte_ether.h> 58 #include <rte_ethdev.h> 59 #include <rte_ethdev_pci.h> 60 #include <rte_malloc.h> 61 #include <rte_random.h> 62 #include <rte_dev.h> 63 #include <rte_hash_crc.h> 64 #ifdef RTE_LIBRTE_SECURITY 65 #include <rte_security_driver.h> 66 #endif 67 68 #include "ixgbe_logs.h" 69 #include "base/ixgbe_api.h" 70 #include "base/ixgbe_vf.h" 71 #include "base/ixgbe_common.h" 72 #include "ixgbe_ethdev.h" 73 #include "ixgbe_bypass.h" 74 #include "ixgbe_rxtx.h" 75 #include "base/ixgbe_type.h" 76 #include "base/ixgbe_phy.h" 77 #include "ixgbe_regs.h" 78 79 /* 80 * High threshold controlling when to start sending XOFF frames. Must be at 81 * least 8 bytes less than receive packet buffer size. This value is in units 82 * of 1024 bytes. 83 */ 84 #define IXGBE_FC_HI 0x80 85 86 /* 87 * Low threshold controlling when to start sending XON frames. This value is 88 * in units of 1024 bytes. 89 */ 90 #define IXGBE_FC_LO 0x40 91 92 /* Default minimum inter-interrupt interval for EITR configuration */ 93 #define IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT 0x79E 94 95 /* Timer value included in XOFF frames. */ 96 #define IXGBE_FC_PAUSE 0x680 97 98 /*Default value of Max Rx Queue*/ 99 #define IXGBE_MAX_RX_QUEUE_NUM 128 100 101 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */ 102 #define IXGBE_LINK_UP_CHECK_TIMEOUT 1000 /* ms */ 103 #define IXGBE_VMDQ_NUM_UC_MAC 4096 /* Maximum nb. of UC MAC addr. */ 104 105 #define IXGBE_MMW_SIZE_DEFAULT 0x4 106 #define IXGBE_MMW_SIZE_JUMBO_FRAME 0x14 107 #define IXGBE_MAX_RING_DESC 4096 /* replicate define from rxtx */ 108 109 /* 110 * Default values for RX/TX configuration 111 */ 112 #define IXGBE_DEFAULT_RX_FREE_THRESH 32 113 #define IXGBE_DEFAULT_RX_PTHRESH 8 114 #define IXGBE_DEFAULT_RX_HTHRESH 8 115 #define IXGBE_DEFAULT_RX_WTHRESH 0 116 117 #define IXGBE_DEFAULT_TX_FREE_THRESH 32 118 #define IXGBE_DEFAULT_TX_PTHRESH 32 119 #define IXGBE_DEFAULT_TX_HTHRESH 0 120 #define IXGBE_DEFAULT_TX_WTHRESH 0 121 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32 122 123 /* Bit shift and mask */ 124 #define IXGBE_4_BIT_WIDTH (CHAR_BIT / 2) 125 #define IXGBE_4_BIT_MASK RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t) 126 #define IXGBE_8_BIT_WIDTH CHAR_BIT 127 #define IXGBE_8_BIT_MASK UINT8_MAX 128 129 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */ 130 131 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0])) 132 133 #define IXGBE_HKEY_MAX_INDEX 10 134 135 /* Additional timesync values. */ 136 #define NSEC_PER_SEC 1000000000L 137 #define IXGBE_INCVAL_10GB 0x66666666 138 #define IXGBE_INCVAL_1GB 0x40000000 139 #define IXGBE_INCVAL_100 0x50000000 140 #define IXGBE_INCVAL_SHIFT_10GB 28 141 #define IXGBE_INCVAL_SHIFT_1GB 24 142 #define IXGBE_INCVAL_SHIFT_100 21 143 #define IXGBE_INCVAL_SHIFT_82599 7 144 #define IXGBE_INCPER_SHIFT_82599 24 145 146 #define IXGBE_CYCLECOUNTER_MASK 0xffffffffffffffffULL 147 148 #define IXGBE_VT_CTL_POOLING_MODE_MASK 0x00030000 149 #define IXGBE_VT_CTL_POOLING_MODE_ETAG 0x00010000 150 #define DEFAULT_ETAG_ETYPE 0x893f 151 #define IXGBE_ETAG_ETYPE 0x00005084 152 #define IXGBE_ETAG_ETYPE_MASK 0x0000ffff 153 #define IXGBE_ETAG_ETYPE_VALID 0x80000000 154 #define IXGBE_RAH_ADTYPE 0x40000000 155 #define IXGBE_RAL_ETAG_FILTER_MASK 0x00003fff 156 #define IXGBE_VMVIR_TAGA_MASK 0x18000000 157 #define IXGBE_VMVIR_TAGA_ETAG_INSERT 0x08000000 158 #define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */ 159 #define IXGBE_QDE_STRIP_TAG 0x00000004 160 #define IXGBE_VTEICR_MASK 0x07 161 162 #define IXGBE_EXVET_VET_EXT_SHIFT 16 163 #define IXGBE_DMATXCTL_VT_MASK 0xFFFF0000 164 165 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev); 166 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev); 167 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev); 168 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev); 169 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev); 170 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev); 171 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev); 172 static int ixgbe_dev_configure(struct rte_eth_dev *dev); 173 static int ixgbe_dev_start(struct rte_eth_dev *dev); 174 static void ixgbe_dev_stop(struct rte_eth_dev *dev); 175 static int ixgbe_dev_set_link_up(struct rte_eth_dev *dev); 176 static int ixgbe_dev_set_link_down(struct rte_eth_dev *dev); 177 static void ixgbe_dev_close(struct rte_eth_dev *dev); 178 static int ixgbe_dev_reset(struct rte_eth_dev *dev); 179 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev); 180 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev); 181 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev); 182 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev); 183 static int ixgbe_dev_link_update(struct rte_eth_dev *dev, 184 int wait_to_complete); 185 static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, 186 struct rte_eth_stats *stats); 187 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev, 188 struct rte_eth_xstat *xstats, unsigned n); 189 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, 190 struct rte_eth_xstat *xstats, unsigned n); 191 static int 192 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 193 uint64_t *values, unsigned int n); 194 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev); 195 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); 196 static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, 197 struct rte_eth_xstat_name *xstats_names, 198 unsigned int size); 199 static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, 200 struct rte_eth_xstat_name *xstats_names, unsigned limit); 201 static int ixgbe_dev_xstats_get_names_by_id( 202 struct rte_eth_dev *dev, 203 struct rte_eth_xstat_name *xstats_names, 204 const uint64_t *ids, 205 unsigned int limit); 206 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 207 uint16_t queue_id, 208 uint8_t stat_idx, 209 uint8_t is_rx); 210 static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 211 size_t fw_size); 212 static void ixgbe_dev_info_get(struct rte_eth_dev *dev, 213 struct rte_eth_dev_info *dev_info); 214 static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev); 215 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev, 216 struct rte_eth_dev_info *dev_info); 217 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 218 219 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev, 220 uint16_t vlan_id, int on); 221 static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 222 enum rte_vlan_type vlan_type, 223 uint16_t tpid_id); 224 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, 225 uint16_t queue, bool on); 226 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, 227 int on); 228 static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); 229 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue); 230 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue); 231 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev); 232 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev); 233 234 static int ixgbe_dev_led_on(struct rte_eth_dev *dev); 235 static int ixgbe_dev_led_off(struct rte_eth_dev *dev); 236 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, 237 struct rte_eth_fc_conf *fc_conf); 238 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, 239 struct rte_eth_fc_conf *fc_conf); 240 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, 241 struct rte_eth_pfc_conf *pfc_conf); 242 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 243 struct rte_eth_rss_reta_entry64 *reta_conf, 244 uint16_t reta_size); 245 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 246 struct rte_eth_rss_reta_entry64 *reta_conf, 247 uint16_t reta_size); 248 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); 249 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 250 static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); 251 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); 252 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); 253 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev, 254 struct rte_intr_handle *handle); 255 static void ixgbe_dev_interrupt_handler(void *param); 256 static void ixgbe_dev_interrupt_delayed_handler(void *param); 257 static int ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 258 uint32_t index, uint32_t pool); 259 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index); 260 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, 261 struct ether_addr *mac_addr); 262 static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config); 263 static bool is_device_supported(struct rte_eth_dev *dev, 264 struct rte_pci_driver *drv); 265 266 /* For Virtual Function support */ 267 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); 268 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); 269 static int ixgbevf_dev_configure(struct rte_eth_dev *dev); 270 static int ixgbevf_dev_start(struct rte_eth_dev *dev); 271 static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, 272 int wait_to_complete); 273 static void ixgbevf_dev_stop(struct rte_eth_dev *dev); 274 static void ixgbevf_dev_close(struct rte_eth_dev *dev); 275 static int ixgbevf_dev_reset(struct rte_eth_dev *dev); 276 static void ixgbevf_intr_disable(struct ixgbe_hw *hw); 277 static void ixgbevf_intr_enable(struct ixgbe_hw *hw); 278 static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, 279 struct rte_eth_stats *stats); 280 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev); 281 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, 282 uint16_t vlan_id, int on); 283 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, 284 uint16_t queue, int on); 285 static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); 286 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on); 287 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 288 uint16_t queue_id); 289 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 290 uint16_t queue_id); 291 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 292 uint8_t queue, uint8_t msix_vector); 293 static void ixgbevf_configure_msix(struct rte_eth_dev *dev); 294 static void ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev); 295 static void ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev); 296 297 /* For Eth VMDQ APIs support */ 298 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct 299 ether_addr * mac_addr, uint8_t on); 300 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on); 301 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 302 struct rte_eth_mirror_conf *mirror_conf, 303 uint8_t rule_id, uint8_t on); 304 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, 305 uint8_t rule_id); 306 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 307 uint16_t queue_id); 308 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 309 uint16_t queue_id); 310 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 311 uint8_t queue, uint8_t msix_vector); 312 static void ixgbe_configure_msix(struct rte_eth_dev *dev); 313 314 static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, 315 struct ether_addr *mac_addr, 316 uint32_t index, uint32_t pool); 317 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index); 318 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 319 struct ether_addr *mac_addr); 320 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev, 321 struct rte_eth_syn_filter *filter); 322 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev, 323 enum rte_filter_op filter_op, 324 void *arg); 325 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 326 struct ixgbe_5tuple_filter *filter); 327 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 328 struct ixgbe_5tuple_filter *filter); 329 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, 330 enum rte_filter_op filter_op, 331 void *arg); 332 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev, 333 struct rte_eth_ntuple_filter *filter); 334 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev, 335 enum rte_filter_op filter_op, 336 void *arg); 337 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev, 338 struct rte_eth_ethertype_filter *filter); 339 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, 340 enum rte_filter_type filter_type, 341 enum rte_filter_op filter_op, 342 void *arg); 343 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); 344 345 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 346 struct ether_addr *mc_addr_set, 347 uint32_t nb_mc_addr); 348 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 349 struct rte_eth_dcb_info *dcb_info); 350 351 static int ixgbe_get_reg_length(struct rte_eth_dev *dev); 352 static int ixgbe_get_regs(struct rte_eth_dev *dev, 353 struct rte_dev_reg_info *regs); 354 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev); 355 static int ixgbe_get_eeprom(struct rte_eth_dev *dev, 356 struct rte_dev_eeprom_info *eeprom); 357 static int ixgbe_set_eeprom(struct rte_eth_dev *dev, 358 struct rte_dev_eeprom_info *eeprom); 359 360 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev); 361 static int ixgbevf_get_regs(struct rte_eth_dev *dev, 362 struct rte_dev_reg_info *regs); 363 364 static int ixgbe_timesync_enable(struct rte_eth_dev *dev); 365 static int ixgbe_timesync_disable(struct rte_eth_dev *dev); 366 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 367 struct timespec *timestamp, 368 uint32_t flags); 369 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 370 struct timespec *timestamp); 371 static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 372 static int ixgbe_timesync_read_time(struct rte_eth_dev *dev, 373 struct timespec *timestamp); 374 static int ixgbe_timesync_write_time(struct rte_eth_dev *dev, 375 const struct timespec *timestamp); 376 static void ixgbevf_dev_interrupt_handler(void *param); 377 378 static int ixgbe_dev_l2_tunnel_eth_type_conf 379 (struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel); 380 static int ixgbe_dev_l2_tunnel_offload_set 381 (struct rte_eth_dev *dev, 382 struct rte_eth_l2_tunnel_conf *l2_tunnel, 383 uint32_t mask, 384 uint8_t en); 385 static int ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, 386 enum rte_filter_op filter_op, 387 void *arg); 388 389 static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 390 struct rte_eth_udp_tunnel *udp_tunnel); 391 static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 392 struct rte_eth_udp_tunnel *udp_tunnel); 393 static int ixgbe_filter_restore(struct rte_eth_dev *dev); 394 static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev); 395 396 /* 397 * Define VF Stats MACRO for Non "cleared on read" register 398 */ 399 #define UPDATE_VF_STAT(reg, last, cur) \ 400 { \ 401 uint32_t latest = IXGBE_READ_REG(hw, reg); \ 402 cur += (latest - last) & UINT_MAX; \ 403 last = latest; \ 404 } 405 406 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur) \ 407 { \ 408 u64 new_lsb = IXGBE_READ_REG(hw, lsb); \ 409 u64 new_msb = IXGBE_READ_REG(hw, msb); \ 410 u64 latest = ((new_msb << 32) | new_lsb); \ 411 cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \ 412 last = latest; \ 413 } 414 415 #define IXGBE_SET_HWSTRIP(h, q) do {\ 416 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 417 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 418 (h)->bitmap[idx] |= 1 << bit;\ 419 } while (0) 420 421 #define IXGBE_CLEAR_HWSTRIP(h, q) do {\ 422 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 423 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 424 (h)->bitmap[idx] &= ~(1 << bit);\ 425 } while (0) 426 427 #define IXGBE_GET_HWSTRIP(h, q, r) do {\ 428 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 429 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 430 (r) = (h)->bitmap[idx] >> bit & 1;\ 431 } while (0) 432 433 /* 434 * The set of PCI devices this driver supports 435 */ 436 static const struct rte_pci_id pci_id_ixgbe_map[] = { 437 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) }, 438 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) }, 439 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) }, 440 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) }, 441 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) }, 442 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) }, 443 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) }, 444 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) }, 445 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) }, 446 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) }, 447 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) }, 448 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) }, 449 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) }, 450 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, 451 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, 452 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, 453 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, 454 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, 455 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, 456 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, 457 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, 458 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) }, 459 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) }, 460 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) }, 461 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) }, 462 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) }, 463 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) }, 464 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_LS) }, 465 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) }, 466 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) }, 467 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) }, 468 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) }, 469 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) }, 470 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) }, 471 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) }, 472 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) }, 473 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) }, 474 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) }, 475 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) }, 476 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) }, 477 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) }, 478 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) }, 479 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) }, 480 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) }, 481 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) }, 482 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, 483 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, 484 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, 485 #ifdef RTE_LIBRTE_IXGBE_BYPASS 486 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, 487 #endif 488 { .vendor_id = 0, /* sentinel */ }, 489 }; 490 491 /* 492 * The set of PCI devices this driver supports (for 82599 VF) 493 */ 494 static const struct rte_pci_id pci_id_ixgbevf_map[] = { 495 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) }, 496 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) }, 497 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) }, 498 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) }, 499 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) }, 500 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) }, 501 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) }, 502 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) }, 503 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) }, 504 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) }, 505 { .vendor_id = 0, /* sentinel */ }, 506 }; 507 508 static const struct rte_eth_desc_lim rx_desc_lim = { 509 .nb_max = IXGBE_MAX_RING_DESC, 510 .nb_min = IXGBE_MIN_RING_DESC, 511 .nb_align = IXGBE_RXD_ALIGN, 512 }; 513 514 static const struct rte_eth_desc_lim tx_desc_lim = { 515 .nb_max = IXGBE_MAX_RING_DESC, 516 .nb_min = IXGBE_MIN_RING_DESC, 517 .nb_align = IXGBE_TXD_ALIGN, 518 .nb_seg_max = IXGBE_TX_MAX_SEG, 519 .nb_mtu_seg_max = IXGBE_TX_MAX_SEG, 520 }; 521 522 static const struct eth_dev_ops ixgbe_eth_dev_ops = { 523 .dev_configure = ixgbe_dev_configure, 524 .dev_start = ixgbe_dev_start, 525 .dev_stop = ixgbe_dev_stop, 526 .dev_set_link_up = ixgbe_dev_set_link_up, 527 .dev_set_link_down = ixgbe_dev_set_link_down, 528 .dev_close = ixgbe_dev_close, 529 .dev_reset = ixgbe_dev_reset, 530 .promiscuous_enable = ixgbe_dev_promiscuous_enable, 531 .promiscuous_disable = ixgbe_dev_promiscuous_disable, 532 .allmulticast_enable = ixgbe_dev_allmulticast_enable, 533 .allmulticast_disable = ixgbe_dev_allmulticast_disable, 534 .link_update = ixgbe_dev_link_update, 535 .stats_get = ixgbe_dev_stats_get, 536 .xstats_get = ixgbe_dev_xstats_get, 537 .xstats_get_by_id = ixgbe_dev_xstats_get_by_id, 538 .stats_reset = ixgbe_dev_stats_reset, 539 .xstats_reset = ixgbe_dev_xstats_reset, 540 .xstats_get_names = ixgbe_dev_xstats_get_names, 541 .xstats_get_names_by_id = ixgbe_dev_xstats_get_names_by_id, 542 .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set, 543 .fw_version_get = ixgbe_fw_version_get, 544 .dev_infos_get = ixgbe_dev_info_get, 545 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 546 .mtu_set = ixgbe_dev_mtu_set, 547 .vlan_filter_set = ixgbe_vlan_filter_set, 548 .vlan_tpid_set = ixgbe_vlan_tpid_set, 549 .vlan_offload_set = ixgbe_vlan_offload_set, 550 .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set, 551 .rx_queue_start = ixgbe_dev_rx_queue_start, 552 .rx_queue_stop = ixgbe_dev_rx_queue_stop, 553 .tx_queue_start = ixgbe_dev_tx_queue_start, 554 .tx_queue_stop = ixgbe_dev_tx_queue_stop, 555 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 556 .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable, 557 .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable, 558 .rx_queue_release = ixgbe_dev_rx_queue_release, 559 .rx_queue_count = ixgbe_dev_rx_queue_count, 560 .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, 561 .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, 562 .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, 563 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 564 .tx_queue_release = ixgbe_dev_tx_queue_release, 565 .dev_led_on = ixgbe_dev_led_on, 566 .dev_led_off = ixgbe_dev_led_off, 567 .flow_ctrl_get = ixgbe_flow_ctrl_get, 568 .flow_ctrl_set = ixgbe_flow_ctrl_set, 569 .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set, 570 .mac_addr_add = ixgbe_add_rar, 571 .mac_addr_remove = ixgbe_remove_rar, 572 .mac_addr_set = ixgbe_set_default_mac_addr, 573 .uc_hash_table_set = ixgbe_uc_hash_table_set, 574 .uc_all_hash_table_set = ixgbe_uc_all_hash_table_set, 575 .mirror_rule_set = ixgbe_mirror_rule_set, 576 .mirror_rule_reset = ixgbe_mirror_rule_reset, 577 .set_queue_rate_limit = ixgbe_set_queue_rate_limit, 578 .reta_update = ixgbe_dev_rss_reta_update, 579 .reta_query = ixgbe_dev_rss_reta_query, 580 .rss_hash_update = ixgbe_dev_rss_hash_update, 581 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 582 .filter_ctrl = ixgbe_dev_filter_ctrl, 583 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 584 .rxq_info_get = ixgbe_rxq_info_get, 585 .txq_info_get = ixgbe_txq_info_get, 586 .timesync_enable = ixgbe_timesync_enable, 587 .timesync_disable = ixgbe_timesync_disable, 588 .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, 589 .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp, 590 .get_reg = ixgbe_get_regs, 591 .get_eeprom_length = ixgbe_get_eeprom_length, 592 .get_eeprom = ixgbe_get_eeprom, 593 .set_eeprom = ixgbe_set_eeprom, 594 .get_dcb_info = ixgbe_dev_get_dcb_info, 595 .timesync_adjust_time = ixgbe_timesync_adjust_time, 596 .timesync_read_time = ixgbe_timesync_read_time, 597 .timesync_write_time = ixgbe_timesync_write_time, 598 .l2_tunnel_eth_type_conf = ixgbe_dev_l2_tunnel_eth_type_conf, 599 .l2_tunnel_offload_set = ixgbe_dev_l2_tunnel_offload_set, 600 .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, 601 .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, 602 .tm_ops_get = ixgbe_tm_ops_get, 603 }; 604 605 /* 606 * dev_ops for virtual function, bare necessities for basic vf 607 * operation have been implemented 608 */ 609 static const struct eth_dev_ops ixgbevf_eth_dev_ops = { 610 .dev_configure = ixgbevf_dev_configure, 611 .dev_start = ixgbevf_dev_start, 612 .dev_stop = ixgbevf_dev_stop, 613 .link_update = ixgbevf_dev_link_update, 614 .stats_get = ixgbevf_dev_stats_get, 615 .xstats_get = ixgbevf_dev_xstats_get, 616 .stats_reset = ixgbevf_dev_stats_reset, 617 .xstats_reset = ixgbevf_dev_stats_reset, 618 .xstats_get_names = ixgbevf_dev_xstats_get_names, 619 .dev_close = ixgbevf_dev_close, 620 .dev_reset = ixgbevf_dev_reset, 621 .allmulticast_enable = ixgbevf_dev_allmulticast_enable, 622 .allmulticast_disable = ixgbevf_dev_allmulticast_disable, 623 .dev_infos_get = ixgbevf_dev_info_get, 624 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 625 .mtu_set = ixgbevf_dev_set_mtu, 626 .vlan_filter_set = ixgbevf_vlan_filter_set, 627 .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set, 628 .vlan_offload_set = ixgbevf_vlan_offload_set, 629 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 630 .rx_queue_release = ixgbe_dev_rx_queue_release, 631 .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, 632 .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, 633 .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, 634 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 635 .tx_queue_release = ixgbe_dev_tx_queue_release, 636 .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable, 637 .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable, 638 .mac_addr_add = ixgbevf_add_mac_addr, 639 .mac_addr_remove = ixgbevf_remove_mac_addr, 640 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 641 .rxq_info_get = ixgbe_rxq_info_get, 642 .txq_info_get = ixgbe_txq_info_get, 643 .mac_addr_set = ixgbevf_set_default_mac_addr, 644 .get_reg = ixgbevf_get_regs, 645 .reta_update = ixgbe_dev_rss_reta_update, 646 .reta_query = ixgbe_dev_rss_reta_query, 647 .rss_hash_update = ixgbe_dev_rss_hash_update, 648 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 649 }; 650 651 /* store statistics names and its offset in stats structure */ 652 struct rte_ixgbe_xstats_name_off { 653 char name[RTE_ETH_XSTATS_NAME_SIZE]; 654 unsigned offset; 655 }; 656 657 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = { 658 {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)}, 659 {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)}, 660 {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)}, 661 {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)}, 662 {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)}, 663 {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)}, 664 {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)}, 665 {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)}, 666 {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)}, 667 {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)}, 668 {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)}, 669 {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)}, 670 {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)}, 671 {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)}, 672 {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 673 prc1023)}, 674 {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 675 prc1522)}, 676 {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)}, 677 {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)}, 678 {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)}, 679 {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)}, 680 {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)}, 681 {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)}, 682 {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)}, 683 {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)}, 684 {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)}, 685 {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)}, 686 {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)}, 687 {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)}, 688 {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)}, 689 {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)}, 690 {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)}, 691 {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)}, 692 {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 693 ptc1023)}, 694 {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 695 ptc1522)}, 696 {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)}, 697 {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)}, 698 {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)}, 699 {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)}, 700 701 {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats, 702 fdirustat_add)}, 703 {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats, 704 fdirustat_remove)}, 705 {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats, 706 fdirfstat_fadd)}, 707 {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats, 708 fdirfstat_fremove)}, 709 {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats, 710 fdirmatch)}, 711 {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats, 712 fdirmiss)}, 713 714 {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)}, 715 {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)}, 716 {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, 717 fclast)}, 718 {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)}, 719 {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)}, 720 {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)}, 721 {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)}, 722 {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats, 723 fcoe_noddp)}, 724 {"rx_fcoe_no_direct_data_placement_ext_buff", 725 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)}, 726 727 {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 728 lxontxc)}, 729 {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 730 lxonrxc)}, 731 {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 732 lxofftxc)}, 733 {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 734 lxoffrxc)}, 735 {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)}, 736 }; 737 738 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \ 739 sizeof(rte_ixgbe_stats_strings[0])) 740 741 /* MACsec statistics */ 742 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = { 743 {"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 744 out_pkts_untagged)}, 745 {"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats, 746 out_pkts_encrypted)}, 747 {"out_pkts_protected", offsetof(struct ixgbe_macsec_stats, 748 out_pkts_protected)}, 749 {"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats, 750 out_octets_encrypted)}, 751 {"out_octets_protected", offsetof(struct ixgbe_macsec_stats, 752 out_octets_protected)}, 753 {"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 754 in_pkts_untagged)}, 755 {"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats, 756 in_pkts_badtag)}, 757 {"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats, 758 in_pkts_nosci)}, 759 {"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats, 760 in_pkts_unknownsci)}, 761 {"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats, 762 in_octets_decrypted)}, 763 {"in_octets_validated", offsetof(struct ixgbe_macsec_stats, 764 in_octets_validated)}, 765 {"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats, 766 in_pkts_unchecked)}, 767 {"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats, 768 in_pkts_delayed)}, 769 {"in_pkts_late", offsetof(struct ixgbe_macsec_stats, 770 in_pkts_late)}, 771 {"in_pkts_ok", offsetof(struct ixgbe_macsec_stats, 772 in_pkts_ok)}, 773 {"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats, 774 in_pkts_invalid)}, 775 {"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats, 776 in_pkts_notvalid)}, 777 {"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats, 778 in_pkts_unusedsa)}, 779 {"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats, 780 in_pkts_notusingsa)}, 781 }; 782 783 #define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \ 784 sizeof(rte_ixgbe_macsec_strings[0])) 785 786 /* Per-queue statistics */ 787 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = { 788 {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)}, 789 {"dropped", offsetof(struct ixgbe_hw_stats, mpc)}, 790 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)}, 791 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)}, 792 }; 793 794 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \ 795 sizeof(rte_ixgbe_rxq_strings[0])) 796 #define IXGBE_NB_RXQ_PRIO_VALUES 8 797 798 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = { 799 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)}, 800 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)}, 801 {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats, 802 pxon2offc)}, 803 }; 804 805 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \ 806 sizeof(rte_ixgbe_txq_strings[0])) 807 #define IXGBE_NB_TXQ_PRIO_VALUES 8 808 809 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = { 810 {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)}, 811 }; 812 813 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) / \ 814 sizeof(rte_ixgbevf_stats_strings[0])) 815 816 /** 817 * Atomically reads the link status information from global 818 * structure rte_eth_dev. 819 * 820 * @param dev 821 * - Pointer to the structure rte_eth_dev to read from. 822 * - Pointer to the buffer to be saved with the link status. 823 * 824 * @return 825 * - On success, zero. 826 * - On failure, negative value. 827 */ 828 static inline int 829 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev, 830 struct rte_eth_link *link) 831 { 832 struct rte_eth_link *dst = link; 833 struct rte_eth_link *src = &(dev->data->dev_link); 834 835 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 836 *(uint64_t *)src) == 0) 837 return -1; 838 839 return 0; 840 } 841 842 /** 843 * Atomically writes the link status information into global 844 * structure rte_eth_dev. 845 * 846 * @param dev 847 * - Pointer to the structure rte_eth_dev to read from. 848 * - Pointer to the buffer to be saved with the link status. 849 * 850 * @return 851 * - On success, zero. 852 * - On failure, negative value. 853 */ 854 static inline int 855 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev, 856 struct rte_eth_link *link) 857 { 858 struct rte_eth_link *dst = &(dev->data->dev_link); 859 struct rte_eth_link *src = link; 860 861 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 862 *(uint64_t *)src) == 0) 863 return -1; 864 865 return 0; 866 } 867 868 /* 869 * This function is the same as ixgbe_is_sfp() in base/ixgbe.h. 870 */ 871 static inline int 872 ixgbe_is_sfp(struct ixgbe_hw *hw) 873 { 874 switch (hw->phy.type) { 875 case ixgbe_phy_sfp_avago: 876 case ixgbe_phy_sfp_ftl: 877 case ixgbe_phy_sfp_intel: 878 case ixgbe_phy_sfp_unknown: 879 case ixgbe_phy_sfp_passive_tyco: 880 case ixgbe_phy_sfp_passive_unknown: 881 return 1; 882 default: 883 return 0; 884 } 885 } 886 887 static inline int32_t 888 ixgbe_pf_reset_hw(struct ixgbe_hw *hw) 889 { 890 uint32_t ctrl_ext; 891 int32_t status; 892 893 status = ixgbe_reset_hw(hw); 894 895 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 896 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 897 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 898 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 899 IXGBE_WRITE_FLUSH(hw); 900 901 if (status == IXGBE_ERR_SFP_NOT_PRESENT) 902 status = IXGBE_SUCCESS; 903 return status; 904 } 905 906 static inline void 907 ixgbe_enable_intr(struct rte_eth_dev *dev) 908 { 909 struct ixgbe_interrupt *intr = 910 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 911 struct ixgbe_hw *hw = 912 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 913 914 IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask); 915 IXGBE_WRITE_FLUSH(hw); 916 } 917 918 /* 919 * This function is based on ixgbe_disable_intr() in base/ixgbe.h. 920 */ 921 static void 922 ixgbe_disable_intr(struct ixgbe_hw *hw) 923 { 924 PMD_INIT_FUNC_TRACE(); 925 926 if (hw->mac.type == ixgbe_mac_82598EB) { 927 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0); 928 } else { 929 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000); 930 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0); 931 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0); 932 } 933 IXGBE_WRITE_FLUSH(hw); 934 } 935 936 /* 937 * This function resets queue statistics mapping registers. 938 * From Niantic datasheet, Initialization of Statistics section: 939 * "...if software requires the queue counters, the RQSMR and TQSM registers 940 * must be re-programmed following a device reset. 941 */ 942 static void 943 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw) 944 { 945 uint32_t i; 946 947 for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) { 948 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0); 949 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0); 950 } 951 } 952 953 954 static int 955 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 956 uint16_t queue_id, 957 uint8_t stat_idx, 958 uint8_t is_rx) 959 { 960 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8 961 #define NB_QMAP_FIELDS_PER_QSM_REG 4 962 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f 963 964 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 965 struct ixgbe_stat_mapping_registers *stat_mappings = 966 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private); 967 uint32_t qsmr_mask = 0; 968 uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK; 969 uint32_t q_map; 970 uint8_t n, offset; 971 972 if ((hw->mac.type != ixgbe_mac_82599EB) && 973 (hw->mac.type != ixgbe_mac_X540) && 974 (hw->mac.type != ixgbe_mac_X550) && 975 (hw->mac.type != ixgbe_mac_X550EM_x) && 976 (hw->mac.type != ixgbe_mac_X550EM_a)) 977 return -ENOSYS; 978 979 PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d", 980 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 981 queue_id, stat_idx); 982 983 n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG); 984 if (n >= IXGBE_NB_STAT_MAPPING_REGS) { 985 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded"); 986 return -EIO; 987 } 988 offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG); 989 990 /* Now clear any previous stat_idx set */ 991 clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 992 if (!is_rx) 993 stat_mappings->tqsm[n] &= ~clearing_mask; 994 else 995 stat_mappings->rqsmr[n] &= ~clearing_mask; 996 997 q_map = (uint32_t)stat_idx; 998 q_map &= QMAP_FIELD_RESERVED_BITS_MASK; 999 qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 1000 if (!is_rx) 1001 stat_mappings->tqsm[n] |= qsmr_mask; 1002 else 1003 stat_mappings->rqsmr[n] |= qsmr_mask; 1004 1005 PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d", 1006 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 1007 queue_id, stat_idx); 1008 PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n, 1009 is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]); 1010 1011 /* Now write the mapping in the appropriate register */ 1012 if (is_rx) { 1013 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d", 1014 stat_mappings->rqsmr[n], n); 1015 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]); 1016 } else { 1017 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d", 1018 stat_mappings->tqsm[n], n); 1019 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]); 1020 } 1021 return 0; 1022 } 1023 1024 static void 1025 ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev) 1026 { 1027 struct ixgbe_stat_mapping_registers *stat_mappings = 1028 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private); 1029 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1030 int i; 1031 1032 /* write whatever was in stat mapping table to the NIC */ 1033 for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) { 1034 /* rx */ 1035 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]); 1036 1037 /* tx */ 1038 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]); 1039 } 1040 } 1041 1042 static void 1043 ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config) 1044 { 1045 uint8_t i; 1046 struct ixgbe_dcb_tc_config *tc; 1047 uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS; 1048 1049 dcb_config->num_tcs.pg_tcs = dcb_max_tc; 1050 dcb_config->num_tcs.pfc_tcs = dcb_max_tc; 1051 for (i = 0; i < dcb_max_tc; i++) { 1052 tc = &dcb_config->tc_config[i]; 1053 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i; 1054 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 1055 (uint8_t)(100/dcb_max_tc + (i & 1)); 1056 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i; 1057 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 1058 (uint8_t)(100/dcb_max_tc + (i & 1)); 1059 tc->pfc = ixgbe_dcb_pfc_disabled; 1060 } 1061 1062 /* Initialize default user to priority mapping, UPx->TC0 */ 1063 tc = &dcb_config->tc_config[0]; 1064 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 1065 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 1066 for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) { 1067 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100; 1068 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100; 1069 } 1070 dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal; 1071 dcb_config->pfc_mode_enable = false; 1072 dcb_config->vt_mode = true; 1073 dcb_config->round_robin_enable = false; 1074 /* support all DCB capabilities in 82599 */ 1075 dcb_config->support.capabilities = 0xFF; 1076 1077 /*we only support 4 Tcs for X540, X550 */ 1078 if (hw->mac.type == ixgbe_mac_X540 || 1079 hw->mac.type == ixgbe_mac_X550 || 1080 hw->mac.type == ixgbe_mac_X550EM_x || 1081 hw->mac.type == ixgbe_mac_X550EM_a) { 1082 dcb_config->num_tcs.pg_tcs = 4; 1083 dcb_config->num_tcs.pfc_tcs = 4; 1084 } 1085 } 1086 1087 /* 1088 * Ensure that all locks are released before first NVM or PHY access 1089 */ 1090 static void 1091 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) 1092 { 1093 uint16_t mask; 1094 1095 /* 1096 * Phy lock should not fail in this early stage. If this is the case, 1097 * it is due to an improper exit of the application. 1098 * So force the release of the faulty lock. Release of common lock 1099 * is done automatically by swfw_sync function. 1100 */ 1101 mask = IXGBE_GSSR_PHY0_SM << hw->bus.func; 1102 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1103 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func); 1104 } 1105 ixgbe_release_swfw_semaphore(hw, mask); 1106 1107 /* 1108 * These ones are more tricky since they are common to all ports; but 1109 * swfw_sync retries last long enough (1s) to be almost sure that if 1110 * lock can not be taken it is due to an improper lock of the 1111 * semaphore. 1112 */ 1113 mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM; 1114 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1115 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 1116 } 1117 ixgbe_release_swfw_semaphore(hw, mask); 1118 } 1119 1120 /* 1121 * This function is based on code in ixgbe_attach() in base/ixgbe.c. 1122 * It returns 0 on success. 1123 */ 1124 static int 1125 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) 1126 { 1127 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1128 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1129 struct ixgbe_hw *hw = 1130 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1131 struct ixgbe_vfta *shadow_vfta = 1132 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1133 struct ixgbe_hwstrip *hwstrip = 1134 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1135 struct ixgbe_dcb_config *dcb_config = 1136 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private); 1137 struct ixgbe_filter_info *filter_info = 1138 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1139 struct ixgbe_bw_conf *bw_conf = 1140 IXGBE_DEV_PRIVATE_TO_BW_CONF(eth_dev->data->dev_private); 1141 uint32_t ctrl_ext; 1142 uint16_t csum; 1143 int diag, i; 1144 1145 PMD_INIT_FUNC_TRACE(); 1146 1147 eth_dev->dev_ops = &ixgbe_eth_dev_ops; 1148 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1149 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1150 eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts; 1151 1152 /* 1153 * For secondary processes, we don't initialise any further as primary 1154 * has already done this work. Only check we don't need a different 1155 * RX and TX function. 1156 */ 1157 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1158 struct ixgbe_tx_queue *txq; 1159 /* TX queue function in primary, set by last queue initialized 1160 * Tx queue may not initialized by primary process 1161 */ 1162 if (eth_dev->data->tx_queues) { 1163 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1]; 1164 ixgbe_set_tx_function(eth_dev, txq); 1165 } else { 1166 /* Use default TX function if we get here */ 1167 PMD_INIT_LOG(NOTICE, "No TX queues configured yet. " 1168 "Using default TX function."); 1169 } 1170 1171 ixgbe_set_rx_function(eth_dev); 1172 1173 return 0; 1174 } 1175 1176 #ifdef RTE_LIBRTE_SECURITY 1177 /* Initialize security_ctx only for primary process*/ 1178 eth_dev->security_ctx = ixgbe_ipsec_ctx_create(eth_dev); 1179 if (eth_dev->security_ctx == NULL) 1180 return -ENOMEM; 1181 #endif 1182 1183 rte_eth_copy_pci_info(eth_dev, pci_dev); 1184 1185 /* Vendor and Device ID need to be set before init of shared code */ 1186 hw->device_id = pci_dev->id.device_id; 1187 hw->vendor_id = pci_dev->id.vendor_id; 1188 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1189 hw->allow_unsupported_sfp = 1; 1190 1191 /* Initialize the shared code (base driver) */ 1192 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1193 diag = ixgbe_bypass_init_shared_code(hw); 1194 #else 1195 diag = ixgbe_init_shared_code(hw); 1196 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1197 1198 if (diag != IXGBE_SUCCESS) { 1199 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); 1200 return -EIO; 1201 } 1202 1203 /* pick up the PCI bus settings for reporting later */ 1204 ixgbe_get_bus_info(hw); 1205 1206 /* Unlock any pending hardware semaphore */ 1207 ixgbe_swfw_lock_reset(hw); 1208 1209 /* Initialize DCB configuration*/ 1210 memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config)); 1211 ixgbe_dcb_init(hw, dcb_config); 1212 /* Get Hardware Flow Control setting */ 1213 hw->fc.requested_mode = ixgbe_fc_full; 1214 hw->fc.current_mode = ixgbe_fc_full; 1215 hw->fc.pause_time = IXGBE_FC_PAUSE; 1216 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 1217 hw->fc.low_water[i] = IXGBE_FC_LO; 1218 hw->fc.high_water[i] = IXGBE_FC_HI; 1219 } 1220 hw->fc.send_xon = 1; 1221 1222 /* Make sure we have a good EEPROM before we read from it */ 1223 diag = ixgbe_validate_eeprom_checksum(hw, &csum); 1224 if (diag != IXGBE_SUCCESS) { 1225 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag); 1226 return -EIO; 1227 } 1228 1229 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1230 diag = ixgbe_bypass_init_hw(hw); 1231 #else 1232 diag = ixgbe_init_hw(hw); 1233 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1234 1235 /* 1236 * Devices with copper phys will fail to initialise if ixgbe_init_hw() 1237 * is called too soon after the kernel driver unbinding/binding occurs. 1238 * The failure occurs in ixgbe_identify_phy_generic() for all devices, 1239 * but for non-copper devies, ixgbe_identify_sfp_module_generic() is 1240 * also called. See ixgbe_identify_phy_82599(). The reason for the 1241 * failure is not known, and only occuts when virtualisation features 1242 * are disabled in the bios. A delay of 100ms was found to be enough by 1243 * trial-and-error, and is doubled to be safe. 1244 */ 1245 if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) { 1246 rte_delay_ms(200); 1247 diag = ixgbe_init_hw(hw); 1248 } 1249 1250 if (diag == IXGBE_ERR_SFP_NOT_PRESENT) 1251 diag = IXGBE_SUCCESS; 1252 1253 if (diag == IXGBE_ERR_EEPROM_VERSION) { 1254 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/" 1255 "LOM. Please be aware there may be issues associated " 1256 "with your hardware."); 1257 PMD_INIT_LOG(ERR, "If you are experiencing problems " 1258 "please contact your Intel or hardware representative " 1259 "who provided you with this hardware."); 1260 } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED) 1261 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module"); 1262 if (diag) { 1263 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag); 1264 return -EIO; 1265 } 1266 1267 /* Reset the hw statistics */ 1268 ixgbe_dev_stats_reset(eth_dev); 1269 1270 /* disable interrupt */ 1271 ixgbe_disable_intr(hw); 1272 1273 /* reset mappings for queue statistics hw counters*/ 1274 ixgbe_reset_qstat_mappings(hw); 1275 1276 /* Allocate memory for storing MAC addresses */ 1277 eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN * 1278 hw->mac.num_rar_entries, 0); 1279 if (eth_dev->data->mac_addrs == NULL) { 1280 PMD_INIT_LOG(ERR, 1281 "Failed to allocate %u bytes needed to store " 1282 "MAC addresses", 1283 ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1284 return -ENOMEM; 1285 } 1286 /* Copy the permanent MAC address */ 1287 ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, 1288 ð_dev->data->mac_addrs[0]); 1289 1290 /* Allocate memory for storing hash filter MAC addresses */ 1291 eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN * 1292 IXGBE_VMDQ_NUM_UC_MAC, 0); 1293 if (eth_dev->data->hash_mac_addrs == NULL) { 1294 PMD_INIT_LOG(ERR, 1295 "Failed to allocate %d bytes needed to store MAC addresses", 1296 ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC); 1297 return -ENOMEM; 1298 } 1299 1300 /* initialize the vfta */ 1301 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1302 1303 /* initialize the hw strip bitmap*/ 1304 memset(hwstrip, 0, sizeof(*hwstrip)); 1305 1306 /* initialize PF if max_vfs not zero */ 1307 ixgbe_pf_host_init(eth_dev); 1308 1309 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 1310 /* let hardware know driver is loaded */ 1311 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 1312 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1313 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 1314 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 1315 IXGBE_WRITE_FLUSH(hw); 1316 1317 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 1318 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d", 1319 (int) hw->mac.type, (int) hw->phy.type, 1320 (int) hw->phy.sfp_type); 1321 else 1322 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d", 1323 (int) hw->mac.type, (int) hw->phy.type); 1324 1325 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1326 eth_dev->data->port_id, pci_dev->id.vendor_id, 1327 pci_dev->id.device_id); 1328 1329 rte_intr_callback_register(intr_handle, 1330 ixgbe_dev_interrupt_handler, eth_dev); 1331 1332 /* enable uio/vfio intr/eventfd mapping */ 1333 rte_intr_enable(intr_handle); 1334 1335 /* enable support intr */ 1336 ixgbe_enable_intr(eth_dev); 1337 1338 /* initialize filter info */ 1339 memset(filter_info, 0, 1340 sizeof(struct ixgbe_filter_info)); 1341 1342 /* initialize 5tuple filter list */ 1343 TAILQ_INIT(&filter_info->fivetuple_list); 1344 1345 /* initialize flow director filter list & hash */ 1346 ixgbe_fdir_filter_init(eth_dev); 1347 1348 /* initialize l2 tunnel filter list & hash */ 1349 ixgbe_l2_tn_filter_init(eth_dev); 1350 1351 /* initialize flow filter lists */ 1352 ixgbe_filterlist_init(); 1353 1354 /* initialize bandwidth configuration info */ 1355 memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); 1356 1357 /* initialize Traffic Manager configuration */ 1358 ixgbe_tm_conf_init(eth_dev); 1359 1360 return 0; 1361 } 1362 1363 static int 1364 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) 1365 { 1366 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1367 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1368 struct ixgbe_hw *hw; 1369 int retries = 0; 1370 int ret; 1371 1372 PMD_INIT_FUNC_TRACE(); 1373 1374 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1375 return -EPERM; 1376 1377 hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1378 1379 if (hw->adapter_stopped == 0) 1380 ixgbe_dev_close(eth_dev); 1381 1382 eth_dev->dev_ops = NULL; 1383 eth_dev->rx_pkt_burst = NULL; 1384 eth_dev->tx_pkt_burst = NULL; 1385 1386 /* Unlock any pending hardware semaphore */ 1387 ixgbe_swfw_lock_reset(hw); 1388 1389 /* disable uio intr before callback unregister */ 1390 rte_intr_disable(intr_handle); 1391 1392 do { 1393 ret = rte_intr_callback_unregister(intr_handle, 1394 ixgbe_dev_interrupt_handler, eth_dev); 1395 if (ret >= 0) { 1396 break; 1397 } else if (ret != -EAGAIN) { 1398 PMD_INIT_LOG(ERR, 1399 "intr callback unregister failed: %d", 1400 ret); 1401 return ret; 1402 } 1403 rte_delay_ms(100); 1404 } while (retries++ < (10 + IXGBE_LINK_UP_TIME)); 1405 1406 /* uninitialize PF if max_vfs not zero */ 1407 ixgbe_pf_host_uninit(eth_dev); 1408 1409 rte_free(eth_dev->data->mac_addrs); 1410 eth_dev->data->mac_addrs = NULL; 1411 1412 rte_free(eth_dev->data->hash_mac_addrs); 1413 eth_dev->data->hash_mac_addrs = NULL; 1414 1415 /* remove all the fdir filters & hash */ 1416 ixgbe_fdir_filter_uninit(eth_dev); 1417 1418 /* remove all the L2 tunnel filters & hash */ 1419 ixgbe_l2_tn_filter_uninit(eth_dev); 1420 1421 /* Remove all ntuple filters of the device */ 1422 ixgbe_ntuple_filter_uninit(eth_dev); 1423 1424 /* clear all the filters list */ 1425 ixgbe_filterlist_flush(); 1426 1427 /* Remove all Traffic Manager configuration */ 1428 ixgbe_tm_conf_uninit(eth_dev); 1429 1430 #ifdef RTE_LIBRTE_SECURITY 1431 rte_free(eth_dev->security_ctx); 1432 #endif 1433 1434 return 0; 1435 } 1436 1437 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 1438 { 1439 struct ixgbe_filter_info *filter_info = 1440 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1441 struct ixgbe_5tuple_filter *p_5tuple; 1442 1443 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 1444 TAILQ_REMOVE(&filter_info->fivetuple_list, 1445 p_5tuple, 1446 entries); 1447 rte_free(p_5tuple); 1448 } 1449 memset(filter_info->fivetuple_mask, 0, 1450 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); 1451 1452 return 0; 1453 } 1454 1455 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev) 1456 { 1457 struct ixgbe_hw_fdir_info *fdir_info = 1458 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1459 struct ixgbe_fdir_filter *fdir_filter; 1460 1461 if (fdir_info->hash_map) 1462 rte_free(fdir_info->hash_map); 1463 if (fdir_info->hash_handle) 1464 rte_hash_free(fdir_info->hash_handle); 1465 1466 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 1467 TAILQ_REMOVE(&fdir_info->fdir_list, 1468 fdir_filter, 1469 entries); 1470 rte_free(fdir_filter); 1471 } 1472 1473 return 0; 1474 } 1475 1476 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev) 1477 { 1478 struct ixgbe_l2_tn_info *l2_tn_info = 1479 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1480 struct ixgbe_l2_tn_filter *l2_tn_filter; 1481 1482 if (l2_tn_info->hash_map) 1483 rte_free(l2_tn_info->hash_map); 1484 if (l2_tn_info->hash_handle) 1485 rte_hash_free(l2_tn_info->hash_handle); 1486 1487 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 1488 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, 1489 l2_tn_filter, 1490 entries); 1491 rte_free(l2_tn_filter); 1492 } 1493 1494 return 0; 1495 } 1496 1497 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) 1498 { 1499 struct ixgbe_hw_fdir_info *fdir_info = 1500 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1501 char fdir_hash_name[RTE_HASH_NAMESIZE]; 1502 struct rte_hash_parameters fdir_hash_params = { 1503 .name = fdir_hash_name, 1504 .entries = IXGBE_MAX_FDIR_FILTER_NUM, 1505 .key_len = sizeof(union ixgbe_atr_input), 1506 .hash_func = rte_hash_crc, 1507 .hash_func_init_val = 0, 1508 .socket_id = rte_socket_id(), 1509 }; 1510 1511 TAILQ_INIT(&fdir_info->fdir_list); 1512 snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, 1513 "fdir_%s", eth_dev->device->name); 1514 fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); 1515 if (!fdir_info->hash_handle) { 1516 PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); 1517 return -EINVAL; 1518 } 1519 fdir_info->hash_map = rte_zmalloc("ixgbe", 1520 sizeof(struct ixgbe_fdir_filter *) * 1521 IXGBE_MAX_FDIR_FILTER_NUM, 1522 0); 1523 if (!fdir_info->hash_map) { 1524 PMD_INIT_LOG(ERR, 1525 "Failed to allocate memory for fdir hash map!"); 1526 return -ENOMEM; 1527 } 1528 fdir_info->mask_added = FALSE; 1529 1530 return 0; 1531 } 1532 1533 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) 1534 { 1535 struct ixgbe_l2_tn_info *l2_tn_info = 1536 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1537 char l2_tn_hash_name[RTE_HASH_NAMESIZE]; 1538 struct rte_hash_parameters l2_tn_hash_params = { 1539 .name = l2_tn_hash_name, 1540 .entries = IXGBE_MAX_L2_TN_FILTER_NUM, 1541 .key_len = sizeof(struct ixgbe_l2_tn_key), 1542 .hash_func = rte_hash_crc, 1543 .hash_func_init_val = 0, 1544 .socket_id = rte_socket_id(), 1545 }; 1546 1547 TAILQ_INIT(&l2_tn_info->l2_tn_list); 1548 snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, 1549 "l2_tn_%s", eth_dev->device->name); 1550 l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); 1551 if (!l2_tn_info->hash_handle) { 1552 PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); 1553 return -EINVAL; 1554 } 1555 l2_tn_info->hash_map = rte_zmalloc("ixgbe", 1556 sizeof(struct ixgbe_l2_tn_filter *) * 1557 IXGBE_MAX_L2_TN_FILTER_NUM, 1558 0); 1559 if (!l2_tn_info->hash_map) { 1560 PMD_INIT_LOG(ERR, 1561 "Failed to allocate memory for L2 TN hash map!"); 1562 return -ENOMEM; 1563 } 1564 l2_tn_info->e_tag_en = FALSE; 1565 l2_tn_info->e_tag_fwd_en = FALSE; 1566 l2_tn_info->e_tag_ether_type = DEFAULT_ETAG_ETYPE; 1567 1568 return 0; 1569 } 1570 /* 1571 * Negotiate mailbox API version with the PF. 1572 * After reset API version is always set to the basic one (ixgbe_mbox_api_10). 1573 * Then we try to negotiate starting with the most recent one. 1574 * If all negotiation attempts fail, then we will proceed with 1575 * the default one (ixgbe_mbox_api_10). 1576 */ 1577 static void 1578 ixgbevf_negotiate_api(struct ixgbe_hw *hw) 1579 { 1580 int32_t i; 1581 1582 /* start with highest supported, proceed down */ 1583 static const enum ixgbe_pfvf_api_rev sup_ver[] = { 1584 ixgbe_mbox_api_12, 1585 ixgbe_mbox_api_11, 1586 ixgbe_mbox_api_10, 1587 }; 1588 1589 for (i = 0; 1590 i != RTE_DIM(sup_ver) && 1591 ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0; 1592 i++) 1593 ; 1594 } 1595 1596 static void 1597 generate_random_mac_addr(struct ether_addr *mac_addr) 1598 { 1599 uint64_t random; 1600 1601 /* Set Organizationally Unique Identifier (OUI) prefix. */ 1602 mac_addr->addr_bytes[0] = 0x00; 1603 mac_addr->addr_bytes[1] = 0x09; 1604 mac_addr->addr_bytes[2] = 0xC0; 1605 /* Force indication of locally assigned MAC address. */ 1606 mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR; 1607 /* Generate the last 3 bytes of the MAC address with a random number. */ 1608 random = rte_rand(); 1609 memcpy(&mac_addr->addr_bytes[3], &random, 3); 1610 } 1611 1612 /* 1613 * Virtual Function device init 1614 */ 1615 static int 1616 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) 1617 { 1618 int diag; 1619 uint32_t tc, tcs; 1620 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1621 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1622 struct ixgbe_hw *hw = 1623 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1624 struct ixgbe_vfta *shadow_vfta = 1625 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1626 struct ixgbe_hwstrip *hwstrip = 1627 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1628 struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr; 1629 1630 PMD_INIT_FUNC_TRACE(); 1631 1632 eth_dev->dev_ops = &ixgbevf_eth_dev_ops; 1633 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1634 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1635 1636 /* for secondary processes, we don't initialise any further as primary 1637 * has already done this work. Only check we don't need a different 1638 * RX function 1639 */ 1640 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1641 struct ixgbe_tx_queue *txq; 1642 /* TX queue function in primary, set by last queue initialized 1643 * Tx queue may not initialized by primary process 1644 */ 1645 if (eth_dev->data->tx_queues) { 1646 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1]; 1647 ixgbe_set_tx_function(eth_dev, txq); 1648 } else { 1649 /* Use default TX function if we get here */ 1650 PMD_INIT_LOG(NOTICE, 1651 "No TX queues configured yet. Using default TX function."); 1652 } 1653 1654 ixgbe_set_rx_function(eth_dev); 1655 1656 return 0; 1657 } 1658 1659 rte_eth_copy_pci_info(eth_dev, pci_dev); 1660 1661 hw->device_id = pci_dev->id.device_id; 1662 hw->vendor_id = pci_dev->id.vendor_id; 1663 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1664 1665 /* initialize the vfta */ 1666 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1667 1668 /* initialize the hw strip bitmap*/ 1669 memset(hwstrip, 0, sizeof(*hwstrip)); 1670 1671 /* Initialize the shared code (base driver) */ 1672 diag = ixgbe_init_shared_code(hw); 1673 if (diag != IXGBE_SUCCESS) { 1674 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag); 1675 return -EIO; 1676 } 1677 1678 /* init_mailbox_params */ 1679 hw->mbx.ops.init_params(hw); 1680 1681 /* Reset the hw statistics */ 1682 ixgbevf_dev_stats_reset(eth_dev); 1683 1684 /* Disable the interrupts for VF */ 1685 ixgbevf_intr_disable(hw); 1686 1687 hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */ 1688 diag = hw->mac.ops.reset_hw(hw); 1689 1690 /* 1691 * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when 1692 * the underlying PF driver has not assigned a MAC address to the VF. 1693 * In this case, assign a random MAC address. 1694 */ 1695 if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) { 1696 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1697 return diag; 1698 } 1699 1700 /* negotiate mailbox API version to use with the PF. */ 1701 ixgbevf_negotiate_api(hw); 1702 1703 /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */ 1704 ixgbevf_get_queues(hw, &tcs, &tc); 1705 1706 /* Allocate memory for storing MAC addresses */ 1707 eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN * 1708 hw->mac.num_rar_entries, 0); 1709 if (eth_dev->data->mac_addrs == NULL) { 1710 PMD_INIT_LOG(ERR, 1711 "Failed to allocate %u bytes needed to store " 1712 "MAC addresses", 1713 ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1714 return -ENOMEM; 1715 } 1716 1717 /* Generate a random MAC address, if none was assigned by PF. */ 1718 if (is_zero_ether_addr(perm_addr)) { 1719 generate_random_mac_addr(perm_addr); 1720 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1); 1721 if (diag) { 1722 rte_free(eth_dev->data->mac_addrs); 1723 eth_dev->data->mac_addrs = NULL; 1724 return diag; 1725 } 1726 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1727 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1728 "%02x:%02x:%02x:%02x:%02x:%02x", 1729 perm_addr->addr_bytes[0], 1730 perm_addr->addr_bytes[1], 1731 perm_addr->addr_bytes[2], 1732 perm_addr->addr_bytes[3], 1733 perm_addr->addr_bytes[4], 1734 perm_addr->addr_bytes[5]); 1735 } 1736 1737 /* Copy the permanent MAC address */ 1738 ether_addr_copy(perm_addr, ð_dev->data->mac_addrs[0]); 1739 1740 /* reset the hardware with the new settings */ 1741 diag = hw->mac.ops.start_hw(hw); 1742 switch (diag) { 1743 case 0: 1744 break; 1745 1746 default: 1747 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1748 return -EIO; 1749 } 1750 1751 rte_intr_callback_register(intr_handle, 1752 ixgbevf_dev_interrupt_handler, eth_dev); 1753 rte_intr_enable(intr_handle); 1754 ixgbevf_intr_enable(hw); 1755 1756 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s", 1757 eth_dev->data->port_id, pci_dev->id.vendor_id, 1758 pci_dev->id.device_id, "ixgbe_mac_82599_vf"); 1759 1760 return 0; 1761 } 1762 1763 /* Virtual Function device uninit */ 1764 1765 static int 1766 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) 1767 { 1768 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1769 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1770 struct ixgbe_hw *hw; 1771 1772 PMD_INIT_FUNC_TRACE(); 1773 1774 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1775 return -EPERM; 1776 1777 hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1778 1779 if (hw->adapter_stopped == 0) 1780 ixgbevf_dev_close(eth_dev); 1781 1782 eth_dev->dev_ops = NULL; 1783 eth_dev->rx_pkt_burst = NULL; 1784 eth_dev->tx_pkt_burst = NULL; 1785 1786 /* Disable the interrupts for VF */ 1787 ixgbevf_intr_disable(hw); 1788 1789 rte_free(eth_dev->data->mac_addrs); 1790 eth_dev->data->mac_addrs = NULL; 1791 1792 rte_intr_disable(intr_handle); 1793 rte_intr_callback_unregister(intr_handle, 1794 ixgbevf_dev_interrupt_handler, eth_dev); 1795 1796 return 0; 1797 } 1798 1799 static int eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1800 struct rte_pci_device *pci_dev) 1801 { 1802 return rte_eth_dev_pci_generic_probe(pci_dev, 1803 sizeof(struct ixgbe_adapter), eth_ixgbe_dev_init); 1804 } 1805 1806 static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev) 1807 { 1808 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbe_dev_uninit); 1809 } 1810 1811 static struct rte_pci_driver rte_ixgbe_pmd = { 1812 .id_table = pci_id_ixgbe_map, 1813 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 1814 RTE_PCI_DRV_IOVA_AS_VA, 1815 .probe = eth_ixgbe_pci_probe, 1816 .remove = eth_ixgbe_pci_remove, 1817 }; 1818 1819 static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1820 struct rte_pci_device *pci_dev) 1821 { 1822 return rte_eth_dev_pci_generic_probe(pci_dev, 1823 sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init); 1824 } 1825 1826 static int eth_ixgbevf_pci_remove(struct rte_pci_device *pci_dev) 1827 { 1828 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbevf_dev_uninit); 1829 } 1830 1831 /* 1832 * virtual function driver struct 1833 */ 1834 static struct rte_pci_driver rte_ixgbevf_pmd = { 1835 .id_table = pci_id_ixgbevf_map, 1836 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1837 .probe = eth_ixgbevf_pci_probe, 1838 .remove = eth_ixgbevf_pci_remove, 1839 }; 1840 1841 static int 1842 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1843 { 1844 struct ixgbe_hw *hw = 1845 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1846 struct ixgbe_vfta *shadow_vfta = 1847 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1848 uint32_t vfta; 1849 uint32_t vid_idx; 1850 uint32_t vid_bit; 1851 1852 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 1853 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 1854 vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx)); 1855 if (on) 1856 vfta |= vid_bit; 1857 else 1858 vfta &= ~vid_bit; 1859 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta); 1860 1861 /* update local VFTA copy */ 1862 shadow_vfta->vfta[vid_idx] = vfta; 1863 1864 return 0; 1865 } 1866 1867 static void 1868 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 1869 { 1870 if (on) 1871 ixgbe_vlan_hw_strip_enable(dev, queue); 1872 else 1873 ixgbe_vlan_hw_strip_disable(dev, queue); 1874 } 1875 1876 static int 1877 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 1878 enum rte_vlan_type vlan_type, 1879 uint16_t tpid) 1880 { 1881 struct ixgbe_hw *hw = 1882 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1883 int ret = 0; 1884 uint32_t reg; 1885 uint32_t qinq; 1886 1887 qinq = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1888 qinq &= IXGBE_DMATXCTL_GDV; 1889 1890 switch (vlan_type) { 1891 case ETH_VLAN_TYPE_INNER: 1892 if (qinq) { 1893 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1894 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1895 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1896 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1897 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1898 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1899 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1900 } else { 1901 ret = -ENOTSUP; 1902 PMD_DRV_LOG(ERR, "Inner type is not supported" 1903 " by single VLAN"); 1904 } 1905 break; 1906 case ETH_VLAN_TYPE_OUTER: 1907 if (qinq) { 1908 /* Only the high 16-bits is valid */ 1909 IXGBE_WRITE_REG(hw, IXGBE_EXVET, (uint32_t)tpid << 1910 IXGBE_EXVET_VET_EXT_SHIFT); 1911 } else { 1912 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1913 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1914 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1915 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1916 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1917 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1918 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1919 } 1920 1921 break; 1922 default: 1923 ret = -EINVAL; 1924 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type); 1925 break; 1926 } 1927 1928 return ret; 1929 } 1930 1931 void 1932 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev) 1933 { 1934 struct ixgbe_hw *hw = 1935 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1936 uint32_t vlnctrl; 1937 1938 PMD_INIT_FUNC_TRACE(); 1939 1940 /* Filter Table Disable */ 1941 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1942 vlnctrl &= ~IXGBE_VLNCTRL_VFE; 1943 1944 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1945 } 1946 1947 void 1948 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1949 { 1950 struct ixgbe_hw *hw = 1951 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1952 struct ixgbe_vfta *shadow_vfta = 1953 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1954 uint32_t vlnctrl; 1955 uint16_t i; 1956 1957 PMD_INIT_FUNC_TRACE(); 1958 1959 /* Filter Table Enable */ 1960 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1961 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 1962 vlnctrl |= IXGBE_VLNCTRL_VFE; 1963 1964 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1965 1966 /* write whatever is in local vfta copy */ 1967 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 1968 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]); 1969 } 1970 1971 static void 1972 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on) 1973 { 1974 struct ixgbe_hwstrip *hwstrip = 1975 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private); 1976 struct ixgbe_rx_queue *rxq; 1977 1978 if (queue >= IXGBE_MAX_RX_QUEUE_NUM) 1979 return; 1980 1981 if (on) 1982 IXGBE_SET_HWSTRIP(hwstrip, queue); 1983 else 1984 IXGBE_CLEAR_HWSTRIP(hwstrip, queue); 1985 1986 if (queue >= dev->data->nb_rx_queues) 1987 return; 1988 1989 rxq = dev->data->rx_queues[queue]; 1990 1991 if (on) 1992 rxq->vlan_flags = PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1993 else 1994 rxq->vlan_flags = PKT_RX_VLAN; 1995 } 1996 1997 static void 1998 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue) 1999 { 2000 struct ixgbe_hw *hw = 2001 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2002 uint32_t ctrl; 2003 2004 PMD_INIT_FUNC_TRACE(); 2005 2006 if (hw->mac.type == ixgbe_mac_82598EB) { 2007 /* No queue level support */ 2008 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2009 return; 2010 } 2011 2012 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2013 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2014 ctrl &= ~IXGBE_RXDCTL_VME; 2015 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2016 2017 /* record those setting for HW strip per queue */ 2018 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0); 2019 } 2020 2021 static void 2022 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue) 2023 { 2024 struct ixgbe_hw *hw = 2025 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2026 uint32_t ctrl; 2027 2028 PMD_INIT_FUNC_TRACE(); 2029 2030 if (hw->mac.type == ixgbe_mac_82598EB) { 2031 /* No queue level supported */ 2032 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2033 return; 2034 } 2035 2036 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2037 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2038 ctrl |= IXGBE_RXDCTL_VME; 2039 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2040 2041 /* record those setting for HW strip per queue */ 2042 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1); 2043 } 2044 2045 void 2046 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev) 2047 { 2048 struct ixgbe_hw *hw = 2049 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2050 uint32_t ctrl; 2051 uint16_t i; 2052 struct ixgbe_rx_queue *rxq; 2053 2054 PMD_INIT_FUNC_TRACE(); 2055 2056 if (hw->mac.type == ixgbe_mac_82598EB) { 2057 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2058 ctrl &= ~IXGBE_VLNCTRL_VME; 2059 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2060 } else { 2061 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2062 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2063 rxq = dev->data->rx_queues[i]; 2064 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 2065 ctrl &= ~IXGBE_RXDCTL_VME; 2066 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); 2067 2068 /* record those setting for HW strip per queue */ 2069 ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0); 2070 } 2071 } 2072 } 2073 2074 void 2075 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev) 2076 { 2077 struct ixgbe_hw *hw = 2078 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2079 uint32_t ctrl; 2080 uint16_t i; 2081 struct ixgbe_rx_queue *rxq; 2082 2083 PMD_INIT_FUNC_TRACE(); 2084 2085 if (hw->mac.type == ixgbe_mac_82598EB) { 2086 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2087 ctrl |= IXGBE_VLNCTRL_VME; 2088 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2089 } else { 2090 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2091 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2092 rxq = dev->data->rx_queues[i]; 2093 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 2094 ctrl |= IXGBE_RXDCTL_VME; 2095 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); 2096 2097 /* record those setting for HW strip per queue */ 2098 ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1); 2099 } 2100 } 2101 } 2102 2103 static void 2104 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2105 { 2106 struct ixgbe_hw *hw = 2107 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2108 uint32_t ctrl; 2109 2110 PMD_INIT_FUNC_TRACE(); 2111 2112 /* DMATXCTRL: Geric Double VLAN Disable */ 2113 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2114 ctrl &= ~IXGBE_DMATXCTL_GDV; 2115 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2116 2117 /* CTRL_EXT: Global Double VLAN Disable */ 2118 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2119 ctrl &= ~IXGBE_EXTENDED_VLAN; 2120 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2121 2122 } 2123 2124 static void 2125 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2126 { 2127 struct ixgbe_hw *hw = 2128 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2129 uint32_t ctrl; 2130 2131 PMD_INIT_FUNC_TRACE(); 2132 2133 /* DMATXCTRL: Geric Double VLAN Enable */ 2134 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2135 ctrl |= IXGBE_DMATXCTL_GDV; 2136 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2137 2138 /* CTRL_EXT: Global Double VLAN Enable */ 2139 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2140 ctrl |= IXGBE_EXTENDED_VLAN; 2141 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2142 2143 /* Clear pooling mode of PFVTCTL. It's required by X550. */ 2144 if (hw->mac.type == ixgbe_mac_X550 || 2145 hw->mac.type == ixgbe_mac_X550EM_x || 2146 hw->mac.type == ixgbe_mac_X550EM_a) { 2147 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 2148 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 2149 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 2150 } 2151 2152 /* 2153 * VET EXT field in the EXVET register = 0x8100 by default 2154 * So no need to change. Same to VT field of DMATXCTL register 2155 */ 2156 } 2157 2158 static int 2159 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2160 { 2161 if (mask & ETH_VLAN_STRIP_MASK) { 2162 if (dev->data->dev_conf.rxmode.hw_vlan_strip) 2163 ixgbe_vlan_hw_strip_enable_all(dev); 2164 else 2165 ixgbe_vlan_hw_strip_disable_all(dev); 2166 } 2167 2168 if (mask & ETH_VLAN_FILTER_MASK) { 2169 if (dev->data->dev_conf.rxmode.hw_vlan_filter) 2170 ixgbe_vlan_hw_filter_enable(dev); 2171 else 2172 ixgbe_vlan_hw_filter_disable(dev); 2173 } 2174 2175 if (mask & ETH_VLAN_EXTEND_MASK) { 2176 if (dev->data->dev_conf.rxmode.hw_vlan_extend) 2177 ixgbe_vlan_hw_extend_enable(dev); 2178 else 2179 ixgbe_vlan_hw_extend_disable(dev); 2180 } 2181 2182 return 0; 2183 } 2184 2185 static void 2186 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2187 { 2188 struct ixgbe_hw *hw = 2189 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2190 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 2191 uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2192 2193 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 2194 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 2195 } 2196 2197 static int 2198 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) 2199 { 2200 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2201 2202 switch (nb_rx_q) { 2203 case 1: 2204 case 2: 2205 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS; 2206 break; 2207 case 4: 2208 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS; 2209 break; 2210 default: 2211 return -EINVAL; 2212 } 2213 2214 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 2215 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2216 RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = 2217 pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2218 return 0; 2219 } 2220 2221 static int 2222 ixgbe_check_mq_mode(struct rte_eth_dev *dev) 2223 { 2224 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 2225 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2226 uint16_t nb_rx_q = dev->data->nb_rx_queues; 2227 uint16_t nb_tx_q = dev->data->nb_tx_queues; 2228 2229 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 2230 /* check multi-queue mode */ 2231 switch (dev_conf->rxmode.mq_mode) { 2232 case ETH_MQ_RX_VMDQ_DCB: 2233 PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV"); 2234 break; 2235 case ETH_MQ_RX_VMDQ_DCB_RSS: 2236 /* DCB/RSS VMDQ in SRIOV mode, not implement yet */ 2237 PMD_INIT_LOG(ERR, "SRIOV active," 2238 " unsupported mq_mode rx %d.", 2239 dev_conf->rxmode.mq_mode); 2240 return -EINVAL; 2241 case ETH_MQ_RX_RSS: 2242 case ETH_MQ_RX_VMDQ_RSS: 2243 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS; 2244 if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) 2245 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) { 2246 PMD_INIT_LOG(ERR, "SRIOV is active," 2247 " invalid queue number" 2248 " for VMDQ RSS, allowed" 2249 " value are 1, 2 or 4."); 2250 return -EINVAL; 2251 } 2252 break; 2253 case ETH_MQ_RX_VMDQ_ONLY: 2254 case ETH_MQ_RX_NONE: 2255 /* if nothing mq mode configure, use default scheme */ 2256 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 2257 break; 2258 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/ 2259 /* SRIOV only works in VMDq enable mode */ 2260 PMD_INIT_LOG(ERR, "SRIOV is active," 2261 " wrong mq_mode rx %d.", 2262 dev_conf->rxmode.mq_mode); 2263 return -EINVAL; 2264 } 2265 2266 switch (dev_conf->txmode.mq_mode) { 2267 case ETH_MQ_TX_VMDQ_DCB: 2268 PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV"); 2269 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2270 break; 2271 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */ 2272 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY; 2273 break; 2274 } 2275 2276 /* check valid queue number */ 2277 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) || 2278 (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) { 2279 PMD_INIT_LOG(ERR, "SRIOV is active," 2280 " nb_rx_q=%d nb_tx_q=%d queue number" 2281 " must be less than or equal to %d.", 2282 nb_rx_q, nb_tx_q, 2283 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool); 2284 return -EINVAL; 2285 } 2286 } else { 2287 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) { 2288 PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is" 2289 " not supported."); 2290 return -EINVAL; 2291 } 2292 /* check configuration for vmdb+dcb mode */ 2293 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) { 2294 const struct rte_eth_vmdq_dcb_conf *conf; 2295 2296 if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2297 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.", 2298 IXGBE_VMDQ_DCB_NB_QUEUES); 2299 return -EINVAL; 2300 } 2301 conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf; 2302 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2303 conf->nb_queue_pools == ETH_32_POOLS)) { 2304 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2305 " nb_queue_pools must be %d or %d.", 2306 ETH_16_POOLS, ETH_32_POOLS); 2307 return -EINVAL; 2308 } 2309 } 2310 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { 2311 const struct rte_eth_vmdq_dcb_tx_conf *conf; 2312 2313 if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2314 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d", 2315 IXGBE_VMDQ_DCB_NB_QUEUES); 2316 return -EINVAL; 2317 } 2318 conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2319 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2320 conf->nb_queue_pools == ETH_32_POOLS)) { 2321 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2322 " nb_queue_pools != %d and" 2323 " nb_queue_pools != %d.", 2324 ETH_16_POOLS, ETH_32_POOLS); 2325 return -EINVAL; 2326 } 2327 } 2328 2329 /* For DCB mode check our configuration before we go further */ 2330 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) { 2331 const struct rte_eth_dcb_rx_conf *conf; 2332 2333 conf = &dev_conf->rx_adv_conf.dcb_rx_conf; 2334 if (!(conf->nb_tcs == ETH_4_TCS || 2335 conf->nb_tcs == ETH_8_TCS)) { 2336 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2337 " and nb_tcs != %d.", 2338 ETH_4_TCS, ETH_8_TCS); 2339 return -EINVAL; 2340 } 2341 } 2342 2343 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { 2344 const struct rte_eth_dcb_tx_conf *conf; 2345 2346 conf = &dev_conf->tx_adv_conf.dcb_tx_conf; 2347 if (!(conf->nb_tcs == ETH_4_TCS || 2348 conf->nb_tcs == ETH_8_TCS)) { 2349 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2350 " and nb_tcs != %d.", 2351 ETH_4_TCS, ETH_8_TCS); 2352 return -EINVAL; 2353 } 2354 } 2355 2356 /* 2357 * When DCB/VT is off, maximum number of queues changes, 2358 * except for 82598EB, which remains constant. 2359 */ 2360 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 2361 hw->mac.type != ixgbe_mac_82598EB) { 2362 if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) { 2363 PMD_INIT_LOG(ERR, 2364 "Neither VT nor DCB are enabled, " 2365 "nb_tx_q > %d.", 2366 IXGBE_NONE_MODE_TX_NB_QUEUES); 2367 return -EINVAL; 2368 } 2369 } 2370 } 2371 return 0; 2372 } 2373 2374 static int 2375 ixgbe_dev_configure(struct rte_eth_dev *dev) 2376 { 2377 struct ixgbe_interrupt *intr = 2378 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2379 struct ixgbe_adapter *adapter = 2380 (struct ixgbe_adapter *)dev->data->dev_private; 2381 int ret; 2382 2383 PMD_INIT_FUNC_TRACE(); 2384 /* multipe queue mode checking */ 2385 ret = ixgbe_check_mq_mode(dev); 2386 if (ret != 0) { 2387 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.", 2388 ret); 2389 return ret; 2390 } 2391 2392 /* set flag to update link status after init */ 2393 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2394 2395 /* 2396 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 2397 * allocation or vector Rx preconditions we will reset it. 2398 */ 2399 adapter->rx_bulk_alloc_allowed = true; 2400 adapter->rx_vec_allowed = true; 2401 2402 return 0; 2403 } 2404 2405 static void 2406 ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) 2407 { 2408 struct ixgbe_hw *hw = 2409 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2410 struct ixgbe_interrupt *intr = 2411 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2412 uint32_t gpie; 2413 2414 /* only set up it on X550EM_X */ 2415 if (hw->mac.type == ixgbe_mac_X550EM_x) { 2416 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 2417 gpie |= IXGBE_SDP0_GPIEN_X550EM_x; 2418 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 2419 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 2420 intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x; 2421 } 2422 } 2423 2424 int 2425 ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, 2426 uint16_t tx_rate, uint64_t q_msk) 2427 { 2428 struct ixgbe_hw *hw; 2429 struct ixgbe_vf_info *vfinfo; 2430 struct rte_eth_link link; 2431 uint8_t nb_q_per_pool; 2432 uint32_t queue_stride; 2433 uint32_t queue_idx, idx = 0, vf_idx; 2434 uint32_t queue_end; 2435 uint16_t total_rate = 0; 2436 struct rte_pci_device *pci_dev; 2437 2438 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2439 rte_eth_link_get_nowait(dev->data->port_id, &link); 2440 2441 if (vf >= pci_dev->max_vfs) 2442 return -EINVAL; 2443 2444 if (tx_rate > link.link_speed) 2445 return -EINVAL; 2446 2447 if (q_msk == 0) 2448 return 0; 2449 2450 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2451 vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); 2452 nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2453 queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2454 queue_idx = vf * queue_stride; 2455 queue_end = queue_idx + nb_q_per_pool - 1; 2456 if (queue_end >= hw->mac.max_tx_queues) 2457 return -EINVAL; 2458 2459 if (vfinfo) { 2460 for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) { 2461 if (vf_idx == vf) 2462 continue; 2463 for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); 2464 idx++) 2465 total_rate += vfinfo[vf_idx].tx_rate[idx]; 2466 } 2467 } else { 2468 return -EINVAL; 2469 } 2470 2471 /* Store tx_rate for this vf. */ 2472 for (idx = 0; idx < nb_q_per_pool; idx++) { 2473 if (((uint64_t)0x1 << idx) & q_msk) { 2474 if (vfinfo[vf].tx_rate[idx] != tx_rate) 2475 vfinfo[vf].tx_rate[idx] = tx_rate; 2476 total_rate += tx_rate; 2477 } 2478 } 2479 2480 if (total_rate > dev->data->dev_link.link_speed) { 2481 /* Reset stored TX rate of the VF if it causes exceed 2482 * link speed. 2483 */ 2484 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); 2485 return -EINVAL; 2486 } 2487 2488 /* Set RTTBCNRC of each queue/pool for vf X */ 2489 for (; queue_idx <= queue_end; queue_idx++) { 2490 if (0x1 & q_msk) 2491 ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); 2492 q_msk = q_msk >> 1; 2493 } 2494 2495 return 0; 2496 } 2497 2498 /* 2499 * Configure device link speed and setup link. 2500 * It returns 0 on success. 2501 */ 2502 static int 2503 ixgbe_dev_start(struct rte_eth_dev *dev) 2504 { 2505 struct ixgbe_hw *hw = 2506 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2507 struct ixgbe_vf_info *vfinfo = 2508 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2509 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2510 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2511 uint32_t intr_vector = 0; 2512 int err, link_up = 0, negotiate = 0; 2513 uint32_t speed = 0; 2514 int mask = 0; 2515 int status; 2516 uint16_t vf, idx; 2517 uint32_t *link_speeds; 2518 struct ixgbe_tm_conf *tm_conf = 2519 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2520 2521 PMD_INIT_FUNC_TRACE(); 2522 2523 /* IXGBE devices don't support: 2524 * - half duplex (checked afterwards for valid speeds) 2525 * - fixed speed: TODO implement 2526 */ 2527 if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) { 2528 PMD_INIT_LOG(ERR, 2529 "Invalid link_speeds for port %u, fix speed not supported", 2530 dev->data->port_id); 2531 return -EINVAL; 2532 } 2533 2534 /* disable uio/vfio intr/eventfd mapping */ 2535 rte_intr_disable(intr_handle); 2536 2537 /* stop adapter */ 2538 hw->adapter_stopped = 0; 2539 ixgbe_stop_adapter(hw); 2540 2541 /* reinitialize adapter 2542 * this calls reset and start 2543 */ 2544 status = ixgbe_pf_reset_hw(hw); 2545 if (status != 0) 2546 return -1; 2547 hw->mac.ops.start_hw(hw); 2548 hw->mac.get_link_status = true; 2549 2550 /* configure PF module if SRIOV enabled */ 2551 ixgbe_pf_host_configure(dev); 2552 2553 ixgbe_dev_phy_intr_setup(dev); 2554 2555 /* check and configure queue intr-vector mapping */ 2556 if ((rte_intr_cap_multiple(intr_handle) || 2557 !RTE_ETH_DEV_SRIOV(dev).active) && 2558 dev->data->dev_conf.intr_conf.rxq != 0) { 2559 intr_vector = dev->data->nb_rx_queues; 2560 if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) { 2561 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 2562 IXGBE_MAX_INTR_QUEUE_NUM); 2563 return -ENOTSUP; 2564 } 2565 if (rte_intr_efd_enable(intr_handle, intr_vector)) 2566 return -1; 2567 } 2568 2569 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 2570 intr_handle->intr_vec = 2571 rte_zmalloc("intr_vec", 2572 dev->data->nb_rx_queues * sizeof(int), 0); 2573 if (intr_handle->intr_vec == NULL) { 2574 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 2575 " intr_vec", dev->data->nb_rx_queues); 2576 return -ENOMEM; 2577 } 2578 } 2579 2580 /* confiugre msix for sleep until rx interrupt */ 2581 ixgbe_configure_msix(dev); 2582 2583 /* initialize transmission unit */ 2584 ixgbe_dev_tx_init(dev); 2585 2586 /* This can fail when allocating mbufs for descriptor rings */ 2587 err = ixgbe_dev_rx_init(dev); 2588 if (err) { 2589 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 2590 goto error; 2591 } 2592 2593 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 2594 ETH_VLAN_EXTEND_MASK; 2595 err = ixgbe_vlan_offload_set(dev, mask); 2596 if (err) { 2597 PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); 2598 goto error; 2599 } 2600 2601 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 2602 /* Enable vlan filtering for VMDq */ 2603 ixgbe_vmdq_vlan_hw_filter_enable(dev); 2604 } 2605 2606 /* Configure DCB hw */ 2607 ixgbe_configure_dcb(dev); 2608 2609 if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { 2610 err = ixgbe_fdir_configure(dev); 2611 if (err) 2612 goto error; 2613 } 2614 2615 /* Restore vf rate limit */ 2616 if (vfinfo != NULL) { 2617 for (vf = 0; vf < pci_dev->max_vfs; vf++) 2618 for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) 2619 if (vfinfo[vf].tx_rate[idx] != 0) 2620 ixgbe_set_vf_rate_limit( 2621 dev, vf, 2622 vfinfo[vf].tx_rate[idx], 2623 1 << idx); 2624 } 2625 2626 ixgbe_restore_statistics_mapping(dev); 2627 2628 err = ixgbe_dev_rxtx_start(dev); 2629 if (err < 0) { 2630 PMD_INIT_LOG(ERR, "Unable to start rxtx queues"); 2631 goto error; 2632 } 2633 2634 /* Skip link setup if loopback mode is enabled for 82599. */ 2635 if (hw->mac.type == ixgbe_mac_82599EB && 2636 dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX) 2637 goto skip_link_setup; 2638 2639 if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) { 2640 err = hw->mac.ops.setup_sfp(hw); 2641 if (err) 2642 goto error; 2643 } 2644 2645 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2646 /* Turn on the copper */ 2647 ixgbe_set_phy_power(hw, true); 2648 } else { 2649 /* Turn on the laser */ 2650 ixgbe_enable_tx_laser(hw); 2651 } 2652 2653 err = ixgbe_check_link(hw, &speed, &link_up, 0); 2654 if (err) 2655 goto error; 2656 dev->data->dev_link.link_status = link_up; 2657 2658 err = ixgbe_get_link_capabilities(hw, &speed, &negotiate); 2659 if (err) 2660 goto error; 2661 2662 link_speeds = &dev->data->dev_conf.link_speeds; 2663 if (*link_speeds & ~(ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2664 ETH_LINK_SPEED_10G)) { 2665 PMD_INIT_LOG(ERR, "Invalid link setting"); 2666 goto error; 2667 } 2668 2669 speed = 0x0; 2670 if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { 2671 switch (hw->mac.type) { 2672 case ixgbe_mac_82598EB: 2673 speed = IXGBE_LINK_SPEED_82598_AUTONEG; 2674 break; 2675 case ixgbe_mac_82599EB: 2676 case ixgbe_mac_X540: 2677 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2678 break; 2679 case ixgbe_mac_X550: 2680 case ixgbe_mac_X550EM_x: 2681 case ixgbe_mac_X550EM_a: 2682 speed = IXGBE_LINK_SPEED_X550_AUTONEG; 2683 break; 2684 default: 2685 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2686 } 2687 } else { 2688 if (*link_speeds & ETH_LINK_SPEED_10G) 2689 speed |= IXGBE_LINK_SPEED_10GB_FULL; 2690 if (*link_speeds & ETH_LINK_SPEED_1G) 2691 speed |= IXGBE_LINK_SPEED_1GB_FULL; 2692 if (*link_speeds & ETH_LINK_SPEED_100M) 2693 speed |= IXGBE_LINK_SPEED_100_FULL; 2694 } 2695 2696 err = ixgbe_setup_link(hw, speed, link_up); 2697 if (err) 2698 goto error; 2699 2700 skip_link_setup: 2701 2702 if (rte_intr_allow_others(intr_handle)) { 2703 /* check if lsc interrupt is enabled */ 2704 if (dev->data->dev_conf.intr_conf.lsc != 0) 2705 ixgbe_dev_lsc_interrupt_setup(dev, TRUE); 2706 else 2707 ixgbe_dev_lsc_interrupt_setup(dev, FALSE); 2708 ixgbe_dev_macsec_interrupt_setup(dev); 2709 } else { 2710 rte_intr_callback_unregister(intr_handle, 2711 ixgbe_dev_interrupt_handler, dev); 2712 if (dev->data->dev_conf.intr_conf.lsc != 0) 2713 PMD_INIT_LOG(INFO, "lsc won't enable because of" 2714 " no intr multiplex"); 2715 } 2716 2717 /* check if rxq interrupt is enabled */ 2718 if (dev->data->dev_conf.intr_conf.rxq != 0 && 2719 rte_intr_dp_is_en(intr_handle)) 2720 ixgbe_dev_rxq_interrupt_setup(dev); 2721 2722 /* enable uio/vfio intr/eventfd mapping */ 2723 rte_intr_enable(intr_handle); 2724 2725 /* resume enabled intr since hw reset */ 2726 ixgbe_enable_intr(dev); 2727 ixgbe_l2_tunnel_conf(dev); 2728 ixgbe_filter_restore(dev); 2729 2730 if (tm_conf->root && !tm_conf->committed) 2731 PMD_DRV_LOG(WARNING, 2732 "please call hierarchy_commit() " 2733 "before starting the port"); 2734 2735 return 0; 2736 2737 error: 2738 PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err); 2739 ixgbe_dev_clear_queues(dev); 2740 return -EIO; 2741 } 2742 2743 /* 2744 * Stop device: disable rx and tx functions to allow for reconfiguring. 2745 */ 2746 static void 2747 ixgbe_dev_stop(struct rte_eth_dev *dev) 2748 { 2749 struct rte_eth_link link; 2750 struct ixgbe_hw *hw = 2751 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2752 struct ixgbe_vf_info *vfinfo = 2753 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2754 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2755 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2756 int vf; 2757 struct ixgbe_tm_conf *tm_conf = 2758 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2759 2760 PMD_INIT_FUNC_TRACE(); 2761 2762 /* disable interrupts */ 2763 ixgbe_disable_intr(hw); 2764 2765 /* reset the NIC */ 2766 ixgbe_pf_reset_hw(hw); 2767 hw->adapter_stopped = 0; 2768 2769 /* stop adapter */ 2770 ixgbe_stop_adapter(hw); 2771 2772 for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) 2773 vfinfo[vf].clear_to_send = false; 2774 2775 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2776 /* Turn off the copper */ 2777 ixgbe_set_phy_power(hw, false); 2778 } else { 2779 /* Turn off the laser */ 2780 ixgbe_disable_tx_laser(hw); 2781 } 2782 2783 ixgbe_dev_clear_queues(dev); 2784 2785 /* Clear stored conf */ 2786 dev->data->scattered_rx = 0; 2787 dev->data->lro = 0; 2788 2789 /* Clear recorded link status */ 2790 memset(&link, 0, sizeof(link)); 2791 rte_ixgbe_dev_atomic_write_link_status(dev, &link); 2792 2793 if (!rte_intr_allow_others(intr_handle)) 2794 /* resume to the default handler */ 2795 rte_intr_callback_register(intr_handle, 2796 ixgbe_dev_interrupt_handler, 2797 (void *)dev); 2798 2799 /* Clean datapath event and queue/vec mapping */ 2800 rte_intr_efd_disable(intr_handle); 2801 if (intr_handle->intr_vec != NULL) { 2802 rte_free(intr_handle->intr_vec); 2803 intr_handle->intr_vec = NULL; 2804 } 2805 2806 /* reset hierarchy commit */ 2807 tm_conf->committed = false; 2808 } 2809 2810 /* 2811 * Set device link up: enable tx. 2812 */ 2813 static int 2814 ixgbe_dev_set_link_up(struct rte_eth_dev *dev) 2815 { 2816 struct ixgbe_hw *hw = 2817 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2818 if (hw->mac.type == ixgbe_mac_82599EB) { 2819 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2820 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2821 /* Not suported in bypass mode */ 2822 PMD_INIT_LOG(ERR, "Set link up is not supported " 2823 "by device id 0x%x", hw->device_id); 2824 return -ENOTSUP; 2825 } 2826 #endif 2827 } 2828 2829 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2830 /* Turn on the copper */ 2831 ixgbe_set_phy_power(hw, true); 2832 } else { 2833 /* Turn on the laser */ 2834 ixgbe_enable_tx_laser(hw); 2835 } 2836 2837 return 0; 2838 } 2839 2840 /* 2841 * Set device link down: disable tx. 2842 */ 2843 static int 2844 ixgbe_dev_set_link_down(struct rte_eth_dev *dev) 2845 { 2846 struct ixgbe_hw *hw = 2847 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2848 if (hw->mac.type == ixgbe_mac_82599EB) { 2849 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2850 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2851 /* Not suported in bypass mode */ 2852 PMD_INIT_LOG(ERR, "Set link down is not supported " 2853 "by device id 0x%x", hw->device_id); 2854 return -ENOTSUP; 2855 } 2856 #endif 2857 } 2858 2859 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2860 /* Turn off the copper */ 2861 ixgbe_set_phy_power(hw, false); 2862 } else { 2863 /* Turn off the laser */ 2864 ixgbe_disable_tx_laser(hw); 2865 } 2866 2867 return 0; 2868 } 2869 2870 /* 2871 * Reset and stop device. 2872 */ 2873 static void 2874 ixgbe_dev_close(struct rte_eth_dev *dev) 2875 { 2876 struct ixgbe_hw *hw = 2877 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2878 2879 PMD_INIT_FUNC_TRACE(); 2880 2881 ixgbe_pf_reset_hw(hw); 2882 2883 ixgbe_dev_stop(dev); 2884 hw->adapter_stopped = 1; 2885 2886 ixgbe_dev_free_queues(dev); 2887 2888 ixgbe_disable_pcie_master(hw); 2889 2890 /* reprogram the RAR[0] in case user changed it. */ 2891 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 2892 } 2893 2894 /* 2895 * Reset PF device. 2896 */ 2897 static int 2898 ixgbe_dev_reset(struct rte_eth_dev *dev) 2899 { 2900 int ret; 2901 2902 /* When a DPDK PMD PF begin to reset PF port, it should notify all 2903 * its VF to make them align with it. The detailed notification 2904 * mechanism is PMD specific. As to ixgbe PF, it is rather complex. 2905 * To avoid unexpected behavior in VF, currently reset of PF with 2906 * SR-IOV activation is not supported. It might be supported later. 2907 */ 2908 if (dev->data->sriov.active) 2909 return -ENOTSUP; 2910 2911 ret = eth_ixgbe_dev_uninit(dev); 2912 if (ret) 2913 return ret; 2914 2915 ret = eth_ixgbe_dev_init(dev); 2916 2917 return ret; 2918 } 2919 2920 static void 2921 ixgbe_read_stats_registers(struct ixgbe_hw *hw, 2922 struct ixgbe_hw_stats *hw_stats, 2923 struct ixgbe_macsec_stats *macsec_stats, 2924 uint64_t *total_missed_rx, uint64_t *total_qbrc, 2925 uint64_t *total_qprc, uint64_t *total_qprdc) 2926 { 2927 uint32_t bprc, lxon, lxoff, total; 2928 uint32_t delta_gprc = 0; 2929 unsigned i; 2930 /* Workaround for RX byte count not including CRC bytes when CRC 2931 * strip is enabled. CRC bytes are removed from counters when crc_strip 2932 * is disabled. 2933 */ 2934 int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) & 2935 IXGBE_HLREG0_RXCRCSTRP); 2936 2937 hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 2938 hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 2939 hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 2940 hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 2941 2942 for (i = 0; i < 8; i++) { 2943 uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 2944 2945 /* global total per queue */ 2946 hw_stats->mpc[i] += mp; 2947 /* Running comprehensive total for stats display */ 2948 *total_missed_rx += hw_stats->mpc[i]; 2949 if (hw->mac.type == ixgbe_mac_82598EB) { 2950 hw_stats->rnbc[i] += 2951 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 2952 hw_stats->pxonrxc[i] += 2953 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 2954 hw_stats->pxoffrxc[i] += 2955 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 2956 } else { 2957 hw_stats->pxonrxc[i] += 2958 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 2959 hw_stats->pxoffrxc[i] += 2960 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 2961 hw_stats->pxon2offc[i] += 2962 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 2963 } 2964 hw_stats->pxontxc[i] += 2965 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 2966 hw_stats->pxofftxc[i] += 2967 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 2968 } 2969 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 2970 uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 2971 uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 2972 uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 2973 2974 delta_gprc += delta_qprc; 2975 2976 hw_stats->qprc[i] += delta_qprc; 2977 hw_stats->qptc[i] += delta_qptc; 2978 2979 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 2980 hw_stats->qbrc[i] += 2981 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32); 2982 if (crc_strip == 0) 2983 hw_stats->qbrc[i] -= delta_qprc * ETHER_CRC_LEN; 2984 2985 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 2986 hw_stats->qbtc[i] += 2987 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32); 2988 2989 hw_stats->qprdc[i] += delta_qprdc; 2990 *total_qprdc += hw_stats->qprdc[i]; 2991 2992 *total_qprc += hw_stats->qprc[i]; 2993 *total_qbrc += hw_stats->qbrc[i]; 2994 } 2995 hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 2996 hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 2997 hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 2998 2999 /* 3000 * An errata states that gprc actually counts good + missed packets: 3001 * Workaround to set gprc to summated queue packet receives 3002 */ 3003 hw_stats->gprc = *total_qprc; 3004 3005 if (hw->mac.type != ixgbe_mac_82598EB) { 3006 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 3007 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3008 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 3009 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3010 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 3011 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3012 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3013 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3014 } else { 3015 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3016 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3017 /* 82598 only has a counter in the high register */ 3018 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3019 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3020 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3021 } 3022 uint64_t old_tpr = hw_stats->tpr; 3023 3024 hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3025 hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3026 3027 if (crc_strip == 0) 3028 hw_stats->gorc -= delta_gprc * ETHER_CRC_LEN; 3029 3030 uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC); 3031 hw_stats->gptc += delta_gptc; 3032 hw_stats->gotc -= delta_gptc * ETHER_CRC_LEN; 3033 hw_stats->tor -= (hw_stats->tpr - old_tpr) * ETHER_CRC_LEN; 3034 3035 /* 3036 * Workaround: mprc hardware is incorrectly counting 3037 * broadcasts, so for now we subtract those. 3038 */ 3039 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3040 hw_stats->bprc += bprc; 3041 hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3042 if (hw->mac.type == ixgbe_mac_82598EB) 3043 hw_stats->mprc -= bprc; 3044 3045 hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3046 hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3047 hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3048 hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3049 hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3050 hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3051 3052 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3053 hw_stats->lxontxc += lxon; 3054 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3055 hw_stats->lxofftxc += lxoff; 3056 total = lxon + lxoff; 3057 3058 hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3059 hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3060 hw_stats->gptc -= total; 3061 hw_stats->mptc -= total; 3062 hw_stats->ptc64 -= total; 3063 hw_stats->gotc -= total * ETHER_MIN_LEN; 3064 3065 hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3066 hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3067 hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3068 hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3069 hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3070 hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3071 hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3072 hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3073 hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3074 hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3075 hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3076 hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3077 hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3078 hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3079 hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3080 hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3081 /* Only read FCOE on 82599 */ 3082 if (hw->mac.type != ixgbe_mac_82598EB) { 3083 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3084 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3085 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3086 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3087 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3088 } 3089 3090 /* Flow Director Stats registers */ 3091 hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 3092 hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 3093 3094 /* MACsec Stats registers */ 3095 macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT); 3096 macsec_stats->out_pkts_encrypted += 3097 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE); 3098 macsec_stats->out_pkts_protected += 3099 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP); 3100 macsec_stats->out_octets_encrypted += 3101 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE); 3102 macsec_stats->out_octets_protected += 3103 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP); 3104 macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT); 3105 macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD); 3106 macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI); 3107 macsec_stats->in_pkts_unknownsci += 3108 IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI); 3109 macsec_stats->in_octets_decrypted += 3110 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD); 3111 macsec_stats->in_octets_validated += 3112 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV); 3113 macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH); 3114 macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY); 3115 macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE); 3116 for (i = 0; i < 2; i++) { 3117 macsec_stats->in_pkts_ok += 3118 IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i)); 3119 macsec_stats->in_pkts_invalid += 3120 IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i)); 3121 macsec_stats->in_pkts_notvalid += 3122 IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i)); 3123 } 3124 macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA); 3125 macsec_stats->in_pkts_notusingsa += 3126 IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA); 3127 } 3128 3129 /* 3130 * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c 3131 */ 3132 static int 3133 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3134 { 3135 struct ixgbe_hw *hw = 3136 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3137 struct ixgbe_hw_stats *hw_stats = 3138 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3139 struct ixgbe_macsec_stats *macsec_stats = 3140 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3141 dev->data->dev_private); 3142 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3143 unsigned i; 3144 3145 total_missed_rx = 0; 3146 total_qbrc = 0; 3147 total_qprc = 0; 3148 total_qprdc = 0; 3149 3150 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3151 &total_qbrc, &total_qprc, &total_qprdc); 3152 3153 if (stats == NULL) 3154 return -EINVAL; 3155 3156 /* Fill out the rte_eth_stats statistics structure */ 3157 stats->ipackets = total_qprc; 3158 stats->ibytes = total_qbrc; 3159 stats->opackets = hw_stats->gptc; 3160 stats->obytes = hw_stats->gotc; 3161 3162 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3163 stats->q_ipackets[i] = hw_stats->qprc[i]; 3164 stats->q_opackets[i] = hw_stats->qptc[i]; 3165 stats->q_ibytes[i] = hw_stats->qbrc[i]; 3166 stats->q_obytes[i] = hw_stats->qbtc[i]; 3167 stats->q_errors[i] = hw_stats->qprdc[i]; 3168 } 3169 3170 /* Rx Errors */ 3171 stats->imissed = total_missed_rx; 3172 stats->ierrors = hw_stats->crcerrs + 3173 hw_stats->mspdc + 3174 hw_stats->rlec + 3175 hw_stats->ruc + 3176 hw_stats->roc + 3177 hw_stats->illerrc + 3178 hw_stats->errbc + 3179 hw_stats->rfc + 3180 hw_stats->fccrc + 3181 hw_stats->fclast; 3182 3183 /* Tx Errors */ 3184 stats->oerrors = 0; 3185 return 0; 3186 } 3187 3188 static void 3189 ixgbe_dev_stats_reset(struct rte_eth_dev *dev) 3190 { 3191 struct ixgbe_hw_stats *stats = 3192 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3193 3194 /* HW registers are cleared on read */ 3195 ixgbe_dev_stats_get(dev, NULL); 3196 3197 /* Reset software totals */ 3198 memset(stats, 0, sizeof(*stats)); 3199 } 3200 3201 /* This function calculates the number of xstats based on the current config */ 3202 static unsigned 3203 ixgbe_xstats_calc_num(void) { 3204 return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS + 3205 (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) + 3206 (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES); 3207 } 3208 3209 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3210 struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int size) 3211 { 3212 const unsigned cnt_stats = ixgbe_xstats_calc_num(); 3213 unsigned stat, i, count; 3214 3215 if (xstats_names != NULL) { 3216 count = 0; 3217 3218 /* Note: limit >= cnt_stats checked upstream 3219 * in rte_eth_xstats_names() 3220 */ 3221 3222 /* Extended stats from ixgbe_hw_stats */ 3223 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3224 snprintf(xstats_names[count].name, 3225 sizeof(xstats_names[count].name), 3226 "%s", 3227 rte_ixgbe_stats_strings[i].name); 3228 count++; 3229 } 3230 3231 /* MACsec Stats */ 3232 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3233 snprintf(xstats_names[count].name, 3234 sizeof(xstats_names[count].name), 3235 "%s", 3236 rte_ixgbe_macsec_strings[i].name); 3237 count++; 3238 } 3239 3240 /* RX Priority Stats */ 3241 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3242 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3243 snprintf(xstats_names[count].name, 3244 sizeof(xstats_names[count].name), 3245 "rx_priority%u_%s", i, 3246 rte_ixgbe_rxq_strings[stat].name); 3247 count++; 3248 } 3249 } 3250 3251 /* TX Priority Stats */ 3252 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3253 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3254 snprintf(xstats_names[count].name, 3255 sizeof(xstats_names[count].name), 3256 "tx_priority%u_%s", i, 3257 rte_ixgbe_txq_strings[stat].name); 3258 count++; 3259 } 3260 } 3261 } 3262 return cnt_stats; 3263 } 3264 3265 static int ixgbe_dev_xstats_get_names_by_id( 3266 struct rte_eth_dev *dev, 3267 struct rte_eth_xstat_name *xstats_names, 3268 const uint64_t *ids, 3269 unsigned int limit) 3270 { 3271 if (!ids) { 3272 const unsigned int cnt_stats = ixgbe_xstats_calc_num(); 3273 unsigned int stat, i, count; 3274 3275 if (xstats_names != NULL) { 3276 count = 0; 3277 3278 /* Note: limit >= cnt_stats checked upstream 3279 * in rte_eth_xstats_names() 3280 */ 3281 3282 /* Extended stats from ixgbe_hw_stats */ 3283 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3284 snprintf(xstats_names[count].name, 3285 sizeof(xstats_names[count].name), 3286 "%s", 3287 rte_ixgbe_stats_strings[i].name); 3288 count++; 3289 } 3290 3291 /* MACsec Stats */ 3292 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3293 snprintf(xstats_names[count].name, 3294 sizeof(xstats_names[count].name), 3295 "%s", 3296 rte_ixgbe_macsec_strings[i].name); 3297 count++; 3298 } 3299 3300 /* RX Priority Stats */ 3301 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3302 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3303 snprintf(xstats_names[count].name, 3304 sizeof(xstats_names[count].name), 3305 "rx_priority%u_%s", i, 3306 rte_ixgbe_rxq_strings[stat].name); 3307 count++; 3308 } 3309 } 3310 3311 /* TX Priority Stats */ 3312 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3313 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3314 snprintf(xstats_names[count].name, 3315 sizeof(xstats_names[count].name), 3316 "tx_priority%u_%s", i, 3317 rte_ixgbe_txq_strings[stat].name); 3318 count++; 3319 } 3320 } 3321 } 3322 return cnt_stats; 3323 } 3324 3325 uint16_t i; 3326 uint16_t size = ixgbe_xstats_calc_num(); 3327 struct rte_eth_xstat_name xstats_names_copy[size]; 3328 3329 ixgbe_dev_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 3330 size); 3331 3332 for (i = 0; i < limit; i++) { 3333 if (ids[i] >= size) { 3334 PMD_INIT_LOG(ERR, "id value isn't valid"); 3335 return -1; 3336 } 3337 strcpy(xstats_names[i].name, 3338 xstats_names_copy[ids[i]].name); 3339 } 3340 return limit; 3341 } 3342 3343 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3344 struct rte_eth_xstat_name *xstats_names, unsigned limit) 3345 { 3346 unsigned i; 3347 3348 if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL) 3349 return -ENOMEM; 3350 3351 if (xstats_names != NULL) 3352 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) 3353 snprintf(xstats_names[i].name, 3354 sizeof(xstats_names[i].name), 3355 "%s", rte_ixgbevf_stats_strings[i].name); 3356 return IXGBEVF_NB_XSTATS; 3357 } 3358 3359 static int 3360 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3361 unsigned n) 3362 { 3363 struct ixgbe_hw *hw = 3364 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3365 struct ixgbe_hw_stats *hw_stats = 3366 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3367 struct ixgbe_macsec_stats *macsec_stats = 3368 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3369 dev->data->dev_private); 3370 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3371 unsigned i, stat, count = 0; 3372 3373 count = ixgbe_xstats_calc_num(); 3374 3375 if (n < count) 3376 return count; 3377 3378 total_missed_rx = 0; 3379 total_qbrc = 0; 3380 total_qprc = 0; 3381 total_qprdc = 0; 3382 3383 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3384 &total_qbrc, &total_qprc, &total_qprdc); 3385 3386 /* If this is a reset xstats is NULL, and we have cleared the 3387 * registers by reading them. 3388 */ 3389 if (!xstats) 3390 return 0; 3391 3392 /* Extended stats from ixgbe_hw_stats */ 3393 count = 0; 3394 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3395 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3396 rte_ixgbe_stats_strings[i].offset); 3397 xstats[count].id = count; 3398 count++; 3399 } 3400 3401 /* MACsec Stats */ 3402 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3403 xstats[count].value = *(uint64_t *)(((char *)macsec_stats) + 3404 rte_ixgbe_macsec_strings[i].offset); 3405 xstats[count].id = count; 3406 count++; 3407 } 3408 3409 /* RX Priority Stats */ 3410 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3411 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3412 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3413 rte_ixgbe_rxq_strings[stat].offset + 3414 (sizeof(uint64_t) * i)); 3415 xstats[count].id = count; 3416 count++; 3417 } 3418 } 3419 3420 /* TX Priority Stats */ 3421 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3422 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3423 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3424 rte_ixgbe_txq_strings[stat].offset + 3425 (sizeof(uint64_t) * i)); 3426 xstats[count].id = count; 3427 count++; 3428 } 3429 } 3430 return count; 3431 } 3432 3433 static int 3434 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 3435 uint64_t *values, unsigned int n) 3436 { 3437 if (!ids) { 3438 struct ixgbe_hw *hw = 3439 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3440 struct ixgbe_hw_stats *hw_stats = 3441 IXGBE_DEV_PRIVATE_TO_STATS( 3442 dev->data->dev_private); 3443 struct ixgbe_macsec_stats *macsec_stats = 3444 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3445 dev->data->dev_private); 3446 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3447 unsigned int i, stat, count = 0; 3448 3449 count = ixgbe_xstats_calc_num(); 3450 3451 if (!ids && n < count) 3452 return count; 3453 3454 total_missed_rx = 0; 3455 total_qbrc = 0; 3456 total_qprc = 0; 3457 total_qprdc = 0; 3458 3459 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, 3460 &total_missed_rx, &total_qbrc, &total_qprc, 3461 &total_qprdc); 3462 3463 /* If this is a reset xstats is NULL, and we have cleared the 3464 * registers by reading them. 3465 */ 3466 if (!ids && !values) 3467 return 0; 3468 3469 /* Extended stats from ixgbe_hw_stats */ 3470 count = 0; 3471 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3472 values[count] = *(uint64_t *)(((char *)hw_stats) + 3473 rte_ixgbe_stats_strings[i].offset); 3474 count++; 3475 } 3476 3477 /* MACsec Stats */ 3478 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3479 values[count] = *(uint64_t *)(((char *)macsec_stats) + 3480 rte_ixgbe_macsec_strings[i].offset); 3481 count++; 3482 } 3483 3484 /* RX Priority Stats */ 3485 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3486 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3487 values[count] = 3488 *(uint64_t *)(((char *)hw_stats) + 3489 rte_ixgbe_rxq_strings[stat].offset + 3490 (sizeof(uint64_t) * i)); 3491 count++; 3492 } 3493 } 3494 3495 /* TX Priority Stats */ 3496 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3497 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3498 values[count] = 3499 *(uint64_t *)(((char *)hw_stats) + 3500 rte_ixgbe_txq_strings[stat].offset + 3501 (sizeof(uint64_t) * i)); 3502 count++; 3503 } 3504 } 3505 return count; 3506 } 3507 3508 uint16_t i; 3509 uint16_t size = ixgbe_xstats_calc_num(); 3510 uint64_t values_copy[size]; 3511 3512 ixgbe_dev_xstats_get_by_id(dev, NULL, values_copy, size); 3513 3514 for (i = 0; i < n; i++) { 3515 if (ids[i] >= size) { 3516 PMD_INIT_LOG(ERR, "id value isn't valid"); 3517 return -1; 3518 } 3519 values[i] = values_copy[ids[i]]; 3520 } 3521 return n; 3522 } 3523 3524 static void 3525 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) 3526 { 3527 struct ixgbe_hw_stats *stats = 3528 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3529 struct ixgbe_macsec_stats *macsec_stats = 3530 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3531 dev->data->dev_private); 3532 3533 unsigned count = ixgbe_xstats_calc_num(); 3534 3535 /* HW registers are cleared on read */ 3536 ixgbe_dev_xstats_get(dev, NULL, count); 3537 3538 /* Reset software totals */ 3539 memset(stats, 0, sizeof(*stats)); 3540 memset(macsec_stats, 0, sizeof(*macsec_stats)); 3541 } 3542 3543 static void 3544 ixgbevf_update_stats(struct rte_eth_dev *dev) 3545 { 3546 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3547 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3548 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3549 3550 /* Good Rx packet, include VF loopback */ 3551 UPDATE_VF_STAT(IXGBE_VFGPRC, 3552 hw_stats->last_vfgprc, hw_stats->vfgprc); 3553 3554 /* Good Rx octets, include VF loopback */ 3555 UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3556 hw_stats->last_vfgorc, hw_stats->vfgorc); 3557 3558 /* Good Tx packet, include VF loopback */ 3559 UPDATE_VF_STAT(IXGBE_VFGPTC, 3560 hw_stats->last_vfgptc, hw_stats->vfgptc); 3561 3562 /* Good Tx octets, include VF loopback */ 3563 UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3564 hw_stats->last_vfgotc, hw_stats->vfgotc); 3565 3566 /* Rx Multicst Packet */ 3567 UPDATE_VF_STAT(IXGBE_VFMPRC, 3568 hw_stats->last_vfmprc, hw_stats->vfmprc); 3569 } 3570 3571 static int 3572 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3573 unsigned n) 3574 { 3575 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3576 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3577 unsigned i; 3578 3579 if (n < IXGBEVF_NB_XSTATS) 3580 return IXGBEVF_NB_XSTATS; 3581 3582 ixgbevf_update_stats(dev); 3583 3584 if (!xstats) 3585 return 0; 3586 3587 /* Extended stats */ 3588 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) { 3589 xstats[i].id = i; 3590 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 3591 rte_ixgbevf_stats_strings[i].offset); 3592 } 3593 3594 return IXGBEVF_NB_XSTATS; 3595 } 3596 3597 static int 3598 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3599 { 3600 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3601 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3602 3603 ixgbevf_update_stats(dev); 3604 3605 if (stats == NULL) 3606 return -EINVAL; 3607 3608 stats->ipackets = hw_stats->vfgprc; 3609 stats->ibytes = hw_stats->vfgorc; 3610 stats->opackets = hw_stats->vfgptc; 3611 stats->obytes = hw_stats->vfgotc; 3612 return 0; 3613 } 3614 3615 static void 3616 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev) 3617 { 3618 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3619 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3620 3621 /* Sync HW register to the last stats */ 3622 ixgbevf_dev_stats_get(dev, NULL); 3623 3624 /* reset HW current stats*/ 3625 hw_stats->vfgprc = 0; 3626 hw_stats->vfgorc = 0; 3627 hw_stats->vfgptc = 0; 3628 hw_stats->vfgotc = 0; 3629 } 3630 3631 static int 3632 ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 3633 { 3634 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3635 u16 eeprom_verh, eeprom_verl; 3636 u32 etrack_id; 3637 int ret; 3638 3639 ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); 3640 ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); 3641 3642 etrack_id = (eeprom_verh << 16) | eeprom_verl; 3643 ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); 3644 3645 ret += 1; /* add the size of '\0' */ 3646 if (fw_size < (u32)ret) 3647 return ret; 3648 else 3649 return 0; 3650 } 3651 3652 static void 3653 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 3654 { 3655 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3656 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3657 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3658 3659 dev_info->pci_dev = pci_dev; 3660 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3661 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3662 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3663 /* 3664 * When DCB/VT is off, maximum number of queues changes, 3665 * except for 82598EB, which remains constant. 3666 */ 3667 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 3668 hw->mac.type != ixgbe_mac_82598EB) 3669 dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES; 3670 } 3671 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */ 3672 dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */ 3673 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3674 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3675 dev_info->max_vfs = pci_dev->max_vfs; 3676 if (hw->mac.type == ixgbe_mac_82598EB) 3677 dev_info->max_vmdq_pools = ETH_16_POOLS; 3678 else 3679 dev_info->max_vmdq_pools = ETH_64_POOLS; 3680 dev_info->vmdq_queue_num = dev_info->max_rx_queues; 3681 dev_info->rx_offload_capa = 3682 DEV_RX_OFFLOAD_VLAN_STRIP | 3683 DEV_RX_OFFLOAD_IPV4_CKSUM | 3684 DEV_RX_OFFLOAD_UDP_CKSUM | 3685 DEV_RX_OFFLOAD_TCP_CKSUM; 3686 3687 /* 3688 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV 3689 * mode. 3690 */ 3691 if ((hw->mac.type == ixgbe_mac_82599EB || 3692 hw->mac.type == ixgbe_mac_X540) && 3693 !RTE_ETH_DEV_SRIOV(dev).active) 3694 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; 3695 3696 if (hw->mac.type == ixgbe_mac_82599EB || 3697 hw->mac.type == ixgbe_mac_X540) 3698 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_MACSEC_STRIP; 3699 3700 if (hw->mac.type == ixgbe_mac_X550 || 3701 hw->mac.type == ixgbe_mac_X550EM_x || 3702 hw->mac.type == ixgbe_mac_X550EM_a) 3703 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; 3704 3705 dev_info->tx_offload_capa = 3706 DEV_TX_OFFLOAD_VLAN_INSERT | 3707 DEV_TX_OFFLOAD_IPV4_CKSUM | 3708 DEV_TX_OFFLOAD_UDP_CKSUM | 3709 DEV_TX_OFFLOAD_TCP_CKSUM | 3710 DEV_TX_OFFLOAD_SCTP_CKSUM | 3711 DEV_TX_OFFLOAD_TCP_TSO; 3712 3713 if (hw->mac.type == ixgbe_mac_82599EB || 3714 hw->mac.type == ixgbe_mac_X540) 3715 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT; 3716 3717 if (hw->mac.type == ixgbe_mac_X550 || 3718 hw->mac.type == ixgbe_mac_X550EM_x || 3719 hw->mac.type == ixgbe_mac_X550EM_a) 3720 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; 3721 3722 #ifdef RTE_LIBRTE_SECURITY 3723 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_SECURITY; 3724 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY; 3725 #endif 3726 3727 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3728 .rx_thresh = { 3729 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3730 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3731 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3732 }, 3733 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3734 .rx_drop_en = 0, 3735 }; 3736 3737 dev_info->default_txconf = (struct rte_eth_txconf) { 3738 .tx_thresh = { 3739 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3740 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3741 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3742 }, 3743 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3744 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3745 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | 3746 ETH_TXQ_FLAGS_NOOFFLOADS, 3747 }; 3748 3749 dev_info->rx_desc_lim = rx_desc_lim; 3750 dev_info->tx_desc_lim = tx_desc_lim; 3751 3752 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3753 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3754 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3755 3756 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; 3757 if (hw->mac.type == ixgbe_mac_X540 || 3758 hw->mac.type == ixgbe_mac_X540_vf || 3759 hw->mac.type == ixgbe_mac_X550 || 3760 hw->mac.type == ixgbe_mac_X550_vf) { 3761 dev_info->speed_capa |= ETH_LINK_SPEED_100M; 3762 } 3763 if (hw->mac.type == ixgbe_mac_X550) { 3764 dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; 3765 dev_info->speed_capa |= ETH_LINK_SPEED_5G; 3766 } 3767 } 3768 3769 static const uint32_t * 3770 ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) 3771 { 3772 static const uint32_t ptypes[] = { 3773 /* For non-vec functions, 3774 * refers to ixgbe_rxd_pkt_info_to_pkt_type(); 3775 * for vec functions, 3776 * refers to _recv_raw_pkts_vec(). 3777 */ 3778 RTE_PTYPE_L2_ETHER, 3779 RTE_PTYPE_L3_IPV4, 3780 RTE_PTYPE_L3_IPV4_EXT, 3781 RTE_PTYPE_L3_IPV6, 3782 RTE_PTYPE_L3_IPV6_EXT, 3783 RTE_PTYPE_L4_SCTP, 3784 RTE_PTYPE_L4_TCP, 3785 RTE_PTYPE_L4_UDP, 3786 RTE_PTYPE_TUNNEL_IP, 3787 RTE_PTYPE_INNER_L3_IPV6, 3788 RTE_PTYPE_INNER_L3_IPV6_EXT, 3789 RTE_PTYPE_INNER_L4_TCP, 3790 RTE_PTYPE_INNER_L4_UDP, 3791 RTE_PTYPE_UNKNOWN 3792 }; 3793 3794 if (dev->rx_pkt_burst == ixgbe_recv_pkts || 3795 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc || 3796 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || 3797 dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) 3798 return ptypes; 3799 3800 #if defined(RTE_ARCH_X86) 3801 if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || 3802 dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) 3803 return ptypes; 3804 #endif 3805 return NULL; 3806 } 3807 3808 static void 3809 ixgbevf_dev_info_get(struct rte_eth_dev *dev, 3810 struct rte_eth_dev_info *dev_info) 3811 { 3812 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3813 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3814 3815 dev_info->pci_dev = pci_dev; 3816 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3817 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3818 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */ 3819 dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */ 3820 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3821 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3822 dev_info->max_vfs = pci_dev->max_vfs; 3823 if (hw->mac.type == ixgbe_mac_82598EB) 3824 dev_info->max_vmdq_pools = ETH_16_POOLS; 3825 else 3826 dev_info->max_vmdq_pools = ETH_64_POOLS; 3827 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | 3828 DEV_RX_OFFLOAD_IPV4_CKSUM | 3829 DEV_RX_OFFLOAD_UDP_CKSUM | 3830 DEV_RX_OFFLOAD_TCP_CKSUM; 3831 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | 3832 DEV_TX_OFFLOAD_IPV4_CKSUM | 3833 DEV_TX_OFFLOAD_UDP_CKSUM | 3834 DEV_TX_OFFLOAD_TCP_CKSUM | 3835 DEV_TX_OFFLOAD_SCTP_CKSUM | 3836 DEV_TX_OFFLOAD_TCP_TSO; 3837 3838 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3839 .rx_thresh = { 3840 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3841 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3842 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3843 }, 3844 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3845 .rx_drop_en = 0, 3846 }; 3847 3848 dev_info->default_txconf = (struct rte_eth_txconf) { 3849 .tx_thresh = { 3850 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3851 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3852 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3853 }, 3854 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3855 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3856 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | 3857 ETH_TXQ_FLAGS_NOOFFLOADS, 3858 }; 3859 3860 dev_info->rx_desc_lim = rx_desc_lim; 3861 dev_info->tx_desc_lim = tx_desc_lim; 3862 } 3863 3864 static int 3865 ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, 3866 int *link_up, int wait_to_complete) 3867 { 3868 /** 3869 * for a quick link status checking, wait_to_compelet == 0, 3870 * skip PF link status checking 3871 */ 3872 bool no_pflink_check = wait_to_complete == 0; 3873 struct ixgbe_mbx_info *mbx = &hw->mbx; 3874 struct ixgbe_mac_info *mac = &hw->mac; 3875 uint32_t links_reg, in_msg; 3876 int ret_val = 0; 3877 3878 /* If we were hit with a reset drop the link */ 3879 if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) 3880 mac->get_link_status = true; 3881 3882 if (!mac->get_link_status) 3883 goto out; 3884 3885 /* if link status is down no point in checking to see if pf is up */ 3886 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 3887 if (!(links_reg & IXGBE_LINKS_UP)) 3888 goto out; 3889 3890 /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs 3891 * before the link status is correct 3892 */ 3893 if (mac->type == ixgbe_mac_82599_vf && wait_to_complete) { 3894 int i; 3895 3896 for (i = 0; i < 5; i++) { 3897 rte_delay_us(100); 3898 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 3899 3900 if (!(links_reg & IXGBE_LINKS_UP)) 3901 goto out; 3902 } 3903 } 3904 3905 switch (links_reg & IXGBE_LINKS_SPEED_82599) { 3906 case IXGBE_LINKS_SPEED_10G_82599: 3907 *speed = IXGBE_LINK_SPEED_10GB_FULL; 3908 if (hw->mac.type >= ixgbe_mac_X550) { 3909 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 3910 *speed = IXGBE_LINK_SPEED_2_5GB_FULL; 3911 } 3912 break; 3913 case IXGBE_LINKS_SPEED_1G_82599: 3914 *speed = IXGBE_LINK_SPEED_1GB_FULL; 3915 break; 3916 case IXGBE_LINKS_SPEED_100_82599: 3917 *speed = IXGBE_LINK_SPEED_100_FULL; 3918 if (hw->mac.type == ixgbe_mac_X550) { 3919 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 3920 *speed = IXGBE_LINK_SPEED_5GB_FULL; 3921 } 3922 break; 3923 case IXGBE_LINKS_SPEED_10_X550EM_A: 3924 *speed = IXGBE_LINK_SPEED_UNKNOWN; 3925 /* Since Reserved in older MAC's */ 3926 if (hw->mac.type >= ixgbe_mac_X550) 3927 *speed = IXGBE_LINK_SPEED_10_FULL; 3928 break; 3929 default: 3930 *speed = IXGBE_LINK_SPEED_UNKNOWN; 3931 } 3932 3933 if (no_pflink_check) { 3934 if (*speed == IXGBE_LINK_SPEED_UNKNOWN) 3935 mac->get_link_status = true; 3936 else 3937 mac->get_link_status = false; 3938 3939 goto out; 3940 } 3941 /* if the read failed it could just be a mailbox collision, best wait 3942 * until we are called again and don't report an error 3943 */ 3944 if (mbx->ops.read(hw, &in_msg, 1, 0)) 3945 goto out; 3946 3947 if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { 3948 /* msg is not CTS and is NACK we must have lost CTS status */ 3949 if (in_msg & IXGBE_VT_MSGTYPE_NACK) 3950 ret_val = -1; 3951 goto out; 3952 } 3953 3954 /* the pf is talking, if we timed out in the past we reinit */ 3955 if (!mbx->timeout) { 3956 ret_val = -1; 3957 goto out; 3958 } 3959 3960 /* if we passed all the tests above then the link is up and we no 3961 * longer need to check for link 3962 */ 3963 mac->get_link_status = false; 3964 3965 out: 3966 *link_up = !mac->get_link_status; 3967 return ret_val; 3968 } 3969 3970 /* return 0 means link status changed, -1 means not changed */ 3971 static int 3972 ixgbe_dev_link_update_share(struct rte_eth_dev *dev, 3973 int wait_to_complete, int vf) 3974 { 3975 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3976 struct rte_eth_link link, old; 3977 ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN; 3978 struct ixgbe_interrupt *intr = 3979 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 3980 int link_up; 3981 int diag; 3982 u32 speed = 0; 3983 int wait = 1; 3984 bool autoneg = false; 3985 3986 link.link_status = ETH_LINK_DOWN; 3987 link.link_speed = 0; 3988 link.link_duplex = ETH_LINK_HALF_DUPLEX; 3989 link.link_autoneg = ETH_LINK_AUTONEG; 3990 memset(&old, 0, sizeof(old)); 3991 rte_ixgbe_dev_atomic_read_link_status(dev, &old); 3992 3993 hw->mac.get_link_status = true; 3994 3995 if ((intr->flags & IXGBE_FLAG_NEED_LINK_CONFIG) && 3996 ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 3997 speed = hw->phy.autoneg_advertised; 3998 if (!speed) 3999 ixgbe_get_link_capabilities(hw, &speed, &autoneg); 4000 ixgbe_setup_link(hw, speed, true); 4001 } 4002 4003 /* check if it needs to wait to complete, if lsc interrupt is enabled */ 4004 if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) 4005 wait = 0; 4006 4007 if (vf) 4008 diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); 4009 else 4010 diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); 4011 4012 if (diag != 0) { 4013 link.link_speed = ETH_SPEED_NUM_100M; 4014 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4015 rte_ixgbe_dev_atomic_write_link_status(dev, &link); 4016 if (link.link_status == old.link_status) 4017 return -1; 4018 return 0; 4019 } 4020 4021 if (link_up == 0) { 4022 rte_ixgbe_dev_atomic_write_link_status(dev, &link); 4023 intr->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 4024 if (link.link_status == old.link_status) 4025 return -1; 4026 return 0; 4027 } 4028 intr->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4029 link.link_status = ETH_LINK_UP; 4030 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4031 4032 switch (link_speed) { 4033 default: 4034 case IXGBE_LINK_SPEED_UNKNOWN: 4035 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4036 link.link_speed = ETH_SPEED_NUM_100M; 4037 break; 4038 4039 case IXGBE_LINK_SPEED_100_FULL: 4040 link.link_speed = ETH_SPEED_NUM_100M; 4041 break; 4042 4043 case IXGBE_LINK_SPEED_1GB_FULL: 4044 link.link_speed = ETH_SPEED_NUM_1G; 4045 break; 4046 4047 case IXGBE_LINK_SPEED_2_5GB_FULL: 4048 link.link_speed = ETH_SPEED_NUM_2_5G; 4049 break; 4050 4051 case IXGBE_LINK_SPEED_5GB_FULL: 4052 link.link_speed = ETH_SPEED_NUM_5G; 4053 break; 4054 4055 case IXGBE_LINK_SPEED_10GB_FULL: 4056 link.link_speed = ETH_SPEED_NUM_10G; 4057 break; 4058 } 4059 rte_ixgbe_dev_atomic_write_link_status(dev, &link); 4060 4061 if (link.link_status == old.link_status) 4062 return -1; 4063 4064 return 0; 4065 } 4066 4067 static int 4068 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4069 { 4070 return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); 4071 } 4072 4073 static int 4074 ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4075 { 4076 return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); 4077 } 4078 4079 static void 4080 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) 4081 { 4082 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4083 uint32_t fctrl; 4084 4085 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4086 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4087 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4088 } 4089 4090 static void 4091 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev) 4092 { 4093 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4094 uint32_t fctrl; 4095 4096 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4097 fctrl &= (~IXGBE_FCTRL_UPE); 4098 if (dev->data->all_multicast == 1) 4099 fctrl |= IXGBE_FCTRL_MPE; 4100 else 4101 fctrl &= (~IXGBE_FCTRL_MPE); 4102 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4103 } 4104 4105 static void 4106 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev) 4107 { 4108 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4109 uint32_t fctrl; 4110 4111 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4112 fctrl |= IXGBE_FCTRL_MPE; 4113 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4114 } 4115 4116 static void 4117 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) 4118 { 4119 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4120 uint32_t fctrl; 4121 4122 if (dev->data->promiscuous == 1) 4123 return; /* must remain in all_multicast mode */ 4124 4125 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4126 fctrl &= (~IXGBE_FCTRL_MPE); 4127 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4128 } 4129 4130 /** 4131 * It clears the interrupt causes and enables the interrupt. 4132 * It will be called once only during nic initialized. 4133 * 4134 * @param dev 4135 * Pointer to struct rte_eth_dev. 4136 * @param on 4137 * Enable or Disable. 4138 * 4139 * @return 4140 * - On success, zero. 4141 * - On failure, a negative value. 4142 */ 4143 static int 4144 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 4145 { 4146 struct ixgbe_interrupt *intr = 4147 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4148 4149 ixgbe_dev_link_status_print(dev); 4150 if (on) 4151 intr->mask |= IXGBE_EICR_LSC; 4152 else 4153 intr->mask &= ~IXGBE_EICR_LSC; 4154 4155 return 0; 4156 } 4157 4158 /** 4159 * It clears the interrupt causes and enables the interrupt. 4160 * It will be called once only during nic initialized. 4161 * 4162 * @param dev 4163 * Pointer to struct rte_eth_dev. 4164 * 4165 * @return 4166 * - On success, zero. 4167 * - On failure, a negative value. 4168 */ 4169 static int 4170 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) 4171 { 4172 struct ixgbe_interrupt *intr = 4173 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4174 4175 intr->mask |= IXGBE_EICR_RTX_QUEUE; 4176 4177 return 0; 4178 } 4179 4180 /** 4181 * It clears the interrupt causes and enables the interrupt. 4182 * It will be called once only during nic initialized. 4183 * 4184 * @param dev 4185 * Pointer to struct rte_eth_dev. 4186 * 4187 * @return 4188 * - On success, zero. 4189 * - On failure, a negative value. 4190 */ 4191 static int 4192 ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev) 4193 { 4194 struct ixgbe_interrupt *intr = 4195 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4196 4197 intr->mask |= IXGBE_EICR_LINKSEC; 4198 4199 return 0; 4200 } 4201 4202 /* 4203 * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update. 4204 * 4205 * @param dev 4206 * Pointer to struct rte_eth_dev. 4207 * 4208 * @return 4209 * - On success, zero. 4210 * - On failure, a negative value. 4211 */ 4212 static int 4213 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) 4214 { 4215 uint32_t eicr; 4216 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4217 struct ixgbe_interrupt *intr = 4218 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4219 4220 /* clear all cause mask */ 4221 ixgbe_disable_intr(hw); 4222 4223 /* read-on-clear nic registers here */ 4224 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4225 PMD_DRV_LOG(DEBUG, "eicr %x", eicr); 4226 4227 intr->flags = 0; 4228 4229 /* set flag for async link update */ 4230 if (eicr & IXGBE_EICR_LSC) 4231 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4232 4233 if (eicr & IXGBE_EICR_MAILBOX) 4234 intr->flags |= IXGBE_FLAG_MAILBOX; 4235 4236 if (eicr & IXGBE_EICR_LINKSEC) 4237 intr->flags |= IXGBE_FLAG_MACSEC; 4238 4239 if (hw->mac.type == ixgbe_mac_X550EM_x && 4240 hw->phy.type == ixgbe_phy_x550em_ext_t && 4241 (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x)) 4242 intr->flags |= IXGBE_FLAG_PHY_INTERRUPT; 4243 4244 return 0; 4245 } 4246 4247 /** 4248 * It gets and then prints the link status. 4249 * 4250 * @param dev 4251 * Pointer to struct rte_eth_dev. 4252 * 4253 * @return 4254 * - On success, zero. 4255 * - On failure, a negative value. 4256 */ 4257 static void 4258 ixgbe_dev_link_status_print(struct rte_eth_dev *dev) 4259 { 4260 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4261 struct rte_eth_link link; 4262 4263 memset(&link, 0, sizeof(link)); 4264 rte_ixgbe_dev_atomic_read_link_status(dev, &link); 4265 if (link.link_status) { 4266 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 4267 (int)(dev->data->port_id), 4268 (unsigned)link.link_speed, 4269 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 4270 "full-duplex" : "half-duplex"); 4271 } else { 4272 PMD_INIT_LOG(INFO, " Port %d: Link Down", 4273 (int)(dev->data->port_id)); 4274 } 4275 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 4276 pci_dev->addr.domain, 4277 pci_dev->addr.bus, 4278 pci_dev->addr.devid, 4279 pci_dev->addr.function); 4280 } 4281 4282 /* 4283 * It executes link_update after knowing an interrupt occurred. 4284 * 4285 * @param dev 4286 * Pointer to struct rte_eth_dev. 4287 * 4288 * @return 4289 * - On success, zero. 4290 * - On failure, a negative value. 4291 */ 4292 static int 4293 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev, 4294 struct rte_intr_handle *intr_handle) 4295 { 4296 struct ixgbe_interrupt *intr = 4297 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4298 int64_t timeout; 4299 struct rte_eth_link link; 4300 struct ixgbe_hw *hw = 4301 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4302 4303 PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags); 4304 4305 if (intr->flags & IXGBE_FLAG_MAILBOX) { 4306 ixgbe_pf_mbx_process(dev); 4307 intr->flags &= ~IXGBE_FLAG_MAILBOX; 4308 } 4309 4310 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4311 ixgbe_handle_lasi(hw); 4312 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4313 } 4314 4315 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4316 /* get the link status before link update, for predicting later */ 4317 memset(&link, 0, sizeof(link)); 4318 rte_ixgbe_dev_atomic_read_link_status(dev, &link); 4319 4320 ixgbe_dev_link_update(dev, 0); 4321 4322 /* likely to up */ 4323 if (!link.link_status) 4324 /* handle it 1 sec later, wait it being stable */ 4325 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT; 4326 /* likely to down */ 4327 else 4328 /* handle it 4 sec later, wait it being stable */ 4329 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; 4330 4331 ixgbe_dev_link_status_print(dev); 4332 if (rte_eal_alarm_set(timeout * 1000, 4333 ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) 4334 PMD_DRV_LOG(ERR, "Error setting alarm"); 4335 else { 4336 /* remember original mask */ 4337 intr->mask_original = intr->mask; 4338 /* only disable lsc interrupt */ 4339 intr->mask &= ~IXGBE_EIMS_LSC; 4340 } 4341 } 4342 4343 PMD_DRV_LOG(DEBUG, "enable intr immediately"); 4344 ixgbe_enable_intr(dev); 4345 rte_intr_enable(intr_handle); 4346 4347 return 0; 4348 } 4349 4350 /** 4351 * Interrupt handler which shall be registered for alarm callback for delayed 4352 * handling specific interrupt to wait for the stable nic state. As the 4353 * NIC interrupt state is not stable for ixgbe after link is just down, 4354 * it needs to wait 4 seconds to get the stable status. 4355 * 4356 * @param handle 4357 * Pointer to interrupt handle. 4358 * @param param 4359 * The address of parameter (struct rte_eth_dev *) regsitered before. 4360 * 4361 * @return 4362 * void 4363 */ 4364 static void 4365 ixgbe_dev_interrupt_delayed_handler(void *param) 4366 { 4367 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4368 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4369 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 4370 struct ixgbe_interrupt *intr = 4371 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4372 struct ixgbe_hw *hw = 4373 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4374 uint32_t eicr; 4375 4376 ixgbe_disable_intr(hw); 4377 4378 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4379 if (eicr & IXGBE_EICR_MAILBOX) 4380 ixgbe_pf_mbx_process(dev); 4381 4382 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4383 ixgbe_handle_lasi(hw); 4384 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4385 } 4386 4387 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4388 ixgbe_dev_link_update(dev, 0); 4389 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4390 ixgbe_dev_link_status_print(dev); 4391 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 4392 NULL, NULL); 4393 } 4394 4395 if (intr->flags & IXGBE_FLAG_MACSEC) { 4396 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, 4397 NULL, NULL); 4398 intr->flags &= ~IXGBE_FLAG_MACSEC; 4399 } 4400 4401 /* restore original mask */ 4402 intr->mask = intr->mask_original; 4403 intr->mask_original = 0; 4404 4405 PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr); 4406 ixgbe_enable_intr(dev); 4407 rte_intr_enable(intr_handle); 4408 } 4409 4410 /** 4411 * Interrupt handler triggered by NIC for handling 4412 * specific interrupt. 4413 * 4414 * @param handle 4415 * Pointer to interrupt handle. 4416 * @param param 4417 * The address of parameter (struct rte_eth_dev *) regsitered before. 4418 * 4419 * @return 4420 * void 4421 */ 4422 static void 4423 ixgbe_dev_interrupt_handler(void *param) 4424 { 4425 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4426 4427 ixgbe_dev_interrupt_get_status(dev); 4428 ixgbe_dev_interrupt_action(dev, dev->intr_handle); 4429 } 4430 4431 static int 4432 ixgbe_dev_led_on(struct rte_eth_dev *dev) 4433 { 4434 struct ixgbe_hw *hw; 4435 4436 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4437 return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4438 } 4439 4440 static int 4441 ixgbe_dev_led_off(struct rte_eth_dev *dev) 4442 { 4443 struct ixgbe_hw *hw; 4444 4445 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4446 return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4447 } 4448 4449 static int 4450 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4451 { 4452 struct ixgbe_hw *hw; 4453 uint32_t mflcn_reg; 4454 uint32_t fccfg_reg; 4455 int rx_pause; 4456 int tx_pause; 4457 4458 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4459 4460 fc_conf->pause_time = hw->fc.pause_time; 4461 fc_conf->high_water = hw->fc.high_water[0]; 4462 fc_conf->low_water = hw->fc.low_water[0]; 4463 fc_conf->send_xon = hw->fc.send_xon; 4464 fc_conf->autoneg = !hw->fc.disable_fc_autoneg; 4465 4466 /* 4467 * Return rx_pause status according to actual setting of 4468 * MFLCN register. 4469 */ 4470 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4471 if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE)) 4472 rx_pause = 1; 4473 else 4474 rx_pause = 0; 4475 4476 /* 4477 * Return tx_pause status according to actual setting of 4478 * FCCFG register. 4479 */ 4480 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4481 if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY)) 4482 tx_pause = 1; 4483 else 4484 tx_pause = 0; 4485 4486 if (rx_pause && tx_pause) 4487 fc_conf->mode = RTE_FC_FULL; 4488 else if (rx_pause) 4489 fc_conf->mode = RTE_FC_RX_PAUSE; 4490 else if (tx_pause) 4491 fc_conf->mode = RTE_FC_TX_PAUSE; 4492 else 4493 fc_conf->mode = RTE_FC_NONE; 4494 4495 return 0; 4496 } 4497 4498 static int 4499 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4500 { 4501 struct ixgbe_hw *hw; 4502 int err; 4503 uint32_t rx_buf_size; 4504 uint32_t max_high_water; 4505 uint32_t mflcn; 4506 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4507 ixgbe_fc_none, 4508 ixgbe_fc_rx_pause, 4509 ixgbe_fc_tx_pause, 4510 ixgbe_fc_full 4511 }; 4512 4513 PMD_INIT_FUNC_TRACE(); 4514 4515 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4516 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)); 4517 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4518 4519 /* 4520 * At least reserve one Ethernet frame for watermark 4521 * high_water/low_water in kilo bytes for ixgbe 4522 */ 4523 max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4524 if ((fc_conf->high_water > max_high_water) || 4525 (fc_conf->high_water < fc_conf->low_water)) { 4526 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4527 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4528 return -EINVAL; 4529 } 4530 4531 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode]; 4532 hw->fc.pause_time = fc_conf->pause_time; 4533 hw->fc.high_water[0] = fc_conf->high_water; 4534 hw->fc.low_water[0] = fc_conf->low_water; 4535 hw->fc.send_xon = fc_conf->send_xon; 4536 hw->fc.disable_fc_autoneg = !fc_conf->autoneg; 4537 4538 err = ixgbe_fc_enable(hw); 4539 4540 /* Not negotiated is not an error case */ 4541 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) { 4542 4543 /* check if we want to forward MAC frames - driver doesn't have native 4544 * capability to do that, so we'll write the registers ourselves */ 4545 4546 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4547 4548 /* set or clear MFLCN.PMCF bit depending on configuration */ 4549 if (fc_conf->mac_ctrl_frame_fwd != 0) 4550 mflcn |= IXGBE_MFLCN_PMCF; 4551 else 4552 mflcn &= ~IXGBE_MFLCN_PMCF; 4553 4554 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn); 4555 IXGBE_WRITE_FLUSH(hw); 4556 4557 return 0; 4558 } 4559 4560 PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err); 4561 return -EIO; 4562 } 4563 4564 /** 4565 * ixgbe_pfc_enable_generic - Enable flow control 4566 * @hw: pointer to hardware structure 4567 * @tc_num: traffic class number 4568 * Enable flow control according to the current settings. 4569 */ 4570 static int 4571 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num) 4572 { 4573 int ret_val = 0; 4574 uint32_t mflcn_reg, fccfg_reg; 4575 uint32_t reg; 4576 uint32_t fcrtl, fcrth; 4577 uint8_t i; 4578 uint8_t nb_rx_en; 4579 4580 /* Validate the water mark configuration */ 4581 if (!hw->fc.pause_time) { 4582 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4583 goto out; 4584 } 4585 4586 /* Low water mark of zero causes XOFF floods */ 4587 if (hw->fc.current_mode & ixgbe_fc_tx_pause) { 4588 /* High/Low water can not be 0 */ 4589 if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) { 4590 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4591 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4592 goto out; 4593 } 4594 4595 if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) { 4596 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4597 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4598 goto out; 4599 } 4600 } 4601 /* Negotiate the fc mode to use */ 4602 ixgbe_fc_autoneg(hw); 4603 4604 /* Disable any previous flow control settings */ 4605 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4606 mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE); 4607 4608 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4609 fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY); 4610 4611 switch (hw->fc.current_mode) { 4612 case ixgbe_fc_none: 4613 /* 4614 * If the count of enabled RX Priority Flow control >1, 4615 * and the TX pause can not be disabled 4616 */ 4617 nb_rx_en = 0; 4618 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4619 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4620 if (reg & IXGBE_FCRTH_FCEN) 4621 nb_rx_en++; 4622 } 4623 if (nb_rx_en > 1) 4624 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4625 break; 4626 case ixgbe_fc_rx_pause: 4627 /* 4628 * Rx Flow control is enabled and Tx Flow control is 4629 * disabled by software override. Since there really 4630 * isn't a way to advertise that we are capable of RX 4631 * Pause ONLY, we will advertise that we support both 4632 * symmetric and asymmetric Rx PAUSE. Later, we will 4633 * disable the adapter's ability to send PAUSE frames. 4634 */ 4635 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4636 /* 4637 * If the count of enabled RX Priority Flow control >1, 4638 * and the TX pause can not be disabled 4639 */ 4640 nb_rx_en = 0; 4641 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4642 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4643 if (reg & IXGBE_FCRTH_FCEN) 4644 nb_rx_en++; 4645 } 4646 if (nb_rx_en > 1) 4647 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4648 break; 4649 case ixgbe_fc_tx_pause: 4650 /* 4651 * Tx Flow control is enabled, and Rx Flow control is 4652 * disabled by software override. 4653 */ 4654 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4655 break; 4656 case ixgbe_fc_full: 4657 /* Flow control (both Rx and Tx) is enabled by SW override. */ 4658 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4659 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4660 break; 4661 default: 4662 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly"); 4663 ret_val = IXGBE_ERR_CONFIG; 4664 goto out; 4665 } 4666 4667 /* Set 802.3x based flow control settings. */ 4668 mflcn_reg |= IXGBE_MFLCN_DPF; 4669 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg); 4670 IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg); 4671 4672 /* Set up and enable Rx high/low water mark thresholds, enable XON. */ 4673 if ((hw->fc.current_mode & ixgbe_fc_tx_pause) && 4674 hw->fc.high_water[tc_num]) { 4675 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE; 4676 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl); 4677 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN; 4678 } else { 4679 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0); 4680 /* 4681 * In order to prevent Tx hangs when the internal Tx 4682 * switch is enabled we must set the high water mark 4683 * to the maximum FCRTH value. This allows the Tx 4684 * switch to function even under heavy Rx workloads. 4685 */ 4686 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32; 4687 } 4688 IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth); 4689 4690 /* Configure pause time (2 TCs per register) */ 4691 reg = hw->fc.pause_time * 0x00010001; 4692 for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++) 4693 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg); 4694 4695 /* Configure flow control refresh threshold value */ 4696 IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2); 4697 4698 out: 4699 return ret_val; 4700 } 4701 4702 static int 4703 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num) 4704 { 4705 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4706 int32_t ret_val = IXGBE_NOT_IMPLEMENTED; 4707 4708 if (hw->mac.type != ixgbe_mac_82598EB) { 4709 ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num); 4710 } 4711 return ret_val; 4712 } 4713 4714 static int 4715 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf) 4716 { 4717 int err; 4718 uint32_t rx_buf_size; 4719 uint32_t max_high_water; 4720 uint8_t tc_num; 4721 uint8_t map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 }; 4722 struct ixgbe_hw *hw = 4723 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4724 struct ixgbe_dcb_config *dcb_config = 4725 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4726 4727 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4728 ixgbe_fc_none, 4729 ixgbe_fc_rx_pause, 4730 ixgbe_fc_tx_pause, 4731 ixgbe_fc_full 4732 }; 4733 4734 PMD_INIT_FUNC_TRACE(); 4735 4736 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4737 tc_num = map[pfc_conf->priority]; 4738 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)); 4739 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4740 /* 4741 * At least reserve one Ethernet frame for watermark 4742 * high_water/low_water in kilo bytes for ixgbe 4743 */ 4744 max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4745 if ((pfc_conf->fc.high_water > max_high_water) || 4746 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) { 4747 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4748 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4749 return -EINVAL; 4750 } 4751 4752 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode]; 4753 hw->fc.pause_time = pfc_conf->fc.pause_time; 4754 hw->fc.send_xon = pfc_conf->fc.send_xon; 4755 hw->fc.low_water[tc_num] = pfc_conf->fc.low_water; 4756 hw->fc.high_water[tc_num] = pfc_conf->fc.high_water; 4757 4758 err = ixgbe_dcb_pfc_enable(dev, tc_num); 4759 4760 /* Not negotiated is not an error case */ 4761 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) 4762 return 0; 4763 4764 PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err); 4765 return -EIO; 4766 } 4767 4768 static int 4769 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 4770 struct rte_eth_rss_reta_entry64 *reta_conf, 4771 uint16_t reta_size) 4772 { 4773 uint16_t i, sp_reta_size; 4774 uint8_t j, mask; 4775 uint32_t reta, r; 4776 uint16_t idx, shift; 4777 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4778 uint32_t reta_reg; 4779 4780 PMD_INIT_FUNC_TRACE(); 4781 4782 if (!ixgbe_rss_update_sp(hw->mac.type)) { 4783 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this " 4784 "NIC."); 4785 return -ENOTSUP; 4786 } 4787 4788 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 4789 if (reta_size != sp_reta_size) { 4790 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 4791 "(%d) doesn't match the number hardware can supported " 4792 "(%d)", reta_size, sp_reta_size); 4793 return -EINVAL; 4794 } 4795 4796 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 4797 idx = i / RTE_RETA_GROUP_SIZE; 4798 shift = i % RTE_RETA_GROUP_SIZE; 4799 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 4800 IXGBE_4_BIT_MASK); 4801 if (!mask) 4802 continue; 4803 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 4804 if (mask == IXGBE_4_BIT_MASK) 4805 r = 0; 4806 else 4807 r = IXGBE_READ_REG(hw, reta_reg); 4808 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) { 4809 if (mask & (0x1 << j)) 4810 reta |= reta_conf[idx].reta[shift + j] << 4811 (CHAR_BIT * j); 4812 else 4813 reta |= r & (IXGBE_8_BIT_MASK << 4814 (CHAR_BIT * j)); 4815 } 4816 IXGBE_WRITE_REG(hw, reta_reg, reta); 4817 } 4818 4819 return 0; 4820 } 4821 4822 static int 4823 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 4824 struct rte_eth_rss_reta_entry64 *reta_conf, 4825 uint16_t reta_size) 4826 { 4827 uint16_t i, sp_reta_size; 4828 uint8_t j, mask; 4829 uint32_t reta; 4830 uint16_t idx, shift; 4831 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4832 uint32_t reta_reg; 4833 4834 PMD_INIT_FUNC_TRACE(); 4835 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 4836 if (reta_size != sp_reta_size) { 4837 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 4838 "(%d) doesn't match the number hardware can supported " 4839 "(%d)", reta_size, sp_reta_size); 4840 return -EINVAL; 4841 } 4842 4843 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 4844 idx = i / RTE_RETA_GROUP_SIZE; 4845 shift = i % RTE_RETA_GROUP_SIZE; 4846 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 4847 IXGBE_4_BIT_MASK); 4848 if (!mask) 4849 continue; 4850 4851 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 4852 reta = IXGBE_READ_REG(hw, reta_reg); 4853 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) { 4854 if (mask & (0x1 << j)) 4855 reta_conf[idx].reta[shift + j] = 4856 ((reta >> (CHAR_BIT * j)) & 4857 IXGBE_8_BIT_MASK); 4858 } 4859 } 4860 4861 return 0; 4862 } 4863 4864 static int 4865 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 4866 uint32_t index, uint32_t pool) 4867 { 4868 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4869 uint32_t enable_addr = 1; 4870 4871 return ixgbe_set_rar(hw, index, mac_addr->addr_bytes, 4872 pool, enable_addr); 4873 } 4874 4875 static void 4876 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) 4877 { 4878 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4879 4880 ixgbe_clear_rar(hw, index); 4881 } 4882 4883 static void 4884 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) 4885 { 4886 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4887 4888 ixgbe_remove_rar(dev, 0); 4889 4890 ixgbe_add_rar(dev, addr, 0, pci_dev->max_vfs); 4891 } 4892 4893 static bool 4894 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) 4895 { 4896 if (strcmp(dev->device->driver->name, drv->driver.name)) 4897 return false; 4898 4899 return true; 4900 } 4901 4902 bool 4903 is_ixgbe_supported(struct rte_eth_dev *dev) 4904 { 4905 return is_device_supported(dev, &rte_ixgbe_pmd); 4906 } 4907 4908 static int 4909 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 4910 { 4911 uint32_t hlreg0; 4912 uint32_t maxfrs; 4913 struct ixgbe_hw *hw; 4914 struct rte_eth_dev_info dev_info; 4915 uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4916 struct rte_eth_dev_data *dev_data = dev->data; 4917 4918 ixgbe_dev_info_get(dev, &dev_info); 4919 4920 /* check that mtu is within the allowed range */ 4921 if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) 4922 return -EINVAL; 4923 4924 /* If device is started, refuse mtu that requires the support of 4925 * scattered packets when this feature has not been enabled before. 4926 */ 4927 if (dev_data->dev_started && !dev_data->scattered_rx && 4928 (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > 4929 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 4930 PMD_INIT_LOG(ERR, "Stop port first."); 4931 return -EINVAL; 4932 } 4933 4934 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4935 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4936 4937 /* switch to jumbo mode if needed */ 4938 if (frame_size > ETHER_MAX_LEN) { 4939 dev->data->dev_conf.rxmode.jumbo_frame = 1; 4940 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 4941 } else { 4942 dev->data->dev_conf.rxmode.jumbo_frame = 0; 4943 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 4944 } 4945 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 4946 4947 /* update max frame size */ 4948 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 4949 4950 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 4951 maxfrs &= 0x0000FFFF; 4952 maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16); 4953 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 4954 4955 return 0; 4956 } 4957 4958 /* 4959 * Virtual Function operations 4960 */ 4961 static void 4962 ixgbevf_intr_disable(struct ixgbe_hw *hw) 4963 { 4964 PMD_INIT_FUNC_TRACE(); 4965 4966 /* Clear interrupt mask to stop from interrupts being generated */ 4967 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK); 4968 4969 IXGBE_WRITE_FLUSH(hw); 4970 } 4971 4972 static void 4973 ixgbevf_intr_enable(struct ixgbe_hw *hw) 4974 { 4975 PMD_INIT_FUNC_TRACE(); 4976 4977 /* VF enable interrupt autoclean */ 4978 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK); 4979 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK); 4980 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK); 4981 4982 IXGBE_WRITE_FLUSH(hw); 4983 } 4984 4985 static int 4986 ixgbevf_dev_configure(struct rte_eth_dev *dev) 4987 { 4988 struct rte_eth_conf *conf = &dev->data->dev_conf; 4989 struct ixgbe_adapter *adapter = 4990 (struct ixgbe_adapter *)dev->data->dev_private; 4991 4992 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 4993 dev->data->port_id); 4994 4995 /* 4996 * VF has no ability to enable/disable HW CRC 4997 * Keep the persistent behavior the same as Host PF 4998 */ 4999 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC 5000 if (!conf->rxmode.hw_strip_crc) { 5001 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 5002 conf->rxmode.hw_strip_crc = 1; 5003 } 5004 #else 5005 if (conf->rxmode.hw_strip_crc) { 5006 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 5007 conf->rxmode.hw_strip_crc = 0; 5008 } 5009 #endif 5010 5011 /* 5012 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 5013 * allocation or vector Rx preconditions we will reset it. 5014 */ 5015 adapter->rx_bulk_alloc_allowed = true; 5016 adapter->rx_vec_allowed = true; 5017 5018 return 0; 5019 } 5020 5021 static int 5022 ixgbevf_dev_start(struct rte_eth_dev *dev) 5023 { 5024 struct ixgbe_hw *hw = 5025 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5026 uint32_t intr_vector = 0; 5027 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5028 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5029 5030 int err, mask = 0; 5031 5032 PMD_INIT_FUNC_TRACE(); 5033 5034 err = hw->mac.ops.reset_hw(hw); 5035 if (err) { 5036 PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); 5037 return err; 5038 } 5039 hw->mac.get_link_status = true; 5040 5041 /* negotiate mailbox API version to use with the PF. */ 5042 ixgbevf_negotiate_api(hw); 5043 5044 ixgbevf_dev_tx_init(dev); 5045 5046 /* This can fail when allocating mbufs for descriptor rings */ 5047 err = ixgbevf_dev_rx_init(dev); 5048 if (err) { 5049 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err); 5050 ixgbe_dev_clear_queues(dev); 5051 return err; 5052 } 5053 5054 /* Set vfta */ 5055 ixgbevf_set_vfta_all(dev, 1); 5056 5057 /* Set HW strip */ 5058 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 5059 ETH_VLAN_EXTEND_MASK; 5060 err = ixgbevf_vlan_offload_set(dev, mask); 5061 if (err) { 5062 PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); 5063 ixgbe_dev_clear_queues(dev); 5064 return err; 5065 } 5066 5067 ixgbevf_dev_rxtx_start(dev); 5068 5069 /* check and configure queue intr-vector mapping */ 5070 if (rte_intr_cap_multiple(intr_handle) && 5071 dev->data->dev_conf.intr_conf.rxq) { 5072 /* According to datasheet, only vector 0/1/2 can be used, 5073 * now only one vector is used for Rx queue 5074 */ 5075 intr_vector = 1; 5076 if (rte_intr_efd_enable(intr_handle, intr_vector)) 5077 return -1; 5078 } 5079 5080 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 5081 intr_handle->intr_vec = 5082 rte_zmalloc("intr_vec", 5083 dev->data->nb_rx_queues * sizeof(int), 0); 5084 if (intr_handle->intr_vec == NULL) { 5085 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 5086 " intr_vec", dev->data->nb_rx_queues); 5087 return -ENOMEM; 5088 } 5089 } 5090 ixgbevf_configure_msix(dev); 5091 5092 /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt 5093 * is mapped to VFIO vector 0 in eth_ixgbevf_dev_init( ). 5094 * If previous VFIO interrupt mapping setting in eth_ixgbevf_dev_init( ) 5095 * is not cleared, it will fail when following rte_intr_enable( ) tries 5096 * to map Rx queue interrupt to other VFIO vectors. 5097 * So clear uio/vfio intr/evevnfd first to avoid failure. 5098 */ 5099 rte_intr_disable(intr_handle); 5100 5101 rte_intr_enable(intr_handle); 5102 5103 /* Re-enable interrupt for VF */ 5104 ixgbevf_intr_enable(hw); 5105 5106 return 0; 5107 } 5108 5109 static void 5110 ixgbevf_dev_stop(struct rte_eth_dev *dev) 5111 { 5112 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5113 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5114 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5115 5116 PMD_INIT_FUNC_TRACE(); 5117 5118 ixgbevf_intr_disable(hw); 5119 5120 hw->adapter_stopped = 1; 5121 ixgbe_stop_adapter(hw); 5122 5123 /* 5124 * Clear what we set, but we still keep shadow_vfta to 5125 * restore after device starts 5126 */ 5127 ixgbevf_set_vfta_all(dev, 0); 5128 5129 /* Clear stored conf */ 5130 dev->data->scattered_rx = 0; 5131 5132 ixgbe_dev_clear_queues(dev); 5133 5134 /* Clean datapath event and queue/vec mapping */ 5135 rte_intr_efd_disable(intr_handle); 5136 if (intr_handle->intr_vec != NULL) { 5137 rte_free(intr_handle->intr_vec); 5138 intr_handle->intr_vec = NULL; 5139 } 5140 } 5141 5142 static void 5143 ixgbevf_dev_close(struct rte_eth_dev *dev) 5144 { 5145 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5146 5147 PMD_INIT_FUNC_TRACE(); 5148 5149 ixgbe_reset_hw(hw); 5150 5151 ixgbevf_dev_stop(dev); 5152 5153 ixgbe_dev_free_queues(dev); 5154 5155 /** 5156 * Remove the VF MAC address ro ensure 5157 * that the VF traffic goes to the PF 5158 * after stop, close and detach of the VF 5159 **/ 5160 ixgbevf_remove_mac_addr(dev, 0); 5161 } 5162 5163 /* 5164 * Reset VF device 5165 */ 5166 static int 5167 ixgbevf_dev_reset(struct rte_eth_dev *dev) 5168 { 5169 int ret; 5170 5171 ret = eth_ixgbevf_dev_uninit(dev); 5172 if (ret) 5173 return ret; 5174 5175 ret = eth_ixgbevf_dev_init(dev); 5176 5177 return ret; 5178 } 5179 5180 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) 5181 { 5182 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5183 struct ixgbe_vfta *shadow_vfta = 5184 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5185 int i = 0, j = 0, vfta = 0, mask = 1; 5186 5187 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 5188 vfta = shadow_vfta->vfta[i]; 5189 if (vfta) { 5190 mask = 1; 5191 for (j = 0; j < 32; j++) { 5192 if (vfta & mask) 5193 ixgbe_set_vfta(hw, (i<<5)+j, 0, 5194 on, false); 5195 mask <<= 1; 5196 } 5197 } 5198 } 5199 5200 } 5201 5202 static int 5203 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 5204 { 5205 struct ixgbe_hw *hw = 5206 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5207 struct ixgbe_vfta *shadow_vfta = 5208 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5209 uint32_t vid_idx = 0; 5210 uint32_t vid_bit = 0; 5211 int ret = 0; 5212 5213 PMD_INIT_FUNC_TRACE(); 5214 5215 /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */ 5216 ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on, false); 5217 if (ret) { 5218 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 5219 return ret; 5220 } 5221 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 5222 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 5223 5224 /* Save what we set and retore it after device reset */ 5225 if (on) 5226 shadow_vfta->vfta[vid_idx] |= vid_bit; 5227 else 5228 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 5229 5230 return 0; 5231 } 5232 5233 static void 5234 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 5235 { 5236 struct ixgbe_hw *hw = 5237 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5238 uint32_t ctrl; 5239 5240 PMD_INIT_FUNC_TRACE(); 5241 5242 if (queue >= hw->mac.max_rx_queues) 5243 return; 5244 5245 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 5246 if (on) 5247 ctrl |= IXGBE_RXDCTL_VME; 5248 else 5249 ctrl &= ~IXGBE_RXDCTL_VME; 5250 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 5251 5252 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on); 5253 } 5254 5255 static int 5256 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) 5257 { 5258 struct ixgbe_hw *hw = 5259 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5260 uint16_t i; 5261 int on = 0; 5262 5263 /* VF function only support hw strip feature, others are not support */ 5264 if (mask & ETH_VLAN_STRIP_MASK) { 5265 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip); 5266 5267 for (i = 0; i < hw->mac.max_rx_queues; i++) 5268 ixgbevf_vlan_strip_queue_set(dev, i, on); 5269 } 5270 5271 return 0; 5272 } 5273 5274 int 5275 ixgbe_vt_check(struct ixgbe_hw *hw) 5276 { 5277 uint32_t reg_val; 5278 5279 /* if Virtualization Technology is enabled */ 5280 reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 5281 if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) { 5282 PMD_INIT_LOG(ERR, "VT must be enabled for this setting"); 5283 return -1; 5284 } 5285 5286 return 0; 5287 } 5288 5289 static uint32_t 5290 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr *uc_addr) 5291 { 5292 uint32_t vector = 0; 5293 5294 switch (hw->mac.mc_filter_type) { 5295 case 0: /* use bits [47:36] of the address */ 5296 vector = ((uc_addr->addr_bytes[4] >> 4) | 5297 (((uint16_t)uc_addr->addr_bytes[5]) << 4)); 5298 break; 5299 case 1: /* use bits [46:35] of the address */ 5300 vector = ((uc_addr->addr_bytes[4] >> 3) | 5301 (((uint16_t)uc_addr->addr_bytes[5]) << 5)); 5302 break; 5303 case 2: /* use bits [45:34] of the address */ 5304 vector = ((uc_addr->addr_bytes[4] >> 2) | 5305 (((uint16_t)uc_addr->addr_bytes[5]) << 6)); 5306 break; 5307 case 3: /* use bits [43:32] of the address */ 5308 vector = ((uc_addr->addr_bytes[4]) | 5309 (((uint16_t)uc_addr->addr_bytes[5]) << 8)); 5310 break; 5311 default: /* Invalid mc_filter_type */ 5312 break; 5313 } 5314 5315 /* vector can only be 12-bits or boundary will be exceeded */ 5316 vector &= 0xFFF; 5317 return vector; 5318 } 5319 5320 static int 5321 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 5322 uint8_t on) 5323 { 5324 uint32_t vector; 5325 uint32_t uta_idx; 5326 uint32_t reg_val; 5327 uint32_t uta_shift; 5328 uint32_t rc; 5329 const uint32_t ixgbe_uta_idx_mask = 0x7F; 5330 const uint32_t ixgbe_uta_bit_shift = 5; 5331 const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1; 5332 const uint32_t bit1 = 0x1; 5333 5334 struct ixgbe_hw *hw = 5335 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5336 struct ixgbe_uta_info *uta_info = 5337 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5338 5339 /* The UTA table only exists on 82599 hardware and newer */ 5340 if (hw->mac.type < ixgbe_mac_82599EB) 5341 return -ENOTSUP; 5342 5343 vector = ixgbe_uta_vector(hw, mac_addr); 5344 uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask; 5345 uta_shift = vector & ixgbe_uta_bit_mask; 5346 5347 rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0); 5348 if (rc == on) 5349 return 0; 5350 5351 reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx)); 5352 if (on) { 5353 uta_info->uta_in_use++; 5354 reg_val |= (bit1 << uta_shift); 5355 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift); 5356 } else { 5357 uta_info->uta_in_use--; 5358 reg_val &= ~(bit1 << uta_shift); 5359 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift); 5360 } 5361 5362 IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val); 5363 5364 if (uta_info->uta_in_use > 0) 5365 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, 5366 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type); 5367 else 5368 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type); 5369 5370 return 0; 5371 } 5372 5373 static int 5374 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on) 5375 { 5376 int i; 5377 struct ixgbe_hw *hw = 5378 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5379 struct ixgbe_uta_info *uta_info = 5380 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5381 5382 /* The UTA table only exists on 82599 hardware and newer */ 5383 if (hw->mac.type < ixgbe_mac_82599EB) 5384 return -ENOTSUP; 5385 5386 if (on) { 5387 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5388 uta_info->uta_shadow[i] = ~0; 5389 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0); 5390 } 5391 } else { 5392 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5393 uta_info->uta_shadow[i] = 0; 5394 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0); 5395 } 5396 } 5397 return 0; 5398 5399 } 5400 5401 uint32_t 5402 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val) 5403 { 5404 uint32_t new_val = orig_val; 5405 5406 if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG) 5407 new_val |= IXGBE_VMOLR_AUPE; 5408 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC) 5409 new_val |= IXGBE_VMOLR_ROMPE; 5410 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC) 5411 new_val |= IXGBE_VMOLR_ROPE; 5412 if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) 5413 new_val |= IXGBE_VMOLR_BAM; 5414 if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) 5415 new_val |= IXGBE_VMOLR_MPE; 5416 5417 return new_val; 5418 } 5419 5420 #define IXGBE_MRCTL_VPME 0x01 /* Virtual Pool Mirroring. */ 5421 #define IXGBE_MRCTL_UPME 0x02 /* Uplink Port Mirroring. */ 5422 #define IXGBE_MRCTL_DPME 0x04 /* Downlink Port Mirroring. */ 5423 #define IXGBE_MRCTL_VLME 0x08 /* VLAN Mirroring. */ 5424 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \ 5425 ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \ 5426 ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN)) 5427 5428 static int 5429 ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 5430 struct rte_eth_mirror_conf *mirror_conf, 5431 uint8_t rule_id, uint8_t on) 5432 { 5433 uint32_t mr_ctl, vlvf; 5434 uint32_t mp_lsb = 0; 5435 uint32_t mv_msb = 0; 5436 uint32_t mv_lsb = 0; 5437 uint32_t mp_msb = 0; 5438 uint8_t i = 0; 5439 int reg_index = 0; 5440 uint64_t vlan_mask = 0; 5441 5442 const uint8_t pool_mask_offset = 32; 5443 const uint8_t vlan_mask_offset = 32; 5444 const uint8_t dst_pool_offset = 8; 5445 const uint8_t rule_mr_offset = 4; 5446 const uint8_t mirror_rule_mask = 0x0F; 5447 5448 struct ixgbe_mirror_info *mr_info = 5449 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5450 struct ixgbe_hw *hw = 5451 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5452 uint8_t mirror_type = 0; 5453 5454 if (ixgbe_vt_check(hw) < 0) 5455 return -ENOTSUP; 5456 5457 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5458 return -EINVAL; 5459 5460 if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) { 5461 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.", 5462 mirror_conf->rule_type); 5463 return -EINVAL; 5464 } 5465 5466 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5467 mirror_type |= IXGBE_MRCTL_VLME; 5468 /* Check if vlan id is valid and find conresponding VLAN ID 5469 * index in VLVF 5470 */ 5471 for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) { 5472 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) { 5473 /* search vlan id related pool vlan filter 5474 * index 5475 */ 5476 reg_index = ixgbe_find_vlvf_slot( 5477 hw, 5478 mirror_conf->vlan.vlan_id[i], 5479 false); 5480 if (reg_index < 0) 5481 return -EINVAL; 5482 vlvf = IXGBE_READ_REG(hw, 5483 IXGBE_VLVF(reg_index)); 5484 if ((vlvf & IXGBE_VLVF_VIEN) && 5485 ((vlvf & IXGBE_VLVF_VLANID_MASK) == 5486 mirror_conf->vlan.vlan_id[i])) 5487 vlan_mask |= (1ULL << reg_index); 5488 else 5489 return -EINVAL; 5490 } 5491 } 5492 5493 if (on) { 5494 mv_lsb = vlan_mask & 0xFFFFFFFF; 5495 mv_msb = vlan_mask >> vlan_mask_offset; 5496 5497 mr_info->mr_conf[rule_id].vlan.vlan_mask = 5498 mirror_conf->vlan.vlan_mask; 5499 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) { 5500 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) 5501 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 5502 mirror_conf->vlan.vlan_id[i]; 5503 } 5504 } else { 5505 mv_lsb = 0; 5506 mv_msb = 0; 5507 mr_info->mr_conf[rule_id].vlan.vlan_mask = 0; 5508 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) 5509 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0; 5510 } 5511 } 5512 5513 /** 5514 * if enable pool mirror, write related pool mask register,if disable 5515 * pool mirror, clear PFMRVM register 5516 */ 5517 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5518 mirror_type |= IXGBE_MRCTL_VPME; 5519 if (on) { 5520 mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF; 5521 mp_msb = mirror_conf->pool_mask >> pool_mask_offset; 5522 mr_info->mr_conf[rule_id].pool_mask = 5523 mirror_conf->pool_mask; 5524 5525 } else { 5526 mp_lsb = 0; 5527 mp_msb = 0; 5528 mr_info->mr_conf[rule_id].pool_mask = 0; 5529 } 5530 } 5531 if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT) 5532 mirror_type |= IXGBE_MRCTL_UPME; 5533 if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT) 5534 mirror_type |= IXGBE_MRCTL_DPME; 5535 5536 /* read mirror control register and recalculate it */ 5537 mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id)); 5538 5539 if (on) { 5540 mr_ctl |= mirror_type; 5541 mr_ctl &= mirror_rule_mask; 5542 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset; 5543 } else { 5544 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask); 5545 } 5546 5547 mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type; 5548 mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool; 5549 5550 /* write mirrror control register */ 5551 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5552 5553 /* write pool mirrror control register */ 5554 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5555 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb); 5556 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), 5557 mp_msb); 5558 } 5559 /* write VLAN mirrror control register */ 5560 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5561 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb); 5562 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), 5563 mv_msb); 5564 } 5565 5566 return 0; 5567 } 5568 5569 static int 5570 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) 5571 { 5572 int mr_ctl = 0; 5573 uint32_t lsb_val = 0; 5574 uint32_t msb_val = 0; 5575 const uint8_t rule_mr_offset = 4; 5576 5577 struct ixgbe_hw *hw = 5578 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5579 struct ixgbe_mirror_info *mr_info = 5580 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5581 5582 if (ixgbe_vt_check(hw) < 0) 5583 return -ENOTSUP; 5584 5585 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5586 return -EINVAL; 5587 5588 memset(&mr_info->mr_conf[rule_id], 0, 5589 sizeof(struct rte_eth_mirror_conf)); 5590 5591 /* clear PFVMCTL register */ 5592 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5593 5594 /* clear pool mask register */ 5595 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val); 5596 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val); 5597 5598 /* clear vlan mask register */ 5599 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val); 5600 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val); 5601 5602 return 0; 5603 } 5604 5605 static int 5606 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5607 { 5608 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5609 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5610 uint32_t mask; 5611 struct ixgbe_hw *hw = 5612 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5613 uint32_t vec = IXGBE_MISC_VEC_ID; 5614 5615 mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS); 5616 if (rte_intr_allow_others(intr_handle)) 5617 vec = IXGBE_RX_VEC_START; 5618 mask |= (1 << vec); 5619 RTE_SET_USED(queue_id); 5620 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 5621 5622 rte_intr_enable(intr_handle); 5623 5624 return 0; 5625 } 5626 5627 static int 5628 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5629 { 5630 uint32_t mask; 5631 struct ixgbe_hw *hw = 5632 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5633 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5634 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5635 uint32_t vec = IXGBE_MISC_VEC_ID; 5636 5637 mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS); 5638 if (rte_intr_allow_others(intr_handle)) 5639 vec = IXGBE_RX_VEC_START; 5640 mask &= ~(1 << vec); 5641 RTE_SET_USED(queue_id); 5642 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 5643 5644 return 0; 5645 } 5646 5647 static int 5648 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5649 { 5650 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5651 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5652 uint32_t mask; 5653 struct ixgbe_hw *hw = 5654 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5655 struct ixgbe_interrupt *intr = 5656 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5657 5658 if (queue_id < 16) { 5659 ixgbe_disable_intr(hw); 5660 intr->mask |= (1 << queue_id); 5661 ixgbe_enable_intr(dev); 5662 } else if (queue_id < 32) { 5663 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5664 mask &= (1 << queue_id); 5665 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5666 } else if (queue_id < 64) { 5667 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5668 mask &= (1 << (queue_id - 32)); 5669 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5670 } 5671 rte_intr_enable(intr_handle); 5672 5673 return 0; 5674 } 5675 5676 static int 5677 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5678 { 5679 uint32_t mask; 5680 struct ixgbe_hw *hw = 5681 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5682 struct ixgbe_interrupt *intr = 5683 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5684 5685 if (queue_id < 16) { 5686 ixgbe_disable_intr(hw); 5687 intr->mask &= ~(1 << queue_id); 5688 ixgbe_enable_intr(dev); 5689 } else if (queue_id < 32) { 5690 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5691 mask &= ~(1 << queue_id); 5692 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5693 } else if (queue_id < 64) { 5694 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5695 mask &= ~(1 << (queue_id - 32)); 5696 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5697 } 5698 5699 return 0; 5700 } 5701 5702 static void 5703 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 5704 uint8_t queue, uint8_t msix_vector) 5705 { 5706 uint32_t tmp, idx; 5707 5708 if (direction == -1) { 5709 /* other causes */ 5710 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5711 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 5712 tmp &= ~0xFF; 5713 tmp |= msix_vector; 5714 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp); 5715 } else { 5716 /* rx or tx cause */ 5717 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5718 idx = ((16 * (queue & 1)) + (8 * direction)); 5719 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 5720 tmp &= ~(0xFF << idx); 5721 tmp |= (msix_vector << idx); 5722 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp); 5723 } 5724 } 5725 5726 /** 5727 * set the IVAR registers, mapping interrupt causes to vectors 5728 * @param hw 5729 * pointer to ixgbe_hw struct 5730 * @direction 5731 * 0 for Rx, 1 for Tx, -1 for other causes 5732 * @queue 5733 * queue to map the corresponding interrupt to 5734 * @msix_vector 5735 * the vector to map to the corresponding queue 5736 */ 5737 static void 5738 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 5739 uint8_t queue, uint8_t msix_vector) 5740 { 5741 uint32_t tmp, idx; 5742 5743 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5744 if (hw->mac.type == ixgbe_mac_82598EB) { 5745 if (direction == -1) 5746 direction = 0; 5747 idx = (((direction * 64) + queue) >> 2) & 0x1F; 5748 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx)); 5749 tmp &= ~(0xFF << (8 * (queue & 0x3))); 5750 tmp |= (msix_vector << (8 * (queue & 0x3))); 5751 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); 5752 } else if ((hw->mac.type == ixgbe_mac_82599EB) || 5753 (hw->mac.type == ixgbe_mac_X540) || 5754 (hw->mac.type == ixgbe_mac_X550)) { 5755 if (direction == -1) { 5756 /* other causes */ 5757 idx = ((queue & 1) * 8); 5758 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5759 tmp &= ~(0xFF << idx); 5760 tmp |= (msix_vector << idx); 5761 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp); 5762 } else { 5763 /* rx or tx causes */ 5764 idx = ((16 * (queue & 1)) + (8 * direction)); 5765 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 5766 tmp &= ~(0xFF << idx); 5767 tmp |= (msix_vector << idx); 5768 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp); 5769 } 5770 } 5771 } 5772 5773 static void 5774 ixgbevf_configure_msix(struct rte_eth_dev *dev) 5775 { 5776 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5777 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5778 struct ixgbe_hw *hw = 5779 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5780 uint32_t q_idx; 5781 uint32_t vector_idx = IXGBE_MISC_VEC_ID; 5782 uint32_t base = IXGBE_MISC_VEC_ID; 5783 5784 /* Configure VF other cause ivar */ 5785 ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); 5786 5787 /* won't configure msix register if no mapping is done 5788 * between intr vector and event fd. 5789 */ 5790 if (!rte_intr_dp_is_en(intr_handle)) 5791 return; 5792 5793 if (rte_intr_allow_others(intr_handle)) { 5794 base = IXGBE_RX_VEC_START; 5795 vector_idx = IXGBE_RX_VEC_START; 5796 } 5797 5798 /* Configure all RX queues of VF */ 5799 for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) { 5800 /* Force all queue use vector 0, 5801 * as IXGBE_VF_MAXMSIVECOTR = 1 5802 */ 5803 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx); 5804 intr_handle->intr_vec[q_idx] = vector_idx; 5805 if (vector_idx < base + intr_handle->nb_efd - 1) 5806 vector_idx++; 5807 } 5808 } 5809 5810 /** 5811 * Sets up the hardware to properly generate MSI-X interrupts 5812 * @hw 5813 * board private structure 5814 */ 5815 static void 5816 ixgbe_configure_msix(struct rte_eth_dev *dev) 5817 { 5818 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5819 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5820 struct ixgbe_hw *hw = 5821 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5822 uint32_t queue_id, base = IXGBE_MISC_VEC_ID; 5823 uint32_t vec = IXGBE_MISC_VEC_ID; 5824 uint32_t mask; 5825 uint32_t gpie; 5826 5827 /* won't configure msix register if no mapping is done 5828 * between intr vector and event fd 5829 * but if misx has been enabled already, need to configure 5830 * auto clean, auto mask and throttling. 5831 */ 5832 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 5833 if (!rte_intr_dp_is_en(intr_handle) && 5834 !(gpie & (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT))) 5835 return; 5836 5837 if (rte_intr_allow_others(intr_handle)) 5838 vec = base = IXGBE_RX_VEC_START; 5839 5840 /* setup GPIE for MSI-x mode */ 5841 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 5842 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 5843 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME; 5844 /* auto clearing and auto setting corresponding bits in EIMS 5845 * when MSI-X interrupt is triggered 5846 */ 5847 if (hw->mac.type == ixgbe_mac_82598EB) { 5848 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 5849 } else { 5850 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 5851 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 5852 } 5853 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 5854 5855 /* Populate the IVAR table and set the ITR values to the 5856 * corresponding register. 5857 */ 5858 if (rte_intr_dp_is_en(intr_handle)) { 5859 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; 5860 queue_id++) { 5861 /* by default, 1:1 mapping */ 5862 ixgbe_set_ivar_map(hw, 0, queue_id, vec); 5863 intr_handle->intr_vec[queue_id] = vec; 5864 if (vec < base + intr_handle->nb_efd - 1) 5865 vec++; 5866 } 5867 5868 switch (hw->mac.type) { 5869 case ixgbe_mac_82598EB: 5870 ixgbe_set_ivar_map(hw, -1, 5871 IXGBE_IVAR_OTHER_CAUSES_INDEX, 5872 IXGBE_MISC_VEC_ID); 5873 break; 5874 case ixgbe_mac_82599EB: 5875 case ixgbe_mac_X540: 5876 case ixgbe_mac_X550: 5877 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); 5878 break; 5879 default: 5880 break; 5881 } 5882 } 5883 IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID), 5884 IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF); 5885 5886 /* set up to autoclear timer, and the vectors */ 5887 mask = IXGBE_EIMS_ENABLE_MASK; 5888 mask &= ~(IXGBE_EIMS_OTHER | 5889 IXGBE_EIMS_MAILBOX | 5890 IXGBE_EIMS_LSC); 5891 5892 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 5893 } 5894 5895 int 5896 ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, 5897 uint16_t queue_idx, uint16_t tx_rate) 5898 { 5899 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5900 uint32_t rf_dec, rf_int; 5901 uint32_t bcnrc_val; 5902 uint16_t link_speed = dev->data->dev_link.link_speed; 5903 5904 if (queue_idx >= hw->mac.max_tx_queues) 5905 return -EINVAL; 5906 5907 if (tx_rate != 0) { 5908 /* Calculate the rate factor values to set */ 5909 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate; 5910 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate; 5911 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate; 5912 5913 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA; 5914 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) & 5915 IXGBE_RTTBCNRC_RF_INT_MASK_M); 5916 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK); 5917 } else { 5918 bcnrc_val = 0; 5919 } 5920 5921 /* 5922 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 5923 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise 5924 * set as 0x4. 5925 */ 5926 if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) && 5927 (dev->data->dev_conf.rxmode.max_rx_pkt_len >= 5928 IXGBE_MAX_JUMBO_FRAME_SIZE)) 5929 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 5930 IXGBE_MMW_SIZE_JUMBO_FRAME); 5931 else 5932 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 5933 IXGBE_MMW_SIZE_DEFAULT); 5934 5935 /* Set RTTBCNRC of queue X */ 5936 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx); 5937 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val); 5938 IXGBE_WRITE_FLUSH(hw); 5939 5940 return 0; 5941 } 5942 5943 static int 5944 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 5945 __attribute__((unused)) uint32_t index, 5946 __attribute__((unused)) uint32_t pool) 5947 { 5948 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5949 int diag; 5950 5951 /* 5952 * On a 82599 VF, adding again the same MAC addr is not an idempotent 5953 * operation. Trap this case to avoid exhausting the [very limited] 5954 * set of PF resources used to store VF MAC addresses. 5955 */ 5956 if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0) 5957 return -1; 5958 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 5959 if (diag != 0) 5960 PMD_DRV_LOG(ERR, "Unable to add MAC address " 5961 "%02x:%02x:%02x:%02x:%02x:%02x - diag=%d", 5962 mac_addr->addr_bytes[0], 5963 mac_addr->addr_bytes[1], 5964 mac_addr->addr_bytes[2], 5965 mac_addr->addr_bytes[3], 5966 mac_addr->addr_bytes[4], 5967 mac_addr->addr_bytes[5], 5968 diag); 5969 return diag; 5970 } 5971 5972 static void 5973 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index) 5974 { 5975 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5976 struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr; 5977 struct ether_addr *mac_addr; 5978 uint32_t i; 5979 int diag; 5980 5981 /* 5982 * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does 5983 * not support the deletion of a given MAC address. 5984 * Instead, it imposes to delete all MAC addresses, then to add again 5985 * all MAC addresses with the exception of the one to be deleted. 5986 */ 5987 (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL); 5988 5989 /* 5990 * Add again all MAC addresses, with the exception of the deleted one 5991 * and of the permanent MAC address. 5992 */ 5993 for (i = 0, mac_addr = dev->data->mac_addrs; 5994 i < hw->mac.num_rar_entries; i++, mac_addr++) { 5995 /* Skip the deleted MAC address */ 5996 if (i == index) 5997 continue; 5998 /* Skip NULL MAC addresses */ 5999 if (is_zero_ether_addr(mac_addr)) 6000 continue; 6001 /* Skip the permanent MAC address */ 6002 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0) 6003 continue; 6004 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6005 if (diag != 0) 6006 PMD_DRV_LOG(ERR, 6007 "Adding again MAC address " 6008 "%02x:%02x:%02x:%02x:%02x:%02x failed " 6009 "diag=%d", 6010 mac_addr->addr_bytes[0], 6011 mac_addr->addr_bytes[1], 6012 mac_addr->addr_bytes[2], 6013 mac_addr->addr_bytes[3], 6014 mac_addr->addr_bytes[4], 6015 mac_addr->addr_bytes[5], 6016 diag); 6017 } 6018 } 6019 6020 static void 6021 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) 6022 { 6023 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6024 6025 hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0); 6026 } 6027 6028 int 6029 ixgbe_syn_filter_set(struct rte_eth_dev *dev, 6030 struct rte_eth_syn_filter *filter, 6031 bool add) 6032 { 6033 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6034 struct ixgbe_filter_info *filter_info = 6035 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6036 uint32_t syn_info; 6037 uint32_t synqf; 6038 6039 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6040 return -EINVAL; 6041 6042 syn_info = filter_info->syn_info; 6043 6044 if (add) { 6045 if (syn_info & IXGBE_SYN_FILTER_ENABLE) 6046 return -EINVAL; 6047 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) & 6048 IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE); 6049 6050 if (filter->hig_pri) 6051 synqf |= IXGBE_SYN_FILTER_SYNQFP; 6052 else 6053 synqf &= ~IXGBE_SYN_FILTER_SYNQFP; 6054 } else { 6055 synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6056 if (!(syn_info & IXGBE_SYN_FILTER_ENABLE)) 6057 return -ENOENT; 6058 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE); 6059 } 6060 6061 filter_info->syn_info = synqf; 6062 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 6063 IXGBE_WRITE_FLUSH(hw); 6064 return 0; 6065 } 6066 6067 static int 6068 ixgbe_syn_filter_get(struct rte_eth_dev *dev, 6069 struct rte_eth_syn_filter *filter) 6070 { 6071 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6072 uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6073 6074 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 6075 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0; 6076 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1); 6077 return 0; 6078 } 6079 return -ENOENT; 6080 } 6081 6082 static int 6083 ixgbe_syn_filter_handle(struct rte_eth_dev *dev, 6084 enum rte_filter_op filter_op, 6085 void *arg) 6086 { 6087 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6088 int ret; 6089 6090 MAC_TYPE_FILTER_SUP(hw->mac.type); 6091 6092 if (filter_op == RTE_ETH_FILTER_NOP) 6093 return 0; 6094 6095 if (arg == NULL) { 6096 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 6097 filter_op); 6098 return -EINVAL; 6099 } 6100 6101 switch (filter_op) { 6102 case RTE_ETH_FILTER_ADD: 6103 ret = ixgbe_syn_filter_set(dev, 6104 (struct rte_eth_syn_filter *)arg, 6105 TRUE); 6106 break; 6107 case RTE_ETH_FILTER_DELETE: 6108 ret = ixgbe_syn_filter_set(dev, 6109 (struct rte_eth_syn_filter *)arg, 6110 FALSE); 6111 break; 6112 case RTE_ETH_FILTER_GET: 6113 ret = ixgbe_syn_filter_get(dev, 6114 (struct rte_eth_syn_filter *)arg); 6115 break; 6116 default: 6117 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 6118 ret = -EINVAL; 6119 break; 6120 } 6121 6122 return ret; 6123 } 6124 6125 6126 static inline enum ixgbe_5tuple_protocol 6127 convert_protocol_type(uint8_t protocol_value) 6128 { 6129 if (protocol_value == IPPROTO_TCP) 6130 return IXGBE_FILTER_PROTOCOL_TCP; 6131 else if (protocol_value == IPPROTO_UDP) 6132 return IXGBE_FILTER_PROTOCOL_UDP; 6133 else if (protocol_value == IPPROTO_SCTP) 6134 return IXGBE_FILTER_PROTOCOL_SCTP; 6135 else 6136 return IXGBE_FILTER_PROTOCOL_NONE; 6137 } 6138 6139 /* inject a 5-tuple filter to HW */ 6140 static inline void 6141 ixgbe_inject_5tuple_filter(struct rte_eth_dev *dev, 6142 struct ixgbe_5tuple_filter *filter) 6143 { 6144 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6145 int i; 6146 uint32_t ftqf, sdpqf; 6147 uint32_t l34timir = 0; 6148 uint8_t mask = 0xff; 6149 6150 i = filter->index; 6151 6152 sdpqf = (uint32_t)(filter->filter_info.dst_port << 6153 IXGBE_SDPQF_DSTPORT_SHIFT); 6154 sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT); 6155 6156 ftqf = (uint32_t)(filter->filter_info.proto & 6157 IXGBE_FTQF_PROTOCOL_MASK); 6158 ftqf |= (uint32_t)((filter->filter_info.priority & 6159 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT); 6160 if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */ 6161 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK; 6162 if (filter->filter_info.dst_ip_mask == 0) 6163 mask &= IXGBE_FTQF_DEST_ADDR_MASK; 6164 if (filter->filter_info.src_port_mask == 0) 6165 mask &= IXGBE_FTQF_SOURCE_PORT_MASK; 6166 if (filter->filter_info.dst_port_mask == 0) 6167 mask &= IXGBE_FTQF_DEST_PORT_MASK; 6168 if (filter->filter_info.proto_mask == 0) 6169 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK; 6170 ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT; 6171 ftqf |= IXGBE_FTQF_POOL_MASK_EN; 6172 ftqf |= IXGBE_FTQF_QUEUE_ENABLE; 6173 6174 IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip); 6175 IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip); 6176 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf); 6177 IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf); 6178 6179 l34timir |= IXGBE_L34T_IMIR_RESERVE; 6180 l34timir |= (uint32_t)(filter->queue << 6181 IXGBE_L34T_IMIR_QUEUE_SHIFT); 6182 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir); 6183 } 6184 6185 /* 6186 * add a 5tuple filter 6187 * 6188 * @param 6189 * dev: Pointer to struct rte_eth_dev. 6190 * index: the index the filter allocates. 6191 * filter: ponter to the filter that will be added. 6192 * rx_queue: the queue id the filter assigned to. 6193 * 6194 * @return 6195 * - On success, zero. 6196 * - On failure, a negative value. 6197 */ 6198 static int 6199 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 6200 struct ixgbe_5tuple_filter *filter) 6201 { 6202 struct ixgbe_filter_info *filter_info = 6203 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6204 int i, idx, shift; 6205 6206 /* 6207 * look for an unused 5tuple filter index, 6208 * and insert the filter to list. 6209 */ 6210 for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { 6211 idx = i / (sizeof(uint32_t) * NBBY); 6212 shift = i % (sizeof(uint32_t) * NBBY); 6213 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { 6214 filter_info->fivetuple_mask[idx] |= 1 << shift; 6215 filter->index = i; 6216 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 6217 filter, 6218 entries); 6219 break; 6220 } 6221 } 6222 if (i >= IXGBE_MAX_FTQF_FILTERS) { 6223 PMD_DRV_LOG(ERR, "5tuple filters are full."); 6224 return -ENOSYS; 6225 } 6226 6227 ixgbe_inject_5tuple_filter(dev, filter); 6228 6229 return 0; 6230 } 6231 6232 /* 6233 * remove a 5tuple filter 6234 * 6235 * @param 6236 * dev: Pointer to struct rte_eth_dev. 6237 * filter: the pointer of the filter will be removed. 6238 */ 6239 static void 6240 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 6241 struct ixgbe_5tuple_filter *filter) 6242 { 6243 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6244 struct ixgbe_filter_info *filter_info = 6245 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6246 uint16_t index = filter->index; 6247 6248 filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &= 6249 ~(1 << (index % (sizeof(uint32_t) * NBBY))); 6250 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 6251 rte_free(filter); 6252 6253 IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0); 6254 IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0); 6255 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0); 6256 IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0); 6257 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0); 6258 } 6259 6260 static int 6261 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 6262 { 6263 struct ixgbe_hw *hw; 6264 uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 6265 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 6266 6267 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6268 6269 if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN)) 6270 return -EINVAL; 6271 6272 /* refuse mtu that requires the support of scattered packets when this 6273 * feature has not been enabled before. 6274 */ 6275 if (!rx_conf->enable_scatter && 6276 (max_frame + 2 * IXGBE_VLAN_TAG_SIZE > 6277 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) 6278 return -EINVAL; 6279 6280 /* 6281 * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU 6282 * request of the version 2.0 of the mailbox API. 6283 * For now, use the IXGBE_VF_SET_LPE request of the version 1.0 6284 * of the mailbox API. 6285 * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers 6286 * prior to 3.11.33 which contains the following change: 6287 * "ixgbe: Enable jumbo frames support w/ SR-IOV" 6288 */ 6289 ixgbevf_rlpml_set_vf(hw, max_frame); 6290 6291 /* update max frame size */ 6292 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame; 6293 return 0; 6294 } 6295 6296 static inline struct ixgbe_5tuple_filter * 6297 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list, 6298 struct ixgbe_5tuple_filter_info *key) 6299 { 6300 struct ixgbe_5tuple_filter *it; 6301 6302 TAILQ_FOREACH(it, filter_list, entries) { 6303 if (memcmp(key, &it->filter_info, 6304 sizeof(struct ixgbe_5tuple_filter_info)) == 0) { 6305 return it; 6306 } 6307 } 6308 return NULL; 6309 } 6310 6311 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/ 6312 static inline int 6313 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter, 6314 struct ixgbe_5tuple_filter_info *filter_info) 6315 { 6316 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM || 6317 filter->priority > IXGBE_5TUPLE_MAX_PRI || 6318 filter->priority < IXGBE_5TUPLE_MIN_PRI) 6319 return -EINVAL; 6320 6321 switch (filter->dst_ip_mask) { 6322 case UINT32_MAX: 6323 filter_info->dst_ip_mask = 0; 6324 filter_info->dst_ip = filter->dst_ip; 6325 break; 6326 case 0: 6327 filter_info->dst_ip_mask = 1; 6328 break; 6329 default: 6330 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 6331 return -EINVAL; 6332 } 6333 6334 switch (filter->src_ip_mask) { 6335 case UINT32_MAX: 6336 filter_info->src_ip_mask = 0; 6337 filter_info->src_ip = filter->src_ip; 6338 break; 6339 case 0: 6340 filter_info->src_ip_mask = 1; 6341 break; 6342 default: 6343 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 6344 return -EINVAL; 6345 } 6346 6347 switch (filter->dst_port_mask) { 6348 case UINT16_MAX: 6349 filter_info->dst_port_mask = 0; 6350 filter_info->dst_port = filter->dst_port; 6351 break; 6352 case 0: 6353 filter_info->dst_port_mask = 1; 6354 break; 6355 default: 6356 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 6357 return -EINVAL; 6358 } 6359 6360 switch (filter->src_port_mask) { 6361 case UINT16_MAX: 6362 filter_info->src_port_mask = 0; 6363 filter_info->src_port = filter->src_port; 6364 break; 6365 case 0: 6366 filter_info->src_port_mask = 1; 6367 break; 6368 default: 6369 PMD_DRV_LOG(ERR, "invalid src_port mask."); 6370 return -EINVAL; 6371 } 6372 6373 switch (filter->proto_mask) { 6374 case UINT8_MAX: 6375 filter_info->proto_mask = 0; 6376 filter_info->proto = 6377 convert_protocol_type(filter->proto); 6378 break; 6379 case 0: 6380 filter_info->proto_mask = 1; 6381 break; 6382 default: 6383 PMD_DRV_LOG(ERR, "invalid protocol mask."); 6384 return -EINVAL; 6385 } 6386 6387 filter_info->priority = (uint8_t)filter->priority; 6388 return 0; 6389 } 6390 6391 /* 6392 * add or delete a ntuple filter 6393 * 6394 * @param 6395 * dev: Pointer to struct rte_eth_dev. 6396 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6397 * add: if true, add filter, if false, remove filter 6398 * 6399 * @return 6400 * - On success, zero. 6401 * - On failure, a negative value. 6402 */ 6403 int 6404 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, 6405 struct rte_eth_ntuple_filter *ntuple_filter, 6406 bool add) 6407 { 6408 struct ixgbe_filter_info *filter_info = 6409 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6410 struct ixgbe_5tuple_filter_info filter_5tuple; 6411 struct ixgbe_5tuple_filter *filter; 6412 int ret; 6413 6414 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6415 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6416 return -EINVAL; 6417 } 6418 6419 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6420 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6421 if (ret < 0) 6422 return ret; 6423 6424 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6425 &filter_5tuple); 6426 if (filter != NULL && add) { 6427 PMD_DRV_LOG(ERR, "filter exists."); 6428 return -EEXIST; 6429 } 6430 if (filter == NULL && !add) { 6431 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6432 return -ENOENT; 6433 } 6434 6435 if (add) { 6436 filter = rte_zmalloc("ixgbe_5tuple_filter", 6437 sizeof(struct ixgbe_5tuple_filter), 0); 6438 if (filter == NULL) 6439 return -ENOMEM; 6440 rte_memcpy(&filter->filter_info, 6441 &filter_5tuple, 6442 sizeof(struct ixgbe_5tuple_filter_info)); 6443 filter->queue = ntuple_filter->queue; 6444 ret = ixgbe_add_5tuple_filter(dev, filter); 6445 if (ret < 0) { 6446 rte_free(filter); 6447 return ret; 6448 } 6449 } else 6450 ixgbe_remove_5tuple_filter(dev, filter); 6451 6452 return 0; 6453 } 6454 6455 /* 6456 * get a ntuple filter 6457 * 6458 * @param 6459 * dev: Pointer to struct rte_eth_dev. 6460 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6461 * 6462 * @return 6463 * - On success, zero. 6464 * - On failure, a negative value. 6465 */ 6466 static int 6467 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev, 6468 struct rte_eth_ntuple_filter *ntuple_filter) 6469 { 6470 struct ixgbe_filter_info *filter_info = 6471 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6472 struct ixgbe_5tuple_filter_info filter_5tuple; 6473 struct ixgbe_5tuple_filter *filter; 6474 int ret; 6475 6476 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6477 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6478 return -EINVAL; 6479 } 6480 6481 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6482 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6483 if (ret < 0) 6484 return ret; 6485 6486 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6487 &filter_5tuple); 6488 if (filter == NULL) { 6489 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6490 return -ENOENT; 6491 } 6492 ntuple_filter->queue = filter->queue; 6493 return 0; 6494 } 6495 6496 /* 6497 * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter. 6498 * @dev: pointer to rte_eth_dev structure 6499 * @filter_op:operation will be taken. 6500 * @arg: a pointer to specific structure corresponding to the filter_op 6501 * 6502 * @return 6503 * - On success, zero. 6504 * - On failure, a negative value. 6505 */ 6506 static int 6507 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, 6508 enum rte_filter_op filter_op, 6509 void *arg) 6510 { 6511 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6512 int ret; 6513 6514 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 6515 6516 if (filter_op == RTE_ETH_FILTER_NOP) 6517 return 0; 6518 6519 if (arg == NULL) { 6520 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 6521 filter_op); 6522 return -EINVAL; 6523 } 6524 6525 switch (filter_op) { 6526 case RTE_ETH_FILTER_ADD: 6527 ret = ixgbe_add_del_ntuple_filter(dev, 6528 (struct rte_eth_ntuple_filter *)arg, 6529 TRUE); 6530 break; 6531 case RTE_ETH_FILTER_DELETE: 6532 ret = ixgbe_add_del_ntuple_filter(dev, 6533 (struct rte_eth_ntuple_filter *)arg, 6534 FALSE); 6535 break; 6536 case RTE_ETH_FILTER_GET: 6537 ret = ixgbe_get_ntuple_filter(dev, 6538 (struct rte_eth_ntuple_filter *)arg); 6539 break; 6540 default: 6541 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 6542 ret = -EINVAL; 6543 break; 6544 } 6545 return ret; 6546 } 6547 6548 int 6549 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, 6550 struct rte_eth_ethertype_filter *filter, 6551 bool add) 6552 { 6553 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6554 struct ixgbe_filter_info *filter_info = 6555 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6556 uint32_t etqf = 0; 6557 uint32_t etqs = 0; 6558 int ret; 6559 struct ixgbe_ethertype_filter ethertype_filter; 6560 6561 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6562 return -EINVAL; 6563 6564 if (filter->ether_type == ETHER_TYPE_IPv4 || 6565 filter->ether_type == ETHER_TYPE_IPv6) { 6566 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 6567 " ethertype filter.", filter->ether_type); 6568 return -EINVAL; 6569 } 6570 6571 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 6572 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 6573 return -EINVAL; 6574 } 6575 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 6576 PMD_DRV_LOG(ERR, "drop option is unsupported."); 6577 return -EINVAL; 6578 } 6579 6580 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6581 if (ret >= 0 && add) { 6582 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 6583 filter->ether_type); 6584 return -EEXIST; 6585 } 6586 if (ret < 0 && !add) { 6587 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6588 filter->ether_type); 6589 return -ENOENT; 6590 } 6591 6592 if (add) { 6593 etqf = IXGBE_ETQF_FILTER_EN; 6594 etqf |= (uint32_t)filter->ether_type; 6595 etqs |= (uint32_t)((filter->queue << 6596 IXGBE_ETQS_RX_QUEUE_SHIFT) & 6597 IXGBE_ETQS_RX_QUEUE); 6598 etqs |= IXGBE_ETQS_QUEUE_EN; 6599 6600 ethertype_filter.ethertype = filter->ether_type; 6601 ethertype_filter.etqf = etqf; 6602 ethertype_filter.etqs = etqs; 6603 ethertype_filter.conf = FALSE; 6604 ret = ixgbe_ethertype_filter_insert(filter_info, 6605 ðertype_filter); 6606 if (ret < 0) { 6607 PMD_DRV_LOG(ERR, "ethertype filters are full."); 6608 return -ENOSPC; 6609 } 6610 } else { 6611 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret); 6612 if (ret < 0) 6613 return -ENOSYS; 6614 } 6615 IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf); 6616 IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs); 6617 IXGBE_WRITE_FLUSH(hw); 6618 6619 return 0; 6620 } 6621 6622 static int 6623 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev, 6624 struct rte_eth_ethertype_filter *filter) 6625 { 6626 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6627 struct ixgbe_filter_info *filter_info = 6628 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6629 uint32_t etqf, etqs; 6630 int ret; 6631 6632 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6633 if (ret < 0) { 6634 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6635 filter->ether_type); 6636 return -ENOENT; 6637 } 6638 6639 etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret)); 6640 if (etqf & IXGBE_ETQF_FILTER_EN) { 6641 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret)); 6642 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE; 6643 filter->flags = 0; 6644 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >> 6645 IXGBE_ETQS_RX_QUEUE_SHIFT; 6646 return 0; 6647 } 6648 return -ENOENT; 6649 } 6650 6651 /* 6652 * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter. 6653 * @dev: pointer to rte_eth_dev structure 6654 * @filter_op:operation will be taken. 6655 * @arg: a pointer to specific structure corresponding to the filter_op 6656 */ 6657 static int 6658 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev, 6659 enum rte_filter_op filter_op, 6660 void *arg) 6661 { 6662 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6663 int ret; 6664 6665 MAC_TYPE_FILTER_SUP(hw->mac.type); 6666 6667 if (filter_op == RTE_ETH_FILTER_NOP) 6668 return 0; 6669 6670 if (arg == NULL) { 6671 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 6672 filter_op); 6673 return -EINVAL; 6674 } 6675 6676 switch (filter_op) { 6677 case RTE_ETH_FILTER_ADD: 6678 ret = ixgbe_add_del_ethertype_filter(dev, 6679 (struct rte_eth_ethertype_filter *)arg, 6680 TRUE); 6681 break; 6682 case RTE_ETH_FILTER_DELETE: 6683 ret = ixgbe_add_del_ethertype_filter(dev, 6684 (struct rte_eth_ethertype_filter *)arg, 6685 FALSE); 6686 break; 6687 case RTE_ETH_FILTER_GET: 6688 ret = ixgbe_get_ethertype_filter(dev, 6689 (struct rte_eth_ethertype_filter *)arg); 6690 break; 6691 default: 6692 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 6693 ret = -EINVAL; 6694 break; 6695 } 6696 return ret; 6697 } 6698 6699 static int 6700 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, 6701 enum rte_filter_type filter_type, 6702 enum rte_filter_op filter_op, 6703 void *arg) 6704 { 6705 int ret = 0; 6706 6707 switch (filter_type) { 6708 case RTE_ETH_FILTER_NTUPLE: 6709 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg); 6710 break; 6711 case RTE_ETH_FILTER_ETHERTYPE: 6712 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg); 6713 break; 6714 case RTE_ETH_FILTER_SYN: 6715 ret = ixgbe_syn_filter_handle(dev, filter_op, arg); 6716 break; 6717 case RTE_ETH_FILTER_FDIR: 6718 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg); 6719 break; 6720 case RTE_ETH_FILTER_L2_TUNNEL: 6721 ret = ixgbe_dev_l2_tunnel_filter_handle(dev, filter_op, arg); 6722 break; 6723 case RTE_ETH_FILTER_GENERIC: 6724 if (filter_op != RTE_ETH_FILTER_GET) 6725 return -EINVAL; 6726 *(const void **)arg = &ixgbe_flow_ops; 6727 break; 6728 default: 6729 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 6730 filter_type); 6731 ret = -EINVAL; 6732 break; 6733 } 6734 6735 return ret; 6736 } 6737 6738 static u8 * 6739 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw, 6740 u8 **mc_addr_ptr, u32 *vmdq) 6741 { 6742 u8 *mc_addr; 6743 6744 *vmdq = 0; 6745 mc_addr = *mc_addr_ptr; 6746 *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr)); 6747 return mc_addr; 6748 } 6749 6750 static int 6751 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 6752 struct ether_addr *mc_addr_set, 6753 uint32_t nb_mc_addr) 6754 { 6755 struct ixgbe_hw *hw; 6756 u8 *mc_addr_list; 6757 6758 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6759 mc_addr_list = (u8 *)mc_addr_set; 6760 return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr, 6761 ixgbe_dev_addr_list_itr, TRUE); 6762 } 6763 6764 static uint64_t 6765 ixgbe_read_systime_cyclecounter(struct rte_eth_dev *dev) 6766 { 6767 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6768 uint64_t systime_cycles; 6769 6770 switch (hw->mac.type) { 6771 case ixgbe_mac_X550: 6772 case ixgbe_mac_X550EM_x: 6773 case ixgbe_mac_X550EM_a: 6774 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 6775 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6776 systime_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6777 * NSEC_PER_SEC; 6778 break; 6779 default: 6780 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6781 systime_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6782 << 32; 6783 } 6784 6785 return systime_cycles; 6786 } 6787 6788 static uint64_t 6789 ixgbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6790 { 6791 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6792 uint64_t rx_tstamp_cycles; 6793 6794 switch (hw->mac.type) { 6795 case ixgbe_mac_X550: 6796 case ixgbe_mac_X550EM_x: 6797 case ixgbe_mac_X550EM_a: 6798 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6799 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6800 rx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6801 * NSEC_PER_SEC; 6802 break; 6803 default: 6804 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 6805 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 6806 rx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 6807 << 32; 6808 } 6809 6810 return rx_tstamp_cycles; 6811 } 6812 6813 static uint64_t 6814 ixgbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6815 { 6816 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6817 uint64_t tx_tstamp_cycles; 6818 6819 switch (hw->mac.type) { 6820 case ixgbe_mac_X550: 6821 case ixgbe_mac_X550EM_x: 6822 case ixgbe_mac_X550EM_a: 6823 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6824 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6825 tx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6826 * NSEC_PER_SEC; 6827 break; 6828 default: 6829 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 6830 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 6831 tx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 6832 << 32; 6833 } 6834 6835 return tx_tstamp_cycles; 6836 } 6837 6838 static void 6839 ixgbe_start_timecounters(struct rte_eth_dev *dev) 6840 { 6841 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6842 struct ixgbe_adapter *adapter = 6843 (struct ixgbe_adapter *)dev->data->dev_private; 6844 struct rte_eth_link link; 6845 uint32_t incval = 0; 6846 uint32_t shift = 0; 6847 6848 /* Get current link speed. */ 6849 memset(&link, 0, sizeof(link)); 6850 ixgbe_dev_link_update(dev, 1); 6851 rte_ixgbe_dev_atomic_read_link_status(dev, &link); 6852 6853 switch (link.link_speed) { 6854 case ETH_SPEED_NUM_100M: 6855 incval = IXGBE_INCVAL_100; 6856 shift = IXGBE_INCVAL_SHIFT_100; 6857 break; 6858 case ETH_SPEED_NUM_1G: 6859 incval = IXGBE_INCVAL_1GB; 6860 shift = IXGBE_INCVAL_SHIFT_1GB; 6861 break; 6862 case ETH_SPEED_NUM_10G: 6863 default: 6864 incval = IXGBE_INCVAL_10GB; 6865 shift = IXGBE_INCVAL_SHIFT_10GB; 6866 break; 6867 } 6868 6869 switch (hw->mac.type) { 6870 case ixgbe_mac_X550: 6871 case ixgbe_mac_X550EM_x: 6872 case ixgbe_mac_X550EM_a: 6873 /* Independent of link speed. */ 6874 incval = 1; 6875 /* Cycles read will be interpreted as ns. */ 6876 shift = 0; 6877 /* Fall-through */ 6878 case ixgbe_mac_X540: 6879 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval); 6880 break; 6881 case ixgbe_mac_82599EB: 6882 incval >>= IXGBE_INCVAL_SHIFT_82599; 6883 shift -= IXGBE_INCVAL_SHIFT_82599; 6884 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 6885 (1 << IXGBE_INCPER_SHIFT_82599) | incval); 6886 break; 6887 default: 6888 /* Not supported. */ 6889 return; 6890 } 6891 6892 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 6893 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6894 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 6895 6896 adapter->systime_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6897 adapter->systime_tc.cc_shift = shift; 6898 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 6899 6900 adapter->rx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6901 adapter->rx_tstamp_tc.cc_shift = shift; 6902 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6903 6904 adapter->tx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 6905 adapter->tx_tstamp_tc.cc_shift = shift; 6906 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 6907 } 6908 6909 static int 6910 ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 6911 { 6912 struct ixgbe_adapter *adapter = 6913 (struct ixgbe_adapter *)dev->data->dev_private; 6914 6915 adapter->systime_tc.nsec += delta; 6916 adapter->rx_tstamp_tc.nsec += delta; 6917 adapter->tx_tstamp_tc.nsec += delta; 6918 6919 return 0; 6920 } 6921 6922 static int 6923 ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 6924 { 6925 uint64_t ns; 6926 struct ixgbe_adapter *adapter = 6927 (struct ixgbe_adapter *)dev->data->dev_private; 6928 6929 ns = rte_timespec_to_ns(ts); 6930 /* Set the timecounters to a new value. */ 6931 adapter->systime_tc.nsec = ns; 6932 adapter->rx_tstamp_tc.nsec = ns; 6933 adapter->tx_tstamp_tc.nsec = ns; 6934 6935 return 0; 6936 } 6937 6938 static int 6939 ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 6940 { 6941 uint64_t ns, systime_cycles; 6942 struct ixgbe_adapter *adapter = 6943 (struct ixgbe_adapter *)dev->data->dev_private; 6944 6945 systime_cycles = ixgbe_read_systime_cyclecounter(dev); 6946 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 6947 *ts = rte_ns_to_timespec(ns); 6948 6949 return 0; 6950 } 6951 6952 static int 6953 ixgbe_timesync_enable(struct rte_eth_dev *dev) 6954 { 6955 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6956 uint32_t tsync_ctl; 6957 uint32_t tsauxc; 6958 6959 /* Stop the timesync system time. */ 6960 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0x0); 6961 /* Reset the timesync system time value. */ 6962 IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x0); 6963 IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x0); 6964 6965 /* Enable system time for platforms where it isn't on by default. */ 6966 tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); 6967 tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME; 6968 IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc); 6969 6970 ixgbe_start_timecounters(dev); 6971 6972 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 6973 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 6974 (ETHER_TYPE_1588 | 6975 IXGBE_ETQF_FILTER_EN | 6976 IXGBE_ETQF_1588)); 6977 6978 /* Enable timestamping of received PTP packets. */ 6979 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 6980 tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED; 6981 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 6982 6983 /* Enable timestamping of transmitted PTP packets. */ 6984 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 6985 tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED; 6986 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 6987 6988 IXGBE_WRITE_FLUSH(hw); 6989 6990 return 0; 6991 } 6992 6993 static int 6994 ixgbe_timesync_disable(struct rte_eth_dev *dev) 6995 { 6996 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6997 uint32_t tsync_ctl; 6998 6999 /* Disable timestamping of transmitted PTP packets. */ 7000 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7001 tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED; 7002 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7003 7004 /* Disable timestamping of received PTP packets. */ 7005 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7006 tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED; 7007 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7008 7009 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7010 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0); 7011 7012 /* Stop incrementating the System Time registers. */ 7013 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0); 7014 7015 return 0; 7016 } 7017 7018 static int 7019 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 7020 struct timespec *timestamp, 7021 uint32_t flags __rte_unused) 7022 { 7023 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7024 struct ixgbe_adapter *adapter = 7025 (struct ixgbe_adapter *)dev->data->dev_private; 7026 uint32_t tsync_rxctl; 7027 uint64_t rx_tstamp_cycles; 7028 uint64_t ns; 7029 7030 tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7031 if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0) 7032 return -EINVAL; 7033 7034 rx_tstamp_cycles = ixgbe_read_rx_tstamp_cyclecounter(dev); 7035 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 7036 *timestamp = rte_ns_to_timespec(ns); 7037 7038 return 0; 7039 } 7040 7041 static int 7042 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 7043 struct timespec *timestamp) 7044 { 7045 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7046 struct ixgbe_adapter *adapter = 7047 (struct ixgbe_adapter *)dev->data->dev_private; 7048 uint32_t tsync_txctl; 7049 uint64_t tx_tstamp_cycles; 7050 uint64_t ns; 7051 7052 tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7053 if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0) 7054 return -EINVAL; 7055 7056 tx_tstamp_cycles = ixgbe_read_tx_tstamp_cyclecounter(dev); 7057 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 7058 *timestamp = rte_ns_to_timespec(ns); 7059 7060 return 0; 7061 } 7062 7063 static int 7064 ixgbe_get_reg_length(struct rte_eth_dev *dev) 7065 { 7066 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7067 int count = 0; 7068 int g_ind = 0; 7069 const struct reg_info *reg_group; 7070 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7071 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7072 7073 while ((reg_group = reg_set[g_ind++])) 7074 count += ixgbe_regs_group_count(reg_group); 7075 7076 return count; 7077 } 7078 7079 static int 7080 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 7081 { 7082 int count = 0; 7083 int g_ind = 0; 7084 const struct reg_info *reg_group; 7085 7086 while ((reg_group = ixgbevf_regs[g_ind++])) 7087 count += ixgbe_regs_group_count(reg_group); 7088 7089 return count; 7090 } 7091 7092 static int 7093 ixgbe_get_regs(struct rte_eth_dev *dev, 7094 struct rte_dev_reg_info *regs) 7095 { 7096 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7097 uint32_t *data = regs->data; 7098 int g_ind = 0; 7099 int count = 0; 7100 const struct reg_info *reg_group; 7101 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7102 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7103 7104 if (data == NULL) { 7105 regs->length = ixgbe_get_reg_length(dev); 7106 regs->width = sizeof(uint32_t); 7107 return 0; 7108 } 7109 7110 /* Support only full register dump */ 7111 if ((regs->length == 0) || 7112 (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) { 7113 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7114 hw->device_id; 7115 while ((reg_group = reg_set[g_ind++])) 7116 count += ixgbe_read_regs_group(dev, &data[count], 7117 reg_group); 7118 return 0; 7119 } 7120 7121 return -ENOTSUP; 7122 } 7123 7124 static int 7125 ixgbevf_get_regs(struct rte_eth_dev *dev, 7126 struct rte_dev_reg_info *regs) 7127 { 7128 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7129 uint32_t *data = regs->data; 7130 int g_ind = 0; 7131 int count = 0; 7132 const struct reg_info *reg_group; 7133 7134 if (data == NULL) { 7135 regs->length = ixgbevf_get_reg_length(dev); 7136 regs->width = sizeof(uint32_t); 7137 return 0; 7138 } 7139 7140 /* Support only full register dump */ 7141 if ((regs->length == 0) || 7142 (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) { 7143 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7144 hw->device_id; 7145 while ((reg_group = ixgbevf_regs[g_ind++])) 7146 count += ixgbe_read_regs_group(dev, &data[count], 7147 reg_group); 7148 return 0; 7149 } 7150 7151 return -ENOTSUP; 7152 } 7153 7154 static int 7155 ixgbe_get_eeprom_length(struct rte_eth_dev *dev) 7156 { 7157 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7158 7159 /* Return unit is byte count */ 7160 return hw->eeprom.word_size * 2; 7161 } 7162 7163 static int 7164 ixgbe_get_eeprom(struct rte_eth_dev *dev, 7165 struct rte_dev_eeprom_info *in_eeprom) 7166 { 7167 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7168 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7169 uint16_t *data = in_eeprom->data; 7170 int first, length; 7171 7172 first = in_eeprom->offset >> 1; 7173 length = in_eeprom->length >> 1; 7174 if ((first > hw->eeprom.word_size) || 7175 ((first + length) > hw->eeprom.word_size)) 7176 return -EINVAL; 7177 7178 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7179 7180 return eeprom->ops.read_buffer(hw, first, length, data); 7181 } 7182 7183 static int 7184 ixgbe_set_eeprom(struct rte_eth_dev *dev, 7185 struct rte_dev_eeprom_info *in_eeprom) 7186 { 7187 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7188 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7189 uint16_t *data = in_eeprom->data; 7190 int first, length; 7191 7192 first = in_eeprom->offset >> 1; 7193 length = in_eeprom->length >> 1; 7194 if ((first > hw->eeprom.word_size) || 7195 ((first + length) > hw->eeprom.word_size)) 7196 return -EINVAL; 7197 7198 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7199 7200 return eeprom->ops.write_buffer(hw, first, length, data); 7201 } 7202 7203 uint16_t 7204 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) { 7205 switch (mac_type) { 7206 case ixgbe_mac_X550: 7207 case ixgbe_mac_X550EM_x: 7208 case ixgbe_mac_X550EM_a: 7209 return ETH_RSS_RETA_SIZE_512; 7210 case ixgbe_mac_X550_vf: 7211 case ixgbe_mac_X550EM_x_vf: 7212 case ixgbe_mac_X550EM_a_vf: 7213 return ETH_RSS_RETA_SIZE_64; 7214 default: 7215 return ETH_RSS_RETA_SIZE_128; 7216 } 7217 } 7218 7219 uint32_t 7220 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) { 7221 switch (mac_type) { 7222 case ixgbe_mac_X550: 7223 case ixgbe_mac_X550EM_x: 7224 case ixgbe_mac_X550EM_a: 7225 if (reta_idx < ETH_RSS_RETA_SIZE_128) 7226 return IXGBE_RETA(reta_idx >> 2); 7227 else 7228 return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2); 7229 case ixgbe_mac_X550_vf: 7230 case ixgbe_mac_X550EM_x_vf: 7231 case ixgbe_mac_X550EM_a_vf: 7232 return IXGBE_VFRETA(reta_idx >> 2); 7233 default: 7234 return IXGBE_RETA(reta_idx >> 2); 7235 } 7236 } 7237 7238 uint32_t 7239 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) { 7240 switch (mac_type) { 7241 case ixgbe_mac_X550_vf: 7242 case ixgbe_mac_X550EM_x_vf: 7243 case ixgbe_mac_X550EM_a_vf: 7244 return IXGBE_VFMRQC; 7245 default: 7246 return IXGBE_MRQC; 7247 } 7248 } 7249 7250 uint32_t 7251 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) { 7252 switch (mac_type) { 7253 case ixgbe_mac_X550_vf: 7254 case ixgbe_mac_X550EM_x_vf: 7255 case ixgbe_mac_X550EM_a_vf: 7256 return IXGBE_VFRSSRK(i); 7257 default: 7258 return IXGBE_RSSRK(i); 7259 } 7260 } 7261 7262 bool 7263 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) { 7264 switch (mac_type) { 7265 case ixgbe_mac_82599_vf: 7266 case ixgbe_mac_X540_vf: 7267 return 0; 7268 default: 7269 return 1; 7270 } 7271 } 7272 7273 static int 7274 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 7275 struct rte_eth_dcb_info *dcb_info) 7276 { 7277 struct ixgbe_dcb_config *dcb_config = 7278 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 7279 struct ixgbe_dcb_tc_config *tc; 7280 struct rte_eth_dcb_tc_queue_mapping *tc_queue; 7281 uint8_t nb_tcs; 7282 uint8_t i, j; 7283 7284 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) 7285 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs; 7286 else 7287 dcb_info->nb_tcs = 1; 7288 7289 tc_queue = &dcb_info->tc_queue; 7290 nb_tcs = dcb_info->nb_tcs; 7291 7292 if (dcb_config->vt_mode) { /* vt is enabled*/ 7293 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 7294 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 7295 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7296 dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i]; 7297 if (RTE_ETH_DEV_SRIOV(dev).active > 0) { 7298 for (j = 0; j < nb_tcs; j++) { 7299 tc_queue->tc_rxq[0][j].base = j; 7300 tc_queue->tc_rxq[0][j].nb_queue = 1; 7301 tc_queue->tc_txq[0][j].base = j; 7302 tc_queue->tc_txq[0][j].nb_queue = 1; 7303 } 7304 } else { 7305 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { 7306 for (j = 0; j < nb_tcs; j++) { 7307 tc_queue->tc_rxq[i][j].base = 7308 i * nb_tcs + j; 7309 tc_queue->tc_rxq[i][j].nb_queue = 1; 7310 tc_queue->tc_txq[i][j].base = 7311 i * nb_tcs + j; 7312 tc_queue->tc_txq[i][j].nb_queue = 1; 7313 } 7314 } 7315 } 7316 } else { /* vt is disabled*/ 7317 struct rte_eth_dcb_rx_conf *rx_conf = 7318 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 7319 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7320 dcb_info->prio_tc[i] = rx_conf->dcb_tc[i]; 7321 if (dcb_info->nb_tcs == ETH_4_TCS) { 7322 for (i = 0; i < dcb_info->nb_tcs; i++) { 7323 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32; 7324 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7325 } 7326 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7327 dcb_info->tc_queue.tc_txq[0][1].base = 64; 7328 dcb_info->tc_queue.tc_txq[0][2].base = 96; 7329 dcb_info->tc_queue.tc_txq[0][3].base = 112; 7330 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64; 7331 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7332 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7333 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7334 } else if (dcb_info->nb_tcs == ETH_8_TCS) { 7335 for (i = 0; i < dcb_info->nb_tcs; i++) { 7336 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16; 7337 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7338 } 7339 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7340 dcb_info->tc_queue.tc_txq[0][1].base = 32; 7341 dcb_info->tc_queue.tc_txq[0][2].base = 64; 7342 dcb_info->tc_queue.tc_txq[0][3].base = 80; 7343 dcb_info->tc_queue.tc_txq[0][4].base = 96; 7344 dcb_info->tc_queue.tc_txq[0][5].base = 104; 7345 dcb_info->tc_queue.tc_txq[0][6].base = 112; 7346 dcb_info->tc_queue.tc_txq[0][7].base = 120; 7347 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32; 7348 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7349 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7350 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7351 dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8; 7352 dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8; 7353 dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8; 7354 dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8; 7355 } 7356 } 7357 for (i = 0; i < dcb_info->nb_tcs; i++) { 7358 tc = &dcb_config->tc_config[i]; 7359 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent; 7360 } 7361 return 0; 7362 } 7363 7364 /* Update e-tag ether type */ 7365 static int 7366 ixgbe_update_e_tag_eth_type(struct ixgbe_hw *hw, 7367 uint16_t ether_type) 7368 { 7369 uint32_t etag_etype; 7370 7371 if (hw->mac.type != ixgbe_mac_X550 && 7372 hw->mac.type != ixgbe_mac_X550EM_x && 7373 hw->mac.type != ixgbe_mac_X550EM_a) { 7374 return -ENOTSUP; 7375 } 7376 7377 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7378 etag_etype &= ~IXGBE_ETAG_ETYPE_MASK; 7379 etag_etype |= ether_type; 7380 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7381 IXGBE_WRITE_FLUSH(hw); 7382 7383 return 0; 7384 } 7385 7386 /* Config l2 tunnel ether type */ 7387 static int 7388 ixgbe_dev_l2_tunnel_eth_type_conf(struct rte_eth_dev *dev, 7389 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7390 { 7391 int ret = 0; 7392 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7393 struct ixgbe_l2_tn_info *l2_tn_info = 7394 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7395 7396 if (l2_tunnel == NULL) 7397 return -EINVAL; 7398 7399 switch (l2_tunnel->l2_tunnel_type) { 7400 case RTE_L2_TUNNEL_TYPE_E_TAG: 7401 l2_tn_info->e_tag_ether_type = l2_tunnel->ether_type; 7402 ret = ixgbe_update_e_tag_eth_type(hw, l2_tunnel->ether_type); 7403 break; 7404 default: 7405 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7406 ret = -EINVAL; 7407 break; 7408 } 7409 7410 return ret; 7411 } 7412 7413 /* Enable e-tag tunnel */ 7414 static int 7415 ixgbe_e_tag_enable(struct ixgbe_hw *hw) 7416 { 7417 uint32_t etag_etype; 7418 7419 if (hw->mac.type != ixgbe_mac_X550 && 7420 hw->mac.type != ixgbe_mac_X550EM_x && 7421 hw->mac.type != ixgbe_mac_X550EM_a) { 7422 return -ENOTSUP; 7423 } 7424 7425 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7426 etag_etype |= IXGBE_ETAG_ETYPE_VALID; 7427 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7428 IXGBE_WRITE_FLUSH(hw); 7429 7430 return 0; 7431 } 7432 7433 /* Enable l2 tunnel */ 7434 static int 7435 ixgbe_dev_l2_tunnel_enable(struct rte_eth_dev *dev, 7436 enum rte_eth_tunnel_type l2_tunnel_type) 7437 { 7438 int ret = 0; 7439 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7440 struct ixgbe_l2_tn_info *l2_tn_info = 7441 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7442 7443 switch (l2_tunnel_type) { 7444 case RTE_L2_TUNNEL_TYPE_E_TAG: 7445 l2_tn_info->e_tag_en = TRUE; 7446 ret = ixgbe_e_tag_enable(hw); 7447 break; 7448 default: 7449 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7450 ret = -EINVAL; 7451 break; 7452 } 7453 7454 return ret; 7455 } 7456 7457 /* Disable e-tag tunnel */ 7458 static int 7459 ixgbe_e_tag_disable(struct ixgbe_hw *hw) 7460 { 7461 uint32_t etag_etype; 7462 7463 if (hw->mac.type != ixgbe_mac_X550 && 7464 hw->mac.type != ixgbe_mac_X550EM_x && 7465 hw->mac.type != ixgbe_mac_X550EM_a) { 7466 return -ENOTSUP; 7467 } 7468 7469 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7470 etag_etype &= ~IXGBE_ETAG_ETYPE_VALID; 7471 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7472 IXGBE_WRITE_FLUSH(hw); 7473 7474 return 0; 7475 } 7476 7477 /* Disable l2 tunnel */ 7478 static int 7479 ixgbe_dev_l2_tunnel_disable(struct rte_eth_dev *dev, 7480 enum rte_eth_tunnel_type l2_tunnel_type) 7481 { 7482 int ret = 0; 7483 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7484 struct ixgbe_l2_tn_info *l2_tn_info = 7485 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7486 7487 switch (l2_tunnel_type) { 7488 case RTE_L2_TUNNEL_TYPE_E_TAG: 7489 l2_tn_info->e_tag_en = FALSE; 7490 ret = ixgbe_e_tag_disable(hw); 7491 break; 7492 default: 7493 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7494 ret = -EINVAL; 7495 break; 7496 } 7497 7498 return ret; 7499 } 7500 7501 static int 7502 ixgbe_e_tag_filter_del(struct rte_eth_dev *dev, 7503 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7504 { 7505 int ret = 0; 7506 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7507 uint32_t i, rar_entries; 7508 uint32_t rar_low, rar_high; 7509 7510 if (hw->mac.type != ixgbe_mac_X550 && 7511 hw->mac.type != ixgbe_mac_X550EM_x && 7512 hw->mac.type != ixgbe_mac_X550EM_a) { 7513 return -ENOTSUP; 7514 } 7515 7516 rar_entries = ixgbe_get_num_rx_addrs(hw); 7517 7518 for (i = 1; i < rar_entries; i++) { 7519 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7520 rar_low = IXGBE_READ_REG(hw, IXGBE_RAL(i)); 7521 if ((rar_high & IXGBE_RAH_AV) && 7522 (rar_high & IXGBE_RAH_ADTYPE) && 7523 ((rar_low & IXGBE_RAL_ETAG_FILTER_MASK) == 7524 l2_tunnel->tunnel_id)) { 7525 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0); 7526 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0); 7527 7528 ixgbe_clear_vmdq(hw, i, IXGBE_CLEAR_VMDQ_ALL); 7529 7530 return ret; 7531 } 7532 } 7533 7534 return ret; 7535 } 7536 7537 static int 7538 ixgbe_e_tag_filter_add(struct rte_eth_dev *dev, 7539 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7540 { 7541 int ret = 0; 7542 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7543 uint32_t i, rar_entries; 7544 uint32_t rar_low, rar_high; 7545 7546 if (hw->mac.type != ixgbe_mac_X550 && 7547 hw->mac.type != ixgbe_mac_X550EM_x && 7548 hw->mac.type != ixgbe_mac_X550EM_a) { 7549 return -ENOTSUP; 7550 } 7551 7552 /* One entry for one tunnel. Try to remove potential existing entry. */ 7553 ixgbe_e_tag_filter_del(dev, l2_tunnel); 7554 7555 rar_entries = ixgbe_get_num_rx_addrs(hw); 7556 7557 for (i = 1; i < rar_entries; i++) { 7558 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7559 if (rar_high & IXGBE_RAH_AV) { 7560 continue; 7561 } else { 7562 ixgbe_set_vmdq(hw, i, l2_tunnel->pool); 7563 rar_high = IXGBE_RAH_AV | IXGBE_RAH_ADTYPE; 7564 rar_low = l2_tunnel->tunnel_id; 7565 7566 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), rar_low); 7567 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), rar_high); 7568 7569 return ret; 7570 } 7571 } 7572 7573 PMD_INIT_LOG(NOTICE, "The table of E-tag forwarding rule is full." 7574 " Please remove a rule before adding a new one."); 7575 return -EINVAL; 7576 } 7577 7578 static inline struct ixgbe_l2_tn_filter * 7579 ixgbe_l2_tn_filter_lookup(struct ixgbe_l2_tn_info *l2_tn_info, 7580 struct ixgbe_l2_tn_key *key) 7581 { 7582 int ret; 7583 7584 ret = rte_hash_lookup(l2_tn_info->hash_handle, (const void *)key); 7585 if (ret < 0) 7586 return NULL; 7587 7588 return l2_tn_info->hash_map[ret]; 7589 } 7590 7591 static inline int 7592 ixgbe_insert_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7593 struct ixgbe_l2_tn_filter *l2_tn_filter) 7594 { 7595 int ret; 7596 7597 ret = rte_hash_add_key(l2_tn_info->hash_handle, 7598 &l2_tn_filter->key); 7599 7600 if (ret < 0) { 7601 PMD_DRV_LOG(ERR, 7602 "Failed to insert L2 tunnel filter" 7603 " to hash table %d!", 7604 ret); 7605 return ret; 7606 } 7607 7608 l2_tn_info->hash_map[ret] = l2_tn_filter; 7609 7610 TAILQ_INSERT_TAIL(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7611 7612 return 0; 7613 } 7614 7615 static inline int 7616 ixgbe_remove_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7617 struct ixgbe_l2_tn_key *key) 7618 { 7619 int ret; 7620 struct ixgbe_l2_tn_filter *l2_tn_filter; 7621 7622 ret = rte_hash_del_key(l2_tn_info->hash_handle, key); 7623 7624 if (ret < 0) { 7625 PMD_DRV_LOG(ERR, 7626 "No such L2 tunnel filter to delete %d!", 7627 ret); 7628 return ret; 7629 } 7630 7631 l2_tn_filter = l2_tn_info->hash_map[ret]; 7632 l2_tn_info->hash_map[ret] = NULL; 7633 7634 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7635 rte_free(l2_tn_filter); 7636 7637 return 0; 7638 } 7639 7640 /* Add l2 tunnel filter */ 7641 int 7642 ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, 7643 struct rte_eth_l2_tunnel_conf *l2_tunnel, 7644 bool restore) 7645 { 7646 int ret; 7647 struct ixgbe_l2_tn_info *l2_tn_info = 7648 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7649 struct ixgbe_l2_tn_key key; 7650 struct ixgbe_l2_tn_filter *node; 7651 7652 if (!restore) { 7653 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7654 key.tn_id = l2_tunnel->tunnel_id; 7655 7656 node = ixgbe_l2_tn_filter_lookup(l2_tn_info, &key); 7657 7658 if (node) { 7659 PMD_DRV_LOG(ERR, 7660 "The L2 tunnel filter already exists!"); 7661 return -EINVAL; 7662 } 7663 7664 node = rte_zmalloc("ixgbe_l2_tn", 7665 sizeof(struct ixgbe_l2_tn_filter), 7666 0); 7667 if (!node) 7668 return -ENOMEM; 7669 7670 rte_memcpy(&node->key, 7671 &key, 7672 sizeof(struct ixgbe_l2_tn_key)); 7673 node->pool = l2_tunnel->pool; 7674 ret = ixgbe_insert_l2_tn_filter(l2_tn_info, node); 7675 if (ret < 0) { 7676 rte_free(node); 7677 return ret; 7678 } 7679 } 7680 7681 switch (l2_tunnel->l2_tunnel_type) { 7682 case RTE_L2_TUNNEL_TYPE_E_TAG: 7683 ret = ixgbe_e_tag_filter_add(dev, l2_tunnel); 7684 break; 7685 default: 7686 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7687 ret = -EINVAL; 7688 break; 7689 } 7690 7691 if ((!restore) && (ret < 0)) 7692 (void)ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7693 7694 return ret; 7695 } 7696 7697 /* Delete l2 tunnel filter */ 7698 int 7699 ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, 7700 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7701 { 7702 int ret; 7703 struct ixgbe_l2_tn_info *l2_tn_info = 7704 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7705 struct ixgbe_l2_tn_key key; 7706 7707 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7708 key.tn_id = l2_tunnel->tunnel_id; 7709 ret = ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7710 if (ret < 0) 7711 return ret; 7712 7713 switch (l2_tunnel->l2_tunnel_type) { 7714 case RTE_L2_TUNNEL_TYPE_E_TAG: 7715 ret = ixgbe_e_tag_filter_del(dev, l2_tunnel); 7716 break; 7717 default: 7718 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7719 ret = -EINVAL; 7720 break; 7721 } 7722 7723 return ret; 7724 } 7725 7726 /** 7727 * ixgbe_dev_l2_tunnel_filter_handle - Handle operations for l2 tunnel filter. 7728 * @dev: pointer to rte_eth_dev structure 7729 * @filter_op:operation will be taken. 7730 * @arg: a pointer to specific structure corresponding to the filter_op 7731 */ 7732 static int 7733 ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, 7734 enum rte_filter_op filter_op, 7735 void *arg) 7736 { 7737 int ret; 7738 7739 if (filter_op == RTE_ETH_FILTER_NOP) 7740 return 0; 7741 7742 if (arg == NULL) { 7743 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 7744 filter_op); 7745 return -EINVAL; 7746 } 7747 7748 switch (filter_op) { 7749 case RTE_ETH_FILTER_ADD: 7750 ret = ixgbe_dev_l2_tunnel_filter_add 7751 (dev, 7752 (struct rte_eth_l2_tunnel_conf *)arg, 7753 FALSE); 7754 break; 7755 case RTE_ETH_FILTER_DELETE: 7756 ret = ixgbe_dev_l2_tunnel_filter_del 7757 (dev, 7758 (struct rte_eth_l2_tunnel_conf *)arg); 7759 break; 7760 default: 7761 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 7762 ret = -EINVAL; 7763 break; 7764 } 7765 return ret; 7766 } 7767 7768 static int 7769 ixgbe_e_tag_forwarding_en_dis(struct rte_eth_dev *dev, bool en) 7770 { 7771 int ret = 0; 7772 uint32_t ctrl; 7773 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7774 7775 if (hw->mac.type != ixgbe_mac_X550 && 7776 hw->mac.type != ixgbe_mac_X550EM_x && 7777 hw->mac.type != ixgbe_mac_X550EM_a) { 7778 return -ENOTSUP; 7779 } 7780 7781 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 7782 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 7783 if (en) 7784 ctrl |= IXGBE_VT_CTL_POOLING_MODE_ETAG; 7785 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 7786 7787 return ret; 7788 } 7789 7790 /* Enable l2 tunnel forwarding */ 7791 static int 7792 ixgbe_dev_l2_tunnel_forwarding_enable 7793 (struct rte_eth_dev *dev, 7794 enum rte_eth_tunnel_type l2_tunnel_type) 7795 { 7796 struct ixgbe_l2_tn_info *l2_tn_info = 7797 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7798 int ret = 0; 7799 7800 switch (l2_tunnel_type) { 7801 case RTE_L2_TUNNEL_TYPE_E_TAG: 7802 l2_tn_info->e_tag_fwd_en = TRUE; 7803 ret = ixgbe_e_tag_forwarding_en_dis(dev, 1); 7804 break; 7805 default: 7806 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7807 ret = -EINVAL; 7808 break; 7809 } 7810 7811 return ret; 7812 } 7813 7814 /* Disable l2 tunnel forwarding */ 7815 static int 7816 ixgbe_dev_l2_tunnel_forwarding_disable 7817 (struct rte_eth_dev *dev, 7818 enum rte_eth_tunnel_type l2_tunnel_type) 7819 { 7820 struct ixgbe_l2_tn_info *l2_tn_info = 7821 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7822 int ret = 0; 7823 7824 switch (l2_tunnel_type) { 7825 case RTE_L2_TUNNEL_TYPE_E_TAG: 7826 l2_tn_info->e_tag_fwd_en = FALSE; 7827 ret = ixgbe_e_tag_forwarding_en_dis(dev, 0); 7828 break; 7829 default: 7830 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7831 ret = -EINVAL; 7832 break; 7833 } 7834 7835 return ret; 7836 } 7837 7838 static int 7839 ixgbe_e_tag_insertion_en_dis(struct rte_eth_dev *dev, 7840 struct rte_eth_l2_tunnel_conf *l2_tunnel, 7841 bool en) 7842 { 7843 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 7844 int ret = 0; 7845 uint32_t vmtir, vmvir; 7846 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7847 7848 if (l2_tunnel->vf_id >= pci_dev->max_vfs) { 7849 PMD_DRV_LOG(ERR, 7850 "VF id %u should be less than %u", 7851 l2_tunnel->vf_id, 7852 pci_dev->max_vfs); 7853 return -EINVAL; 7854 } 7855 7856 if (hw->mac.type != ixgbe_mac_X550 && 7857 hw->mac.type != ixgbe_mac_X550EM_x && 7858 hw->mac.type != ixgbe_mac_X550EM_a) { 7859 return -ENOTSUP; 7860 } 7861 7862 if (en) 7863 vmtir = l2_tunnel->tunnel_id; 7864 else 7865 vmtir = 0; 7866 7867 IXGBE_WRITE_REG(hw, IXGBE_VMTIR(l2_tunnel->vf_id), vmtir); 7868 7869 vmvir = IXGBE_READ_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id)); 7870 vmvir &= ~IXGBE_VMVIR_TAGA_MASK; 7871 if (en) 7872 vmvir |= IXGBE_VMVIR_TAGA_ETAG_INSERT; 7873 IXGBE_WRITE_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id), vmvir); 7874 7875 return ret; 7876 } 7877 7878 /* Enable l2 tunnel tag insertion */ 7879 static int 7880 ixgbe_dev_l2_tunnel_insertion_enable(struct rte_eth_dev *dev, 7881 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7882 { 7883 int ret = 0; 7884 7885 switch (l2_tunnel->l2_tunnel_type) { 7886 case RTE_L2_TUNNEL_TYPE_E_TAG: 7887 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 1); 7888 break; 7889 default: 7890 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7891 ret = -EINVAL; 7892 break; 7893 } 7894 7895 return ret; 7896 } 7897 7898 /* Disable l2 tunnel tag insertion */ 7899 static int 7900 ixgbe_dev_l2_tunnel_insertion_disable 7901 (struct rte_eth_dev *dev, 7902 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7903 { 7904 int ret = 0; 7905 7906 switch (l2_tunnel->l2_tunnel_type) { 7907 case RTE_L2_TUNNEL_TYPE_E_TAG: 7908 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 0); 7909 break; 7910 default: 7911 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7912 ret = -EINVAL; 7913 break; 7914 } 7915 7916 return ret; 7917 } 7918 7919 static int 7920 ixgbe_e_tag_stripping_en_dis(struct rte_eth_dev *dev, 7921 bool en) 7922 { 7923 int ret = 0; 7924 uint32_t qde; 7925 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7926 7927 if (hw->mac.type != ixgbe_mac_X550 && 7928 hw->mac.type != ixgbe_mac_X550EM_x && 7929 hw->mac.type != ixgbe_mac_X550EM_a) { 7930 return -ENOTSUP; 7931 } 7932 7933 qde = IXGBE_READ_REG(hw, IXGBE_QDE); 7934 if (en) 7935 qde |= IXGBE_QDE_STRIP_TAG; 7936 else 7937 qde &= ~IXGBE_QDE_STRIP_TAG; 7938 qde &= ~IXGBE_QDE_READ; 7939 qde |= IXGBE_QDE_WRITE; 7940 IXGBE_WRITE_REG(hw, IXGBE_QDE, qde); 7941 7942 return ret; 7943 } 7944 7945 /* Enable l2 tunnel tag stripping */ 7946 static int 7947 ixgbe_dev_l2_tunnel_stripping_enable 7948 (struct rte_eth_dev *dev, 7949 enum rte_eth_tunnel_type l2_tunnel_type) 7950 { 7951 int ret = 0; 7952 7953 switch (l2_tunnel_type) { 7954 case RTE_L2_TUNNEL_TYPE_E_TAG: 7955 ret = ixgbe_e_tag_stripping_en_dis(dev, 1); 7956 break; 7957 default: 7958 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7959 ret = -EINVAL; 7960 break; 7961 } 7962 7963 return ret; 7964 } 7965 7966 /* Disable l2 tunnel tag stripping */ 7967 static int 7968 ixgbe_dev_l2_tunnel_stripping_disable 7969 (struct rte_eth_dev *dev, 7970 enum rte_eth_tunnel_type l2_tunnel_type) 7971 { 7972 int ret = 0; 7973 7974 switch (l2_tunnel_type) { 7975 case RTE_L2_TUNNEL_TYPE_E_TAG: 7976 ret = ixgbe_e_tag_stripping_en_dis(dev, 0); 7977 break; 7978 default: 7979 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7980 ret = -EINVAL; 7981 break; 7982 } 7983 7984 return ret; 7985 } 7986 7987 /* Enable/disable l2 tunnel offload functions */ 7988 static int 7989 ixgbe_dev_l2_tunnel_offload_set 7990 (struct rte_eth_dev *dev, 7991 struct rte_eth_l2_tunnel_conf *l2_tunnel, 7992 uint32_t mask, 7993 uint8_t en) 7994 { 7995 int ret = 0; 7996 7997 if (l2_tunnel == NULL) 7998 return -EINVAL; 7999 8000 ret = -EINVAL; 8001 if (mask & ETH_L2_TUNNEL_ENABLE_MASK) { 8002 if (en) 8003 ret = ixgbe_dev_l2_tunnel_enable( 8004 dev, 8005 l2_tunnel->l2_tunnel_type); 8006 else 8007 ret = ixgbe_dev_l2_tunnel_disable( 8008 dev, 8009 l2_tunnel->l2_tunnel_type); 8010 } 8011 8012 if (mask & ETH_L2_TUNNEL_INSERTION_MASK) { 8013 if (en) 8014 ret = ixgbe_dev_l2_tunnel_insertion_enable( 8015 dev, 8016 l2_tunnel); 8017 else 8018 ret = ixgbe_dev_l2_tunnel_insertion_disable( 8019 dev, 8020 l2_tunnel); 8021 } 8022 8023 if (mask & ETH_L2_TUNNEL_STRIPPING_MASK) { 8024 if (en) 8025 ret = ixgbe_dev_l2_tunnel_stripping_enable( 8026 dev, 8027 l2_tunnel->l2_tunnel_type); 8028 else 8029 ret = ixgbe_dev_l2_tunnel_stripping_disable( 8030 dev, 8031 l2_tunnel->l2_tunnel_type); 8032 } 8033 8034 if (mask & ETH_L2_TUNNEL_FORWARDING_MASK) { 8035 if (en) 8036 ret = ixgbe_dev_l2_tunnel_forwarding_enable( 8037 dev, 8038 l2_tunnel->l2_tunnel_type); 8039 else 8040 ret = ixgbe_dev_l2_tunnel_forwarding_disable( 8041 dev, 8042 l2_tunnel->l2_tunnel_type); 8043 } 8044 8045 return ret; 8046 } 8047 8048 static int 8049 ixgbe_update_vxlan_port(struct ixgbe_hw *hw, 8050 uint16_t port) 8051 { 8052 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, port); 8053 IXGBE_WRITE_FLUSH(hw); 8054 8055 return 0; 8056 } 8057 8058 /* There's only one register for VxLAN UDP port. 8059 * So, we cannot add several ports. Will update it. 8060 */ 8061 static int 8062 ixgbe_add_vxlan_port(struct ixgbe_hw *hw, 8063 uint16_t port) 8064 { 8065 if (port == 0) { 8066 PMD_DRV_LOG(ERR, "Add VxLAN port 0 is not allowed."); 8067 return -EINVAL; 8068 } 8069 8070 return ixgbe_update_vxlan_port(hw, port); 8071 } 8072 8073 /* We cannot delete the VxLAN port. For there's a register for VxLAN 8074 * UDP port, it must have a value. 8075 * So, will reset it to the original value 0. 8076 */ 8077 static int 8078 ixgbe_del_vxlan_port(struct ixgbe_hw *hw, 8079 uint16_t port) 8080 { 8081 uint16_t cur_port; 8082 8083 cur_port = (uint16_t)IXGBE_READ_REG(hw, IXGBE_VXLANCTRL); 8084 8085 if (cur_port != port) { 8086 PMD_DRV_LOG(ERR, "Port %u does not exist.", port); 8087 return -EINVAL; 8088 } 8089 8090 return ixgbe_update_vxlan_port(hw, 0); 8091 } 8092 8093 /* Add UDP tunneling port */ 8094 static int 8095 ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 8096 struct rte_eth_udp_tunnel *udp_tunnel) 8097 { 8098 int ret = 0; 8099 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8100 8101 if (hw->mac.type != ixgbe_mac_X550 && 8102 hw->mac.type != ixgbe_mac_X550EM_x && 8103 hw->mac.type != ixgbe_mac_X550EM_a) { 8104 return -ENOTSUP; 8105 } 8106 8107 if (udp_tunnel == NULL) 8108 return -EINVAL; 8109 8110 switch (udp_tunnel->prot_type) { 8111 case RTE_TUNNEL_TYPE_VXLAN: 8112 ret = ixgbe_add_vxlan_port(hw, udp_tunnel->udp_port); 8113 break; 8114 8115 case RTE_TUNNEL_TYPE_GENEVE: 8116 case RTE_TUNNEL_TYPE_TEREDO: 8117 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 8118 ret = -EINVAL; 8119 break; 8120 8121 default: 8122 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8123 ret = -EINVAL; 8124 break; 8125 } 8126 8127 return ret; 8128 } 8129 8130 /* Remove UDP tunneling port */ 8131 static int 8132 ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 8133 struct rte_eth_udp_tunnel *udp_tunnel) 8134 { 8135 int ret = 0; 8136 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8137 8138 if (hw->mac.type != ixgbe_mac_X550 && 8139 hw->mac.type != ixgbe_mac_X550EM_x && 8140 hw->mac.type != ixgbe_mac_X550EM_a) { 8141 return -ENOTSUP; 8142 } 8143 8144 if (udp_tunnel == NULL) 8145 return -EINVAL; 8146 8147 switch (udp_tunnel->prot_type) { 8148 case RTE_TUNNEL_TYPE_VXLAN: 8149 ret = ixgbe_del_vxlan_port(hw, udp_tunnel->udp_port); 8150 break; 8151 case RTE_TUNNEL_TYPE_GENEVE: 8152 case RTE_TUNNEL_TYPE_TEREDO: 8153 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 8154 ret = -EINVAL; 8155 break; 8156 default: 8157 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8158 ret = -EINVAL; 8159 break; 8160 } 8161 8162 return ret; 8163 } 8164 8165 static void 8166 ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) 8167 { 8168 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8169 8170 hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_ALLMULTI); 8171 } 8172 8173 static void 8174 ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) 8175 { 8176 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8177 8178 hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 8179 } 8180 8181 static void ixgbevf_mbx_process(struct rte_eth_dev *dev) 8182 { 8183 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8184 u32 in_msg = 0; 8185 8186 if (ixgbe_read_mbx(hw, &in_msg, 1, 0)) 8187 return; 8188 8189 /* PF reset VF event */ 8190 if (in_msg == IXGBE_PF_CONTROL_MSG) 8191 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 8192 NULL, NULL); 8193 } 8194 8195 static int 8196 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev) 8197 { 8198 uint32_t eicr; 8199 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8200 struct ixgbe_interrupt *intr = 8201 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8202 ixgbevf_intr_disable(hw); 8203 8204 /* read-on-clear nic registers here */ 8205 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR); 8206 intr->flags = 0; 8207 8208 /* only one misc vector supported - mailbox */ 8209 eicr &= IXGBE_VTEICR_MASK; 8210 if (eicr == IXGBE_MISC_VEC_ID) 8211 intr->flags |= IXGBE_FLAG_MAILBOX; 8212 8213 return 0; 8214 } 8215 8216 static int 8217 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev) 8218 { 8219 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8220 struct ixgbe_interrupt *intr = 8221 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8222 8223 if (intr->flags & IXGBE_FLAG_MAILBOX) { 8224 ixgbevf_mbx_process(dev); 8225 intr->flags &= ~IXGBE_FLAG_MAILBOX; 8226 } 8227 8228 ixgbevf_intr_enable(hw); 8229 8230 return 0; 8231 } 8232 8233 static void 8234 ixgbevf_dev_interrupt_handler(void *param) 8235 { 8236 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 8237 8238 ixgbevf_dev_interrupt_get_status(dev); 8239 ixgbevf_dev_interrupt_action(dev); 8240 } 8241 8242 /** 8243 * ixgbe_disable_sec_tx_path_generic - Stops the transmit data path 8244 * @hw: pointer to hardware structure 8245 * 8246 * Stops the transmit data path and waits for the HW to internally empty 8247 * the Tx security block 8248 **/ 8249 int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw) 8250 { 8251 #define IXGBE_MAX_SECTX_POLL 40 8252 8253 int i; 8254 int sectxreg; 8255 8256 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8257 sectxreg |= IXGBE_SECTXCTRL_TX_DIS; 8258 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8259 for (i = 0; i < IXGBE_MAX_SECTX_POLL; i++) { 8260 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT); 8261 if (sectxreg & IXGBE_SECTXSTAT_SECTX_RDY) 8262 break; 8263 /* Use interrupt-safe sleep just in case */ 8264 usec_delay(1000); 8265 } 8266 8267 /* For informational purposes only */ 8268 if (i >= IXGBE_MAX_SECTX_POLL) 8269 PMD_DRV_LOG(DEBUG, "Tx unit being enabled before security " 8270 "path fully disabled. Continuing with init."); 8271 8272 return IXGBE_SUCCESS; 8273 } 8274 8275 /** 8276 * ixgbe_enable_sec_tx_path_generic - Enables the transmit data path 8277 * @hw: pointer to hardware structure 8278 * 8279 * Enables the transmit data path. 8280 **/ 8281 int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw) 8282 { 8283 uint32_t sectxreg; 8284 8285 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8286 sectxreg &= ~IXGBE_SECTXCTRL_TX_DIS; 8287 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8288 IXGBE_WRITE_FLUSH(hw); 8289 8290 return IXGBE_SUCCESS; 8291 } 8292 8293 /* restore n-tuple filter */ 8294 static inline void 8295 ixgbe_ntuple_filter_restore(struct rte_eth_dev *dev) 8296 { 8297 struct ixgbe_filter_info *filter_info = 8298 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8299 struct ixgbe_5tuple_filter *node; 8300 8301 TAILQ_FOREACH(node, &filter_info->fivetuple_list, entries) { 8302 ixgbe_inject_5tuple_filter(dev, node); 8303 } 8304 } 8305 8306 /* restore ethernet type filter */ 8307 static inline void 8308 ixgbe_ethertype_filter_restore(struct rte_eth_dev *dev) 8309 { 8310 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8311 struct ixgbe_filter_info *filter_info = 8312 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8313 int i; 8314 8315 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8316 if (filter_info->ethertype_mask & (1 << i)) { 8317 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 8318 filter_info->ethertype_filters[i].etqf); 8319 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 8320 filter_info->ethertype_filters[i].etqs); 8321 IXGBE_WRITE_FLUSH(hw); 8322 } 8323 } 8324 } 8325 8326 /* restore SYN filter */ 8327 static inline void 8328 ixgbe_syn_filter_restore(struct rte_eth_dev *dev) 8329 { 8330 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8331 struct ixgbe_filter_info *filter_info = 8332 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8333 uint32_t synqf; 8334 8335 synqf = filter_info->syn_info; 8336 8337 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 8338 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 8339 IXGBE_WRITE_FLUSH(hw); 8340 } 8341 } 8342 8343 /* restore L2 tunnel filter */ 8344 static inline void 8345 ixgbe_l2_tn_filter_restore(struct rte_eth_dev *dev) 8346 { 8347 struct ixgbe_l2_tn_info *l2_tn_info = 8348 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8349 struct ixgbe_l2_tn_filter *node; 8350 struct rte_eth_l2_tunnel_conf l2_tn_conf; 8351 8352 TAILQ_FOREACH(node, &l2_tn_info->l2_tn_list, entries) { 8353 l2_tn_conf.l2_tunnel_type = node->key.l2_tn_type; 8354 l2_tn_conf.tunnel_id = node->key.tn_id; 8355 l2_tn_conf.pool = node->pool; 8356 (void)ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_conf, TRUE); 8357 } 8358 } 8359 8360 static int 8361 ixgbe_filter_restore(struct rte_eth_dev *dev) 8362 { 8363 ixgbe_ntuple_filter_restore(dev); 8364 ixgbe_ethertype_filter_restore(dev); 8365 ixgbe_syn_filter_restore(dev); 8366 ixgbe_fdir_filter_restore(dev); 8367 ixgbe_l2_tn_filter_restore(dev); 8368 8369 return 0; 8370 } 8371 8372 static void 8373 ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev) 8374 { 8375 struct ixgbe_l2_tn_info *l2_tn_info = 8376 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8377 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8378 8379 if (l2_tn_info->e_tag_en) 8380 (void)ixgbe_e_tag_enable(hw); 8381 8382 if (l2_tn_info->e_tag_fwd_en) 8383 (void)ixgbe_e_tag_forwarding_en_dis(dev, 1); 8384 8385 (void)ixgbe_update_e_tag_eth_type(hw, l2_tn_info->e_tag_ether_type); 8386 } 8387 8388 /* remove all the n-tuple filters */ 8389 void 8390 ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev) 8391 { 8392 struct ixgbe_filter_info *filter_info = 8393 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8394 struct ixgbe_5tuple_filter *p_5tuple; 8395 8396 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) 8397 ixgbe_remove_5tuple_filter(dev, p_5tuple); 8398 } 8399 8400 /* remove all the ether type filters */ 8401 void 8402 ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev) 8403 { 8404 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8405 struct ixgbe_filter_info *filter_info = 8406 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8407 int i; 8408 8409 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8410 if (filter_info->ethertype_mask & (1 << i) && 8411 !filter_info->ethertype_filters[i].conf) { 8412 (void)ixgbe_ethertype_filter_remove(filter_info, 8413 (uint8_t)i); 8414 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 0); 8415 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 0); 8416 IXGBE_WRITE_FLUSH(hw); 8417 } 8418 } 8419 } 8420 8421 /* remove the SYN filter */ 8422 void 8423 ixgbe_clear_syn_filter(struct rte_eth_dev *dev) 8424 { 8425 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8426 struct ixgbe_filter_info *filter_info = 8427 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8428 8429 if (filter_info->syn_info & IXGBE_SYN_FILTER_ENABLE) { 8430 filter_info->syn_info = 0; 8431 8432 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, 0); 8433 IXGBE_WRITE_FLUSH(hw); 8434 } 8435 } 8436 8437 /* remove all the L2 tunnel filters */ 8438 int 8439 ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev) 8440 { 8441 struct ixgbe_l2_tn_info *l2_tn_info = 8442 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8443 struct ixgbe_l2_tn_filter *l2_tn_filter; 8444 struct rte_eth_l2_tunnel_conf l2_tn_conf; 8445 int ret = 0; 8446 8447 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 8448 l2_tn_conf.l2_tunnel_type = l2_tn_filter->key.l2_tn_type; 8449 l2_tn_conf.tunnel_id = l2_tn_filter->key.tn_id; 8450 l2_tn_conf.pool = l2_tn_filter->pool; 8451 ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_conf); 8452 if (ret < 0) 8453 return ret; 8454 } 8455 8456 return 0; 8457 } 8458 8459 RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 8460 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe, pci_id_ixgbe_map); 8461 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe, "* igb_uio | uio_pci_generic | vfio-pci"); 8462 RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd); 8463 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map); 8464 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci"); 8465