1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <netinet/in.h> 14 #include <rte_string_fns.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <rte_bus_pci.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_kvargs.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <rte_ethdev_driver.h> 31 #include <rte_ethdev_pci.h> 32 #include <rte_malloc.h> 33 #include <rte_random.h> 34 #include <rte_dev.h> 35 #include <rte_hash_crc.h> 36 #ifdef RTE_LIBRTE_SECURITY 37 #include <rte_security_driver.h> 38 #endif 39 40 #include "ixgbe_logs.h" 41 #include "base/ixgbe_api.h" 42 #include "base/ixgbe_vf.h" 43 #include "base/ixgbe_common.h" 44 #include "ixgbe_ethdev.h" 45 #include "ixgbe_bypass.h" 46 #include "ixgbe_rxtx.h" 47 #include "base/ixgbe_type.h" 48 #include "base/ixgbe_phy.h" 49 #include "ixgbe_regs.h" 50 51 /* 52 * High threshold controlling when to start sending XOFF frames. Must be at 53 * least 8 bytes less than receive packet buffer size. This value is in units 54 * of 1024 bytes. 55 */ 56 #define IXGBE_FC_HI 0x80 57 58 /* 59 * Low threshold controlling when to start sending XON frames. This value is 60 * in units of 1024 bytes. 61 */ 62 #define IXGBE_FC_LO 0x40 63 64 /* Timer value included in XOFF frames. */ 65 #define IXGBE_FC_PAUSE 0x680 66 67 /*Default value of Max Rx Queue*/ 68 #define IXGBE_MAX_RX_QUEUE_NUM 128 69 70 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */ 71 #define IXGBE_LINK_UP_CHECK_TIMEOUT 1000 /* ms */ 72 #define IXGBE_VMDQ_NUM_UC_MAC 4096 /* Maximum nb. of UC MAC addr. */ 73 74 #define IXGBE_MMW_SIZE_DEFAULT 0x4 75 #define IXGBE_MMW_SIZE_JUMBO_FRAME 0x14 76 #define IXGBE_MAX_RING_DESC 4096 /* replicate define from rxtx */ 77 78 /* 79 * Default values for RX/TX configuration 80 */ 81 #define IXGBE_DEFAULT_RX_FREE_THRESH 32 82 #define IXGBE_DEFAULT_RX_PTHRESH 8 83 #define IXGBE_DEFAULT_RX_HTHRESH 8 84 #define IXGBE_DEFAULT_RX_WTHRESH 0 85 86 #define IXGBE_DEFAULT_TX_FREE_THRESH 32 87 #define IXGBE_DEFAULT_TX_PTHRESH 32 88 #define IXGBE_DEFAULT_TX_HTHRESH 0 89 #define IXGBE_DEFAULT_TX_WTHRESH 0 90 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32 91 92 /* Bit shift and mask */ 93 #define IXGBE_4_BIT_WIDTH (CHAR_BIT / 2) 94 #define IXGBE_4_BIT_MASK RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t) 95 #define IXGBE_8_BIT_WIDTH CHAR_BIT 96 #define IXGBE_8_BIT_MASK UINT8_MAX 97 98 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */ 99 100 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0])) 101 102 /* Additional timesync values. */ 103 #define NSEC_PER_SEC 1000000000L 104 #define IXGBE_INCVAL_10GB 0x66666666 105 #define IXGBE_INCVAL_1GB 0x40000000 106 #define IXGBE_INCVAL_100 0x50000000 107 #define IXGBE_INCVAL_SHIFT_10GB 28 108 #define IXGBE_INCVAL_SHIFT_1GB 24 109 #define IXGBE_INCVAL_SHIFT_100 21 110 #define IXGBE_INCVAL_SHIFT_82599 7 111 #define IXGBE_INCPER_SHIFT_82599 24 112 113 #define IXGBE_CYCLECOUNTER_MASK 0xffffffffffffffffULL 114 115 #define IXGBE_VT_CTL_POOLING_MODE_MASK 0x00030000 116 #define IXGBE_VT_CTL_POOLING_MODE_ETAG 0x00010000 117 #define IXGBE_ETAG_ETYPE 0x00005084 118 #define IXGBE_ETAG_ETYPE_MASK 0x0000ffff 119 #define IXGBE_ETAG_ETYPE_VALID 0x80000000 120 #define IXGBE_RAH_ADTYPE 0x40000000 121 #define IXGBE_RAL_ETAG_FILTER_MASK 0x00003fff 122 #define IXGBE_VMVIR_TAGA_MASK 0x18000000 123 #define IXGBE_VMVIR_TAGA_ETAG_INSERT 0x08000000 124 #define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */ 125 #define IXGBE_QDE_STRIP_TAG 0x00000004 126 #define IXGBE_VTEICR_MASK 0x07 127 128 #define IXGBE_EXVET_VET_EXT_SHIFT 16 129 #define IXGBE_DMATXCTL_VT_MASK 0xFFFF0000 130 131 #define IXGBEVF_DEVARG_PFLINK_FULLCHK "pflink_fullchk" 132 133 static const char * const ixgbevf_valid_arguments[] = { 134 IXGBEVF_DEVARG_PFLINK_FULLCHK, 135 NULL 136 }; 137 138 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params); 139 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev); 140 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev); 141 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev); 142 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev); 143 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev); 144 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev); 145 static int ixgbe_dev_configure(struct rte_eth_dev *dev); 146 static int ixgbe_dev_start(struct rte_eth_dev *dev); 147 static void ixgbe_dev_stop(struct rte_eth_dev *dev); 148 static int ixgbe_dev_set_link_up(struct rte_eth_dev *dev); 149 static int ixgbe_dev_set_link_down(struct rte_eth_dev *dev); 150 static void ixgbe_dev_close(struct rte_eth_dev *dev); 151 static int ixgbe_dev_reset(struct rte_eth_dev *dev); 152 static int ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev); 153 static int ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev); 154 static int ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev); 155 static int ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev); 156 static int ixgbe_dev_link_update(struct rte_eth_dev *dev, 157 int wait_to_complete); 158 static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, 159 struct rte_eth_stats *stats); 160 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev, 161 struct rte_eth_xstat *xstats, unsigned n); 162 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, 163 struct rte_eth_xstat *xstats, unsigned n); 164 static int 165 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 166 uint64_t *values, unsigned int n); 167 static int ixgbe_dev_stats_reset(struct rte_eth_dev *dev); 168 static int ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); 169 static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, 170 struct rte_eth_xstat_name *xstats_names, 171 unsigned int size); 172 static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, 173 struct rte_eth_xstat_name *xstats_names, unsigned limit); 174 static int ixgbe_dev_xstats_get_names_by_id( 175 struct rte_eth_dev *dev, 176 struct rte_eth_xstat_name *xstats_names, 177 const uint64_t *ids, 178 unsigned int limit); 179 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 180 uint16_t queue_id, 181 uint8_t stat_idx, 182 uint8_t is_rx); 183 static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, 184 size_t fw_size); 185 static int ixgbe_dev_info_get(struct rte_eth_dev *dev, 186 struct rte_eth_dev_info *dev_info); 187 static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev); 188 static int ixgbevf_dev_info_get(struct rte_eth_dev *dev, 189 struct rte_eth_dev_info *dev_info); 190 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 191 192 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev, 193 uint16_t vlan_id, int on); 194 static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 195 enum rte_vlan_type vlan_type, 196 uint16_t tpid_id); 197 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, 198 uint16_t queue, bool on); 199 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, 200 int on); 201 static void ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, 202 int mask); 203 static int ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask); 204 static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); 205 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue); 206 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue); 207 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev); 208 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev); 209 210 static int ixgbe_dev_led_on(struct rte_eth_dev *dev); 211 static int ixgbe_dev_led_off(struct rte_eth_dev *dev); 212 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, 213 struct rte_eth_fc_conf *fc_conf); 214 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, 215 struct rte_eth_fc_conf *fc_conf); 216 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, 217 struct rte_eth_pfc_conf *pfc_conf); 218 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 219 struct rte_eth_rss_reta_entry64 *reta_conf, 220 uint16_t reta_size); 221 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 222 struct rte_eth_rss_reta_entry64 *reta_conf, 223 uint16_t reta_size); 224 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); 225 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); 226 static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); 227 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); 228 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); 229 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev); 230 static void ixgbe_dev_interrupt_handler(void *param); 231 static void ixgbe_dev_interrupt_delayed_handler(void *param); 232 static void ixgbe_dev_setup_link_alarm_handler(void *param); 233 234 static int ixgbe_add_rar(struct rte_eth_dev *dev, 235 struct rte_ether_addr *mac_addr, 236 uint32_t index, uint32_t pool); 237 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index); 238 static int ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, 239 struct rte_ether_addr *mac_addr); 240 static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config); 241 static bool is_device_supported(struct rte_eth_dev *dev, 242 struct rte_pci_driver *drv); 243 244 /* For Virtual Function support */ 245 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); 246 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); 247 static int ixgbevf_dev_configure(struct rte_eth_dev *dev); 248 static int ixgbevf_dev_start(struct rte_eth_dev *dev); 249 static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, 250 int wait_to_complete); 251 static void ixgbevf_dev_stop(struct rte_eth_dev *dev); 252 static void ixgbevf_dev_close(struct rte_eth_dev *dev); 253 static int ixgbevf_dev_reset(struct rte_eth_dev *dev); 254 static void ixgbevf_intr_disable(struct rte_eth_dev *dev); 255 static void ixgbevf_intr_enable(struct rte_eth_dev *dev); 256 static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, 257 struct rte_eth_stats *stats); 258 static int ixgbevf_dev_stats_reset(struct rte_eth_dev *dev); 259 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, 260 uint16_t vlan_id, int on); 261 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, 262 uint16_t queue, int on); 263 static int ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask); 264 static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); 265 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on); 266 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 267 uint16_t queue_id); 268 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 269 uint16_t queue_id); 270 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 271 uint8_t queue, uint8_t msix_vector); 272 static void ixgbevf_configure_msix(struct rte_eth_dev *dev); 273 static int ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev); 274 static int ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev); 275 static int ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev); 276 static int ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev); 277 278 /* For Eth VMDQ APIs support */ 279 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct 280 rte_ether_addr * mac_addr, uint8_t on); 281 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on); 282 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 283 struct rte_eth_mirror_conf *mirror_conf, 284 uint8_t rule_id, uint8_t on); 285 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, 286 uint8_t rule_id); 287 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 288 uint16_t queue_id); 289 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 290 uint16_t queue_id); 291 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 292 uint8_t queue, uint8_t msix_vector); 293 static void ixgbe_configure_msix(struct rte_eth_dev *dev); 294 295 static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, 296 struct rte_ether_addr *mac_addr, 297 uint32_t index, uint32_t pool); 298 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index); 299 static int ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 300 struct rte_ether_addr *mac_addr); 301 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev, 302 struct rte_eth_syn_filter *filter); 303 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev, 304 enum rte_filter_op filter_op, 305 void *arg); 306 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 307 struct ixgbe_5tuple_filter *filter); 308 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 309 struct ixgbe_5tuple_filter *filter); 310 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, 311 enum rte_filter_op filter_op, 312 void *arg); 313 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev, 314 struct rte_eth_ntuple_filter *filter); 315 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev, 316 enum rte_filter_op filter_op, 317 void *arg); 318 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev, 319 struct rte_eth_ethertype_filter *filter); 320 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, 321 enum rte_filter_type filter_type, 322 enum rte_filter_op filter_op, 323 void *arg); 324 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); 325 326 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 327 struct rte_ether_addr *mc_addr_set, 328 uint32_t nb_mc_addr); 329 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 330 struct rte_eth_dcb_info *dcb_info); 331 332 static int ixgbe_get_reg_length(struct rte_eth_dev *dev); 333 static int ixgbe_get_regs(struct rte_eth_dev *dev, 334 struct rte_dev_reg_info *regs); 335 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev); 336 static int ixgbe_get_eeprom(struct rte_eth_dev *dev, 337 struct rte_dev_eeprom_info *eeprom); 338 static int ixgbe_set_eeprom(struct rte_eth_dev *dev, 339 struct rte_dev_eeprom_info *eeprom); 340 341 static int ixgbe_get_module_info(struct rte_eth_dev *dev, 342 struct rte_eth_dev_module_info *modinfo); 343 static int ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 344 struct rte_dev_eeprom_info *info); 345 346 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev); 347 static int ixgbevf_get_regs(struct rte_eth_dev *dev, 348 struct rte_dev_reg_info *regs); 349 350 static int ixgbe_timesync_enable(struct rte_eth_dev *dev); 351 static int ixgbe_timesync_disable(struct rte_eth_dev *dev); 352 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 353 struct timespec *timestamp, 354 uint32_t flags); 355 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 356 struct timespec *timestamp); 357 static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); 358 static int ixgbe_timesync_read_time(struct rte_eth_dev *dev, 359 struct timespec *timestamp); 360 static int ixgbe_timesync_write_time(struct rte_eth_dev *dev, 361 const struct timespec *timestamp); 362 static void ixgbevf_dev_interrupt_handler(void *param); 363 364 static int ixgbe_dev_l2_tunnel_eth_type_conf 365 (struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel); 366 static int ixgbe_dev_l2_tunnel_offload_set 367 (struct rte_eth_dev *dev, 368 struct rte_eth_l2_tunnel_conf *l2_tunnel, 369 uint32_t mask, 370 uint8_t en); 371 static int ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, 372 enum rte_filter_op filter_op, 373 void *arg); 374 375 static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 376 struct rte_eth_udp_tunnel *udp_tunnel); 377 static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 378 struct rte_eth_udp_tunnel *udp_tunnel); 379 static int ixgbe_filter_restore(struct rte_eth_dev *dev); 380 static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev); 381 382 /* 383 * Define VF Stats MACRO for Non "cleared on read" register 384 */ 385 #define UPDATE_VF_STAT(reg, last, cur) \ 386 { \ 387 uint32_t latest = IXGBE_READ_REG(hw, reg); \ 388 cur += (latest - last) & UINT_MAX; \ 389 last = latest; \ 390 } 391 392 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur) \ 393 { \ 394 u64 new_lsb = IXGBE_READ_REG(hw, lsb); \ 395 u64 new_msb = IXGBE_READ_REG(hw, msb); \ 396 u64 latest = ((new_msb << 32) | new_lsb); \ 397 cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \ 398 last = latest; \ 399 } 400 401 #define IXGBE_SET_HWSTRIP(h, q) do {\ 402 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 403 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 404 (h)->bitmap[idx] |= 1 << bit;\ 405 } while (0) 406 407 #define IXGBE_CLEAR_HWSTRIP(h, q) do {\ 408 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 409 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 410 (h)->bitmap[idx] &= ~(1 << bit);\ 411 } while (0) 412 413 #define IXGBE_GET_HWSTRIP(h, q, r) do {\ 414 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \ 415 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \ 416 (r) = (h)->bitmap[idx] >> bit & 1;\ 417 } while (0) 418 419 int ixgbe_logtype_init; 420 int ixgbe_logtype_driver; 421 422 #ifdef RTE_LIBRTE_IXGBE_DEBUG_RX 423 int ixgbe_logtype_rx; 424 #endif 425 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX 426 int ixgbe_logtype_tx; 427 #endif 428 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX_FREE 429 int ixgbe_logtype_tx_free; 430 #endif 431 432 /* 433 * The set of PCI devices this driver supports 434 */ 435 static const struct rte_pci_id pci_id_ixgbe_map[] = { 436 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) }, 437 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) }, 438 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) }, 439 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) }, 440 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) }, 441 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) }, 442 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) }, 443 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) }, 444 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) }, 445 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) }, 446 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) }, 447 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) }, 448 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) }, 449 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, 450 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, 451 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, 452 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, 453 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, 454 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, 455 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, 456 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, 457 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) }, 458 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) }, 459 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) }, 460 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) }, 461 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) }, 462 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) }, 463 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) }, 464 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) }, 465 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) }, 466 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) }, 467 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) }, 468 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) }, 469 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) }, 470 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) }, 471 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) }, 472 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) }, 473 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) }, 474 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) }, 475 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) }, 476 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) }, 477 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) }, 478 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) }, 479 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) }, 480 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, 481 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, 482 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, 483 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI) }, 484 #ifdef RTE_LIBRTE_IXGBE_BYPASS 485 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, 486 #endif 487 { .vendor_id = 0, /* sentinel */ }, 488 }; 489 490 /* 491 * The set of PCI devices this driver supports (for 82599 VF) 492 */ 493 static const struct rte_pci_id pci_id_ixgbevf_map[] = { 494 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) }, 495 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) }, 496 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) }, 497 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) }, 498 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) }, 499 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) }, 500 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) }, 501 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) }, 502 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) }, 503 { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) }, 504 { .vendor_id = 0, /* sentinel */ }, 505 }; 506 507 static const struct rte_eth_desc_lim rx_desc_lim = { 508 .nb_max = IXGBE_MAX_RING_DESC, 509 .nb_min = IXGBE_MIN_RING_DESC, 510 .nb_align = IXGBE_RXD_ALIGN, 511 }; 512 513 static const struct rte_eth_desc_lim tx_desc_lim = { 514 .nb_max = IXGBE_MAX_RING_DESC, 515 .nb_min = IXGBE_MIN_RING_DESC, 516 .nb_align = IXGBE_TXD_ALIGN, 517 .nb_seg_max = IXGBE_TX_MAX_SEG, 518 .nb_mtu_seg_max = IXGBE_TX_MAX_SEG, 519 }; 520 521 static const struct eth_dev_ops ixgbe_eth_dev_ops = { 522 .dev_configure = ixgbe_dev_configure, 523 .dev_start = ixgbe_dev_start, 524 .dev_stop = ixgbe_dev_stop, 525 .dev_set_link_up = ixgbe_dev_set_link_up, 526 .dev_set_link_down = ixgbe_dev_set_link_down, 527 .dev_close = ixgbe_dev_close, 528 .dev_reset = ixgbe_dev_reset, 529 .promiscuous_enable = ixgbe_dev_promiscuous_enable, 530 .promiscuous_disable = ixgbe_dev_promiscuous_disable, 531 .allmulticast_enable = ixgbe_dev_allmulticast_enable, 532 .allmulticast_disable = ixgbe_dev_allmulticast_disable, 533 .link_update = ixgbe_dev_link_update, 534 .stats_get = ixgbe_dev_stats_get, 535 .xstats_get = ixgbe_dev_xstats_get, 536 .xstats_get_by_id = ixgbe_dev_xstats_get_by_id, 537 .stats_reset = ixgbe_dev_stats_reset, 538 .xstats_reset = ixgbe_dev_xstats_reset, 539 .xstats_get_names = ixgbe_dev_xstats_get_names, 540 .xstats_get_names_by_id = ixgbe_dev_xstats_get_names_by_id, 541 .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set, 542 .fw_version_get = ixgbe_fw_version_get, 543 .dev_infos_get = ixgbe_dev_info_get, 544 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 545 .mtu_set = ixgbe_dev_mtu_set, 546 .vlan_filter_set = ixgbe_vlan_filter_set, 547 .vlan_tpid_set = ixgbe_vlan_tpid_set, 548 .vlan_offload_set = ixgbe_vlan_offload_set, 549 .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set, 550 .rx_queue_start = ixgbe_dev_rx_queue_start, 551 .rx_queue_stop = ixgbe_dev_rx_queue_stop, 552 .tx_queue_start = ixgbe_dev_tx_queue_start, 553 .tx_queue_stop = ixgbe_dev_tx_queue_stop, 554 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 555 .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable, 556 .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable, 557 .rx_queue_release = ixgbe_dev_rx_queue_release, 558 .rx_queue_count = ixgbe_dev_rx_queue_count, 559 .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, 560 .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, 561 .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, 562 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 563 .tx_queue_release = ixgbe_dev_tx_queue_release, 564 .dev_led_on = ixgbe_dev_led_on, 565 .dev_led_off = ixgbe_dev_led_off, 566 .flow_ctrl_get = ixgbe_flow_ctrl_get, 567 .flow_ctrl_set = ixgbe_flow_ctrl_set, 568 .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set, 569 .mac_addr_add = ixgbe_add_rar, 570 .mac_addr_remove = ixgbe_remove_rar, 571 .mac_addr_set = ixgbe_set_default_mac_addr, 572 .uc_hash_table_set = ixgbe_uc_hash_table_set, 573 .uc_all_hash_table_set = ixgbe_uc_all_hash_table_set, 574 .mirror_rule_set = ixgbe_mirror_rule_set, 575 .mirror_rule_reset = ixgbe_mirror_rule_reset, 576 .set_queue_rate_limit = ixgbe_set_queue_rate_limit, 577 .reta_update = ixgbe_dev_rss_reta_update, 578 .reta_query = ixgbe_dev_rss_reta_query, 579 .rss_hash_update = ixgbe_dev_rss_hash_update, 580 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 581 .filter_ctrl = ixgbe_dev_filter_ctrl, 582 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 583 .rxq_info_get = ixgbe_rxq_info_get, 584 .txq_info_get = ixgbe_txq_info_get, 585 .timesync_enable = ixgbe_timesync_enable, 586 .timesync_disable = ixgbe_timesync_disable, 587 .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, 588 .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp, 589 .get_reg = ixgbe_get_regs, 590 .get_eeprom_length = ixgbe_get_eeprom_length, 591 .get_eeprom = ixgbe_get_eeprom, 592 .set_eeprom = ixgbe_set_eeprom, 593 .get_module_info = ixgbe_get_module_info, 594 .get_module_eeprom = ixgbe_get_module_eeprom, 595 .get_dcb_info = ixgbe_dev_get_dcb_info, 596 .timesync_adjust_time = ixgbe_timesync_adjust_time, 597 .timesync_read_time = ixgbe_timesync_read_time, 598 .timesync_write_time = ixgbe_timesync_write_time, 599 .l2_tunnel_eth_type_conf = ixgbe_dev_l2_tunnel_eth_type_conf, 600 .l2_tunnel_offload_set = ixgbe_dev_l2_tunnel_offload_set, 601 .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, 602 .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, 603 .tm_ops_get = ixgbe_tm_ops_get, 604 }; 605 606 /* 607 * dev_ops for virtual function, bare necessities for basic vf 608 * operation have been implemented 609 */ 610 static const struct eth_dev_ops ixgbevf_eth_dev_ops = { 611 .dev_configure = ixgbevf_dev_configure, 612 .dev_start = ixgbevf_dev_start, 613 .dev_stop = ixgbevf_dev_stop, 614 .link_update = ixgbevf_dev_link_update, 615 .stats_get = ixgbevf_dev_stats_get, 616 .xstats_get = ixgbevf_dev_xstats_get, 617 .stats_reset = ixgbevf_dev_stats_reset, 618 .xstats_reset = ixgbevf_dev_stats_reset, 619 .xstats_get_names = ixgbevf_dev_xstats_get_names, 620 .dev_close = ixgbevf_dev_close, 621 .dev_reset = ixgbevf_dev_reset, 622 .promiscuous_enable = ixgbevf_dev_promiscuous_enable, 623 .promiscuous_disable = ixgbevf_dev_promiscuous_disable, 624 .allmulticast_enable = ixgbevf_dev_allmulticast_enable, 625 .allmulticast_disable = ixgbevf_dev_allmulticast_disable, 626 .dev_infos_get = ixgbevf_dev_info_get, 627 .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, 628 .mtu_set = ixgbevf_dev_set_mtu, 629 .vlan_filter_set = ixgbevf_vlan_filter_set, 630 .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set, 631 .vlan_offload_set = ixgbevf_vlan_offload_set, 632 .rx_queue_setup = ixgbe_dev_rx_queue_setup, 633 .rx_queue_release = ixgbe_dev_rx_queue_release, 634 .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, 635 .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, 636 .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, 637 .tx_queue_setup = ixgbe_dev_tx_queue_setup, 638 .tx_queue_release = ixgbe_dev_tx_queue_release, 639 .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable, 640 .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable, 641 .mac_addr_add = ixgbevf_add_mac_addr, 642 .mac_addr_remove = ixgbevf_remove_mac_addr, 643 .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, 644 .rxq_info_get = ixgbe_rxq_info_get, 645 .txq_info_get = ixgbe_txq_info_get, 646 .mac_addr_set = ixgbevf_set_default_mac_addr, 647 .get_reg = ixgbevf_get_regs, 648 .reta_update = ixgbe_dev_rss_reta_update, 649 .reta_query = ixgbe_dev_rss_reta_query, 650 .rss_hash_update = ixgbe_dev_rss_hash_update, 651 .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, 652 }; 653 654 /* store statistics names and its offset in stats structure */ 655 struct rte_ixgbe_xstats_name_off { 656 char name[RTE_ETH_XSTATS_NAME_SIZE]; 657 unsigned offset; 658 }; 659 660 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = { 661 {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)}, 662 {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)}, 663 {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)}, 664 {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)}, 665 {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)}, 666 {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)}, 667 {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)}, 668 {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)}, 669 {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)}, 670 {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)}, 671 {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)}, 672 {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)}, 673 {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)}, 674 {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)}, 675 {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 676 prc1023)}, 677 {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 678 prc1522)}, 679 {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)}, 680 {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)}, 681 {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)}, 682 {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)}, 683 {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)}, 684 {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)}, 685 {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)}, 686 {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)}, 687 {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)}, 688 {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)}, 689 {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)}, 690 {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)}, 691 {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)}, 692 {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)}, 693 {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)}, 694 {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)}, 695 {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats, 696 ptc1023)}, 697 {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats, 698 ptc1522)}, 699 {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)}, 700 {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)}, 701 {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)}, 702 {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)}, 703 704 {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats, 705 fdirustat_add)}, 706 {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats, 707 fdirustat_remove)}, 708 {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats, 709 fdirfstat_fadd)}, 710 {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats, 711 fdirfstat_fremove)}, 712 {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats, 713 fdirmatch)}, 714 {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats, 715 fdirmiss)}, 716 717 {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)}, 718 {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)}, 719 {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, 720 fclast)}, 721 {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)}, 722 {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)}, 723 {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)}, 724 {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)}, 725 {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats, 726 fcoe_noddp)}, 727 {"rx_fcoe_no_direct_data_placement_ext_buff", 728 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)}, 729 730 {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 731 lxontxc)}, 732 {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats, 733 lxonrxc)}, 734 {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 735 lxofftxc)}, 736 {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats, 737 lxoffrxc)}, 738 {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)}, 739 }; 740 741 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \ 742 sizeof(rte_ixgbe_stats_strings[0])) 743 744 /* MACsec statistics */ 745 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = { 746 {"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 747 out_pkts_untagged)}, 748 {"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats, 749 out_pkts_encrypted)}, 750 {"out_pkts_protected", offsetof(struct ixgbe_macsec_stats, 751 out_pkts_protected)}, 752 {"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats, 753 out_octets_encrypted)}, 754 {"out_octets_protected", offsetof(struct ixgbe_macsec_stats, 755 out_octets_protected)}, 756 {"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats, 757 in_pkts_untagged)}, 758 {"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats, 759 in_pkts_badtag)}, 760 {"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats, 761 in_pkts_nosci)}, 762 {"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats, 763 in_pkts_unknownsci)}, 764 {"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats, 765 in_octets_decrypted)}, 766 {"in_octets_validated", offsetof(struct ixgbe_macsec_stats, 767 in_octets_validated)}, 768 {"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats, 769 in_pkts_unchecked)}, 770 {"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats, 771 in_pkts_delayed)}, 772 {"in_pkts_late", offsetof(struct ixgbe_macsec_stats, 773 in_pkts_late)}, 774 {"in_pkts_ok", offsetof(struct ixgbe_macsec_stats, 775 in_pkts_ok)}, 776 {"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats, 777 in_pkts_invalid)}, 778 {"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats, 779 in_pkts_notvalid)}, 780 {"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats, 781 in_pkts_unusedsa)}, 782 {"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats, 783 in_pkts_notusingsa)}, 784 }; 785 786 #define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \ 787 sizeof(rte_ixgbe_macsec_strings[0])) 788 789 /* Per-queue statistics */ 790 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = { 791 {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)}, 792 {"dropped", offsetof(struct ixgbe_hw_stats, mpc)}, 793 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)}, 794 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)}, 795 }; 796 797 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \ 798 sizeof(rte_ixgbe_rxq_strings[0])) 799 #define IXGBE_NB_RXQ_PRIO_VALUES 8 800 801 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = { 802 {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)}, 803 {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)}, 804 {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats, 805 pxon2offc)}, 806 }; 807 808 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \ 809 sizeof(rte_ixgbe_txq_strings[0])) 810 #define IXGBE_NB_TXQ_PRIO_VALUES 8 811 812 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = { 813 {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)}, 814 }; 815 816 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) / \ 817 sizeof(rte_ixgbevf_stats_strings[0])) 818 819 /* 820 * This function is the same as ixgbe_is_sfp() in base/ixgbe.h. 821 */ 822 static inline int 823 ixgbe_is_sfp(struct ixgbe_hw *hw) 824 { 825 switch (hw->phy.type) { 826 case ixgbe_phy_sfp_avago: 827 case ixgbe_phy_sfp_ftl: 828 case ixgbe_phy_sfp_intel: 829 case ixgbe_phy_sfp_unknown: 830 case ixgbe_phy_sfp_passive_tyco: 831 case ixgbe_phy_sfp_passive_unknown: 832 return 1; 833 default: 834 return 0; 835 } 836 } 837 838 static inline int32_t 839 ixgbe_pf_reset_hw(struct ixgbe_hw *hw) 840 { 841 uint32_t ctrl_ext; 842 int32_t status; 843 844 status = ixgbe_reset_hw(hw); 845 846 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 847 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 848 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 849 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 850 IXGBE_WRITE_FLUSH(hw); 851 852 if (status == IXGBE_ERR_SFP_NOT_PRESENT) 853 status = IXGBE_SUCCESS; 854 return status; 855 } 856 857 static inline void 858 ixgbe_enable_intr(struct rte_eth_dev *dev) 859 { 860 struct ixgbe_interrupt *intr = 861 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 862 struct ixgbe_hw *hw = 863 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 864 865 IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask); 866 IXGBE_WRITE_FLUSH(hw); 867 } 868 869 /* 870 * This function is based on ixgbe_disable_intr() in base/ixgbe.h. 871 */ 872 static void 873 ixgbe_disable_intr(struct ixgbe_hw *hw) 874 { 875 PMD_INIT_FUNC_TRACE(); 876 877 if (hw->mac.type == ixgbe_mac_82598EB) { 878 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0); 879 } else { 880 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000); 881 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0); 882 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0); 883 } 884 IXGBE_WRITE_FLUSH(hw); 885 } 886 887 /* 888 * This function resets queue statistics mapping registers. 889 * From Niantic datasheet, Initialization of Statistics section: 890 * "...if software requires the queue counters, the RQSMR and TQSM registers 891 * must be re-programmed following a device reset. 892 */ 893 static void 894 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw) 895 { 896 uint32_t i; 897 898 for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) { 899 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0); 900 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0); 901 } 902 } 903 904 905 static int 906 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, 907 uint16_t queue_id, 908 uint8_t stat_idx, 909 uint8_t is_rx) 910 { 911 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8 912 #define NB_QMAP_FIELDS_PER_QSM_REG 4 913 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f 914 915 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 916 struct ixgbe_stat_mapping_registers *stat_mappings = 917 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private); 918 uint32_t qsmr_mask = 0; 919 uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK; 920 uint32_t q_map; 921 uint8_t n, offset; 922 923 if ((hw->mac.type != ixgbe_mac_82599EB) && 924 (hw->mac.type != ixgbe_mac_X540) && 925 (hw->mac.type != ixgbe_mac_X550) && 926 (hw->mac.type != ixgbe_mac_X550EM_x) && 927 (hw->mac.type != ixgbe_mac_X550EM_a)) 928 return -ENOSYS; 929 930 PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d", 931 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 932 queue_id, stat_idx); 933 934 n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG); 935 if (n >= IXGBE_NB_STAT_MAPPING_REGS) { 936 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded"); 937 return -EIO; 938 } 939 offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG); 940 941 /* Now clear any previous stat_idx set */ 942 clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 943 if (!is_rx) 944 stat_mappings->tqsm[n] &= ~clearing_mask; 945 else 946 stat_mappings->rqsmr[n] &= ~clearing_mask; 947 948 q_map = (uint32_t)stat_idx; 949 q_map &= QMAP_FIELD_RESERVED_BITS_MASK; 950 qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset); 951 if (!is_rx) 952 stat_mappings->tqsm[n] |= qsmr_mask; 953 else 954 stat_mappings->rqsmr[n] |= qsmr_mask; 955 956 PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d", 957 (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", 958 queue_id, stat_idx); 959 PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n, 960 is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]); 961 962 /* Now write the mapping in the appropriate register */ 963 if (is_rx) { 964 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d", 965 stat_mappings->rqsmr[n], n); 966 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]); 967 } else { 968 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d", 969 stat_mappings->tqsm[n], n); 970 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]); 971 } 972 return 0; 973 } 974 975 static void 976 ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev) 977 { 978 struct ixgbe_stat_mapping_registers *stat_mappings = 979 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private); 980 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 981 int i; 982 983 /* write whatever was in stat mapping table to the NIC */ 984 for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) { 985 /* rx */ 986 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]); 987 988 /* tx */ 989 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]); 990 } 991 } 992 993 static void 994 ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config) 995 { 996 uint8_t i; 997 struct ixgbe_dcb_tc_config *tc; 998 uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS; 999 1000 dcb_config->num_tcs.pg_tcs = dcb_max_tc; 1001 dcb_config->num_tcs.pfc_tcs = dcb_max_tc; 1002 for (i = 0; i < dcb_max_tc; i++) { 1003 tc = &dcb_config->tc_config[i]; 1004 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i; 1005 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 1006 (uint8_t)(100/dcb_max_tc + (i & 1)); 1007 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i; 1008 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 1009 (uint8_t)(100/dcb_max_tc + (i & 1)); 1010 tc->pfc = ixgbe_dcb_pfc_disabled; 1011 } 1012 1013 /* Initialize default user to priority mapping, UPx->TC0 */ 1014 tc = &dcb_config->tc_config[0]; 1015 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 1016 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 1017 for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) { 1018 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100; 1019 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100; 1020 } 1021 dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal; 1022 dcb_config->pfc_mode_enable = false; 1023 dcb_config->vt_mode = true; 1024 dcb_config->round_robin_enable = false; 1025 /* support all DCB capabilities in 82599 */ 1026 dcb_config->support.capabilities = 0xFF; 1027 1028 /*we only support 4 Tcs for X540, X550 */ 1029 if (hw->mac.type == ixgbe_mac_X540 || 1030 hw->mac.type == ixgbe_mac_X550 || 1031 hw->mac.type == ixgbe_mac_X550EM_x || 1032 hw->mac.type == ixgbe_mac_X550EM_a) { 1033 dcb_config->num_tcs.pg_tcs = 4; 1034 dcb_config->num_tcs.pfc_tcs = 4; 1035 } 1036 } 1037 1038 /* 1039 * Ensure that all locks are released before first NVM or PHY access 1040 */ 1041 static void 1042 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) 1043 { 1044 uint16_t mask; 1045 1046 /* 1047 * Phy lock should not fail in this early stage. If this is the case, 1048 * it is due to an improper exit of the application. 1049 * So force the release of the faulty lock. Release of common lock 1050 * is done automatically by swfw_sync function. 1051 */ 1052 mask = IXGBE_GSSR_PHY0_SM << hw->bus.func; 1053 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1054 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func); 1055 } 1056 ixgbe_release_swfw_semaphore(hw, mask); 1057 1058 /* 1059 * These ones are more tricky since they are common to all ports; but 1060 * swfw_sync retries last long enough (1s) to be almost sure that if 1061 * lock can not be taken it is due to an improper lock of the 1062 * semaphore. 1063 */ 1064 mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM; 1065 if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) { 1066 PMD_DRV_LOG(DEBUG, "SWFW common locks released"); 1067 } 1068 ixgbe_release_swfw_semaphore(hw, mask); 1069 } 1070 1071 /* 1072 * This function is based on code in ixgbe_attach() in base/ixgbe.c. 1073 * It returns 0 on success. 1074 */ 1075 static int 1076 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) 1077 { 1078 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1079 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1080 struct ixgbe_hw *hw = 1081 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1082 struct ixgbe_vfta *shadow_vfta = 1083 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1084 struct ixgbe_hwstrip *hwstrip = 1085 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1086 struct ixgbe_dcb_config *dcb_config = 1087 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private); 1088 struct ixgbe_filter_info *filter_info = 1089 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1090 struct ixgbe_bw_conf *bw_conf = 1091 IXGBE_DEV_PRIVATE_TO_BW_CONF(eth_dev->data->dev_private); 1092 uint32_t ctrl_ext; 1093 uint16_t csum; 1094 int diag, i; 1095 1096 PMD_INIT_FUNC_TRACE(); 1097 1098 eth_dev->dev_ops = &ixgbe_eth_dev_ops; 1099 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1100 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1101 eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts; 1102 1103 /* 1104 * For secondary processes, we don't initialise any further as primary 1105 * has already done this work. Only check we don't need a different 1106 * RX and TX function. 1107 */ 1108 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1109 struct ixgbe_tx_queue *txq; 1110 /* TX queue function in primary, set by last queue initialized 1111 * Tx queue may not initialized by primary process 1112 */ 1113 if (eth_dev->data->tx_queues) { 1114 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1]; 1115 ixgbe_set_tx_function(eth_dev, txq); 1116 } else { 1117 /* Use default TX function if we get here */ 1118 PMD_INIT_LOG(NOTICE, "No TX queues configured yet. " 1119 "Using default TX function."); 1120 } 1121 1122 ixgbe_set_rx_function(eth_dev); 1123 1124 return 0; 1125 } 1126 1127 rte_eth_copy_pci_info(eth_dev, pci_dev); 1128 1129 /* Vendor and Device ID need to be set before init of shared code */ 1130 hw->device_id = pci_dev->id.device_id; 1131 hw->vendor_id = pci_dev->id.vendor_id; 1132 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1133 hw->allow_unsupported_sfp = 1; 1134 1135 /* Initialize the shared code (base driver) */ 1136 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1137 diag = ixgbe_bypass_init_shared_code(hw); 1138 #else 1139 diag = ixgbe_init_shared_code(hw); 1140 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1141 1142 if (diag != IXGBE_SUCCESS) { 1143 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); 1144 return -EIO; 1145 } 1146 1147 if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) { 1148 PMD_INIT_LOG(ERR, "\nERROR: " 1149 "Firmware recovery mode detected. Limiting functionality.\n" 1150 "Refer to the Intel(R) Ethernet Adapters and Devices " 1151 "User Guide for details on firmware recovery mode."); 1152 return -EIO; 1153 } 1154 1155 /* pick up the PCI bus settings for reporting later */ 1156 ixgbe_get_bus_info(hw); 1157 1158 /* Unlock any pending hardware semaphore */ 1159 ixgbe_swfw_lock_reset(hw); 1160 1161 #ifdef RTE_LIBRTE_SECURITY 1162 /* Initialize security_ctx only for primary process*/ 1163 if (ixgbe_ipsec_ctx_create(eth_dev)) 1164 return -ENOMEM; 1165 #endif 1166 1167 /* Initialize DCB configuration*/ 1168 memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config)); 1169 ixgbe_dcb_init(hw, dcb_config); 1170 /* Get Hardware Flow Control setting */ 1171 hw->fc.requested_mode = ixgbe_fc_full; 1172 hw->fc.current_mode = ixgbe_fc_full; 1173 hw->fc.pause_time = IXGBE_FC_PAUSE; 1174 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 1175 hw->fc.low_water[i] = IXGBE_FC_LO; 1176 hw->fc.high_water[i] = IXGBE_FC_HI; 1177 } 1178 hw->fc.send_xon = 1; 1179 1180 /* Make sure we have a good EEPROM before we read from it */ 1181 diag = ixgbe_validate_eeprom_checksum(hw, &csum); 1182 if (diag != IXGBE_SUCCESS) { 1183 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag); 1184 return -EIO; 1185 } 1186 1187 #ifdef RTE_LIBRTE_IXGBE_BYPASS 1188 diag = ixgbe_bypass_init_hw(hw); 1189 #else 1190 diag = ixgbe_init_hw(hw); 1191 #endif /* RTE_LIBRTE_IXGBE_BYPASS */ 1192 1193 /* 1194 * Devices with copper phys will fail to initialise if ixgbe_init_hw() 1195 * is called too soon after the kernel driver unbinding/binding occurs. 1196 * The failure occurs in ixgbe_identify_phy_generic() for all devices, 1197 * but for non-copper devies, ixgbe_identify_sfp_module_generic() is 1198 * also called. See ixgbe_identify_phy_82599(). The reason for the 1199 * failure is not known, and only occuts when virtualisation features 1200 * are disabled in the bios. A delay of 100ms was found to be enough by 1201 * trial-and-error, and is doubled to be safe. 1202 */ 1203 if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) { 1204 rte_delay_ms(200); 1205 diag = ixgbe_init_hw(hw); 1206 } 1207 1208 if (diag == IXGBE_ERR_SFP_NOT_PRESENT) 1209 diag = IXGBE_SUCCESS; 1210 1211 if (diag == IXGBE_ERR_EEPROM_VERSION) { 1212 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/" 1213 "LOM. Please be aware there may be issues associated " 1214 "with your hardware."); 1215 PMD_INIT_LOG(ERR, "If you are experiencing problems " 1216 "please contact your Intel or hardware representative " 1217 "who provided you with this hardware."); 1218 } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED) 1219 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module"); 1220 if (diag) { 1221 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag); 1222 return -EIO; 1223 } 1224 1225 /* Reset the hw statistics */ 1226 ixgbe_dev_stats_reset(eth_dev); 1227 1228 /* disable interrupt */ 1229 ixgbe_disable_intr(hw); 1230 1231 /* reset mappings for queue statistics hw counters*/ 1232 ixgbe_reset_qstat_mappings(hw); 1233 1234 /* Allocate memory for storing MAC addresses */ 1235 eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", RTE_ETHER_ADDR_LEN * 1236 hw->mac.num_rar_entries, 0); 1237 if (eth_dev->data->mac_addrs == NULL) { 1238 PMD_INIT_LOG(ERR, 1239 "Failed to allocate %u bytes needed to store " 1240 "MAC addresses", 1241 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1242 return -ENOMEM; 1243 } 1244 /* Copy the permanent MAC address */ 1245 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr, 1246 ð_dev->data->mac_addrs[0]); 1247 1248 /* Allocate memory for storing hash filter MAC addresses */ 1249 eth_dev->data->hash_mac_addrs = rte_zmalloc( 1250 "ixgbe", RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC, 0); 1251 if (eth_dev->data->hash_mac_addrs == NULL) { 1252 PMD_INIT_LOG(ERR, 1253 "Failed to allocate %d bytes needed to store MAC addresses", 1254 RTE_ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC); 1255 return -ENOMEM; 1256 } 1257 1258 /* Pass the information to the rte_eth_dev_close() that it should also 1259 * release the private port resources. 1260 */ 1261 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1262 1263 /* initialize the vfta */ 1264 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1265 1266 /* initialize the hw strip bitmap*/ 1267 memset(hwstrip, 0, sizeof(*hwstrip)); 1268 1269 /* initialize PF if max_vfs not zero */ 1270 ixgbe_pf_host_init(eth_dev); 1271 1272 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 1273 /* let hardware know driver is loaded */ 1274 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 1275 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 1276 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 1277 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 1278 IXGBE_WRITE_FLUSH(hw); 1279 1280 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 1281 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d", 1282 (int) hw->mac.type, (int) hw->phy.type, 1283 (int) hw->phy.sfp_type); 1284 else 1285 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d", 1286 (int) hw->mac.type, (int) hw->phy.type); 1287 1288 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1289 eth_dev->data->port_id, pci_dev->id.vendor_id, 1290 pci_dev->id.device_id); 1291 1292 rte_intr_callback_register(intr_handle, 1293 ixgbe_dev_interrupt_handler, eth_dev); 1294 1295 /* enable uio/vfio intr/eventfd mapping */ 1296 rte_intr_enable(intr_handle); 1297 1298 /* enable support intr */ 1299 ixgbe_enable_intr(eth_dev); 1300 1301 /* initialize filter info */ 1302 memset(filter_info, 0, 1303 sizeof(struct ixgbe_filter_info)); 1304 1305 /* initialize 5tuple filter list */ 1306 TAILQ_INIT(&filter_info->fivetuple_list); 1307 1308 /* initialize flow director filter list & hash */ 1309 ixgbe_fdir_filter_init(eth_dev); 1310 1311 /* initialize l2 tunnel filter list & hash */ 1312 ixgbe_l2_tn_filter_init(eth_dev); 1313 1314 /* initialize flow filter lists */ 1315 ixgbe_filterlist_init(); 1316 1317 /* initialize bandwidth configuration info */ 1318 memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); 1319 1320 /* initialize Traffic Manager configuration */ 1321 ixgbe_tm_conf_init(eth_dev); 1322 1323 return 0; 1324 } 1325 1326 static int 1327 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) 1328 { 1329 PMD_INIT_FUNC_TRACE(); 1330 1331 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1332 return 0; 1333 1334 ixgbe_dev_close(eth_dev); 1335 1336 return 0; 1337 } 1338 1339 static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) 1340 { 1341 struct ixgbe_filter_info *filter_info = 1342 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); 1343 struct ixgbe_5tuple_filter *p_5tuple; 1344 1345 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { 1346 TAILQ_REMOVE(&filter_info->fivetuple_list, 1347 p_5tuple, 1348 entries); 1349 rte_free(p_5tuple); 1350 } 1351 memset(filter_info->fivetuple_mask, 0, 1352 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); 1353 1354 return 0; 1355 } 1356 1357 static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev) 1358 { 1359 struct ixgbe_hw_fdir_info *fdir_info = 1360 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1361 struct ixgbe_fdir_filter *fdir_filter; 1362 1363 if (fdir_info->hash_map) 1364 rte_free(fdir_info->hash_map); 1365 if (fdir_info->hash_handle) 1366 rte_hash_free(fdir_info->hash_handle); 1367 1368 while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { 1369 TAILQ_REMOVE(&fdir_info->fdir_list, 1370 fdir_filter, 1371 entries); 1372 rte_free(fdir_filter); 1373 } 1374 1375 return 0; 1376 } 1377 1378 static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev) 1379 { 1380 struct ixgbe_l2_tn_info *l2_tn_info = 1381 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1382 struct ixgbe_l2_tn_filter *l2_tn_filter; 1383 1384 if (l2_tn_info->hash_map) 1385 rte_free(l2_tn_info->hash_map); 1386 if (l2_tn_info->hash_handle) 1387 rte_hash_free(l2_tn_info->hash_handle); 1388 1389 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 1390 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, 1391 l2_tn_filter, 1392 entries); 1393 rte_free(l2_tn_filter); 1394 } 1395 1396 return 0; 1397 } 1398 1399 static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) 1400 { 1401 struct ixgbe_hw_fdir_info *fdir_info = 1402 IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); 1403 char fdir_hash_name[RTE_HASH_NAMESIZE]; 1404 struct rte_hash_parameters fdir_hash_params = { 1405 .name = fdir_hash_name, 1406 .entries = IXGBE_MAX_FDIR_FILTER_NUM, 1407 .key_len = sizeof(union ixgbe_atr_input), 1408 .hash_func = rte_hash_crc, 1409 .hash_func_init_val = 0, 1410 .socket_id = rte_socket_id(), 1411 }; 1412 1413 TAILQ_INIT(&fdir_info->fdir_list); 1414 snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, 1415 "fdir_%s", eth_dev->device->name); 1416 fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); 1417 if (!fdir_info->hash_handle) { 1418 PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); 1419 return -EINVAL; 1420 } 1421 fdir_info->hash_map = rte_zmalloc("ixgbe", 1422 sizeof(struct ixgbe_fdir_filter *) * 1423 IXGBE_MAX_FDIR_FILTER_NUM, 1424 0); 1425 if (!fdir_info->hash_map) { 1426 PMD_INIT_LOG(ERR, 1427 "Failed to allocate memory for fdir hash map!"); 1428 return -ENOMEM; 1429 } 1430 fdir_info->mask_added = FALSE; 1431 1432 return 0; 1433 } 1434 1435 static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) 1436 { 1437 struct ixgbe_l2_tn_info *l2_tn_info = 1438 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); 1439 char l2_tn_hash_name[RTE_HASH_NAMESIZE]; 1440 struct rte_hash_parameters l2_tn_hash_params = { 1441 .name = l2_tn_hash_name, 1442 .entries = IXGBE_MAX_L2_TN_FILTER_NUM, 1443 .key_len = sizeof(struct ixgbe_l2_tn_key), 1444 .hash_func = rte_hash_crc, 1445 .hash_func_init_val = 0, 1446 .socket_id = rte_socket_id(), 1447 }; 1448 1449 TAILQ_INIT(&l2_tn_info->l2_tn_list); 1450 snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, 1451 "l2_tn_%s", eth_dev->device->name); 1452 l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); 1453 if (!l2_tn_info->hash_handle) { 1454 PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); 1455 return -EINVAL; 1456 } 1457 l2_tn_info->hash_map = rte_zmalloc("ixgbe", 1458 sizeof(struct ixgbe_l2_tn_filter *) * 1459 IXGBE_MAX_L2_TN_FILTER_NUM, 1460 0); 1461 if (!l2_tn_info->hash_map) { 1462 PMD_INIT_LOG(ERR, 1463 "Failed to allocate memory for L2 TN hash map!"); 1464 return -ENOMEM; 1465 } 1466 l2_tn_info->e_tag_en = FALSE; 1467 l2_tn_info->e_tag_fwd_en = FALSE; 1468 l2_tn_info->e_tag_ether_type = RTE_ETHER_TYPE_ETAG; 1469 1470 return 0; 1471 } 1472 /* 1473 * Negotiate mailbox API version with the PF. 1474 * After reset API version is always set to the basic one (ixgbe_mbox_api_10). 1475 * Then we try to negotiate starting with the most recent one. 1476 * If all negotiation attempts fail, then we will proceed with 1477 * the default one (ixgbe_mbox_api_10). 1478 */ 1479 static void 1480 ixgbevf_negotiate_api(struct ixgbe_hw *hw) 1481 { 1482 int32_t i; 1483 1484 /* start with highest supported, proceed down */ 1485 static const enum ixgbe_pfvf_api_rev sup_ver[] = { 1486 ixgbe_mbox_api_13, 1487 ixgbe_mbox_api_12, 1488 ixgbe_mbox_api_11, 1489 ixgbe_mbox_api_10, 1490 }; 1491 1492 for (i = 0; 1493 i != RTE_DIM(sup_ver) && 1494 ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0; 1495 i++) 1496 ; 1497 } 1498 1499 static void 1500 generate_random_mac_addr(struct rte_ether_addr *mac_addr) 1501 { 1502 uint64_t random; 1503 1504 /* Set Organizationally Unique Identifier (OUI) prefix. */ 1505 mac_addr->addr_bytes[0] = 0x00; 1506 mac_addr->addr_bytes[1] = 0x09; 1507 mac_addr->addr_bytes[2] = 0xC0; 1508 /* Force indication of locally assigned MAC address. */ 1509 mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR; 1510 /* Generate the last 3 bytes of the MAC address with a random number. */ 1511 random = rte_rand(); 1512 memcpy(&mac_addr->addr_bytes[3], &random, 3); 1513 } 1514 1515 static int 1516 devarg_handle_int(__rte_unused const char *key, const char *value, 1517 void *extra_args) 1518 { 1519 uint16_t *n = extra_args; 1520 1521 if (value == NULL || extra_args == NULL) 1522 return -EINVAL; 1523 1524 *n = (uint16_t)strtoul(value, NULL, 0); 1525 if (*n == USHRT_MAX && errno == ERANGE) 1526 return -1; 1527 1528 return 0; 1529 } 1530 1531 static void 1532 ixgbevf_parse_devargs(struct ixgbe_adapter *adapter, 1533 struct rte_devargs *devargs) 1534 { 1535 struct rte_kvargs *kvlist; 1536 uint16_t pflink_fullchk; 1537 1538 if (devargs == NULL) 1539 return; 1540 1541 kvlist = rte_kvargs_parse(devargs->args, ixgbevf_valid_arguments); 1542 if (kvlist == NULL) 1543 return; 1544 1545 if (rte_kvargs_count(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK) == 1 && 1546 rte_kvargs_process(kvlist, IXGBEVF_DEVARG_PFLINK_FULLCHK, 1547 devarg_handle_int, &pflink_fullchk) == 0 && 1548 pflink_fullchk == 1) 1549 adapter->pflink_fullchk = 1; 1550 1551 rte_kvargs_free(kvlist); 1552 } 1553 1554 /* 1555 * Virtual Function device init 1556 */ 1557 static int 1558 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) 1559 { 1560 int diag; 1561 uint32_t tc, tcs; 1562 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1563 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1564 struct ixgbe_hw *hw = 1565 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1566 struct ixgbe_vfta *shadow_vfta = 1567 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); 1568 struct ixgbe_hwstrip *hwstrip = 1569 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private); 1570 struct rte_ether_addr *perm_addr = 1571 (struct rte_ether_addr *)hw->mac.perm_addr; 1572 1573 PMD_INIT_FUNC_TRACE(); 1574 1575 eth_dev->dev_ops = &ixgbevf_eth_dev_ops; 1576 eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; 1577 eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; 1578 1579 /* for secondary processes, we don't initialise any further as primary 1580 * has already done this work. Only check we don't need a different 1581 * RX function 1582 */ 1583 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1584 struct ixgbe_tx_queue *txq; 1585 /* TX queue function in primary, set by last queue initialized 1586 * Tx queue may not initialized by primary process 1587 */ 1588 if (eth_dev->data->tx_queues) { 1589 txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1]; 1590 ixgbe_set_tx_function(eth_dev, txq); 1591 } else { 1592 /* Use default TX function if we get here */ 1593 PMD_INIT_LOG(NOTICE, 1594 "No TX queues configured yet. Using default TX function."); 1595 } 1596 1597 ixgbe_set_rx_function(eth_dev); 1598 1599 return 0; 1600 } 1601 1602 ixgbevf_parse_devargs(eth_dev->data->dev_private, 1603 pci_dev->device.devargs); 1604 1605 rte_eth_copy_pci_info(eth_dev, pci_dev); 1606 1607 hw->device_id = pci_dev->id.device_id; 1608 hw->vendor_id = pci_dev->id.vendor_id; 1609 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1610 1611 /* initialize the vfta */ 1612 memset(shadow_vfta, 0, sizeof(*shadow_vfta)); 1613 1614 /* initialize the hw strip bitmap*/ 1615 memset(hwstrip, 0, sizeof(*hwstrip)); 1616 1617 /* Initialize the shared code (base driver) */ 1618 diag = ixgbe_init_shared_code(hw); 1619 if (diag != IXGBE_SUCCESS) { 1620 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag); 1621 return -EIO; 1622 } 1623 1624 /* init_mailbox_params */ 1625 hw->mbx.ops.init_params(hw); 1626 1627 /* Reset the hw statistics */ 1628 ixgbevf_dev_stats_reset(eth_dev); 1629 1630 /* Disable the interrupts for VF */ 1631 ixgbevf_intr_disable(eth_dev); 1632 1633 hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */ 1634 diag = hw->mac.ops.reset_hw(hw); 1635 1636 /* 1637 * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when 1638 * the underlying PF driver has not assigned a MAC address to the VF. 1639 * In this case, assign a random MAC address. 1640 */ 1641 if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) { 1642 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1643 /* 1644 * This error code will be propagated to the app by 1645 * rte_eth_dev_reset, so use a public error code rather than 1646 * the internal-only IXGBE_ERR_RESET_FAILED 1647 */ 1648 return -EAGAIN; 1649 } 1650 1651 /* negotiate mailbox API version to use with the PF. */ 1652 ixgbevf_negotiate_api(hw); 1653 1654 /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */ 1655 ixgbevf_get_queues(hw, &tcs, &tc); 1656 1657 /* Allocate memory for storing MAC addresses */ 1658 eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", RTE_ETHER_ADDR_LEN * 1659 hw->mac.num_rar_entries, 0); 1660 if (eth_dev->data->mac_addrs == NULL) { 1661 PMD_INIT_LOG(ERR, 1662 "Failed to allocate %u bytes needed to store " 1663 "MAC addresses", 1664 RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries); 1665 return -ENOMEM; 1666 } 1667 1668 /* Pass the information to the rte_eth_dev_close() that it should also 1669 * release the private port resources. 1670 */ 1671 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1672 1673 /* Generate a random MAC address, if none was assigned by PF. */ 1674 if (rte_is_zero_ether_addr(perm_addr)) { 1675 generate_random_mac_addr(perm_addr); 1676 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1); 1677 if (diag) { 1678 rte_free(eth_dev->data->mac_addrs); 1679 eth_dev->data->mac_addrs = NULL; 1680 return diag; 1681 } 1682 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); 1683 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " 1684 "%02x:%02x:%02x:%02x:%02x:%02x", 1685 perm_addr->addr_bytes[0], 1686 perm_addr->addr_bytes[1], 1687 perm_addr->addr_bytes[2], 1688 perm_addr->addr_bytes[3], 1689 perm_addr->addr_bytes[4], 1690 perm_addr->addr_bytes[5]); 1691 } 1692 1693 /* Copy the permanent MAC address */ 1694 rte_ether_addr_copy(perm_addr, ð_dev->data->mac_addrs[0]); 1695 1696 /* reset the hardware with the new settings */ 1697 diag = hw->mac.ops.start_hw(hw); 1698 switch (diag) { 1699 case 0: 1700 break; 1701 1702 default: 1703 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag); 1704 return -EIO; 1705 } 1706 1707 rte_intr_callback_register(intr_handle, 1708 ixgbevf_dev_interrupt_handler, eth_dev); 1709 rte_intr_enable(intr_handle); 1710 ixgbevf_intr_enable(eth_dev); 1711 1712 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s", 1713 eth_dev->data->port_id, pci_dev->id.vendor_id, 1714 pci_dev->id.device_id, "ixgbe_mac_82599_vf"); 1715 1716 return 0; 1717 } 1718 1719 /* Virtual Function device uninit */ 1720 1721 static int 1722 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) 1723 { 1724 PMD_INIT_FUNC_TRACE(); 1725 1726 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1727 return 0; 1728 1729 ixgbevf_dev_close(eth_dev); 1730 1731 return 0; 1732 } 1733 1734 static int 1735 eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1736 struct rte_pci_device *pci_dev) 1737 { 1738 char name[RTE_ETH_NAME_MAX_LEN]; 1739 struct rte_eth_dev *pf_ethdev; 1740 struct rte_eth_devargs eth_da; 1741 int i, retval; 1742 1743 if (pci_dev->device.devargs) { 1744 retval = rte_eth_devargs_parse(pci_dev->device.devargs->args, 1745 ð_da); 1746 if (retval) 1747 return retval; 1748 } else 1749 memset(ð_da, 0, sizeof(eth_da)); 1750 1751 retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name, 1752 sizeof(struct ixgbe_adapter), 1753 eth_dev_pci_specific_init, pci_dev, 1754 eth_ixgbe_dev_init, NULL); 1755 1756 if (retval || eth_da.nb_representor_ports < 1) 1757 return retval; 1758 1759 pf_ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1760 if (pf_ethdev == NULL) 1761 return -ENODEV; 1762 1763 /* probe VF representor ports */ 1764 for (i = 0; i < eth_da.nb_representor_ports; i++) { 1765 struct ixgbe_vf_info *vfinfo; 1766 struct ixgbe_vf_representor representor; 1767 1768 vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA( 1769 pf_ethdev->data->dev_private); 1770 if (vfinfo == NULL) { 1771 PMD_DRV_LOG(ERR, 1772 "no virtual functions supported by PF"); 1773 break; 1774 } 1775 1776 representor.vf_id = eth_da.representor_ports[i]; 1777 representor.switch_domain_id = vfinfo->switch_domain_id; 1778 representor.pf_ethdev = pf_ethdev; 1779 1780 /* representor port net_bdf_port */ 1781 snprintf(name, sizeof(name), "net_%s_representor_%d", 1782 pci_dev->device.name, 1783 eth_da.representor_ports[i]); 1784 1785 retval = rte_eth_dev_create(&pci_dev->device, name, 1786 sizeof(struct ixgbe_vf_representor), NULL, NULL, 1787 ixgbe_vf_representor_init, &representor); 1788 1789 if (retval) 1790 PMD_DRV_LOG(ERR, "failed to create ixgbe vf " 1791 "representor %s.", name); 1792 } 1793 1794 return 0; 1795 } 1796 1797 static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev) 1798 { 1799 struct rte_eth_dev *ethdev; 1800 1801 ethdev = rte_eth_dev_allocated(pci_dev->device.name); 1802 if (!ethdev) 1803 return -ENODEV; 1804 1805 if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) 1806 return rte_eth_dev_destroy(ethdev, ixgbe_vf_representor_uninit); 1807 else 1808 return rte_eth_dev_destroy(ethdev, eth_ixgbe_dev_uninit); 1809 } 1810 1811 static struct rte_pci_driver rte_ixgbe_pmd = { 1812 .id_table = pci_id_ixgbe_map, 1813 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1814 .probe = eth_ixgbe_pci_probe, 1815 .remove = eth_ixgbe_pci_remove, 1816 }; 1817 1818 static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1819 struct rte_pci_device *pci_dev) 1820 { 1821 return rte_eth_dev_pci_generic_probe(pci_dev, 1822 sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init); 1823 } 1824 1825 static int eth_ixgbevf_pci_remove(struct rte_pci_device *pci_dev) 1826 { 1827 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbevf_dev_uninit); 1828 } 1829 1830 /* 1831 * virtual function driver struct 1832 */ 1833 static struct rte_pci_driver rte_ixgbevf_pmd = { 1834 .id_table = pci_id_ixgbevf_map, 1835 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1836 .probe = eth_ixgbevf_pci_probe, 1837 .remove = eth_ixgbevf_pci_remove, 1838 }; 1839 1840 static int 1841 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1842 { 1843 struct ixgbe_hw *hw = 1844 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1845 struct ixgbe_vfta *shadow_vfta = 1846 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1847 uint32_t vfta; 1848 uint32_t vid_idx; 1849 uint32_t vid_bit; 1850 1851 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 1852 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 1853 vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx)); 1854 if (on) 1855 vfta |= vid_bit; 1856 else 1857 vfta &= ~vid_bit; 1858 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta); 1859 1860 /* update local VFTA copy */ 1861 shadow_vfta->vfta[vid_idx] = vfta; 1862 1863 return 0; 1864 } 1865 1866 static void 1867 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 1868 { 1869 if (on) 1870 ixgbe_vlan_hw_strip_enable(dev, queue); 1871 else 1872 ixgbe_vlan_hw_strip_disable(dev, queue); 1873 } 1874 1875 static int 1876 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, 1877 enum rte_vlan_type vlan_type, 1878 uint16_t tpid) 1879 { 1880 struct ixgbe_hw *hw = 1881 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1882 int ret = 0; 1883 uint32_t reg; 1884 uint32_t qinq; 1885 1886 qinq = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1887 qinq &= IXGBE_DMATXCTL_GDV; 1888 1889 switch (vlan_type) { 1890 case ETH_VLAN_TYPE_INNER: 1891 if (qinq) { 1892 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1893 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1894 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1895 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1896 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1897 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1898 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1899 } else { 1900 ret = -ENOTSUP; 1901 PMD_DRV_LOG(ERR, "Inner type is not supported" 1902 " by single VLAN"); 1903 } 1904 break; 1905 case ETH_VLAN_TYPE_OUTER: 1906 if (qinq) { 1907 /* Only the high 16-bits is valid */ 1908 IXGBE_WRITE_REG(hw, IXGBE_EXVET, (uint32_t)tpid << 1909 IXGBE_EXVET_VET_EXT_SHIFT); 1910 } else { 1911 reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1912 reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid; 1913 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg); 1914 reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 1915 reg = (reg & (~IXGBE_DMATXCTL_VT_MASK)) 1916 | ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT); 1917 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg); 1918 } 1919 1920 break; 1921 default: 1922 ret = -EINVAL; 1923 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type); 1924 break; 1925 } 1926 1927 return ret; 1928 } 1929 1930 void 1931 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev) 1932 { 1933 struct ixgbe_hw *hw = 1934 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1935 uint32_t vlnctrl; 1936 1937 PMD_INIT_FUNC_TRACE(); 1938 1939 /* Filter Table Disable */ 1940 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1941 vlnctrl &= ~IXGBE_VLNCTRL_VFE; 1942 1943 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1944 } 1945 1946 void 1947 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev) 1948 { 1949 struct ixgbe_hw *hw = 1950 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1951 struct ixgbe_vfta *shadow_vfta = 1952 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 1953 uint32_t vlnctrl; 1954 uint16_t i; 1955 1956 PMD_INIT_FUNC_TRACE(); 1957 1958 /* Filter Table Enable */ 1959 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 1960 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 1961 vlnctrl |= IXGBE_VLNCTRL_VFE; 1962 1963 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 1964 1965 /* write whatever is in local vfta copy */ 1966 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 1967 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]); 1968 } 1969 1970 static void 1971 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on) 1972 { 1973 struct ixgbe_hwstrip *hwstrip = 1974 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private); 1975 struct ixgbe_rx_queue *rxq; 1976 1977 if (queue >= IXGBE_MAX_RX_QUEUE_NUM) 1978 return; 1979 1980 if (on) 1981 IXGBE_SET_HWSTRIP(hwstrip, queue); 1982 else 1983 IXGBE_CLEAR_HWSTRIP(hwstrip, queue); 1984 1985 if (queue >= dev->data->nb_rx_queues) 1986 return; 1987 1988 rxq = dev->data->rx_queues[queue]; 1989 1990 if (on) { 1991 rxq->vlan_flags = PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1992 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 1993 } else { 1994 rxq->vlan_flags = PKT_RX_VLAN; 1995 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 1996 } 1997 } 1998 1999 static void 2000 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue) 2001 { 2002 struct ixgbe_hw *hw = 2003 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2004 uint32_t ctrl; 2005 2006 PMD_INIT_FUNC_TRACE(); 2007 2008 if (hw->mac.type == ixgbe_mac_82598EB) { 2009 /* No queue level support */ 2010 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2011 return; 2012 } 2013 2014 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2015 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2016 ctrl &= ~IXGBE_RXDCTL_VME; 2017 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2018 2019 /* record those setting for HW strip per queue */ 2020 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0); 2021 } 2022 2023 static void 2024 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue) 2025 { 2026 struct ixgbe_hw *hw = 2027 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2028 uint32_t ctrl; 2029 2030 PMD_INIT_FUNC_TRACE(); 2031 2032 if (hw->mac.type == ixgbe_mac_82598EB) { 2033 /* No queue level supported */ 2034 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip"); 2035 return; 2036 } 2037 2038 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ 2039 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 2040 ctrl |= IXGBE_RXDCTL_VME; 2041 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 2042 2043 /* record those setting for HW strip per queue */ 2044 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1); 2045 } 2046 2047 static void 2048 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev) 2049 { 2050 struct ixgbe_hw *hw = 2051 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2052 uint32_t ctrl; 2053 2054 PMD_INIT_FUNC_TRACE(); 2055 2056 /* DMATXCTRL: Geric Double VLAN Disable */ 2057 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2058 ctrl &= ~IXGBE_DMATXCTL_GDV; 2059 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2060 2061 /* CTRL_EXT: Global Double VLAN Disable */ 2062 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2063 ctrl &= ~IXGBE_EXTENDED_VLAN; 2064 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2065 2066 } 2067 2068 static void 2069 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2070 { 2071 struct ixgbe_hw *hw = 2072 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2073 uint32_t ctrl; 2074 2075 PMD_INIT_FUNC_TRACE(); 2076 2077 /* DMATXCTRL: Geric Double VLAN Enable */ 2078 ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2079 ctrl |= IXGBE_DMATXCTL_GDV; 2080 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl); 2081 2082 /* CTRL_EXT: Global Double VLAN Enable */ 2083 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 2084 ctrl |= IXGBE_EXTENDED_VLAN; 2085 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl); 2086 2087 /* Clear pooling mode of PFVTCTL. It's required by X550. */ 2088 if (hw->mac.type == ixgbe_mac_X550 || 2089 hw->mac.type == ixgbe_mac_X550EM_x || 2090 hw->mac.type == ixgbe_mac_X550EM_a) { 2091 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 2092 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 2093 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 2094 } 2095 2096 /* 2097 * VET EXT field in the EXVET register = 0x8100 by default 2098 * So no need to change. Same to VT field of DMATXCTL register 2099 */ 2100 } 2101 2102 void 2103 ixgbe_vlan_hw_strip_config(struct rte_eth_dev *dev) 2104 { 2105 struct ixgbe_hw *hw = 2106 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2107 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2108 uint32_t ctrl; 2109 uint16_t i; 2110 struct ixgbe_rx_queue *rxq; 2111 bool on; 2112 2113 PMD_INIT_FUNC_TRACE(); 2114 2115 if (hw->mac.type == ixgbe_mac_82598EB) { 2116 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2117 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2118 ctrl |= IXGBE_VLNCTRL_VME; 2119 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2120 } else { 2121 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2122 ctrl &= ~IXGBE_VLNCTRL_VME; 2123 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2124 } 2125 } else { 2126 /* 2127 * Other 10G NIC, the VLAN strip can be setup 2128 * per queue in RXDCTL 2129 */ 2130 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2131 rxq = dev->data->rx_queues[i]; 2132 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 2133 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 2134 ctrl |= IXGBE_RXDCTL_VME; 2135 on = TRUE; 2136 } else { 2137 ctrl &= ~IXGBE_RXDCTL_VME; 2138 on = FALSE; 2139 } 2140 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); 2141 2142 /* record those setting for HW strip per queue */ 2143 ixgbe_vlan_hw_strip_bitmap_set(dev, i, on); 2144 } 2145 } 2146 } 2147 2148 static void 2149 ixgbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask) 2150 { 2151 uint16_t i; 2152 struct rte_eth_rxmode *rxmode; 2153 struct ixgbe_rx_queue *rxq; 2154 2155 if (mask & ETH_VLAN_STRIP_MASK) { 2156 rxmode = &dev->data->dev_conf.rxmode; 2157 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2158 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2159 rxq = dev->data->rx_queues[i]; 2160 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 2161 } 2162 else 2163 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2164 rxq = dev->data->rx_queues[i]; 2165 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 2166 } 2167 } 2168 } 2169 2170 static int 2171 ixgbe_vlan_offload_config(struct rte_eth_dev *dev, int mask) 2172 { 2173 struct rte_eth_rxmode *rxmode; 2174 rxmode = &dev->data->dev_conf.rxmode; 2175 2176 if (mask & ETH_VLAN_STRIP_MASK) { 2177 ixgbe_vlan_hw_strip_config(dev); 2178 } 2179 2180 if (mask & ETH_VLAN_FILTER_MASK) { 2181 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2182 ixgbe_vlan_hw_filter_enable(dev); 2183 else 2184 ixgbe_vlan_hw_filter_disable(dev); 2185 } 2186 2187 if (mask & ETH_VLAN_EXTEND_MASK) { 2188 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) 2189 ixgbe_vlan_hw_extend_enable(dev); 2190 else 2191 ixgbe_vlan_hw_extend_disable(dev); 2192 } 2193 2194 return 0; 2195 } 2196 2197 static int 2198 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2199 { 2200 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 2201 2202 ixgbe_vlan_offload_config(dev, mask); 2203 2204 return 0; 2205 } 2206 2207 static void 2208 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) 2209 { 2210 struct ixgbe_hw *hw = 2211 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2212 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 2213 uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2214 2215 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 2216 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 2217 } 2218 2219 static int 2220 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) 2221 { 2222 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2223 2224 switch (nb_rx_q) { 2225 case 1: 2226 case 2: 2227 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS; 2228 break; 2229 case 4: 2230 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS; 2231 break; 2232 default: 2233 return -EINVAL; 2234 } 2235 2236 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 2237 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2238 RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = 2239 pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2240 return 0; 2241 } 2242 2243 static int 2244 ixgbe_check_mq_mode(struct rte_eth_dev *dev) 2245 { 2246 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 2247 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2248 uint16_t nb_rx_q = dev->data->nb_rx_queues; 2249 uint16_t nb_tx_q = dev->data->nb_tx_queues; 2250 2251 if (RTE_ETH_DEV_SRIOV(dev).active != 0) { 2252 /* check multi-queue mode */ 2253 switch (dev_conf->rxmode.mq_mode) { 2254 case ETH_MQ_RX_VMDQ_DCB: 2255 PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV"); 2256 break; 2257 case ETH_MQ_RX_VMDQ_DCB_RSS: 2258 /* DCB/RSS VMDQ in SRIOV mode, not implement yet */ 2259 PMD_INIT_LOG(ERR, "SRIOV active," 2260 " unsupported mq_mode rx %d.", 2261 dev_conf->rxmode.mq_mode); 2262 return -EINVAL; 2263 case ETH_MQ_RX_RSS: 2264 case ETH_MQ_RX_VMDQ_RSS: 2265 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS; 2266 if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) 2267 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) { 2268 PMD_INIT_LOG(ERR, "SRIOV is active," 2269 " invalid queue number" 2270 " for VMDQ RSS, allowed" 2271 " value are 1, 2 or 4."); 2272 return -EINVAL; 2273 } 2274 break; 2275 case ETH_MQ_RX_VMDQ_ONLY: 2276 case ETH_MQ_RX_NONE: 2277 /* if nothing mq mode configure, use default scheme */ 2278 dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; 2279 break; 2280 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/ 2281 /* SRIOV only works in VMDq enable mode */ 2282 PMD_INIT_LOG(ERR, "SRIOV is active," 2283 " wrong mq_mode rx %d.", 2284 dev_conf->rxmode.mq_mode); 2285 return -EINVAL; 2286 } 2287 2288 switch (dev_conf->txmode.mq_mode) { 2289 case ETH_MQ_TX_VMDQ_DCB: 2290 PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV"); 2291 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2292 break; 2293 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */ 2294 dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY; 2295 break; 2296 } 2297 2298 /* check valid queue number */ 2299 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) || 2300 (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) { 2301 PMD_INIT_LOG(ERR, "SRIOV is active," 2302 " nb_rx_q=%d nb_tx_q=%d queue number" 2303 " must be less than or equal to %d.", 2304 nb_rx_q, nb_tx_q, 2305 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool); 2306 return -EINVAL; 2307 } 2308 } else { 2309 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) { 2310 PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is" 2311 " not supported."); 2312 return -EINVAL; 2313 } 2314 /* check configuration for vmdb+dcb mode */ 2315 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) { 2316 const struct rte_eth_vmdq_dcb_conf *conf; 2317 2318 if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2319 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.", 2320 IXGBE_VMDQ_DCB_NB_QUEUES); 2321 return -EINVAL; 2322 } 2323 conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf; 2324 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2325 conf->nb_queue_pools == ETH_32_POOLS)) { 2326 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2327 " nb_queue_pools must be %d or %d.", 2328 ETH_16_POOLS, ETH_32_POOLS); 2329 return -EINVAL; 2330 } 2331 } 2332 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { 2333 const struct rte_eth_vmdq_dcb_tx_conf *conf; 2334 2335 if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) { 2336 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d", 2337 IXGBE_VMDQ_DCB_NB_QUEUES); 2338 return -EINVAL; 2339 } 2340 conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2341 if (!(conf->nb_queue_pools == ETH_16_POOLS || 2342 conf->nb_queue_pools == ETH_32_POOLS)) { 2343 PMD_INIT_LOG(ERR, "VMDQ+DCB selected," 2344 " nb_queue_pools != %d and" 2345 " nb_queue_pools != %d.", 2346 ETH_16_POOLS, ETH_32_POOLS); 2347 return -EINVAL; 2348 } 2349 } 2350 2351 /* For DCB mode check our configuration before we go further */ 2352 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) { 2353 const struct rte_eth_dcb_rx_conf *conf; 2354 2355 conf = &dev_conf->rx_adv_conf.dcb_rx_conf; 2356 if (!(conf->nb_tcs == ETH_4_TCS || 2357 conf->nb_tcs == ETH_8_TCS)) { 2358 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2359 " and nb_tcs != %d.", 2360 ETH_4_TCS, ETH_8_TCS); 2361 return -EINVAL; 2362 } 2363 } 2364 2365 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { 2366 const struct rte_eth_dcb_tx_conf *conf; 2367 2368 conf = &dev_conf->tx_adv_conf.dcb_tx_conf; 2369 if (!(conf->nb_tcs == ETH_4_TCS || 2370 conf->nb_tcs == ETH_8_TCS)) { 2371 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d" 2372 " and nb_tcs != %d.", 2373 ETH_4_TCS, ETH_8_TCS); 2374 return -EINVAL; 2375 } 2376 } 2377 2378 /* 2379 * When DCB/VT is off, maximum number of queues changes, 2380 * except for 82598EB, which remains constant. 2381 */ 2382 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 2383 hw->mac.type != ixgbe_mac_82598EB) { 2384 if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) { 2385 PMD_INIT_LOG(ERR, 2386 "Neither VT nor DCB are enabled, " 2387 "nb_tx_q > %d.", 2388 IXGBE_NONE_MODE_TX_NB_QUEUES); 2389 return -EINVAL; 2390 } 2391 } 2392 } 2393 return 0; 2394 } 2395 2396 static int 2397 ixgbe_dev_configure(struct rte_eth_dev *dev) 2398 { 2399 struct ixgbe_interrupt *intr = 2400 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2401 struct ixgbe_adapter *adapter = dev->data->dev_private; 2402 int ret; 2403 2404 PMD_INIT_FUNC_TRACE(); 2405 /* multipe queue mode checking */ 2406 ret = ixgbe_check_mq_mode(dev); 2407 if (ret != 0) { 2408 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.", 2409 ret); 2410 return ret; 2411 } 2412 2413 /* set flag to update link status after init */ 2414 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2415 2416 /* 2417 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 2418 * allocation or vector Rx preconditions we will reset it. 2419 */ 2420 adapter->rx_bulk_alloc_allowed = true; 2421 adapter->rx_vec_allowed = true; 2422 2423 return 0; 2424 } 2425 2426 static void 2427 ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) 2428 { 2429 struct ixgbe_hw *hw = 2430 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2431 struct ixgbe_interrupt *intr = 2432 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 2433 uint32_t gpie; 2434 2435 /* only set up it on X550EM_X */ 2436 if (hw->mac.type == ixgbe_mac_X550EM_x) { 2437 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 2438 gpie |= IXGBE_SDP0_GPIEN_X550EM_x; 2439 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 2440 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 2441 intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x; 2442 } 2443 } 2444 2445 int 2446 ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, 2447 uint16_t tx_rate, uint64_t q_msk) 2448 { 2449 struct ixgbe_hw *hw; 2450 struct ixgbe_vf_info *vfinfo; 2451 struct rte_eth_link link; 2452 uint8_t nb_q_per_pool; 2453 uint32_t queue_stride; 2454 uint32_t queue_idx, idx = 0, vf_idx; 2455 uint32_t queue_end; 2456 uint16_t total_rate = 0; 2457 struct rte_pci_device *pci_dev; 2458 int ret; 2459 2460 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2461 ret = rte_eth_link_get_nowait(dev->data->port_id, &link); 2462 if (ret < 0) 2463 return ret; 2464 2465 if (vf >= pci_dev->max_vfs) 2466 return -EINVAL; 2467 2468 if (tx_rate > link.link_speed) 2469 return -EINVAL; 2470 2471 if (q_msk == 0) 2472 return 0; 2473 2474 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2475 vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); 2476 nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; 2477 queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; 2478 queue_idx = vf * queue_stride; 2479 queue_end = queue_idx + nb_q_per_pool - 1; 2480 if (queue_end >= hw->mac.max_tx_queues) 2481 return -EINVAL; 2482 2483 if (vfinfo) { 2484 for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) { 2485 if (vf_idx == vf) 2486 continue; 2487 for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); 2488 idx++) 2489 total_rate += vfinfo[vf_idx].tx_rate[idx]; 2490 } 2491 } else { 2492 return -EINVAL; 2493 } 2494 2495 /* Store tx_rate for this vf. */ 2496 for (idx = 0; idx < nb_q_per_pool; idx++) { 2497 if (((uint64_t)0x1 << idx) & q_msk) { 2498 if (vfinfo[vf].tx_rate[idx] != tx_rate) 2499 vfinfo[vf].tx_rate[idx] = tx_rate; 2500 total_rate += tx_rate; 2501 } 2502 } 2503 2504 if (total_rate > dev->data->dev_link.link_speed) { 2505 /* Reset stored TX rate of the VF if it causes exceed 2506 * link speed. 2507 */ 2508 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); 2509 return -EINVAL; 2510 } 2511 2512 /* Set RTTBCNRC of each queue/pool for vf X */ 2513 for (; queue_idx <= queue_end; queue_idx++) { 2514 if (0x1 & q_msk) 2515 ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); 2516 q_msk = q_msk >> 1; 2517 } 2518 2519 return 0; 2520 } 2521 2522 /* 2523 * Configure device link speed and setup link. 2524 * It returns 0 on success. 2525 */ 2526 static int 2527 ixgbe_dev_start(struct rte_eth_dev *dev) 2528 { 2529 struct ixgbe_hw *hw = 2530 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2531 struct ixgbe_vf_info *vfinfo = 2532 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2533 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2534 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2535 uint32_t intr_vector = 0; 2536 int err, link_up = 0, negotiate = 0; 2537 uint32_t speed = 0; 2538 uint32_t allowed_speeds = 0; 2539 int mask = 0; 2540 int status; 2541 uint16_t vf, idx; 2542 uint32_t *link_speeds; 2543 struct ixgbe_tm_conf *tm_conf = 2544 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2545 2546 PMD_INIT_FUNC_TRACE(); 2547 2548 /* IXGBE devices don't support: 2549 * - half duplex (checked afterwards for valid speeds) 2550 * - fixed speed: TODO implement 2551 */ 2552 if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) { 2553 PMD_INIT_LOG(ERR, 2554 "Invalid link_speeds for port %u, fix speed not supported", 2555 dev->data->port_id); 2556 return -EINVAL; 2557 } 2558 2559 /* Stop the link setup handler before resetting the HW. */ 2560 rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev); 2561 2562 /* disable uio/vfio intr/eventfd mapping */ 2563 rte_intr_disable(intr_handle); 2564 2565 /* stop adapter */ 2566 hw->adapter_stopped = 0; 2567 ixgbe_stop_adapter(hw); 2568 2569 /* reinitialize adapter 2570 * this calls reset and start 2571 */ 2572 status = ixgbe_pf_reset_hw(hw); 2573 if (status != 0) 2574 return -1; 2575 hw->mac.ops.start_hw(hw); 2576 hw->mac.get_link_status = true; 2577 2578 /* configure PF module if SRIOV enabled */ 2579 ixgbe_pf_host_configure(dev); 2580 2581 ixgbe_dev_phy_intr_setup(dev); 2582 2583 /* check and configure queue intr-vector mapping */ 2584 if ((rte_intr_cap_multiple(intr_handle) || 2585 !RTE_ETH_DEV_SRIOV(dev).active) && 2586 dev->data->dev_conf.intr_conf.rxq != 0) { 2587 intr_vector = dev->data->nb_rx_queues; 2588 if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) { 2589 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 2590 IXGBE_MAX_INTR_QUEUE_NUM); 2591 return -ENOTSUP; 2592 } 2593 if (rte_intr_efd_enable(intr_handle, intr_vector)) 2594 return -1; 2595 } 2596 2597 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 2598 intr_handle->intr_vec = 2599 rte_zmalloc("intr_vec", 2600 dev->data->nb_rx_queues * sizeof(int), 0); 2601 if (intr_handle->intr_vec == NULL) { 2602 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 2603 " intr_vec", dev->data->nb_rx_queues); 2604 return -ENOMEM; 2605 } 2606 } 2607 2608 /* confiugre msix for sleep until rx interrupt */ 2609 ixgbe_configure_msix(dev); 2610 2611 /* initialize transmission unit */ 2612 ixgbe_dev_tx_init(dev); 2613 2614 /* This can fail when allocating mbufs for descriptor rings */ 2615 err = ixgbe_dev_rx_init(dev); 2616 if (err) { 2617 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware"); 2618 goto error; 2619 } 2620 2621 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 2622 ETH_VLAN_EXTEND_MASK; 2623 err = ixgbe_vlan_offload_config(dev, mask); 2624 if (err) { 2625 PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); 2626 goto error; 2627 } 2628 2629 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { 2630 /* Enable vlan filtering for VMDq */ 2631 ixgbe_vmdq_vlan_hw_filter_enable(dev); 2632 } 2633 2634 /* Configure DCB hw */ 2635 ixgbe_configure_dcb(dev); 2636 2637 if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { 2638 err = ixgbe_fdir_configure(dev); 2639 if (err) 2640 goto error; 2641 } 2642 2643 /* Restore vf rate limit */ 2644 if (vfinfo != NULL) { 2645 for (vf = 0; vf < pci_dev->max_vfs; vf++) 2646 for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) 2647 if (vfinfo[vf].tx_rate[idx] != 0) 2648 ixgbe_set_vf_rate_limit( 2649 dev, vf, 2650 vfinfo[vf].tx_rate[idx], 2651 1 << idx); 2652 } 2653 2654 ixgbe_restore_statistics_mapping(dev); 2655 2656 err = ixgbe_dev_rxtx_start(dev); 2657 if (err < 0) { 2658 PMD_INIT_LOG(ERR, "Unable to start rxtx queues"); 2659 goto error; 2660 } 2661 2662 /* Skip link setup if loopback mode is enabled. */ 2663 if (dev->data->dev_conf.lpbk_mode != 0) { 2664 err = ixgbe_check_supported_loopback_mode(dev); 2665 if (err < 0) { 2666 PMD_INIT_LOG(ERR, "Unsupported loopback mode"); 2667 goto error; 2668 } else { 2669 goto skip_link_setup; 2670 } 2671 } 2672 2673 if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) { 2674 err = hw->mac.ops.setup_sfp(hw); 2675 if (err) 2676 goto error; 2677 } 2678 2679 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2680 /* Turn on the copper */ 2681 ixgbe_set_phy_power(hw, true); 2682 } else { 2683 /* Turn on the laser */ 2684 ixgbe_enable_tx_laser(hw); 2685 } 2686 2687 err = ixgbe_check_link(hw, &speed, &link_up, 0); 2688 if (err) 2689 goto error; 2690 dev->data->dev_link.link_status = link_up; 2691 2692 err = ixgbe_get_link_capabilities(hw, &speed, &negotiate); 2693 if (err) 2694 goto error; 2695 2696 switch (hw->mac.type) { 2697 case ixgbe_mac_X550: 2698 case ixgbe_mac_X550EM_x: 2699 case ixgbe_mac_X550EM_a: 2700 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2701 ETH_LINK_SPEED_2_5G | ETH_LINK_SPEED_5G | 2702 ETH_LINK_SPEED_10G; 2703 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 2704 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 2705 allowed_speeds = ETH_LINK_SPEED_10M | 2706 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 2707 break; 2708 default: 2709 allowed_speeds = ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | 2710 ETH_LINK_SPEED_10G; 2711 } 2712 2713 link_speeds = &dev->data->dev_conf.link_speeds; 2714 if (*link_speeds & ~allowed_speeds) { 2715 PMD_INIT_LOG(ERR, "Invalid link setting"); 2716 goto error; 2717 } 2718 2719 speed = 0x0; 2720 if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { 2721 switch (hw->mac.type) { 2722 case ixgbe_mac_82598EB: 2723 speed = IXGBE_LINK_SPEED_82598_AUTONEG; 2724 break; 2725 case ixgbe_mac_82599EB: 2726 case ixgbe_mac_X540: 2727 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2728 break; 2729 case ixgbe_mac_X550: 2730 case ixgbe_mac_X550EM_x: 2731 case ixgbe_mac_X550EM_a: 2732 speed = IXGBE_LINK_SPEED_X550_AUTONEG; 2733 break; 2734 default: 2735 speed = IXGBE_LINK_SPEED_82599_AUTONEG; 2736 } 2737 } else { 2738 if (*link_speeds & ETH_LINK_SPEED_10G) 2739 speed |= IXGBE_LINK_SPEED_10GB_FULL; 2740 if (*link_speeds & ETH_LINK_SPEED_5G) 2741 speed |= IXGBE_LINK_SPEED_5GB_FULL; 2742 if (*link_speeds & ETH_LINK_SPEED_2_5G) 2743 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 2744 if (*link_speeds & ETH_LINK_SPEED_1G) 2745 speed |= IXGBE_LINK_SPEED_1GB_FULL; 2746 if (*link_speeds & ETH_LINK_SPEED_100M) 2747 speed |= IXGBE_LINK_SPEED_100_FULL; 2748 if (*link_speeds & ETH_LINK_SPEED_10M) 2749 speed |= IXGBE_LINK_SPEED_10_FULL; 2750 } 2751 2752 err = ixgbe_setup_link(hw, speed, link_up); 2753 if (err) 2754 goto error; 2755 2756 skip_link_setup: 2757 2758 if (rte_intr_allow_others(intr_handle)) { 2759 /* check if lsc interrupt is enabled */ 2760 if (dev->data->dev_conf.intr_conf.lsc != 0) 2761 ixgbe_dev_lsc_interrupt_setup(dev, TRUE); 2762 else 2763 ixgbe_dev_lsc_interrupt_setup(dev, FALSE); 2764 ixgbe_dev_macsec_interrupt_setup(dev); 2765 } else { 2766 rte_intr_callback_unregister(intr_handle, 2767 ixgbe_dev_interrupt_handler, dev); 2768 if (dev->data->dev_conf.intr_conf.lsc != 0) 2769 PMD_INIT_LOG(INFO, "lsc won't enable because of" 2770 " no intr multiplex"); 2771 } 2772 2773 /* check if rxq interrupt is enabled */ 2774 if (dev->data->dev_conf.intr_conf.rxq != 0 && 2775 rte_intr_dp_is_en(intr_handle)) 2776 ixgbe_dev_rxq_interrupt_setup(dev); 2777 2778 /* enable uio/vfio intr/eventfd mapping */ 2779 rte_intr_enable(intr_handle); 2780 2781 /* resume enabled intr since hw reset */ 2782 ixgbe_enable_intr(dev); 2783 ixgbe_l2_tunnel_conf(dev); 2784 ixgbe_filter_restore(dev); 2785 2786 if (tm_conf->root && !tm_conf->committed) 2787 PMD_DRV_LOG(WARNING, 2788 "please call hierarchy_commit() " 2789 "before starting the port"); 2790 2791 /* 2792 * Update link status right before return, because it may 2793 * start link configuration process in a separate thread. 2794 */ 2795 ixgbe_dev_link_update(dev, 0); 2796 2797 return 0; 2798 2799 error: 2800 PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err); 2801 ixgbe_dev_clear_queues(dev); 2802 return -EIO; 2803 } 2804 2805 /* 2806 * Stop device: disable rx and tx functions to allow for reconfiguring. 2807 */ 2808 static void 2809 ixgbe_dev_stop(struct rte_eth_dev *dev) 2810 { 2811 struct rte_eth_link link; 2812 struct ixgbe_adapter *adapter = dev->data->dev_private; 2813 struct ixgbe_hw *hw = 2814 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2815 struct ixgbe_vf_info *vfinfo = 2816 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); 2817 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2818 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2819 int vf; 2820 struct ixgbe_tm_conf *tm_conf = 2821 IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); 2822 2823 if (hw->adapter_stopped) 2824 return; 2825 2826 PMD_INIT_FUNC_TRACE(); 2827 2828 rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev); 2829 2830 /* disable interrupts */ 2831 ixgbe_disable_intr(hw); 2832 2833 /* reset the NIC */ 2834 ixgbe_pf_reset_hw(hw); 2835 hw->adapter_stopped = 0; 2836 2837 /* stop adapter */ 2838 ixgbe_stop_adapter(hw); 2839 2840 for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) 2841 vfinfo[vf].clear_to_send = false; 2842 2843 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2844 /* Turn off the copper */ 2845 ixgbe_set_phy_power(hw, false); 2846 } else { 2847 /* Turn off the laser */ 2848 ixgbe_disable_tx_laser(hw); 2849 } 2850 2851 ixgbe_dev_clear_queues(dev); 2852 2853 /* Clear stored conf */ 2854 dev->data->scattered_rx = 0; 2855 dev->data->lro = 0; 2856 2857 /* Clear recorded link status */ 2858 memset(&link, 0, sizeof(link)); 2859 rte_eth_linkstatus_set(dev, &link); 2860 2861 if (!rte_intr_allow_others(intr_handle)) 2862 /* resume to the default handler */ 2863 rte_intr_callback_register(intr_handle, 2864 ixgbe_dev_interrupt_handler, 2865 (void *)dev); 2866 2867 /* Clean datapath event and queue/vec mapping */ 2868 rte_intr_efd_disable(intr_handle); 2869 if (intr_handle->intr_vec != NULL) { 2870 rte_free(intr_handle->intr_vec); 2871 intr_handle->intr_vec = NULL; 2872 } 2873 2874 /* reset hierarchy commit */ 2875 tm_conf->committed = false; 2876 2877 adapter->rss_reta_updated = 0; 2878 2879 hw->adapter_stopped = true; 2880 } 2881 2882 /* 2883 * Set device link up: enable tx. 2884 */ 2885 static int 2886 ixgbe_dev_set_link_up(struct rte_eth_dev *dev) 2887 { 2888 struct ixgbe_hw *hw = 2889 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2890 if (hw->mac.type == ixgbe_mac_82599EB) { 2891 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2892 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2893 /* Not suported in bypass mode */ 2894 PMD_INIT_LOG(ERR, "Set link up is not supported " 2895 "by device id 0x%x", hw->device_id); 2896 return -ENOTSUP; 2897 } 2898 #endif 2899 } 2900 2901 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2902 /* Turn on the copper */ 2903 ixgbe_set_phy_power(hw, true); 2904 } else { 2905 /* Turn on the laser */ 2906 ixgbe_enable_tx_laser(hw); 2907 } 2908 2909 return 0; 2910 } 2911 2912 /* 2913 * Set device link down: disable tx. 2914 */ 2915 static int 2916 ixgbe_dev_set_link_down(struct rte_eth_dev *dev) 2917 { 2918 struct ixgbe_hw *hw = 2919 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2920 if (hw->mac.type == ixgbe_mac_82599EB) { 2921 #ifdef RTE_LIBRTE_IXGBE_BYPASS 2922 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { 2923 /* Not suported in bypass mode */ 2924 PMD_INIT_LOG(ERR, "Set link down is not supported " 2925 "by device id 0x%x", hw->device_id); 2926 return -ENOTSUP; 2927 } 2928 #endif 2929 } 2930 2931 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { 2932 /* Turn off the copper */ 2933 ixgbe_set_phy_power(hw, false); 2934 } else { 2935 /* Turn off the laser */ 2936 ixgbe_disable_tx_laser(hw); 2937 } 2938 2939 return 0; 2940 } 2941 2942 /* 2943 * Reset and stop device. 2944 */ 2945 static void 2946 ixgbe_dev_close(struct rte_eth_dev *dev) 2947 { 2948 struct ixgbe_hw *hw = 2949 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2950 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2951 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2952 int retries = 0; 2953 int ret; 2954 2955 PMD_INIT_FUNC_TRACE(); 2956 2957 ixgbe_pf_reset_hw(hw); 2958 2959 ixgbe_dev_stop(dev); 2960 2961 ixgbe_dev_free_queues(dev); 2962 2963 ixgbe_disable_pcie_master(hw); 2964 2965 /* reprogram the RAR[0] in case user changed it. */ 2966 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 2967 2968 dev->dev_ops = NULL; 2969 dev->rx_pkt_burst = NULL; 2970 dev->tx_pkt_burst = NULL; 2971 2972 /* Unlock any pending hardware semaphore */ 2973 ixgbe_swfw_lock_reset(hw); 2974 2975 /* disable uio intr before callback unregister */ 2976 rte_intr_disable(intr_handle); 2977 2978 do { 2979 ret = rte_intr_callback_unregister(intr_handle, 2980 ixgbe_dev_interrupt_handler, dev); 2981 if (ret >= 0) { 2982 break; 2983 } else if (ret != -EAGAIN) { 2984 PMD_INIT_LOG(ERR, 2985 "intr callback unregister failed: %d", 2986 ret); 2987 } 2988 rte_delay_ms(100); 2989 } while (retries++ < (10 + IXGBE_LINK_UP_TIME)); 2990 2991 /* cancel the delay handler before remove dev */ 2992 rte_eal_alarm_cancel(ixgbe_dev_interrupt_delayed_handler, dev); 2993 2994 /* uninitialize PF if max_vfs not zero */ 2995 ixgbe_pf_host_uninit(dev); 2996 2997 /* remove all the fdir filters & hash */ 2998 ixgbe_fdir_filter_uninit(dev); 2999 3000 /* remove all the L2 tunnel filters & hash */ 3001 ixgbe_l2_tn_filter_uninit(dev); 3002 3003 /* Remove all ntuple filters of the device */ 3004 ixgbe_ntuple_filter_uninit(dev); 3005 3006 /* clear all the filters list */ 3007 ixgbe_filterlist_flush(); 3008 3009 /* Remove all Traffic Manager configuration */ 3010 ixgbe_tm_conf_uninit(dev); 3011 3012 #ifdef RTE_LIBRTE_SECURITY 3013 rte_free(dev->security_ctx); 3014 #endif 3015 3016 } 3017 3018 /* 3019 * Reset PF device. 3020 */ 3021 static int 3022 ixgbe_dev_reset(struct rte_eth_dev *dev) 3023 { 3024 int ret; 3025 3026 /* When a DPDK PMD PF begin to reset PF port, it should notify all 3027 * its VF to make them align with it. The detailed notification 3028 * mechanism is PMD specific. As to ixgbe PF, it is rather complex. 3029 * To avoid unexpected behavior in VF, currently reset of PF with 3030 * SR-IOV activation is not supported. It might be supported later. 3031 */ 3032 if (dev->data->sriov.active) 3033 return -ENOTSUP; 3034 3035 ret = eth_ixgbe_dev_uninit(dev); 3036 if (ret) 3037 return ret; 3038 3039 ret = eth_ixgbe_dev_init(dev, NULL); 3040 3041 return ret; 3042 } 3043 3044 static void 3045 ixgbe_read_stats_registers(struct ixgbe_hw *hw, 3046 struct ixgbe_hw_stats *hw_stats, 3047 struct ixgbe_macsec_stats *macsec_stats, 3048 uint64_t *total_missed_rx, uint64_t *total_qbrc, 3049 uint64_t *total_qprc, uint64_t *total_qprdc) 3050 { 3051 uint32_t bprc, lxon, lxoff, total; 3052 uint32_t delta_gprc = 0; 3053 unsigned i; 3054 /* Workaround for RX byte count not including CRC bytes when CRC 3055 * strip is enabled. CRC bytes are removed from counters when crc_strip 3056 * is disabled. 3057 */ 3058 int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) & 3059 IXGBE_HLREG0_RXCRCSTRP); 3060 3061 hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3062 hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 3063 hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); 3064 hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); 3065 3066 for (i = 0; i < 8; i++) { 3067 uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3068 3069 /* global total per queue */ 3070 hw_stats->mpc[i] += mp; 3071 /* Running comprehensive total for stats display */ 3072 *total_missed_rx += hw_stats->mpc[i]; 3073 if (hw->mac.type == ixgbe_mac_82598EB) { 3074 hw_stats->rnbc[i] += 3075 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3076 hw_stats->pxonrxc[i] += 3077 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 3078 hw_stats->pxoffrxc[i] += 3079 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 3080 } else { 3081 hw_stats->pxonrxc[i] += 3082 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 3083 hw_stats->pxoffrxc[i] += 3084 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 3085 hw_stats->pxon2offc[i] += 3086 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 3087 } 3088 hw_stats->pxontxc[i] += 3089 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 3090 hw_stats->pxofftxc[i] += 3091 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 3092 } 3093 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3094 uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3095 uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 3096 uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3097 3098 delta_gprc += delta_qprc; 3099 3100 hw_stats->qprc[i] += delta_qprc; 3101 hw_stats->qptc[i] += delta_qptc; 3102 3103 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 3104 hw_stats->qbrc[i] += 3105 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32); 3106 if (crc_strip == 0) 3107 hw_stats->qbrc[i] -= delta_qprc * RTE_ETHER_CRC_LEN; 3108 3109 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 3110 hw_stats->qbtc[i] += 3111 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32); 3112 3113 hw_stats->qprdc[i] += delta_qprdc; 3114 *total_qprdc += hw_stats->qprdc[i]; 3115 3116 *total_qprc += hw_stats->qprc[i]; 3117 *total_qbrc += hw_stats->qbrc[i]; 3118 } 3119 hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); 3120 hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC); 3121 hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3122 3123 /* 3124 * An errata states that gprc actually counts good + missed packets: 3125 * Workaround to set gprc to summated queue packet receives 3126 */ 3127 hw_stats->gprc = *total_qprc; 3128 3129 if (hw->mac.type != ixgbe_mac_82598EB) { 3130 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 3131 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 3132 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 3133 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32); 3134 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 3135 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 3136 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3137 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3138 } else { 3139 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3140 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3141 /* 82598 only has a counter in the high register */ 3142 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3143 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3144 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3145 } 3146 uint64_t old_tpr = hw_stats->tpr; 3147 3148 hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3149 hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT); 3150 3151 if (crc_strip == 0) 3152 hw_stats->gorc -= delta_gprc * RTE_ETHER_CRC_LEN; 3153 3154 uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC); 3155 hw_stats->gptc += delta_gptc; 3156 hw_stats->gotc -= delta_gptc * RTE_ETHER_CRC_LEN; 3157 hw_stats->tor -= (hw_stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN; 3158 3159 /* 3160 * Workaround: mprc hardware is incorrectly counting 3161 * broadcasts, so for now we subtract those. 3162 */ 3163 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3164 hw_stats->bprc += bprc; 3165 hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3166 if (hw->mac.type == ixgbe_mac_82598EB) 3167 hw_stats->mprc -= bprc; 3168 3169 hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3170 hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3171 hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3172 hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3173 hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3174 hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3175 3176 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3177 hw_stats->lxontxc += lxon; 3178 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3179 hw_stats->lxofftxc += lxoff; 3180 total = lxon + lxoff; 3181 3182 hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3183 hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3184 hw_stats->gptc -= total; 3185 hw_stats->mptc -= total; 3186 hw_stats->ptc64 -= total; 3187 hw_stats->gotc -= total * RTE_ETHER_MIN_LEN; 3188 3189 hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3190 hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3191 hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3192 hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3193 hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 3194 hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 3195 hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 3196 hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3197 hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3198 hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3199 hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3200 hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3201 hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3202 hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC); 3203 hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 3204 hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST); 3205 /* Only read FCOE on 82599 */ 3206 if (hw->mac.type != ixgbe_mac_82598EB) { 3207 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 3208 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 3209 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 3210 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 3211 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 3212 } 3213 3214 /* Flow Director Stats registers */ 3215 if (hw->mac.type != ixgbe_mac_82598EB) { 3216 hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 3217 hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 3218 hw_stats->fdirustat_add += IXGBE_READ_REG(hw, 3219 IXGBE_FDIRUSTAT) & 0xFFFF; 3220 hw_stats->fdirustat_remove += (IXGBE_READ_REG(hw, 3221 IXGBE_FDIRUSTAT) >> 16) & 0xFFFF; 3222 hw_stats->fdirfstat_fadd += IXGBE_READ_REG(hw, 3223 IXGBE_FDIRFSTAT) & 0xFFFF; 3224 hw_stats->fdirfstat_fremove += (IXGBE_READ_REG(hw, 3225 IXGBE_FDIRFSTAT) >> 16) & 0xFFFF; 3226 } 3227 /* MACsec Stats registers */ 3228 macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT); 3229 macsec_stats->out_pkts_encrypted += 3230 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE); 3231 macsec_stats->out_pkts_protected += 3232 IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP); 3233 macsec_stats->out_octets_encrypted += 3234 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE); 3235 macsec_stats->out_octets_protected += 3236 IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP); 3237 macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT); 3238 macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD); 3239 macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI); 3240 macsec_stats->in_pkts_unknownsci += 3241 IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI); 3242 macsec_stats->in_octets_decrypted += 3243 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD); 3244 macsec_stats->in_octets_validated += 3245 IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV); 3246 macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH); 3247 macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY); 3248 macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE); 3249 for (i = 0; i < 2; i++) { 3250 macsec_stats->in_pkts_ok += 3251 IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i)); 3252 macsec_stats->in_pkts_invalid += 3253 IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i)); 3254 macsec_stats->in_pkts_notvalid += 3255 IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i)); 3256 } 3257 macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA); 3258 macsec_stats->in_pkts_notusingsa += 3259 IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA); 3260 } 3261 3262 /* 3263 * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c 3264 */ 3265 static int 3266 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3267 { 3268 struct ixgbe_hw *hw = 3269 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3270 struct ixgbe_hw_stats *hw_stats = 3271 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3272 struct ixgbe_macsec_stats *macsec_stats = 3273 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3274 dev->data->dev_private); 3275 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3276 unsigned i; 3277 3278 total_missed_rx = 0; 3279 total_qbrc = 0; 3280 total_qprc = 0; 3281 total_qprdc = 0; 3282 3283 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3284 &total_qbrc, &total_qprc, &total_qprdc); 3285 3286 if (stats == NULL) 3287 return -EINVAL; 3288 3289 /* Fill out the rte_eth_stats statistics structure */ 3290 stats->ipackets = total_qprc; 3291 stats->ibytes = total_qbrc; 3292 stats->opackets = hw_stats->gptc; 3293 stats->obytes = hw_stats->gotc; 3294 3295 for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) { 3296 stats->q_ipackets[i] = hw_stats->qprc[i]; 3297 stats->q_opackets[i] = hw_stats->qptc[i]; 3298 stats->q_ibytes[i] = hw_stats->qbrc[i]; 3299 stats->q_obytes[i] = hw_stats->qbtc[i]; 3300 stats->q_errors[i] = hw_stats->qprdc[i]; 3301 } 3302 3303 /* Rx Errors */ 3304 stats->imissed = total_missed_rx; 3305 stats->ierrors = hw_stats->crcerrs + 3306 hw_stats->mspdc + 3307 hw_stats->rlec + 3308 hw_stats->ruc + 3309 hw_stats->roc + 3310 hw_stats->illerrc + 3311 hw_stats->errbc + 3312 hw_stats->rfc + 3313 hw_stats->fccrc + 3314 hw_stats->fclast; 3315 3316 /* Tx Errors */ 3317 stats->oerrors = 0; 3318 return 0; 3319 } 3320 3321 static int 3322 ixgbe_dev_stats_reset(struct rte_eth_dev *dev) 3323 { 3324 struct ixgbe_hw_stats *stats = 3325 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3326 3327 /* HW registers are cleared on read */ 3328 ixgbe_dev_stats_get(dev, NULL); 3329 3330 /* Reset software totals */ 3331 memset(stats, 0, sizeof(*stats)); 3332 3333 return 0; 3334 } 3335 3336 /* This function calculates the number of xstats based on the current config */ 3337 static unsigned 3338 ixgbe_xstats_calc_num(void) { 3339 return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS + 3340 (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) + 3341 (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES); 3342 } 3343 3344 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3345 struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int size) 3346 { 3347 const unsigned cnt_stats = ixgbe_xstats_calc_num(); 3348 unsigned stat, i, count; 3349 3350 if (xstats_names != NULL) { 3351 count = 0; 3352 3353 /* Note: limit >= cnt_stats checked upstream 3354 * in rte_eth_xstats_names() 3355 */ 3356 3357 /* Extended stats from ixgbe_hw_stats */ 3358 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3359 strlcpy(xstats_names[count].name, 3360 rte_ixgbe_stats_strings[i].name, 3361 sizeof(xstats_names[count].name)); 3362 count++; 3363 } 3364 3365 /* MACsec Stats */ 3366 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3367 strlcpy(xstats_names[count].name, 3368 rte_ixgbe_macsec_strings[i].name, 3369 sizeof(xstats_names[count].name)); 3370 count++; 3371 } 3372 3373 /* RX Priority Stats */ 3374 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3375 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3376 snprintf(xstats_names[count].name, 3377 sizeof(xstats_names[count].name), 3378 "rx_priority%u_%s", i, 3379 rte_ixgbe_rxq_strings[stat].name); 3380 count++; 3381 } 3382 } 3383 3384 /* TX Priority Stats */ 3385 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3386 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3387 snprintf(xstats_names[count].name, 3388 sizeof(xstats_names[count].name), 3389 "tx_priority%u_%s", i, 3390 rte_ixgbe_txq_strings[stat].name); 3391 count++; 3392 } 3393 } 3394 } 3395 return cnt_stats; 3396 } 3397 3398 static int ixgbe_dev_xstats_get_names_by_id( 3399 struct rte_eth_dev *dev, 3400 struct rte_eth_xstat_name *xstats_names, 3401 const uint64_t *ids, 3402 unsigned int limit) 3403 { 3404 if (!ids) { 3405 const unsigned int cnt_stats = ixgbe_xstats_calc_num(); 3406 unsigned int stat, i, count; 3407 3408 if (xstats_names != NULL) { 3409 count = 0; 3410 3411 /* Note: limit >= cnt_stats checked upstream 3412 * in rte_eth_xstats_names() 3413 */ 3414 3415 /* Extended stats from ixgbe_hw_stats */ 3416 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3417 strlcpy(xstats_names[count].name, 3418 rte_ixgbe_stats_strings[i].name, 3419 sizeof(xstats_names[count].name)); 3420 count++; 3421 } 3422 3423 /* MACsec Stats */ 3424 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3425 strlcpy(xstats_names[count].name, 3426 rte_ixgbe_macsec_strings[i].name, 3427 sizeof(xstats_names[count].name)); 3428 count++; 3429 } 3430 3431 /* RX Priority Stats */ 3432 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3433 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3434 snprintf(xstats_names[count].name, 3435 sizeof(xstats_names[count].name), 3436 "rx_priority%u_%s", i, 3437 rte_ixgbe_rxq_strings[stat].name); 3438 count++; 3439 } 3440 } 3441 3442 /* TX Priority Stats */ 3443 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3444 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3445 snprintf(xstats_names[count].name, 3446 sizeof(xstats_names[count].name), 3447 "tx_priority%u_%s", i, 3448 rte_ixgbe_txq_strings[stat].name); 3449 count++; 3450 } 3451 } 3452 } 3453 return cnt_stats; 3454 } 3455 3456 uint16_t i; 3457 uint16_t size = ixgbe_xstats_calc_num(); 3458 struct rte_eth_xstat_name xstats_names_copy[size]; 3459 3460 ixgbe_dev_xstats_get_names_by_id(dev, xstats_names_copy, NULL, 3461 size); 3462 3463 for (i = 0; i < limit; i++) { 3464 if (ids[i] >= size) { 3465 PMD_INIT_LOG(ERR, "id value isn't valid"); 3466 return -1; 3467 } 3468 strcpy(xstats_names[i].name, 3469 xstats_names_copy[ids[i]].name); 3470 } 3471 return limit; 3472 } 3473 3474 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 3475 struct rte_eth_xstat_name *xstats_names, unsigned limit) 3476 { 3477 unsigned i; 3478 3479 if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL) 3480 return -ENOMEM; 3481 3482 if (xstats_names != NULL) 3483 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) 3484 strlcpy(xstats_names[i].name, 3485 rte_ixgbevf_stats_strings[i].name, 3486 sizeof(xstats_names[i].name)); 3487 return IXGBEVF_NB_XSTATS; 3488 } 3489 3490 static int 3491 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3492 unsigned n) 3493 { 3494 struct ixgbe_hw *hw = 3495 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3496 struct ixgbe_hw_stats *hw_stats = 3497 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3498 struct ixgbe_macsec_stats *macsec_stats = 3499 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3500 dev->data->dev_private); 3501 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3502 unsigned i, stat, count = 0; 3503 3504 count = ixgbe_xstats_calc_num(); 3505 3506 if (n < count) 3507 return count; 3508 3509 total_missed_rx = 0; 3510 total_qbrc = 0; 3511 total_qprc = 0; 3512 total_qprdc = 0; 3513 3514 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, 3515 &total_qbrc, &total_qprc, &total_qprdc); 3516 3517 /* If this is a reset xstats is NULL, and we have cleared the 3518 * registers by reading them. 3519 */ 3520 if (!xstats) 3521 return 0; 3522 3523 /* Extended stats from ixgbe_hw_stats */ 3524 count = 0; 3525 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3526 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3527 rte_ixgbe_stats_strings[i].offset); 3528 xstats[count].id = count; 3529 count++; 3530 } 3531 3532 /* MACsec Stats */ 3533 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3534 xstats[count].value = *(uint64_t *)(((char *)macsec_stats) + 3535 rte_ixgbe_macsec_strings[i].offset); 3536 xstats[count].id = count; 3537 count++; 3538 } 3539 3540 /* RX Priority Stats */ 3541 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3542 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3543 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3544 rte_ixgbe_rxq_strings[stat].offset + 3545 (sizeof(uint64_t) * i)); 3546 xstats[count].id = count; 3547 count++; 3548 } 3549 } 3550 3551 /* TX Priority Stats */ 3552 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3553 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3554 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 3555 rte_ixgbe_txq_strings[stat].offset + 3556 (sizeof(uint64_t) * i)); 3557 xstats[count].id = count; 3558 count++; 3559 } 3560 } 3561 return count; 3562 } 3563 3564 static int 3565 ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, 3566 uint64_t *values, unsigned int n) 3567 { 3568 if (!ids) { 3569 struct ixgbe_hw *hw = 3570 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3571 struct ixgbe_hw_stats *hw_stats = 3572 IXGBE_DEV_PRIVATE_TO_STATS( 3573 dev->data->dev_private); 3574 struct ixgbe_macsec_stats *macsec_stats = 3575 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3576 dev->data->dev_private); 3577 uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; 3578 unsigned int i, stat, count = 0; 3579 3580 count = ixgbe_xstats_calc_num(); 3581 3582 if (!ids && n < count) 3583 return count; 3584 3585 total_missed_rx = 0; 3586 total_qbrc = 0; 3587 total_qprc = 0; 3588 total_qprdc = 0; 3589 3590 ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, 3591 &total_missed_rx, &total_qbrc, &total_qprc, 3592 &total_qprdc); 3593 3594 /* If this is a reset xstats is NULL, and we have cleared the 3595 * registers by reading them. 3596 */ 3597 if (!ids && !values) 3598 return 0; 3599 3600 /* Extended stats from ixgbe_hw_stats */ 3601 count = 0; 3602 for (i = 0; i < IXGBE_NB_HW_STATS; i++) { 3603 values[count] = *(uint64_t *)(((char *)hw_stats) + 3604 rte_ixgbe_stats_strings[i].offset); 3605 count++; 3606 } 3607 3608 /* MACsec Stats */ 3609 for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { 3610 values[count] = *(uint64_t *)(((char *)macsec_stats) + 3611 rte_ixgbe_macsec_strings[i].offset); 3612 count++; 3613 } 3614 3615 /* RX Priority Stats */ 3616 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { 3617 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { 3618 values[count] = 3619 *(uint64_t *)(((char *)hw_stats) + 3620 rte_ixgbe_rxq_strings[stat].offset + 3621 (sizeof(uint64_t) * i)); 3622 count++; 3623 } 3624 } 3625 3626 /* TX Priority Stats */ 3627 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { 3628 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { 3629 values[count] = 3630 *(uint64_t *)(((char *)hw_stats) + 3631 rte_ixgbe_txq_strings[stat].offset + 3632 (sizeof(uint64_t) * i)); 3633 count++; 3634 } 3635 } 3636 return count; 3637 } 3638 3639 uint16_t i; 3640 uint16_t size = ixgbe_xstats_calc_num(); 3641 uint64_t values_copy[size]; 3642 3643 ixgbe_dev_xstats_get_by_id(dev, NULL, values_copy, size); 3644 3645 for (i = 0; i < n; i++) { 3646 if (ids[i] >= size) { 3647 PMD_INIT_LOG(ERR, "id value isn't valid"); 3648 return -1; 3649 } 3650 values[i] = values_copy[ids[i]]; 3651 } 3652 return n; 3653 } 3654 3655 static int 3656 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) 3657 { 3658 struct ixgbe_hw_stats *stats = 3659 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3660 struct ixgbe_macsec_stats *macsec_stats = 3661 IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( 3662 dev->data->dev_private); 3663 3664 unsigned count = ixgbe_xstats_calc_num(); 3665 3666 /* HW registers are cleared on read */ 3667 ixgbe_dev_xstats_get(dev, NULL, count); 3668 3669 /* Reset software totals */ 3670 memset(stats, 0, sizeof(*stats)); 3671 memset(macsec_stats, 0, sizeof(*macsec_stats)); 3672 3673 return 0; 3674 } 3675 3676 static void 3677 ixgbevf_update_stats(struct rte_eth_dev *dev) 3678 { 3679 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3680 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3681 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3682 3683 /* Good Rx packet, include VF loopback */ 3684 UPDATE_VF_STAT(IXGBE_VFGPRC, 3685 hw_stats->last_vfgprc, hw_stats->vfgprc); 3686 3687 /* Good Rx octets, include VF loopback */ 3688 UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3689 hw_stats->last_vfgorc, hw_stats->vfgorc); 3690 3691 /* Good Tx packet, include VF loopback */ 3692 UPDATE_VF_STAT(IXGBE_VFGPTC, 3693 hw_stats->last_vfgptc, hw_stats->vfgptc); 3694 3695 /* Good Tx octets, include VF loopback */ 3696 UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3697 hw_stats->last_vfgotc, hw_stats->vfgotc); 3698 3699 /* Rx Multicst Packet */ 3700 UPDATE_VF_STAT(IXGBE_VFMPRC, 3701 hw_stats->last_vfmprc, hw_stats->vfmprc); 3702 } 3703 3704 static int 3705 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 3706 unsigned n) 3707 { 3708 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3709 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3710 unsigned i; 3711 3712 if (n < IXGBEVF_NB_XSTATS) 3713 return IXGBEVF_NB_XSTATS; 3714 3715 ixgbevf_update_stats(dev); 3716 3717 if (!xstats) 3718 return 0; 3719 3720 /* Extended stats */ 3721 for (i = 0; i < IXGBEVF_NB_XSTATS; i++) { 3722 xstats[i].id = i; 3723 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + 3724 rte_ixgbevf_stats_strings[i].offset); 3725 } 3726 3727 return IXGBEVF_NB_XSTATS; 3728 } 3729 3730 static int 3731 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 3732 { 3733 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3734 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3735 3736 ixgbevf_update_stats(dev); 3737 3738 if (stats == NULL) 3739 return -EINVAL; 3740 3741 stats->ipackets = hw_stats->vfgprc; 3742 stats->ibytes = hw_stats->vfgorc; 3743 stats->opackets = hw_stats->vfgptc; 3744 stats->obytes = hw_stats->vfgotc; 3745 return 0; 3746 } 3747 3748 static int 3749 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev) 3750 { 3751 struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) 3752 IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 3753 3754 /* Sync HW register to the last stats */ 3755 ixgbevf_dev_stats_get(dev, NULL); 3756 3757 /* reset HW current stats*/ 3758 hw_stats->vfgprc = 0; 3759 hw_stats->vfgorc = 0; 3760 hw_stats->vfgptc = 0; 3761 hw_stats->vfgotc = 0; 3762 3763 return 0; 3764 } 3765 3766 static int 3767 ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) 3768 { 3769 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3770 u16 eeprom_verh, eeprom_verl; 3771 u32 etrack_id; 3772 int ret; 3773 3774 ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); 3775 ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); 3776 3777 etrack_id = (eeprom_verh << 16) | eeprom_verl; 3778 ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); 3779 3780 ret += 1; /* add the size of '\0' */ 3781 if (fw_size < (u32)ret) 3782 return ret; 3783 else 3784 return 0; 3785 } 3786 3787 static int 3788 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 3789 { 3790 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3791 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3792 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3793 3794 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3795 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3796 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3797 /* 3798 * When DCB/VT is off, maximum number of queues changes, 3799 * except for 82598EB, which remains constant. 3800 */ 3801 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE && 3802 hw->mac.type != ixgbe_mac_82598EB) 3803 dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES; 3804 } 3805 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */ 3806 dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */ 3807 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3808 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3809 dev_info->max_vfs = pci_dev->max_vfs; 3810 if (hw->mac.type == ixgbe_mac_82598EB) 3811 dev_info->max_vmdq_pools = ETH_16_POOLS; 3812 else 3813 dev_info->max_vmdq_pools = ETH_64_POOLS; 3814 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3815 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 3816 dev_info->vmdq_queue_num = dev_info->max_rx_queues; 3817 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3818 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3819 dev_info->rx_queue_offload_capa); 3820 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3821 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3822 3823 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3824 .rx_thresh = { 3825 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3826 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3827 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3828 }, 3829 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3830 .rx_drop_en = 0, 3831 .offloads = 0, 3832 }; 3833 3834 dev_info->default_txconf = (struct rte_eth_txconf) { 3835 .tx_thresh = { 3836 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3837 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3838 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3839 }, 3840 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3841 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3842 .offloads = 0, 3843 }; 3844 3845 dev_info->rx_desc_lim = rx_desc_lim; 3846 dev_info->tx_desc_lim = tx_desc_lim; 3847 3848 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3849 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3850 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3851 3852 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; 3853 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 3854 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 3855 dev_info->speed_capa = ETH_LINK_SPEED_10M | 3856 ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G; 3857 3858 if (hw->mac.type == ixgbe_mac_X540 || 3859 hw->mac.type == ixgbe_mac_X540_vf || 3860 hw->mac.type == ixgbe_mac_X550 || 3861 hw->mac.type == ixgbe_mac_X550_vf) { 3862 dev_info->speed_capa |= ETH_LINK_SPEED_100M; 3863 } 3864 if (hw->mac.type == ixgbe_mac_X550) { 3865 dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; 3866 dev_info->speed_capa |= ETH_LINK_SPEED_5G; 3867 } 3868 3869 /* Driver-preferred Rx/Tx parameters */ 3870 dev_info->default_rxportconf.burst_size = 32; 3871 dev_info->default_txportconf.burst_size = 32; 3872 dev_info->default_rxportconf.nb_queues = 1; 3873 dev_info->default_txportconf.nb_queues = 1; 3874 dev_info->default_rxportconf.ring_size = 256; 3875 dev_info->default_txportconf.ring_size = 256; 3876 3877 return 0; 3878 } 3879 3880 static const uint32_t * 3881 ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) 3882 { 3883 static const uint32_t ptypes[] = { 3884 /* For non-vec functions, 3885 * refers to ixgbe_rxd_pkt_info_to_pkt_type(); 3886 * for vec functions, 3887 * refers to _recv_raw_pkts_vec(). 3888 */ 3889 RTE_PTYPE_L2_ETHER, 3890 RTE_PTYPE_L3_IPV4, 3891 RTE_PTYPE_L3_IPV4_EXT, 3892 RTE_PTYPE_L3_IPV6, 3893 RTE_PTYPE_L3_IPV6_EXT, 3894 RTE_PTYPE_L4_SCTP, 3895 RTE_PTYPE_L4_TCP, 3896 RTE_PTYPE_L4_UDP, 3897 RTE_PTYPE_TUNNEL_IP, 3898 RTE_PTYPE_INNER_L3_IPV6, 3899 RTE_PTYPE_INNER_L3_IPV6_EXT, 3900 RTE_PTYPE_INNER_L4_TCP, 3901 RTE_PTYPE_INNER_L4_UDP, 3902 RTE_PTYPE_UNKNOWN 3903 }; 3904 3905 if (dev->rx_pkt_burst == ixgbe_recv_pkts || 3906 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc || 3907 dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || 3908 dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) 3909 return ptypes; 3910 3911 #if defined(RTE_ARCH_X86) 3912 if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || 3913 dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) 3914 return ptypes; 3915 #endif 3916 return NULL; 3917 } 3918 3919 static int 3920 ixgbevf_dev_info_get(struct rte_eth_dev *dev, 3921 struct rte_eth_dev_info *dev_info) 3922 { 3923 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 3924 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3925 3926 dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; 3927 dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; 3928 dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */ 3929 dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */ 3930 dev_info->max_mtu = dev_info->max_rx_pktlen - IXGBE_ETH_OVERHEAD; 3931 dev_info->max_mac_addrs = hw->mac.num_rar_entries; 3932 dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; 3933 dev_info->max_vfs = pci_dev->max_vfs; 3934 if (hw->mac.type == ixgbe_mac_82598EB) 3935 dev_info->max_vmdq_pools = ETH_16_POOLS; 3936 else 3937 dev_info->max_vmdq_pools = ETH_64_POOLS; 3938 dev_info->rx_queue_offload_capa = ixgbe_get_rx_queue_offloads(dev); 3939 dev_info->rx_offload_capa = (ixgbe_get_rx_port_offloads(dev) | 3940 dev_info->rx_queue_offload_capa); 3941 dev_info->tx_queue_offload_capa = ixgbe_get_tx_queue_offloads(dev); 3942 dev_info->tx_offload_capa = ixgbe_get_tx_port_offloads(dev); 3943 dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t); 3944 dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type); 3945 dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL; 3946 3947 dev_info->default_rxconf = (struct rte_eth_rxconf) { 3948 .rx_thresh = { 3949 .pthresh = IXGBE_DEFAULT_RX_PTHRESH, 3950 .hthresh = IXGBE_DEFAULT_RX_HTHRESH, 3951 .wthresh = IXGBE_DEFAULT_RX_WTHRESH, 3952 }, 3953 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH, 3954 .rx_drop_en = 0, 3955 .offloads = 0, 3956 }; 3957 3958 dev_info->default_txconf = (struct rte_eth_txconf) { 3959 .tx_thresh = { 3960 .pthresh = IXGBE_DEFAULT_TX_PTHRESH, 3961 .hthresh = IXGBE_DEFAULT_TX_HTHRESH, 3962 .wthresh = IXGBE_DEFAULT_TX_WTHRESH, 3963 }, 3964 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH, 3965 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH, 3966 .offloads = 0, 3967 }; 3968 3969 dev_info->rx_desc_lim = rx_desc_lim; 3970 dev_info->tx_desc_lim = tx_desc_lim; 3971 3972 return 0; 3973 } 3974 3975 static int 3976 ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, 3977 int *link_up, int wait_to_complete) 3978 { 3979 struct ixgbe_adapter *adapter = container_of(hw, 3980 struct ixgbe_adapter, hw); 3981 struct ixgbe_mbx_info *mbx = &hw->mbx; 3982 struct ixgbe_mac_info *mac = &hw->mac; 3983 uint32_t links_reg, in_msg; 3984 int ret_val = 0; 3985 3986 /* If we were hit with a reset drop the link */ 3987 if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) 3988 mac->get_link_status = true; 3989 3990 if (!mac->get_link_status) 3991 goto out; 3992 3993 /* if link status is down no point in checking to see if pf is up */ 3994 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 3995 if (!(links_reg & IXGBE_LINKS_UP)) 3996 goto out; 3997 3998 /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs 3999 * before the link status is correct 4000 */ 4001 if (mac->type == ixgbe_mac_82599_vf && wait_to_complete) { 4002 int i; 4003 4004 for (i = 0; i < 5; i++) { 4005 rte_delay_us(100); 4006 links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); 4007 4008 if (!(links_reg & IXGBE_LINKS_UP)) 4009 goto out; 4010 } 4011 } 4012 4013 switch (links_reg & IXGBE_LINKS_SPEED_82599) { 4014 case IXGBE_LINKS_SPEED_10G_82599: 4015 *speed = IXGBE_LINK_SPEED_10GB_FULL; 4016 if (hw->mac.type >= ixgbe_mac_X550) { 4017 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4018 *speed = IXGBE_LINK_SPEED_2_5GB_FULL; 4019 } 4020 break; 4021 case IXGBE_LINKS_SPEED_1G_82599: 4022 *speed = IXGBE_LINK_SPEED_1GB_FULL; 4023 break; 4024 case IXGBE_LINKS_SPEED_100_82599: 4025 *speed = IXGBE_LINK_SPEED_100_FULL; 4026 if (hw->mac.type == ixgbe_mac_X550) { 4027 if (links_reg & IXGBE_LINKS_SPEED_NON_STD) 4028 *speed = IXGBE_LINK_SPEED_5GB_FULL; 4029 } 4030 break; 4031 case IXGBE_LINKS_SPEED_10_X550EM_A: 4032 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4033 /* Since Reserved in older MAC's */ 4034 if (hw->mac.type >= ixgbe_mac_X550) 4035 *speed = IXGBE_LINK_SPEED_10_FULL; 4036 break; 4037 default: 4038 *speed = IXGBE_LINK_SPEED_UNKNOWN; 4039 } 4040 4041 if (wait_to_complete == 0 && adapter->pflink_fullchk == 0) { 4042 if (*speed == IXGBE_LINK_SPEED_UNKNOWN) 4043 mac->get_link_status = true; 4044 else 4045 mac->get_link_status = false; 4046 4047 goto out; 4048 } 4049 4050 /* if the read failed it could just be a mailbox collision, best wait 4051 * until we are called again and don't report an error 4052 */ 4053 if (mbx->ops.read(hw, &in_msg, 1, 0)) 4054 goto out; 4055 4056 if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { 4057 /* msg is not CTS and is NACK we must have lost CTS status */ 4058 if (in_msg & IXGBE_VT_MSGTYPE_NACK) 4059 mac->get_link_status = false; 4060 goto out; 4061 } 4062 4063 /* the pf is talking, if we timed out in the past we reinit */ 4064 if (!mbx->timeout) { 4065 ret_val = -1; 4066 goto out; 4067 } 4068 4069 /* if we passed all the tests above then the link is up and we no 4070 * longer need to check for link 4071 */ 4072 mac->get_link_status = false; 4073 4074 out: 4075 *link_up = !mac->get_link_status; 4076 return ret_val; 4077 } 4078 4079 static void 4080 ixgbe_dev_setup_link_alarm_handler(void *param) 4081 { 4082 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4083 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4084 struct ixgbe_interrupt *intr = 4085 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4086 u32 speed; 4087 bool autoneg = false; 4088 4089 speed = hw->phy.autoneg_advertised; 4090 if (!speed) 4091 ixgbe_get_link_capabilities(hw, &speed, &autoneg); 4092 4093 ixgbe_setup_link(hw, speed, true); 4094 4095 intr->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4096 } 4097 4098 /* return 0 means link status changed, -1 means not changed */ 4099 int 4100 ixgbe_dev_link_update_share(struct rte_eth_dev *dev, 4101 int wait_to_complete, int vf) 4102 { 4103 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4104 struct rte_eth_link link; 4105 ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN; 4106 struct ixgbe_interrupt *intr = 4107 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4108 int link_up; 4109 int diag; 4110 int wait = 1; 4111 4112 memset(&link, 0, sizeof(link)); 4113 link.link_status = ETH_LINK_DOWN; 4114 link.link_speed = ETH_SPEED_NUM_NONE; 4115 link.link_duplex = ETH_LINK_HALF_DUPLEX; 4116 link.link_autoneg = ETH_LINK_AUTONEG; 4117 4118 hw->mac.get_link_status = true; 4119 4120 if (intr->flags & IXGBE_FLAG_NEED_LINK_CONFIG) 4121 return rte_eth_linkstatus_set(dev, &link); 4122 4123 /* check if it needs to wait to complete, if lsc interrupt is enabled */ 4124 if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) 4125 wait = 0; 4126 4127 if (vf) 4128 diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); 4129 else 4130 diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); 4131 4132 if (diag != 0) { 4133 link.link_speed = ETH_SPEED_NUM_100M; 4134 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4135 return rte_eth_linkstatus_set(dev, &link); 4136 } 4137 4138 if (link_up == 0) { 4139 if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { 4140 intr->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 4141 rte_eal_alarm_set(10, 4142 ixgbe_dev_setup_link_alarm_handler, dev); 4143 } 4144 return rte_eth_linkstatus_set(dev, &link); 4145 } 4146 4147 link.link_status = ETH_LINK_UP; 4148 link.link_duplex = ETH_LINK_FULL_DUPLEX; 4149 4150 switch (link_speed) { 4151 default: 4152 case IXGBE_LINK_SPEED_UNKNOWN: 4153 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || 4154 hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) 4155 link.link_speed = ETH_SPEED_NUM_10M; 4156 else 4157 link.link_speed = ETH_SPEED_NUM_100M; 4158 break; 4159 4160 case IXGBE_LINK_SPEED_100_FULL: 4161 link.link_speed = ETH_SPEED_NUM_100M; 4162 break; 4163 4164 case IXGBE_LINK_SPEED_1GB_FULL: 4165 link.link_speed = ETH_SPEED_NUM_1G; 4166 break; 4167 4168 case IXGBE_LINK_SPEED_2_5GB_FULL: 4169 link.link_speed = ETH_SPEED_NUM_2_5G; 4170 break; 4171 4172 case IXGBE_LINK_SPEED_5GB_FULL: 4173 link.link_speed = ETH_SPEED_NUM_5G; 4174 break; 4175 4176 case IXGBE_LINK_SPEED_10GB_FULL: 4177 link.link_speed = ETH_SPEED_NUM_10G; 4178 break; 4179 } 4180 4181 return rte_eth_linkstatus_set(dev, &link); 4182 } 4183 4184 static int 4185 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4186 { 4187 return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); 4188 } 4189 4190 static int 4191 ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 4192 { 4193 return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); 4194 } 4195 4196 static int 4197 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) 4198 { 4199 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4200 uint32_t fctrl; 4201 4202 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4203 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4204 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4205 4206 return 0; 4207 } 4208 4209 static int 4210 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev) 4211 { 4212 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4213 uint32_t fctrl; 4214 4215 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4216 fctrl &= (~IXGBE_FCTRL_UPE); 4217 if (dev->data->all_multicast == 1) 4218 fctrl |= IXGBE_FCTRL_MPE; 4219 else 4220 fctrl &= (~IXGBE_FCTRL_MPE); 4221 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4222 4223 return 0; 4224 } 4225 4226 static int 4227 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev) 4228 { 4229 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4230 uint32_t fctrl; 4231 4232 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4233 fctrl |= IXGBE_FCTRL_MPE; 4234 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4235 4236 return 0; 4237 } 4238 4239 static int 4240 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) 4241 { 4242 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4243 uint32_t fctrl; 4244 4245 if (dev->data->promiscuous == 1) 4246 return 0; /* must remain in all_multicast mode */ 4247 4248 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4249 fctrl &= (~IXGBE_FCTRL_MPE); 4250 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4251 4252 return 0; 4253 } 4254 4255 /** 4256 * It clears the interrupt causes and enables the interrupt. 4257 * It will be called once only during nic initialized. 4258 * 4259 * @param dev 4260 * Pointer to struct rte_eth_dev. 4261 * @param on 4262 * Enable or Disable. 4263 * 4264 * @return 4265 * - On success, zero. 4266 * - On failure, a negative value. 4267 */ 4268 static int 4269 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) 4270 { 4271 struct ixgbe_interrupt *intr = 4272 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4273 4274 ixgbe_dev_link_status_print(dev); 4275 if (on) 4276 intr->mask |= IXGBE_EICR_LSC; 4277 else 4278 intr->mask &= ~IXGBE_EICR_LSC; 4279 4280 return 0; 4281 } 4282 4283 /** 4284 * It clears the interrupt causes and enables the interrupt. 4285 * It will be called once only during nic initialized. 4286 * 4287 * @param dev 4288 * Pointer to struct rte_eth_dev. 4289 * 4290 * @return 4291 * - On success, zero. 4292 * - On failure, a negative value. 4293 */ 4294 static int 4295 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) 4296 { 4297 struct ixgbe_interrupt *intr = 4298 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4299 4300 intr->mask |= IXGBE_EICR_RTX_QUEUE; 4301 4302 return 0; 4303 } 4304 4305 /** 4306 * It clears the interrupt causes and enables the interrupt. 4307 * It will be called once only during nic initialized. 4308 * 4309 * @param dev 4310 * Pointer to struct rte_eth_dev. 4311 * 4312 * @return 4313 * - On success, zero. 4314 * - On failure, a negative value. 4315 */ 4316 static int 4317 ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev) 4318 { 4319 struct ixgbe_interrupt *intr = 4320 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4321 4322 intr->mask |= IXGBE_EICR_LINKSEC; 4323 4324 return 0; 4325 } 4326 4327 /* 4328 * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update. 4329 * 4330 * @param dev 4331 * Pointer to struct rte_eth_dev. 4332 * 4333 * @return 4334 * - On success, zero. 4335 * - On failure, a negative value. 4336 */ 4337 static int 4338 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) 4339 { 4340 uint32_t eicr; 4341 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4342 struct ixgbe_interrupt *intr = 4343 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4344 4345 /* clear all cause mask */ 4346 ixgbe_disable_intr(hw); 4347 4348 /* read-on-clear nic registers here */ 4349 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4350 PMD_DRV_LOG(DEBUG, "eicr %x", eicr); 4351 4352 intr->flags = 0; 4353 4354 /* set flag for async link update */ 4355 if (eicr & IXGBE_EICR_LSC) 4356 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4357 4358 if (eicr & IXGBE_EICR_MAILBOX) 4359 intr->flags |= IXGBE_FLAG_MAILBOX; 4360 4361 if (eicr & IXGBE_EICR_LINKSEC) 4362 intr->flags |= IXGBE_FLAG_MACSEC; 4363 4364 if (hw->mac.type == ixgbe_mac_X550EM_x && 4365 hw->phy.type == ixgbe_phy_x550em_ext_t && 4366 (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x)) 4367 intr->flags |= IXGBE_FLAG_PHY_INTERRUPT; 4368 4369 return 0; 4370 } 4371 4372 /** 4373 * It gets and then prints the link status. 4374 * 4375 * @param dev 4376 * Pointer to struct rte_eth_dev. 4377 * 4378 * @return 4379 * - On success, zero. 4380 * - On failure, a negative value. 4381 */ 4382 static void 4383 ixgbe_dev_link_status_print(struct rte_eth_dev *dev) 4384 { 4385 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4386 struct rte_eth_link link; 4387 4388 rte_eth_linkstatus_get(dev, &link); 4389 4390 if (link.link_status) { 4391 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 4392 (int)(dev->data->port_id), 4393 (unsigned)link.link_speed, 4394 link.link_duplex == ETH_LINK_FULL_DUPLEX ? 4395 "full-duplex" : "half-duplex"); 4396 } else { 4397 PMD_INIT_LOG(INFO, " Port %d: Link Down", 4398 (int)(dev->data->port_id)); 4399 } 4400 PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, 4401 pci_dev->addr.domain, 4402 pci_dev->addr.bus, 4403 pci_dev->addr.devid, 4404 pci_dev->addr.function); 4405 } 4406 4407 /* 4408 * It executes link_update after knowing an interrupt occurred. 4409 * 4410 * @param dev 4411 * Pointer to struct rte_eth_dev. 4412 * 4413 * @return 4414 * - On success, zero. 4415 * - On failure, a negative value. 4416 */ 4417 static int 4418 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) 4419 { 4420 struct ixgbe_interrupt *intr = 4421 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4422 int64_t timeout; 4423 struct ixgbe_hw *hw = 4424 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4425 4426 PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags); 4427 4428 if (intr->flags & IXGBE_FLAG_MAILBOX) { 4429 ixgbe_pf_mbx_process(dev); 4430 intr->flags &= ~IXGBE_FLAG_MAILBOX; 4431 } 4432 4433 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4434 ixgbe_handle_lasi(hw); 4435 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4436 } 4437 4438 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4439 struct rte_eth_link link; 4440 4441 /* get the link status before link update, for predicting later */ 4442 rte_eth_linkstatus_get(dev, &link); 4443 4444 ixgbe_dev_link_update(dev, 0); 4445 4446 /* likely to up */ 4447 if (!link.link_status) 4448 /* handle it 1 sec later, wait it being stable */ 4449 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT; 4450 /* likely to down */ 4451 else 4452 /* handle it 4 sec later, wait it being stable */ 4453 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; 4454 4455 ixgbe_dev_link_status_print(dev); 4456 if (rte_eal_alarm_set(timeout * 1000, 4457 ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) 4458 PMD_DRV_LOG(ERR, "Error setting alarm"); 4459 else { 4460 /* remember original mask */ 4461 intr->mask_original = intr->mask; 4462 /* only disable lsc interrupt */ 4463 intr->mask &= ~IXGBE_EIMS_LSC; 4464 } 4465 } 4466 4467 PMD_DRV_LOG(DEBUG, "enable intr immediately"); 4468 ixgbe_enable_intr(dev); 4469 4470 return 0; 4471 } 4472 4473 /** 4474 * Interrupt handler which shall be registered for alarm callback for delayed 4475 * handling specific interrupt to wait for the stable nic state. As the 4476 * NIC interrupt state is not stable for ixgbe after link is just down, 4477 * it needs to wait 4 seconds to get the stable status. 4478 * 4479 * @param handle 4480 * Pointer to interrupt handle. 4481 * @param param 4482 * The address of parameter (struct rte_eth_dev *) regsitered before. 4483 * 4484 * @return 4485 * void 4486 */ 4487 static void 4488 ixgbe_dev_interrupt_delayed_handler(void *param) 4489 { 4490 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4491 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 4492 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 4493 struct ixgbe_interrupt *intr = 4494 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 4495 struct ixgbe_hw *hw = 4496 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4497 uint32_t eicr; 4498 4499 ixgbe_disable_intr(hw); 4500 4501 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4502 if (eicr & IXGBE_EICR_MAILBOX) 4503 ixgbe_pf_mbx_process(dev); 4504 4505 if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) { 4506 ixgbe_handle_lasi(hw); 4507 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT; 4508 } 4509 4510 if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) { 4511 ixgbe_dev_link_update(dev, 0); 4512 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4513 ixgbe_dev_link_status_print(dev); 4514 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, 4515 NULL); 4516 } 4517 4518 if (intr->flags & IXGBE_FLAG_MACSEC) { 4519 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, 4520 NULL); 4521 intr->flags &= ~IXGBE_FLAG_MACSEC; 4522 } 4523 4524 /* restore original mask */ 4525 intr->mask = intr->mask_original; 4526 intr->mask_original = 0; 4527 4528 PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr); 4529 ixgbe_enable_intr(dev); 4530 rte_intr_ack(intr_handle); 4531 } 4532 4533 /** 4534 * Interrupt handler triggered by NIC for handling 4535 * specific interrupt. 4536 * 4537 * @param handle 4538 * Pointer to interrupt handle. 4539 * @param param 4540 * The address of parameter (struct rte_eth_dev *) regsitered before. 4541 * 4542 * @return 4543 * void 4544 */ 4545 static void 4546 ixgbe_dev_interrupt_handler(void *param) 4547 { 4548 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 4549 4550 ixgbe_dev_interrupt_get_status(dev); 4551 ixgbe_dev_interrupt_action(dev); 4552 } 4553 4554 static int 4555 ixgbe_dev_led_on(struct rte_eth_dev *dev) 4556 { 4557 struct ixgbe_hw *hw; 4558 4559 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4560 return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4561 } 4562 4563 static int 4564 ixgbe_dev_led_off(struct rte_eth_dev *dev) 4565 { 4566 struct ixgbe_hw *hw; 4567 4568 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4569 return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP; 4570 } 4571 4572 static int 4573 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4574 { 4575 struct ixgbe_hw *hw; 4576 uint32_t mflcn_reg; 4577 uint32_t fccfg_reg; 4578 int rx_pause; 4579 int tx_pause; 4580 4581 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4582 4583 fc_conf->pause_time = hw->fc.pause_time; 4584 fc_conf->high_water = hw->fc.high_water[0]; 4585 fc_conf->low_water = hw->fc.low_water[0]; 4586 fc_conf->send_xon = hw->fc.send_xon; 4587 fc_conf->autoneg = !hw->fc.disable_fc_autoneg; 4588 4589 /* 4590 * Return rx_pause status according to actual setting of 4591 * MFLCN register. 4592 */ 4593 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4594 if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE)) 4595 rx_pause = 1; 4596 else 4597 rx_pause = 0; 4598 4599 /* 4600 * Return tx_pause status according to actual setting of 4601 * FCCFG register. 4602 */ 4603 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4604 if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY)) 4605 tx_pause = 1; 4606 else 4607 tx_pause = 0; 4608 4609 if (rx_pause && tx_pause) 4610 fc_conf->mode = RTE_FC_FULL; 4611 else if (rx_pause) 4612 fc_conf->mode = RTE_FC_RX_PAUSE; 4613 else if (tx_pause) 4614 fc_conf->mode = RTE_FC_TX_PAUSE; 4615 else 4616 fc_conf->mode = RTE_FC_NONE; 4617 4618 return 0; 4619 } 4620 4621 static int 4622 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) 4623 { 4624 struct ixgbe_hw *hw; 4625 int err; 4626 uint32_t rx_buf_size; 4627 uint32_t max_high_water; 4628 uint32_t mflcn; 4629 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4630 ixgbe_fc_none, 4631 ixgbe_fc_rx_pause, 4632 ixgbe_fc_tx_pause, 4633 ixgbe_fc_full 4634 }; 4635 4636 PMD_INIT_FUNC_TRACE(); 4637 4638 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4639 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)); 4640 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4641 4642 /* 4643 * At least reserve one Ethernet frame for watermark 4644 * high_water/low_water in kilo bytes for ixgbe 4645 */ 4646 max_high_water = (rx_buf_size - 4647 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4648 if ((fc_conf->high_water > max_high_water) || 4649 (fc_conf->high_water < fc_conf->low_water)) { 4650 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4651 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4652 return -EINVAL; 4653 } 4654 4655 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode]; 4656 hw->fc.pause_time = fc_conf->pause_time; 4657 hw->fc.high_water[0] = fc_conf->high_water; 4658 hw->fc.low_water[0] = fc_conf->low_water; 4659 hw->fc.send_xon = fc_conf->send_xon; 4660 hw->fc.disable_fc_autoneg = !fc_conf->autoneg; 4661 4662 err = ixgbe_fc_enable(hw); 4663 4664 /* Not negotiated is not an error case */ 4665 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) { 4666 4667 /* check if we want to forward MAC frames - driver doesn't have native 4668 * capability to do that, so we'll write the registers ourselves */ 4669 4670 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4671 4672 /* set or clear MFLCN.PMCF bit depending on configuration */ 4673 if (fc_conf->mac_ctrl_frame_fwd != 0) 4674 mflcn |= IXGBE_MFLCN_PMCF; 4675 else 4676 mflcn &= ~IXGBE_MFLCN_PMCF; 4677 4678 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn); 4679 IXGBE_WRITE_FLUSH(hw); 4680 4681 return 0; 4682 } 4683 4684 PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err); 4685 return -EIO; 4686 } 4687 4688 /** 4689 * ixgbe_pfc_enable_generic - Enable flow control 4690 * @hw: pointer to hardware structure 4691 * @tc_num: traffic class number 4692 * Enable flow control according to the current settings. 4693 */ 4694 static int 4695 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num) 4696 { 4697 int ret_val = 0; 4698 uint32_t mflcn_reg, fccfg_reg; 4699 uint32_t reg; 4700 uint32_t fcrtl, fcrth; 4701 uint8_t i; 4702 uint8_t nb_rx_en; 4703 4704 /* Validate the water mark configuration */ 4705 if (!hw->fc.pause_time) { 4706 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4707 goto out; 4708 } 4709 4710 /* Low water mark of zero causes XOFF floods */ 4711 if (hw->fc.current_mode & ixgbe_fc_tx_pause) { 4712 /* High/Low water can not be 0 */ 4713 if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) { 4714 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4715 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4716 goto out; 4717 } 4718 4719 if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) { 4720 PMD_INIT_LOG(ERR, "Invalid water mark configuration"); 4721 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS; 4722 goto out; 4723 } 4724 } 4725 /* Negotiate the fc mode to use */ 4726 ixgbe_fc_autoneg(hw); 4727 4728 /* Disable any previous flow control settings */ 4729 mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN); 4730 mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE); 4731 4732 fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 4733 fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY); 4734 4735 switch (hw->fc.current_mode) { 4736 case ixgbe_fc_none: 4737 /* 4738 * If the count of enabled RX Priority Flow control >1, 4739 * and the TX pause can not be disabled 4740 */ 4741 nb_rx_en = 0; 4742 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4743 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4744 if (reg & IXGBE_FCRTH_FCEN) 4745 nb_rx_en++; 4746 } 4747 if (nb_rx_en > 1) 4748 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4749 break; 4750 case ixgbe_fc_rx_pause: 4751 /* 4752 * Rx Flow control is enabled and Tx Flow control is 4753 * disabled by software override. Since there really 4754 * isn't a way to advertise that we are capable of RX 4755 * Pause ONLY, we will advertise that we support both 4756 * symmetric and asymmetric Rx PAUSE. Later, we will 4757 * disable the adapter's ability to send PAUSE frames. 4758 */ 4759 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4760 /* 4761 * If the count of enabled RX Priority Flow control >1, 4762 * and the TX pause can not be disabled 4763 */ 4764 nb_rx_en = 0; 4765 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4766 reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i)); 4767 if (reg & IXGBE_FCRTH_FCEN) 4768 nb_rx_en++; 4769 } 4770 if (nb_rx_en > 1) 4771 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4772 break; 4773 case ixgbe_fc_tx_pause: 4774 /* 4775 * Tx Flow control is enabled, and Rx Flow control is 4776 * disabled by software override. 4777 */ 4778 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4779 break; 4780 case ixgbe_fc_full: 4781 /* Flow control (both Rx and Tx) is enabled by SW override. */ 4782 mflcn_reg |= IXGBE_MFLCN_RPFCE; 4783 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY; 4784 break; 4785 default: 4786 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly"); 4787 ret_val = IXGBE_ERR_CONFIG; 4788 goto out; 4789 } 4790 4791 /* Set 802.3x based flow control settings. */ 4792 mflcn_reg |= IXGBE_MFLCN_DPF; 4793 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg); 4794 IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg); 4795 4796 /* Set up and enable Rx high/low water mark thresholds, enable XON. */ 4797 if ((hw->fc.current_mode & ixgbe_fc_tx_pause) && 4798 hw->fc.high_water[tc_num]) { 4799 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE; 4800 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl); 4801 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN; 4802 } else { 4803 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0); 4804 /* 4805 * In order to prevent Tx hangs when the internal Tx 4806 * switch is enabled we must set the high water mark 4807 * to the maximum FCRTH value. This allows the Tx 4808 * switch to function even under heavy Rx workloads. 4809 */ 4810 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32; 4811 } 4812 IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth); 4813 4814 /* Configure pause time (2 TCs per register) */ 4815 reg = hw->fc.pause_time * 0x00010001; 4816 for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++) 4817 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg); 4818 4819 /* Configure flow control refresh threshold value */ 4820 IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2); 4821 4822 out: 4823 return ret_val; 4824 } 4825 4826 static int 4827 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num) 4828 { 4829 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4830 int32_t ret_val = IXGBE_NOT_IMPLEMENTED; 4831 4832 if (hw->mac.type != ixgbe_mac_82598EB) { 4833 ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num); 4834 } 4835 return ret_val; 4836 } 4837 4838 static int 4839 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf) 4840 { 4841 int err; 4842 uint32_t rx_buf_size; 4843 uint32_t max_high_water; 4844 uint8_t tc_num; 4845 uint8_t map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 }; 4846 struct ixgbe_hw *hw = 4847 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4848 struct ixgbe_dcb_config *dcb_config = 4849 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4850 4851 enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = { 4852 ixgbe_fc_none, 4853 ixgbe_fc_rx_pause, 4854 ixgbe_fc_tx_pause, 4855 ixgbe_fc_full 4856 }; 4857 4858 PMD_INIT_FUNC_TRACE(); 4859 4860 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4861 tc_num = map[pfc_conf->priority]; 4862 rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)); 4863 PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); 4864 /* 4865 * At least reserve one Ethernet frame for watermark 4866 * high_water/low_water in kilo bytes for ixgbe 4867 */ 4868 max_high_water = (rx_buf_size - 4869 RTE_ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT; 4870 if ((pfc_conf->fc.high_water > max_high_water) || 4871 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) { 4872 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB"); 4873 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water); 4874 return -EINVAL; 4875 } 4876 4877 hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode]; 4878 hw->fc.pause_time = pfc_conf->fc.pause_time; 4879 hw->fc.send_xon = pfc_conf->fc.send_xon; 4880 hw->fc.low_water[tc_num] = pfc_conf->fc.low_water; 4881 hw->fc.high_water[tc_num] = pfc_conf->fc.high_water; 4882 4883 err = ixgbe_dcb_pfc_enable(dev, tc_num); 4884 4885 /* Not negotiated is not an error case */ 4886 if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) 4887 return 0; 4888 4889 PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err); 4890 return -EIO; 4891 } 4892 4893 static int 4894 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, 4895 struct rte_eth_rss_reta_entry64 *reta_conf, 4896 uint16_t reta_size) 4897 { 4898 uint16_t i, sp_reta_size; 4899 uint8_t j, mask; 4900 uint32_t reta, r; 4901 uint16_t idx, shift; 4902 struct ixgbe_adapter *adapter = dev->data->dev_private; 4903 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4904 uint32_t reta_reg; 4905 4906 PMD_INIT_FUNC_TRACE(); 4907 4908 if (!ixgbe_rss_update_sp(hw->mac.type)) { 4909 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this " 4910 "NIC."); 4911 return -ENOTSUP; 4912 } 4913 4914 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 4915 if (reta_size != sp_reta_size) { 4916 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 4917 "(%d) doesn't match the number hardware can supported " 4918 "(%d)", reta_size, sp_reta_size); 4919 return -EINVAL; 4920 } 4921 4922 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 4923 idx = i / RTE_RETA_GROUP_SIZE; 4924 shift = i % RTE_RETA_GROUP_SIZE; 4925 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 4926 IXGBE_4_BIT_MASK); 4927 if (!mask) 4928 continue; 4929 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 4930 if (mask == IXGBE_4_BIT_MASK) 4931 r = 0; 4932 else 4933 r = IXGBE_READ_REG(hw, reta_reg); 4934 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) { 4935 if (mask & (0x1 << j)) 4936 reta |= reta_conf[idx].reta[shift + j] << 4937 (CHAR_BIT * j); 4938 else 4939 reta |= r & (IXGBE_8_BIT_MASK << 4940 (CHAR_BIT * j)); 4941 } 4942 IXGBE_WRITE_REG(hw, reta_reg, reta); 4943 } 4944 adapter->rss_reta_updated = 1; 4945 4946 return 0; 4947 } 4948 4949 static int 4950 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, 4951 struct rte_eth_rss_reta_entry64 *reta_conf, 4952 uint16_t reta_size) 4953 { 4954 uint16_t i, sp_reta_size; 4955 uint8_t j, mask; 4956 uint32_t reta; 4957 uint16_t idx, shift; 4958 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4959 uint32_t reta_reg; 4960 4961 PMD_INIT_FUNC_TRACE(); 4962 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 4963 if (reta_size != sp_reta_size) { 4964 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 4965 "(%d) doesn't match the number hardware can supported " 4966 "(%d)", reta_size, sp_reta_size); 4967 return -EINVAL; 4968 } 4969 4970 for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) { 4971 idx = i / RTE_RETA_GROUP_SIZE; 4972 shift = i % RTE_RETA_GROUP_SIZE; 4973 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 4974 IXGBE_4_BIT_MASK); 4975 if (!mask) 4976 continue; 4977 4978 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 4979 reta = IXGBE_READ_REG(hw, reta_reg); 4980 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) { 4981 if (mask & (0x1 << j)) 4982 reta_conf[idx].reta[shift + j] = 4983 ((reta >> (CHAR_BIT * j)) & 4984 IXGBE_8_BIT_MASK); 4985 } 4986 } 4987 4988 return 0; 4989 } 4990 4991 static int 4992 ixgbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 4993 uint32_t index, uint32_t pool) 4994 { 4995 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4996 uint32_t enable_addr = 1; 4997 4998 return ixgbe_set_rar(hw, index, mac_addr->addr_bytes, 4999 pool, enable_addr); 5000 } 5001 5002 static void 5003 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) 5004 { 5005 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5006 5007 ixgbe_clear_rar(hw, index); 5008 } 5009 5010 static int 5011 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr) 5012 { 5013 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5014 5015 ixgbe_remove_rar(dev, 0); 5016 ixgbe_add_rar(dev, addr, 0, pci_dev->max_vfs); 5017 5018 return 0; 5019 } 5020 5021 static bool 5022 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) 5023 { 5024 if (strcmp(dev->device->driver->name, drv->driver.name)) 5025 return false; 5026 5027 return true; 5028 } 5029 5030 bool 5031 is_ixgbe_supported(struct rte_eth_dev *dev) 5032 { 5033 return is_device_supported(dev, &rte_ixgbe_pmd); 5034 } 5035 5036 static int 5037 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 5038 { 5039 uint32_t hlreg0; 5040 uint32_t maxfrs; 5041 struct ixgbe_hw *hw; 5042 struct rte_eth_dev_info dev_info; 5043 uint32_t frame_size = mtu + IXGBE_ETH_OVERHEAD; 5044 struct rte_eth_dev_data *dev_data = dev->data; 5045 int ret; 5046 5047 ret = ixgbe_dev_info_get(dev, &dev_info); 5048 if (ret != 0) 5049 return ret; 5050 5051 /* check that mtu is within the allowed range */ 5052 if (mtu < RTE_ETHER_MIN_MTU || frame_size > dev_info.max_rx_pktlen) 5053 return -EINVAL; 5054 5055 /* If device is started, refuse mtu that requires the support of 5056 * scattered packets when this feature has not been enabled before. 5057 */ 5058 if (dev_data->dev_started && !dev_data->scattered_rx && 5059 (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > 5060 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 5061 PMD_INIT_LOG(ERR, "Stop port first."); 5062 return -EINVAL; 5063 } 5064 5065 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5066 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5067 5068 /* switch to jumbo mode if needed */ 5069 if (frame_size > RTE_ETHER_MAX_LEN) { 5070 dev->data->dev_conf.rxmode.offloads |= 5071 DEV_RX_OFFLOAD_JUMBO_FRAME; 5072 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 5073 } else { 5074 dev->data->dev_conf.rxmode.offloads &= 5075 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 5076 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 5077 } 5078 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5079 5080 /* update max frame size */ 5081 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 5082 5083 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 5084 maxfrs &= 0x0000FFFF; 5085 maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16); 5086 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 5087 5088 return 0; 5089 } 5090 5091 /* 5092 * Virtual Function operations 5093 */ 5094 static void 5095 ixgbevf_intr_disable(struct rte_eth_dev *dev) 5096 { 5097 struct ixgbe_interrupt *intr = 5098 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5099 struct ixgbe_hw *hw = 5100 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5101 5102 PMD_INIT_FUNC_TRACE(); 5103 5104 /* Clear interrupt mask to stop from interrupts being generated */ 5105 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK); 5106 5107 IXGBE_WRITE_FLUSH(hw); 5108 5109 /* Clear mask value. */ 5110 intr->mask = 0; 5111 } 5112 5113 static void 5114 ixgbevf_intr_enable(struct rte_eth_dev *dev) 5115 { 5116 struct ixgbe_interrupt *intr = 5117 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5118 struct ixgbe_hw *hw = 5119 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5120 5121 PMD_INIT_FUNC_TRACE(); 5122 5123 /* VF enable interrupt autoclean */ 5124 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK); 5125 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK); 5126 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK); 5127 5128 IXGBE_WRITE_FLUSH(hw); 5129 5130 /* Save IXGBE_VTEIMS value to mask. */ 5131 intr->mask = IXGBE_VF_IRQ_ENABLE_MASK; 5132 } 5133 5134 static int 5135 ixgbevf_dev_configure(struct rte_eth_dev *dev) 5136 { 5137 struct rte_eth_conf *conf = &dev->data->dev_conf; 5138 struct ixgbe_adapter *adapter = dev->data->dev_private; 5139 5140 PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", 5141 dev->data->port_id); 5142 5143 /* 5144 * VF has no ability to enable/disable HW CRC 5145 * Keep the persistent behavior the same as Host PF 5146 */ 5147 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC 5148 if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 5149 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip"); 5150 conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 5151 } 5152 #else 5153 if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) { 5154 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip"); 5155 conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC; 5156 } 5157 #endif 5158 5159 /* 5160 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk 5161 * allocation or vector Rx preconditions we will reset it. 5162 */ 5163 adapter->rx_bulk_alloc_allowed = true; 5164 adapter->rx_vec_allowed = true; 5165 5166 return 0; 5167 } 5168 5169 static int 5170 ixgbevf_dev_start(struct rte_eth_dev *dev) 5171 { 5172 struct ixgbe_hw *hw = 5173 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5174 uint32_t intr_vector = 0; 5175 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5176 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5177 5178 int err, mask = 0; 5179 5180 PMD_INIT_FUNC_TRACE(); 5181 5182 /* Stop the link setup handler before resetting the HW. */ 5183 rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev); 5184 5185 err = hw->mac.ops.reset_hw(hw); 5186 if (err) { 5187 PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); 5188 return err; 5189 } 5190 hw->mac.get_link_status = true; 5191 5192 /* negotiate mailbox API version to use with the PF. */ 5193 ixgbevf_negotiate_api(hw); 5194 5195 ixgbevf_dev_tx_init(dev); 5196 5197 /* This can fail when allocating mbufs for descriptor rings */ 5198 err = ixgbevf_dev_rx_init(dev); 5199 if (err) { 5200 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err); 5201 ixgbe_dev_clear_queues(dev); 5202 return err; 5203 } 5204 5205 /* Set vfta */ 5206 ixgbevf_set_vfta_all(dev, 1); 5207 5208 /* Set HW strip */ 5209 mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | 5210 ETH_VLAN_EXTEND_MASK; 5211 err = ixgbevf_vlan_offload_config(dev, mask); 5212 if (err) { 5213 PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); 5214 ixgbe_dev_clear_queues(dev); 5215 return err; 5216 } 5217 5218 ixgbevf_dev_rxtx_start(dev); 5219 5220 /* check and configure queue intr-vector mapping */ 5221 if (rte_intr_cap_multiple(intr_handle) && 5222 dev->data->dev_conf.intr_conf.rxq) { 5223 /* According to datasheet, only vector 0/1/2 can be used, 5224 * now only one vector is used for Rx queue 5225 */ 5226 intr_vector = 1; 5227 if (rte_intr_efd_enable(intr_handle, intr_vector)) 5228 return -1; 5229 } 5230 5231 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 5232 intr_handle->intr_vec = 5233 rte_zmalloc("intr_vec", 5234 dev->data->nb_rx_queues * sizeof(int), 0); 5235 if (intr_handle->intr_vec == NULL) { 5236 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 5237 " intr_vec", dev->data->nb_rx_queues); 5238 return -ENOMEM; 5239 } 5240 } 5241 ixgbevf_configure_msix(dev); 5242 5243 /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt 5244 * is mapped to VFIO vector 0 in eth_ixgbevf_dev_init( ). 5245 * If previous VFIO interrupt mapping setting in eth_ixgbevf_dev_init( ) 5246 * is not cleared, it will fail when following rte_intr_enable( ) tries 5247 * to map Rx queue interrupt to other VFIO vectors. 5248 * So clear uio/vfio intr/evevnfd first to avoid failure. 5249 */ 5250 rte_intr_disable(intr_handle); 5251 5252 rte_intr_enable(intr_handle); 5253 5254 /* Re-enable interrupt for VF */ 5255 ixgbevf_intr_enable(dev); 5256 5257 /* 5258 * Update link status right before return, because it may 5259 * start link configuration process in a separate thread. 5260 */ 5261 ixgbevf_dev_link_update(dev, 0); 5262 5263 hw->adapter_stopped = false; 5264 5265 return 0; 5266 } 5267 5268 static void 5269 ixgbevf_dev_stop(struct rte_eth_dev *dev) 5270 { 5271 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5272 struct ixgbe_adapter *adapter = dev->data->dev_private; 5273 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5274 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5275 5276 if (hw->adapter_stopped) 5277 return; 5278 5279 PMD_INIT_FUNC_TRACE(); 5280 5281 rte_eal_alarm_cancel(ixgbe_dev_setup_link_alarm_handler, dev); 5282 5283 ixgbevf_intr_disable(dev); 5284 5285 hw->adapter_stopped = 1; 5286 ixgbe_stop_adapter(hw); 5287 5288 /* 5289 * Clear what we set, but we still keep shadow_vfta to 5290 * restore after device starts 5291 */ 5292 ixgbevf_set_vfta_all(dev, 0); 5293 5294 /* Clear stored conf */ 5295 dev->data->scattered_rx = 0; 5296 5297 ixgbe_dev_clear_queues(dev); 5298 5299 /* Clean datapath event and queue/vec mapping */ 5300 rte_intr_efd_disable(intr_handle); 5301 if (intr_handle->intr_vec != NULL) { 5302 rte_free(intr_handle->intr_vec); 5303 intr_handle->intr_vec = NULL; 5304 } 5305 5306 adapter->rss_reta_updated = 0; 5307 } 5308 5309 static void 5310 ixgbevf_dev_close(struct rte_eth_dev *dev) 5311 { 5312 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5313 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5314 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5315 5316 PMD_INIT_FUNC_TRACE(); 5317 5318 ixgbe_reset_hw(hw); 5319 5320 ixgbevf_dev_stop(dev); 5321 5322 ixgbe_dev_free_queues(dev); 5323 5324 /** 5325 * Remove the VF MAC address ro ensure 5326 * that the VF traffic goes to the PF 5327 * after stop, close and detach of the VF 5328 **/ 5329 ixgbevf_remove_mac_addr(dev, 0); 5330 5331 dev->dev_ops = NULL; 5332 dev->rx_pkt_burst = NULL; 5333 dev->tx_pkt_burst = NULL; 5334 5335 rte_intr_disable(intr_handle); 5336 rte_intr_callback_unregister(intr_handle, 5337 ixgbevf_dev_interrupt_handler, dev); 5338 } 5339 5340 /* 5341 * Reset VF device 5342 */ 5343 static int 5344 ixgbevf_dev_reset(struct rte_eth_dev *dev) 5345 { 5346 int ret; 5347 5348 ret = eth_ixgbevf_dev_uninit(dev); 5349 if (ret) 5350 return ret; 5351 5352 ret = eth_ixgbevf_dev_init(dev); 5353 5354 return ret; 5355 } 5356 5357 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) 5358 { 5359 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5360 struct ixgbe_vfta *shadow_vfta = 5361 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5362 int i = 0, j = 0, vfta = 0, mask = 1; 5363 5364 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 5365 vfta = shadow_vfta->vfta[i]; 5366 if (vfta) { 5367 mask = 1; 5368 for (j = 0; j < 32; j++) { 5369 if (vfta & mask) 5370 ixgbe_set_vfta(hw, (i<<5)+j, 0, 5371 on, false); 5372 mask <<= 1; 5373 } 5374 } 5375 } 5376 5377 } 5378 5379 static int 5380 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 5381 { 5382 struct ixgbe_hw *hw = 5383 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5384 struct ixgbe_vfta *shadow_vfta = 5385 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); 5386 uint32_t vid_idx = 0; 5387 uint32_t vid_bit = 0; 5388 int ret = 0; 5389 5390 PMD_INIT_FUNC_TRACE(); 5391 5392 /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */ 5393 ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on, false); 5394 if (ret) { 5395 PMD_INIT_LOG(ERR, "Unable to set VF vlan"); 5396 return ret; 5397 } 5398 vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F); 5399 vid_bit = (uint32_t) (1 << (vlan_id & 0x1F)); 5400 5401 /* Save what we set and retore it after device reset */ 5402 if (on) 5403 shadow_vfta->vfta[vid_idx] |= vid_bit; 5404 else 5405 shadow_vfta->vfta[vid_idx] &= ~vid_bit; 5406 5407 return 0; 5408 } 5409 5410 static void 5411 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) 5412 { 5413 struct ixgbe_hw *hw = 5414 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5415 uint32_t ctrl; 5416 5417 PMD_INIT_FUNC_TRACE(); 5418 5419 if (queue >= hw->mac.max_rx_queues) 5420 return; 5421 5422 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue)); 5423 if (on) 5424 ctrl |= IXGBE_RXDCTL_VME; 5425 else 5426 ctrl &= ~IXGBE_RXDCTL_VME; 5427 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl); 5428 5429 ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on); 5430 } 5431 5432 static int 5433 ixgbevf_vlan_offload_config(struct rte_eth_dev *dev, int mask) 5434 { 5435 struct ixgbe_rx_queue *rxq; 5436 uint16_t i; 5437 int on = 0; 5438 5439 /* VF function only support hw strip feature, others are not support */ 5440 if (mask & ETH_VLAN_STRIP_MASK) { 5441 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5442 rxq = dev->data->rx_queues[i]; 5443 on = !!(rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 5444 ixgbevf_vlan_strip_queue_set(dev, i, on); 5445 } 5446 } 5447 5448 return 0; 5449 } 5450 5451 static int 5452 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) 5453 { 5454 ixgbe_config_vlan_strip_on_all_queues(dev, mask); 5455 5456 ixgbevf_vlan_offload_config(dev, mask); 5457 5458 return 0; 5459 } 5460 5461 int 5462 ixgbe_vt_check(struct ixgbe_hw *hw) 5463 { 5464 uint32_t reg_val; 5465 5466 /* if Virtualization Technology is enabled */ 5467 reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 5468 if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) { 5469 PMD_INIT_LOG(ERR, "VT must be enabled for this setting"); 5470 return -1; 5471 } 5472 5473 return 0; 5474 } 5475 5476 static uint32_t 5477 ixgbe_uta_vector(struct ixgbe_hw *hw, struct rte_ether_addr *uc_addr) 5478 { 5479 uint32_t vector = 0; 5480 5481 switch (hw->mac.mc_filter_type) { 5482 case 0: /* use bits [47:36] of the address */ 5483 vector = ((uc_addr->addr_bytes[4] >> 4) | 5484 (((uint16_t)uc_addr->addr_bytes[5]) << 4)); 5485 break; 5486 case 1: /* use bits [46:35] of the address */ 5487 vector = ((uc_addr->addr_bytes[4] >> 3) | 5488 (((uint16_t)uc_addr->addr_bytes[5]) << 5)); 5489 break; 5490 case 2: /* use bits [45:34] of the address */ 5491 vector = ((uc_addr->addr_bytes[4] >> 2) | 5492 (((uint16_t)uc_addr->addr_bytes[5]) << 6)); 5493 break; 5494 case 3: /* use bits [43:32] of the address */ 5495 vector = ((uc_addr->addr_bytes[4]) | 5496 (((uint16_t)uc_addr->addr_bytes[5]) << 8)); 5497 break; 5498 default: /* Invalid mc_filter_type */ 5499 break; 5500 } 5501 5502 /* vector can only be 12-bits or boundary will be exceeded */ 5503 vector &= 0xFFF; 5504 return vector; 5505 } 5506 5507 static int 5508 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, 5509 struct rte_ether_addr *mac_addr, uint8_t on) 5510 { 5511 uint32_t vector; 5512 uint32_t uta_idx; 5513 uint32_t reg_val; 5514 uint32_t uta_shift; 5515 uint32_t rc; 5516 const uint32_t ixgbe_uta_idx_mask = 0x7F; 5517 const uint32_t ixgbe_uta_bit_shift = 5; 5518 const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1; 5519 const uint32_t bit1 = 0x1; 5520 5521 struct ixgbe_hw *hw = 5522 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5523 struct ixgbe_uta_info *uta_info = 5524 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5525 5526 /* The UTA table only exists on 82599 hardware and newer */ 5527 if (hw->mac.type < ixgbe_mac_82599EB) 5528 return -ENOTSUP; 5529 5530 vector = ixgbe_uta_vector(hw, mac_addr); 5531 uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask; 5532 uta_shift = vector & ixgbe_uta_bit_mask; 5533 5534 rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0); 5535 if (rc == on) 5536 return 0; 5537 5538 reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx)); 5539 if (on) { 5540 uta_info->uta_in_use++; 5541 reg_val |= (bit1 << uta_shift); 5542 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift); 5543 } else { 5544 uta_info->uta_in_use--; 5545 reg_val &= ~(bit1 << uta_shift); 5546 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift); 5547 } 5548 5549 IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val); 5550 5551 if (uta_info->uta_in_use > 0) 5552 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, 5553 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type); 5554 else 5555 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type); 5556 5557 return 0; 5558 } 5559 5560 static int 5561 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on) 5562 { 5563 int i; 5564 struct ixgbe_hw *hw = 5565 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5566 struct ixgbe_uta_info *uta_info = 5567 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private); 5568 5569 /* The UTA table only exists on 82599 hardware and newer */ 5570 if (hw->mac.type < ixgbe_mac_82599EB) 5571 return -ENOTSUP; 5572 5573 if (on) { 5574 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5575 uta_info->uta_shadow[i] = ~0; 5576 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0); 5577 } 5578 } else { 5579 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) { 5580 uta_info->uta_shadow[i] = 0; 5581 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0); 5582 } 5583 } 5584 return 0; 5585 5586 } 5587 5588 uint32_t 5589 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val) 5590 { 5591 uint32_t new_val = orig_val; 5592 5593 if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG) 5594 new_val |= IXGBE_VMOLR_AUPE; 5595 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC) 5596 new_val |= IXGBE_VMOLR_ROMPE; 5597 if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC) 5598 new_val |= IXGBE_VMOLR_ROPE; 5599 if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) 5600 new_val |= IXGBE_VMOLR_BAM; 5601 if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) 5602 new_val |= IXGBE_VMOLR_MPE; 5603 5604 return new_val; 5605 } 5606 5607 #define IXGBE_MRCTL_VPME 0x01 /* Virtual Pool Mirroring. */ 5608 #define IXGBE_MRCTL_UPME 0x02 /* Uplink Port Mirroring. */ 5609 #define IXGBE_MRCTL_DPME 0x04 /* Downlink Port Mirroring. */ 5610 #define IXGBE_MRCTL_VLME 0x08 /* VLAN Mirroring. */ 5611 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \ 5612 ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \ 5613 ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN)) 5614 5615 static int 5616 ixgbe_mirror_rule_set(struct rte_eth_dev *dev, 5617 struct rte_eth_mirror_conf *mirror_conf, 5618 uint8_t rule_id, uint8_t on) 5619 { 5620 uint32_t mr_ctl, vlvf; 5621 uint32_t mp_lsb = 0; 5622 uint32_t mv_msb = 0; 5623 uint32_t mv_lsb = 0; 5624 uint32_t mp_msb = 0; 5625 uint8_t i = 0; 5626 int reg_index = 0; 5627 uint64_t vlan_mask = 0; 5628 5629 const uint8_t pool_mask_offset = 32; 5630 const uint8_t vlan_mask_offset = 32; 5631 const uint8_t dst_pool_offset = 8; 5632 const uint8_t rule_mr_offset = 4; 5633 const uint8_t mirror_rule_mask = 0x0F; 5634 5635 struct ixgbe_mirror_info *mr_info = 5636 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5637 struct ixgbe_hw *hw = 5638 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5639 uint8_t mirror_type = 0; 5640 5641 if (ixgbe_vt_check(hw) < 0) 5642 return -ENOTSUP; 5643 5644 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5645 return -EINVAL; 5646 5647 if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) { 5648 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.", 5649 mirror_conf->rule_type); 5650 return -EINVAL; 5651 } 5652 5653 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5654 mirror_type |= IXGBE_MRCTL_VLME; 5655 /* Check if vlan id is valid and find conresponding VLAN ID 5656 * index in VLVF 5657 */ 5658 for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) { 5659 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) { 5660 /* search vlan id related pool vlan filter 5661 * index 5662 */ 5663 reg_index = ixgbe_find_vlvf_slot( 5664 hw, 5665 mirror_conf->vlan.vlan_id[i], 5666 false); 5667 if (reg_index < 0) 5668 return -EINVAL; 5669 vlvf = IXGBE_READ_REG(hw, 5670 IXGBE_VLVF(reg_index)); 5671 if ((vlvf & IXGBE_VLVF_VIEN) && 5672 ((vlvf & IXGBE_VLVF_VLANID_MASK) == 5673 mirror_conf->vlan.vlan_id[i])) 5674 vlan_mask |= (1ULL << reg_index); 5675 else 5676 return -EINVAL; 5677 } 5678 } 5679 5680 if (on) { 5681 mv_lsb = vlan_mask & 0xFFFFFFFF; 5682 mv_msb = vlan_mask >> vlan_mask_offset; 5683 5684 mr_info->mr_conf[rule_id].vlan.vlan_mask = 5685 mirror_conf->vlan.vlan_mask; 5686 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) { 5687 if (mirror_conf->vlan.vlan_mask & (1ULL << i)) 5688 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 5689 mirror_conf->vlan.vlan_id[i]; 5690 } 5691 } else { 5692 mv_lsb = 0; 5693 mv_msb = 0; 5694 mr_info->mr_conf[rule_id].vlan.vlan_mask = 0; 5695 for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) 5696 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0; 5697 } 5698 } 5699 5700 /** 5701 * if enable pool mirror, write related pool mask register,if disable 5702 * pool mirror, clear PFMRVM register 5703 */ 5704 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5705 mirror_type |= IXGBE_MRCTL_VPME; 5706 if (on) { 5707 mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF; 5708 mp_msb = mirror_conf->pool_mask >> pool_mask_offset; 5709 mr_info->mr_conf[rule_id].pool_mask = 5710 mirror_conf->pool_mask; 5711 5712 } else { 5713 mp_lsb = 0; 5714 mp_msb = 0; 5715 mr_info->mr_conf[rule_id].pool_mask = 0; 5716 } 5717 } 5718 if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT) 5719 mirror_type |= IXGBE_MRCTL_UPME; 5720 if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT) 5721 mirror_type |= IXGBE_MRCTL_DPME; 5722 5723 /* read mirror control register and recalculate it */ 5724 mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id)); 5725 5726 if (on) { 5727 mr_ctl |= mirror_type; 5728 mr_ctl &= mirror_rule_mask; 5729 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset; 5730 } else { 5731 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask); 5732 } 5733 5734 mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type; 5735 mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool; 5736 5737 /* write mirrror control register */ 5738 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5739 5740 /* write pool mirrror control register */ 5741 if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { 5742 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb); 5743 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), 5744 mp_msb); 5745 } 5746 /* write VLAN mirrror control register */ 5747 if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { 5748 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb); 5749 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), 5750 mv_msb); 5751 } 5752 5753 return 0; 5754 } 5755 5756 static int 5757 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) 5758 { 5759 int mr_ctl = 0; 5760 uint32_t lsb_val = 0; 5761 uint32_t msb_val = 0; 5762 const uint8_t rule_mr_offset = 4; 5763 5764 struct ixgbe_hw *hw = 5765 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5766 struct ixgbe_mirror_info *mr_info = 5767 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); 5768 5769 if (ixgbe_vt_check(hw) < 0) 5770 return -ENOTSUP; 5771 5772 if (rule_id >= IXGBE_MAX_MIRROR_RULES) 5773 return -EINVAL; 5774 5775 memset(&mr_info->mr_conf[rule_id], 0, 5776 sizeof(struct rte_eth_mirror_conf)); 5777 5778 /* clear PFVMCTL register */ 5779 IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); 5780 5781 /* clear pool mask register */ 5782 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val); 5783 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val); 5784 5785 /* clear vlan mask register */ 5786 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val); 5787 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val); 5788 5789 return 0; 5790 } 5791 5792 static int 5793 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5794 { 5795 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5796 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5797 struct ixgbe_interrupt *intr = 5798 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5799 struct ixgbe_hw *hw = 5800 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5801 uint32_t vec = IXGBE_MISC_VEC_ID; 5802 5803 if (rte_intr_allow_others(intr_handle)) 5804 vec = IXGBE_RX_VEC_START; 5805 intr->mask |= (1 << vec); 5806 RTE_SET_USED(queue_id); 5807 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5808 5809 rte_intr_ack(intr_handle); 5810 5811 return 0; 5812 } 5813 5814 static int 5815 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5816 { 5817 struct ixgbe_interrupt *intr = 5818 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5819 struct ixgbe_hw *hw = 5820 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5821 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5822 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5823 uint32_t vec = IXGBE_MISC_VEC_ID; 5824 5825 if (rte_intr_allow_others(intr_handle)) 5826 vec = IXGBE_RX_VEC_START; 5827 intr->mask &= ~(1 << vec); 5828 RTE_SET_USED(queue_id); 5829 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, intr->mask); 5830 5831 return 0; 5832 } 5833 5834 static int 5835 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 5836 { 5837 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5838 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5839 uint32_t mask; 5840 struct ixgbe_hw *hw = 5841 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5842 struct ixgbe_interrupt *intr = 5843 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5844 5845 if (queue_id < 16) { 5846 ixgbe_disable_intr(hw); 5847 intr->mask |= (1 << queue_id); 5848 ixgbe_enable_intr(dev); 5849 } else if (queue_id < 32) { 5850 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5851 mask &= (1 << queue_id); 5852 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5853 } else if (queue_id < 64) { 5854 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5855 mask &= (1 << (queue_id - 32)); 5856 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5857 } 5858 rte_intr_ack(intr_handle); 5859 5860 return 0; 5861 } 5862 5863 static int 5864 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 5865 { 5866 uint32_t mask; 5867 struct ixgbe_hw *hw = 5868 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5869 struct ixgbe_interrupt *intr = 5870 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 5871 5872 if (queue_id < 16) { 5873 ixgbe_disable_intr(hw); 5874 intr->mask &= ~(1 << queue_id); 5875 ixgbe_enable_intr(dev); 5876 } else if (queue_id < 32) { 5877 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)); 5878 mask &= ~(1 << queue_id); 5879 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 5880 } else if (queue_id < 64) { 5881 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)); 5882 mask &= ~(1 << (queue_id - 32)); 5883 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 5884 } 5885 5886 return 0; 5887 } 5888 5889 static void 5890 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 5891 uint8_t queue, uint8_t msix_vector) 5892 { 5893 uint32_t tmp, idx; 5894 5895 if (direction == -1) { 5896 /* other causes */ 5897 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5898 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 5899 tmp &= ~0xFF; 5900 tmp |= msix_vector; 5901 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp); 5902 } else { 5903 /* rx or tx cause */ 5904 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5905 idx = ((16 * (queue & 1)) + (8 * direction)); 5906 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 5907 tmp &= ~(0xFF << idx); 5908 tmp |= (msix_vector << idx); 5909 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp); 5910 } 5911 } 5912 5913 /** 5914 * set the IVAR registers, mapping interrupt causes to vectors 5915 * @param hw 5916 * pointer to ixgbe_hw struct 5917 * @direction 5918 * 0 for Rx, 1 for Tx, -1 for other causes 5919 * @queue 5920 * queue to map the corresponding interrupt to 5921 * @msix_vector 5922 * the vector to map to the corresponding queue 5923 */ 5924 static void 5925 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, 5926 uint8_t queue, uint8_t msix_vector) 5927 { 5928 uint32_t tmp, idx; 5929 5930 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5931 if (hw->mac.type == ixgbe_mac_82598EB) { 5932 if (direction == -1) 5933 direction = 0; 5934 idx = (((direction * 64) + queue) >> 2) & 0x1F; 5935 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx)); 5936 tmp &= ~(0xFF << (8 * (queue & 0x3))); 5937 tmp |= (msix_vector << (8 * (queue & 0x3))); 5938 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); 5939 } else if ((hw->mac.type == ixgbe_mac_82599EB) || 5940 (hw->mac.type == ixgbe_mac_X540) || 5941 (hw->mac.type == ixgbe_mac_X550) || 5942 (hw->mac.type == ixgbe_mac_X550EM_x)) { 5943 if (direction == -1) { 5944 /* other causes */ 5945 idx = ((queue & 1) * 8); 5946 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5947 tmp &= ~(0xFF << idx); 5948 tmp |= (msix_vector << idx); 5949 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp); 5950 } else { 5951 /* rx or tx causes */ 5952 idx = ((16 * (queue & 1)) + (8 * direction)); 5953 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 5954 tmp &= ~(0xFF << idx); 5955 tmp |= (msix_vector << idx); 5956 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp); 5957 } 5958 } 5959 } 5960 5961 static void 5962 ixgbevf_configure_msix(struct rte_eth_dev *dev) 5963 { 5964 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 5965 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 5966 struct ixgbe_hw *hw = 5967 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5968 uint32_t q_idx; 5969 uint32_t vector_idx = IXGBE_MISC_VEC_ID; 5970 uint32_t base = IXGBE_MISC_VEC_ID; 5971 5972 /* Configure VF other cause ivar */ 5973 ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); 5974 5975 /* won't configure msix register if no mapping is done 5976 * between intr vector and event fd. 5977 */ 5978 if (!rte_intr_dp_is_en(intr_handle)) 5979 return; 5980 5981 if (rte_intr_allow_others(intr_handle)) { 5982 base = IXGBE_RX_VEC_START; 5983 vector_idx = IXGBE_RX_VEC_START; 5984 } 5985 5986 /* Configure all RX queues of VF */ 5987 for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) { 5988 /* Force all queue use vector 0, 5989 * as IXGBE_VF_MAXMSIVECOTR = 1 5990 */ 5991 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx); 5992 intr_handle->intr_vec[q_idx] = vector_idx; 5993 if (vector_idx < base + intr_handle->nb_efd - 1) 5994 vector_idx++; 5995 } 5996 5997 /* As RX queue setting above show, all queues use the vector 0. 5998 * Set only the ITR value of IXGBE_MISC_VEC_ID. 5999 */ 6000 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(IXGBE_MISC_VEC_ID), 6001 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6002 | IXGBE_EITR_CNT_WDIS); 6003 } 6004 6005 /** 6006 * Sets up the hardware to properly generate MSI-X interrupts 6007 * @hw 6008 * board private structure 6009 */ 6010 static void 6011 ixgbe_configure_msix(struct rte_eth_dev *dev) 6012 { 6013 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 6014 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 6015 struct ixgbe_hw *hw = 6016 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6017 uint32_t queue_id, base = IXGBE_MISC_VEC_ID; 6018 uint32_t vec = IXGBE_MISC_VEC_ID; 6019 uint32_t mask; 6020 uint32_t gpie; 6021 6022 /* won't configure msix register if no mapping is done 6023 * between intr vector and event fd 6024 * but if misx has been enabled already, need to configure 6025 * auto clean, auto mask and throttling. 6026 */ 6027 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6028 if (!rte_intr_dp_is_en(intr_handle) && 6029 !(gpie & (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT))) 6030 return; 6031 6032 if (rte_intr_allow_others(intr_handle)) 6033 vec = base = IXGBE_RX_VEC_START; 6034 6035 /* setup GPIE for MSI-x mode */ 6036 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 6037 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 6038 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME; 6039 /* auto clearing and auto setting corresponding bits in EIMS 6040 * when MSI-X interrupt is triggered 6041 */ 6042 if (hw->mac.type == ixgbe_mac_82598EB) { 6043 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 6044 } else { 6045 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 6046 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 6047 } 6048 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 6049 6050 /* Populate the IVAR table and set the ITR values to the 6051 * corresponding register. 6052 */ 6053 if (rte_intr_dp_is_en(intr_handle)) { 6054 for (queue_id = 0; queue_id < dev->data->nb_rx_queues; 6055 queue_id++) { 6056 /* by default, 1:1 mapping */ 6057 ixgbe_set_ivar_map(hw, 0, queue_id, vec); 6058 intr_handle->intr_vec[queue_id] = vec; 6059 if (vec < base + intr_handle->nb_efd - 1) 6060 vec++; 6061 } 6062 6063 switch (hw->mac.type) { 6064 case ixgbe_mac_82598EB: 6065 ixgbe_set_ivar_map(hw, -1, 6066 IXGBE_IVAR_OTHER_CAUSES_INDEX, 6067 IXGBE_MISC_VEC_ID); 6068 break; 6069 case ixgbe_mac_82599EB: 6070 case ixgbe_mac_X540: 6071 case ixgbe_mac_X550: 6072 case ixgbe_mac_X550EM_x: 6073 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); 6074 break; 6075 default: 6076 break; 6077 } 6078 } 6079 IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID), 6080 IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT) 6081 | IXGBE_EITR_CNT_WDIS); 6082 6083 /* set up to autoclear timer, and the vectors */ 6084 mask = IXGBE_EIMS_ENABLE_MASK; 6085 mask &= ~(IXGBE_EIMS_OTHER | 6086 IXGBE_EIMS_MAILBOX | 6087 IXGBE_EIMS_LSC); 6088 6089 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 6090 } 6091 6092 int 6093 ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, 6094 uint16_t queue_idx, uint16_t tx_rate) 6095 { 6096 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6097 struct rte_eth_rxmode *rxmode; 6098 uint32_t rf_dec, rf_int; 6099 uint32_t bcnrc_val; 6100 uint16_t link_speed = dev->data->dev_link.link_speed; 6101 6102 if (queue_idx >= hw->mac.max_tx_queues) 6103 return -EINVAL; 6104 6105 if (tx_rate != 0) { 6106 /* Calculate the rate factor values to set */ 6107 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate; 6108 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate; 6109 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate; 6110 6111 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA; 6112 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) & 6113 IXGBE_RTTBCNRC_RF_INT_MASK_M); 6114 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK); 6115 } else { 6116 bcnrc_val = 0; 6117 } 6118 6119 rxmode = &dev->data->dev_conf.rxmode; 6120 /* 6121 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 6122 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise 6123 * set as 0x4. 6124 */ 6125 if ((rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) && 6126 (rxmode->max_rx_pkt_len >= IXGBE_MAX_JUMBO_FRAME_SIZE)) 6127 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6128 IXGBE_MMW_SIZE_JUMBO_FRAME); 6129 else 6130 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM, 6131 IXGBE_MMW_SIZE_DEFAULT); 6132 6133 /* Set RTTBCNRC of queue X */ 6134 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx); 6135 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val); 6136 IXGBE_WRITE_FLUSH(hw); 6137 6138 return 0; 6139 } 6140 6141 static int 6142 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 6143 __attribute__((unused)) uint32_t index, 6144 __attribute__((unused)) uint32_t pool) 6145 { 6146 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6147 int diag; 6148 6149 /* 6150 * On a 82599 VF, adding again the same MAC addr is not an idempotent 6151 * operation. Trap this case to avoid exhausting the [very limited] 6152 * set of PF resources used to store VF MAC addresses. 6153 */ 6154 if (memcmp(hw->mac.perm_addr, mac_addr, 6155 sizeof(struct rte_ether_addr)) == 0) 6156 return -1; 6157 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6158 if (diag != 0) 6159 PMD_DRV_LOG(ERR, "Unable to add MAC address " 6160 "%02x:%02x:%02x:%02x:%02x:%02x - diag=%d", 6161 mac_addr->addr_bytes[0], 6162 mac_addr->addr_bytes[1], 6163 mac_addr->addr_bytes[2], 6164 mac_addr->addr_bytes[3], 6165 mac_addr->addr_bytes[4], 6166 mac_addr->addr_bytes[5], 6167 diag); 6168 return diag; 6169 } 6170 6171 static void 6172 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index) 6173 { 6174 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6175 struct rte_ether_addr *perm_addr = 6176 (struct rte_ether_addr *)hw->mac.perm_addr; 6177 struct rte_ether_addr *mac_addr; 6178 uint32_t i; 6179 int diag; 6180 6181 /* 6182 * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does 6183 * not support the deletion of a given MAC address. 6184 * Instead, it imposes to delete all MAC addresses, then to add again 6185 * all MAC addresses with the exception of the one to be deleted. 6186 */ 6187 (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL); 6188 6189 /* 6190 * Add again all MAC addresses, with the exception of the deleted one 6191 * and of the permanent MAC address. 6192 */ 6193 for (i = 0, mac_addr = dev->data->mac_addrs; 6194 i < hw->mac.num_rar_entries; i++, mac_addr++) { 6195 /* Skip the deleted MAC address */ 6196 if (i == index) 6197 continue; 6198 /* Skip NULL MAC addresses */ 6199 if (rte_is_zero_ether_addr(mac_addr)) 6200 continue; 6201 /* Skip the permanent MAC address */ 6202 if (memcmp(perm_addr, mac_addr, 6203 sizeof(struct rte_ether_addr)) == 0) 6204 continue; 6205 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); 6206 if (diag != 0) 6207 PMD_DRV_LOG(ERR, 6208 "Adding again MAC address " 6209 "%02x:%02x:%02x:%02x:%02x:%02x failed " 6210 "diag=%d", 6211 mac_addr->addr_bytes[0], 6212 mac_addr->addr_bytes[1], 6213 mac_addr->addr_bytes[2], 6214 mac_addr->addr_bytes[3], 6215 mac_addr->addr_bytes[4], 6216 mac_addr->addr_bytes[5], 6217 diag); 6218 } 6219 } 6220 6221 static int 6222 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, 6223 struct rte_ether_addr *addr) 6224 { 6225 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6226 6227 hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0); 6228 6229 return 0; 6230 } 6231 6232 int 6233 ixgbe_syn_filter_set(struct rte_eth_dev *dev, 6234 struct rte_eth_syn_filter *filter, 6235 bool add) 6236 { 6237 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6238 struct ixgbe_filter_info *filter_info = 6239 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6240 uint32_t syn_info; 6241 uint32_t synqf; 6242 6243 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6244 return -EINVAL; 6245 6246 syn_info = filter_info->syn_info; 6247 6248 if (add) { 6249 if (syn_info & IXGBE_SYN_FILTER_ENABLE) 6250 return -EINVAL; 6251 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) & 6252 IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE); 6253 6254 if (filter->hig_pri) 6255 synqf |= IXGBE_SYN_FILTER_SYNQFP; 6256 else 6257 synqf &= ~IXGBE_SYN_FILTER_SYNQFP; 6258 } else { 6259 synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6260 if (!(syn_info & IXGBE_SYN_FILTER_ENABLE)) 6261 return -ENOENT; 6262 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE); 6263 } 6264 6265 filter_info->syn_info = synqf; 6266 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 6267 IXGBE_WRITE_FLUSH(hw); 6268 return 0; 6269 } 6270 6271 static int 6272 ixgbe_syn_filter_get(struct rte_eth_dev *dev, 6273 struct rte_eth_syn_filter *filter) 6274 { 6275 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6276 uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); 6277 6278 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 6279 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0; 6280 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1); 6281 return 0; 6282 } 6283 return -ENOENT; 6284 } 6285 6286 static int 6287 ixgbe_syn_filter_handle(struct rte_eth_dev *dev, 6288 enum rte_filter_op filter_op, 6289 void *arg) 6290 { 6291 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6292 int ret; 6293 6294 MAC_TYPE_FILTER_SUP(hw->mac.type); 6295 6296 if (filter_op == RTE_ETH_FILTER_NOP) 6297 return 0; 6298 6299 if (arg == NULL) { 6300 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", 6301 filter_op); 6302 return -EINVAL; 6303 } 6304 6305 switch (filter_op) { 6306 case RTE_ETH_FILTER_ADD: 6307 ret = ixgbe_syn_filter_set(dev, 6308 (struct rte_eth_syn_filter *)arg, 6309 TRUE); 6310 break; 6311 case RTE_ETH_FILTER_DELETE: 6312 ret = ixgbe_syn_filter_set(dev, 6313 (struct rte_eth_syn_filter *)arg, 6314 FALSE); 6315 break; 6316 case RTE_ETH_FILTER_GET: 6317 ret = ixgbe_syn_filter_get(dev, 6318 (struct rte_eth_syn_filter *)arg); 6319 break; 6320 default: 6321 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); 6322 ret = -EINVAL; 6323 break; 6324 } 6325 6326 return ret; 6327 } 6328 6329 6330 static inline enum ixgbe_5tuple_protocol 6331 convert_protocol_type(uint8_t protocol_value) 6332 { 6333 if (protocol_value == IPPROTO_TCP) 6334 return IXGBE_FILTER_PROTOCOL_TCP; 6335 else if (protocol_value == IPPROTO_UDP) 6336 return IXGBE_FILTER_PROTOCOL_UDP; 6337 else if (protocol_value == IPPROTO_SCTP) 6338 return IXGBE_FILTER_PROTOCOL_SCTP; 6339 else 6340 return IXGBE_FILTER_PROTOCOL_NONE; 6341 } 6342 6343 /* inject a 5-tuple filter to HW */ 6344 static inline void 6345 ixgbe_inject_5tuple_filter(struct rte_eth_dev *dev, 6346 struct ixgbe_5tuple_filter *filter) 6347 { 6348 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6349 int i; 6350 uint32_t ftqf, sdpqf; 6351 uint32_t l34timir = 0; 6352 uint8_t mask = 0xff; 6353 6354 i = filter->index; 6355 6356 sdpqf = (uint32_t)(filter->filter_info.dst_port << 6357 IXGBE_SDPQF_DSTPORT_SHIFT); 6358 sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT); 6359 6360 ftqf = (uint32_t)(filter->filter_info.proto & 6361 IXGBE_FTQF_PROTOCOL_MASK); 6362 ftqf |= (uint32_t)((filter->filter_info.priority & 6363 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT); 6364 if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */ 6365 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK; 6366 if (filter->filter_info.dst_ip_mask == 0) 6367 mask &= IXGBE_FTQF_DEST_ADDR_MASK; 6368 if (filter->filter_info.src_port_mask == 0) 6369 mask &= IXGBE_FTQF_SOURCE_PORT_MASK; 6370 if (filter->filter_info.dst_port_mask == 0) 6371 mask &= IXGBE_FTQF_DEST_PORT_MASK; 6372 if (filter->filter_info.proto_mask == 0) 6373 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK; 6374 ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT; 6375 ftqf |= IXGBE_FTQF_POOL_MASK_EN; 6376 ftqf |= IXGBE_FTQF_QUEUE_ENABLE; 6377 6378 IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip); 6379 IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip); 6380 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf); 6381 IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf); 6382 6383 l34timir |= IXGBE_L34T_IMIR_RESERVE; 6384 l34timir |= (uint32_t)(filter->queue << 6385 IXGBE_L34T_IMIR_QUEUE_SHIFT); 6386 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir); 6387 } 6388 6389 /* 6390 * add a 5tuple filter 6391 * 6392 * @param 6393 * dev: Pointer to struct rte_eth_dev. 6394 * index: the index the filter allocates. 6395 * filter: ponter to the filter that will be added. 6396 * rx_queue: the queue id the filter assigned to. 6397 * 6398 * @return 6399 * - On success, zero. 6400 * - On failure, a negative value. 6401 */ 6402 static int 6403 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, 6404 struct ixgbe_5tuple_filter *filter) 6405 { 6406 struct ixgbe_filter_info *filter_info = 6407 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6408 int i, idx, shift; 6409 6410 /* 6411 * look for an unused 5tuple filter index, 6412 * and insert the filter to list. 6413 */ 6414 for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { 6415 idx = i / (sizeof(uint32_t) * NBBY); 6416 shift = i % (sizeof(uint32_t) * NBBY); 6417 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { 6418 filter_info->fivetuple_mask[idx] |= 1 << shift; 6419 filter->index = i; 6420 TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, 6421 filter, 6422 entries); 6423 break; 6424 } 6425 } 6426 if (i >= IXGBE_MAX_FTQF_FILTERS) { 6427 PMD_DRV_LOG(ERR, "5tuple filters are full."); 6428 return -ENOSYS; 6429 } 6430 6431 ixgbe_inject_5tuple_filter(dev, filter); 6432 6433 return 0; 6434 } 6435 6436 /* 6437 * remove a 5tuple filter 6438 * 6439 * @param 6440 * dev: Pointer to struct rte_eth_dev. 6441 * filter: the pointer of the filter will be removed. 6442 */ 6443 static void 6444 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, 6445 struct ixgbe_5tuple_filter *filter) 6446 { 6447 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6448 struct ixgbe_filter_info *filter_info = 6449 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6450 uint16_t index = filter->index; 6451 6452 filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &= 6453 ~(1 << (index % (sizeof(uint32_t) * NBBY))); 6454 TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); 6455 rte_free(filter); 6456 6457 IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0); 6458 IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0); 6459 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0); 6460 IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0); 6461 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0); 6462 } 6463 6464 static int 6465 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) 6466 { 6467 struct ixgbe_hw *hw; 6468 uint32_t max_frame = mtu + IXGBE_ETH_OVERHEAD; 6469 struct rte_eth_dev_data *dev_data = dev->data; 6470 6471 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6472 6473 if (mtu < RTE_ETHER_MIN_MTU || 6474 max_frame > RTE_ETHER_MAX_JUMBO_FRAME_LEN) 6475 return -EINVAL; 6476 6477 /* If device is started, refuse mtu that requires the support of 6478 * scattered packets when this feature has not been enabled before. 6479 */ 6480 if (dev_data->dev_started && !dev_data->scattered_rx && 6481 (max_frame + 2 * IXGBE_VLAN_TAG_SIZE > 6482 dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { 6483 PMD_INIT_LOG(ERR, "Stop port first."); 6484 return -EINVAL; 6485 } 6486 6487 /* 6488 * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU 6489 * request of the version 2.0 of the mailbox API. 6490 * For now, use the IXGBE_VF_SET_LPE request of the version 1.0 6491 * of the mailbox API. 6492 * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers 6493 * prior to 3.11.33 which contains the following change: 6494 * "ixgbe: Enable jumbo frames support w/ SR-IOV" 6495 */ 6496 ixgbevf_rlpml_set_vf(hw, max_frame); 6497 6498 /* update max frame size */ 6499 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame; 6500 return 0; 6501 } 6502 6503 static inline struct ixgbe_5tuple_filter * 6504 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list, 6505 struct ixgbe_5tuple_filter_info *key) 6506 { 6507 struct ixgbe_5tuple_filter *it; 6508 6509 TAILQ_FOREACH(it, filter_list, entries) { 6510 if (memcmp(key, &it->filter_info, 6511 sizeof(struct ixgbe_5tuple_filter_info)) == 0) { 6512 return it; 6513 } 6514 } 6515 return NULL; 6516 } 6517 6518 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/ 6519 static inline int 6520 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter, 6521 struct ixgbe_5tuple_filter_info *filter_info) 6522 { 6523 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM || 6524 filter->priority > IXGBE_5TUPLE_MAX_PRI || 6525 filter->priority < IXGBE_5TUPLE_MIN_PRI) 6526 return -EINVAL; 6527 6528 switch (filter->dst_ip_mask) { 6529 case UINT32_MAX: 6530 filter_info->dst_ip_mask = 0; 6531 filter_info->dst_ip = filter->dst_ip; 6532 break; 6533 case 0: 6534 filter_info->dst_ip_mask = 1; 6535 break; 6536 default: 6537 PMD_DRV_LOG(ERR, "invalid dst_ip mask."); 6538 return -EINVAL; 6539 } 6540 6541 switch (filter->src_ip_mask) { 6542 case UINT32_MAX: 6543 filter_info->src_ip_mask = 0; 6544 filter_info->src_ip = filter->src_ip; 6545 break; 6546 case 0: 6547 filter_info->src_ip_mask = 1; 6548 break; 6549 default: 6550 PMD_DRV_LOG(ERR, "invalid src_ip mask."); 6551 return -EINVAL; 6552 } 6553 6554 switch (filter->dst_port_mask) { 6555 case UINT16_MAX: 6556 filter_info->dst_port_mask = 0; 6557 filter_info->dst_port = filter->dst_port; 6558 break; 6559 case 0: 6560 filter_info->dst_port_mask = 1; 6561 break; 6562 default: 6563 PMD_DRV_LOG(ERR, "invalid dst_port mask."); 6564 return -EINVAL; 6565 } 6566 6567 switch (filter->src_port_mask) { 6568 case UINT16_MAX: 6569 filter_info->src_port_mask = 0; 6570 filter_info->src_port = filter->src_port; 6571 break; 6572 case 0: 6573 filter_info->src_port_mask = 1; 6574 break; 6575 default: 6576 PMD_DRV_LOG(ERR, "invalid src_port mask."); 6577 return -EINVAL; 6578 } 6579 6580 switch (filter->proto_mask) { 6581 case UINT8_MAX: 6582 filter_info->proto_mask = 0; 6583 filter_info->proto = 6584 convert_protocol_type(filter->proto); 6585 break; 6586 case 0: 6587 filter_info->proto_mask = 1; 6588 break; 6589 default: 6590 PMD_DRV_LOG(ERR, "invalid protocol mask."); 6591 return -EINVAL; 6592 } 6593 6594 filter_info->priority = (uint8_t)filter->priority; 6595 return 0; 6596 } 6597 6598 /* 6599 * add or delete a ntuple filter 6600 * 6601 * @param 6602 * dev: Pointer to struct rte_eth_dev. 6603 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6604 * add: if true, add filter, if false, remove filter 6605 * 6606 * @return 6607 * - On success, zero. 6608 * - On failure, a negative value. 6609 */ 6610 int 6611 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, 6612 struct rte_eth_ntuple_filter *ntuple_filter, 6613 bool add) 6614 { 6615 struct ixgbe_filter_info *filter_info = 6616 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6617 struct ixgbe_5tuple_filter_info filter_5tuple; 6618 struct ixgbe_5tuple_filter *filter; 6619 int ret; 6620 6621 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6622 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6623 return -EINVAL; 6624 } 6625 6626 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6627 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6628 if (ret < 0) 6629 return ret; 6630 6631 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6632 &filter_5tuple); 6633 if (filter != NULL && add) { 6634 PMD_DRV_LOG(ERR, "filter exists."); 6635 return -EEXIST; 6636 } 6637 if (filter == NULL && !add) { 6638 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6639 return -ENOENT; 6640 } 6641 6642 if (add) { 6643 filter = rte_zmalloc("ixgbe_5tuple_filter", 6644 sizeof(struct ixgbe_5tuple_filter), 0); 6645 if (filter == NULL) 6646 return -ENOMEM; 6647 rte_memcpy(&filter->filter_info, 6648 &filter_5tuple, 6649 sizeof(struct ixgbe_5tuple_filter_info)); 6650 filter->queue = ntuple_filter->queue; 6651 ret = ixgbe_add_5tuple_filter(dev, filter); 6652 if (ret < 0) { 6653 rte_free(filter); 6654 return ret; 6655 } 6656 } else 6657 ixgbe_remove_5tuple_filter(dev, filter); 6658 6659 return 0; 6660 } 6661 6662 /* 6663 * get a ntuple filter 6664 * 6665 * @param 6666 * dev: Pointer to struct rte_eth_dev. 6667 * ntuple_filter: Pointer to struct rte_eth_ntuple_filter 6668 * 6669 * @return 6670 * - On success, zero. 6671 * - On failure, a negative value. 6672 */ 6673 static int 6674 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev, 6675 struct rte_eth_ntuple_filter *ntuple_filter) 6676 { 6677 struct ixgbe_filter_info *filter_info = 6678 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6679 struct ixgbe_5tuple_filter_info filter_5tuple; 6680 struct ixgbe_5tuple_filter *filter; 6681 int ret; 6682 6683 if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) { 6684 PMD_DRV_LOG(ERR, "only 5tuple is supported."); 6685 return -EINVAL; 6686 } 6687 6688 memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info)); 6689 ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple); 6690 if (ret < 0) 6691 return ret; 6692 6693 filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list, 6694 &filter_5tuple); 6695 if (filter == NULL) { 6696 PMD_DRV_LOG(ERR, "filter doesn't exist."); 6697 return -ENOENT; 6698 } 6699 ntuple_filter->queue = filter->queue; 6700 return 0; 6701 } 6702 6703 /* 6704 * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter. 6705 * @dev: pointer to rte_eth_dev structure 6706 * @filter_op:operation will be taken. 6707 * @arg: a pointer to specific structure corresponding to the filter_op 6708 * 6709 * @return 6710 * - On success, zero. 6711 * - On failure, a negative value. 6712 */ 6713 static int 6714 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, 6715 enum rte_filter_op filter_op, 6716 void *arg) 6717 { 6718 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6719 int ret; 6720 6721 MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); 6722 6723 if (filter_op == RTE_ETH_FILTER_NOP) 6724 return 0; 6725 6726 if (arg == NULL) { 6727 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 6728 filter_op); 6729 return -EINVAL; 6730 } 6731 6732 switch (filter_op) { 6733 case RTE_ETH_FILTER_ADD: 6734 ret = ixgbe_add_del_ntuple_filter(dev, 6735 (struct rte_eth_ntuple_filter *)arg, 6736 TRUE); 6737 break; 6738 case RTE_ETH_FILTER_DELETE: 6739 ret = ixgbe_add_del_ntuple_filter(dev, 6740 (struct rte_eth_ntuple_filter *)arg, 6741 FALSE); 6742 break; 6743 case RTE_ETH_FILTER_GET: 6744 ret = ixgbe_get_ntuple_filter(dev, 6745 (struct rte_eth_ntuple_filter *)arg); 6746 break; 6747 default: 6748 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 6749 ret = -EINVAL; 6750 break; 6751 } 6752 return ret; 6753 } 6754 6755 int 6756 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, 6757 struct rte_eth_ethertype_filter *filter, 6758 bool add) 6759 { 6760 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6761 struct ixgbe_filter_info *filter_info = 6762 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6763 uint32_t etqf = 0; 6764 uint32_t etqs = 0; 6765 int ret; 6766 struct ixgbe_ethertype_filter ethertype_filter; 6767 6768 if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) 6769 return -EINVAL; 6770 6771 if (filter->ether_type == RTE_ETHER_TYPE_IPV4 || 6772 filter->ether_type == RTE_ETHER_TYPE_IPV6) { 6773 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" 6774 " ethertype filter.", filter->ether_type); 6775 return -EINVAL; 6776 } 6777 6778 if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { 6779 PMD_DRV_LOG(ERR, "mac compare is unsupported."); 6780 return -EINVAL; 6781 } 6782 if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { 6783 PMD_DRV_LOG(ERR, "drop option is unsupported."); 6784 return -EINVAL; 6785 } 6786 6787 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6788 if (ret >= 0 && add) { 6789 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", 6790 filter->ether_type); 6791 return -EEXIST; 6792 } 6793 if (ret < 0 && !add) { 6794 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6795 filter->ether_type); 6796 return -ENOENT; 6797 } 6798 6799 if (add) { 6800 etqf = IXGBE_ETQF_FILTER_EN; 6801 etqf |= (uint32_t)filter->ether_type; 6802 etqs |= (uint32_t)((filter->queue << 6803 IXGBE_ETQS_RX_QUEUE_SHIFT) & 6804 IXGBE_ETQS_RX_QUEUE); 6805 etqs |= IXGBE_ETQS_QUEUE_EN; 6806 6807 ethertype_filter.ethertype = filter->ether_type; 6808 ethertype_filter.etqf = etqf; 6809 ethertype_filter.etqs = etqs; 6810 ethertype_filter.conf = FALSE; 6811 ret = ixgbe_ethertype_filter_insert(filter_info, 6812 ðertype_filter); 6813 if (ret < 0) { 6814 PMD_DRV_LOG(ERR, "ethertype filters are full."); 6815 return -ENOSPC; 6816 } 6817 } else { 6818 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret); 6819 if (ret < 0) 6820 return -ENOSYS; 6821 } 6822 IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf); 6823 IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs); 6824 IXGBE_WRITE_FLUSH(hw); 6825 6826 return 0; 6827 } 6828 6829 static int 6830 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev, 6831 struct rte_eth_ethertype_filter *filter) 6832 { 6833 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6834 struct ixgbe_filter_info *filter_info = 6835 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 6836 uint32_t etqf, etqs; 6837 int ret; 6838 6839 ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type); 6840 if (ret < 0) { 6841 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", 6842 filter->ether_type); 6843 return -ENOENT; 6844 } 6845 6846 etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret)); 6847 if (etqf & IXGBE_ETQF_FILTER_EN) { 6848 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret)); 6849 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE; 6850 filter->flags = 0; 6851 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >> 6852 IXGBE_ETQS_RX_QUEUE_SHIFT; 6853 return 0; 6854 } 6855 return -ENOENT; 6856 } 6857 6858 /* 6859 * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter. 6860 * @dev: pointer to rte_eth_dev structure 6861 * @filter_op:operation will be taken. 6862 * @arg: a pointer to specific structure corresponding to the filter_op 6863 */ 6864 static int 6865 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev, 6866 enum rte_filter_op filter_op, 6867 void *arg) 6868 { 6869 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6870 int ret; 6871 6872 MAC_TYPE_FILTER_SUP(hw->mac.type); 6873 6874 if (filter_op == RTE_ETH_FILTER_NOP) 6875 return 0; 6876 6877 if (arg == NULL) { 6878 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 6879 filter_op); 6880 return -EINVAL; 6881 } 6882 6883 switch (filter_op) { 6884 case RTE_ETH_FILTER_ADD: 6885 ret = ixgbe_add_del_ethertype_filter(dev, 6886 (struct rte_eth_ethertype_filter *)arg, 6887 TRUE); 6888 break; 6889 case RTE_ETH_FILTER_DELETE: 6890 ret = ixgbe_add_del_ethertype_filter(dev, 6891 (struct rte_eth_ethertype_filter *)arg, 6892 FALSE); 6893 break; 6894 case RTE_ETH_FILTER_GET: 6895 ret = ixgbe_get_ethertype_filter(dev, 6896 (struct rte_eth_ethertype_filter *)arg); 6897 break; 6898 default: 6899 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 6900 ret = -EINVAL; 6901 break; 6902 } 6903 return ret; 6904 } 6905 6906 static int 6907 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, 6908 enum rte_filter_type filter_type, 6909 enum rte_filter_op filter_op, 6910 void *arg) 6911 { 6912 int ret = 0; 6913 6914 switch (filter_type) { 6915 case RTE_ETH_FILTER_NTUPLE: 6916 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg); 6917 break; 6918 case RTE_ETH_FILTER_ETHERTYPE: 6919 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg); 6920 break; 6921 case RTE_ETH_FILTER_SYN: 6922 ret = ixgbe_syn_filter_handle(dev, filter_op, arg); 6923 break; 6924 case RTE_ETH_FILTER_FDIR: 6925 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg); 6926 break; 6927 case RTE_ETH_FILTER_L2_TUNNEL: 6928 ret = ixgbe_dev_l2_tunnel_filter_handle(dev, filter_op, arg); 6929 break; 6930 case RTE_ETH_FILTER_GENERIC: 6931 if (filter_op != RTE_ETH_FILTER_GET) 6932 return -EINVAL; 6933 *(const void **)arg = &ixgbe_flow_ops; 6934 break; 6935 default: 6936 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 6937 filter_type); 6938 ret = -EINVAL; 6939 break; 6940 } 6941 6942 return ret; 6943 } 6944 6945 static u8 * 6946 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw, 6947 u8 **mc_addr_ptr, u32 *vmdq) 6948 { 6949 u8 *mc_addr; 6950 6951 *vmdq = 0; 6952 mc_addr = *mc_addr_ptr; 6953 *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr)); 6954 return mc_addr; 6955 } 6956 6957 static int 6958 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev, 6959 struct rte_ether_addr *mc_addr_set, 6960 uint32_t nb_mc_addr) 6961 { 6962 struct ixgbe_hw *hw; 6963 u8 *mc_addr_list; 6964 6965 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6966 mc_addr_list = (u8 *)mc_addr_set; 6967 return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr, 6968 ixgbe_dev_addr_list_itr, TRUE); 6969 } 6970 6971 static uint64_t 6972 ixgbe_read_systime_cyclecounter(struct rte_eth_dev *dev) 6973 { 6974 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6975 uint64_t systime_cycles; 6976 6977 switch (hw->mac.type) { 6978 case ixgbe_mac_X550: 6979 case ixgbe_mac_X550EM_x: 6980 case ixgbe_mac_X550EM_a: 6981 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */ 6982 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6983 systime_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6984 * NSEC_PER_SEC; 6985 break; 6986 default: 6987 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML); 6988 systime_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) 6989 << 32; 6990 } 6991 6992 return systime_cycles; 6993 } 6994 6995 static uint64_t 6996 ixgbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev) 6997 { 6998 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 6999 uint64_t rx_tstamp_cycles; 7000 7001 switch (hw->mac.type) { 7002 case ixgbe_mac_X550: 7003 case ixgbe_mac_X550EM_x: 7004 case ixgbe_mac_X550EM_a: 7005 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 7006 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 7007 rx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 7008 * NSEC_PER_SEC; 7009 break; 7010 default: 7011 /* RXSTMPL stores ns and RXSTMPH stores seconds. */ 7012 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL); 7013 rx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) 7014 << 32; 7015 } 7016 7017 return rx_tstamp_cycles; 7018 } 7019 7020 static uint64_t 7021 ixgbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) 7022 { 7023 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7024 uint64_t tx_tstamp_cycles; 7025 7026 switch (hw->mac.type) { 7027 case ixgbe_mac_X550: 7028 case ixgbe_mac_X550EM_x: 7029 case ixgbe_mac_X550EM_a: 7030 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 7031 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 7032 tx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 7033 * NSEC_PER_SEC; 7034 break; 7035 default: 7036 /* TXSTMPL stores ns and TXSTMPH stores seconds. */ 7037 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL); 7038 tx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) 7039 << 32; 7040 } 7041 7042 return tx_tstamp_cycles; 7043 } 7044 7045 static void 7046 ixgbe_start_timecounters(struct rte_eth_dev *dev) 7047 { 7048 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7049 struct ixgbe_adapter *adapter = dev->data->dev_private; 7050 struct rte_eth_link link; 7051 uint32_t incval = 0; 7052 uint32_t shift = 0; 7053 7054 /* Get current link speed. */ 7055 ixgbe_dev_link_update(dev, 1); 7056 rte_eth_linkstatus_get(dev, &link); 7057 7058 switch (link.link_speed) { 7059 case ETH_SPEED_NUM_100M: 7060 incval = IXGBE_INCVAL_100; 7061 shift = IXGBE_INCVAL_SHIFT_100; 7062 break; 7063 case ETH_SPEED_NUM_1G: 7064 incval = IXGBE_INCVAL_1GB; 7065 shift = IXGBE_INCVAL_SHIFT_1GB; 7066 break; 7067 case ETH_SPEED_NUM_10G: 7068 default: 7069 incval = IXGBE_INCVAL_10GB; 7070 shift = IXGBE_INCVAL_SHIFT_10GB; 7071 break; 7072 } 7073 7074 switch (hw->mac.type) { 7075 case ixgbe_mac_X550: 7076 case ixgbe_mac_X550EM_x: 7077 case ixgbe_mac_X550EM_a: 7078 /* Independent of link speed. */ 7079 incval = 1; 7080 /* Cycles read will be interpreted as ns. */ 7081 shift = 0; 7082 /* Fall-through */ 7083 case ixgbe_mac_X540: 7084 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval); 7085 break; 7086 case ixgbe_mac_82599EB: 7087 incval >>= IXGBE_INCVAL_SHIFT_82599; 7088 shift -= IXGBE_INCVAL_SHIFT_82599; 7089 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 7090 (1 << IXGBE_INCPER_SHIFT_82599) | incval); 7091 break; 7092 default: 7093 /* Not supported. */ 7094 return; 7095 } 7096 7097 memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); 7098 memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 7099 memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); 7100 7101 adapter->systime_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 7102 adapter->systime_tc.cc_shift = shift; 7103 adapter->systime_tc.nsec_mask = (1ULL << shift) - 1; 7104 7105 adapter->rx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 7106 adapter->rx_tstamp_tc.cc_shift = shift; 7107 adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 7108 7109 adapter->tx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK; 7110 adapter->tx_tstamp_tc.cc_shift = shift; 7111 adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1; 7112 } 7113 7114 static int 7115 ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) 7116 { 7117 struct ixgbe_adapter *adapter = dev->data->dev_private; 7118 7119 adapter->systime_tc.nsec += delta; 7120 adapter->rx_tstamp_tc.nsec += delta; 7121 adapter->tx_tstamp_tc.nsec += delta; 7122 7123 return 0; 7124 } 7125 7126 static int 7127 ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) 7128 { 7129 uint64_t ns; 7130 struct ixgbe_adapter *adapter = dev->data->dev_private; 7131 7132 ns = rte_timespec_to_ns(ts); 7133 /* Set the timecounters to a new value. */ 7134 adapter->systime_tc.nsec = ns; 7135 adapter->rx_tstamp_tc.nsec = ns; 7136 adapter->tx_tstamp_tc.nsec = ns; 7137 7138 return 0; 7139 } 7140 7141 static int 7142 ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) 7143 { 7144 uint64_t ns, systime_cycles; 7145 struct ixgbe_adapter *adapter = dev->data->dev_private; 7146 7147 systime_cycles = ixgbe_read_systime_cyclecounter(dev); 7148 ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); 7149 *ts = rte_ns_to_timespec(ns); 7150 7151 return 0; 7152 } 7153 7154 static int 7155 ixgbe_timesync_enable(struct rte_eth_dev *dev) 7156 { 7157 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7158 uint32_t tsync_ctl; 7159 uint32_t tsauxc; 7160 7161 /* Stop the timesync system time. */ 7162 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0x0); 7163 /* Reset the timesync system time value. */ 7164 IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x0); 7165 IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x0); 7166 7167 /* Enable system time for platforms where it isn't on by default. */ 7168 tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); 7169 tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME; 7170 IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc); 7171 7172 ixgbe_start_timecounters(dev); 7173 7174 /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7175 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 7176 (RTE_ETHER_TYPE_1588 | 7177 IXGBE_ETQF_FILTER_EN | 7178 IXGBE_ETQF_1588)); 7179 7180 /* Enable timestamping of received PTP packets. */ 7181 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7182 tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED; 7183 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7184 7185 /* Enable timestamping of transmitted PTP packets. */ 7186 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7187 tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED; 7188 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7189 7190 IXGBE_WRITE_FLUSH(hw); 7191 7192 return 0; 7193 } 7194 7195 static int 7196 ixgbe_timesync_disable(struct rte_eth_dev *dev) 7197 { 7198 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7199 uint32_t tsync_ctl; 7200 7201 /* Disable timestamping of transmitted PTP packets. */ 7202 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7203 tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED; 7204 IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl); 7205 7206 /* Disable timestamping of received PTP packets. */ 7207 tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7208 tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED; 7209 IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl); 7210 7211 /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */ 7212 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0); 7213 7214 /* Stop incrementating the System Time registers. */ 7215 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0); 7216 7217 return 0; 7218 } 7219 7220 static int 7221 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev, 7222 struct timespec *timestamp, 7223 uint32_t flags __rte_unused) 7224 { 7225 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7226 struct ixgbe_adapter *adapter = dev->data->dev_private; 7227 uint32_t tsync_rxctl; 7228 uint64_t rx_tstamp_cycles; 7229 uint64_t ns; 7230 7231 tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); 7232 if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0) 7233 return -EINVAL; 7234 7235 rx_tstamp_cycles = ixgbe_read_rx_tstamp_cyclecounter(dev); 7236 ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); 7237 *timestamp = rte_ns_to_timespec(ns); 7238 7239 return 0; 7240 } 7241 7242 static int 7243 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev, 7244 struct timespec *timestamp) 7245 { 7246 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7247 struct ixgbe_adapter *adapter = dev->data->dev_private; 7248 uint32_t tsync_txctl; 7249 uint64_t tx_tstamp_cycles; 7250 uint64_t ns; 7251 7252 tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); 7253 if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0) 7254 return -EINVAL; 7255 7256 tx_tstamp_cycles = ixgbe_read_tx_tstamp_cyclecounter(dev); 7257 ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); 7258 *timestamp = rte_ns_to_timespec(ns); 7259 7260 return 0; 7261 } 7262 7263 static int 7264 ixgbe_get_reg_length(struct rte_eth_dev *dev) 7265 { 7266 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7267 int count = 0; 7268 int g_ind = 0; 7269 const struct reg_info *reg_group; 7270 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7271 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7272 7273 while ((reg_group = reg_set[g_ind++])) 7274 count += ixgbe_regs_group_count(reg_group); 7275 7276 return count; 7277 } 7278 7279 static int 7280 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused) 7281 { 7282 int count = 0; 7283 int g_ind = 0; 7284 const struct reg_info *reg_group; 7285 7286 while ((reg_group = ixgbevf_regs[g_ind++])) 7287 count += ixgbe_regs_group_count(reg_group); 7288 7289 return count; 7290 } 7291 7292 static int 7293 ixgbe_get_regs(struct rte_eth_dev *dev, 7294 struct rte_dev_reg_info *regs) 7295 { 7296 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7297 uint32_t *data = regs->data; 7298 int g_ind = 0; 7299 int count = 0; 7300 const struct reg_info *reg_group; 7301 const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ? 7302 ixgbe_regs_mac_82598EB : ixgbe_regs_others; 7303 7304 if (data == NULL) { 7305 regs->length = ixgbe_get_reg_length(dev); 7306 regs->width = sizeof(uint32_t); 7307 return 0; 7308 } 7309 7310 /* Support only full register dump */ 7311 if ((regs->length == 0) || 7312 (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) { 7313 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7314 hw->device_id; 7315 while ((reg_group = reg_set[g_ind++])) 7316 count += ixgbe_read_regs_group(dev, &data[count], 7317 reg_group); 7318 return 0; 7319 } 7320 7321 return -ENOTSUP; 7322 } 7323 7324 static int 7325 ixgbevf_get_regs(struct rte_eth_dev *dev, 7326 struct rte_dev_reg_info *regs) 7327 { 7328 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7329 uint32_t *data = regs->data; 7330 int g_ind = 0; 7331 int count = 0; 7332 const struct reg_info *reg_group; 7333 7334 if (data == NULL) { 7335 regs->length = ixgbevf_get_reg_length(dev); 7336 regs->width = sizeof(uint32_t); 7337 return 0; 7338 } 7339 7340 /* Support only full register dump */ 7341 if ((regs->length == 0) || 7342 (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) { 7343 regs->version = hw->mac.type << 24 | hw->revision_id << 16 | 7344 hw->device_id; 7345 while ((reg_group = ixgbevf_regs[g_ind++])) 7346 count += ixgbe_read_regs_group(dev, &data[count], 7347 reg_group); 7348 return 0; 7349 } 7350 7351 return -ENOTSUP; 7352 } 7353 7354 static int 7355 ixgbe_get_eeprom_length(struct rte_eth_dev *dev) 7356 { 7357 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7358 7359 /* Return unit is byte count */ 7360 return hw->eeprom.word_size * 2; 7361 } 7362 7363 static int 7364 ixgbe_get_eeprom(struct rte_eth_dev *dev, 7365 struct rte_dev_eeprom_info *in_eeprom) 7366 { 7367 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7368 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7369 uint16_t *data = in_eeprom->data; 7370 int first, length; 7371 7372 first = in_eeprom->offset >> 1; 7373 length = in_eeprom->length >> 1; 7374 if ((first > hw->eeprom.word_size) || 7375 ((first + length) > hw->eeprom.word_size)) 7376 return -EINVAL; 7377 7378 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7379 7380 return eeprom->ops.read_buffer(hw, first, length, data); 7381 } 7382 7383 static int 7384 ixgbe_set_eeprom(struct rte_eth_dev *dev, 7385 struct rte_dev_eeprom_info *in_eeprom) 7386 { 7387 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7388 struct ixgbe_eeprom_info *eeprom = &hw->eeprom; 7389 uint16_t *data = in_eeprom->data; 7390 int first, length; 7391 7392 first = in_eeprom->offset >> 1; 7393 length = in_eeprom->length >> 1; 7394 if ((first > hw->eeprom.word_size) || 7395 ((first + length) > hw->eeprom.word_size)) 7396 return -EINVAL; 7397 7398 in_eeprom->magic = hw->vendor_id | (hw->device_id << 16); 7399 7400 return eeprom->ops.write_buffer(hw, first, length, data); 7401 } 7402 7403 static int 7404 ixgbe_get_module_info(struct rte_eth_dev *dev, 7405 struct rte_eth_dev_module_info *modinfo) 7406 { 7407 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7408 uint32_t status; 7409 uint8_t sff8472_rev, addr_mode; 7410 bool page_swap = false; 7411 7412 /* Check whether we support SFF-8472 or not */ 7413 status = hw->phy.ops.read_i2c_eeprom(hw, 7414 IXGBE_SFF_SFF_8472_COMP, 7415 &sff8472_rev); 7416 if (status != 0) 7417 return -EIO; 7418 7419 /* addressing mode is not supported */ 7420 status = hw->phy.ops.read_i2c_eeprom(hw, 7421 IXGBE_SFF_SFF_8472_SWAP, 7422 &addr_mode); 7423 if (status != 0) 7424 return -EIO; 7425 7426 if (addr_mode & IXGBE_SFF_ADDRESSING_MODE) { 7427 PMD_DRV_LOG(ERR, 7428 "Address change required to access page 0xA2, " 7429 "but not supported. Please report the module " 7430 "type to the driver maintainers."); 7431 page_swap = true; 7432 } 7433 7434 if (sff8472_rev == IXGBE_SFF_SFF_8472_UNSUP || page_swap) { 7435 /* We have a SFP, but it does not support SFF-8472 */ 7436 modinfo->type = RTE_ETH_MODULE_SFF_8079; 7437 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN; 7438 } else { 7439 /* We have a SFP which supports a revision of SFF-8472. */ 7440 modinfo->type = RTE_ETH_MODULE_SFF_8472; 7441 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN; 7442 } 7443 7444 return 0; 7445 } 7446 7447 static int 7448 ixgbe_get_module_eeprom(struct rte_eth_dev *dev, 7449 struct rte_dev_eeprom_info *info) 7450 { 7451 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7452 uint32_t status = IXGBE_ERR_PHY_ADDR_INVALID; 7453 uint8_t databyte = 0xFF; 7454 uint8_t *data = info->data; 7455 uint32_t i = 0; 7456 7457 if (info->length == 0) 7458 return -EINVAL; 7459 7460 for (i = info->offset; i < info->offset + info->length; i++) { 7461 if (i < RTE_ETH_MODULE_SFF_8079_LEN) 7462 status = hw->phy.ops.read_i2c_eeprom(hw, i, &databyte); 7463 else 7464 status = hw->phy.ops.read_i2c_sff8472(hw, i, &databyte); 7465 7466 if (status != 0) 7467 return -EIO; 7468 7469 data[i - info->offset] = databyte; 7470 } 7471 7472 return 0; 7473 } 7474 7475 uint16_t 7476 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) { 7477 switch (mac_type) { 7478 case ixgbe_mac_X550: 7479 case ixgbe_mac_X550EM_x: 7480 case ixgbe_mac_X550EM_a: 7481 return ETH_RSS_RETA_SIZE_512; 7482 case ixgbe_mac_X550_vf: 7483 case ixgbe_mac_X550EM_x_vf: 7484 case ixgbe_mac_X550EM_a_vf: 7485 return ETH_RSS_RETA_SIZE_64; 7486 case ixgbe_mac_X540_vf: 7487 case ixgbe_mac_82599_vf: 7488 return 0; 7489 default: 7490 return ETH_RSS_RETA_SIZE_128; 7491 } 7492 } 7493 7494 uint32_t 7495 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) { 7496 switch (mac_type) { 7497 case ixgbe_mac_X550: 7498 case ixgbe_mac_X550EM_x: 7499 case ixgbe_mac_X550EM_a: 7500 if (reta_idx < ETH_RSS_RETA_SIZE_128) 7501 return IXGBE_RETA(reta_idx >> 2); 7502 else 7503 return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2); 7504 case ixgbe_mac_X550_vf: 7505 case ixgbe_mac_X550EM_x_vf: 7506 case ixgbe_mac_X550EM_a_vf: 7507 return IXGBE_VFRETA(reta_idx >> 2); 7508 default: 7509 return IXGBE_RETA(reta_idx >> 2); 7510 } 7511 } 7512 7513 uint32_t 7514 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) { 7515 switch (mac_type) { 7516 case ixgbe_mac_X550_vf: 7517 case ixgbe_mac_X550EM_x_vf: 7518 case ixgbe_mac_X550EM_a_vf: 7519 return IXGBE_VFMRQC; 7520 default: 7521 return IXGBE_MRQC; 7522 } 7523 } 7524 7525 uint32_t 7526 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) { 7527 switch (mac_type) { 7528 case ixgbe_mac_X550_vf: 7529 case ixgbe_mac_X550EM_x_vf: 7530 case ixgbe_mac_X550EM_a_vf: 7531 return IXGBE_VFRSSRK(i); 7532 default: 7533 return IXGBE_RSSRK(i); 7534 } 7535 } 7536 7537 bool 7538 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) { 7539 switch (mac_type) { 7540 case ixgbe_mac_82599_vf: 7541 case ixgbe_mac_X540_vf: 7542 return 0; 7543 default: 7544 return 1; 7545 } 7546 } 7547 7548 static int 7549 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, 7550 struct rte_eth_dcb_info *dcb_info) 7551 { 7552 struct ixgbe_dcb_config *dcb_config = 7553 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 7554 struct ixgbe_dcb_tc_config *tc; 7555 struct rte_eth_dcb_tc_queue_mapping *tc_queue; 7556 uint8_t nb_tcs; 7557 uint8_t i, j; 7558 7559 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) 7560 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs; 7561 else 7562 dcb_info->nb_tcs = 1; 7563 7564 tc_queue = &dcb_info->tc_queue; 7565 nb_tcs = dcb_info->nb_tcs; 7566 7567 if (dcb_config->vt_mode) { /* vt is enabled*/ 7568 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 7569 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 7570 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7571 dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i]; 7572 if (RTE_ETH_DEV_SRIOV(dev).active > 0) { 7573 for (j = 0; j < nb_tcs; j++) { 7574 tc_queue->tc_rxq[0][j].base = j; 7575 tc_queue->tc_rxq[0][j].nb_queue = 1; 7576 tc_queue->tc_txq[0][j].base = j; 7577 tc_queue->tc_txq[0][j].nb_queue = 1; 7578 } 7579 } else { 7580 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { 7581 for (j = 0; j < nb_tcs; j++) { 7582 tc_queue->tc_rxq[i][j].base = 7583 i * nb_tcs + j; 7584 tc_queue->tc_rxq[i][j].nb_queue = 1; 7585 tc_queue->tc_txq[i][j].base = 7586 i * nb_tcs + j; 7587 tc_queue->tc_txq[i][j].nb_queue = 1; 7588 } 7589 } 7590 } 7591 } else { /* vt is disabled*/ 7592 struct rte_eth_dcb_rx_conf *rx_conf = 7593 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 7594 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 7595 dcb_info->prio_tc[i] = rx_conf->dcb_tc[i]; 7596 if (dcb_info->nb_tcs == ETH_4_TCS) { 7597 for (i = 0; i < dcb_info->nb_tcs; i++) { 7598 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32; 7599 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7600 } 7601 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7602 dcb_info->tc_queue.tc_txq[0][1].base = 64; 7603 dcb_info->tc_queue.tc_txq[0][2].base = 96; 7604 dcb_info->tc_queue.tc_txq[0][3].base = 112; 7605 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64; 7606 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7607 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7608 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7609 } else if (dcb_info->nb_tcs == ETH_8_TCS) { 7610 for (i = 0; i < dcb_info->nb_tcs; i++) { 7611 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16; 7612 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16; 7613 } 7614 dcb_info->tc_queue.tc_txq[0][0].base = 0; 7615 dcb_info->tc_queue.tc_txq[0][1].base = 32; 7616 dcb_info->tc_queue.tc_txq[0][2].base = 64; 7617 dcb_info->tc_queue.tc_txq[0][3].base = 80; 7618 dcb_info->tc_queue.tc_txq[0][4].base = 96; 7619 dcb_info->tc_queue.tc_txq[0][5].base = 104; 7620 dcb_info->tc_queue.tc_txq[0][6].base = 112; 7621 dcb_info->tc_queue.tc_txq[0][7].base = 120; 7622 dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32; 7623 dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32; 7624 dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16; 7625 dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16; 7626 dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8; 7627 dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8; 7628 dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8; 7629 dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8; 7630 } 7631 } 7632 for (i = 0; i < dcb_info->nb_tcs; i++) { 7633 tc = &dcb_config->tc_config[i]; 7634 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent; 7635 } 7636 return 0; 7637 } 7638 7639 /* Update e-tag ether type */ 7640 static int 7641 ixgbe_update_e_tag_eth_type(struct ixgbe_hw *hw, 7642 uint16_t ether_type) 7643 { 7644 uint32_t etag_etype; 7645 7646 if (hw->mac.type != ixgbe_mac_X550 && 7647 hw->mac.type != ixgbe_mac_X550EM_x && 7648 hw->mac.type != ixgbe_mac_X550EM_a) { 7649 return -ENOTSUP; 7650 } 7651 7652 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7653 etag_etype &= ~IXGBE_ETAG_ETYPE_MASK; 7654 etag_etype |= ether_type; 7655 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7656 IXGBE_WRITE_FLUSH(hw); 7657 7658 return 0; 7659 } 7660 7661 /* Config l2 tunnel ether type */ 7662 static int 7663 ixgbe_dev_l2_tunnel_eth_type_conf(struct rte_eth_dev *dev, 7664 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7665 { 7666 int ret = 0; 7667 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7668 struct ixgbe_l2_tn_info *l2_tn_info = 7669 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7670 7671 if (l2_tunnel == NULL) 7672 return -EINVAL; 7673 7674 switch (l2_tunnel->l2_tunnel_type) { 7675 case RTE_L2_TUNNEL_TYPE_E_TAG: 7676 l2_tn_info->e_tag_ether_type = l2_tunnel->ether_type; 7677 ret = ixgbe_update_e_tag_eth_type(hw, l2_tunnel->ether_type); 7678 break; 7679 default: 7680 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7681 ret = -EINVAL; 7682 break; 7683 } 7684 7685 return ret; 7686 } 7687 7688 /* Enable e-tag tunnel */ 7689 static int 7690 ixgbe_e_tag_enable(struct ixgbe_hw *hw) 7691 { 7692 uint32_t etag_etype; 7693 7694 if (hw->mac.type != ixgbe_mac_X550 && 7695 hw->mac.type != ixgbe_mac_X550EM_x && 7696 hw->mac.type != ixgbe_mac_X550EM_a) { 7697 return -ENOTSUP; 7698 } 7699 7700 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7701 etag_etype |= IXGBE_ETAG_ETYPE_VALID; 7702 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7703 IXGBE_WRITE_FLUSH(hw); 7704 7705 return 0; 7706 } 7707 7708 /* Enable l2 tunnel */ 7709 static int 7710 ixgbe_dev_l2_tunnel_enable(struct rte_eth_dev *dev, 7711 enum rte_eth_tunnel_type l2_tunnel_type) 7712 { 7713 int ret = 0; 7714 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7715 struct ixgbe_l2_tn_info *l2_tn_info = 7716 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7717 7718 switch (l2_tunnel_type) { 7719 case RTE_L2_TUNNEL_TYPE_E_TAG: 7720 l2_tn_info->e_tag_en = TRUE; 7721 ret = ixgbe_e_tag_enable(hw); 7722 break; 7723 default: 7724 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7725 ret = -EINVAL; 7726 break; 7727 } 7728 7729 return ret; 7730 } 7731 7732 /* Disable e-tag tunnel */ 7733 static int 7734 ixgbe_e_tag_disable(struct ixgbe_hw *hw) 7735 { 7736 uint32_t etag_etype; 7737 7738 if (hw->mac.type != ixgbe_mac_X550 && 7739 hw->mac.type != ixgbe_mac_X550EM_x && 7740 hw->mac.type != ixgbe_mac_X550EM_a) { 7741 return -ENOTSUP; 7742 } 7743 7744 etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE); 7745 etag_etype &= ~IXGBE_ETAG_ETYPE_VALID; 7746 IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype); 7747 IXGBE_WRITE_FLUSH(hw); 7748 7749 return 0; 7750 } 7751 7752 /* Disable l2 tunnel */ 7753 static int 7754 ixgbe_dev_l2_tunnel_disable(struct rte_eth_dev *dev, 7755 enum rte_eth_tunnel_type l2_tunnel_type) 7756 { 7757 int ret = 0; 7758 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7759 struct ixgbe_l2_tn_info *l2_tn_info = 7760 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7761 7762 switch (l2_tunnel_type) { 7763 case RTE_L2_TUNNEL_TYPE_E_TAG: 7764 l2_tn_info->e_tag_en = FALSE; 7765 ret = ixgbe_e_tag_disable(hw); 7766 break; 7767 default: 7768 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7769 ret = -EINVAL; 7770 break; 7771 } 7772 7773 return ret; 7774 } 7775 7776 static int 7777 ixgbe_e_tag_filter_del(struct rte_eth_dev *dev, 7778 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7779 { 7780 int ret = 0; 7781 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7782 uint32_t i, rar_entries; 7783 uint32_t rar_low, rar_high; 7784 7785 if (hw->mac.type != ixgbe_mac_X550 && 7786 hw->mac.type != ixgbe_mac_X550EM_x && 7787 hw->mac.type != ixgbe_mac_X550EM_a) { 7788 return -ENOTSUP; 7789 } 7790 7791 rar_entries = ixgbe_get_num_rx_addrs(hw); 7792 7793 for (i = 1; i < rar_entries; i++) { 7794 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7795 rar_low = IXGBE_READ_REG(hw, IXGBE_RAL(i)); 7796 if ((rar_high & IXGBE_RAH_AV) && 7797 (rar_high & IXGBE_RAH_ADTYPE) && 7798 ((rar_low & IXGBE_RAL_ETAG_FILTER_MASK) == 7799 l2_tunnel->tunnel_id)) { 7800 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0); 7801 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0); 7802 7803 ixgbe_clear_vmdq(hw, i, IXGBE_CLEAR_VMDQ_ALL); 7804 7805 return ret; 7806 } 7807 } 7808 7809 return ret; 7810 } 7811 7812 static int 7813 ixgbe_e_tag_filter_add(struct rte_eth_dev *dev, 7814 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7815 { 7816 int ret = 0; 7817 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 7818 uint32_t i, rar_entries; 7819 uint32_t rar_low, rar_high; 7820 7821 if (hw->mac.type != ixgbe_mac_X550 && 7822 hw->mac.type != ixgbe_mac_X550EM_x && 7823 hw->mac.type != ixgbe_mac_X550EM_a) { 7824 return -ENOTSUP; 7825 } 7826 7827 /* One entry for one tunnel. Try to remove potential existing entry. */ 7828 ixgbe_e_tag_filter_del(dev, l2_tunnel); 7829 7830 rar_entries = ixgbe_get_num_rx_addrs(hw); 7831 7832 for (i = 1; i < rar_entries; i++) { 7833 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i)); 7834 if (rar_high & IXGBE_RAH_AV) { 7835 continue; 7836 } else { 7837 ixgbe_set_vmdq(hw, i, l2_tunnel->pool); 7838 rar_high = IXGBE_RAH_AV | IXGBE_RAH_ADTYPE; 7839 rar_low = l2_tunnel->tunnel_id; 7840 7841 IXGBE_WRITE_REG(hw, IXGBE_RAL(i), rar_low); 7842 IXGBE_WRITE_REG(hw, IXGBE_RAH(i), rar_high); 7843 7844 return ret; 7845 } 7846 } 7847 7848 PMD_INIT_LOG(NOTICE, "The table of E-tag forwarding rule is full." 7849 " Please remove a rule before adding a new one."); 7850 return -EINVAL; 7851 } 7852 7853 static inline struct ixgbe_l2_tn_filter * 7854 ixgbe_l2_tn_filter_lookup(struct ixgbe_l2_tn_info *l2_tn_info, 7855 struct ixgbe_l2_tn_key *key) 7856 { 7857 int ret; 7858 7859 ret = rte_hash_lookup(l2_tn_info->hash_handle, (const void *)key); 7860 if (ret < 0) 7861 return NULL; 7862 7863 return l2_tn_info->hash_map[ret]; 7864 } 7865 7866 static inline int 7867 ixgbe_insert_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7868 struct ixgbe_l2_tn_filter *l2_tn_filter) 7869 { 7870 int ret; 7871 7872 ret = rte_hash_add_key(l2_tn_info->hash_handle, 7873 &l2_tn_filter->key); 7874 7875 if (ret < 0) { 7876 PMD_DRV_LOG(ERR, 7877 "Failed to insert L2 tunnel filter" 7878 " to hash table %d!", 7879 ret); 7880 return ret; 7881 } 7882 7883 l2_tn_info->hash_map[ret] = l2_tn_filter; 7884 7885 TAILQ_INSERT_TAIL(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7886 7887 return 0; 7888 } 7889 7890 static inline int 7891 ixgbe_remove_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, 7892 struct ixgbe_l2_tn_key *key) 7893 { 7894 int ret; 7895 struct ixgbe_l2_tn_filter *l2_tn_filter; 7896 7897 ret = rte_hash_del_key(l2_tn_info->hash_handle, key); 7898 7899 if (ret < 0) { 7900 PMD_DRV_LOG(ERR, 7901 "No such L2 tunnel filter to delete %d!", 7902 ret); 7903 return ret; 7904 } 7905 7906 l2_tn_filter = l2_tn_info->hash_map[ret]; 7907 l2_tn_info->hash_map[ret] = NULL; 7908 7909 TAILQ_REMOVE(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); 7910 rte_free(l2_tn_filter); 7911 7912 return 0; 7913 } 7914 7915 /* Add l2 tunnel filter */ 7916 int 7917 ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, 7918 struct rte_eth_l2_tunnel_conf *l2_tunnel, 7919 bool restore) 7920 { 7921 int ret; 7922 struct ixgbe_l2_tn_info *l2_tn_info = 7923 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7924 struct ixgbe_l2_tn_key key; 7925 struct ixgbe_l2_tn_filter *node; 7926 7927 if (!restore) { 7928 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7929 key.tn_id = l2_tunnel->tunnel_id; 7930 7931 node = ixgbe_l2_tn_filter_lookup(l2_tn_info, &key); 7932 7933 if (node) { 7934 PMD_DRV_LOG(ERR, 7935 "The L2 tunnel filter already exists!"); 7936 return -EINVAL; 7937 } 7938 7939 node = rte_zmalloc("ixgbe_l2_tn", 7940 sizeof(struct ixgbe_l2_tn_filter), 7941 0); 7942 if (!node) 7943 return -ENOMEM; 7944 7945 rte_memcpy(&node->key, 7946 &key, 7947 sizeof(struct ixgbe_l2_tn_key)); 7948 node->pool = l2_tunnel->pool; 7949 ret = ixgbe_insert_l2_tn_filter(l2_tn_info, node); 7950 if (ret < 0) { 7951 rte_free(node); 7952 return ret; 7953 } 7954 } 7955 7956 switch (l2_tunnel->l2_tunnel_type) { 7957 case RTE_L2_TUNNEL_TYPE_E_TAG: 7958 ret = ixgbe_e_tag_filter_add(dev, l2_tunnel); 7959 break; 7960 default: 7961 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7962 ret = -EINVAL; 7963 break; 7964 } 7965 7966 if ((!restore) && (ret < 0)) 7967 (void)ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7968 7969 return ret; 7970 } 7971 7972 /* Delete l2 tunnel filter */ 7973 int 7974 ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, 7975 struct rte_eth_l2_tunnel_conf *l2_tunnel) 7976 { 7977 int ret; 7978 struct ixgbe_l2_tn_info *l2_tn_info = 7979 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 7980 struct ixgbe_l2_tn_key key; 7981 7982 key.l2_tn_type = l2_tunnel->l2_tunnel_type; 7983 key.tn_id = l2_tunnel->tunnel_id; 7984 ret = ixgbe_remove_l2_tn_filter(l2_tn_info, &key); 7985 if (ret < 0) 7986 return ret; 7987 7988 switch (l2_tunnel->l2_tunnel_type) { 7989 case RTE_L2_TUNNEL_TYPE_E_TAG: 7990 ret = ixgbe_e_tag_filter_del(dev, l2_tunnel); 7991 break; 7992 default: 7993 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 7994 ret = -EINVAL; 7995 break; 7996 } 7997 7998 return ret; 7999 } 8000 8001 /** 8002 * ixgbe_dev_l2_tunnel_filter_handle - Handle operations for l2 tunnel filter. 8003 * @dev: pointer to rte_eth_dev structure 8004 * @filter_op:operation will be taken. 8005 * @arg: a pointer to specific structure corresponding to the filter_op 8006 */ 8007 static int 8008 ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, 8009 enum rte_filter_op filter_op, 8010 void *arg) 8011 { 8012 int ret; 8013 8014 if (filter_op == RTE_ETH_FILTER_NOP) 8015 return 0; 8016 8017 if (arg == NULL) { 8018 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", 8019 filter_op); 8020 return -EINVAL; 8021 } 8022 8023 switch (filter_op) { 8024 case RTE_ETH_FILTER_ADD: 8025 ret = ixgbe_dev_l2_tunnel_filter_add 8026 (dev, 8027 (struct rte_eth_l2_tunnel_conf *)arg, 8028 FALSE); 8029 break; 8030 case RTE_ETH_FILTER_DELETE: 8031 ret = ixgbe_dev_l2_tunnel_filter_del 8032 (dev, 8033 (struct rte_eth_l2_tunnel_conf *)arg); 8034 break; 8035 default: 8036 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); 8037 ret = -EINVAL; 8038 break; 8039 } 8040 return ret; 8041 } 8042 8043 static int 8044 ixgbe_e_tag_forwarding_en_dis(struct rte_eth_dev *dev, bool en) 8045 { 8046 int ret = 0; 8047 uint32_t ctrl; 8048 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8049 8050 if (hw->mac.type != ixgbe_mac_X550 && 8051 hw->mac.type != ixgbe_mac_X550EM_x && 8052 hw->mac.type != ixgbe_mac_X550EM_a) { 8053 return -ENOTSUP; 8054 } 8055 8056 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 8057 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK; 8058 if (en) 8059 ctrl |= IXGBE_VT_CTL_POOLING_MODE_ETAG; 8060 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl); 8061 8062 return ret; 8063 } 8064 8065 /* Enable l2 tunnel forwarding */ 8066 static int 8067 ixgbe_dev_l2_tunnel_forwarding_enable 8068 (struct rte_eth_dev *dev, 8069 enum rte_eth_tunnel_type l2_tunnel_type) 8070 { 8071 struct ixgbe_l2_tn_info *l2_tn_info = 8072 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8073 int ret = 0; 8074 8075 switch (l2_tunnel_type) { 8076 case RTE_L2_TUNNEL_TYPE_E_TAG: 8077 l2_tn_info->e_tag_fwd_en = TRUE; 8078 ret = ixgbe_e_tag_forwarding_en_dis(dev, 1); 8079 break; 8080 default: 8081 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8082 ret = -EINVAL; 8083 break; 8084 } 8085 8086 return ret; 8087 } 8088 8089 /* Disable l2 tunnel forwarding */ 8090 static int 8091 ixgbe_dev_l2_tunnel_forwarding_disable 8092 (struct rte_eth_dev *dev, 8093 enum rte_eth_tunnel_type l2_tunnel_type) 8094 { 8095 struct ixgbe_l2_tn_info *l2_tn_info = 8096 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8097 int ret = 0; 8098 8099 switch (l2_tunnel_type) { 8100 case RTE_L2_TUNNEL_TYPE_E_TAG: 8101 l2_tn_info->e_tag_fwd_en = FALSE; 8102 ret = ixgbe_e_tag_forwarding_en_dis(dev, 0); 8103 break; 8104 default: 8105 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8106 ret = -EINVAL; 8107 break; 8108 } 8109 8110 return ret; 8111 } 8112 8113 static int 8114 ixgbe_e_tag_insertion_en_dis(struct rte_eth_dev *dev, 8115 struct rte_eth_l2_tunnel_conf *l2_tunnel, 8116 bool en) 8117 { 8118 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 8119 int ret = 0; 8120 uint32_t vmtir, vmvir; 8121 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8122 8123 if (l2_tunnel->vf_id >= pci_dev->max_vfs) { 8124 PMD_DRV_LOG(ERR, 8125 "VF id %u should be less than %u", 8126 l2_tunnel->vf_id, 8127 pci_dev->max_vfs); 8128 return -EINVAL; 8129 } 8130 8131 if (hw->mac.type != ixgbe_mac_X550 && 8132 hw->mac.type != ixgbe_mac_X550EM_x && 8133 hw->mac.type != ixgbe_mac_X550EM_a) { 8134 return -ENOTSUP; 8135 } 8136 8137 if (en) 8138 vmtir = l2_tunnel->tunnel_id; 8139 else 8140 vmtir = 0; 8141 8142 IXGBE_WRITE_REG(hw, IXGBE_VMTIR(l2_tunnel->vf_id), vmtir); 8143 8144 vmvir = IXGBE_READ_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id)); 8145 vmvir &= ~IXGBE_VMVIR_TAGA_MASK; 8146 if (en) 8147 vmvir |= IXGBE_VMVIR_TAGA_ETAG_INSERT; 8148 IXGBE_WRITE_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id), vmvir); 8149 8150 return ret; 8151 } 8152 8153 /* Enable l2 tunnel tag insertion */ 8154 static int 8155 ixgbe_dev_l2_tunnel_insertion_enable(struct rte_eth_dev *dev, 8156 struct rte_eth_l2_tunnel_conf *l2_tunnel) 8157 { 8158 int ret = 0; 8159 8160 switch (l2_tunnel->l2_tunnel_type) { 8161 case RTE_L2_TUNNEL_TYPE_E_TAG: 8162 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 1); 8163 break; 8164 default: 8165 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8166 ret = -EINVAL; 8167 break; 8168 } 8169 8170 return ret; 8171 } 8172 8173 /* Disable l2 tunnel tag insertion */ 8174 static int 8175 ixgbe_dev_l2_tunnel_insertion_disable 8176 (struct rte_eth_dev *dev, 8177 struct rte_eth_l2_tunnel_conf *l2_tunnel) 8178 { 8179 int ret = 0; 8180 8181 switch (l2_tunnel->l2_tunnel_type) { 8182 case RTE_L2_TUNNEL_TYPE_E_TAG: 8183 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 0); 8184 break; 8185 default: 8186 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8187 ret = -EINVAL; 8188 break; 8189 } 8190 8191 return ret; 8192 } 8193 8194 static int 8195 ixgbe_e_tag_stripping_en_dis(struct rte_eth_dev *dev, 8196 bool en) 8197 { 8198 int ret = 0; 8199 uint32_t qde; 8200 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8201 8202 if (hw->mac.type != ixgbe_mac_X550 && 8203 hw->mac.type != ixgbe_mac_X550EM_x && 8204 hw->mac.type != ixgbe_mac_X550EM_a) { 8205 return -ENOTSUP; 8206 } 8207 8208 qde = IXGBE_READ_REG(hw, IXGBE_QDE); 8209 if (en) 8210 qde |= IXGBE_QDE_STRIP_TAG; 8211 else 8212 qde &= ~IXGBE_QDE_STRIP_TAG; 8213 qde &= ~IXGBE_QDE_READ; 8214 qde |= IXGBE_QDE_WRITE; 8215 IXGBE_WRITE_REG(hw, IXGBE_QDE, qde); 8216 8217 return ret; 8218 } 8219 8220 /* Enable l2 tunnel tag stripping */ 8221 static int 8222 ixgbe_dev_l2_tunnel_stripping_enable 8223 (struct rte_eth_dev *dev, 8224 enum rte_eth_tunnel_type l2_tunnel_type) 8225 { 8226 int ret = 0; 8227 8228 switch (l2_tunnel_type) { 8229 case RTE_L2_TUNNEL_TYPE_E_TAG: 8230 ret = ixgbe_e_tag_stripping_en_dis(dev, 1); 8231 break; 8232 default: 8233 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8234 ret = -EINVAL; 8235 break; 8236 } 8237 8238 return ret; 8239 } 8240 8241 /* Disable l2 tunnel tag stripping */ 8242 static int 8243 ixgbe_dev_l2_tunnel_stripping_disable 8244 (struct rte_eth_dev *dev, 8245 enum rte_eth_tunnel_type l2_tunnel_type) 8246 { 8247 int ret = 0; 8248 8249 switch (l2_tunnel_type) { 8250 case RTE_L2_TUNNEL_TYPE_E_TAG: 8251 ret = ixgbe_e_tag_stripping_en_dis(dev, 0); 8252 break; 8253 default: 8254 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8255 ret = -EINVAL; 8256 break; 8257 } 8258 8259 return ret; 8260 } 8261 8262 /* Enable/disable l2 tunnel offload functions */ 8263 static int 8264 ixgbe_dev_l2_tunnel_offload_set 8265 (struct rte_eth_dev *dev, 8266 struct rte_eth_l2_tunnel_conf *l2_tunnel, 8267 uint32_t mask, 8268 uint8_t en) 8269 { 8270 int ret = 0; 8271 8272 if (l2_tunnel == NULL) 8273 return -EINVAL; 8274 8275 ret = -EINVAL; 8276 if (mask & ETH_L2_TUNNEL_ENABLE_MASK) { 8277 if (en) 8278 ret = ixgbe_dev_l2_tunnel_enable( 8279 dev, 8280 l2_tunnel->l2_tunnel_type); 8281 else 8282 ret = ixgbe_dev_l2_tunnel_disable( 8283 dev, 8284 l2_tunnel->l2_tunnel_type); 8285 } 8286 8287 if (mask & ETH_L2_TUNNEL_INSERTION_MASK) { 8288 if (en) 8289 ret = ixgbe_dev_l2_tunnel_insertion_enable( 8290 dev, 8291 l2_tunnel); 8292 else 8293 ret = ixgbe_dev_l2_tunnel_insertion_disable( 8294 dev, 8295 l2_tunnel); 8296 } 8297 8298 if (mask & ETH_L2_TUNNEL_STRIPPING_MASK) { 8299 if (en) 8300 ret = ixgbe_dev_l2_tunnel_stripping_enable( 8301 dev, 8302 l2_tunnel->l2_tunnel_type); 8303 else 8304 ret = ixgbe_dev_l2_tunnel_stripping_disable( 8305 dev, 8306 l2_tunnel->l2_tunnel_type); 8307 } 8308 8309 if (mask & ETH_L2_TUNNEL_FORWARDING_MASK) { 8310 if (en) 8311 ret = ixgbe_dev_l2_tunnel_forwarding_enable( 8312 dev, 8313 l2_tunnel->l2_tunnel_type); 8314 else 8315 ret = ixgbe_dev_l2_tunnel_forwarding_disable( 8316 dev, 8317 l2_tunnel->l2_tunnel_type); 8318 } 8319 8320 return ret; 8321 } 8322 8323 static int 8324 ixgbe_update_vxlan_port(struct ixgbe_hw *hw, 8325 uint16_t port) 8326 { 8327 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, port); 8328 IXGBE_WRITE_FLUSH(hw); 8329 8330 return 0; 8331 } 8332 8333 /* There's only one register for VxLAN UDP port. 8334 * So, we cannot add several ports. Will update it. 8335 */ 8336 static int 8337 ixgbe_add_vxlan_port(struct ixgbe_hw *hw, 8338 uint16_t port) 8339 { 8340 if (port == 0) { 8341 PMD_DRV_LOG(ERR, "Add VxLAN port 0 is not allowed."); 8342 return -EINVAL; 8343 } 8344 8345 return ixgbe_update_vxlan_port(hw, port); 8346 } 8347 8348 /* We cannot delete the VxLAN port. For there's a register for VxLAN 8349 * UDP port, it must have a value. 8350 * So, will reset it to the original value 0. 8351 */ 8352 static int 8353 ixgbe_del_vxlan_port(struct ixgbe_hw *hw, 8354 uint16_t port) 8355 { 8356 uint16_t cur_port; 8357 8358 cur_port = (uint16_t)IXGBE_READ_REG(hw, IXGBE_VXLANCTRL); 8359 8360 if (cur_port != port) { 8361 PMD_DRV_LOG(ERR, "Port %u does not exist.", port); 8362 return -EINVAL; 8363 } 8364 8365 return ixgbe_update_vxlan_port(hw, 0); 8366 } 8367 8368 /* Add UDP tunneling port */ 8369 static int 8370 ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, 8371 struct rte_eth_udp_tunnel *udp_tunnel) 8372 { 8373 int ret = 0; 8374 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8375 8376 if (hw->mac.type != ixgbe_mac_X550 && 8377 hw->mac.type != ixgbe_mac_X550EM_x && 8378 hw->mac.type != ixgbe_mac_X550EM_a) { 8379 return -ENOTSUP; 8380 } 8381 8382 if (udp_tunnel == NULL) 8383 return -EINVAL; 8384 8385 switch (udp_tunnel->prot_type) { 8386 case RTE_TUNNEL_TYPE_VXLAN: 8387 ret = ixgbe_add_vxlan_port(hw, udp_tunnel->udp_port); 8388 break; 8389 8390 case RTE_TUNNEL_TYPE_GENEVE: 8391 case RTE_TUNNEL_TYPE_TEREDO: 8392 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 8393 ret = -EINVAL; 8394 break; 8395 8396 default: 8397 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8398 ret = -EINVAL; 8399 break; 8400 } 8401 8402 return ret; 8403 } 8404 8405 /* Remove UDP tunneling port */ 8406 static int 8407 ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, 8408 struct rte_eth_udp_tunnel *udp_tunnel) 8409 { 8410 int ret = 0; 8411 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8412 8413 if (hw->mac.type != ixgbe_mac_X550 && 8414 hw->mac.type != ixgbe_mac_X550EM_x && 8415 hw->mac.type != ixgbe_mac_X550EM_a) { 8416 return -ENOTSUP; 8417 } 8418 8419 if (udp_tunnel == NULL) 8420 return -EINVAL; 8421 8422 switch (udp_tunnel->prot_type) { 8423 case RTE_TUNNEL_TYPE_VXLAN: 8424 ret = ixgbe_del_vxlan_port(hw, udp_tunnel->udp_port); 8425 break; 8426 case RTE_TUNNEL_TYPE_GENEVE: 8427 case RTE_TUNNEL_TYPE_TEREDO: 8428 PMD_DRV_LOG(ERR, "Tunnel type is not supported now."); 8429 ret = -EINVAL; 8430 break; 8431 default: 8432 PMD_DRV_LOG(ERR, "Invalid tunnel type"); 8433 ret = -EINVAL; 8434 break; 8435 } 8436 8437 return ret; 8438 } 8439 8440 static int 8441 ixgbevf_dev_promiscuous_enable(struct rte_eth_dev *dev) 8442 { 8443 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8444 int ret; 8445 8446 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_PROMISC)) { 8447 case IXGBE_SUCCESS: 8448 ret = 0; 8449 break; 8450 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 8451 ret = -ENOTSUP; 8452 break; 8453 default: 8454 ret = -EAGAIN; 8455 break; 8456 } 8457 8458 return ret; 8459 } 8460 8461 static int 8462 ixgbevf_dev_promiscuous_disable(struct rte_eth_dev *dev) 8463 { 8464 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8465 int ret; 8466 8467 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_NONE)) { 8468 case IXGBE_SUCCESS: 8469 ret = 0; 8470 break; 8471 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 8472 ret = -ENOTSUP; 8473 break; 8474 default: 8475 ret = -EAGAIN; 8476 break; 8477 } 8478 8479 return ret; 8480 } 8481 8482 static int 8483 ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) 8484 { 8485 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8486 int ret; 8487 int mode = IXGBEVF_XCAST_MODE_ALLMULTI; 8488 8489 switch (hw->mac.ops.update_xcast_mode(hw, mode)) { 8490 case IXGBE_SUCCESS: 8491 ret = 0; 8492 break; 8493 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 8494 ret = -ENOTSUP; 8495 break; 8496 default: 8497 ret = -EAGAIN; 8498 break; 8499 } 8500 8501 return ret; 8502 } 8503 8504 static int 8505 ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) 8506 { 8507 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8508 int ret; 8509 8510 switch (hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI)) { 8511 case IXGBE_SUCCESS: 8512 ret = 0; 8513 break; 8514 case IXGBE_ERR_FEATURE_NOT_SUPPORTED: 8515 ret = -ENOTSUP; 8516 break; 8517 default: 8518 ret = -EAGAIN; 8519 break; 8520 } 8521 8522 return ret; 8523 } 8524 8525 static void ixgbevf_mbx_process(struct rte_eth_dev *dev) 8526 { 8527 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8528 u32 in_msg = 0; 8529 8530 /* peek the message first */ 8531 in_msg = IXGBE_READ_REG(hw, IXGBE_VFMBMEM); 8532 8533 /* PF reset VF event */ 8534 if (in_msg == IXGBE_PF_CONTROL_MSG) { 8535 /* dummy mbx read to ack pf */ 8536 if (ixgbe_read_mbx(hw, &in_msg, 1, 0)) 8537 return; 8538 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 8539 NULL); 8540 } 8541 } 8542 8543 static int 8544 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev) 8545 { 8546 uint32_t eicr; 8547 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8548 struct ixgbe_interrupt *intr = 8549 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8550 ixgbevf_intr_disable(dev); 8551 8552 /* read-on-clear nic registers here */ 8553 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR); 8554 intr->flags = 0; 8555 8556 /* only one misc vector supported - mailbox */ 8557 eicr &= IXGBE_VTEICR_MASK; 8558 if (eicr == IXGBE_MISC_VEC_ID) 8559 intr->flags |= IXGBE_FLAG_MAILBOX; 8560 8561 return 0; 8562 } 8563 8564 static int 8565 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev) 8566 { 8567 struct ixgbe_interrupt *intr = 8568 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); 8569 8570 if (intr->flags & IXGBE_FLAG_MAILBOX) { 8571 ixgbevf_mbx_process(dev); 8572 intr->flags &= ~IXGBE_FLAG_MAILBOX; 8573 } 8574 8575 ixgbevf_intr_enable(dev); 8576 8577 return 0; 8578 } 8579 8580 static void 8581 ixgbevf_dev_interrupt_handler(void *param) 8582 { 8583 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 8584 8585 ixgbevf_dev_interrupt_get_status(dev); 8586 ixgbevf_dev_interrupt_action(dev); 8587 } 8588 8589 /** 8590 * ixgbe_disable_sec_tx_path_generic - Stops the transmit data path 8591 * @hw: pointer to hardware structure 8592 * 8593 * Stops the transmit data path and waits for the HW to internally empty 8594 * the Tx security block 8595 **/ 8596 int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw) 8597 { 8598 #define IXGBE_MAX_SECTX_POLL 40 8599 8600 int i; 8601 int sectxreg; 8602 8603 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8604 sectxreg |= IXGBE_SECTXCTRL_TX_DIS; 8605 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8606 for (i = 0; i < IXGBE_MAX_SECTX_POLL; i++) { 8607 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT); 8608 if (sectxreg & IXGBE_SECTXSTAT_SECTX_RDY) 8609 break; 8610 /* Use interrupt-safe sleep just in case */ 8611 usec_delay(1000); 8612 } 8613 8614 /* For informational purposes only */ 8615 if (i >= IXGBE_MAX_SECTX_POLL) 8616 PMD_DRV_LOG(DEBUG, "Tx unit being enabled before security " 8617 "path fully disabled. Continuing with init."); 8618 8619 return IXGBE_SUCCESS; 8620 } 8621 8622 /** 8623 * ixgbe_enable_sec_tx_path_generic - Enables the transmit data path 8624 * @hw: pointer to hardware structure 8625 * 8626 * Enables the transmit data path. 8627 **/ 8628 int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw) 8629 { 8630 uint32_t sectxreg; 8631 8632 sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); 8633 sectxreg &= ~IXGBE_SECTXCTRL_TX_DIS; 8634 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); 8635 IXGBE_WRITE_FLUSH(hw); 8636 8637 return IXGBE_SUCCESS; 8638 } 8639 8640 /* restore n-tuple filter */ 8641 static inline void 8642 ixgbe_ntuple_filter_restore(struct rte_eth_dev *dev) 8643 { 8644 struct ixgbe_filter_info *filter_info = 8645 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8646 struct ixgbe_5tuple_filter *node; 8647 8648 TAILQ_FOREACH(node, &filter_info->fivetuple_list, entries) { 8649 ixgbe_inject_5tuple_filter(dev, node); 8650 } 8651 } 8652 8653 /* restore ethernet type filter */ 8654 static inline void 8655 ixgbe_ethertype_filter_restore(struct rte_eth_dev *dev) 8656 { 8657 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8658 struct ixgbe_filter_info *filter_info = 8659 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8660 int i; 8661 8662 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8663 if (filter_info->ethertype_mask & (1 << i)) { 8664 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 8665 filter_info->ethertype_filters[i].etqf); 8666 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 8667 filter_info->ethertype_filters[i].etqs); 8668 IXGBE_WRITE_FLUSH(hw); 8669 } 8670 } 8671 } 8672 8673 /* restore SYN filter */ 8674 static inline void 8675 ixgbe_syn_filter_restore(struct rte_eth_dev *dev) 8676 { 8677 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8678 struct ixgbe_filter_info *filter_info = 8679 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8680 uint32_t synqf; 8681 8682 synqf = filter_info->syn_info; 8683 8684 if (synqf & IXGBE_SYN_FILTER_ENABLE) { 8685 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); 8686 IXGBE_WRITE_FLUSH(hw); 8687 } 8688 } 8689 8690 /* restore L2 tunnel filter */ 8691 static inline void 8692 ixgbe_l2_tn_filter_restore(struct rte_eth_dev *dev) 8693 { 8694 struct ixgbe_l2_tn_info *l2_tn_info = 8695 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8696 struct ixgbe_l2_tn_filter *node; 8697 struct rte_eth_l2_tunnel_conf l2_tn_conf; 8698 8699 TAILQ_FOREACH(node, &l2_tn_info->l2_tn_list, entries) { 8700 l2_tn_conf.l2_tunnel_type = node->key.l2_tn_type; 8701 l2_tn_conf.tunnel_id = node->key.tn_id; 8702 l2_tn_conf.pool = node->pool; 8703 (void)ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_conf, TRUE); 8704 } 8705 } 8706 8707 /* restore rss filter */ 8708 static inline void 8709 ixgbe_rss_filter_restore(struct rte_eth_dev *dev) 8710 { 8711 struct ixgbe_filter_info *filter_info = 8712 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8713 8714 if (filter_info->rss_info.conf.queue_num) 8715 ixgbe_config_rss_filter(dev, 8716 &filter_info->rss_info, TRUE); 8717 } 8718 8719 static int 8720 ixgbe_filter_restore(struct rte_eth_dev *dev) 8721 { 8722 ixgbe_ntuple_filter_restore(dev); 8723 ixgbe_ethertype_filter_restore(dev); 8724 ixgbe_syn_filter_restore(dev); 8725 ixgbe_fdir_filter_restore(dev); 8726 ixgbe_l2_tn_filter_restore(dev); 8727 ixgbe_rss_filter_restore(dev); 8728 8729 return 0; 8730 } 8731 8732 static void 8733 ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev) 8734 { 8735 struct ixgbe_l2_tn_info *l2_tn_info = 8736 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8737 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8738 8739 if (l2_tn_info->e_tag_en) 8740 (void)ixgbe_e_tag_enable(hw); 8741 8742 if (l2_tn_info->e_tag_fwd_en) 8743 (void)ixgbe_e_tag_forwarding_en_dis(dev, 1); 8744 8745 (void)ixgbe_update_e_tag_eth_type(hw, l2_tn_info->e_tag_ether_type); 8746 } 8747 8748 /* remove all the n-tuple filters */ 8749 void 8750 ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev) 8751 { 8752 struct ixgbe_filter_info *filter_info = 8753 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8754 struct ixgbe_5tuple_filter *p_5tuple; 8755 8756 while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) 8757 ixgbe_remove_5tuple_filter(dev, p_5tuple); 8758 } 8759 8760 /* remove all the ether type filters */ 8761 void 8762 ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev) 8763 { 8764 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8765 struct ixgbe_filter_info *filter_info = 8766 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8767 int i; 8768 8769 for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { 8770 if (filter_info->ethertype_mask & (1 << i) && 8771 !filter_info->ethertype_filters[i].conf) { 8772 (void)ixgbe_ethertype_filter_remove(filter_info, 8773 (uint8_t)i); 8774 IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 0); 8775 IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 0); 8776 IXGBE_WRITE_FLUSH(hw); 8777 } 8778 } 8779 } 8780 8781 /* remove the SYN filter */ 8782 void 8783 ixgbe_clear_syn_filter(struct rte_eth_dev *dev) 8784 { 8785 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 8786 struct ixgbe_filter_info *filter_info = 8787 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 8788 8789 if (filter_info->syn_info & IXGBE_SYN_FILTER_ENABLE) { 8790 filter_info->syn_info = 0; 8791 8792 IXGBE_WRITE_REG(hw, IXGBE_SYNQF, 0); 8793 IXGBE_WRITE_FLUSH(hw); 8794 } 8795 } 8796 8797 /* remove all the L2 tunnel filters */ 8798 int 8799 ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev) 8800 { 8801 struct ixgbe_l2_tn_info *l2_tn_info = 8802 IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); 8803 struct ixgbe_l2_tn_filter *l2_tn_filter; 8804 struct rte_eth_l2_tunnel_conf l2_tn_conf; 8805 int ret = 0; 8806 8807 while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { 8808 l2_tn_conf.l2_tunnel_type = l2_tn_filter->key.l2_tn_type; 8809 l2_tn_conf.tunnel_id = l2_tn_filter->key.tn_id; 8810 l2_tn_conf.pool = l2_tn_filter->pool; 8811 ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_conf); 8812 if (ret < 0) 8813 return ret; 8814 } 8815 8816 return 0; 8817 } 8818 8819 RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); 8820 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe, pci_id_ixgbe_map); 8821 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe, "* igb_uio | uio_pci_generic | vfio-pci"); 8822 RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd); 8823 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map); 8824 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci"); 8825 RTE_PMD_REGISTER_PARAM_STRING(net_ixgbe_vf, 8826 IXGBEVF_DEVARG_PFLINK_FULLCHK "=<0|1>"); 8827 8828 RTE_INIT(ixgbe_init_log) 8829 { 8830 ixgbe_logtype_init = rte_log_register("pmd.net.ixgbe.init"); 8831 if (ixgbe_logtype_init >= 0) 8832 rte_log_set_level(ixgbe_logtype_init, RTE_LOG_NOTICE); 8833 ixgbe_logtype_driver = rte_log_register("pmd.net.ixgbe.driver"); 8834 if (ixgbe_logtype_driver >= 0) 8835 rte_log_set_level(ixgbe_logtype_driver, RTE_LOG_NOTICE); 8836 #ifdef RTE_LIBRTE_IXGBE_DEBUG_RX 8837 ixgbe_logtype_rx = rte_log_register("pmd.net.ixgbe.rx"); 8838 if (ixgbe_logtype_rx >= 0) 8839 rte_log_set_level(ixgbe_logtype_rx, RTE_LOG_DEBUG); 8840 #endif 8841 8842 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX 8843 ixgbe_logtype_tx = rte_log_register("pmd.net.ixgbe.tx"); 8844 if (ixgbe_logtype_tx >= 0) 8845 rte_log_set_level(ixgbe_logtype_tx, RTE_LOG_DEBUG); 8846 #endif 8847 8848 #ifdef RTE_LIBRTE_IXGBE_DEBUG_TX_FREE 8849 ixgbe_logtype_tx_free = rte_log_register("pmd.net.ixgbe.tx_free"); 8850 if (ixgbe_logtype_tx_free >= 0) 8851 rte_log_set_level(ixgbe_logtype_tx_free, RTE_LOG_DEBUG); 8852 #endif 8853 } 8854