xref: /f-stack/dpdk/drivers/net/i40e/i40e_ethdev.c (revision 851ac5c0) 
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2017 Intel Corporation
3  */
4 
5 #include <stdio.h>
6 #include <errno.h>
7 #include <stdint.h>
8 #include <string.h>
9 #include <unistd.h>
10 #include <stdarg.h>
11 #include <inttypes.h>
12 #include <assert.h>
13 
14 #include <rte_common.h>
15 #include <rte_eal.h>
16 #include <rte_string_fns.h>
17 #include <rte_pci.h>
18 #include <rte_bus_pci.h>
19 #include <rte_ether.h>
20 #include <rte_ethdev_driver.h>
21 #include <rte_ethdev_pci.h>
22 #include <rte_memzone.h>
23 #include <rte_malloc.h>
24 #include <rte_memcpy.h>
25 #include <rte_alarm.h>
26 #include <rte_dev.h>
27 #include <rte_eth_ctrl.h>
28 #include <rte_tailq.h>
29 #include <rte_hash_crc.h>
30 
31 #include "i40e_logs.h"
32 #include "base/i40e_prototype.h"
33 #include "base/i40e_adminq_cmd.h"
34 #include "base/i40e_type.h"
35 #include "base/i40e_register.h"
36 #include "base/i40e_dcb.h"
37 #include "i40e_ethdev.h"
38 #include "i40e_rxtx.h"
39 #include "i40e_pf.h"
40 #include "i40e_regs.h"
41 #include "rte_pmd_i40e.h"
42 
43 #define ETH_I40E_FLOATING_VEB_ARG	"enable_floating_veb"
44 #define ETH_I40E_FLOATING_VEB_LIST_ARG	"floating_veb_list"
45 #define ETH_I40E_SUPPORT_MULTI_DRIVER	"support-multi-driver"
46 #define ETH_I40E_QUEUE_NUM_PER_VF_ARG	"queue-num-per-vf"
47 #define ETH_I40E_USE_LATEST_VEC	"use-latest-supported-vec"
48 
49 #define I40E_CLEAR_PXE_WAIT_MS     200
50 
51 /* Maximun number of capability elements */
52 #define I40E_MAX_CAP_ELE_NUM       128
53 
54 /* Wait count and interval */
55 #define I40E_CHK_Q_ENA_COUNT       1000
56 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
57 
58 /* Maximun number of VSI */
59 #define I40E_MAX_NUM_VSIS          (384UL)
60 
61 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
62 
63 /* Flow control default timer */
64 #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU
65 
66 /* Flow control enable fwd bit */
67 #define I40E_PRTMAC_FWD_CTRL   0x00000001
68 
69 /* Receive Packet Buffer size */
70 #define I40E_RXPBSIZE (968 * 1024)
71 
72 /* Kilobytes shift */
73 #define I40E_KILOSHIFT 10
74 
75 /* Flow control default high water */
76 #define I40E_DEFAULT_HIGH_WATER (0xF2000 >> I40E_KILOSHIFT)
77 
78 /* Flow control default low water */
79 #define I40E_DEFAULT_LOW_WATER  (0xF2000 >> I40E_KILOSHIFT)
80 
81 /* Receive Average Packet Size in Byte*/
82 #define I40E_PACKET_AVERAGE_SIZE 128
83 
84 /* Mask of PF interrupt causes */
85 #define I40E_PFINT_ICR0_ENA_MASK ( \
86 		I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
87 		I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
88 		I40E_PFINT_ICR0_ENA_GRST_MASK | \
89 		I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
90 		I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
91 		I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
92 		I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
93 		I40E_PFINT_ICR0_ENA_VFLR_MASK | \
94 		I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
95 
96 #define I40E_FLOW_TYPES ( \
97 	(1UL << RTE_ETH_FLOW_FRAG_IPV4) | \
98 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
99 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
100 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
101 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
102 	(1UL << RTE_ETH_FLOW_FRAG_IPV6) | \
103 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
104 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
105 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
106 	(1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
107 	(1UL << RTE_ETH_FLOW_L2_PAYLOAD))
108 
109 /* Additional timesync values. */
110 #define I40E_PTP_40GB_INCVAL     0x0199999999ULL
111 #define I40E_PTP_10GB_INCVAL     0x0333333333ULL
112 #define I40E_PTP_1GB_INCVAL      0x2000000000ULL
113 #define I40E_PRTTSYN_TSYNENA     0x80000000
114 #define I40E_PRTTSYN_TSYNTYPE    0x0e000000
115 #define I40E_CYCLECOUNTER_MASK   0xffffffffffffffffULL
116 
117 /**
118  * Below are values for writing un-exposed registers suggested
119  * by silicon experts
120  */
121 /* Destination MAC address */
122 #define I40E_REG_INSET_L2_DMAC                   0xE000000000000000ULL
123 /* Source MAC address */
124 #define I40E_REG_INSET_L2_SMAC                   0x1C00000000000000ULL
125 /* Outer (S-Tag) VLAN tag in the outer L2 header */
126 #define I40E_REG_INSET_L2_OUTER_VLAN             0x0000000004000000ULL
127 /* Inner (C-Tag) or single VLAN tag in the outer L2 header */
128 #define I40E_REG_INSET_L2_INNER_VLAN             0x0080000000000000ULL
129 /* Single VLAN tag in the inner L2 header */
130 #define I40E_REG_INSET_TUNNEL_VLAN               0x0100000000000000ULL
131 /* Source IPv4 address */
132 #define I40E_REG_INSET_L3_SRC_IP4                0x0001800000000000ULL
133 /* Destination IPv4 address */
134 #define I40E_REG_INSET_L3_DST_IP4                0x0000001800000000ULL
135 /* Source IPv4 address for X722 */
136 #define I40E_X722_REG_INSET_L3_SRC_IP4           0x0006000000000000ULL
137 /* Destination IPv4 address for X722 */
138 #define I40E_X722_REG_INSET_L3_DST_IP4           0x0000060000000000ULL
139 /* IPv4 Protocol for X722 */
140 #define I40E_X722_REG_INSET_L3_IP4_PROTO         0x0010000000000000ULL
141 /* IPv4 Time to Live for X722 */
142 #define I40E_X722_REG_INSET_L3_IP4_TTL           0x0010000000000000ULL
143 /* IPv4 Type of Service (TOS) */
144 #define I40E_REG_INSET_L3_IP4_TOS                0x0040000000000000ULL
145 /* IPv4 Protocol */
146 #define I40E_REG_INSET_L3_IP4_PROTO              0x0004000000000000ULL
147 /* IPv4 Time to Live */
148 #define I40E_REG_INSET_L3_IP4_TTL                0x0004000000000000ULL
149 /* Source IPv6 address */
150 #define I40E_REG_INSET_L3_SRC_IP6                0x0007F80000000000ULL
151 /* Destination IPv6 address */
152 #define I40E_REG_INSET_L3_DST_IP6                0x000007F800000000ULL
153 /* IPv6 Traffic Class (TC) */
154 #define I40E_REG_INSET_L3_IP6_TC                 0x0040000000000000ULL
155 /* IPv6 Next Header */
156 #define I40E_REG_INSET_L3_IP6_NEXT_HDR           0x0008000000000000ULL
157 /* IPv6 Hop Limit */
158 #define I40E_REG_INSET_L3_IP6_HOP_LIMIT          0x0008000000000000ULL
159 /* Source L4 port */
160 #define I40E_REG_INSET_L4_SRC_PORT               0x0000000400000000ULL
161 /* Destination L4 port */
162 #define I40E_REG_INSET_L4_DST_PORT               0x0000000200000000ULL
163 /* SCTP verification tag */
164 #define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG  0x0000000180000000ULL
165 /* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/
166 #define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC   0x0000000001C00000ULL
167 /* Source port of tunneling UDP */
168 #define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT    0x0000000000200000ULL
169 /* Destination port of tunneling UDP */
170 #define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT    0x0000000000100000ULL
171 /* UDP Tunneling ID, NVGRE/GRE key */
172 #define I40E_REG_INSET_TUNNEL_ID                 0x00000000000C0000ULL
173 /* Last ether type */
174 #define I40E_REG_INSET_LAST_ETHER_TYPE           0x0000000000004000ULL
175 /* Tunneling outer destination IPv4 address */
176 #define I40E_REG_INSET_TUNNEL_L3_DST_IP4         0x00000000000000C0ULL
177 /* Tunneling outer destination IPv6 address */
178 #define I40E_REG_INSET_TUNNEL_L3_DST_IP6         0x0000000000003FC0ULL
179 /* 1st word of flex payload */
180 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD1        0x0000000000002000ULL
181 /* 2nd word of flex payload */
182 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD2        0x0000000000001000ULL
183 /* 3rd word of flex payload */
184 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD3        0x0000000000000800ULL
185 /* 4th word of flex payload */
186 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD4        0x0000000000000400ULL
187 /* 5th word of flex payload */
188 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD5        0x0000000000000200ULL
189 /* 6th word of flex payload */
190 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD6        0x0000000000000100ULL
191 /* 7th word of flex payload */
192 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD7        0x0000000000000080ULL
193 /* 8th word of flex payload */
194 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD8        0x0000000000000040ULL
195 /* all 8 words flex payload */
196 #define I40E_REG_INSET_FLEX_PAYLOAD_WORDS        0x0000000000003FC0ULL
197 #define I40E_REG_INSET_MASK_DEFAULT              0x0000000000000000ULL
198 
199 #define I40E_TRANSLATE_INSET 0
200 #define I40E_TRANSLATE_REG   1
201 
202 #define I40E_INSET_IPV4_TOS_MASK        0x0009FF00UL
203 #define I40E_INSET_IPv4_TTL_MASK        0x000D00FFUL
204 #define I40E_INSET_IPV4_PROTO_MASK      0x000DFF00UL
205 #define I40E_INSET_IPV6_TC_MASK         0x0009F00FUL
206 #define I40E_INSET_IPV6_HOP_LIMIT_MASK  0x000CFF00UL
207 #define I40E_INSET_IPV6_NEXT_HDR_MASK   0x000C00FFUL
208 
209 /* PCI offset for querying capability */
210 #define PCI_DEV_CAP_REG            0xA4
211 /* PCI offset for enabling/disabling Extended Tag */
212 #define PCI_DEV_CTRL_REG           0xA8
213 /* Bit mask of Extended Tag capability */
214 #define PCI_DEV_CAP_EXT_TAG_MASK   0x20
215 /* Bit shift of Extended Tag enable/disable */
216 #define PCI_DEV_CTRL_EXT_TAG_SHIFT 8
217 /* Bit mask of Extended Tag enable/disable */
218 #define PCI_DEV_CTRL_EXT_TAG_MASK  (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT)
219 
220 static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev, void *init_params);
221 static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev);
222 static int i40e_dev_configure(struct rte_eth_dev *dev);
223 static int i40e_dev_start(struct rte_eth_dev *dev);
224 static void i40e_dev_stop(struct rte_eth_dev *dev);
225 static void i40e_dev_close(struct rte_eth_dev *dev);
226 static int  i40e_dev_reset(struct rte_eth_dev *dev);
227 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
228 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
229 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
230 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
231 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
232 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
233 static int i40e_dev_stats_get(struct rte_eth_dev *dev,
234 			       struct rte_eth_stats *stats);
235 static int i40e_dev_xstats_get(struct rte_eth_dev *dev,
236 			       struct rte_eth_xstat *xstats, unsigned n);
237 static int i40e_dev_xstats_get_names(struct rte_eth_dev *dev,
238 				     struct rte_eth_xstat_name *xstats_names,
239 				     unsigned limit);
240 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
241 static int i40e_fw_version_get(struct rte_eth_dev *dev,
242 				char *fw_version, size_t fw_size);
243 static void i40e_dev_info_get(struct rte_eth_dev *dev,
244 			      struct rte_eth_dev_info *dev_info);
245 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
246 				uint16_t vlan_id,
247 				int on);
248 static int i40e_vlan_tpid_set(struct rte_eth_dev *dev,
249 			      enum rte_vlan_type vlan_type,
250 			      uint16_t tpid);
251 static int i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
252 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
253 				      uint16_t queue,
254 				      int on);
255 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
256 static int i40e_dev_led_on(struct rte_eth_dev *dev);
257 static int i40e_dev_led_off(struct rte_eth_dev *dev);
258 static int i40e_flow_ctrl_get(struct rte_eth_dev *dev,
259 			      struct rte_eth_fc_conf *fc_conf);
260 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
261 			      struct rte_eth_fc_conf *fc_conf);
262 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
263 				       struct rte_eth_pfc_conf *pfc_conf);
264 static int i40e_macaddr_add(struct rte_eth_dev *dev,
265 			    struct ether_addr *mac_addr,
266 			    uint32_t index,
267 			    uint32_t pool);
268 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
269 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
270 				    struct rte_eth_rss_reta_entry64 *reta_conf,
271 				    uint16_t reta_size);
272 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
273 				   struct rte_eth_rss_reta_entry64 *reta_conf,
274 				   uint16_t reta_size);
275 
276 static int i40e_get_cap(struct i40e_hw *hw);
277 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
278 static int i40e_pf_setup(struct i40e_pf *pf);
279 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
280 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
281 static int i40e_dcb_setup(struct rte_eth_dev *dev);
282 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
283 		bool offset_loaded, uint64_t *offset, uint64_t *stat);
284 static void i40e_stat_update_48(struct i40e_hw *hw,
285 			       uint32_t hireg,
286 			       uint32_t loreg,
287 			       bool offset_loaded,
288 			       uint64_t *offset,
289 			       uint64_t *stat);
290 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
291 static void i40e_dev_interrupt_handler(void *param);
292 static void i40e_dev_alarm_handler(void *param);
293 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
294 				uint32_t base, uint32_t num);
295 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
296 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
297 			uint32_t base);
298 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
299 			uint16_t num);
300 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
301 static int i40e_veb_release(struct i40e_veb *veb);
302 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
303 						struct i40e_vsi *vsi);
304 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
305 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
306 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
307 					     struct i40e_macvlan_filter *mv_f,
308 					     int num,
309 					     uint16_t vlan);
310 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
311 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
312 				    struct rte_eth_rss_conf *rss_conf);
313 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
314 				      struct rte_eth_rss_conf *rss_conf);
315 static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
316 					struct rte_eth_udp_tunnel *udp_tunnel);
317 static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
318 					struct rte_eth_udp_tunnel *udp_tunnel);
319 static void i40e_filter_input_set_init(struct i40e_pf *pf);
320 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
321 				enum rte_filter_op filter_op,
322 				void *arg);
323 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
324 				enum rte_filter_type filter_type,
325 				enum rte_filter_op filter_op,
326 				void *arg);
327 static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
328 				  struct rte_eth_dcb_info *dcb_info);
329 static int i40e_dev_sync_phy_type(struct i40e_hw *hw);
330 static void i40e_configure_registers(struct i40e_hw *hw);
331 static void i40e_hw_init(struct rte_eth_dev *dev);
332 static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi);
333 static enum i40e_status_code i40e_aq_del_mirror_rule(struct i40e_hw *hw,
334 						     uint16_t seid,
335 						     uint16_t rule_type,
336 						     uint16_t *entries,
337 						     uint16_t count,
338 						     uint16_t rule_id);
339 static int i40e_mirror_rule_set(struct rte_eth_dev *dev,
340 			struct rte_eth_mirror_conf *mirror_conf,
341 			uint8_t sw_id, uint8_t on);
342 static int i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id);
343 
344 static int i40e_timesync_enable(struct rte_eth_dev *dev);
345 static int i40e_timesync_disable(struct rte_eth_dev *dev);
346 static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
347 					   struct timespec *timestamp,
348 					   uint32_t flags);
349 static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
350 					   struct timespec *timestamp);
351 static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw);
352 
353 static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
354 
355 static int i40e_timesync_read_time(struct rte_eth_dev *dev,
356 				   struct timespec *timestamp);
357 static int i40e_timesync_write_time(struct rte_eth_dev *dev,
358 				    const struct timespec *timestamp);
359 
360 static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
361 					 uint16_t queue_id);
362 static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
363 					  uint16_t queue_id);
364 
365 static int i40e_get_regs(struct rte_eth_dev *dev,
366 			 struct rte_dev_reg_info *regs);
367 
368 static int i40e_get_eeprom_length(struct rte_eth_dev *dev);
369 
370 static int i40e_get_eeprom(struct rte_eth_dev *dev,
371 			   struct rte_dev_eeprom_info *eeprom);
372 
373 static int i40e_get_module_info(struct rte_eth_dev *dev,
374 				struct rte_eth_dev_module_info *modinfo);
375 static int i40e_get_module_eeprom(struct rte_eth_dev *dev,
376 				  struct rte_dev_eeprom_info *info);
377 
378 static int i40e_set_default_mac_addr(struct rte_eth_dev *dev,
379 				      struct ether_addr *mac_addr);
380 
381 static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
382 
383 static int i40e_ethertype_filter_convert(
384 	const struct rte_eth_ethertype_filter *input,
385 	struct i40e_ethertype_filter *filter);
386 static int i40e_sw_ethertype_filter_insert(struct i40e_pf *pf,
387 				   struct i40e_ethertype_filter *filter);
388 
389 static int i40e_tunnel_filter_convert(
390 	struct i40e_aqc_cloud_filters_element_bb *cld_filter,
391 	struct i40e_tunnel_filter *tunnel_filter);
392 static int i40e_sw_tunnel_filter_insert(struct i40e_pf *pf,
393 				struct i40e_tunnel_filter *tunnel_filter);
394 static int i40e_cloud_filter_qinq_create(struct i40e_pf *pf);
395 
396 static void i40e_ethertype_filter_restore(struct i40e_pf *pf);
397 static void i40e_tunnel_filter_restore(struct i40e_pf *pf);
398 static void i40e_filter_restore(struct i40e_pf *pf);
399 static void i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev);
400 
401 int i40e_logtype_init;
402 int i40e_logtype_driver;
403 
404 static const char *const valid_keys[] = {
405 	ETH_I40E_FLOATING_VEB_ARG,
406 	ETH_I40E_FLOATING_VEB_LIST_ARG,
407 	ETH_I40E_SUPPORT_MULTI_DRIVER,
408 	ETH_I40E_QUEUE_NUM_PER_VF_ARG,
409 	ETH_I40E_USE_LATEST_VEC,
410 	NULL};
411 
412 static const struct rte_pci_id pci_id_i40e_map[] = {
413 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) },
414 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) },
415 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B) },
416 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C) },
417 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A) },
418 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B) },
419 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C) },
420 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T) },
421 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2) },
422 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A) },
423 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4) },
424 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B) },
425 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28) },
426 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0) },
427 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722) },
428 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722) },
429 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722) },
430 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722) },
431 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722) },
432 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722) },
433 	{ .vendor_id = 0, /* sentinel */ },
434 };
435 
436 static const struct eth_dev_ops i40e_eth_dev_ops = {
437 	.dev_configure                = i40e_dev_configure,
438 	.dev_start                    = i40e_dev_start,
439 	.dev_stop                     = i40e_dev_stop,
440 	.dev_close                    = i40e_dev_close,
441 	.dev_reset		      = i40e_dev_reset,
442 	.promiscuous_enable           = i40e_dev_promiscuous_enable,
443 	.promiscuous_disable          = i40e_dev_promiscuous_disable,
444 	.allmulticast_enable          = i40e_dev_allmulticast_enable,
445 	.allmulticast_disable         = i40e_dev_allmulticast_disable,
446 	.dev_set_link_up              = i40e_dev_set_link_up,
447 	.dev_set_link_down            = i40e_dev_set_link_down,
448 	.link_update                  = i40e_dev_link_update,
449 	.stats_get                    = i40e_dev_stats_get,
450 	.xstats_get                   = i40e_dev_xstats_get,
451 	.xstats_get_names             = i40e_dev_xstats_get_names,
452 	.stats_reset                  = i40e_dev_stats_reset,
453 	.xstats_reset                 = i40e_dev_stats_reset,
454 	.fw_version_get               = i40e_fw_version_get,
455 	.dev_infos_get                = i40e_dev_info_get,
456 	.dev_supported_ptypes_get     = i40e_dev_supported_ptypes_get,
457 	.vlan_filter_set              = i40e_vlan_filter_set,
458 	.vlan_tpid_set                = i40e_vlan_tpid_set,
459 	.vlan_offload_set             = i40e_vlan_offload_set,
460 	.vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
461 	.vlan_pvid_set                = i40e_vlan_pvid_set,
462 	.rx_queue_start               = i40e_dev_rx_queue_start,
463 	.rx_queue_stop                = i40e_dev_rx_queue_stop,
464 	.tx_queue_start               = i40e_dev_tx_queue_start,
465 	.tx_queue_stop                = i40e_dev_tx_queue_stop,
466 	.rx_queue_setup               = i40e_dev_rx_queue_setup,
467 	.rx_queue_intr_enable         = i40e_dev_rx_queue_intr_enable,
468 	.rx_queue_intr_disable        = i40e_dev_rx_queue_intr_disable,
469 	.rx_queue_release             = i40e_dev_rx_queue_release,
470 	.rx_queue_count               = i40e_dev_rx_queue_count,
471 	.rx_descriptor_done           = i40e_dev_rx_descriptor_done,
472 	.rx_descriptor_status         = i40e_dev_rx_descriptor_status,
473 	.tx_descriptor_status         = i40e_dev_tx_descriptor_status,
474 	.tx_queue_setup               = i40e_dev_tx_queue_setup,
475 	.tx_queue_release             = i40e_dev_tx_queue_release,
476 	.dev_led_on                   = i40e_dev_led_on,
477 	.dev_led_off                  = i40e_dev_led_off,
478 	.flow_ctrl_get                = i40e_flow_ctrl_get,
479 	.flow_ctrl_set                = i40e_flow_ctrl_set,
480 	.priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
481 	.mac_addr_add                 = i40e_macaddr_add,
482 	.mac_addr_remove              = i40e_macaddr_remove,
483 	.reta_update                  = i40e_dev_rss_reta_update,
484 	.reta_query                   = i40e_dev_rss_reta_query,
485 	.rss_hash_update              = i40e_dev_rss_hash_update,
486 	.rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
487 	.udp_tunnel_port_add          = i40e_dev_udp_tunnel_port_add,
488 	.udp_tunnel_port_del          = i40e_dev_udp_tunnel_port_del,
489 	.filter_ctrl                  = i40e_dev_filter_ctrl,
490 	.rxq_info_get                 = i40e_rxq_info_get,
491 	.txq_info_get                 = i40e_txq_info_get,
492 	.mirror_rule_set              = i40e_mirror_rule_set,
493 	.mirror_rule_reset            = i40e_mirror_rule_reset,
494 	.timesync_enable              = i40e_timesync_enable,
495 	.timesync_disable             = i40e_timesync_disable,
496 	.timesync_read_rx_timestamp   = i40e_timesync_read_rx_timestamp,
497 	.timesync_read_tx_timestamp   = i40e_timesync_read_tx_timestamp,
498 	.get_dcb_info                 = i40e_dev_get_dcb_info,
499 	.timesync_adjust_time         = i40e_timesync_adjust_time,
500 	.timesync_read_time           = i40e_timesync_read_time,
501 	.timesync_write_time          = i40e_timesync_write_time,
502 	.get_reg                      = i40e_get_regs,
503 	.get_eeprom_length            = i40e_get_eeprom_length,
504 	.get_eeprom                   = i40e_get_eeprom,
505 	.get_module_info              = i40e_get_module_info,
506 	.get_module_eeprom            = i40e_get_module_eeprom,
507 	.mac_addr_set                 = i40e_set_default_mac_addr,
508 	.mtu_set                      = i40e_dev_mtu_set,
509 	.tm_ops_get                   = i40e_tm_ops_get,
510 };
511 
512 /* store statistics names and its offset in stats structure */
513 struct rte_i40e_xstats_name_off {
514 	char name[RTE_ETH_XSTATS_NAME_SIZE];
515 	unsigned offset;
516 };
517 
518 static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = {
519 	{"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
520 	{"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
521 	{"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
522 	{"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
523 	{"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
524 		rx_unknown_protocol)},
525 	{"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
526 	{"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
527 	{"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
528 	{"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_discards)},
529 };
530 
531 #define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \
532 		sizeof(rte_i40e_stats_strings[0]))
533 
534 static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = {
535 	{"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats,
536 		tx_dropped_link_down)},
537 	{"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)},
538 	{"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats,
539 		illegal_bytes)},
540 	{"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)},
541 	{"mac_local_errors", offsetof(struct i40e_hw_port_stats,
542 		mac_local_faults)},
543 	{"mac_remote_errors", offsetof(struct i40e_hw_port_stats,
544 		mac_remote_faults)},
545 	{"rx_length_errors", offsetof(struct i40e_hw_port_stats,
546 		rx_length_errors)},
547 	{"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)},
548 	{"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)},
549 	{"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)},
550 	{"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)},
551 	{"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)},
552 	{"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
553 		rx_size_127)},
554 	{"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
555 		rx_size_255)},
556 	{"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
557 		rx_size_511)},
558 	{"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
559 		rx_size_1023)},
560 	{"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
561 		rx_size_1522)},
562 	{"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
563 		rx_size_big)},
564 	{"rx_undersized_errors", offsetof(struct i40e_hw_port_stats,
565 		rx_undersize)},
566 	{"rx_oversize_errors", offsetof(struct i40e_hw_port_stats,
567 		rx_oversize)},
568 	{"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats,
569 		mac_short_packet_dropped)},
570 	{"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats,
571 		rx_fragments)},
572 	{"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)},
573 	{"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)},
574 	{"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
575 		tx_size_127)},
576 	{"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
577 		tx_size_255)},
578 	{"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
579 		tx_size_511)},
580 	{"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
581 		tx_size_1023)},
582 	{"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
583 		tx_size_1522)},
584 	{"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
585 		tx_size_big)},
586 	{"rx_flow_director_atr_match_packets",
587 		offsetof(struct i40e_hw_port_stats, fd_atr_match)},
588 	{"rx_flow_director_sb_match_packets",
589 		offsetof(struct i40e_hw_port_stats, fd_sb_match)},
590 	{"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
591 		tx_lpi_status)},
592 	{"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
593 		rx_lpi_status)},
594 	{"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
595 		tx_lpi_count)},
596 	{"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
597 		rx_lpi_count)},
598 };
599 
600 #define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \
601 		sizeof(rte_i40e_hw_port_strings[0]))
602 
603 static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = {
604 	{"xon_packets", offsetof(struct i40e_hw_port_stats,
605 		priority_xon_rx)},
606 	{"xoff_packets", offsetof(struct i40e_hw_port_stats,
607 		priority_xoff_rx)},
608 };
609 
610 #define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \
611 		sizeof(rte_i40e_rxq_prio_strings[0]))
612 
613 static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = {
614 	{"xon_packets", offsetof(struct i40e_hw_port_stats,
615 		priority_xon_tx)},
616 	{"xoff_packets", offsetof(struct i40e_hw_port_stats,
617 		priority_xoff_tx)},
618 	{"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats,
619 		priority_xon_2_xoff)},
620 };
621 
622 #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \
623 		sizeof(rte_i40e_txq_prio_strings[0]))
624 
625 static int
626 eth_i40e_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
627 	struct rte_pci_device *pci_dev)
628 {
629 	char name[RTE_ETH_NAME_MAX_LEN];
630 	struct rte_eth_devargs eth_da = { .nb_representor_ports = 0 };
631 	int i, retval;
632 
633 	if (pci_dev->device.devargs) {
634 		retval = rte_eth_devargs_parse(pci_dev->device.devargs->args,
635 				&eth_da);
636 		if (retval)
637 			return retval;
638 	}
639 
640 	retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
641 		sizeof(struct i40e_adapter),
642 		eth_dev_pci_specific_init, pci_dev,
643 		eth_i40e_dev_init, NULL);
644 
645 	if (retval || eth_da.nb_representor_ports < 1)
646 		return retval;
647 
648 	/* probe VF representor ports */
649 	struct rte_eth_dev *pf_ethdev = rte_eth_dev_allocated(
650 		pci_dev->device.name);
651 
652 	if (pf_ethdev == NULL)
653 		return -ENODEV;
654 
655 	for (i = 0; i < eth_da.nb_representor_ports; i++) {
656 		struct i40e_vf_representor representor = {
657 			.vf_id = eth_da.representor_ports[i],
658 			.switch_domain_id = I40E_DEV_PRIVATE_TO_PF(
659 				pf_ethdev->data->dev_private)->switch_domain_id,
660 			.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(
661 				pf_ethdev->data->dev_private)
662 		};
663 
664 		/* representor port net_bdf_port */
665 		snprintf(name, sizeof(name), "net_%s_representor_%d",
666 			pci_dev->device.name, eth_da.representor_ports[i]);
667 
668 		retval = rte_eth_dev_create(&pci_dev->device, name,
669 			sizeof(struct i40e_vf_representor), NULL, NULL,
670 			i40e_vf_representor_init, &representor);
671 
672 		if (retval)
673 			PMD_DRV_LOG(ERR, "failed to create i40e vf "
674 				"representor %s.", name);
675 	}
676 
677 	return 0;
678 }
679 
680 static int eth_i40e_pci_remove(struct rte_pci_device *pci_dev)
681 {
682 	struct rte_eth_dev *ethdev;
683 
684 	ethdev = rte_eth_dev_allocated(pci_dev->device.name);
685 	if (!ethdev)
686 		return -ENODEV;
687 
688 
689 	if (ethdev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
690 		return rte_eth_dev_destroy(ethdev, i40e_vf_representor_uninit);
691 	else
692 		return rte_eth_dev_destroy(ethdev, eth_i40e_dev_uninit);
693 }
694 
695 static struct rte_pci_driver rte_i40e_pmd = {
696 	.id_table = pci_id_i40e_map,
697 	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
698 		     RTE_PCI_DRV_IOVA_AS_VA,
699 	.probe = eth_i40e_pci_probe,
700 	.remove = eth_i40e_pci_remove,
701 };
702 
703 static inline void
704 i40e_write_global_rx_ctl(struct i40e_hw *hw, uint32_t reg_addr,
705 			 uint32_t reg_val)
706 {
707 	uint32_t ori_reg_val;
708 	struct rte_eth_dev *dev;
709 
710 	ori_reg_val = i40e_read_rx_ctl(hw, reg_addr);
711 	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
712 	i40e_write_rx_ctl(hw, reg_addr, reg_val);
713 	if (ori_reg_val != reg_val)
714 		PMD_DRV_LOG(WARNING,
715 			    "i40e device %s changed global register [0x%08x]."
716 			    " original: 0x%08x, new: 0x%08x",
717 			    dev->device->name, reg_addr, ori_reg_val, reg_val);
718 }
719 
720 RTE_PMD_REGISTER_PCI(net_i40e, rte_i40e_pmd);
721 RTE_PMD_REGISTER_PCI_TABLE(net_i40e, pci_id_i40e_map);
722 RTE_PMD_REGISTER_KMOD_DEP(net_i40e, "* igb_uio | uio_pci_generic | vfio-pci");
723 
724 #ifndef I40E_GLQF_ORT
725 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
726 #endif
727 #ifndef I40E_GLQF_PIT
728 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
729 #endif
730 #ifndef I40E_GLQF_L3_MAP
731 #define I40E_GLQF_L3_MAP(_i) (0x0026C700 + ((_i) * 4))
732 #endif
733 
734 static inline void i40e_GLQF_reg_init(struct i40e_hw *hw)
735 {
736 	/*
737 	 * Initialize registers for parsing packet type of QinQ
738 	 * This should be removed from code once proper
739 	 * configuration API is added to avoid configuration conflicts
740 	 * between ports of the same device.
741 	 */
742 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(40), 0x00000029);
743 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_PIT(9), 0x00009420);
744 }
745 
746 static inline void i40e_config_automask(struct i40e_pf *pf)
747 {
748 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
749 	uint32_t val;
750 
751 	/* INTENA flag is not auto-cleared for interrupt */
752 	val = I40E_READ_REG(hw, I40E_GLINT_CTL);
753 	val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK |
754 		I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK;
755 
756 	/* If support multi-driver, PF will use INT0. */
757 	if (!pf->support_multi_driver)
758 		val |= I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK;
759 
760 	I40E_WRITE_REG(hw, I40E_GLINT_CTL, val);
761 }
762 
763 #define I40E_FLOW_CONTROL_ETHERTYPE  0x8808
764 
765 /*
766  * Add a ethertype filter to drop all flow control frames transmitted
767  * from VSIs.
768 */
769 static void
770 i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf)
771 {
772 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
773 	uint16_t flags = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
774 			I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
775 			I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
776 	int ret;
777 
778 	ret = i40e_aq_add_rem_control_packet_filter(hw, NULL,
779 				I40E_FLOW_CONTROL_ETHERTYPE, flags,
780 				pf->main_vsi_seid, 0,
781 				TRUE, NULL, NULL);
782 	if (ret)
783 		PMD_INIT_LOG(ERR,
784 			"Failed to add filter to drop flow control frames from VSIs.");
785 }
786 
787 static int
788 floating_veb_list_handler(__rte_unused const char *key,
789 			  const char *floating_veb_value,
790 			  void *opaque)
791 {
792 	int idx = 0;
793 	unsigned int count = 0;
794 	char *end = NULL;
795 	int min, max;
796 	bool *vf_floating_veb = opaque;
797 
798 	while (isblank(*floating_veb_value))
799 		floating_veb_value++;
800 
801 	/* Reset floating VEB configuration for VFs */
802 	for (idx = 0; idx < I40E_MAX_VF; idx++)
803 		vf_floating_veb[idx] = false;
804 
805 	min = I40E_MAX_VF;
806 	do {
807 		while (isblank(*floating_veb_value))
808 			floating_veb_value++;
809 		if (*floating_veb_value == '\0')
810 			return -1;
811 		errno = 0;
812 		idx = strtoul(floating_veb_value, &end, 10);
813 		if (errno || end == NULL)
814 			return -1;
815 		while (isblank(*end))
816 			end++;
817 		if (*end == '-') {
818 			min = idx;
819 		} else if ((*end == ';') || (*end == '\0')) {
820 			max = idx;
821 			if (min == I40E_MAX_VF)
822 				min = idx;
823 			if (max >= I40E_MAX_VF)
824 				max = I40E_MAX_VF - 1;
825 			for (idx = min; idx <= max; idx++) {
826 				vf_floating_veb[idx] = true;
827 				count++;
828 			}
829 			min = I40E_MAX_VF;
830 		} else {
831 			return -1;
832 		}
833 		floating_veb_value = end + 1;
834 	} while (*end != '\0');
835 
836 	if (count == 0)
837 		return -1;
838 
839 	return 0;
840 }
841 
842 static void
843 config_vf_floating_veb(struct rte_devargs *devargs,
844 		       uint16_t floating_veb,
845 		       bool *vf_floating_veb)
846 {
847 	struct rte_kvargs *kvlist;
848 	int i;
849 	const char *floating_veb_list = ETH_I40E_FLOATING_VEB_LIST_ARG;
850 
851 	if (!floating_veb)
852 		return;
853 	/* All the VFs attach to the floating VEB by default
854 	 * when the floating VEB is enabled.
855 	 */
856 	for (i = 0; i < I40E_MAX_VF; i++)
857 		vf_floating_veb[i] = true;
858 
859 	if (devargs == NULL)
860 		return;
861 
862 	kvlist = rte_kvargs_parse(devargs->args, valid_keys);
863 	if (kvlist == NULL)
864 		return;
865 
866 	if (!rte_kvargs_count(kvlist, floating_veb_list)) {
867 		rte_kvargs_free(kvlist);
868 		return;
869 	}
870 	/* When the floating_veb_list parameter exists, all the VFs
871 	 * will attach to the legacy VEB firstly, then configure VFs
872 	 * to the floating VEB according to the floating_veb_list.
873 	 */
874 	if (rte_kvargs_process(kvlist, floating_veb_list,
875 			       floating_veb_list_handler,
876 			       vf_floating_veb) < 0) {
877 		rte_kvargs_free(kvlist);
878 		return;
879 	}
880 	rte_kvargs_free(kvlist);
881 }
882 
883 static int
884 i40e_check_floating_handler(__rte_unused const char *key,
885 			    const char *value,
886 			    __rte_unused void *opaque)
887 {
888 	if (strcmp(value, "1"))
889 		return -1;
890 
891 	return 0;
892 }
893 
894 static int
895 is_floating_veb_supported(struct rte_devargs *devargs)
896 {
897 	struct rte_kvargs *kvlist;
898 	const char *floating_veb_key = ETH_I40E_FLOATING_VEB_ARG;
899 
900 	if (devargs == NULL)
901 		return 0;
902 
903 	kvlist = rte_kvargs_parse(devargs->args, valid_keys);
904 	if (kvlist == NULL)
905 		return 0;
906 
907 	if (!rte_kvargs_count(kvlist, floating_veb_key)) {
908 		rte_kvargs_free(kvlist);
909 		return 0;
910 	}
911 	/* Floating VEB is enabled when there's key-value:
912 	 * enable_floating_veb=1
913 	 */
914 	if (rte_kvargs_process(kvlist, floating_veb_key,
915 			       i40e_check_floating_handler, NULL) < 0) {
916 		rte_kvargs_free(kvlist);
917 		return 0;
918 	}
919 	rte_kvargs_free(kvlist);
920 
921 	return 1;
922 }
923 
924 static void
925 config_floating_veb(struct rte_eth_dev *dev)
926 {
927 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
928 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
929 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
930 
931 	memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list));
932 
933 	if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) {
934 		pf->floating_veb =
935 			is_floating_veb_supported(pci_dev->device.devargs);
936 		config_vf_floating_veb(pci_dev->device.devargs,
937 				       pf->floating_veb,
938 				       pf->floating_veb_list);
939 	} else {
940 		pf->floating_veb = false;
941 	}
942 }
943 
944 #define I40E_L2_TAGS_S_TAG_SHIFT 1
945 #define I40E_L2_TAGS_S_TAG_MASK I40E_MASK(0x1, I40E_L2_TAGS_S_TAG_SHIFT)
946 
947 static int
948 i40e_init_ethtype_filter_list(struct rte_eth_dev *dev)
949 {
950 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
951 	struct i40e_ethertype_rule *ethertype_rule = &pf->ethertype;
952 	char ethertype_hash_name[RTE_HASH_NAMESIZE];
953 	int ret;
954 
955 	struct rte_hash_parameters ethertype_hash_params = {
956 		.name = ethertype_hash_name,
957 		.entries = I40E_MAX_ETHERTYPE_FILTER_NUM,
958 		.key_len = sizeof(struct i40e_ethertype_filter_input),
959 		.hash_func = rte_hash_crc,
960 		.hash_func_init_val = 0,
961 		.socket_id = rte_socket_id(),
962 	};
963 
964 	/* Initialize ethertype filter rule list and hash */
965 	TAILQ_INIT(&ethertype_rule->ethertype_list);
966 	snprintf(ethertype_hash_name, RTE_HASH_NAMESIZE,
967 		 "ethertype_%s", dev->device->name);
968 	ethertype_rule->hash_table = rte_hash_create(&ethertype_hash_params);
969 	if (!ethertype_rule->hash_table) {
970 		PMD_INIT_LOG(ERR, "Failed to create ethertype hash table!");
971 		return -EINVAL;
972 	}
973 	ethertype_rule->hash_map = rte_zmalloc("i40e_ethertype_hash_map",
974 				       sizeof(struct i40e_ethertype_filter *) *
975 				       I40E_MAX_ETHERTYPE_FILTER_NUM,
976 				       0);
977 	if (!ethertype_rule->hash_map) {
978 		PMD_INIT_LOG(ERR,
979 			     "Failed to allocate memory for ethertype hash map!");
980 		ret = -ENOMEM;
981 		goto err_ethertype_hash_map_alloc;
982 	}
983 
984 	return 0;
985 
986 err_ethertype_hash_map_alloc:
987 	rte_hash_free(ethertype_rule->hash_table);
988 
989 	return ret;
990 }
991 
992 static int
993 i40e_init_tunnel_filter_list(struct rte_eth_dev *dev)
994 {
995 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
996 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
997 	char tunnel_hash_name[RTE_HASH_NAMESIZE];
998 	int ret;
999 
1000 	struct rte_hash_parameters tunnel_hash_params = {
1001 		.name = tunnel_hash_name,
1002 		.entries = I40E_MAX_TUNNEL_FILTER_NUM,
1003 		.key_len = sizeof(struct i40e_tunnel_filter_input),
1004 		.hash_func = rte_hash_crc,
1005 		.hash_func_init_val = 0,
1006 		.socket_id = rte_socket_id(),
1007 	};
1008 
1009 	/* Initialize tunnel filter rule list and hash */
1010 	TAILQ_INIT(&tunnel_rule->tunnel_list);
1011 	snprintf(tunnel_hash_name, RTE_HASH_NAMESIZE,
1012 		 "tunnel_%s", dev->device->name);
1013 	tunnel_rule->hash_table = rte_hash_create(&tunnel_hash_params);
1014 	if (!tunnel_rule->hash_table) {
1015 		PMD_INIT_LOG(ERR, "Failed to create tunnel hash table!");
1016 		return -EINVAL;
1017 	}
1018 	tunnel_rule->hash_map = rte_zmalloc("i40e_tunnel_hash_map",
1019 				    sizeof(struct i40e_tunnel_filter *) *
1020 				    I40E_MAX_TUNNEL_FILTER_NUM,
1021 				    0);
1022 	if (!tunnel_rule->hash_map) {
1023 		PMD_INIT_LOG(ERR,
1024 			     "Failed to allocate memory for tunnel hash map!");
1025 		ret = -ENOMEM;
1026 		goto err_tunnel_hash_map_alloc;
1027 	}
1028 
1029 	return 0;
1030 
1031 err_tunnel_hash_map_alloc:
1032 	rte_hash_free(tunnel_rule->hash_table);
1033 
1034 	return ret;
1035 }
1036 
1037 static int
1038 i40e_init_fdir_filter_list(struct rte_eth_dev *dev)
1039 {
1040 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1041 	struct i40e_fdir_info *fdir_info = &pf->fdir;
1042 	char fdir_hash_name[RTE_HASH_NAMESIZE];
1043 	int ret;
1044 
1045 	struct rte_hash_parameters fdir_hash_params = {
1046 		.name = fdir_hash_name,
1047 		.entries = I40E_MAX_FDIR_FILTER_NUM,
1048 		.key_len = sizeof(struct i40e_fdir_input),
1049 		.hash_func = rte_hash_crc,
1050 		.hash_func_init_val = 0,
1051 		.socket_id = rte_socket_id(),
1052 	};
1053 
1054 	/* Initialize flow director filter rule list and hash */
1055 	TAILQ_INIT(&fdir_info->fdir_list);
1056 	snprintf(fdir_hash_name, RTE_HASH_NAMESIZE,
1057 		 "fdir_%s", dev->device->name);
1058 	fdir_info->hash_table = rte_hash_create(&fdir_hash_params);
1059 	if (!fdir_info->hash_table) {
1060 		PMD_INIT_LOG(ERR, "Failed to create fdir hash table!");
1061 		return -EINVAL;
1062 	}
1063 	fdir_info->hash_map = rte_zmalloc("i40e_fdir_hash_map",
1064 					  sizeof(struct i40e_fdir_filter *) *
1065 					  I40E_MAX_FDIR_FILTER_NUM,
1066 					  0);
1067 	if (!fdir_info->hash_map) {
1068 		PMD_INIT_LOG(ERR,
1069 			     "Failed to allocate memory for fdir hash map!");
1070 		ret = -ENOMEM;
1071 		goto err_fdir_hash_map_alloc;
1072 	}
1073 	return 0;
1074 
1075 err_fdir_hash_map_alloc:
1076 	rte_hash_free(fdir_info->hash_table);
1077 
1078 	return ret;
1079 }
1080 
1081 static void
1082 i40e_init_customized_info(struct i40e_pf *pf)
1083 {
1084 	int i;
1085 
1086 	/* Initialize customized pctype */
1087 	for (i = I40E_CUSTOMIZED_GTPC; i < I40E_CUSTOMIZED_MAX; i++) {
1088 		pf->customized_pctype[i].index = i;
1089 		pf->customized_pctype[i].pctype = I40E_FILTER_PCTYPE_INVALID;
1090 		pf->customized_pctype[i].valid = false;
1091 	}
1092 
1093 	pf->gtp_support = false;
1094 }
1095 
1096 void
1097 i40e_init_queue_region_conf(struct rte_eth_dev *dev)
1098 {
1099 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1100 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1101 	struct i40e_queue_regions *info = &pf->queue_region;
1102 	uint16_t i;
1103 
1104 	for (i = 0; i < I40E_PFQF_HREGION_MAX_INDEX; i++)
1105 		i40e_write_rx_ctl(hw, I40E_PFQF_HREGION(i), 0);
1106 
1107 	memset(info, 0, sizeof(struct i40e_queue_regions));
1108 }
1109 
1110 static int
1111 i40e_parse_multi_drv_handler(__rte_unused const char *key,
1112 			       const char *value,
1113 			       void *opaque)
1114 {
1115 	struct i40e_pf *pf;
1116 	unsigned long support_multi_driver;
1117 	char *end;
1118 
1119 	pf = (struct i40e_pf *)opaque;
1120 
1121 	errno = 0;
1122 	support_multi_driver = strtoul(value, &end, 10);
1123 	if (errno != 0 || end == value || *end != 0) {
1124 		PMD_DRV_LOG(WARNING, "Wrong global configuration");
1125 		return -(EINVAL);
1126 	}
1127 
1128 	if (support_multi_driver == 1 || support_multi_driver == 0)
1129 		pf->support_multi_driver = (bool)support_multi_driver;
1130 	else
1131 		PMD_DRV_LOG(WARNING, "%s must be 1 or 0,",
1132 			    "enable global configuration by default."
1133 			    ETH_I40E_SUPPORT_MULTI_DRIVER);
1134 	return 0;
1135 }
1136 
1137 static int
1138 i40e_support_multi_driver(struct rte_eth_dev *dev)
1139 {
1140 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1141 	struct rte_kvargs *kvlist;
1142 	int kvargs_count;
1143 
1144 	/* Enable global configuration by default */
1145 	pf->support_multi_driver = false;
1146 
1147 	if (!dev->device->devargs)
1148 		return 0;
1149 
1150 	kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
1151 	if (!kvlist)
1152 		return -EINVAL;
1153 
1154 	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER);
1155 	if (!kvargs_count) {
1156 		rte_kvargs_free(kvlist);
1157 		return 0;
1158 	}
1159 
1160 	if (kvargs_count > 1)
1161 		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
1162 			    "the first invalid or last valid one is used !",
1163 			    ETH_I40E_SUPPORT_MULTI_DRIVER);
1164 
1165 	if (rte_kvargs_process(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER,
1166 			       i40e_parse_multi_drv_handler, pf) < 0) {
1167 		rte_kvargs_free(kvlist);
1168 		return -EINVAL;
1169 	}
1170 
1171 	rte_kvargs_free(kvlist);
1172 	return 0;
1173 }
1174 
1175 static int
1176 i40e_aq_debug_write_global_register(struct i40e_hw *hw,
1177 				    uint32_t reg_addr, uint64_t reg_val,
1178 				    struct i40e_asq_cmd_details *cmd_details)
1179 {
1180 	uint64_t ori_reg_val;
1181 	struct rte_eth_dev *dev;
1182 	int ret;
1183 
1184 	ret = i40e_aq_debug_read_register(hw, reg_addr, &ori_reg_val, NULL);
1185 	if (ret != I40E_SUCCESS) {
1186 		PMD_DRV_LOG(ERR,
1187 			    "Fail to debug read from 0x%08x",
1188 			    reg_addr);
1189 		return -EIO;
1190 	}
1191 	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
1192 
1193 	if (ori_reg_val != reg_val)
1194 		PMD_DRV_LOG(WARNING,
1195 			    "i40e device %s changed global register [0x%08x]."
1196 			    " original: 0x%"PRIx64", after: 0x%"PRIx64,
1197 			    dev->device->name, reg_addr, ori_reg_val, reg_val);
1198 
1199 	return i40e_aq_debug_write_register(hw, reg_addr, reg_val, cmd_details);
1200 }
1201 
1202 static int
1203 i40e_parse_latest_vec_handler(__rte_unused const char *key,
1204 				const char *value,
1205 				void *opaque)
1206 {
1207 	struct i40e_adapter *ad = opaque;
1208 	int use_latest_vec;
1209 
1210 	use_latest_vec = atoi(value);
1211 
1212 	if (use_latest_vec != 0 && use_latest_vec != 1)
1213 		PMD_DRV_LOG(WARNING, "Value should be 0 or 1, set it as 1!");
1214 
1215 	ad->use_latest_vec = (uint8_t)use_latest_vec;
1216 
1217 	return 0;
1218 }
1219 
1220 static int
1221 i40e_use_latest_vec(struct rte_eth_dev *dev)
1222 {
1223 	struct i40e_adapter *ad =
1224 		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1225 	struct rte_kvargs *kvlist;
1226 	int kvargs_count;
1227 
1228 	ad->use_latest_vec = false;
1229 
1230 	if (!dev->device->devargs)
1231 		return 0;
1232 
1233 	kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
1234 	if (!kvlist)
1235 		return -EINVAL;
1236 
1237 	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_USE_LATEST_VEC);
1238 	if (!kvargs_count) {
1239 		rte_kvargs_free(kvlist);
1240 		return 0;
1241 	}
1242 
1243 	if (kvargs_count > 1)
1244 		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
1245 			    "the first invalid or last valid one is used !",
1246 			    ETH_I40E_USE_LATEST_VEC);
1247 
1248 	if (rte_kvargs_process(kvlist, ETH_I40E_USE_LATEST_VEC,
1249 				i40e_parse_latest_vec_handler, ad) < 0) {
1250 		rte_kvargs_free(kvlist);
1251 		return -EINVAL;
1252 	}
1253 
1254 	rte_kvargs_free(kvlist);
1255 	return 0;
1256 }
1257 
1258 #define I40E_ALARM_INTERVAL 50000 /* us */
1259 
1260 static int
1261 eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
1262 {
1263 	struct rte_pci_device *pci_dev;
1264 	struct rte_intr_handle *intr_handle;
1265 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1266 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1267 	struct i40e_vsi *vsi;
1268 	int ret;
1269 	uint32_t len, val;
1270 	uint8_t aq_fail = 0;
1271 
1272 	PMD_INIT_FUNC_TRACE();
1273 
1274 	dev->dev_ops = &i40e_eth_dev_ops;
1275 	dev->rx_pkt_burst = i40e_recv_pkts;
1276 	dev->tx_pkt_burst = i40e_xmit_pkts;
1277 	dev->tx_pkt_prepare = i40e_prep_pkts;
1278 
1279 	/* for secondary processes, we don't initialise any further as primary
1280 	 * has already done this work. Only check we don't need a different
1281 	 * RX function */
1282 	if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1283 		i40e_set_rx_function(dev);
1284 		i40e_set_tx_function(dev);
1285 		return 0;
1286 	}
1287 	i40e_set_default_ptype_table(dev);
1288 	pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1289 	intr_handle = &pci_dev->intr_handle;
1290 
1291 	rte_eth_copy_pci_info(dev, pci_dev);
1292 
1293 	pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1294 	pf->adapter->eth_dev = dev;
1295 	pf->dev_data = dev->data;
1296 
1297 	hw->back = I40E_PF_TO_ADAPTER(pf);
1298 	hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
1299 	if (!hw->hw_addr) {
1300 		PMD_INIT_LOG(ERR,
1301 			"Hardware is not available, as address is NULL");
1302 		return -ENODEV;
1303 	}
1304 
1305 	hw->vendor_id = pci_dev->id.vendor_id;
1306 	hw->device_id = pci_dev->id.device_id;
1307 	hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1308 	hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1309 	hw->bus.device = pci_dev->addr.devid;
1310 	hw->bus.func = pci_dev->addr.function;
1311 	hw->adapter_stopped = 0;
1312 	hw->adapter_closed = 0;
1313 
1314 	/*
1315 	 * Switch Tag value should not be identical to either the First Tag
1316 	 * or Second Tag values. So set something other than common Ethertype
1317 	 * for internal switching.
1318 	 */
1319 	hw->switch_tag = 0xffff;
1320 
1321 	val = I40E_READ_REG(hw, I40E_GL_FWSTS);
1322 	if (val & I40E_GL_FWSTS_FWS1B_MASK) {
1323 		PMD_INIT_LOG(ERR, "\nERROR: "
1324 			"Firmware recovery mode detected. Limiting functionality.\n"
1325 			"Refer to the Intel(R) Ethernet Adapters and Devices "
1326 			"User Guide for details on firmware recovery mode.");
1327 		return -EIO;
1328 	}
1329 
1330 	/* Check if need to support multi-driver */
1331 	i40e_support_multi_driver(dev);
1332 	/* Check if users want the latest supported vec path */
1333 	i40e_use_latest_vec(dev);
1334 
1335 	/* Make sure all is clean before doing PF reset */
1336 	i40e_clear_hw(hw);
1337 
1338 	/* Reset here to make sure all is clean for each PF */
1339 	ret = i40e_pf_reset(hw);
1340 	if (ret) {
1341 		PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
1342 		return ret;
1343 	}
1344 
1345 	/* Initialize the shared code (base driver) */
1346 	ret = i40e_init_shared_code(hw);
1347 	if (ret) {
1348 		PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
1349 		return ret;
1350 	}
1351 
1352 	/* Initialize the parameters for adminq */
1353 	i40e_init_adminq_parameter(hw);
1354 	ret = i40e_init_adminq(hw);
1355 	if (ret != I40E_SUCCESS) {
1356 		PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
1357 		return -EIO;
1358 	}
1359 	/* Firmware of SFP x722 does not support adminq option */
1360 	if (hw->device_id == I40E_DEV_ID_SFP_X722)
1361 		hw->flags &= ~I40E_HW_FLAG_802_1AD_CAPABLE;
1362 
1363 	PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
1364 		     hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
1365 		     hw->aq.api_maj_ver, hw->aq.api_min_ver,
1366 		     ((hw->nvm.version >> 12) & 0xf),
1367 		     ((hw->nvm.version >> 4) & 0xff),
1368 		     (hw->nvm.version & 0xf), hw->nvm.eetrack);
1369 
1370 	/* Initialize the hardware */
1371 	i40e_hw_init(dev);
1372 
1373 	i40e_config_automask(pf);
1374 
1375 	i40e_set_default_pctype_table(dev);
1376 
1377 	/*
1378 	 * To work around the NVM issue, initialize registers
1379 	 * for packet type of QinQ by software.
1380 	 * It should be removed once issues are fixed in NVM.
1381 	 */
1382 	if (!pf->support_multi_driver)
1383 		i40e_GLQF_reg_init(hw);
1384 
1385 	/* Initialize the input set for filters (hash and fd) to default value */
1386 	i40e_filter_input_set_init(pf);
1387 
1388 	/* initialise the L3_MAP register */
1389 	if (!pf->support_multi_driver) {
1390 		ret = i40e_aq_debug_write_global_register(hw,
1391 						   I40E_GLQF_L3_MAP(40),
1392 						   0x00000028,	NULL);
1393 		if (ret)
1394 			PMD_INIT_LOG(ERR, "Failed to write L3 MAP register %d",
1395 				     ret);
1396 		PMD_INIT_LOG(DEBUG,
1397 			     "Global register 0x%08x is changed with 0x28",
1398 			     I40E_GLQF_L3_MAP(40));
1399 	}
1400 
1401 	/* Need the special FW version to support floating VEB */
1402 	config_floating_veb(dev);
1403 	/* Clear PXE mode */
1404 	i40e_clear_pxe_mode(hw);
1405 	i40e_dev_sync_phy_type(hw);
1406 
1407 	/*
1408 	 * On X710, performance number is far from the expectation on recent
1409 	 * firmware versions. The fix for this issue may not be integrated in
1410 	 * the following firmware version. So the workaround in software driver
1411 	 * is needed. It needs to modify the initial values of 3 internal only
1412 	 * registers. Note that the workaround can be removed when it is fixed
1413 	 * in firmware in the future.
1414 	 */
1415 	i40e_configure_registers(hw);
1416 
1417 	/* Get hw capabilities */
1418 	ret = i40e_get_cap(hw);
1419 	if (ret != I40E_SUCCESS) {
1420 		PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
1421 		goto err_get_capabilities;
1422 	}
1423 
1424 	/* Initialize parameters for PF */
1425 	ret = i40e_pf_parameter_init(dev);
1426 	if (ret != 0) {
1427 		PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
1428 		goto err_parameter_init;
1429 	}
1430 
1431 	/* Initialize the queue management */
1432 	ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
1433 	if (ret < 0) {
1434 		PMD_INIT_LOG(ERR, "Failed to init queue pool");
1435 		goto err_qp_pool_init;
1436 	}
1437 	ret = i40e_res_pool_init(&pf->msix_pool, 1,
1438 				hw->func_caps.num_msix_vectors - 1);
1439 	if (ret < 0) {
1440 		PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
1441 		goto err_msix_pool_init;
1442 	}
1443 
1444 	/* Initialize lan hmc */
1445 	ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1446 				hw->func_caps.num_rx_qp, 0, 0);
1447 	if (ret != I40E_SUCCESS) {
1448 		PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
1449 		goto err_init_lan_hmc;
1450 	}
1451 
1452 	/* Configure lan hmc */
1453 	ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1454 	if (ret != I40E_SUCCESS) {
1455 		PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
1456 		goto err_configure_lan_hmc;
1457 	}
1458 
1459 	/* Get and check the mac address */
1460 	i40e_get_mac_addr(hw, hw->mac.addr);
1461 	if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
1462 		PMD_INIT_LOG(ERR, "mac address is not valid");
1463 		ret = -EIO;
1464 		goto err_get_mac_addr;
1465 	}
1466 	/* Copy the permanent MAC address */
1467 	ether_addr_copy((struct ether_addr *) hw->mac.addr,
1468 			(struct ether_addr *) hw->mac.perm_addr);
1469 
1470 	/* Disable flow control */
1471 	hw->fc.requested_mode = I40E_FC_NONE;
1472 	i40e_set_fc(hw, &aq_fail, TRUE);
1473 
1474 	/* Set the global registers with default ether type value */
1475 	if (!pf->support_multi_driver) {
1476 		ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
1477 					 ETHER_TYPE_VLAN);
1478 		if (ret != I40E_SUCCESS) {
1479 			PMD_INIT_LOG(ERR,
1480 				     "Failed to set the default outer "
1481 				     "VLAN ether type");
1482 			goto err_setup_pf_switch;
1483 		}
1484 	}
1485 
1486 	/* PF setup, which includes VSI setup */
1487 	ret = i40e_pf_setup(pf);
1488 	if (ret) {
1489 		PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
1490 		goto err_setup_pf_switch;
1491 	}
1492 
1493 	vsi = pf->main_vsi;
1494 
1495 	/* Disable double vlan by default */
1496 	i40e_vsi_config_double_vlan(vsi, FALSE);
1497 
1498 	/* Disable S-TAG identification when floating_veb is disabled */
1499 	if (!pf->floating_veb) {
1500 		ret = I40E_READ_REG(hw, I40E_PRT_L2TAGSEN);
1501 		if (ret & I40E_L2_TAGS_S_TAG_MASK) {
1502 			ret &= ~I40E_L2_TAGS_S_TAG_MASK;
1503 			I40E_WRITE_REG(hw, I40E_PRT_L2TAGSEN, ret);
1504 		}
1505 	}
1506 
1507 	if (!vsi->max_macaddrs)
1508 		len = ETHER_ADDR_LEN;
1509 	else
1510 		len = ETHER_ADDR_LEN * vsi->max_macaddrs;
1511 
1512 	/* Should be after VSI initialized */
1513 	dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
1514 	if (!dev->data->mac_addrs) {
1515 		PMD_INIT_LOG(ERR,
1516 			"Failed to allocated memory for storing mac address");
1517 		goto err_mac_alloc;
1518 	}
1519 	ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
1520 					&dev->data->mac_addrs[0]);
1521 
1522 	/* Init dcb to sw mode by default */
1523 	ret = i40e_dcb_init_configure(dev, TRUE);
1524 	if (ret != I40E_SUCCESS) {
1525 		PMD_INIT_LOG(INFO, "Failed to init dcb.");
1526 		pf->flags &= ~I40E_FLAG_DCB;
1527 	}
1528 	/* Update HW struct after DCB configuration */
1529 	i40e_get_cap(hw);
1530 
1531 	/* initialize pf host driver to setup SRIOV resource if applicable */
1532 	i40e_pf_host_init(dev);
1533 
1534 	/* register callback func to eal lib */
1535 	rte_intr_callback_register(intr_handle,
1536 				   i40e_dev_interrupt_handler, dev);
1537 
1538 	/* configure and enable device interrupt */
1539 	i40e_pf_config_irq0(hw, TRUE);
1540 	i40e_pf_enable_irq0(hw);
1541 
1542 	/* enable uio intr after callback register */
1543 	rte_intr_enable(intr_handle);
1544 
1545 	/* By default disable flexible payload in global configuration */
1546 	if (!pf->support_multi_driver)
1547 		i40e_flex_payload_reg_set_default(hw);
1548 
1549 	/*
1550 	 * Add an ethertype filter to drop all flow control frames transmitted
1551 	 * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC
1552 	 * frames to wire.
1553 	 */
1554 	i40e_add_tx_flow_control_drop_filter(pf);
1555 
1556 	/* Set the max frame size to 0x2600 by default,
1557 	 * in case other drivers changed the default value.
1558 	 */
1559 	i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL);
1560 
1561 	/* initialize mirror rule list */
1562 	TAILQ_INIT(&pf->mirror_list);
1563 
1564 	/* initialize Traffic Manager configuration */
1565 	i40e_tm_conf_init(dev);
1566 
1567 	/* Initialize customized information */
1568 	i40e_init_customized_info(pf);
1569 
1570 	ret = i40e_init_ethtype_filter_list(dev);
1571 	if (ret < 0)
1572 		goto err_init_ethtype_filter_list;
1573 	ret = i40e_init_tunnel_filter_list(dev);
1574 	if (ret < 0)
1575 		goto err_init_tunnel_filter_list;
1576 	ret = i40e_init_fdir_filter_list(dev);
1577 	if (ret < 0)
1578 		goto err_init_fdir_filter_list;
1579 
1580 	/* initialize queue region configuration */
1581 	i40e_init_queue_region_conf(dev);
1582 
1583 	/* initialize rss configuration from rte_flow */
1584 	memset(&pf->rss_info, 0,
1585 		sizeof(struct i40e_rte_flow_rss_conf));
1586 
1587 	/* reset all stats of the device, including pf and main vsi */
1588 	i40e_dev_stats_reset(dev);
1589 
1590 	return 0;
1591 
1592 err_init_fdir_filter_list:
1593 	rte_free(pf->tunnel.hash_table);
1594 	rte_free(pf->tunnel.hash_map);
1595 err_init_tunnel_filter_list:
1596 	rte_free(pf->ethertype.hash_table);
1597 	rte_free(pf->ethertype.hash_map);
1598 err_init_ethtype_filter_list:
1599 	rte_free(dev->data->mac_addrs);
1600 	dev->data->mac_addrs = NULL;
1601 err_mac_alloc:
1602 	i40e_vsi_release(pf->main_vsi);
1603 err_setup_pf_switch:
1604 err_get_mac_addr:
1605 err_configure_lan_hmc:
1606 	(void)i40e_shutdown_lan_hmc(hw);
1607 err_init_lan_hmc:
1608 	i40e_res_pool_destroy(&pf->msix_pool);
1609 err_msix_pool_init:
1610 	i40e_res_pool_destroy(&pf->qp_pool);
1611 err_qp_pool_init:
1612 err_parameter_init:
1613 err_get_capabilities:
1614 	(void)i40e_shutdown_adminq(hw);
1615 
1616 	return ret;
1617 }
1618 
1619 static void
1620 i40e_rm_ethtype_filter_list(struct i40e_pf *pf)
1621 {
1622 	struct i40e_ethertype_filter *p_ethertype;
1623 	struct i40e_ethertype_rule *ethertype_rule;
1624 
1625 	ethertype_rule = &pf->ethertype;
1626 	/* Remove all ethertype filter rules and hash */
1627 	if (ethertype_rule->hash_map)
1628 		rte_free(ethertype_rule->hash_map);
1629 	if (ethertype_rule->hash_table)
1630 		rte_hash_free(ethertype_rule->hash_table);
1631 
1632 	while ((p_ethertype = TAILQ_FIRST(&ethertype_rule->ethertype_list))) {
1633 		TAILQ_REMOVE(&ethertype_rule->ethertype_list,
1634 			     p_ethertype, rules);
1635 		rte_free(p_ethertype);
1636 	}
1637 }
1638 
1639 static void
1640 i40e_rm_tunnel_filter_list(struct i40e_pf *pf)
1641 {
1642 	struct i40e_tunnel_filter *p_tunnel;
1643 	struct i40e_tunnel_rule *tunnel_rule;
1644 
1645 	tunnel_rule = &pf->tunnel;
1646 	/* Remove all tunnel director rules and hash */
1647 	if (tunnel_rule->hash_map)
1648 		rte_free(tunnel_rule->hash_map);
1649 	if (tunnel_rule->hash_table)
1650 		rte_hash_free(tunnel_rule->hash_table);
1651 
1652 	while ((p_tunnel = TAILQ_FIRST(&tunnel_rule->tunnel_list))) {
1653 		TAILQ_REMOVE(&tunnel_rule->tunnel_list, p_tunnel, rules);
1654 		rte_free(p_tunnel);
1655 	}
1656 }
1657 
1658 static void
1659 i40e_rm_fdir_filter_list(struct i40e_pf *pf)
1660 {
1661 	struct i40e_fdir_filter *p_fdir;
1662 	struct i40e_fdir_info *fdir_info;
1663 
1664 	fdir_info = &pf->fdir;
1665 	/* Remove all flow director rules and hash */
1666 	if (fdir_info->hash_map)
1667 		rte_free(fdir_info->hash_map);
1668 	if (fdir_info->hash_table)
1669 		rte_hash_free(fdir_info->hash_table);
1670 
1671 	while ((p_fdir = TAILQ_FIRST(&fdir_info->fdir_list))) {
1672 		TAILQ_REMOVE(&fdir_info->fdir_list, p_fdir, rules);
1673 		rte_free(p_fdir);
1674 	}
1675 }
1676 
1677 void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
1678 {
1679 	/*
1680 	 * Disable by default flexible payload
1681 	 * for corresponding L2/L3/L4 layers.
1682 	 */
1683 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33), 0x00000000);
1684 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(34), 0x00000000);
1685 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(35), 0x00000000);
1686 }
1687 
1688 static int
1689 eth_i40e_dev_uninit(struct rte_eth_dev *dev)
1690 {
1691 	struct i40e_pf *pf;
1692 	struct rte_pci_device *pci_dev;
1693 	struct rte_intr_handle *intr_handle;
1694 	struct i40e_hw *hw;
1695 	struct i40e_filter_control_settings settings;
1696 	struct rte_flow *p_flow;
1697 	int ret;
1698 	uint8_t aq_fail = 0;
1699 	int retries = 0;
1700 
1701 	PMD_INIT_FUNC_TRACE();
1702 
1703 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1704 		return 0;
1705 
1706 	pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1707 	hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1708 	pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1709 	intr_handle = &pci_dev->intr_handle;
1710 
1711 	ret = rte_eth_switch_domain_free(pf->switch_domain_id);
1712 	if (ret)
1713 		PMD_INIT_LOG(WARNING, "failed to free switch domain: %d", ret);
1714 
1715 	if (hw->adapter_closed == 0)
1716 		i40e_dev_close(dev);
1717 
1718 	dev->dev_ops = NULL;
1719 	dev->rx_pkt_burst = NULL;
1720 	dev->tx_pkt_burst = NULL;
1721 
1722 	/* Clear PXE mode */
1723 	i40e_clear_pxe_mode(hw);
1724 
1725 	/* Unconfigure filter control */
1726 	memset(&settings, 0, sizeof(settings));
1727 	ret = i40e_set_filter_control(hw, &settings);
1728 	if (ret)
1729 		PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
1730 					ret);
1731 
1732 	/* Disable flow control */
1733 	hw->fc.requested_mode = I40E_FC_NONE;
1734 	i40e_set_fc(hw, &aq_fail, TRUE);
1735 
1736 	/* uninitialize pf host driver */
1737 	i40e_pf_host_uninit(dev);
1738 
1739 	/* disable uio intr before callback unregister */
1740 	rte_intr_disable(intr_handle);
1741 
1742 	/* unregister callback func to eal lib */
1743 	do {
1744 		ret = rte_intr_callback_unregister(intr_handle,
1745 				i40e_dev_interrupt_handler, dev);
1746 		if (ret >= 0) {
1747 			break;
1748 		} else if (ret != -EAGAIN) {
1749 			PMD_INIT_LOG(ERR,
1750 				 "intr callback unregister failed: %d",
1751 				 ret);
1752 			return ret;
1753 		}
1754 		i40e_msec_delay(500);
1755 	} while (retries++ < 5);
1756 
1757 	i40e_rm_ethtype_filter_list(pf);
1758 	i40e_rm_tunnel_filter_list(pf);
1759 	i40e_rm_fdir_filter_list(pf);
1760 
1761 	/* Remove all flows */
1762 	while ((p_flow = TAILQ_FIRST(&pf->flow_list))) {
1763 		TAILQ_REMOVE(&pf->flow_list, p_flow, node);
1764 		rte_free(p_flow);
1765 	}
1766 
1767 	/* Remove all Traffic Manager configuration */
1768 	i40e_tm_conf_uninit(dev);
1769 
1770 	return 0;
1771 }
1772 
1773 static int
1774 i40e_dev_configure(struct rte_eth_dev *dev)
1775 {
1776 	struct i40e_adapter *ad =
1777 		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1778 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1779 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1780 	enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1781 	int i, ret;
1782 
1783 	ret = i40e_dev_sync_phy_type(hw);
1784 	if (ret)
1785 		return ret;
1786 
1787 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
1788 	 * bulk allocation or vector Rx preconditions we will reset it.
1789 	 */
1790 	ad->rx_bulk_alloc_allowed = true;
1791 	ad->rx_vec_allowed = true;
1792 	ad->tx_simple_allowed = true;
1793 	ad->tx_vec_allowed = true;
1794 
1795 	/* Only legacy filter API needs the following fdir config. So when the
1796 	 * legacy filter API is deprecated, the following codes should also be
1797 	 * removed.
1798 	 */
1799 	if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
1800 		ret = i40e_fdir_setup(pf);
1801 		if (ret != I40E_SUCCESS) {
1802 			PMD_DRV_LOG(ERR, "Failed to setup flow director.");
1803 			return -ENOTSUP;
1804 		}
1805 		ret = i40e_fdir_configure(dev);
1806 		if (ret < 0) {
1807 			PMD_DRV_LOG(ERR, "failed to configure fdir.");
1808 			goto err;
1809 		}
1810 	} else
1811 		i40e_fdir_teardown(pf);
1812 
1813 	ret = i40e_dev_init_vlan(dev);
1814 	if (ret < 0)
1815 		goto err;
1816 
1817 	/* VMDQ setup.
1818 	 *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
1819 	 *  RSS setting have different requirements.
1820 	 *  General PMD driver call sequence are NIC init, configure,
1821 	 *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
1822 	 *  will try to lookup the VSI that specific queue belongs to if VMDQ
1823 	 *  applicable. So, VMDQ setting has to be done before
1824 	 *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
1825 	 *  For RSS setting, it will try to calculate actual configured RX queue
1826 	 *  number, which will be available after rx_queue_setup(). dev_start()
1827 	 *  function is good to place RSS setup.
1828 	 */
1829 	if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
1830 		ret = i40e_vmdq_setup(dev);
1831 		if (ret)
1832 			goto err;
1833 	}
1834 
1835 	if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
1836 		ret = i40e_dcb_setup(dev);
1837 		if (ret) {
1838 			PMD_DRV_LOG(ERR, "failed to configure DCB.");
1839 			goto err_dcb;
1840 		}
1841 	}
1842 
1843 	TAILQ_INIT(&pf->flow_list);
1844 
1845 	return 0;
1846 
1847 err_dcb:
1848 	/* need to release vmdq resource if exists */
1849 	for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1850 		i40e_vsi_release(pf->vmdq[i].vsi);
1851 		pf->vmdq[i].vsi = NULL;
1852 	}
1853 	rte_free(pf->vmdq);
1854 	pf->vmdq = NULL;
1855 err:
1856 	/* Need to release fdir resource if exists.
1857 	 * Only legacy filter API needs the following fdir config. So when the
1858 	 * legacy filter API is deprecated, the following code should also be
1859 	 * removed.
1860 	 */
1861 	i40e_fdir_teardown(pf);
1862 	return ret;
1863 }
1864 
1865 void
1866 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
1867 {
1868 	struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1869 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1870 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1871 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1872 	uint16_t msix_vect = vsi->msix_intr;
1873 	uint16_t i;
1874 
1875 	for (i = 0; i < vsi->nb_qps; i++) {
1876 		I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1877 		I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1878 		rte_wmb();
1879 	}
1880 
1881 	if (vsi->type != I40E_VSI_SRIOV) {
1882 		if (!rte_intr_allow_others(intr_handle)) {
1883 			I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1884 				       I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
1885 			I40E_WRITE_REG(hw,
1886 				       I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1887 				       0);
1888 		} else {
1889 			I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1890 				       I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK);
1891 			I40E_WRITE_REG(hw,
1892 				       I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1893 						       msix_vect - 1), 0);
1894 		}
1895 	} else {
1896 		uint32_t reg;
1897 		reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1898 			vsi->user_param + (msix_vect - 1);
1899 
1900 		I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1901 			       I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
1902 	}
1903 	I40E_WRITE_FLUSH(hw);
1904 }
1905 
1906 static void
1907 __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
1908 		       int base_queue, int nb_queue,
1909 		       uint16_t itr_idx)
1910 {
1911 	int i;
1912 	uint32_t val;
1913 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1914 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
1915 
1916 	/* Bind all RX queues to allocated MSIX interrupt */
1917 	for (i = 0; i < nb_queue; i++) {
1918 		val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
1919 			itr_idx << I40E_QINT_RQCTL_ITR_INDX_SHIFT |
1920 			((base_queue + i + 1) <<
1921 			 I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
1922 			(0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
1923 			I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1924 
1925 		if (i == nb_queue - 1)
1926 			val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
1927 		I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val);
1928 	}
1929 
1930 	/* Write first RX queue to Link list register as the head element */
1931 	if (vsi->type != I40E_VSI_SRIOV) {
1932 		uint16_t interval =
1933 			i40e_calc_itr_interval(1, pf->support_multi_driver);
1934 
1935 		if (msix_vect == I40E_MISC_VEC_ID) {
1936 			I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1937 				       (base_queue <<
1938 					I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1939 				       (0x0 <<
1940 					I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1941 			I40E_WRITE_REG(hw,
1942 				       I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1943 				       interval);
1944 		} else {
1945 			I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1946 				       (base_queue <<
1947 					I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1948 				       (0x0 <<
1949 					I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1950 			I40E_WRITE_REG(hw,
1951 				       I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1952 						       msix_vect - 1),
1953 				       interval);
1954 		}
1955 	} else {
1956 		uint32_t reg;
1957 
1958 		if (msix_vect == I40E_MISC_VEC_ID) {
1959 			I40E_WRITE_REG(hw,
1960 				       I40E_VPINT_LNKLST0(vsi->user_param),
1961 				       (base_queue <<
1962 					I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1963 				       (0x0 <<
1964 					I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1965 		} else {
1966 			/* num_msix_vectors_vf needs to minus irq0 */
1967 			reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1968 				vsi->user_param + (msix_vect - 1);
1969 
1970 			I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1971 				       (base_queue <<
1972 					I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1973 				       (0x0 <<
1974 					I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1975 		}
1976 	}
1977 
1978 	I40E_WRITE_FLUSH(hw);
1979 }
1980 
1981 void
1982 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t itr_idx)
1983 {
1984 	struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1985 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1986 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1987 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1988 	uint16_t msix_vect = vsi->msix_intr;
1989 	uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
1990 	uint16_t queue_idx = 0;
1991 	int record = 0;
1992 	int i;
1993 
1994 	for (i = 0; i < vsi->nb_qps; i++) {
1995 		I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1996 		I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1997 	}
1998 
1999 	/* VF bind interrupt */
2000 	if (vsi->type == I40E_VSI_SRIOV) {
2001 		__vsi_queues_bind_intr(vsi, msix_vect,
2002 				       vsi->base_queue, vsi->nb_qps,
2003 				       itr_idx);
2004 		return;
2005 	}
2006 
2007 	/* PF & VMDq bind interrupt */
2008 	if (rte_intr_dp_is_en(intr_handle)) {
2009 		if (vsi->type == I40E_VSI_MAIN) {
2010 			queue_idx = 0;
2011 			record = 1;
2012 		} else if (vsi->type == I40E_VSI_VMDQ2) {
2013 			struct i40e_vsi *main_vsi =
2014 				I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter);
2015 			queue_idx = vsi->base_queue - main_vsi->nb_qps;
2016 			record = 1;
2017 		}
2018 	}
2019 
2020 	for (i = 0; i < vsi->nb_used_qps; i++) {
2021 		if (nb_msix <= 1) {
2022 			if (!rte_intr_allow_others(intr_handle))
2023 				/* allow to share MISC_VEC_ID */
2024 				msix_vect = I40E_MISC_VEC_ID;
2025 
2026 			/* no enough msix_vect, map all to one */
2027 			__vsi_queues_bind_intr(vsi, msix_vect,
2028 					       vsi->base_queue + i,
2029 					       vsi->nb_used_qps - i,
2030 					       itr_idx);
2031 			for (; !!record && i < vsi->nb_used_qps; i++)
2032 				intr_handle->intr_vec[queue_idx + i] =
2033 					msix_vect;
2034 			break;
2035 		}
2036 		/* 1:1 queue/msix_vect mapping */
2037 		__vsi_queues_bind_intr(vsi, msix_vect,
2038 				       vsi->base_queue + i, 1,
2039 				       itr_idx);
2040 		if (!!record)
2041 			intr_handle->intr_vec[queue_idx + i] = msix_vect;
2042 
2043 		msix_vect++;
2044 		nb_msix--;
2045 	}
2046 }
2047 
2048 static void
2049 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
2050 {
2051 	struct rte_eth_dev *dev = vsi->adapter->eth_dev;
2052 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2053 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2054 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2055 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2056 	uint16_t msix_intr, i;
2057 
2058 	if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver)
2059 		for (i = 0; i < vsi->nb_msix; i++) {
2060 			msix_intr = vsi->msix_intr + i;
2061 			I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
2062 				I40E_PFINT_DYN_CTLN_INTENA_MASK |
2063 				I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2064 				I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
2065 		}
2066 	else
2067 		I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
2068 			       I40E_PFINT_DYN_CTL0_INTENA_MASK |
2069 			       I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2070 			       I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
2071 
2072 	I40E_WRITE_FLUSH(hw);
2073 }
2074 
2075 static void
2076 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
2077 {
2078 	struct rte_eth_dev *dev = vsi->adapter->eth_dev;
2079 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2080 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2081 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2082 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2083 	uint16_t msix_intr, i;
2084 
2085 	if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver)
2086 		for (i = 0; i < vsi->nb_msix; i++) {
2087 			msix_intr = vsi->msix_intr + i;
2088 			I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
2089 				       I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
2090 		}
2091 	else
2092 		I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
2093 			       I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
2094 
2095 	I40E_WRITE_FLUSH(hw);
2096 }
2097 
2098 static inline uint8_t
2099 i40e_parse_link_speeds(uint16_t link_speeds)
2100 {
2101 	uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
2102 
2103 	if (link_speeds & ETH_LINK_SPEED_40G)
2104 		link_speed |= I40E_LINK_SPEED_40GB;
2105 	if (link_speeds & ETH_LINK_SPEED_25G)
2106 		link_speed |= I40E_LINK_SPEED_25GB;
2107 	if (link_speeds & ETH_LINK_SPEED_20G)
2108 		link_speed |= I40E_LINK_SPEED_20GB;
2109 	if (link_speeds & ETH_LINK_SPEED_10G)
2110 		link_speed |= I40E_LINK_SPEED_10GB;
2111 	if (link_speeds & ETH_LINK_SPEED_1G)
2112 		link_speed |= I40E_LINK_SPEED_1GB;
2113 	if (link_speeds & ETH_LINK_SPEED_100M)
2114 		link_speed |= I40E_LINK_SPEED_100MB;
2115 
2116 	return link_speed;
2117 }
2118 
2119 static int
2120 i40e_phy_conf_link(struct i40e_hw *hw,
2121 		   uint8_t abilities,
2122 		   uint8_t force_speed,
2123 		   bool is_up)
2124 {
2125 	enum i40e_status_code status;
2126 	struct i40e_aq_get_phy_abilities_resp phy_ab;
2127 	struct i40e_aq_set_phy_config phy_conf;
2128 	enum i40e_aq_phy_type cnt;
2129 	uint8_t avail_speed;
2130 	uint32_t phy_type_mask = 0;
2131 
2132 	const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
2133 			I40E_AQ_PHY_FLAG_PAUSE_RX |
2134 			I40E_AQ_PHY_FLAG_PAUSE_RX |
2135 			I40E_AQ_PHY_FLAG_LOW_POWER;
2136 	int ret = -ENOTSUP;
2137 
2138 	/* To get phy capabilities of available speeds. */
2139 	status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab,
2140 					      NULL);
2141 	if (status) {
2142 		PMD_DRV_LOG(ERR, "Failed to get PHY capabilities: %d\n",
2143 				status);
2144 		return ret;
2145 	}
2146 	avail_speed = phy_ab.link_speed;
2147 
2148 	/* To get the current phy config. */
2149 	status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
2150 					      NULL);
2151 	if (status) {
2152 		PMD_DRV_LOG(ERR, "Failed to get the current PHY config: %d\n",
2153 				status);
2154 		return ret;
2155 	}
2156 
2157 	/* If link needs to go up and it is in autoneg mode the speed is OK,
2158 	 * no need to set up again.
2159 	 */
2160 	if (is_up && phy_ab.phy_type != 0 &&
2161 		     abilities & I40E_AQ_PHY_AN_ENABLED &&
2162 		     phy_ab.link_speed != 0)
2163 		return I40E_SUCCESS;
2164 
2165 	memset(&phy_conf, 0, sizeof(phy_conf));
2166 
2167 	/* bits 0-2 use the values from get_phy_abilities_resp */
2168 	abilities &= ~mask;
2169 	abilities |= phy_ab.abilities & mask;
2170 
2171 	phy_conf.abilities = abilities;
2172 
2173 	/* If link needs to go up, but the force speed is not supported,
2174 	 * Warn users and config the default available speeds.
2175 	 */
2176 	if (is_up && !(force_speed & avail_speed)) {
2177 		PMD_DRV_LOG(WARNING, "Invalid speed setting, set to default!\n");
2178 		phy_conf.link_speed = avail_speed;
2179 	} else {
2180 		phy_conf.link_speed = is_up ? force_speed : avail_speed;
2181 	}
2182 
2183 	/* PHY type mask needs to include each type except PHY type extension */
2184 	for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_25GBASE_KR; cnt++)
2185 		phy_type_mask |= 1 << cnt;
2186 
2187 	/* use get_phy_abilities_resp value for the rest */
2188 	phy_conf.phy_type = is_up ? cpu_to_le32(phy_type_mask) : 0;
2189 	phy_conf.phy_type_ext = is_up ? (I40E_AQ_PHY_TYPE_EXT_25G_KR |
2190 		I40E_AQ_PHY_TYPE_EXT_25G_CR | I40E_AQ_PHY_TYPE_EXT_25G_SR |
2191 		I40E_AQ_PHY_TYPE_EXT_25G_LR) : 0;
2192 	phy_conf.fec_config = phy_ab.fec_cfg_curr_mod_ext_info;
2193 	phy_conf.eee_capability = phy_ab.eee_capability;
2194 	phy_conf.eeer = phy_ab.eeer_val;
2195 	phy_conf.low_power_ctrl = phy_ab.d3_lpan;
2196 
2197 	PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
2198 		    phy_ab.abilities, phy_ab.link_speed);
2199 	PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
2200 		    phy_conf.abilities, phy_conf.link_speed);
2201 
2202 	status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
2203 	if (status)
2204 		return ret;
2205 
2206 	return I40E_SUCCESS;
2207 }
2208 
2209 static int
2210 i40e_apply_link_speed(struct rte_eth_dev *dev)
2211 {
2212 	uint8_t speed;
2213 	uint8_t abilities = 0;
2214 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2215 	struct rte_eth_conf *conf = &dev->data->dev_conf;
2216 
2217 	if (conf->link_speeds == ETH_LINK_SPEED_AUTONEG) {
2218 		conf->link_speeds = ETH_LINK_SPEED_40G |
2219 				    ETH_LINK_SPEED_25G |
2220 				    ETH_LINK_SPEED_20G |
2221 				    ETH_LINK_SPEED_10G |
2222 				    ETH_LINK_SPEED_1G |
2223 				    ETH_LINK_SPEED_100M;
2224 	}
2225 	speed = i40e_parse_link_speeds(conf->link_speeds);
2226 	abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK |
2227 		     I40E_AQ_PHY_AN_ENABLED |
2228 		     I40E_AQ_PHY_LINK_ENABLED;
2229 
2230 	return i40e_phy_conf_link(hw, abilities, speed, true);
2231 }
2232 
2233 static int
2234 i40e_dev_start(struct rte_eth_dev *dev)
2235 {
2236 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2237 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2238 	struct i40e_vsi *main_vsi = pf->main_vsi;
2239 	int ret, i;
2240 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2241 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2242 	uint32_t intr_vector = 0;
2243 	struct i40e_vsi *vsi;
2244 
2245 	hw->adapter_stopped = 0;
2246 
2247 	if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
2248 		PMD_INIT_LOG(ERR,
2249 		"Invalid link_speeds for port %u, autonegotiation disabled",
2250 			      dev->data->port_id);
2251 		return -EINVAL;
2252 	}
2253 
2254 	rte_intr_disable(intr_handle);
2255 
2256 	if ((rte_intr_cap_multiple(intr_handle) ||
2257 	     !RTE_ETH_DEV_SRIOV(dev).active) &&
2258 	    dev->data->dev_conf.intr_conf.rxq != 0) {
2259 		intr_vector = dev->data->nb_rx_queues;
2260 		ret = rte_intr_efd_enable(intr_handle, intr_vector);
2261 		if (ret)
2262 			return ret;
2263 	}
2264 
2265 	if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2266 		intr_handle->intr_vec =
2267 			rte_zmalloc("intr_vec",
2268 				    dev->data->nb_rx_queues * sizeof(int),
2269 				    0);
2270 		if (!intr_handle->intr_vec) {
2271 			PMD_INIT_LOG(ERR,
2272 				"Failed to allocate %d rx_queues intr_vec",
2273 				dev->data->nb_rx_queues);
2274 			return -ENOMEM;
2275 		}
2276 	}
2277 
2278 	/* Initialize VSI */
2279 	ret = i40e_dev_rxtx_init(pf);
2280 	if (ret != I40E_SUCCESS) {
2281 		PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
2282 		goto err_up;
2283 	}
2284 
2285 	/* Map queues with MSIX interrupt */
2286 	main_vsi->nb_used_qps = dev->data->nb_rx_queues -
2287 		pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
2288 	i40e_vsi_queues_bind_intr(main_vsi, I40E_ITR_INDEX_DEFAULT);
2289 	i40e_vsi_enable_queues_intr(main_vsi);
2290 
2291 	/* Map VMDQ VSI queues with MSIX interrupt */
2292 	for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
2293 		pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
2294 		i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi,
2295 					  I40E_ITR_INDEX_DEFAULT);
2296 		i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
2297 	}
2298 
2299 	/* enable FDIR MSIX interrupt */
2300 	if (pf->fdir.fdir_vsi) {
2301 		i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi,
2302 					  I40E_ITR_INDEX_NONE);
2303 		i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
2304 	}
2305 
2306 	/* Enable all queues which have been configured */
2307 	ret = i40e_dev_switch_queues(pf, TRUE);
2308 	if (ret != I40E_SUCCESS) {
2309 		PMD_DRV_LOG(ERR, "Failed to enable VSI");
2310 		goto err_up;
2311 	}
2312 
2313 	/* Enable receiving broadcast packets */
2314 	ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
2315 	if (ret != I40E_SUCCESS)
2316 		PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
2317 
2318 	for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
2319 		ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
2320 						true, NULL);
2321 		if (ret != I40E_SUCCESS)
2322 			PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
2323 	}
2324 
2325 	/* Enable the VLAN promiscuous mode. */
2326 	if (pf->vfs) {
2327 		for (i = 0; i < pf->vf_num; i++) {
2328 			vsi = pf->vfs[i].vsi;
2329 			i40e_aq_set_vsi_vlan_promisc(hw, vsi->seid,
2330 						     true, NULL);
2331 		}
2332 	}
2333 
2334 	/* Enable mac loopback mode */
2335 	if (dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_MODE_NONE ||
2336 	    dev->data->dev_conf.lpbk_mode == I40E_AQ_LB_PHY_LOCAL) {
2337 		ret = i40e_aq_set_lb_modes(hw, dev->data->dev_conf.lpbk_mode, NULL);
2338 		if (ret != I40E_SUCCESS) {
2339 			PMD_DRV_LOG(ERR, "fail to set loopback link");
2340 			goto err_up;
2341 		}
2342 	}
2343 
2344 	/* Apply link configure */
2345 	ret = i40e_apply_link_speed(dev);
2346 	if (I40E_SUCCESS != ret) {
2347 		PMD_DRV_LOG(ERR, "Fail to apply link setting");
2348 		goto err_up;
2349 	}
2350 
2351 	if (!rte_intr_allow_others(intr_handle)) {
2352 		rte_intr_callback_unregister(intr_handle,
2353 					     i40e_dev_interrupt_handler,
2354 					     (void *)dev);
2355 		/* configure and enable device interrupt */
2356 		i40e_pf_config_irq0(hw, FALSE);
2357 		i40e_pf_enable_irq0(hw);
2358 
2359 		if (dev->data->dev_conf.intr_conf.lsc != 0)
2360 			PMD_INIT_LOG(INFO,
2361 				"lsc won't enable because of no intr multiplex");
2362 	} else {
2363 		ret = i40e_aq_set_phy_int_mask(hw,
2364 					       ~(I40E_AQ_EVENT_LINK_UPDOWN |
2365 					       I40E_AQ_EVENT_MODULE_QUAL_FAIL |
2366 					       I40E_AQ_EVENT_MEDIA_NA), NULL);
2367 		if (ret != I40E_SUCCESS)
2368 			PMD_DRV_LOG(WARNING, "Fail to set phy mask");
2369 
2370 		/* Call get_link_info aq commond to enable/disable LSE */
2371 		i40e_dev_link_update(dev, 0);
2372 	}
2373 
2374 	if (dev->data->dev_conf.intr_conf.rxq == 0) {
2375 		rte_eal_alarm_set(I40E_ALARM_INTERVAL,
2376 				  i40e_dev_alarm_handler, dev);
2377 	} else {
2378 		/* enable uio intr after callback register */
2379 		rte_intr_enable(intr_handle);
2380 	}
2381 
2382 	i40e_filter_restore(pf);
2383 
2384 	if (pf->tm_conf.root && !pf->tm_conf.committed)
2385 		PMD_DRV_LOG(WARNING,
2386 			    "please call hierarchy_commit() "
2387 			    "before starting the port");
2388 
2389 	return I40E_SUCCESS;
2390 
2391 err_up:
2392 	i40e_dev_switch_queues(pf, FALSE);
2393 	i40e_dev_clear_queues(dev);
2394 
2395 	return ret;
2396 }
2397 
2398 static void
2399 i40e_dev_stop(struct rte_eth_dev *dev)
2400 {
2401 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2402 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2403 	struct i40e_vsi *main_vsi = pf->main_vsi;
2404 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2405 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2406 	int i;
2407 
2408 	if (hw->adapter_stopped == 1)
2409 		return;
2410 
2411 	if (dev->data->dev_conf.intr_conf.rxq == 0) {
2412 		rte_eal_alarm_cancel(i40e_dev_alarm_handler, dev);
2413 		rte_intr_enable(intr_handle);
2414 	}
2415 
2416 	/* Disable all queues */
2417 	i40e_dev_switch_queues(pf, FALSE);
2418 
2419 	/* un-map queues with interrupt registers */
2420 	i40e_vsi_disable_queues_intr(main_vsi);
2421 	i40e_vsi_queues_unbind_intr(main_vsi);
2422 
2423 	for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
2424 		i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
2425 		i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
2426 	}
2427 
2428 	if (pf->fdir.fdir_vsi) {
2429 		i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi);
2430 		i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi);
2431 	}
2432 	/* Clear all queues and release memory */
2433 	i40e_dev_clear_queues(dev);
2434 
2435 	/* Set link down */
2436 	i40e_dev_set_link_down(dev);
2437 
2438 	if (!rte_intr_allow_others(intr_handle))
2439 		/* resume to the default handler */
2440 		rte_intr_callback_register(intr_handle,
2441 					   i40e_dev_interrupt_handler,
2442 					   (void *)dev);
2443 
2444 	/* Clean datapath event and queue/vec mapping */
2445 	rte_intr_efd_disable(intr_handle);
2446 	if (intr_handle->intr_vec) {
2447 		rte_free(intr_handle->intr_vec);
2448 		intr_handle->intr_vec = NULL;
2449 	}
2450 
2451 	/* reset hierarchy commit */
2452 	pf->tm_conf.committed = false;
2453 
2454 	hw->adapter_stopped = 1;
2455 
2456 	pf->adapter->rss_reta_updated = 0;
2457 }
2458 
2459 static void
2460 i40e_dev_close(struct rte_eth_dev *dev)
2461 {
2462 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2463 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2464 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2465 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2466 	struct i40e_mirror_rule *p_mirror;
2467 	uint32_t reg;
2468 	int i;
2469 	int ret;
2470 
2471 	PMD_INIT_FUNC_TRACE();
2472 
2473 	i40e_dev_stop(dev);
2474 
2475 	/* Remove all mirror rules */
2476 	while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) {
2477 		ret = i40e_aq_del_mirror_rule(hw,
2478 					      pf->main_vsi->veb->seid,
2479 					      p_mirror->rule_type,
2480 					      p_mirror->entries,
2481 					      p_mirror->num_entries,
2482 					      p_mirror->id);
2483 		if (ret < 0)
2484 			PMD_DRV_LOG(ERR, "failed to remove mirror rule: "
2485 				    "status = %d, aq_err = %d.", ret,
2486 				    hw->aq.asq_last_status);
2487 
2488 		/* remove mirror software resource anyway */
2489 		TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules);
2490 		rte_free(p_mirror);
2491 		pf->nb_mirror_rule--;
2492 	}
2493 
2494 	i40e_dev_free_queues(dev);
2495 
2496 	/* Disable interrupt */
2497 	i40e_pf_disable_irq0(hw);
2498 	rte_intr_disable(intr_handle);
2499 
2500 	/*
2501 	 * Only legacy filter API needs the following fdir config. So when the
2502 	 * legacy filter API is deprecated, the following code should also be
2503 	 * removed.
2504 	 */
2505 	i40e_fdir_teardown(pf);
2506 
2507 	/* shutdown and destroy the HMC */
2508 	i40e_shutdown_lan_hmc(hw);
2509 
2510 	for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
2511 		i40e_vsi_release(pf->vmdq[i].vsi);
2512 		pf->vmdq[i].vsi = NULL;
2513 	}
2514 	rte_free(pf->vmdq);
2515 	pf->vmdq = NULL;
2516 
2517 	/* release all the existing VSIs and VEBs */
2518 	i40e_vsi_release(pf->main_vsi);
2519 
2520 	/* shutdown the adminq */
2521 	i40e_aq_queue_shutdown(hw, true);
2522 	i40e_shutdown_adminq(hw);
2523 
2524 	i40e_res_pool_destroy(&pf->qp_pool);
2525 	i40e_res_pool_destroy(&pf->msix_pool);
2526 
2527 	/* Disable flexible payload in global configuration */
2528 	if (!pf->support_multi_driver)
2529 		i40e_flex_payload_reg_set_default(hw);
2530 
2531 	/* force a PF reset to clean anything leftover */
2532 	reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
2533 	I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
2534 			(reg | I40E_PFGEN_CTRL_PFSWR_MASK));
2535 	I40E_WRITE_FLUSH(hw);
2536 
2537 	hw->adapter_closed = 1;
2538 }
2539 
2540 /*
2541  * Reset PF device only to re-initialize resources in PMD layer
2542  */
2543 static int
2544 i40e_dev_reset(struct rte_eth_dev *dev)
2545 {
2546 	int ret;
2547 
2548 	/* When a DPDK PMD PF begin to reset PF port, it should notify all
2549 	 * its VF to make them align with it. The detailed notification
2550 	 * mechanism is PMD specific. As to i40e PF, it is rather complex.
2551 	 * To avoid unexpected behavior in VF, currently reset of PF with
2552 	 * SR-IOV activation is not supported. It might be supported later.
2553 	 */
2554 	if (dev->data->sriov.active)
2555 		return -ENOTSUP;
2556 
2557 	ret = eth_i40e_dev_uninit(dev);
2558 	if (ret)
2559 		return ret;
2560 
2561 	ret = eth_i40e_dev_init(dev, NULL);
2562 
2563 	return ret;
2564 }
2565 
2566 static void
2567 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
2568 {
2569 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2570 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2571 	struct i40e_vsi *vsi = pf->main_vsi;
2572 	int status;
2573 
2574 	status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
2575 						     true, NULL, true);
2576 	if (status != I40E_SUCCESS)
2577 		PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
2578 
2579 	status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
2580 							TRUE, NULL);
2581 	if (status != I40E_SUCCESS)
2582 		PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
2583 
2584 }
2585 
2586 static void
2587 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
2588 {
2589 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2590 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2591 	struct i40e_vsi *vsi = pf->main_vsi;
2592 	int status;
2593 
2594 	status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
2595 						     false, NULL, true);
2596 	if (status != I40E_SUCCESS)
2597 		PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
2598 
2599 	/* must remain in all_multicast mode */
2600 	if (dev->data->all_multicast == 1)
2601 		return;
2602 
2603 	status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
2604 							false, NULL);
2605 	if (status != I40E_SUCCESS)
2606 		PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
2607 }
2608 
2609 static void
2610 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
2611 {
2612 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2613 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2614 	struct i40e_vsi *vsi = pf->main_vsi;
2615 	int ret;
2616 
2617 	ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
2618 	if (ret != I40E_SUCCESS)
2619 		PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
2620 }
2621 
2622 static void
2623 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
2624 {
2625 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2626 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2627 	struct i40e_vsi *vsi = pf->main_vsi;
2628 	int ret;
2629 
2630 	if (dev->data->promiscuous == 1)
2631 		return; /* must remain in all_multicast mode */
2632 
2633 	ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
2634 				vsi->seid, FALSE, NULL);
2635 	if (ret != I40E_SUCCESS)
2636 		PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
2637 }
2638 
2639 /*
2640  * Set device link up.
2641  */
2642 static int
2643 i40e_dev_set_link_up(struct rte_eth_dev *dev)
2644 {
2645 	/* re-apply link speed setting */
2646 	return i40e_apply_link_speed(dev);
2647 }
2648 
2649 /*
2650  * Set device link down.
2651  */
2652 static int
2653 i40e_dev_set_link_down(struct rte_eth_dev *dev)
2654 {
2655 	uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
2656 	uint8_t abilities = 0;
2657 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2658 
2659 	abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
2660 	return i40e_phy_conf_link(hw, abilities, speed, false);
2661 }
2662 
2663 static __rte_always_inline void
2664 update_link_reg(struct i40e_hw *hw, struct rte_eth_link *link)
2665 {
2666 /* Link status registers and values*/
2667 #define I40E_PRTMAC_LINKSTA		0x001E2420
2668 #define I40E_REG_LINK_UP		0x40000080
2669 #define I40E_PRTMAC_MACC		0x001E24E0
2670 #define I40E_REG_MACC_25GB		0x00020000
2671 #define I40E_REG_SPEED_MASK		0x38000000
2672 #define I40E_REG_SPEED_0		0x00000000
2673 #define I40E_REG_SPEED_1		0x08000000
2674 #define I40E_REG_SPEED_2		0x10000000
2675 #define I40E_REG_SPEED_3		0x18000000
2676 #define I40E_REG_SPEED_4		0x20000000
2677 	uint32_t link_speed;
2678 	uint32_t reg_val;
2679 
2680 	reg_val = I40E_READ_REG(hw, I40E_PRTMAC_LINKSTA);
2681 	link_speed = reg_val & I40E_REG_SPEED_MASK;
2682 	reg_val &= I40E_REG_LINK_UP;
2683 	link->link_status = (reg_val == I40E_REG_LINK_UP) ? 1 : 0;
2684 
2685 	if (unlikely(link->link_status == 0))
2686 		return;
2687 
2688 	/* Parse the link status */
2689 	switch (link_speed) {
2690 	case I40E_REG_SPEED_0:
2691 		link->link_speed = ETH_SPEED_NUM_100M;
2692 		break;
2693 	case I40E_REG_SPEED_1:
2694 		link->link_speed = ETH_SPEED_NUM_1G;
2695 		break;
2696 	case I40E_REG_SPEED_2:
2697 		if (hw->mac.type == I40E_MAC_X722)
2698 			link->link_speed = ETH_SPEED_NUM_2_5G;
2699 		else
2700 			link->link_speed = ETH_SPEED_NUM_10G;
2701 		break;
2702 	case I40E_REG_SPEED_3:
2703 		if (hw->mac.type == I40E_MAC_X722) {
2704 			link->link_speed = ETH_SPEED_NUM_5G;
2705 		} else {
2706 			reg_val = I40E_READ_REG(hw, I40E_PRTMAC_MACC);
2707 
2708 			if (reg_val & I40E_REG_MACC_25GB)
2709 				link->link_speed = ETH_SPEED_NUM_25G;
2710 			else
2711 				link->link_speed = ETH_SPEED_NUM_40G;
2712 		}
2713 		break;
2714 	case I40E_REG_SPEED_4:
2715 		if (hw->mac.type == I40E_MAC_X722)
2716 			link->link_speed = ETH_SPEED_NUM_10G;
2717 		else
2718 			link->link_speed = ETH_SPEED_NUM_20G;
2719 		break;
2720 	default:
2721 		PMD_DRV_LOG(ERR, "Unknown link speed info %u", link_speed);
2722 		break;
2723 	}
2724 }
2725 
2726 static __rte_always_inline void
2727 update_link_aq(struct i40e_hw *hw, struct rte_eth_link *link,
2728 	bool enable_lse, int wait_to_complete)
2729 {
2730 #define CHECK_INTERVAL             100  /* 100ms */
2731 #define MAX_REPEAT_TIME            10  /* 1s (10 * 100ms) in total */
2732 	uint32_t rep_cnt = MAX_REPEAT_TIME;
2733 	struct i40e_link_status link_status;
2734 	int status;
2735 
2736 	memset(&link_status, 0, sizeof(link_status));
2737 
2738 	do {
2739 		memset(&link_status, 0, sizeof(link_status));
2740 
2741 		/* Get link status information from hardware */
2742 		status = i40e_aq_get_link_info(hw, enable_lse,
2743 						&link_status, NULL);
2744 		if (unlikely(status != I40E_SUCCESS)) {
2745 			link->link_speed = ETH_SPEED_NUM_100M;
2746 			link->link_duplex = ETH_LINK_FULL_DUPLEX;
2747 			PMD_DRV_LOG(ERR, "Failed to get link info");
2748 			return;
2749 		}
2750 
2751 		link->link_status = link_status.link_info & I40E_AQ_LINK_UP;
2752 		if (!wait_to_complete || link->link_status)
2753 			break;
2754 
2755 		rte_delay_ms(CHECK_INTERVAL);
2756 	} while (--rep_cnt);
2757 
2758 	/* Parse the link status */
2759 	switch (link_status.link_speed) {
2760 	case I40E_LINK_SPEED_100MB:
2761 		link->link_speed = ETH_SPEED_NUM_100M;
2762 		break;
2763 	case I40E_LINK_SPEED_1GB:
2764 		link->link_speed = ETH_SPEED_NUM_1G;
2765 		break;
2766 	case I40E_LINK_SPEED_10GB:
2767 		link->link_speed = ETH_SPEED_NUM_10G;
2768 		break;
2769 	case I40E_LINK_SPEED_20GB:
2770 		link->link_speed = ETH_SPEED_NUM_20G;
2771 		break;
2772 	case I40E_LINK_SPEED_25GB:
2773 		link->link_speed = ETH_SPEED_NUM_25G;
2774 		break;
2775 	case I40E_LINK_SPEED_40GB:
2776 		link->link_speed = ETH_SPEED_NUM_40G;
2777 		break;
2778 	default:
2779 		link->link_speed = ETH_SPEED_NUM_100M;
2780 		break;
2781 	}
2782 }
2783 
2784 int
2785 i40e_dev_link_update(struct rte_eth_dev *dev,
2786 		     int wait_to_complete)
2787 {
2788 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2789 	struct rte_eth_link link;
2790 	bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
2791 	int ret;
2792 
2793 	memset(&link, 0, sizeof(link));
2794 
2795 	/* i40e uses full duplex only */
2796 	link.link_duplex = ETH_LINK_FULL_DUPLEX;
2797 	link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2798 			ETH_LINK_SPEED_FIXED);
2799 
2800 	if (!wait_to_complete && !enable_lse)
2801 		update_link_reg(hw, &link);
2802 	else
2803 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
2804 
2805 	ret = rte_eth_linkstatus_set(dev, &link);
2806 	i40e_notify_all_vfs_link_status(dev);
2807 
2808 	return ret;
2809 }
2810 
2811 /* Get all the statistics of a VSI */
2812 void
2813 i40e_update_vsi_stats(struct i40e_vsi *vsi)
2814 {
2815 	struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
2816 	struct i40e_eth_stats *nes = &vsi->eth_stats;
2817 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2818 	int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
2819 
2820 	i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
2821 			    vsi->offset_loaded, &oes->rx_bytes,
2822 			    &nes->rx_bytes);
2823 	i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
2824 			    vsi->offset_loaded, &oes->rx_unicast,
2825 			    &nes->rx_unicast);
2826 	i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
2827 			    vsi->offset_loaded, &oes->rx_multicast,
2828 			    &nes->rx_multicast);
2829 	i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
2830 			    vsi->offset_loaded, &oes->rx_broadcast,
2831 			    &nes->rx_broadcast);
2832 	/* exclude CRC bytes */
2833 	nes->rx_bytes -= (nes->rx_unicast + nes->rx_multicast +
2834 		nes->rx_broadcast) * ETHER_CRC_LEN;
2835 
2836 	i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
2837 			    &oes->rx_discards, &nes->rx_discards);
2838 	/* GLV_REPC not supported */
2839 	/* GLV_RMPC not supported */
2840 	i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
2841 			    &oes->rx_unknown_protocol,
2842 			    &nes->rx_unknown_protocol);
2843 	i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
2844 			    vsi->offset_loaded, &oes->tx_bytes,
2845 			    &nes->tx_bytes);
2846 	i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
2847 			    vsi->offset_loaded, &oes->tx_unicast,
2848 			    &nes->tx_unicast);
2849 	i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
2850 			    vsi->offset_loaded, &oes->tx_multicast,
2851 			    &nes->tx_multicast);
2852 	i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
2853 			    vsi->offset_loaded,  &oes->tx_broadcast,
2854 			    &nes->tx_broadcast);
2855 	/* GLV_TDPC not supported */
2856 	i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
2857 			    &oes->tx_errors, &nes->tx_errors);
2858 	vsi->offset_loaded = true;
2859 
2860 	PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
2861 		    vsi->vsi_id);
2862 	PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
2863 	PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
2864 	PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
2865 	PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
2866 	PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
2867 	PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2868 		    nes->rx_unknown_protocol);
2869 	PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
2870 	PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
2871 	PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
2872 	PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
2873 	PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
2874 	PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
2875 	PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
2876 		    vsi->vsi_id);
2877 }
2878 
2879 static void
2880 i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw)
2881 {
2882 	unsigned int i;
2883 	struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2884 	struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
2885 
2886 	/* Get rx/tx bytes of internal transfer packets */
2887 	i40e_stat_update_48(hw, I40E_GLV_GORCH(hw->port),
2888 			I40E_GLV_GORCL(hw->port),
2889 			pf->offset_loaded,
2890 			&pf->internal_stats_offset.rx_bytes,
2891 			&pf->internal_stats.rx_bytes);
2892 
2893 	i40e_stat_update_48(hw, I40E_GLV_GOTCH(hw->port),
2894 			I40E_GLV_GOTCL(hw->port),
2895 			pf->offset_loaded,
2896 			&pf->internal_stats_offset.tx_bytes,
2897 			&pf->internal_stats.tx_bytes);
2898 	/* Get total internal rx packet count */
2899 	i40e_stat_update_48(hw, I40E_GLV_UPRCH(hw->port),
2900 			    I40E_GLV_UPRCL(hw->port),
2901 			    pf->offset_loaded,
2902 			    &pf->internal_stats_offset.rx_unicast,
2903 			    &pf->internal_stats.rx_unicast);
2904 	i40e_stat_update_48(hw, I40E_GLV_MPRCH(hw->port),
2905 			    I40E_GLV_MPRCL(hw->port),
2906 			    pf->offset_loaded,
2907 			    &pf->internal_stats_offset.rx_multicast,
2908 			    &pf->internal_stats.rx_multicast);
2909 	i40e_stat_update_48(hw, I40E_GLV_BPRCH(hw->port),
2910 			    I40E_GLV_BPRCL(hw->port),
2911 			    pf->offset_loaded,
2912 			    &pf->internal_stats_offset.rx_broadcast,
2913 			    &pf->internal_stats.rx_broadcast);
2914 	/* Get total internal tx packet count */
2915 	i40e_stat_update_48(hw, I40E_GLV_UPTCH(hw->port),
2916 			    I40E_GLV_UPTCL(hw->port),
2917 			    pf->offset_loaded,
2918 			    &pf->internal_stats_offset.tx_unicast,
2919 			    &pf->internal_stats.tx_unicast);
2920 	i40e_stat_update_48(hw, I40E_GLV_MPTCH(hw->port),
2921 			    I40E_GLV_MPTCL(hw->port),
2922 			    pf->offset_loaded,
2923 			    &pf->internal_stats_offset.tx_multicast,
2924 			    &pf->internal_stats.tx_multicast);
2925 	i40e_stat_update_48(hw, I40E_GLV_BPTCH(hw->port),
2926 			    I40E_GLV_BPTCL(hw->port),
2927 			    pf->offset_loaded,
2928 			    &pf->internal_stats_offset.tx_broadcast,
2929 			    &pf->internal_stats.tx_broadcast);
2930 
2931 	/* exclude CRC size */
2932 	pf->internal_stats.rx_bytes -= (pf->internal_stats.rx_unicast +
2933 		pf->internal_stats.rx_multicast +
2934 		pf->internal_stats.rx_broadcast) * ETHER_CRC_LEN;
2935 
2936 	/* Get statistics of struct i40e_eth_stats */
2937 	i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
2938 			    I40E_GLPRT_GORCL(hw->port),
2939 			    pf->offset_loaded, &os->eth.rx_bytes,
2940 			    &ns->eth.rx_bytes);
2941 	i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
2942 			    I40E_GLPRT_UPRCL(hw->port),
2943 			    pf->offset_loaded, &os->eth.rx_unicast,
2944 			    &ns->eth.rx_unicast);
2945 	i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
2946 			    I40E_GLPRT_MPRCL(hw->port),
2947 			    pf->offset_loaded, &os->eth.rx_multicast,
2948 			    &ns->eth.rx_multicast);
2949 	i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
2950 			    I40E_GLPRT_BPRCL(hw->port),
2951 			    pf->offset_loaded, &os->eth.rx_broadcast,
2952 			    &ns->eth.rx_broadcast);
2953 	/* Workaround: CRC size should not be included in byte statistics,
2954 	 * so subtract ETHER_CRC_LEN from the byte counter for each rx packet.
2955 	 */
2956 	ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
2957 		ns->eth.rx_broadcast) * ETHER_CRC_LEN;
2958 
2959 	/* exclude internal rx bytes
2960 	 * Workaround: it is possible I40E_GLV_GORCH[H/L] is updated before
2961 	 * I40E_GLPRT_GORCH[H/L], so there is a small window that cause negative
2962 	 * value.
2963 	 * same to I40E_GLV_UPRC[H/L], I40E_GLV_MPRC[H/L], I40E_GLV_BPRC[H/L].
2964 	 */
2965 	if (ns->eth.rx_bytes < pf->internal_stats.rx_bytes)
2966 		ns->eth.rx_bytes = 0;
2967 	else
2968 		ns->eth.rx_bytes -= pf->internal_stats.rx_bytes;
2969 
2970 	if (ns->eth.rx_unicast < pf->internal_stats.rx_unicast)
2971 		ns->eth.rx_unicast = 0;
2972 	else
2973 		ns->eth.rx_unicast -= pf->internal_stats.rx_unicast;
2974 
2975 	if (ns->eth.rx_multicast < pf->internal_stats.rx_multicast)
2976 		ns->eth.rx_multicast = 0;
2977 	else
2978 		ns->eth.rx_multicast -= pf->internal_stats.rx_multicast;
2979 
2980 	if (ns->eth.rx_broadcast < pf->internal_stats.rx_broadcast)
2981 		ns->eth.rx_broadcast = 0;
2982 	else
2983 		ns->eth.rx_broadcast -= pf->internal_stats.rx_broadcast;
2984 
2985 	i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
2986 			    pf->offset_loaded, &os->eth.rx_discards,
2987 			    &ns->eth.rx_discards);
2988 	/* GLPRT_REPC not supported */
2989 	/* GLPRT_RMPC not supported */
2990 	i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
2991 			    pf->offset_loaded,
2992 			    &os->eth.rx_unknown_protocol,
2993 			    &ns->eth.rx_unknown_protocol);
2994 	i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
2995 			    I40E_GLPRT_GOTCL(hw->port),
2996 			    pf->offset_loaded, &os->eth.tx_bytes,
2997 			    &ns->eth.tx_bytes);
2998 	i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
2999 			    I40E_GLPRT_UPTCL(hw->port),
3000 			    pf->offset_loaded, &os->eth.tx_unicast,
3001 			    &ns->eth.tx_unicast);
3002 	i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
3003 			    I40E_GLPRT_MPTCL(hw->port),
3004 			    pf->offset_loaded, &os->eth.tx_multicast,
3005 			    &ns->eth.tx_multicast);
3006 	i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
3007 			    I40E_GLPRT_BPTCL(hw->port),
3008 			    pf->offset_loaded, &os->eth.tx_broadcast,
3009 			    &ns->eth.tx_broadcast);
3010 	ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
3011 		ns->eth.tx_broadcast) * ETHER_CRC_LEN;
3012 
3013 	/* exclude internal tx bytes
3014 	 * Workaround: it is possible I40E_GLV_GOTCH[H/L] is updated before
3015 	 * I40E_GLPRT_GOTCH[H/L], so there is a small window that cause negative
3016 	 * value.
3017 	 * same to I40E_GLV_UPTC[H/L], I40E_GLV_MPTC[H/L], I40E_GLV_BPTC[H/L].
3018 	 */
3019 	if (ns->eth.tx_bytes < pf->internal_stats.tx_bytes)
3020 		ns->eth.tx_bytes = 0;
3021 	else
3022 		ns->eth.tx_bytes -= pf->internal_stats.tx_bytes;
3023 
3024 	if (ns->eth.tx_unicast < pf->internal_stats.tx_unicast)
3025 		ns->eth.tx_unicast = 0;
3026 	else
3027 		ns->eth.tx_unicast -= pf->internal_stats.tx_unicast;
3028 
3029 	if (ns->eth.tx_multicast < pf->internal_stats.tx_multicast)
3030 		ns->eth.tx_multicast = 0;
3031 	else
3032 		ns->eth.tx_multicast -= pf->internal_stats.tx_multicast;
3033 
3034 	if (ns->eth.tx_broadcast < pf->internal_stats.tx_broadcast)
3035 		ns->eth.tx_broadcast = 0;
3036 	else
3037 		ns->eth.tx_broadcast -= pf->internal_stats.tx_broadcast;
3038 
3039 	/* GLPRT_TEPC not supported */
3040 
3041 	/* additional port specific stats */
3042 	i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
3043 			    pf->offset_loaded, &os->tx_dropped_link_down,
3044 			    &ns->tx_dropped_link_down);
3045 	i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
3046 			    pf->offset_loaded, &os->crc_errors,
3047 			    &ns->crc_errors);
3048 	i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
3049 			    pf->offset_loaded, &os->illegal_bytes,
3050 			    &ns->illegal_bytes);
3051 	/* GLPRT_ERRBC not supported */
3052 	i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
3053 			    pf->offset_loaded, &os->mac_local_faults,
3054 			    &ns->mac_local_faults);
3055 	i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
3056 			    pf->offset_loaded, &os->mac_remote_faults,
3057 			    &ns->mac_remote_faults);
3058 	i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
3059 			    pf->offset_loaded, &os->rx_length_errors,
3060 			    &ns->rx_length_errors);
3061 	i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
3062 			    pf->offset_loaded, &os->link_xon_rx,
3063 			    &ns->link_xon_rx);
3064 	i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
3065 			    pf->offset_loaded, &os->link_xoff_rx,
3066 			    &ns->link_xoff_rx);
3067 	for (i = 0; i < 8; i++) {
3068 		i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
3069 				    pf->offset_loaded,
3070 				    &os->priority_xon_rx[i],
3071 				    &ns->priority_xon_rx[i]);
3072 		i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
3073 				    pf->offset_loaded,
3074 				    &os->priority_xoff_rx[i],
3075 				    &ns->priority_xoff_rx[i]);
3076 	}
3077 	i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
3078 			    pf->offset_loaded, &os->link_xon_tx,
3079 			    &ns->link_xon_tx);
3080 	i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
3081 			    pf->offset_loaded, &os->link_xoff_tx,
3082 			    &ns->link_xoff_tx);
3083 	for (i = 0; i < 8; i++) {
3084 		i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
3085 				    pf->offset_loaded,
3086 				    &os->priority_xon_tx[i],
3087 				    &ns->priority_xon_tx[i]);
3088 		i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
3089 				    pf->offset_loaded,
3090 				    &os->priority_xoff_tx[i],
3091 				    &ns->priority_xoff_tx[i]);
3092 		i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
3093 				    pf->offset_loaded,
3094 				    &os->priority_xon_2_xoff[i],
3095 				    &ns->priority_xon_2_xoff[i]);
3096 	}
3097 	i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
3098 			    I40E_GLPRT_PRC64L(hw->port),
3099 			    pf->offset_loaded, &os->rx_size_64,
3100 			    &ns->rx_size_64);
3101 	i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
3102 			    I40E_GLPRT_PRC127L(hw->port),
3103 			    pf->offset_loaded, &os->rx_size_127,
3104 			    &ns->rx_size_127);
3105 	i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
3106 			    I40E_GLPRT_PRC255L(hw->port),
3107 			    pf->offset_loaded, &os->rx_size_255,
3108 			    &ns->rx_size_255);
3109 	i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
3110 			    I40E_GLPRT_PRC511L(hw->port),
3111 			    pf->offset_loaded, &os->rx_size_511,
3112 			    &ns->rx_size_511);
3113 	i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
3114 			    I40E_GLPRT_PRC1023L(hw->port),
3115 			    pf->offset_loaded, &os->rx_size_1023,
3116 			    &ns->rx_size_1023);
3117 	i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
3118 			    I40E_GLPRT_PRC1522L(hw->port),
3119 			    pf->offset_loaded, &os->rx_size_1522,
3120 			    &ns->rx_size_1522);
3121 	i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
3122 			    I40E_GLPRT_PRC9522L(hw->port),
3123 			    pf->offset_loaded, &os->rx_size_big,
3124 			    &ns->rx_size_big);
3125 	i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
3126 			    pf->offset_loaded, &os->rx_undersize,
3127 			    &ns->rx_undersize);
3128 	i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
3129 			    pf->offset_loaded, &os->rx_fragments,
3130 			    &ns->rx_fragments);
3131 	i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
3132 			    pf->offset_loaded, &os->rx_oversize,
3133 			    &ns->rx_oversize);
3134 	i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
3135 			    pf->offset_loaded, &os->rx_jabber,
3136 			    &ns->rx_jabber);
3137 	i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
3138 			    I40E_GLPRT_PTC64L(hw->port),
3139 			    pf->offset_loaded, &os->tx_size_64,
3140 			    &ns->tx_size_64);
3141 	i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
3142 			    I40E_GLPRT_PTC127L(hw->port),
3143 			    pf->offset_loaded, &os->tx_size_127,
3144 			    &ns->tx_size_127);
3145 	i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
3146 			    I40E_GLPRT_PTC255L(hw->port),
3147 			    pf->offset_loaded, &os->tx_size_255,
3148 			    &ns->tx_size_255);
3149 	i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
3150 			    I40E_GLPRT_PTC511L(hw->port),
3151 			    pf->offset_loaded, &os->tx_size_511,
3152 			    &ns->tx_size_511);
3153 	i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
3154 			    I40E_GLPRT_PTC1023L(hw->port),
3155 			    pf->offset_loaded, &os->tx_size_1023,
3156 			    &ns->tx_size_1023);
3157 	i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
3158 			    I40E_GLPRT_PTC1522L(hw->port),
3159 			    pf->offset_loaded, &os->tx_size_1522,
3160 			    &ns->tx_size_1522);
3161 	i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
3162 			    I40E_GLPRT_PTC9522L(hw->port),
3163 			    pf->offset_loaded, &os->tx_size_big,
3164 			    &ns->tx_size_big);
3165 	i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
3166 			   pf->offset_loaded,
3167 			   &os->fd_sb_match, &ns->fd_sb_match);
3168 	/* GLPRT_MSPDC not supported */
3169 	/* GLPRT_XEC not supported */
3170 
3171 	pf->offset_loaded = true;
3172 
3173 	if (pf->main_vsi)
3174 		i40e_update_vsi_stats(pf->main_vsi);
3175 }
3176 
3177 /* Get all statistics of a port */
3178 static int
3179 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
3180 {
3181 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3182 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3183 	struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
3184 	struct i40e_vsi *vsi;
3185 	unsigned i;
3186 
3187 	/* call read registers - updates values, now write them to struct */
3188 	i40e_read_stats_registers(pf, hw);
3189 
3190 	stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
3191 			pf->main_vsi->eth_stats.rx_multicast +
3192 			pf->main_vsi->eth_stats.rx_broadcast -
3193 			pf->main_vsi->eth_stats.rx_discards;
3194 	stats->opackets = ns->eth.tx_unicast +
3195 			ns->eth.tx_multicast +
3196 			ns->eth.tx_broadcast;
3197 	stats->ibytes   = pf->main_vsi->eth_stats.rx_bytes;
3198 	stats->obytes   = ns->eth.tx_bytes;
3199 	stats->oerrors  = ns->eth.tx_errors +
3200 			pf->main_vsi->eth_stats.tx_errors;
3201 
3202 	/* Rx Errors */
3203 	stats->imissed  = ns->eth.rx_discards +
3204 			pf->main_vsi->eth_stats.rx_discards;
3205 	stats->ierrors  = ns->crc_errors +
3206 			ns->rx_length_errors + ns->rx_undersize +
3207 			ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
3208 
3209 	if (pf->vfs) {
3210 		for (i = 0; i < pf->vf_num; i++) {
3211 			vsi = pf->vfs[i].vsi;
3212 			i40e_update_vsi_stats(vsi);
3213 
3214 			stats->ipackets += (vsi->eth_stats.rx_unicast +
3215 					vsi->eth_stats.rx_multicast +
3216 					vsi->eth_stats.rx_broadcast -
3217 					vsi->eth_stats.rx_discards);
3218 			stats->ibytes   += vsi->eth_stats.rx_bytes;
3219 			stats->oerrors  += vsi->eth_stats.tx_errors;
3220 			stats->imissed  += vsi->eth_stats.rx_discards;
3221 		}
3222 	}
3223 
3224 	PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
3225 	PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", ns->eth.rx_bytes);
3226 	PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", ns->eth.rx_unicast);
3227 	PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", ns->eth.rx_multicast);
3228 	PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", ns->eth.rx_broadcast);
3229 	PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", ns->eth.rx_discards);
3230 	PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
3231 		    ns->eth.rx_unknown_protocol);
3232 	PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", ns->eth.tx_bytes);
3233 	PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", ns->eth.tx_unicast);
3234 	PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", ns->eth.tx_multicast);
3235 	PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", ns->eth.tx_broadcast);
3236 	PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", ns->eth.tx_discards);
3237 	PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", ns->eth.tx_errors);
3238 
3239 	PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %"PRIu64"",
3240 		    ns->tx_dropped_link_down);
3241 	PMD_DRV_LOG(DEBUG, "crc_errors:               %"PRIu64"", ns->crc_errors);
3242 	PMD_DRV_LOG(DEBUG, "illegal_bytes:            %"PRIu64"",
3243 		    ns->illegal_bytes);
3244 	PMD_DRV_LOG(DEBUG, "error_bytes:              %"PRIu64"", ns->error_bytes);
3245 	PMD_DRV_LOG(DEBUG, "mac_local_faults:         %"PRIu64"",
3246 		    ns->mac_local_faults);
3247 	PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %"PRIu64"",
3248 		    ns->mac_remote_faults);
3249 	PMD_DRV_LOG(DEBUG, "rx_length_errors:         %"PRIu64"",
3250 		    ns->rx_length_errors);
3251 	PMD_DRV_LOG(DEBUG, "link_xon_rx:              %"PRIu64"", ns->link_xon_rx);
3252 	PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %"PRIu64"", ns->link_xoff_rx);
3253 	for (i = 0; i < 8; i++) {
3254 		PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %"PRIu64"",
3255 				i, ns->priority_xon_rx[i]);
3256 		PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %"PRIu64"",
3257 				i, ns->priority_xoff_rx[i]);
3258 	}
3259 	PMD_DRV_LOG(DEBUG, "link_xon_tx:              %"PRIu64"", ns->link_xon_tx);
3260 	PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %"PRIu64"", ns->link_xoff_tx);
3261 	for (i = 0; i < 8; i++) {
3262 		PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %"PRIu64"",
3263 				i, ns->priority_xon_tx[i]);
3264 		PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %"PRIu64"",
3265 				i, ns->priority_xoff_tx[i]);
3266 		PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %"PRIu64"",
3267 				i, ns->priority_xon_2_xoff[i]);
3268 	}
3269 	PMD_DRV_LOG(DEBUG, "rx_size_64:               %"PRIu64"", ns->rx_size_64);
3270 	PMD_DRV_LOG(DEBUG, "rx_size_127:              %"PRIu64"", ns->rx_size_127);
3271 	PMD_DRV_LOG(DEBUG, "rx_size_255:              %"PRIu64"", ns->rx_size_255);
3272 	PMD_DRV_LOG(DEBUG, "rx_size_511:              %"PRIu64"", ns->rx_size_511);
3273 	PMD_DRV_LOG(DEBUG, "rx_size_1023:             %"PRIu64"", ns->rx_size_1023);
3274 	PMD_DRV_LOG(DEBUG, "rx_size_1522:             %"PRIu64"", ns->rx_size_1522);
3275 	PMD_DRV_LOG(DEBUG, "rx_size_big:              %"PRIu64"", ns->rx_size_big);
3276 	PMD_DRV_LOG(DEBUG, "rx_undersize:             %"PRIu64"", ns->rx_undersize);
3277 	PMD_DRV_LOG(DEBUG, "rx_fragments:             %"PRIu64"", ns->rx_fragments);
3278 	PMD_DRV_LOG(DEBUG, "rx_oversize:              %"PRIu64"", ns->rx_oversize);
3279 	PMD_DRV_LOG(DEBUG, "rx_jabber:                %"PRIu64"", ns->rx_jabber);
3280 	PMD_DRV_LOG(DEBUG, "tx_size_64:               %"PRIu64"", ns->tx_size_64);
3281 	PMD_DRV_LOG(DEBUG, "tx_size_127:              %"PRIu64"", ns->tx_size_127);
3282 	PMD_DRV_LOG(DEBUG, "tx_size_255:              %"PRIu64"", ns->tx_size_255);
3283 	PMD_DRV_LOG(DEBUG, "tx_size_511:              %"PRIu64"", ns->tx_size_511);
3284 	PMD_DRV_LOG(DEBUG, "tx_size_1023:             %"PRIu64"", ns->tx_size_1023);
3285 	PMD_DRV_LOG(DEBUG, "tx_size_1522:             %"PRIu64"", ns->tx_size_1522);
3286 	PMD_DRV_LOG(DEBUG, "tx_size_big:              %"PRIu64"", ns->tx_size_big);
3287 	PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %"PRIu64"",
3288 			ns->mac_short_packet_dropped);
3289 	PMD_DRV_LOG(DEBUG, "checksum_error:           %"PRIu64"",
3290 		    ns->checksum_error);
3291 	PMD_DRV_LOG(DEBUG, "fdir_match:               %"PRIu64"", ns->fd_sb_match);
3292 	PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
3293 	return 0;
3294 }
3295 
3296 /* Reset the statistics */
3297 static void
3298 i40e_dev_stats_reset(struct rte_eth_dev *dev)
3299 {
3300 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3301 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3302 
3303 	/* Mark PF and VSI stats to update the offset, aka "reset" */
3304 	pf->offset_loaded = false;
3305 	if (pf->main_vsi)
3306 		pf->main_vsi->offset_loaded = false;
3307 
3308 	/* read the stats, reading current register values into offset */
3309 	i40e_read_stats_registers(pf, hw);
3310 }
3311 
3312 static uint32_t
3313 i40e_xstats_calc_num(void)
3314 {
3315 	return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS +
3316 		(I40E_NB_RXQ_PRIO_XSTATS * 8) +
3317 		(I40E_NB_TXQ_PRIO_XSTATS * 8);
3318 }
3319 
3320 static int i40e_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
3321 				     struct rte_eth_xstat_name *xstats_names,
3322 				     __rte_unused unsigned limit)
3323 {
3324 	unsigned count = 0;
3325 	unsigned i, prio;
3326 
3327 	if (xstats_names == NULL)
3328 		return i40e_xstats_calc_num();
3329 
3330 	/* Note: limit checked in rte_eth_xstats_names() */
3331 
3332 	/* Get stats from i40e_eth_stats struct */
3333 	for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
3334 		snprintf(xstats_names[count].name,
3335 			 sizeof(xstats_names[count].name),
3336 			 "%s", rte_i40e_stats_strings[i].name);
3337 		count++;
3338 	}
3339 
3340 	/* Get individiual stats from i40e_hw_port struct */
3341 	for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
3342 		snprintf(xstats_names[count].name,
3343 			sizeof(xstats_names[count].name),
3344 			 "%s", rte_i40e_hw_port_strings[i].name);
3345 		count++;
3346 	}
3347 
3348 	for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
3349 		for (prio = 0; prio < 8; prio++) {
3350 			snprintf(xstats_names[count].name,
3351 				 sizeof(xstats_names[count].name),
3352 				 "rx_priority%u_%s", prio,
3353 				 rte_i40e_rxq_prio_strings[i].name);
3354 			count++;
3355 		}
3356 	}
3357 
3358 	for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
3359 		for (prio = 0; prio < 8; prio++) {
3360 			snprintf(xstats_names[count].name,
3361 				 sizeof(xstats_names[count].name),
3362 				 "tx_priority%u_%s", prio,
3363 				 rte_i40e_txq_prio_strings[i].name);
3364 			count++;
3365 		}
3366 	}
3367 	return count;
3368 }
3369 
3370 static int
3371 i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
3372 		    unsigned n)
3373 {
3374 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3375 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3376 	unsigned i, count, prio;
3377 	struct i40e_hw_port_stats *hw_stats = &pf->stats;
3378 
3379 	count = i40e_xstats_calc_num();
3380 	if (n < count)
3381 		return count;
3382 
3383 	i40e_read_stats_registers(pf, hw);
3384 
3385 	if (xstats == NULL)
3386 		return 0;
3387 
3388 	count = 0;
3389 
3390 	/* Get stats from i40e_eth_stats struct */
3391 	for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
3392 		xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) +
3393 			rte_i40e_stats_strings[i].offset);
3394 		xstats[count].id = count;
3395 		count++;
3396 	}
3397 
3398 	/* Get individiual stats from i40e_hw_port struct */
3399 	for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
3400 		xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
3401 			rte_i40e_hw_port_strings[i].offset);
3402 		xstats[count].id = count;
3403 		count++;
3404 	}
3405 
3406 	for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
3407 		for (prio = 0; prio < 8; prio++) {
3408 			xstats[count].value =
3409 				*(uint64_t *)(((char *)hw_stats) +
3410 				rte_i40e_rxq_prio_strings[i].offset +
3411 				(sizeof(uint64_t) * prio));
3412 			xstats[count].id = count;
3413 			count++;
3414 		}
3415 	}
3416 
3417 	for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
3418 		for (prio = 0; prio < 8; prio++) {
3419 			xstats[count].value =
3420 				*(uint64_t *)(((char *)hw_stats) +
3421 				rte_i40e_txq_prio_strings[i].offset +
3422 				(sizeof(uint64_t) * prio));
3423 			xstats[count].id = count;
3424 			count++;
3425 		}
3426 	}
3427 
3428 	return count;
3429 }
3430 
3431 static int
3432 i40e_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
3433 {
3434 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3435 	u32 full_ver;
3436 	u8 ver, patch;
3437 	u16 build;
3438 	int ret;
3439 
3440 	full_ver = hw->nvm.oem_ver;
3441 	ver = (u8)(full_ver >> 24);
3442 	build = (u16)((full_ver >> 8) & 0xffff);
3443 	patch = (u8)(full_ver & 0xff);
3444 
3445 	ret = snprintf(fw_version, fw_size,
3446 		 "%d.%d%d 0x%08x %d.%d.%d",
3447 		 ((hw->nvm.version >> 12) & 0xf),
3448 		 ((hw->nvm.version >> 4) & 0xff),
3449 		 (hw->nvm.version & 0xf), hw->nvm.eetrack,
3450 		 ver, build, patch);
3451 
3452 	ret += 1; /* add the size of '\0' */
3453 	if (fw_size < (u32)ret)
3454 		return ret;
3455 	else
3456 		return 0;
3457 }
3458 
3459 /*
3460  * When using NVM 6.01(for X710 XL710 XXV710)/3.33(for X722) or later,
3461  * the Rx data path does not hang if the FW LLDP is stopped.
3462  * return true if lldp need to stop
3463  * return false if we cannot disable the LLDP to avoid Rx data path blocking.
3464  */
3465 static bool
3466 i40e_need_stop_lldp(struct rte_eth_dev *dev)
3467 {
3468 	double nvm_ver;
3469 	char ver_str[64] = {0};
3470 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3471 
3472 	i40e_fw_version_get(dev, ver_str, 64);
3473 	nvm_ver = atof(ver_str);
3474 	if ((hw->mac.type == I40E_MAC_X722 ||
3475 	     hw->mac.type == I40E_MAC_X722_VF) &&
3476 	     ((uint32_t)(nvm_ver * 1000) >= (uint32_t)(3.33 * 1000)))
3477 		return true;
3478 	else if ((uint32_t)(nvm_ver * 1000) >= (uint32_t)(6.01 * 1000))
3479 		return true;
3480 
3481 	return false;
3482 }
3483 
3484 static void
3485 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
3486 {
3487 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3488 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3489 	struct i40e_vsi *vsi = pf->main_vsi;
3490 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
3491 
3492 	dev_info->max_rx_queues = vsi->nb_qps;
3493 	dev_info->max_tx_queues = vsi->nb_qps;
3494 	dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
3495 	dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
3496 	dev_info->max_mac_addrs = vsi->max_macaddrs;
3497 	dev_info->max_vfs = pci_dev->max_vfs;
3498 	dev_info->rx_queue_offload_capa = 0;
3499 	dev_info->rx_offload_capa =
3500 		DEV_RX_OFFLOAD_VLAN_STRIP |
3501 		DEV_RX_OFFLOAD_QINQ_STRIP |
3502 		DEV_RX_OFFLOAD_IPV4_CKSUM |
3503 		DEV_RX_OFFLOAD_UDP_CKSUM |
3504 		DEV_RX_OFFLOAD_TCP_CKSUM |
3505 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
3506 		DEV_RX_OFFLOAD_KEEP_CRC |
3507 		DEV_RX_OFFLOAD_SCATTER |
3508 		DEV_RX_OFFLOAD_VLAN_EXTEND |
3509 		DEV_RX_OFFLOAD_VLAN_FILTER |
3510 		DEV_RX_OFFLOAD_JUMBO_FRAME;
3511 
3512 	dev_info->tx_queue_offload_capa = DEV_TX_OFFLOAD_MBUF_FAST_FREE;
3513 	dev_info->tx_offload_capa =
3514 		DEV_TX_OFFLOAD_VLAN_INSERT |
3515 		DEV_TX_OFFLOAD_QINQ_INSERT |
3516 		DEV_TX_OFFLOAD_IPV4_CKSUM |
3517 		DEV_TX_OFFLOAD_UDP_CKSUM |
3518 		DEV_TX_OFFLOAD_TCP_CKSUM |
3519 		DEV_TX_OFFLOAD_SCTP_CKSUM |
3520 		DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
3521 		DEV_TX_OFFLOAD_TCP_TSO |
3522 		DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
3523 		DEV_TX_OFFLOAD_GRE_TNL_TSO |
3524 		DEV_TX_OFFLOAD_IPIP_TNL_TSO |
3525 		DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
3526 		DEV_TX_OFFLOAD_MULTI_SEGS |
3527 		dev_info->tx_queue_offload_capa;
3528 	dev_info->dev_capa =
3529 		RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
3530 		RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
3531 
3532 	dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
3533 						sizeof(uint32_t);
3534 	dev_info->reta_size = pf->hash_lut_size;
3535 	dev_info->flow_type_rss_offloads = pf->adapter->flow_types_mask;
3536 
3537 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
3538 		.rx_thresh = {
3539 			.pthresh = I40E_DEFAULT_RX_PTHRESH,
3540 			.hthresh = I40E_DEFAULT_RX_HTHRESH,
3541 			.wthresh = I40E_DEFAULT_RX_WTHRESH,
3542 		},
3543 		.rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
3544 		.rx_drop_en = 0,
3545 		.offloads = 0,
3546 	};
3547 
3548 	dev_info->default_txconf = (struct rte_eth_txconf) {
3549 		.tx_thresh = {
3550 			.pthresh = I40E_DEFAULT_TX_PTHRESH,
3551 			.hthresh = I40E_DEFAULT_TX_HTHRESH,
3552 			.wthresh = I40E_DEFAULT_TX_WTHRESH,
3553 		},
3554 		.tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
3555 		.tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
3556 		.offloads = 0,
3557 	};
3558 
3559 	dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
3560 		.nb_max = I40E_MAX_RING_DESC,
3561 		.nb_min = I40E_MIN_RING_DESC,
3562 		.nb_align = I40E_ALIGN_RING_DESC,
3563 	};
3564 
3565 	dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
3566 		.nb_max = I40E_MAX_RING_DESC,
3567 		.nb_min = I40E_MIN_RING_DESC,
3568 		.nb_align = I40E_ALIGN_RING_DESC,
3569 		.nb_seg_max = I40E_TX_MAX_SEG,
3570 		.nb_mtu_seg_max = I40E_TX_MAX_MTU_SEG,
3571 	};
3572 
3573 	if (pf->flags & I40E_FLAG_VMDQ) {
3574 		dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
3575 		dev_info->vmdq_queue_base = dev_info->max_rx_queues;
3576 		dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
3577 						pf->max_nb_vmdq_vsi;
3578 		dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
3579 		dev_info->max_rx_queues += dev_info->vmdq_queue_num;
3580 		dev_info->max_tx_queues += dev_info->vmdq_queue_num;
3581 	}
3582 
3583 	if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) {
3584 		/* For XL710 */
3585 		dev_info->speed_capa = ETH_LINK_SPEED_40G;
3586 		dev_info->default_rxportconf.nb_queues = 2;
3587 		dev_info->default_txportconf.nb_queues = 2;
3588 		if (dev->data->nb_rx_queues == 1)
3589 			dev_info->default_rxportconf.ring_size = 2048;
3590 		else
3591 			dev_info->default_rxportconf.ring_size = 1024;
3592 		if (dev->data->nb_tx_queues == 1)
3593 			dev_info->default_txportconf.ring_size = 1024;
3594 		else
3595 			dev_info->default_txportconf.ring_size = 512;
3596 
3597 	} else if (I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types)) {
3598 		/* For XXV710 */
3599 		dev_info->speed_capa = ETH_LINK_SPEED_25G;
3600 		dev_info->default_rxportconf.nb_queues = 1;
3601 		dev_info->default_txportconf.nb_queues = 1;
3602 		dev_info->default_rxportconf.ring_size = 256;
3603 		dev_info->default_txportconf.ring_size = 256;
3604 	} else {
3605 		/* For X710 */
3606 		dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
3607 		dev_info->default_rxportconf.nb_queues = 1;
3608 		dev_info->default_txportconf.nb_queues = 1;
3609 		if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_10G) {
3610 			dev_info->default_rxportconf.ring_size = 512;
3611 			dev_info->default_txportconf.ring_size = 256;
3612 		} else {
3613 			dev_info->default_rxportconf.ring_size = 256;
3614 			dev_info->default_txportconf.ring_size = 256;
3615 		}
3616 	}
3617 	dev_info->default_rxportconf.burst_size = 32;
3618 	dev_info->default_txportconf.burst_size = 32;
3619 }
3620 
3621 static int
3622 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3623 {
3624 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3625 	struct i40e_vsi *vsi = pf->main_vsi;
3626 	PMD_INIT_FUNC_TRACE();
3627 
3628 	if (on)
3629 		return i40e_vsi_add_vlan(vsi, vlan_id);
3630 	else
3631 		return i40e_vsi_delete_vlan(vsi, vlan_id);
3632 }
3633 
3634 static int
3635 i40e_vlan_tpid_set_by_registers(struct rte_eth_dev *dev,
3636 				enum rte_vlan_type vlan_type,
3637 				uint16_t tpid, int qinq)
3638 {
3639 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3640 	uint64_t reg_r = 0;
3641 	uint64_t reg_w = 0;
3642 	uint16_t reg_id = 3;
3643 	int ret;
3644 
3645 	if (qinq) {
3646 		if (vlan_type == ETH_VLAN_TYPE_OUTER)
3647 			reg_id = 2;
3648 	}
3649 
3650 	ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
3651 					  &reg_r, NULL);
3652 	if (ret != I40E_SUCCESS) {
3653 		PMD_DRV_LOG(ERR,
3654 			   "Fail to debug read from I40E_GL_SWT_L2TAGCTRL[%d]",
3655 			   reg_id);
3656 		return -EIO;
3657 	}
3658 	PMD_DRV_LOG(DEBUG,
3659 		    "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: 0x%08"PRIx64,
3660 		    reg_id, reg_r);
3661 
3662 	reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK));
3663 	reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT);
3664 	if (reg_r == reg_w) {
3665 		PMD_DRV_LOG(DEBUG, "No need to write");
3666 		return 0;
3667 	}
3668 
3669 	ret = i40e_aq_debug_write_global_register(hw,
3670 					   I40E_GL_SWT_L2TAGCTRL(reg_id),
3671 					   reg_w, NULL);
3672 	if (ret != I40E_SUCCESS) {
3673 		PMD_DRV_LOG(ERR,
3674 			    "Fail to debug write to I40E_GL_SWT_L2TAGCTRL[%d]",
3675 			    reg_id);
3676 		return -EIO;
3677 	}
3678 	PMD_DRV_LOG(DEBUG,
3679 		    "Global register 0x%08x is changed with value 0x%08x",
3680 		    I40E_GL_SWT_L2TAGCTRL(reg_id), (uint32_t)reg_w);
3681 
3682 	return 0;
3683 }
3684 
3685 static int
3686 i40e_vlan_tpid_set(struct rte_eth_dev *dev,
3687 		   enum rte_vlan_type vlan_type,
3688 		   uint16_t tpid)
3689 {
3690 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3691 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3692 	int qinq = dev->data->dev_conf.rxmode.offloads &
3693 		   DEV_RX_OFFLOAD_VLAN_EXTEND;
3694 	int ret = 0;
3695 
3696 	if ((vlan_type != ETH_VLAN_TYPE_INNER &&
3697 	     vlan_type != ETH_VLAN_TYPE_OUTER) ||
3698 	    (!qinq && vlan_type == ETH_VLAN_TYPE_INNER)) {
3699 		PMD_DRV_LOG(ERR,
3700 			    "Unsupported vlan type.");
3701 		return -EINVAL;
3702 	}
3703 
3704 	if (pf->support_multi_driver) {
3705 		PMD_DRV_LOG(ERR, "Setting TPID is not supported.");
3706 		return -ENOTSUP;
3707 	}
3708 
3709 	/* 802.1ad frames ability is added in NVM API 1.7*/
3710 	if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) {
3711 		if (qinq) {
3712 			if (vlan_type == ETH_VLAN_TYPE_OUTER)
3713 				hw->first_tag = rte_cpu_to_le_16(tpid);
3714 			else if (vlan_type == ETH_VLAN_TYPE_INNER)
3715 				hw->second_tag = rte_cpu_to_le_16(tpid);
3716 		} else {
3717 			if (vlan_type == ETH_VLAN_TYPE_OUTER)
3718 				hw->second_tag = rte_cpu_to_le_16(tpid);
3719 		}
3720 		ret = i40e_aq_set_switch_config(hw, 0, 0, 0, NULL);
3721 		if (ret != I40E_SUCCESS) {
3722 			PMD_DRV_LOG(ERR,
3723 				    "Set switch config failed aq_err: %d",
3724 				    hw->aq.asq_last_status);
3725 			ret = -EIO;
3726 		}
3727 	} else
3728 		/* If NVM API < 1.7, keep the register setting */
3729 		ret = i40e_vlan_tpid_set_by_registers(dev, vlan_type,
3730 						      tpid, qinq);
3731 
3732 	return ret;
3733 }
3734 
3735 static int
3736 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3737 {
3738 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3739 	struct i40e_vsi *vsi = pf->main_vsi;
3740 	struct rte_eth_rxmode *rxmode;
3741 
3742 	rxmode = &dev->data->dev_conf.rxmode;
3743 	if (mask & ETH_VLAN_FILTER_MASK) {
3744 		if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
3745 			i40e_vsi_config_vlan_filter(vsi, TRUE);
3746 		else
3747 			i40e_vsi_config_vlan_filter(vsi, FALSE);
3748 	}
3749 
3750 	if (mask & ETH_VLAN_STRIP_MASK) {
3751 		/* Enable or disable VLAN stripping */
3752 		if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
3753 			i40e_vsi_config_vlan_stripping(vsi, TRUE);
3754 		else
3755 			i40e_vsi_config_vlan_stripping(vsi, FALSE);
3756 	}
3757 
3758 	if (mask & ETH_VLAN_EXTEND_MASK) {
3759 		if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) {
3760 			i40e_vsi_config_double_vlan(vsi, TRUE);
3761 			/* Set global registers with default ethertype. */
3762 			i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
3763 					   ETHER_TYPE_VLAN);
3764 			i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER,
3765 					   ETHER_TYPE_VLAN);
3766 		}
3767 		else
3768 			i40e_vsi_config_double_vlan(vsi, FALSE);
3769 	}
3770 
3771 	return 0;
3772 }
3773 
3774 static void
3775 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
3776 			  __rte_unused uint16_t queue,
3777 			  __rte_unused int on)
3778 {
3779 	PMD_INIT_FUNC_TRACE();
3780 }
3781 
3782 static int
3783 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
3784 {
3785 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3786 	struct i40e_vsi *vsi = pf->main_vsi;
3787 	struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
3788 	struct i40e_vsi_vlan_pvid_info info;
3789 
3790 	memset(&info, 0, sizeof(info));
3791 	info.on = on;
3792 	if (info.on)
3793 		info.config.pvid = pvid;
3794 	else {
3795 		info.config.reject.tagged =
3796 				data->dev_conf.txmode.hw_vlan_reject_tagged;
3797 		info.config.reject.untagged =
3798 				data->dev_conf.txmode.hw_vlan_reject_untagged;
3799 	}
3800 
3801 	return i40e_vsi_vlan_pvid_set(vsi, &info);
3802 }
3803 
3804 static int
3805 i40e_dev_led_on(struct rte_eth_dev *dev)
3806 {
3807 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3808 	uint32_t mode = i40e_led_get(hw);
3809 
3810 	if (mode == 0)
3811 		i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
3812 
3813 	return 0;
3814 }
3815 
3816 static int
3817 i40e_dev_led_off(struct rte_eth_dev *dev)
3818 {
3819 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3820 	uint32_t mode = i40e_led_get(hw);
3821 
3822 	if (mode != 0)
3823 		i40e_led_set(hw, 0, false);
3824 
3825 	return 0;
3826 }
3827 
3828 static int
3829 i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3830 {
3831 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3832 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3833 
3834 	fc_conf->pause_time = pf->fc_conf.pause_time;
3835 
3836 	/* read out from register, in case they are modified by other port */
3837 	pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] =
3838 		I40E_READ_REG(hw, I40E_GLRPB_GHW) >> I40E_KILOSHIFT;
3839 	pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] =
3840 		I40E_READ_REG(hw, I40E_GLRPB_GLW) >> I40E_KILOSHIFT;
3841 
3842 	fc_conf->high_water =  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS];
3843 	fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS];
3844 
3845 	 /* Return current mode according to actual setting*/
3846 	switch (hw->fc.current_mode) {
3847 	case I40E_FC_FULL:
3848 		fc_conf->mode = RTE_FC_FULL;
3849 		break;
3850 	case I40E_FC_TX_PAUSE:
3851 		fc_conf->mode = RTE_FC_TX_PAUSE;
3852 		break;
3853 	case I40E_FC_RX_PAUSE:
3854 		fc_conf->mode = RTE_FC_RX_PAUSE;
3855 		break;
3856 	case I40E_FC_NONE:
3857 	default:
3858 		fc_conf->mode = RTE_FC_NONE;
3859 	};
3860 
3861 	return 0;
3862 }
3863 
3864 static int
3865 i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3866 {
3867 	uint32_t mflcn_reg, fctrl_reg, reg;
3868 	uint32_t max_high_water;
3869 	uint8_t i, aq_failure;
3870 	int err;
3871 	struct i40e_hw *hw;
3872 	struct i40e_pf *pf;
3873 	enum i40e_fc_mode rte_fcmode_2_i40e_fcmode[] = {
3874 		[RTE_FC_NONE] = I40E_FC_NONE,
3875 		[RTE_FC_RX_PAUSE] = I40E_FC_RX_PAUSE,
3876 		[RTE_FC_TX_PAUSE] = I40E_FC_TX_PAUSE,
3877 		[RTE_FC_FULL] = I40E_FC_FULL
3878 	};
3879 
3880 	/* high_water field in the rte_eth_fc_conf using the kilobytes unit */
3881 
3882 	max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT;
3883 	if ((fc_conf->high_water > max_high_water) ||
3884 			(fc_conf->high_water < fc_conf->low_water)) {
3885 		PMD_INIT_LOG(ERR,
3886 			"Invalid high/low water setup value in KB, High_water must be <= %d.",
3887 			max_high_water);
3888 		return -EINVAL;
3889 	}
3890 
3891 	hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3892 	pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3893 	hw->fc.requested_mode = rte_fcmode_2_i40e_fcmode[fc_conf->mode];
3894 
3895 	pf->fc_conf.pause_time = fc_conf->pause_time;
3896 	pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->high_water;
3897 	pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->low_water;
3898 
3899 	PMD_INIT_FUNC_TRACE();
3900 
3901 	/* All the link flow control related enable/disable register
3902 	 * configuration is handle by the F/W
3903 	 */
3904 	err = i40e_set_fc(hw, &aq_failure, true);
3905 	if (err < 0)
3906 		return -ENOSYS;
3907 
3908 	if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) {
3909 		/* Configure flow control refresh threshold,
3910 		 * the value for stat_tx_pause_refresh_timer[8]
3911 		 * is used for global pause operation.
3912 		 */
3913 
3914 		I40E_WRITE_REG(hw,
3915 			       I40E_PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(8),
3916 			       pf->fc_conf.pause_time);
3917 
3918 		/* configure the timer value included in transmitted pause
3919 		 * frame,
3920 		 * the value for stat_tx_pause_quanta[8] is used for global
3921 		 * pause operation
3922 		 */
3923 		I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(8),
3924 			       pf->fc_conf.pause_time);
3925 
3926 		fctrl_reg = I40E_READ_REG(hw,
3927 					  I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL);
3928 
3929 		if (fc_conf->mac_ctrl_frame_fwd != 0)
3930 			fctrl_reg |= I40E_PRTMAC_FWD_CTRL;
3931 		else
3932 			fctrl_reg &= ~I40E_PRTMAC_FWD_CTRL;
3933 
3934 		I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL,
3935 			       fctrl_reg);
3936 	} else {
3937 		/* Configure pause time (2 TCs per register) */
3938 		reg = (uint32_t)pf->fc_conf.pause_time * (uint32_t)0x00010001;
3939 		for (i = 0; i < I40E_MAX_TRAFFIC_CLASS / 2; i++)
3940 			I40E_WRITE_REG(hw, I40E_PRTDCB_FCTTVN(i), reg);
3941 
3942 		/* Configure flow control refresh threshold value */
3943 		I40E_WRITE_REG(hw, I40E_PRTDCB_FCRTV,
3944 			       pf->fc_conf.pause_time / 2);
3945 
3946 		mflcn_reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
3947 
3948 		/* set or clear MFLCN.PMCF & MFLCN.DPF bits
3949 		 *depending on configuration
3950 		 */
3951 		if (fc_conf->mac_ctrl_frame_fwd != 0) {
3952 			mflcn_reg |= I40E_PRTDCB_MFLCN_PMCF_MASK;
3953 			mflcn_reg &= ~I40E_PRTDCB_MFLCN_DPF_MASK;
3954 		} else {
3955 			mflcn_reg &= ~I40E_PRTDCB_MFLCN_PMCF_MASK;
3956 			mflcn_reg |= I40E_PRTDCB_MFLCN_DPF_MASK;
3957 		}
3958 
3959 		I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg);
3960 	}
3961 
3962 	if (!pf->support_multi_driver) {
3963 		/* config water marker both based on the packets and bytes */
3964 		I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PHW,
3965 				 (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
3966 				 << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
3967 		I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PLW,
3968 				  (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
3969 				 << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
3970 		I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GHW,
3971 				  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
3972 				  << I40E_KILOSHIFT);
3973 		I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GLW,
3974 				   pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
3975 				   << I40E_KILOSHIFT);
3976 	} else {
3977 		PMD_DRV_LOG(ERR,
3978 			    "Water marker configuration is not supported.");
3979 	}
3980 
3981 	I40E_WRITE_FLUSH(hw);
3982 
3983 	return 0;
3984 }
3985 
3986 static int
3987 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
3988 			    __rte_unused struct rte_eth_pfc_conf *pfc_conf)
3989 {
3990 	PMD_INIT_FUNC_TRACE();
3991 
3992 	return -ENOSYS;
3993 }
3994 
3995 /* Add a MAC address, and update filters */
3996 static int
3997 i40e_macaddr_add(struct rte_eth_dev *dev,
3998 		 struct ether_addr *mac_addr,
3999 		 __rte_unused uint32_t index,
4000 		 uint32_t pool)
4001 {
4002 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4003 	struct i40e_mac_filter_info mac_filter;
4004 	struct i40e_vsi *vsi;
4005 	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
4006 	int ret;
4007 
4008 	/* If VMDQ not enabled or configured, return */
4009 	if (pool != 0 && (!(pf->flags & I40E_FLAG_VMDQ) ||
4010 			  !pf->nb_cfg_vmdq_vsi)) {
4011 		PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
4012 			pf->flags & I40E_FLAG_VMDQ ? "configured" : "enabled",
4013 			pool);
4014 		return -ENOTSUP;
4015 	}
4016 
4017 	if (pool > pf->nb_cfg_vmdq_vsi) {
4018 		PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
4019 				pool, pf->nb_cfg_vmdq_vsi);
4020 		return -EINVAL;
4021 	}
4022 
4023 	rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
4024 	if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
4025 		mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4026 	else
4027 		mac_filter.filter_type = RTE_MAC_PERFECT_MATCH;
4028 
4029 	if (pool == 0)
4030 		vsi = pf->main_vsi;
4031 	else
4032 		vsi = pf->vmdq[pool - 1].vsi;
4033 
4034 	ret = i40e_vsi_add_mac(vsi, &mac_filter);
4035 	if (ret != I40E_SUCCESS) {
4036 		PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
4037 		return -ENODEV;
4038 	}
4039 	return 0;
4040 }
4041 
4042 /* Remove a MAC address, and update filters */
4043 static void
4044 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
4045 {
4046 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4047 	struct i40e_vsi *vsi;
4048 	struct rte_eth_dev_data *data = dev->data;
4049 	struct ether_addr *macaddr;
4050 	int ret;
4051 	uint32_t i;
4052 	uint64_t pool_sel;
4053 
4054 	macaddr = &(data->mac_addrs[index]);
4055 
4056 	pool_sel = dev->data->mac_pool_sel[index];
4057 
4058 	for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
4059 		if (pool_sel & (1ULL << i)) {
4060 			if (i == 0)
4061 				vsi = pf->main_vsi;
4062 			else {
4063 				/* No VMDQ pool enabled or configured */
4064 				if (!(pf->flags & I40E_FLAG_VMDQ) ||
4065 					(i > pf->nb_cfg_vmdq_vsi)) {
4066 					PMD_DRV_LOG(ERR,
4067 						"No VMDQ pool enabled/configured");
4068 					return;
4069 				}
4070 				vsi = pf->vmdq[i - 1].vsi;
4071 			}
4072 			ret = i40e_vsi_delete_mac(vsi, macaddr);
4073 
4074 			if (ret) {
4075 				PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
4076 				return;
4077 			}
4078 		}
4079 	}
4080 }
4081 
4082 /* Set perfect match or hash match of MAC and VLAN for a VF */
4083 static int
4084 i40e_vf_mac_filter_set(struct i40e_pf *pf,
4085 		 struct rte_eth_mac_filter *filter,
4086 		 bool add)
4087 {
4088 	struct i40e_hw *hw;
4089 	struct i40e_mac_filter_info mac_filter;
4090 	struct ether_addr old_mac;
4091 	struct ether_addr *new_mac;
4092 	struct i40e_pf_vf *vf = NULL;
4093 	uint16_t vf_id;
4094 	int ret;
4095 
4096 	if (pf == NULL) {
4097 		PMD_DRV_LOG(ERR, "Invalid PF argument.");
4098 		return -EINVAL;
4099 	}
4100 	hw = I40E_PF_TO_HW(pf);
4101 
4102 	if (filter == NULL) {
4103 		PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
4104 		return -EINVAL;
4105 	}
4106 
4107 	new_mac = &filter->mac_addr;
4108 
4109 	if (is_zero_ether_addr(new_mac)) {
4110 		PMD_DRV_LOG(ERR, "Invalid ethernet address.");
4111 		return -EINVAL;
4112 	}
4113 
4114 	vf_id = filter->dst_id;
4115 
4116 	if (vf_id > pf->vf_num - 1 || !pf->vfs) {
4117 		PMD_DRV_LOG(ERR, "Invalid argument.");
4118 		return -EINVAL;
4119 	}
4120 	vf = &pf->vfs[vf_id];
4121 
4122 	if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
4123 		PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
4124 		return -EINVAL;
4125 	}
4126 
4127 	if (add) {
4128 		rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
4129 		rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
4130 				ETHER_ADDR_LEN);
4131 		rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
4132 				 ETHER_ADDR_LEN);
4133 
4134 		mac_filter.filter_type = filter->filter_type;
4135 		ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
4136 		if (ret != I40E_SUCCESS) {
4137 			PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
4138 			return -1;
4139 		}
4140 		ether_addr_copy(new_mac, &pf->dev_addr);
4141 	} else {
4142 		rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
4143 				ETHER_ADDR_LEN);
4144 		ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
4145 		if (ret != I40E_SUCCESS) {
4146 			PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
4147 			return -1;
4148 		}
4149 
4150 		/* Clear device address as it has been removed */
4151 		if (is_same_ether_addr(&(pf->dev_addr), new_mac))
4152 			memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
4153 	}
4154 
4155 	return 0;
4156 }
4157 
4158 /* MAC filter handle */
4159 static int
4160 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
4161 		void *arg)
4162 {
4163 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4164 	struct rte_eth_mac_filter *filter;
4165 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4166 	int ret = I40E_NOT_SUPPORTED;
4167 
4168 	filter = (struct rte_eth_mac_filter *)(arg);
4169 
4170 	switch (filter_op) {
4171 	case RTE_ETH_FILTER_NOP:
4172 		ret = I40E_SUCCESS;
4173 		break;
4174 	case RTE_ETH_FILTER_ADD:
4175 		i40e_pf_disable_irq0(hw);
4176 		if (filter->is_vf)
4177 			ret = i40e_vf_mac_filter_set(pf, filter, 1);
4178 		i40e_pf_enable_irq0(hw);
4179 		break;
4180 	case RTE_ETH_FILTER_DELETE:
4181 		i40e_pf_disable_irq0(hw);
4182 		if (filter->is_vf)
4183 			ret = i40e_vf_mac_filter_set(pf, filter, 0);
4184 		i40e_pf_enable_irq0(hw);
4185 		break;
4186 	default:
4187 		PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
4188 		ret = I40E_ERR_PARAM;
4189 		break;
4190 	}
4191 
4192 	return ret;
4193 }
4194 
4195 static int
4196 i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
4197 {
4198 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
4199 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4200 	uint32_t reg;
4201 	int ret;
4202 
4203 	if (!lut)
4204 		return -EINVAL;
4205 
4206 	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
4207 		ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id,
4208 					  vsi->type != I40E_VSI_SRIOV,
4209 					  lut, lut_size);
4210 		if (ret) {
4211 			PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
4212 			return ret;
4213 		}
4214 	} else {
4215 		uint32_t *lut_dw = (uint32_t *)lut;
4216 		uint16_t i, lut_size_dw = lut_size / 4;
4217 
4218 		if (vsi->type == I40E_VSI_SRIOV) {
4219 			for (i = 0; i <= lut_size_dw; i++) {
4220 				reg = I40E_VFQF_HLUT1(i, vsi->user_param);
4221 				lut_dw[i] = i40e_read_rx_ctl(hw, reg);
4222 			}
4223 		} else {
4224 			for (i = 0; i < lut_size_dw; i++)
4225 				lut_dw[i] = I40E_READ_REG(hw,
4226 							  I40E_PFQF_HLUT(i));
4227 		}
4228 	}
4229 
4230 	return 0;
4231 }
4232 
4233 int
4234 i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
4235 {
4236 	struct i40e_pf *pf;
4237 	struct i40e_hw *hw;
4238 	int ret;
4239 
4240 	if (!vsi || !lut)
4241 		return -EINVAL;
4242 
4243 	pf = I40E_VSI_TO_PF(vsi);
4244 	hw = I40E_VSI_TO_HW(vsi);
4245 
4246 	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
4247 		ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id,
4248 					  vsi->type != I40E_VSI_SRIOV,
4249 					  lut, lut_size);
4250 		if (ret) {
4251 			PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
4252 			return ret;
4253 		}
4254 	} else {
4255 		uint32_t *lut_dw = (uint32_t *)lut;
4256 		uint16_t i, lut_size_dw = lut_size / 4;
4257 
4258 		if (vsi->type == I40E_VSI_SRIOV) {
4259 			for (i = 0; i < lut_size_dw; i++)
4260 				I40E_WRITE_REG(
4261 					hw,
4262 					I40E_VFQF_HLUT1(i, vsi->user_param),
4263 					lut_dw[i]);
4264 		} else {
4265 			for (i = 0; i < lut_size_dw; i++)
4266 				I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i),
4267 					       lut_dw[i]);
4268 		}
4269 		I40E_WRITE_FLUSH(hw);
4270 	}
4271 
4272 	return 0;
4273 }
4274 
4275 static int
4276 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
4277 			 struct rte_eth_rss_reta_entry64 *reta_conf,
4278 			 uint16_t reta_size)
4279 {
4280 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4281 	uint16_t i, lut_size = pf->hash_lut_size;
4282 	uint16_t idx, shift;
4283 	uint8_t *lut;
4284 	int ret;
4285 
4286 	if (reta_size != lut_size ||
4287 		reta_size > ETH_RSS_RETA_SIZE_512) {
4288 		PMD_DRV_LOG(ERR,
4289 			"The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)",
4290 			reta_size, lut_size);
4291 		return -EINVAL;
4292 	}
4293 
4294 	lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
4295 	if (!lut) {
4296 		PMD_DRV_LOG(ERR, "No memory can be allocated");
4297 		return -ENOMEM;
4298 	}
4299 	ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
4300 	if (ret)
4301 		goto out;
4302 	for (i = 0; i < reta_size; i++) {
4303 		idx = i / RTE_RETA_GROUP_SIZE;
4304 		shift = i % RTE_RETA_GROUP_SIZE;
4305 		if (reta_conf[idx].mask & (1ULL << shift))
4306 			lut[i] = reta_conf[idx].reta[shift];
4307 	}
4308 	ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size);
4309 
4310 	pf->adapter->rss_reta_updated = 1;
4311 
4312 out:
4313 	rte_free(lut);
4314 
4315 	return ret;
4316 }
4317 
4318 static int
4319 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
4320 			struct rte_eth_rss_reta_entry64 *reta_conf,
4321 			uint16_t reta_size)
4322 {
4323 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4324 	uint16_t i, lut_size = pf->hash_lut_size;
4325 	uint16_t idx, shift;
4326 	uint8_t *lut;
4327 	int ret;
4328 
4329 	if (reta_size != lut_size ||
4330 		reta_size > ETH_RSS_RETA_SIZE_512) {
4331 		PMD_DRV_LOG(ERR,
4332 			"The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)",
4333 			reta_size, lut_size);
4334 		return -EINVAL;
4335 	}
4336 
4337 	lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
4338 	if (!lut) {
4339 		PMD_DRV_LOG(ERR, "No memory can be allocated");
4340 		return -ENOMEM;
4341 	}
4342 
4343 	ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
4344 	if (ret)
4345 		goto out;
4346 	for (i = 0; i < reta_size; i++) {
4347 		idx = i / RTE_RETA_GROUP_SIZE;
4348 		shift = i % RTE_RETA_GROUP_SIZE;
4349 		if (reta_conf[idx].mask & (1ULL << shift))
4350 			reta_conf[idx].reta[shift] = lut[i];
4351 	}
4352 
4353 out:
4354 	rte_free(lut);
4355 
4356 	return ret;
4357 }
4358 
4359 /**
4360  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
4361  * @hw:   pointer to the HW structure
4362  * @mem:  pointer to mem struct to fill out
4363  * @size: size of memory requested
4364  * @alignment: what to align the allocation to
4365  **/
4366 enum i40e_status_code
4367 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
4368 			struct i40e_dma_mem *mem,
4369 			u64 size,
4370 			u32 alignment)
4371 {
4372 	const struct rte_memzone *mz = NULL;
4373 	char z_name[RTE_MEMZONE_NAMESIZE];
4374 
4375 	if (!mem)
4376 		return I40E_ERR_PARAM;
4377 
4378 	snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
4379 	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY,
4380 			RTE_MEMZONE_IOVA_CONTIG, alignment, RTE_PGSIZE_2M);
4381 	if (!mz)
4382 		return I40E_ERR_NO_MEMORY;
4383 
4384 	mem->size = size;
4385 	mem->va = mz->addr;
4386 	mem->pa = mz->iova;
4387 	mem->zone = (const void *)mz;
4388 	PMD_DRV_LOG(DEBUG,
4389 		"memzone %s allocated with physical address: %"PRIu64,
4390 		mz->name, mem->pa);
4391 
4392 	return I40E_SUCCESS;
4393 }
4394 
4395 /**
4396  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
4397  * @hw:   pointer to the HW structure
4398  * @mem:  ptr to mem struct to free
4399  **/
4400 enum i40e_status_code
4401 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
4402 		    struct i40e_dma_mem *mem)
4403 {
4404 	if (!mem)
4405 		return I40E_ERR_PARAM;
4406 
4407 	PMD_DRV_LOG(DEBUG,
4408 		"memzone %s to be freed with physical address: %"PRIu64,
4409 		((const struct rte_memzone *)mem->zone)->name, mem->pa);
4410 	rte_memzone_free((const struct rte_memzone *)mem->zone);
4411 	mem->zone = NULL;
4412 	mem->va = NULL;
4413 	mem->pa = (u64)0;
4414 
4415 	return I40E_SUCCESS;
4416 }
4417 
4418 /**
4419  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
4420  * @hw:   pointer to the HW structure
4421  * @mem:  pointer to mem struct to fill out
4422  * @size: size of memory requested
4423  **/
4424 enum i40e_status_code
4425 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
4426 			 struct i40e_virt_mem *mem,
4427 			 u32 size)
4428 {
4429 	if (!mem)
4430 		return I40E_ERR_PARAM;
4431 
4432 	mem->size = size;
4433 	mem->va = rte_zmalloc("i40e", size, 0);
4434 
4435 	if (mem->va)
4436 		return I40E_SUCCESS;
4437 	else
4438 		return I40E_ERR_NO_MEMORY;
4439 }
4440 
4441 /**
4442  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
4443  * @hw:   pointer to the HW structure
4444  * @mem:  pointer to mem struct to free
4445  **/
4446 enum i40e_status_code
4447 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
4448 		     struct i40e_virt_mem *mem)
4449 {
4450 	if (!mem)
4451 		return I40E_ERR_PARAM;
4452 
4453 	rte_free(mem->va);
4454 	mem->va = NULL;
4455 
4456 	return I40E_SUCCESS;
4457 }
4458 
4459 void
4460 i40e_init_spinlock_d(struct i40e_spinlock *sp)
4461 {
4462 	rte_spinlock_init(&sp->spinlock);
4463 }
4464 
4465 void
4466 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
4467 {
4468 	rte_spinlock_lock(&sp->spinlock);
4469 }
4470 
4471 void
4472 i40e_release_spinlock_d(struct i40e_spinlock *sp)
4473 {
4474 	rte_spinlock_unlock(&sp->spinlock);
4475 }
4476 
4477 void
4478 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
4479 {
4480 	return;
4481 }
4482 
4483 /**
4484  * Get the hardware capabilities, which will be parsed
4485  * and saved into struct i40e_hw.
4486  */
4487 static int
4488 i40e_get_cap(struct i40e_hw *hw)
4489 {
4490 	struct i40e_aqc_list_capabilities_element_resp *buf;
4491 	uint16_t len, size = 0;
4492 	int ret;
4493 
4494 	/* Calculate a huge enough buff for saving response data temporarily */
4495 	len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
4496 						I40E_MAX_CAP_ELE_NUM;
4497 	buf = rte_zmalloc("i40e", len, 0);
4498 	if (!buf) {
4499 		PMD_DRV_LOG(ERR, "Failed to allocate memory");
4500 		return I40E_ERR_NO_MEMORY;
4501 	}
4502 
4503 	/* Get, parse the capabilities and save it to hw */
4504 	ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
4505 			i40e_aqc_opc_list_func_capabilities, NULL);
4506 	if (ret != I40E_SUCCESS)
4507 		PMD_DRV_LOG(ERR, "Failed to discover capabilities");
4508 
4509 	/* Free the temporary buffer after being used */
4510 	rte_free(buf);
4511 
4512 	return ret;
4513 }
4514 
4515 #define RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF	4
4516 
4517 static int i40e_pf_parse_vf_queue_number_handler(const char *key,
4518 		const char *value,
4519 		void *opaque)
4520 {
4521 	struct i40e_pf *pf;
4522 	unsigned long num;
4523 	char *end;
4524 
4525 	pf = (struct i40e_pf *)opaque;
4526 	RTE_SET_USED(key);
4527 
4528 	errno = 0;
4529 	num = strtoul(value, &end, 0);
4530 	if (errno != 0 || end == value || *end != 0) {
4531 		PMD_DRV_LOG(WARNING, "Wrong VF queue number = %s, Now it is "
4532 			    "kept the value = %hu", value, pf->vf_nb_qp_max);
4533 		return -(EINVAL);
4534 	}
4535 
4536 	if (num <= I40E_MAX_QP_NUM_PER_VF && rte_is_power_of_2(num))
4537 		pf->vf_nb_qp_max = (uint16_t)num;
4538 	else
4539 		/* here return 0 to make next valid same argument work */
4540 		PMD_DRV_LOG(WARNING, "Wrong VF queue number = %lu, it must be "
4541 			    "power of 2 and equal or less than 16 !, Now it is "
4542 			    "kept the value = %hu", num, pf->vf_nb_qp_max);
4543 
4544 	return 0;
4545 }
4546 
4547 static int i40e_pf_config_vf_rxq_number(struct rte_eth_dev *dev)
4548 {
4549 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4550 	struct rte_kvargs *kvlist;
4551 	int kvargs_count;
4552 
4553 	/* set default queue number per VF as 4 */
4554 	pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
4555 
4556 	if (dev->device->devargs == NULL)
4557 		return 0;
4558 
4559 	kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
4560 	if (kvlist == NULL)
4561 		return -(EINVAL);
4562 
4563 	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_QUEUE_NUM_PER_VF_ARG);
4564 	if (!kvargs_count) {
4565 		rte_kvargs_free(kvlist);
4566 		return 0;
4567 	}
4568 
4569 	if (kvargs_count > 1)
4570 		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
4571 			    "the first invalid or last valid one is used !",
4572 			    ETH_I40E_QUEUE_NUM_PER_VF_ARG);
4573 
4574 	rte_kvargs_process(kvlist, ETH_I40E_QUEUE_NUM_PER_VF_ARG,
4575 			   i40e_pf_parse_vf_queue_number_handler, pf);
4576 
4577 	rte_kvargs_free(kvlist);
4578 
4579 	return 0;
4580 }
4581 
4582 static int
4583 i40e_pf_parameter_init(struct rte_eth_dev *dev)
4584 {
4585 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4586 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4587 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
4588 	uint16_t qp_count = 0, vsi_count = 0;
4589 
4590 	if (pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
4591 		PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
4592 		return -EINVAL;
4593 	}
4594 
4595 	i40e_pf_config_vf_rxq_number(dev);
4596 
4597 	/* Add the parameter init for LFC */
4598 	pf->fc_conf.pause_time = I40E_DEFAULT_PAUSE_TIME;
4599 	pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_HIGH_WATER;
4600 	pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER;
4601 
4602 	pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
4603 	pf->max_num_vsi = hw->func_caps.num_vsis;
4604 	pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
4605 	pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
4606 
4607 	/* FDir queue/VSI allocation */
4608 	pf->fdir_qp_offset = 0;
4609 	if (hw->func_caps.fd) {
4610 		pf->flags |= I40E_FLAG_FDIR;
4611 		pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
4612 	} else {
4613 		pf->fdir_nb_qps = 0;
4614 	}
4615 	qp_count += pf->fdir_nb_qps;
4616 	vsi_count += 1;
4617 
4618 	/* LAN queue/VSI allocation */
4619 	pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps;
4620 	if (!hw->func_caps.rss) {
4621 		pf->lan_nb_qps = 1;
4622 	} else {
4623 		pf->flags |= I40E_FLAG_RSS;
4624 		if (hw->mac.type == I40E_MAC_X722)
4625 			pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE;
4626 		pf->lan_nb_qps = pf->lan_nb_qp_max;
4627 	}
4628 	qp_count += pf->lan_nb_qps;
4629 	vsi_count += 1;
4630 
4631 	/* VF queue/VSI allocation */
4632 	pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps;
4633 	if (hw->func_caps.sr_iov_1_1 && pci_dev->max_vfs) {
4634 		pf->flags |= I40E_FLAG_SRIOV;
4635 		pf->vf_nb_qps = pf->vf_nb_qp_max;
4636 		pf->vf_num = pci_dev->max_vfs;
4637 		PMD_DRV_LOG(DEBUG,
4638 			"%u VF VSIs, %u queues per VF VSI, in total %u queues",
4639 			pf->vf_num, pf->vf_nb_qps, pf->vf_nb_qps * pf->vf_num);
4640 	} else {
4641 		pf->vf_nb_qps = 0;
4642 		pf->vf_num = 0;
4643 	}
4644 	qp_count += pf->vf_nb_qps * pf->vf_num;
4645 	vsi_count += pf->vf_num;
4646 
4647 	/* VMDq queue/VSI allocation */
4648 	pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num;
4649 	pf->vmdq_nb_qps = 0;
4650 	pf->max_nb_vmdq_vsi = 0;
4651 	if (hw->func_caps.vmdq) {
4652 		if (qp_count < hw->func_caps.num_tx_qp &&
4653 			vsi_count < hw->func_caps.num_vsis) {
4654 			pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp -
4655 				qp_count) / pf->vmdq_nb_qp_max;
4656 
4657 			/* Limit the maximum number of VMDq vsi to the maximum
4658 			 * ethdev can support
4659 			 */
4660 			pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
4661 				hw->func_caps.num_vsis - vsi_count);
4662 			pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
4663 				ETH_64_POOLS);
4664 			if (pf->max_nb_vmdq_vsi) {
4665 				pf->flags |= I40E_FLAG_VMDQ;
4666 				pf->vmdq_nb_qps = pf->vmdq_nb_qp_max;
4667 				PMD_DRV_LOG(DEBUG,
4668 					"%u VMDQ VSIs, %u queues per VMDQ VSI, in total %u queues",
4669 					pf->max_nb_vmdq_vsi, pf->vmdq_nb_qps,
4670 					pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi);
4671 			} else {
4672 				PMD_DRV_LOG(INFO,
4673 					"No enough queues left for VMDq");
4674 			}
4675 		} else {
4676 			PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq");
4677 		}
4678 	}
4679 	qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
4680 	vsi_count += pf->max_nb_vmdq_vsi;
4681 
4682 	if (hw->func_caps.dcb)
4683 		pf->flags |= I40E_FLAG_DCB;
4684 
4685 	if (qp_count > hw->func_caps.num_tx_qp) {
4686 		PMD_DRV_LOG(ERR,
4687 			"Failed to allocate %u queues, which exceeds the hardware maximum %u",
4688 			qp_count, hw->func_caps.num_tx_qp);
4689 		return -EINVAL;
4690 	}
4691 	if (vsi_count > hw->func_caps.num_vsis) {
4692 		PMD_DRV_LOG(ERR,
4693 			"Failed to allocate %u VSIs, which exceeds the hardware maximum %u",
4694 			vsi_count, hw->func_caps.num_vsis);
4695 		return -EINVAL;
4696 	}
4697 
4698 	return 0;
4699 }
4700 
4701 static int
4702 i40e_pf_get_switch_config(struct i40e_pf *pf)
4703 {
4704 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4705 	struct i40e_aqc_get_switch_config_resp *switch_config;
4706 	struct i40e_aqc_switch_config_element_resp *element;
4707 	uint16_t start_seid = 0, num_reported;
4708 	int ret;
4709 
4710 	switch_config = (struct i40e_aqc_get_switch_config_resp *)\
4711 			rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
4712 	if (!switch_config) {
4713 		PMD_DRV_LOG(ERR, "Failed to allocated memory");
4714 		return -ENOMEM;
4715 	}
4716 
4717 	/* Get the switch configurations */
4718 	ret = i40e_aq_get_switch_config(hw, switch_config,
4719 		I40E_AQ_LARGE_BUF, &start_seid, NULL);
4720 	if (ret != I40E_SUCCESS) {
4721 		PMD_DRV_LOG(ERR, "Failed to get switch configurations");
4722 		goto fail;
4723 	}
4724 	num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
4725 	if (num_reported != 1) { /* The number should be 1 */
4726 		PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
4727 		goto fail;
4728 	}
4729 
4730 	/* Parse the switch configuration elements */
4731 	element = &(switch_config->element[0]);
4732 	if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
4733 		pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
4734 		pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
4735 	} else
4736 		PMD_DRV_LOG(INFO, "Unknown element type");
4737 
4738 fail:
4739 	rte_free(switch_config);
4740 
4741 	return ret;
4742 }
4743 
4744 static int
4745 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
4746 			uint32_t num)
4747 {
4748 	struct pool_entry *entry;
4749 
4750 	if (pool == NULL || num == 0)
4751 		return -EINVAL;
4752 
4753 	entry = rte_zmalloc("i40e", sizeof(*entry), 0);
4754 	if (entry == NULL) {
4755 		PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
4756 		return -ENOMEM;
4757 	}
4758 
4759 	/* queue heap initialize */
4760 	pool->num_free = num;
4761 	pool->num_alloc = 0;
4762 	pool->base = base;
4763 	LIST_INIT(&pool->alloc_list);
4764 	LIST_INIT(&pool->free_list);
4765 
4766 	/* Initialize element  */
4767 	entry->base = 0;
4768 	entry->len = num;
4769 
4770 	LIST_INSERT_HEAD(&pool->free_list, entry, next);
4771 	return 0;
4772 }
4773 
4774 static void
4775 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
4776 {
4777 	struct pool_entry *entry, *next_entry;
4778 
4779 	if (pool == NULL)
4780 		return;
4781 
4782 	for (entry = LIST_FIRST(&pool->alloc_list);
4783 			entry && (next_entry = LIST_NEXT(entry, next), 1);
4784 			entry = next_entry) {
4785 		LIST_REMOVE(entry, next);
4786 		rte_free(entry);
4787 	}
4788 
4789 	for (entry = LIST_FIRST(&pool->free_list);
4790 			entry && (next_entry = LIST_NEXT(entry, next), 1);
4791 			entry = next_entry) {
4792 		LIST_REMOVE(entry, next);
4793 		rte_free(entry);
4794 	}
4795 
4796 	pool->num_free = 0;
4797 	pool->num_alloc = 0;
4798 	pool->base = 0;
4799 	LIST_INIT(&pool->alloc_list);
4800 	LIST_INIT(&pool->free_list);
4801 }
4802 
4803 static int
4804 i40e_res_pool_free(struct i40e_res_pool_info *pool,
4805 		       uint32_t base)
4806 {
4807 	struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
4808 	uint32_t pool_offset;
4809 	int insert;
4810 
4811 	if (pool == NULL) {
4812 		PMD_DRV_LOG(ERR, "Invalid parameter");
4813 		return -EINVAL;
4814 	}
4815 
4816 	pool_offset = base - pool->base;
4817 	/* Lookup in alloc list */
4818 	LIST_FOREACH(entry, &pool->alloc_list, next) {
4819 		if (entry->base == pool_offset) {
4820 			valid_entry = entry;
4821 			LIST_REMOVE(entry, next);
4822 			break;
4823 		}
4824 	}
4825 
4826 	/* Not find, return */
4827 	if (valid_entry == NULL) {
4828 		PMD_DRV_LOG(ERR, "Failed to find entry");
4829 		return -EINVAL;
4830 	}
4831 
4832 	/**
4833 	 * Found it, move it to free list  and try to merge.
4834 	 * In order to make merge easier, always sort it by qbase.
4835 	 * Find adjacent prev and last entries.
4836 	 */
4837 	prev = next = NULL;
4838 	LIST_FOREACH(entry, &pool->free_list, next) {
4839 		if (entry->base > valid_entry->base) {
4840 			next = entry;
4841 			break;
4842 		}
4843 		prev = entry;
4844 	}
4845 
4846 	insert = 0;
4847 	/* Try to merge with next one*/
4848 	if (next != NULL) {
4849 		/* Merge with next one */
4850 		if (valid_entry->base + valid_entry->len == next->base) {
4851 			next->base = valid_entry->base;
4852 			next->len += valid_entry->len;
4853 			rte_free(valid_entry);
4854 			valid_entry = next;
4855 			insert = 1;
4856 		}
4857 	}
4858 
4859 	if (prev != NULL) {
4860 		/* Merge with previous one */
4861 		if (prev->base + prev->len == valid_entry->base) {
4862 			prev->len += valid_entry->len;
4863 			/* If it merge with next one, remove next node */
4864 			if (insert == 1) {
4865 				LIST_REMOVE(valid_entry, next);
4866 				rte_free(valid_entry);
4867 			} else {
4868 				rte_free(valid_entry);
4869 				insert = 1;
4870 			}
4871 		}
4872 	}
4873 
4874 	/* Not find any entry to merge, insert */
4875 	if (insert == 0) {
4876 		if (prev != NULL)
4877 			LIST_INSERT_AFTER(prev, valid_entry, next);
4878 		else if (next != NULL)
4879 			LIST_INSERT_BEFORE(next, valid_entry, next);
4880 		else /* It's empty list, insert to head */
4881 			LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
4882 	}
4883 
4884 	pool->num_free += valid_entry->len;
4885 	pool->num_alloc -= valid_entry->len;
4886 
4887 	return 0;
4888 }
4889 
4890 static int
4891 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
4892 		       uint16_t num)
4893 {
4894 	struct pool_entry *entry, *valid_entry;
4895 
4896 	if (pool == NULL || num == 0) {
4897 		PMD_DRV_LOG(ERR, "Invalid parameter");
4898 		return -EINVAL;
4899 	}
4900 
4901 	if (pool->num_free < num) {
4902 		PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
4903 			    num, pool->num_free);
4904 		return -ENOMEM;
4905 	}
4906 
4907 	valid_entry = NULL;
4908 	/* Lookup  in free list and find most fit one */
4909 	LIST_FOREACH(entry, &pool->free_list, next) {
4910 		if (entry->len >= num) {
4911 			/* Find best one */
4912 			if (entry->len == num) {
4913 				valid_entry = entry;
4914 				break;
4915 			}
4916 			if (valid_entry == NULL || valid_entry->len > entry->len)
4917 				valid_entry = entry;
4918 		}
4919 	}
4920 
4921 	/* Not find one to satisfy the request, return */
4922 	if (valid_entry == NULL) {
4923 		PMD_DRV_LOG(ERR, "No valid entry found");
4924 		return -ENOMEM;
4925 	}
4926 	/**
4927 	 * The entry have equal queue number as requested,
4928 	 * remove it from alloc_list.
4929 	 */
4930 	if (valid_entry->len == num) {
4931 		LIST_REMOVE(valid_entry, next);
4932 	} else {
4933 		/**
4934 		 * The entry have more numbers than requested,
4935 		 * create a new entry for alloc_list and minus its
4936 		 * queue base and number in free_list.
4937 		 */
4938 		entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
4939 		if (entry == NULL) {
4940 			PMD_DRV_LOG(ERR,
4941 				"Failed to allocate memory for resource pool");
4942 			return -ENOMEM;
4943 		}
4944 		entry->base = valid_entry->base;
4945 		entry->len = num;
4946 		valid_entry->base += num;
4947 		valid_entry->len -= num;
4948 		valid_entry = entry;
4949 	}
4950 
4951 	/* Insert it into alloc list, not sorted */
4952 	LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
4953 
4954 	pool->num_free -= valid_entry->len;
4955 	pool->num_alloc += valid_entry->len;
4956 
4957 	return valid_entry->base + pool->base;
4958 }
4959 
4960 /**
4961  * bitmap_is_subset - Check whether src2 is subset of src1
4962  **/
4963 static inline int
4964 bitmap_is_subset(uint8_t src1, uint8_t src2)
4965 {
4966 	return !((src1 ^ src2) & src2);
4967 }
4968 
4969 static enum i40e_status_code
4970 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
4971 {
4972 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4973 
4974 	/* If DCB is not supported, only default TC is supported */
4975 	if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
4976 		PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
4977 		return I40E_NOT_SUPPORTED;
4978 	}
4979 
4980 	if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
4981 		PMD_DRV_LOG(ERR,
4982 			"Enabled TC map 0x%x not applicable to HW support 0x%x",
4983 			hw->func_caps.enabled_tcmap, enabled_tcmap);
4984 		return I40E_NOT_SUPPORTED;
4985 	}
4986 	return I40E_SUCCESS;
4987 }
4988 
4989 int
4990 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
4991 				struct i40e_vsi_vlan_pvid_info *info)
4992 {
4993 	struct i40e_hw *hw;
4994 	struct i40e_vsi_context ctxt;
4995 	uint8_t vlan_flags = 0;
4996 	int ret;
4997 
4998 	if (vsi == NULL || info == NULL) {
4999 		PMD_DRV_LOG(ERR, "invalid parameters");
5000 		return I40E_ERR_PARAM;
5001 	}
5002 
5003 	if (info->on) {
5004 		vsi->info.pvid = info->config.pvid;
5005 		/**
5006 		 * If insert pvid is enabled, only tagged pkts are
5007 		 * allowed to be sent out.
5008 		 */
5009 		vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
5010 				I40E_AQ_VSI_PVLAN_MODE_TAGGED;
5011 	} else {
5012 		vsi->info.pvid = 0;
5013 		if (info->config.reject.tagged == 0)
5014 			vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
5015 
5016 		if (info->config.reject.untagged == 0)
5017 			vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
5018 	}
5019 	vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
5020 					I40E_AQ_VSI_PVLAN_MODE_MASK);
5021 	vsi->info.port_vlan_flags |= vlan_flags;
5022 	vsi->info.valid_sections =
5023 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5024 	memset(&ctxt, 0, sizeof(ctxt));
5025 	rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
5026 	ctxt.seid = vsi->seid;
5027 
5028 	hw = I40E_VSI_TO_HW(vsi);
5029 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5030 	if (ret != I40E_SUCCESS)
5031 		PMD_DRV_LOG(ERR, "Failed to update VSI params");
5032 
5033 	return ret;
5034 }
5035 
5036 static int
5037 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
5038 {
5039 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5040 	int i, ret;
5041 	struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
5042 
5043 	ret = validate_tcmap_parameter(vsi, enabled_tcmap);
5044 	if (ret != I40E_SUCCESS)
5045 		return ret;
5046 
5047 	if (!vsi->seid) {
5048 		PMD_DRV_LOG(ERR, "seid not valid");
5049 		return -EINVAL;
5050 	}
5051 
5052 	memset(&tc_bw_data, 0, sizeof(tc_bw_data));
5053 	tc_bw_data.tc_valid_bits = enabled_tcmap;
5054 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5055 		tc_bw_data.tc_bw_credits[i] =
5056 			(enabled_tcmap & (1 << i)) ? 1 : 0;
5057 
5058 	ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
5059 	if (ret != I40E_SUCCESS) {
5060 		PMD_DRV_LOG(ERR, "Failed to configure TC BW");
5061 		return ret;
5062 	}
5063 
5064 	rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
5065 					sizeof(vsi->info.qs_handle));
5066 	return I40E_SUCCESS;
5067 }
5068 
5069 static enum i40e_status_code
5070 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
5071 				 struct i40e_aqc_vsi_properties_data *info,
5072 				 uint8_t enabled_tcmap)
5073 {
5074 	enum i40e_status_code ret;
5075 	int i, total_tc = 0;
5076 	uint16_t qpnum_per_tc, bsf, qp_idx;
5077 
5078 	ret = validate_tcmap_parameter(vsi, enabled_tcmap);
5079 	if (ret != I40E_SUCCESS)
5080 		return ret;
5081 
5082 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5083 		if (enabled_tcmap & (1 << i))
5084 			total_tc++;
5085 	if (total_tc == 0)
5086 		total_tc = 1;
5087 	vsi->enabled_tc = enabled_tcmap;
5088 
5089 	/* Number of queues per enabled TC */
5090 	qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
5091 	qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
5092 	bsf = rte_bsf32(qpnum_per_tc);
5093 
5094 	/* Adjust the queue number to actual queues that can be applied */
5095 	if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1))
5096 		vsi->nb_qps = qpnum_per_tc * total_tc;
5097 
5098 	/**
5099 	 * Configure TC and queue mapping parameters, for enabled TC,
5100 	 * allocate qpnum_per_tc queues to this traffic. For disabled TC,
5101 	 * default queue will serve it.
5102 	 */
5103 	qp_idx = 0;
5104 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5105 		if (vsi->enabled_tc & (1 << i)) {
5106 			info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
5107 					I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5108 				(bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5109 			qp_idx += qpnum_per_tc;
5110 		} else
5111 			info->tc_mapping[i] = 0;
5112 	}
5113 
5114 	/* Associate queue number with VSI */
5115 	if (vsi->type == I40E_VSI_SRIOV) {
5116 		info->mapping_flags |=
5117 			rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5118 		for (i = 0; i < vsi->nb_qps; i++)
5119 			info->queue_mapping[i] =
5120 				rte_cpu_to_le_16(vsi->base_queue + i);
5121 	} else {
5122 		info->mapping_flags |=
5123 			rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5124 		info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
5125 	}
5126 	info->valid_sections |=
5127 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5128 
5129 	return I40E_SUCCESS;
5130 }
5131 
5132 static int
5133 i40e_veb_release(struct i40e_veb *veb)
5134 {
5135 	struct i40e_vsi *vsi;
5136 	struct i40e_hw *hw;
5137 
5138 	if (veb == NULL)
5139 		return -EINVAL;
5140 
5141 	if (!TAILQ_EMPTY(&veb->head)) {
5142 		PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
5143 		return -EACCES;
5144 	}
5145 	/* associate_vsi field is NULL for floating VEB */
5146 	if (veb->associate_vsi != NULL) {
5147 		vsi = veb->associate_vsi;
5148 		hw = I40E_VSI_TO_HW(vsi);
5149 
5150 		vsi->uplink_seid = veb->uplink_seid;
5151 		vsi->veb = NULL;
5152 	} else {
5153 		veb->associate_pf->main_vsi->floating_veb = NULL;
5154 		hw = I40E_VSI_TO_HW(veb->associate_pf->main_vsi);
5155 	}
5156 
5157 	i40e_aq_delete_element(hw, veb->seid, NULL);
5158 	rte_free(veb);
5159 	return I40E_SUCCESS;
5160 }
5161 
5162 /* Setup a veb */
5163 static struct i40e_veb *
5164 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
5165 {
5166 	struct i40e_veb *veb;
5167 	int ret;
5168 	struct i40e_hw *hw;
5169 
5170 	if (pf == NULL) {
5171 		PMD_DRV_LOG(ERR,
5172 			    "veb setup failed, associated PF shouldn't null");
5173 		return NULL;
5174 	}
5175 	hw = I40E_PF_TO_HW(pf);
5176 
5177 	veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
5178 	if (!veb) {
5179 		PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
5180 		goto fail;
5181 	}
5182 
5183 	veb->associate_vsi = vsi;
5184 	veb->associate_pf = pf;
5185 	TAILQ_INIT(&veb->head);
5186 	veb->uplink_seid = vsi ? vsi->uplink_seid : 0;
5187 
5188 	/* create floating veb if vsi is NULL */
5189 	if (vsi != NULL) {
5190 		ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
5191 				      I40E_DEFAULT_TCMAP, false,
5192 				      &veb->seid, false, NULL);
5193 	} else {
5194 		ret = i40e_aq_add_veb(hw, 0, 0, I40E_DEFAULT_TCMAP,
5195 				      true, &veb->seid, false, NULL);
5196 	}
5197 
5198 	if (ret != I40E_SUCCESS) {
5199 		PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
5200 			    hw->aq.asq_last_status);
5201 		goto fail;
5202 	}
5203 	veb->enabled_tc = I40E_DEFAULT_TCMAP;
5204 
5205 	/* get statistics index */
5206 	ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
5207 				&veb->stats_idx, NULL, NULL, NULL);
5208 	if (ret != I40E_SUCCESS) {
5209 		PMD_DRV_LOG(ERR, "Get veb statistics index failed, aq_err: %d",
5210 			    hw->aq.asq_last_status);
5211 		goto fail;
5212 	}
5213 	/* Get VEB bandwidth, to be implemented */
5214 	/* Now associated vsi binding to the VEB, set uplink to this VEB */
5215 	if (vsi)
5216 		vsi->uplink_seid = veb->seid;
5217 
5218 	return veb;
5219 fail:
5220 	rte_free(veb);
5221 	return NULL;
5222 }
5223 
5224 int
5225 i40e_vsi_release(struct i40e_vsi *vsi)
5226 {
5227 	struct i40e_pf *pf;
5228 	struct i40e_hw *hw;
5229 	struct i40e_vsi_list *vsi_list;
5230 	void *temp;
5231 	int ret;
5232 	struct i40e_mac_filter *f;
5233 	uint16_t user_param;
5234 
5235 	if (!vsi)
5236 		return I40E_SUCCESS;
5237 
5238 	if (!vsi->adapter)
5239 		return -EFAULT;
5240 
5241 	user_param = vsi->user_param;
5242 
5243 	pf = I40E_VSI_TO_PF(vsi);
5244 	hw = I40E_VSI_TO_HW(vsi);
5245 
5246 	/* VSI has child to attach, release child first */
5247 	if (vsi->veb) {
5248 		TAILQ_FOREACH_SAFE(vsi_list, &vsi->veb->head, list, temp) {
5249 			if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
5250 				return -1;
5251 		}
5252 		i40e_veb_release(vsi->veb);
5253 	}
5254 
5255 	if (vsi->floating_veb) {
5256 		TAILQ_FOREACH_SAFE(vsi_list, &vsi->floating_veb->head, list, temp) {
5257 			if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
5258 				return -1;
5259 		}
5260 	}
5261 
5262 	/* Remove all macvlan filters of the VSI */
5263 	i40e_vsi_remove_all_macvlan_filter(vsi);
5264 	TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp)
5265 		rte_free(f);
5266 
5267 	if (vsi->type != I40E_VSI_MAIN &&
5268 	    ((vsi->type != I40E_VSI_SRIOV) ||
5269 	    !pf->floating_veb_list[user_param])) {
5270 		/* Remove vsi from parent's sibling list */
5271 		if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
5272 			PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
5273 			return I40E_ERR_PARAM;
5274 		}
5275 		TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
5276 				&vsi->sib_vsi_list, list);
5277 
5278 		/* Remove all switch element of the VSI */
5279 		ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
5280 		if (ret != I40E_SUCCESS)
5281 			PMD_DRV_LOG(ERR, "Failed to delete element");
5282 	}
5283 
5284 	if ((vsi->type == I40E_VSI_SRIOV) &&
5285 	    pf->floating_veb_list[user_param]) {
5286 		/* Remove vsi from parent's sibling list */
5287 		if (vsi->parent_vsi == NULL ||
5288 		    vsi->parent_vsi->floating_veb == NULL) {
5289 			PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
5290 			return I40E_ERR_PARAM;
5291 		}
5292 		TAILQ_REMOVE(&vsi->parent_vsi->floating_veb->head,
5293 			     &vsi->sib_vsi_list, list);
5294 
5295 		/* Remove all switch element of the VSI */
5296 		ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
5297 		if (ret != I40E_SUCCESS)
5298 			PMD_DRV_LOG(ERR, "Failed to delete element");
5299 	}
5300 
5301 	i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
5302 
5303 	if (vsi->type != I40E_VSI_SRIOV)
5304 		i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
5305 	rte_free(vsi);
5306 
5307 	return I40E_SUCCESS;
5308 }
5309 
5310 static int
5311 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
5312 {
5313 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5314 	struct i40e_aqc_remove_macvlan_element_data def_filter;
5315 	struct i40e_mac_filter_info filter;
5316 	int ret;
5317 
5318 	if (vsi->type != I40E_VSI_MAIN)
5319 		return I40E_ERR_CONFIG;
5320 	memset(&def_filter, 0, sizeof(def_filter));
5321 	rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
5322 					ETH_ADDR_LEN);
5323 	def_filter.vlan_tag = 0;
5324 	def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
5325 				I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5326 	ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
5327 	if (ret != I40E_SUCCESS) {
5328 		struct i40e_mac_filter *f;
5329 		struct ether_addr *mac;
5330 
5331 		PMD_DRV_LOG(DEBUG,
5332 			    "Cannot remove the default macvlan filter");
5333 		/* It needs to add the permanent mac into mac list */
5334 		f = rte_zmalloc("macv_filter", sizeof(*f), 0);
5335 		if (f == NULL) {
5336 			PMD_DRV_LOG(ERR, "failed to allocate memory");
5337 			return I40E_ERR_NO_MEMORY;
5338 		}
5339 		mac = &f->mac_info.mac_addr;
5340 		rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
5341 				ETH_ADDR_LEN);
5342 		f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
5343 		TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
5344 		vsi->mac_num++;
5345 
5346 		return ret;
5347 	}
5348 	rte_memcpy(&filter.mac_addr,
5349 		(struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
5350 	filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
5351 	return i40e_vsi_add_mac(vsi, &filter);
5352 }
5353 
5354 /*
5355  * i40e_vsi_get_bw_config - Query VSI BW Information
5356  * @vsi: the VSI to be queried
5357  *
5358  * Returns 0 on success, negative value on failure
5359  */
5360 static enum i40e_status_code
5361 i40e_vsi_get_bw_config(struct i40e_vsi *vsi)
5362 {
5363 	struct i40e_aqc_query_vsi_bw_config_resp bw_config;
5364 	struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
5365 	struct i40e_hw *hw = &vsi->adapter->hw;
5366 	i40e_status ret;
5367 	int i;
5368 	uint32_t bw_max;
5369 
5370 	memset(&bw_config, 0, sizeof(bw_config));
5371 	ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5372 	if (ret != I40E_SUCCESS) {
5373 		PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
5374 			    hw->aq.asq_last_status);
5375 		return ret;
5376 	}
5377 
5378 	memset(&ets_sla_config, 0, sizeof(ets_sla_config));
5379 	ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
5380 					&ets_sla_config, NULL);
5381 	if (ret != I40E_SUCCESS) {
5382 		PMD_DRV_LOG(ERR,
5383 			"VSI failed to get TC bandwdith configuration %u",
5384 			hw->aq.asq_last_status);
5385 		return ret;
5386 	}
5387 
5388 	/* store and print out BW info */
5389 	vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit);
5390 	vsi->bw_info.bw_max = bw_config.max_bw;
5391 	PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit);
5392 	PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max);
5393 	bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) |
5394 		    (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) <<
5395 		     I40E_16_BIT_WIDTH);
5396 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5397 		vsi->bw_info.bw_ets_share_credits[i] =
5398 				ets_sla_config.share_credits[i];
5399 		vsi->bw_info.bw_ets_credits[i] =
5400 				rte_le_to_cpu_16(ets_sla_config.credits[i]);
5401 		/* 4 bits per TC, 4th bit is reserved */
5402 		vsi->bw_info.bw_ets_max[i] =
5403 			(uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
5404 				  RTE_LEN2MASK(3, uint8_t));
5405 		PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i,
5406 			    vsi->bw_info.bw_ets_share_credits[i]);
5407 		PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i,
5408 			    vsi->bw_info.bw_ets_credits[i]);
5409 		PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i,
5410 			    vsi->bw_info.bw_ets_max[i]);
5411 	}
5412 
5413 	return I40E_SUCCESS;
5414 }
5415 
5416 /* i40e_enable_pf_lb
5417  * @pf: pointer to the pf structure
5418  *
5419  * allow loopback on pf
5420  */
5421 static inline void
5422 i40e_enable_pf_lb(struct i40e_pf *pf)
5423 {
5424 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5425 	struct i40e_vsi_context ctxt;
5426 	int ret;
5427 
5428 	/* Use the FW API if FW >= v5.0 */
5429 	if (hw->aq.fw_maj_ver < 5 && hw->mac.type != I40E_MAC_X722) {
5430 		PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
5431 		return;
5432 	}
5433 
5434 	memset(&ctxt, 0, sizeof(ctxt));
5435 	ctxt.seid = pf->main_vsi_seid;
5436 	ctxt.pf_num = hw->pf_id;
5437 	ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
5438 	if (ret) {
5439 		PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d",
5440 			    ret, hw->aq.asq_last_status);
5441 		return;
5442 	}
5443 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
5444 	ctxt.info.valid_sections =
5445 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5446 	ctxt.info.switch_id |=
5447 		rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5448 
5449 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5450 	if (ret)
5451 		PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d",
5452 			    hw->aq.asq_last_status);
5453 }
5454 
5455 /* Setup a VSI */
5456 struct i40e_vsi *
5457 i40e_vsi_setup(struct i40e_pf *pf,
5458 	       enum i40e_vsi_type type,
5459 	       struct i40e_vsi *uplink_vsi,
5460 	       uint16_t user_param)
5461 {
5462 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5463 	struct i40e_vsi *vsi;
5464 	struct i40e_mac_filter_info filter;
5465 	int ret;
5466 	struct i40e_vsi_context ctxt;
5467 	struct ether_addr broadcast =
5468 		{.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
5469 
5470 	if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV &&
5471 	    uplink_vsi == NULL) {
5472 		PMD_DRV_LOG(ERR,
5473 			"VSI setup failed, VSI link shouldn't be NULL");
5474 		return NULL;
5475 	}
5476 
5477 	if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
5478 		PMD_DRV_LOG(ERR,
5479 			"VSI setup failed, MAIN VSI uplink VSI should be NULL");
5480 		return NULL;
5481 	}
5482 
5483 	/* two situations
5484 	 * 1.type is not MAIN and uplink vsi is not NULL
5485 	 * If uplink vsi didn't setup VEB, create one first under veb field
5486 	 * 2.type is SRIOV and the uplink is NULL
5487 	 * If floating VEB is NULL, create one veb under floating veb field
5488 	 */
5489 
5490 	if (type != I40E_VSI_MAIN && uplink_vsi != NULL &&
5491 	    uplink_vsi->veb == NULL) {
5492 		uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
5493 
5494 		if (uplink_vsi->veb == NULL) {
5495 			PMD_DRV_LOG(ERR, "VEB setup failed");
5496 			return NULL;
5497 		}
5498 		/* set ALLOWLOOPBACk on pf, when veb is created */
5499 		i40e_enable_pf_lb(pf);
5500 	}
5501 
5502 	if (type == I40E_VSI_SRIOV && uplink_vsi == NULL &&
5503 	    pf->main_vsi->floating_veb == NULL) {
5504 		pf->main_vsi->floating_veb = i40e_veb_setup(pf, uplink_vsi);
5505 
5506 		if (pf->main_vsi->floating_veb == NULL) {
5507 			PMD_DRV_LOG(ERR, "VEB setup failed");
5508 			return NULL;
5509 		}
5510 	}
5511 
5512 	vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
5513 	if (!vsi) {
5514 		PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
5515 		return NULL;
5516 	}
5517 	TAILQ_INIT(&vsi->mac_list);
5518 	vsi->type = type;
5519 	vsi->adapter = I40E_PF_TO_ADAPTER(pf);
5520 	vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
5521 	vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi;
5522 	vsi->user_param = user_param;
5523 	vsi->vlan_anti_spoof_on = 0;
5524 	vsi->vlan_filter_on = 0;
5525 	/* Allocate queues */
5526 	switch (vsi->type) {
5527 	case I40E_VSI_MAIN  :
5528 		vsi->nb_qps = pf->lan_nb_qps;
5529 		break;
5530 	case I40E_VSI_SRIOV :
5531 		vsi->nb_qps = pf->vf_nb_qps;
5532 		break;
5533 	case I40E_VSI_VMDQ2:
5534 		vsi->nb_qps = pf->vmdq_nb_qps;
5535 		break;
5536 	case I40E_VSI_FDIR:
5537 		vsi->nb_qps = pf->fdir_nb_qps;
5538 		break;
5539 	default:
5540 		goto fail_mem;
5541 	}
5542 	/*
5543 	 * The filter status descriptor is reported in rx queue 0,
5544 	 * while the tx queue for fdir filter programming has no
5545 	 * such constraints, can be non-zero queues.
5546 	 * To simplify it, choose FDIR vsi use queue 0 pair.
5547 	 * To make sure it will use queue 0 pair, queue allocation
5548 	 * need be done before this function is called
5549 	 */
5550 	if (type != I40E_VSI_FDIR) {
5551 		ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
5552 			if (ret < 0) {
5553 				PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
5554 						vsi->seid, ret);
5555 				goto fail_mem;
5556 			}
5557 			vsi->base_queue = ret;
5558 	} else
5559 		vsi->base_queue = I40E_FDIR_QUEUE_ID;
5560 
5561 	/* VF has MSIX interrupt in VF range, don't allocate here */
5562 	if (type == I40E_VSI_MAIN) {
5563 		if (pf->support_multi_driver) {
5564 			/* If support multi-driver, need to use INT0 instead of
5565 			 * allocating from msix pool. The Msix pool is init from
5566 			 * INT1, so it's OK just set msix_intr to 0 and nb_msix
5567 			 * to 1 without calling i40e_res_pool_alloc.
5568 			 */
5569 			vsi->msix_intr = 0;
5570 			vsi->nb_msix = 1;
5571 		} else {
5572 			ret = i40e_res_pool_alloc(&pf->msix_pool,
5573 						  RTE_MIN(vsi->nb_qps,
5574 						     RTE_MAX_RXTX_INTR_VEC_ID));
5575 			if (ret < 0) {
5576 				PMD_DRV_LOG(ERR,
5577 					    "VSI MAIN %d get heap failed %d",
5578 					    vsi->seid, ret);
5579 				goto fail_queue_alloc;
5580 			}
5581 			vsi->msix_intr = ret;
5582 			vsi->nb_msix = RTE_MIN(vsi->nb_qps,
5583 					       RTE_MAX_RXTX_INTR_VEC_ID);
5584 		}
5585 	} else if (type != I40E_VSI_SRIOV) {
5586 		ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
5587 		if (ret < 0) {
5588 			PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
5589 			goto fail_queue_alloc;
5590 		}
5591 		vsi->msix_intr = ret;
5592 		vsi->nb_msix = 1;
5593 	} else {
5594 		vsi->msix_intr = 0;
5595 		vsi->nb_msix = 0;
5596 	}
5597 
5598 	/* Add VSI */
5599 	if (type == I40E_VSI_MAIN) {
5600 		/* For main VSI, no need to add since it's default one */
5601 		vsi->uplink_seid = pf->mac_seid;
5602 		vsi->seid = pf->main_vsi_seid;
5603 		/* Bind queues with specific MSIX interrupt */
5604 		/**
5605 		 * Needs 2 interrupt at least, one for misc cause which will
5606 		 * enabled from OS side, Another for queues binding the
5607 		 * interrupt from device side only.
5608 		 */
5609 
5610 		/* Get default VSI parameters from hardware */
5611 		memset(&ctxt, 0, sizeof(ctxt));
5612 		ctxt.seid = vsi->seid;
5613 		ctxt.pf_num = hw->pf_id;
5614 		ctxt.uplink_seid = vsi->uplink_seid;
5615 		ctxt.vf_num = 0;
5616 		ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
5617 		if (ret != I40E_SUCCESS) {
5618 			PMD_DRV_LOG(ERR, "Failed to get VSI params");
5619 			goto fail_msix_alloc;
5620 		}
5621 		rte_memcpy(&vsi->info, &ctxt.info,
5622 			sizeof(struct i40e_aqc_vsi_properties_data));
5623 		vsi->vsi_id = ctxt.vsi_number;
5624 		vsi->info.valid_sections = 0;
5625 
5626 		/* Configure tc, enabled TC0 only */
5627 		if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
5628 			I40E_SUCCESS) {
5629 			PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
5630 			goto fail_msix_alloc;
5631 		}
5632 
5633 		/* TC, queue mapping */
5634 		memset(&ctxt, 0, sizeof(ctxt));
5635 		vsi->info.valid_sections |=
5636 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5637 		vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5638 					I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
5639 		rte_memcpy(&ctxt.info, &vsi->info,
5640 			sizeof(struct i40e_aqc_vsi_properties_data));
5641 		ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
5642 						I40E_DEFAULT_TCMAP);
5643 		if (ret != I40E_SUCCESS) {
5644 			PMD_DRV_LOG(ERR,
5645 				"Failed to configure TC queue mapping");
5646 			goto fail_msix_alloc;
5647 		}
5648 		ctxt.seid = vsi->seid;
5649 		ctxt.pf_num = hw->pf_id;
5650 		ctxt.uplink_seid = vsi->uplink_seid;
5651 		ctxt.vf_num = 0;
5652 
5653 		/* Update VSI parameters */
5654 		ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5655 		if (ret != I40E_SUCCESS) {
5656 			PMD_DRV_LOG(ERR, "Failed to update VSI params");
5657 			goto fail_msix_alloc;
5658 		}
5659 
5660 		rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
5661 						sizeof(vsi->info.tc_mapping));
5662 		rte_memcpy(&vsi->info.queue_mapping,
5663 				&ctxt.info.queue_mapping,
5664 			sizeof(vsi->info.queue_mapping));
5665 		vsi->info.mapping_flags = ctxt.info.mapping_flags;
5666 		vsi->info.valid_sections = 0;
5667 
5668 		rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
5669 				ETH_ADDR_LEN);
5670 
5671 		/**
5672 		 * Updating default filter settings are necessary to prevent
5673 		 * reception of tagged packets.
5674 		 * Some old firmware configurations load a default macvlan
5675 		 * filter which accepts both tagged and untagged packets.
5676 		 * The updating is to use a normal filter instead if needed.
5677 		 * For NVM 4.2.2 or after, the updating is not needed anymore.
5678 		 * The firmware with correct configurations load the default
5679 		 * macvlan filter which is expected and cannot be removed.
5680 		 */
5681 		i40e_update_default_filter_setting(vsi);
5682 		i40e_config_qinq(hw, vsi);
5683 	} else if (type == I40E_VSI_SRIOV) {
5684 		memset(&ctxt, 0, sizeof(ctxt));
5685 		/**
5686 		 * For other VSI, the uplink_seid equals to uplink VSI's
5687 		 * uplink_seid since they share same VEB
5688 		 */
5689 		if (uplink_vsi == NULL)
5690 			vsi->uplink_seid = pf->main_vsi->floating_veb->seid;
5691 		else
5692 			vsi->uplink_seid = uplink_vsi->uplink_seid;
5693 		ctxt.pf_num = hw->pf_id;
5694 		ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
5695 		ctxt.uplink_seid = vsi->uplink_seid;
5696 		ctxt.connection_type = 0x1;
5697 		ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5698 
5699 		/* Use the VEB configuration if FW >= v5.0 */
5700 		if (hw->aq.fw_maj_ver >= 5 || hw->mac.type == I40E_MAC_X722) {
5701 			/* Configure switch ID */
5702 			ctxt.info.valid_sections |=
5703 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5704 			ctxt.info.switch_id =
5705 			rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5706 		}
5707 
5708 		/* Configure port/vlan */
5709 		ctxt.info.valid_sections |=
5710 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5711 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
5712 		ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
5713 						hw->func_caps.enabled_tcmap);
5714 		if (ret != I40E_SUCCESS) {
5715 			PMD_DRV_LOG(ERR,
5716 				"Failed to configure TC queue mapping");
5717 			goto fail_msix_alloc;
5718 		}
5719 
5720 		ctxt.info.up_enable_bits = hw->func_caps.enabled_tcmap;
5721 		ctxt.info.valid_sections |=
5722 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
5723 		/**
5724 		 * Since VSI is not created yet, only configure parameter,
5725 		 * will add vsi below.
5726 		 */
5727 
5728 		i40e_config_qinq(hw, vsi);
5729 	} else if (type == I40E_VSI_VMDQ2) {
5730 		memset(&ctxt, 0, sizeof(ctxt));
5731 		/*
5732 		 * For other VSI, the uplink_seid equals to uplink VSI's
5733 		 * uplink_seid since they share same VEB
5734 		 */
5735 		vsi->uplink_seid = uplink_vsi->uplink_seid;
5736 		ctxt.pf_num = hw->pf_id;
5737 		ctxt.vf_num = 0;
5738 		ctxt.uplink_seid = vsi->uplink_seid;
5739 		ctxt.connection_type = 0x1;
5740 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5741 
5742 		ctxt.info.valid_sections |=
5743 				rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5744 		/* user_param carries flag to enable loop back */
5745 		if (user_param) {
5746 			ctxt.info.switch_id =
5747 			rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
5748 			ctxt.info.switch_id |=
5749 			rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5750 		}
5751 
5752 		/* Configure port/vlan */
5753 		ctxt.info.valid_sections |=
5754 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5755 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
5756 		ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
5757 						I40E_DEFAULT_TCMAP);
5758 		if (ret != I40E_SUCCESS) {
5759 			PMD_DRV_LOG(ERR,
5760 				"Failed to configure TC queue mapping");
5761 			goto fail_msix_alloc;
5762 		}
5763 		ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
5764 		ctxt.info.valid_sections |=
5765 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
5766 	} else if (type == I40E_VSI_FDIR) {
5767 		memset(&ctxt, 0, sizeof(ctxt));
5768 		vsi->uplink_seid = uplink_vsi->uplink_seid;
5769 		ctxt.pf_num = hw->pf_id;
5770 		ctxt.vf_num = 0;
5771 		ctxt.uplink_seid = vsi->uplink_seid;
5772 		ctxt.connection_type = 0x1;     /* regular data port */
5773 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
5774 		ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
5775 						I40E_DEFAULT_TCMAP);
5776 		if (ret != I40E_SUCCESS) {
5777 			PMD_DRV_LOG(ERR,
5778 				"Failed to configure TC queue mapping.");
5779 			goto fail_msix_alloc;
5780 		}
5781 		ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
5782 		ctxt.info.valid_sections |=
5783 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
5784 	} else {
5785 		PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
5786 		goto fail_msix_alloc;
5787 	}
5788 
5789 	if (vsi->type != I40E_VSI_MAIN) {
5790 		ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5791 		if (ret != I40E_SUCCESS) {
5792 			PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
5793 				    hw->aq.asq_last_status);
5794 			goto fail_msix_alloc;
5795 		}
5796 		memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
5797 		vsi->info.valid_sections = 0;
5798 		vsi->seid = ctxt.seid;
5799 		vsi->vsi_id = ctxt.vsi_number;
5800 		vsi->sib_vsi_list.vsi = vsi;
5801 		if (vsi->type == I40E_VSI_SRIOV && uplink_vsi == NULL) {
5802 			TAILQ_INSERT_TAIL(&pf->main_vsi->floating_veb->head,
5803 					  &vsi->sib_vsi_list, list);
5804 		} else {
5805 			TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
5806 					  &vsi->sib_vsi_list, list);
5807 		}
5808 	}
5809 
5810 	/* MAC/VLAN configuration */
5811 	rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
5812 	filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
5813 
5814 	ret = i40e_vsi_add_mac(vsi, &filter);
5815 	if (ret != I40E_SUCCESS) {
5816 		PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
5817 		goto fail_msix_alloc;
5818 	}
5819 
5820 	/* Get VSI BW information */
5821 	i40e_vsi_get_bw_config(vsi);
5822 	return vsi;
5823 fail_msix_alloc:
5824 	i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
5825 fail_queue_alloc:
5826 	i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
5827 fail_mem:
5828 	rte_free(vsi);
5829 	return NULL;
5830 }
5831 
5832 /* Configure vlan filter on or off */
5833 int
5834 i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on)
5835 {
5836 	int i, num;
5837 	struct i40e_mac_filter *f;
5838 	void *temp;
5839 	struct i40e_mac_filter_info *mac_filter;
5840 	enum rte_mac_filter_type desired_filter;
5841 	int ret = I40E_SUCCESS;
5842 
5843 	if (on) {
5844 		/* Filter to match MAC and VLAN */
5845 		desired_filter = RTE_MACVLAN_PERFECT_MATCH;
5846 	} else {
5847 		/* Filter to match only MAC */
5848 		desired_filter = RTE_MAC_PERFECT_MATCH;
5849 	}
5850 
5851 	num = vsi->mac_num;
5852 
5853 	mac_filter = rte_zmalloc("mac_filter_info_data",
5854 				 num * sizeof(*mac_filter), 0);
5855 	if (mac_filter == NULL) {
5856 		PMD_DRV_LOG(ERR, "failed to allocate memory");
5857 		return I40E_ERR_NO_MEMORY;
5858 	}
5859 
5860 	i = 0;
5861 
5862 	/* Remove all existing mac */
5863 	TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) {
5864 		mac_filter[i] = f->mac_info;
5865 		ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr);
5866 		if (ret) {
5867 			PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
5868 				    on ? "enable" : "disable");
5869 			goto DONE;
5870 		}
5871 		i++;
5872 	}
5873 
5874 	/* Override with new filter */
5875 	for (i = 0; i < num; i++) {
5876 		mac_filter[i].filter_type = desired_filter;
5877 		ret = i40e_vsi_add_mac(vsi, &mac_filter[i]);
5878 		if (ret) {
5879 			PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
5880 				    on ? "enable" : "disable");
5881 			goto DONE;
5882 		}
5883 	}
5884 
5885 DONE:
5886 	rte_free(mac_filter);
5887 	return ret;
5888 }
5889 
5890 /* Configure vlan stripping on or off */
5891 int
5892 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
5893 {
5894 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5895 	struct i40e_vsi_context ctxt;
5896 	uint8_t vlan_flags;
5897 	int ret = I40E_SUCCESS;
5898 
5899 	/* Check if it has been already on or off */
5900 	if (vsi->info.valid_sections &
5901 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
5902 		if (on) {
5903 			if ((vsi->info.port_vlan_flags &
5904 				I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
5905 				return 0; /* already on */
5906 		} else {
5907 			if ((vsi->info.port_vlan_flags &
5908 				I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
5909 				I40E_AQ_VSI_PVLAN_EMOD_MASK)
5910 				return 0; /* already off */
5911 		}
5912 	}
5913 
5914 	if (on)
5915 		vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
5916 	else
5917 		vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5918 	vsi->info.valid_sections =
5919 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5920 	vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
5921 	vsi->info.port_vlan_flags |= vlan_flags;
5922 	ctxt.seid = vsi->seid;
5923 	rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
5924 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5925 	if (ret)
5926 		PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
5927 			    on ? "enable" : "disable");
5928 
5929 	return ret;
5930 }
5931 
5932 static int
5933 i40e_dev_init_vlan(struct rte_eth_dev *dev)
5934 {
5935 	struct rte_eth_dev_data *data = dev->data;
5936 	int ret;
5937 	int mask = 0;
5938 
5939 	/* Apply vlan offload setting */
5940 	mask = ETH_VLAN_STRIP_MASK |
5941 	       ETH_VLAN_FILTER_MASK |
5942 	       ETH_VLAN_EXTEND_MASK;
5943 	ret = i40e_vlan_offload_set(dev, mask);
5944 	if (ret) {
5945 		PMD_DRV_LOG(INFO, "Failed to update vlan offload");
5946 		return ret;
5947 	}
5948 
5949 	/* Apply pvid setting */
5950 	ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
5951 				data->dev_conf.txmode.hw_vlan_insert_pvid);
5952 	if (ret)
5953 		PMD_DRV_LOG(INFO, "Failed to update VSI params");
5954 
5955 	return ret;
5956 }
5957 
5958 static int
5959 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
5960 {
5961 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5962 
5963 	return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
5964 }
5965 
5966 static int
5967 i40e_update_flow_control(struct i40e_hw *hw)
5968 {
5969 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
5970 	struct i40e_link_status link_status;
5971 	uint32_t rxfc = 0, txfc = 0, reg;
5972 	uint8_t an_info;
5973 	int ret;
5974 
5975 	memset(&link_status, 0, sizeof(link_status));
5976 	ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
5977 	if (ret != I40E_SUCCESS) {
5978 		PMD_DRV_LOG(ERR, "Failed to get link status information");
5979 		goto write_reg; /* Disable flow control */
5980 	}
5981 
5982 	an_info = hw->phy.link_info.an_info;
5983 	if (!(an_info & I40E_AQ_AN_COMPLETED)) {
5984 		PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
5985 		ret = I40E_ERR_NOT_READY;
5986 		goto write_reg; /* Disable flow control */
5987 	}
5988 	/**
5989 	 * If link auto negotiation is enabled, flow control needs to
5990 	 * be configured according to it
5991 	 */
5992 	switch (an_info & I40E_LINK_PAUSE_RXTX) {
5993 	case I40E_LINK_PAUSE_RXTX:
5994 		rxfc = 1;
5995 		txfc = 1;
5996 		hw->fc.current_mode = I40E_FC_FULL;
5997 		break;
5998 	case I40E_AQ_LINK_PAUSE_RX:
5999 		rxfc = 1;
6000 		hw->fc.current_mode = I40E_FC_RX_PAUSE;
6001 		break;
6002 	case I40E_AQ_LINK_PAUSE_TX:
6003 		txfc = 1;
6004 		hw->fc.current_mode = I40E_FC_TX_PAUSE;
6005 		break;
6006 	default:
6007 		hw->fc.current_mode = I40E_FC_NONE;
6008 		break;
6009 	}
6010 
6011 write_reg:
6012 	I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
6013 		txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
6014 	reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
6015 	reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
6016 	reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
6017 	I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
6018 
6019 	return ret;
6020 }
6021 
6022 /* PF setup */
6023 static int
6024 i40e_pf_setup(struct i40e_pf *pf)
6025 {
6026 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6027 	struct i40e_filter_control_settings settings;
6028 	struct i40e_vsi *vsi;
6029 	int ret;
6030 
6031 	/* Clear all stats counters */
6032 	pf->offset_loaded = FALSE;
6033 	memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
6034 	memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
6035 	memset(&pf->internal_stats, 0, sizeof(struct i40e_eth_stats));
6036 	memset(&pf->internal_stats_offset, 0, sizeof(struct i40e_eth_stats));
6037 
6038 	ret = i40e_pf_get_switch_config(pf);
6039 	if (ret != I40E_SUCCESS) {
6040 		PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
6041 		return ret;
6042 	}
6043 
6044 	ret = rte_eth_switch_domain_alloc(&pf->switch_domain_id);
6045 	if (ret)
6046 		PMD_INIT_LOG(WARNING,
6047 			"failed to allocate switch domain for device %d", ret);
6048 
6049 	if (pf->flags & I40E_FLAG_FDIR) {
6050 		/* make queue allocated first, let FDIR use queue pair 0*/
6051 		ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
6052 		if (ret != I40E_FDIR_QUEUE_ID) {
6053 			PMD_DRV_LOG(ERR,
6054 				"queue allocation fails for FDIR: ret =%d",
6055 				ret);
6056 			pf->flags &= ~I40E_FLAG_FDIR;
6057 		}
6058 	}
6059 	/*  main VSI setup */
6060 	vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
6061 	if (!vsi) {
6062 		PMD_DRV_LOG(ERR, "Setup of main vsi failed");
6063 		return I40E_ERR_NOT_READY;
6064 	}
6065 	pf->main_vsi = vsi;
6066 
6067 	/* Configure filter control */
6068 	memset(&settings, 0, sizeof(settings));
6069 	if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
6070 		settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
6071 	else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
6072 		settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
6073 	else {
6074 		PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported",
6075 			hw->func_caps.rss_table_size);
6076 		return I40E_ERR_PARAM;
6077 	}
6078 	PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table size: %u",
6079 		hw->func_caps.rss_table_size);
6080 	pf->hash_lut_size = hw->func_caps.rss_table_size;
6081 
6082 	/* Enable ethtype and macvlan filters */
6083 	settings.enable_ethtype = TRUE;
6084 	settings.enable_macvlan = TRUE;
6085 	ret = i40e_set_filter_control(hw, &settings);
6086 	if (ret)
6087 		PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
6088 								ret);
6089 
6090 	/* Update flow control according to the auto negotiation */
6091 	i40e_update_flow_control(hw);
6092 
6093 	return I40E_SUCCESS;
6094 }
6095 
6096 int
6097 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
6098 {
6099 	uint32_t reg;
6100 	uint16_t j;
6101 
6102 	/**
6103 	 * Set or clear TX Queue Disable flags,
6104 	 * which is required by hardware.
6105 	 */
6106 	i40e_pre_tx_queue_cfg(hw, q_idx, on);
6107 	rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
6108 
6109 	/* Wait until the request is finished */
6110 	for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
6111 		rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
6112 		reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
6113 		if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
6114 			((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
6115 							& 0x1))) {
6116 			break;
6117 		}
6118 	}
6119 	if (on) {
6120 		if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
6121 			return I40E_SUCCESS; /* already on, skip next steps */
6122 
6123 		I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
6124 		reg |= I40E_QTX_ENA_QENA_REQ_MASK;
6125 	} else {
6126 		if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
6127 			return I40E_SUCCESS; /* already off, skip next steps */
6128 		reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
6129 	}
6130 	/* Write the register */
6131 	I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
6132 	/* Check the result */
6133 	for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
6134 		rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
6135 		reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
6136 		if (on) {
6137 			if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
6138 				(reg & I40E_QTX_ENA_QENA_STAT_MASK))
6139 				break;
6140 		} else {
6141 			if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
6142 				!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
6143 				break;
6144 		}
6145 	}
6146 	/* Check if it is timeout */
6147 	if (j >= I40E_CHK_Q_ENA_COUNT) {
6148 		PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
6149 			    (on ? "enable" : "disable"), q_idx);
6150 		return I40E_ERR_TIMEOUT;
6151 	}
6152 
6153 	return I40E_SUCCESS;
6154 }
6155 
6156 /* Swith on or off the tx queues */
6157 static int
6158 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
6159 {
6160 	struct rte_eth_dev_data *dev_data = pf->dev_data;
6161 	struct i40e_tx_queue *txq;
6162 	struct rte_eth_dev *dev = pf->adapter->eth_dev;
6163 	uint16_t i;
6164 	int ret;
6165 
6166 	for (i = 0; i < dev_data->nb_tx_queues; i++) {
6167 		txq = dev_data->tx_queues[i];
6168 		/* Don't operate the queue if not configured or
6169 		 * if starting only per queue */
6170 		if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
6171 			continue;
6172 		if (on)
6173 			ret = i40e_dev_tx_queue_start(dev, i);
6174 		else
6175 			ret = i40e_dev_tx_queue_stop(dev, i);
6176 		if ( ret != I40E_SUCCESS)
6177 			return ret;
6178 	}
6179 
6180 	return I40E_SUCCESS;
6181 }
6182 
6183 int
6184 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
6185 {
6186 	uint32_t reg;
6187 	uint16_t j;
6188 
6189 	/* Wait until the request is finished */
6190 	for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
6191 		rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
6192 		reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
6193 		if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
6194 			((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
6195 			break;
6196 	}
6197 
6198 	if (on) {
6199 		if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
6200 			return I40E_SUCCESS; /* Already on, skip next steps */
6201 		reg |= I40E_QRX_ENA_QENA_REQ_MASK;
6202 	} else {
6203 		if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
6204 			return I40E_SUCCESS; /* Already off, skip next steps */
6205 		reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
6206 	}
6207 
6208 	/* Write the register */
6209 	I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
6210 	/* Check the result */
6211 	for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
6212 		rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
6213 		reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
6214 		if (on) {
6215 			if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
6216 				(reg & I40E_QRX_ENA_QENA_STAT_MASK))
6217 				break;
6218 		} else {
6219 			if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
6220 				!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
6221 				break;
6222 		}
6223 	}
6224 
6225 	/* Check if it is timeout */
6226 	if (j >= I40E_CHK_Q_ENA_COUNT) {
6227 		PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
6228 			    (on ? "enable" : "disable"), q_idx);
6229 		return I40E_ERR_TIMEOUT;
6230 	}
6231 
6232 	return I40E_SUCCESS;
6233 }
6234 /* Switch on or off the rx queues */
6235 static int
6236 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
6237 {
6238 	struct rte_eth_dev_data *dev_data = pf->dev_data;
6239 	struct i40e_rx_queue *rxq;
6240 	struct rte_eth_dev *dev = pf->adapter->eth_dev;
6241 	uint16_t i;
6242 	int ret;
6243 
6244 	for (i = 0; i < dev_data->nb_rx_queues; i++) {
6245 		rxq = dev_data->rx_queues[i];
6246 		/* Don't operate the queue if not configured or
6247 		 * if starting only per queue */
6248 		if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
6249 			continue;
6250 		if (on)
6251 			ret = i40e_dev_rx_queue_start(dev, i);
6252 		else
6253 			ret = i40e_dev_rx_queue_stop(dev, i);
6254 		if (ret != I40E_SUCCESS)
6255 			return ret;
6256 	}
6257 
6258 	return I40E_SUCCESS;
6259 }
6260 
6261 /* Switch on or off all the rx/tx queues */
6262 int
6263 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
6264 {
6265 	int ret;
6266 
6267 	if (on) {
6268 		/* enable rx queues before enabling tx queues */
6269 		ret = i40e_dev_switch_rx_queues(pf, on);
6270 		if (ret) {
6271 			PMD_DRV_LOG(ERR, "Failed to switch rx queues");
6272 			return ret;
6273 		}
6274 		ret = i40e_dev_switch_tx_queues(pf, on);
6275 	} else {
6276 		/* Stop tx queues before stopping rx queues */
6277 		ret = i40e_dev_switch_tx_queues(pf, on);
6278 		if (ret) {
6279 			PMD_DRV_LOG(ERR, "Failed to switch tx queues");
6280 			return ret;
6281 		}
6282 		ret = i40e_dev_switch_rx_queues(pf, on);
6283 	}
6284 
6285 	return ret;
6286 }
6287 
6288 /* Initialize VSI for TX */
6289 static int
6290 i40e_dev_tx_init(struct i40e_pf *pf)
6291 {
6292 	struct rte_eth_dev_data *data = pf->dev_data;
6293 	uint16_t i;
6294 	uint32_t ret = I40E_SUCCESS;
6295 	struct i40e_tx_queue *txq;
6296 
6297 	for (i = 0; i < data->nb_tx_queues; i++) {
6298 		txq = data->tx_queues[i];
6299 		if (!txq || !txq->q_set)
6300 			continue;
6301 		ret = i40e_tx_queue_init(txq);
6302 		if (ret != I40E_SUCCESS)
6303 			break;
6304 	}
6305 	if (ret == I40E_SUCCESS)
6306 		i40e_set_tx_function(container_of(pf, struct i40e_adapter, pf)
6307 				     ->eth_dev);
6308 
6309 	return ret;
6310 }
6311 
6312 /* Initialize VSI for RX */
6313 static int
6314 i40e_dev_rx_init(struct i40e_pf *pf)
6315 {
6316 	struct rte_eth_dev_data *data = pf->dev_data;
6317 	int ret = I40E_SUCCESS;
6318 	uint16_t i;
6319 	struct i40e_rx_queue *rxq;
6320 
6321 	i40e_pf_config_mq_rx(pf);
6322 	for (i = 0; i < data->nb_rx_queues; i++) {
6323 		rxq = data->rx_queues[i];
6324 		if (!rxq || !rxq->q_set)
6325 			continue;
6326 
6327 		ret = i40e_rx_queue_init(rxq);
6328 		if (ret != I40E_SUCCESS) {
6329 			PMD_DRV_LOG(ERR,
6330 				"Failed to do RX queue initialization");
6331 			break;
6332 		}
6333 	}
6334 	if (ret == I40E_SUCCESS)
6335 		i40e_set_rx_function(container_of(pf, struct i40e_adapter, pf)
6336 				     ->eth_dev);
6337 
6338 	return ret;
6339 }
6340 
6341 static int
6342 i40e_dev_rxtx_init(struct i40e_pf *pf)
6343 {
6344 	int err;
6345 
6346 	err = i40e_dev_tx_init(pf);
6347 	if (err) {
6348 		PMD_DRV_LOG(ERR, "Failed to do TX initialization");
6349 		return err;
6350 	}
6351 	err = i40e_dev_rx_init(pf);
6352 	if (err) {
6353 		PMD_DRV_LOG(ERR, "Failed to do RX initialization");
6354 		return err;
6355 	}
6356 
6357 	return err;
6358 }
6359 
6360 static int
6361 i40e_vmdq_setup(struct rte_eth_dev *dev)
6362 {
6363 	struct rte_eth_conf *conf = &dev->data->dev_conf;
6364 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6365 	int i, err, conf_vsis, j, loop;
6366 	struct i40e_vsi *vsi;
6367 	struct i40e_vmdq_info *vmdq_info;
6368 	struct rte_eth_vmdq_rx_conf *vmdq_conf;
6369 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6370 
6371 	/*
6372 	 * Disable interrupt to avoid message from VF. Furthermore, it will
6373 	 * avoid race condition in VSI creation/destroy.
6374 	 */
6375 	i40e_pf_disable_irq0(hw);
6376 
6377 	if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
6378 		PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
6379 		return -ENOTSUP;
6380 	}
6381 
6382 	conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
6383 	if (conf_vsis > pf->max_nb_vmdq_vsi) {
6384 		PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
6385 			conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
6386 			pf->max_nb_vmdq_vsi);
6387 		return -ENOTSUP;
6388 	}
6389 
6390 	if (pf->vmdq != NULL) {
6391 		PMD_INIT_LOG(INFO, "VMDQ already configured");
6392 		return 0;
6393 	}
6394 
6395 	pf->vmdq = rte_zmalloc("vmdq_info_struct",
6396 				sizeof(*vmdq_info) * conf_vsis, 0);
6397 
6398 	if (pf->vmdq == NULL) {
6399 		PMD_INIT_LOG(ERR, "Failed to allocate memory");
6400 		return -ENOMEM;
6401 	}
6402 
6403 	vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
6404 
6405 	/* Create VMDQ VSI */
6406 	for (i = 0; i < conf_vsis; i++) {
6407 		vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
6408 				vmdq_conf->enable_loop_back);
6409 		if (vsi == NULL) {
6410 			PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
6411 			err = -1;
6412 			goto err_vsi_setup;
6413 		}
6414 		vmdq_info = &pf->vmdq[i];
6415 		vmdq_info->pf = pf;
6416 		vmdq_info->vsi = vsi;
6417 	}
6418 	pf->nb_cfg_vmdq_vsi = conf_vsis;
6419 
6420 	/* Configure Vlan */
6421 	loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
6422 	for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
6423 		for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
6424 			if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
6425 				PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
6426 					vmdq_conf->pool_map[i].vlan_id, j);
6427 
6428 				err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
6429 						vmdq_conf->pool_map[i].vlan_id);
6430 				if (err) {
6431 					PMD_INIT_LOG(ERR, "Failed to add vlan");
6432 					err = -1;
6433 					goto err_vsi_setup;
6434 				}
6435 			}
6436 		}
6437 	}
6438 
6439 	i40e_pf_enable_irq0(hw);
6440 
6441 	return 0;
6442 
6443 err_vsi_setup:
6444 	for (i = 0; i < conf_vsis; i++)
6445 		if (pf->vmdq[i].vsi == NULL)
6446 			break;
6447 		else
6448 			i40e_vsi_release(pf->vmdq[i].vsi);
6449 
6450 	rte_free(pf->vmdq);
6451 	pf->vmdq = NULL;
6452 	i40e_pf_enable_irq0(hw);
6453 	return err;
6454 }
6455 
6456 static void
6457 i40e_stat_update_32(struct i40e_hw *hw,
6458 		   uint32_t reg,
6459 		   bool offset_loaded,
6460 		   uint64_t *offset,
6461 		   uint64_t *stat)
6462 {
6463 	uint64_t new_data;
6464 
6465 	new_data = (uint64_t)I40E_READ_REG(hw, reg);
6466 	if (!offset_loaded)
6467 		*offset = new_data;
6468 
6469 	if (new_data >= *offset)
6470 		*stat = (uint64_t)(new_data - *offset);
6471 	else
6472 		*stat = (uint64_t)((new_data +
6473 			((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
6474 }
6475 
6476 static void
6477 i40e_stat_update_48(struct i40e_hw *hw,
6478 		   uint32_t hireg,
6479 		   uint32_t loreg,
6480 		   bool offset_loaded,
6481 		   uint64_t *offset,
6482 		   uint64_t *stat)
6483 {
6484 	uint64_t new_data;
6485 
6486 	new_data = (uint64_t)I40E_READ_REG(hw, loreg);
6487 	new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
6488 			I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
6489 
6490 	if (!offset_loaded)
6491 		*offset = new_data;
6492 
6493 	if (new_data >= *offset)
6494 		*stat = new_data - *offset;
6495 	else
6496 		*stat = (uint64_t)((new_data +
6497 			((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
6498 
6499 	*stat &= I40E_48_BIT_MASK;
6500 }
6501 
6502 /* Disable IRQ0 */
6503 void
6504 i40e_pf_disable_irq0(struct i40e_hw *hw)
6505 {
6506 	/* Disable all interrupt types */
6507 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
6508 		       I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
6509 	I40E_WRITE_FLUSH(hw);
6510 }
6511 
6512 /* Enable IRQ0 */
6513 void
6514 i40e_pf_enable_irq0(struct i40e_hw *hw)
6515 {
6516 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
6517 		I40E_PFINT_DYN_CTL0_INTENA_MASK |
6518 		I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
6519 		I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
6520 	I40E_WRITE_FLUSH(hw);
6521 }
6522 
6523 static void
6524 i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue)
6525 {
6526 	/* read pending request and disable first */
6527 	i40e_pf_disable_irq0(hw);
6528 	I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
6529 	I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
6530 		I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
6531 
6532 	if (no_queue)
6533 		/* Link no queues with irq0 */
6534 		I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
6535 			       I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
6536 }
6537 
6538 static void
6539 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
6540 {
6541 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6542 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6543 	int i;
6544 	uint16_t abs_vf_id;
6545 	uint32_t index, offset, val;
6546 
6547 	if (!pf->vfs)
6548 		return;
6549 	/**
6550 	 * Try to find which VF trigger a reset, use absolute VF id to access
6551 	 * since the reg is global register.
6552 	 */
6553 	for (i = 0; i < pf->vf_num; i++) {
6554 		abs_vf_id = hw->func_caps.vf_base_id + i;
6555 		index = abs_vf_id / I40E_UINT32_BIT_SIZE;
6556 		offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
6557 		val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
6558 		/* VFR event occurred */
6559 		if (val & (0x1 << offset)) {
6560 			int ret;
6561 
6562 			/* Clear the event first */
6563 			I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
6564 							(0x1 << offset));
6565 			PMD_DRV_LOG(INFO, "VF %u reset occurred", abs_vf_id);
6566 			/**
6567 			 * Only notify a VF reset event occurred,
6568 			 * don't trigger another SW reset
6569 			 */
6570 			ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
6571 			if (ret != I40E_SUCCESS)
6572 				PMD_DRV_LOG(ERR, "Failed to do VF reset");
6573 		}
6574 	}
6575 }
6576 
6577 static void
6578 i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev)
6579 {
6580 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6581 	int i;
6582 
6583 	for (i = 0; i < pf->vf_num; i++)
6584 		i40e_notify_vf_link_status(dev, &pf->vfs[i]);
6585 }
6586 
6587 static void
6588 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
6589 {
6590 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6591 	struct i40e_arq_event_info info;
6592 	uint16_t pending, opcode;
6593 	int ret;
6594 
6595 	info.buf_len = I40E_AQ_BUF_SZ;
6596 	info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
6597 	if (!info.msg_buf) {
6598 		PMD_DRV_LOG(ERR, "Failed to allocate mem");
6599 		return;
6600 	}
6601 
6602 	pending = 1;
6603 	while (pending) {
6604 		ret = i40e_clean_arq_element(hw, &info, &pending);
6605 
6606 		if (ret != I40E_SUCCESS) {
6607 			PMD_DRV_LOG(INFO,
6608 				"Failed to read msg from AdminQ, aq_err: %u",
6609 				hw->aq.asq_last_status);
6610 			break;
6611 		}
6612 		opcode = rte_le_to_cpu_16(info.desc.opcode);
6613 
6614 		switch (opcode) {
6615 		case i40e_aqc_opc_send_msg_to_pf:
6616 			/* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
6617 			i40e_pf_host_handle_vf_msg(dev,
6618 					rte_le_to_cpu_16(info.desc.retval),
6619 					rte_le_to_cpu_32(info.desc.cookie_high),
6620 					rte_le_to_cpu_32(info.desc.cookie_low),
6621 					info.msg_buf,
6622 					info.msg_len);
6623 			break;
6624 		case i40e_aqc_opc_get_link_status:
6625 			ret = i40e_dev_link_update(dev, 0);
6626 			if (!ret)
6627 				_rte_eth_dev_callback_process(dev,
6628 					RTE_ETH_EVENT_INTR_LSC, NULL);
6629 			break;
6630 		default:
6631 			PMD_DRV_LOG(DEBUG, "Request %u is not supported yet",
6632 				    opcode);
6633 			break;
6634 		}
6635 	}
6636 	rte_free(info.msg_buf);
6637 }
6638 
6639 /**
6640  * Interrupt handler triggered by NIC  for handling
6641  * specific interrupt.
6642  *
6643  * @param handle
6644  *  Pointer to interrupt handle.
6645  * @param param
6646  *  The address of parameter (struct rte_eth_dev *) regsitered before.
6647  *
6648  * @return
6649  *  void
6650  */
6651 static void
6652 i40e_dev_interrupt_handler(void *param)
6653 {
6654 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
6655 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6656 	uint32_t icr0;
6657 
6658 	/* Disable interrupt */
6659 	i40e_pf_disable_irq0(hw);
6660 
6661 	/* read out interrupt causes */
6662 	icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
6663 
6664 	/* No interrupt event indicated */
6665 	if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
6666 		PMD_DRV_LOG(INFO, "No interrupt event");
6667 		goto done;
6668 	}
6669 	if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
6670 		PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
6671 	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
6672 		PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
6673 	if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
6674 		PMD_DRV_LOG(INFO, "ICR0: global reset requested");
6675 	if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
6676 		PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
6677 	if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
6678 		PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
6679 	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
6680 		PMD_DRV_LOG(ERR, "ICR0: HMC error");
6681 	if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
6682 		PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
6683 
6684 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
6685 		PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
6686 		i40e_dev_handle_vfr_event(dev);
6687 	}
6688 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
6689 		PMD_DRV_LOG(INFO, "ICR0: adminq event");
6690 		i40e_dev_handle_aq_msg(dev);
6691 	}
6692 
6693 done:
6694 	/* Enable interrupt */
6695 	i40e_pf_enable_irq0(hw);
6696 }
6697 
6698 static void
6699 i40e_dev_alarm_handler(void *param)
6700 {
6701 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
6702 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6703 	uint32_t icr0;
6704 
6705 	/* Disable interrupt */
6706 	i40e_pf_disable_irq0(hw);
6707 
6708 	/* read out interrupt causes */
6709 	icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
6710 
6711 	/* No interrupt event indicated */
6712 	if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK))
6713 		goto done;
6714 	if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
6715 		PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
6716 	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
6717 		PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
6718 	if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
6719 		PMD_DRV_LOG(INFO, "ICR0: global reset requested");
6720 	if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
6721 		PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
6722 	if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
6723 		PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
6724 	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
6725 		PMD_DRV_LOG(ERR, "ICR0: HMC error");
6726 	if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
6727 		PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
6728 
6729 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
6730 		PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
6731 		i40e_dev_handle_vfr_event(dev);
6732 	}
6733 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
6734 		PMD_DRV_LOG(INFO, "ICR0: adminq event");
6735 		i40e_dev_handle_aq_msg(dev);
6736 	}
6737 
6738 done:
6739 	/* Enable interrupt */
6740 	i40e_pf_enable_irq0(hw);
6741 	rte_eal_alarm_set(I40E_ALARM_INTERVAL,
6742 			  i40e_dev_alarm_handler, dev);
6743 }
6744 
6745 int
6746 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
6747 			 struct i40e_macvlan_filter *filter,
6748 			 int total)
6749 {
6750 	int ele_num, ele_buff_size;
6751 	int num, actual_num, i;
6752 	uint16_t flags;
6753 	int ret = I40E_SUCCESS;
6754 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6755 	struct i40e_aqc_add_macvlan_element_data *req_list;
6756 
6757 	if (filter == NULL  || total == 0)
6758 		return I40E_ERR_PARAM;
6759 	ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
6760 	ele_buff_size = hw->aq.asq_buf_size;
6761 
6762 	req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
6763 	if (req_list == NULL) {
6764 		PMD_DRV_LOG(ERR, "Fail to allocate memory");
6765 		return I40E_ERR_NO_MEMORY;
6766 	}
6767 
6768 	num = 0;
6769 	do {
6770 		actual_num = (num + ele_num > total) ? (total - num) : ele_num;
6771 		memset(req_list, 0, ele_buff_size);
6772 
6773 		for (i = 0; i < actual_num; i++) {
6774 			rte_memcpy(req_list[i].mac_addr,
6775 				&filter[num + i].macaddr, ETH_ADDR_LEN);
6776 			req_list[i].vlan_tag =
6777 				rte_cpu_to_le_16(filter[num + i].vlan_id);
6778 
6779 			switch (filter[num + i].filter_type) {
6780 			case RTE_MAC_PERFECT_MATCH:
6781 				flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
6782 					I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
6783 				break;
6784 			case RTE_MACVLAN_PERFECT_MATCH:
6785 				flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
6786 				break;
6787 			case RTE_MAC_HASH_MATCH:
6788 				flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
6789 					I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
6790 				break;
6791 			case RTE_MACVLAN_HASH_MATCH:
6792 				flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
6793 				break;
6794 			default:
6795 				PMD_DRV_LOG(ERR, "Invalid MAC match type");
6796 				ret = I40E_ERR_PARAM;
6797 				goto DONE;
6798 			}
6799 
6800 			req_list[i].queue_number = 0;
6801 
6802 			req_list[i].flags = rte_cpu_to_le_16(flags);
6803 		}
6804 
6805 		ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
6806 						actual_num, NULL);
6807 		if (ret != I40E_SUCCESS) {
6808 			PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
6809 			goto DONE;
6810 		}
6811 		num += actual_num;
6812 	} while (num < total);
6813 
6814 DONE:
6815 	rte_free(req_list);
6816 	return ret;
6817 }
6818 
6819 int
6820 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
6821 			    struct i40e_macvlan_filter *filter,
6822 			    int total)
6823 {
6824 	int ele_num, ele_buff_size;
6825 	int num, actual_num, i;
6826 	uint16_t flags;
6827 	int ret = I40E_SUCCESS;
6828 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6829 	struct i40e_aqc_remove_macvlan_element_data *req_list;
6830 
6831 	if (filter == NULL  || total == 0)
6832 		return I40E_ERR_PARAM;
6833 
6834 	ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
6835 	ele_buff_size = hw->aq.asq_buf_size;
6836 
6837 	req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
6838 	if (req_list == NULL) {
6839 		PMD_DRV_LOG(ERR, "Fail to allocate memory");
6840 		return I40E_ERR_NO_MEMORY;
6841 	}
6842 
6843 	num = 0;
6844 	do {
6845 		actual_num = (num + ele_num > total) ? (total - num) : ele_num;
6846 		memset(req_list, 0, ele_buff_size);
6847 
6848 		for (i = 0; i < actual_num; i++) {
6849 			rte_memcpy(req_list[i].mac_addr,
6850 				&filter[num + i].macaddr, ETH_ADDR_LEN);
6851 			req_list[i].vlan_tag =
6852 				rte_cpu_to_le_16(filter[num + i].vlan_id);
6853 
6854 			switch (filter[num + i].filter_type) {
6855 			case RTE_MAC_PERFECT_MATCH:
6856 				flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
6857 					I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
6858 				break;
6859 			case RTE_MACVLAN_PERFECT_MATCH:
6860 				flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
6861 				break;
6862 			case RTE_MAC_HASH_MATCH:
6863 				flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
6864 					I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
6865 				break;
6866 			case RTE_MACVLAN_HASH_MATCH:
6867 				flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
6868 				break;
6869 			default:
6870 				PMD_DRV_LOG(ERR, "Invalid MAC filter type");
6871 				ret = I40E_ERR_PARAM;
6872 				goto DONE;
6873 			}
6874 			req_list[i].flags = rte_cpu_to_le_16(flags);
6875 		}
6876 
6877 		ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
6878 						actual_num, NULL);
6879 		if (ret != I40E_SUCCESS) {
6880 			PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
6881 			goto DONE;
6882 		}
6883 		num += actual_num;
6884 	} while (num < total);
6885 
6886 DONE:
6887 	rte_free(req_list);
6888 	return ret;
6889 }
6890 
6891 /* Find out specific MAC filter */
6892 static struct i40e_mac_filter *
6893 i40e_find_mac_filter(struct i40e_vsi *vsi,
6894 			 struct ether_addr *macaddr)
6895 {
6896 	struct i40e_mac_filter *f;
6897 
6898 	TAILQ_FOREACH(f, &vsi->mac_list, next) {
6899 		if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
6900 			return f;
6901 	}
6902 
6903 	return NULL;
6904 }
6905 
6906 static bool
6907 i40e_find_vlan_filter(struct i40e_vsi *vsi,
6908 			 uint16_t vlan_id)
6909 {
6910 	uint32_t vid_idx, vid_bit;
6911 
6912 	if (vlan_id > ETH_VLAN_ID_MAX)
6913 		return 0;
6914 
6915 	vid_idx = I40E_VFTA_IDX(vlan_id);
6916 	vid_bit = I40E_VFTA_BIT(vlan_id);
6917 
6918 	if (vsi->vfta[vid_idx] & vid_bit)
6919 		return 1;
6920 	else
6921 		return 0;
6922 }
6923 
6924 static void
6925 i40e_store_vlan_filter(struct i40e_vsi *vsi,
6926 		       uint16_t vlan_id, bool on)
6927 {
6928 	uint32_t vid_idx, vid_bit;
6929 
6930 	vid_idx = I40E_VFTA_IDX(vlan_id);
6931 	vid_bit = I40E_VFTA_BIT(vlan_id);
6932 
6933 	if (on)
6934 		vsi->vfta[vid_idx] |= vid_bit;
6935 	else
6936 		vsi->vfta[vid_idx] &= ~vid_bit;
6937 }
6938 
6939 void
6940 i40e_set_vlan_filter(struct i40e_vsi *vsi,
6941 		     uint16_t vlan_id, bool on)
6942 {
6943 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6944 	struct i40e_aqc_add_remove_vlan_element_data vlan_data = {0};
6945 	int ret;
6946 
6947 	if (vlan_id > ETH_VLAN_ID_MAX)
6948 		return;
6949 
6950 	i40e_store_vlan_filter(vsi, vlan_id, on);
6951 
6952 	if ((!vsi->vlan_anti_spoof_on && !vsi->vlan_filter_on) || !vlan_id)
6953 		return;
6954 
6955 	vlan_data.vlan_tag = rte_cpu_to_le_16(vlan_id);
6956 
6957 	if (on) {
6958 		ret = i40e_aq_add_vlan(hw, vsi->seid,
6959 				       &vlan_data, 1, NULL);
6960 		if (ret != I40E_SUCCESS)
6961 			PMD_DRV_LOG(ERR, "Failed to add vlan filter");
6962 	} else {
6963 		ret = i40e_aq_remove_vlan(hw, vsi->seid,
6964 					  &vlan_data, 1, NULL);
6965 		if (ret != I40E_SUCCESS)
6966 			PMD_DRV_LOG(ERR,
6967 				    "Failed to remove vlan filter");
6968 	}
6969 }
6970 
6971 /**
6972  * Find all vlan options for specific mac addr,
6973  * return with actual vlan found.
6974  */
6975 int
6976 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
6977 			   struct i40e_macvlan_filter *mv_f,
6978 			   int num, struct ether_addr *addr)
6979 {
6980 	int i;
6981 	uint32_t j, k;
6982 
6983 	/**
6984 	 * Not to use i40e_find_vlan_filter to decrease the loop time,
6985 	 * although the code looks complex.
6986 	  */
6987 	if (num < vsi->vlan_num)
6988 		return I40E_ERR_PARAM;
6989 
6990 	i = 0;
6991 	for (j = 0; j < I40E_VFTA_SIZE; j++) {
6992 		if (vsi->vfta[j]) {
6993 			for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
6994 				if (vsi->vfta[j] & (1 << k)) {
6995 					if (i > num - 1) {
6996 						PMD_DRV_LOG(ERR,
6997 							"vlan number doesn't match");
6998 						return I40E_ERR_PARAM;
6999 					}
7000 					rte_memcpy(&mv_f[i].macaddr,
7001 							addr, ETH_ADDR_LEN);
7002 					mv_f[i].vlan_id =
7003 						j * I40E_UINT32_BIT_SIZE + k;
7004 					i++;
7005 				}
7006 			}
7007 		}
7008 	}
7009 	return I40E_SUCCESS;
7010 }
7011 
7012 static inline int
7013 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
7014 			   struct i40e_macvlan_filter *mv_f,
7015 			   int num,
7016 			   uint16_t vlan)
7017 {
7018 	int i = 0;
7019 	struct i40e_mac_filter *f;
7020 
7021 	if (num < vsi->mac_num)
7022 		return I40E_ERR_PARAM;
7023 
7024 	TAILQ_FOREACH(f, &vsi->mac_list, next) {
7025 		if (i > num - 1) {
7026 			PMD_DRV_LOG(ERR, "buffer number not match");
7027 			return I40E_ERR_PARAM;
7028 		}
7029 		rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
7030 				ETH_ADDR_LEN);
7031 		mv_f[i].vlan_id = vlan;
7032 		mv_f[i].filter_type = f->mac_info.filter_type;
7033 		i++;
7034 	}
7035 
7036 	return I40E_SUCCESS;
7037 }
7038 
7039 static int
7040 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
7041 {
7042 	int i, j, num;
7043 	struct i40e_mac_filter *f;
7044 	struct i40e_macvlan_filter *mv_f;
7045 	int ret = I40E_SUCCESS;
7046 
7047 	if (vsi == NULL || vsi->mac_num == 0)
7048 		return I40E_ERR_PARAM;
7049 
7050 	/* Case that no vlan is set */
7051 	if (vsi->vlan_num == 0)
7052 		num = vsi->mac_num;
7053 	else
7054 		num = vsi->mac_num * vsi->vlan_num;
7055 
7056 	mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
7057 	if (mv_f == NULL) {
7058 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7059 		return I40E_ERR_NO_MEMORY;
7060 	}
7061 
7062 	i = 0;
7063 	if (vsi->vlan_num == 0) {
7064 		TAILQ_FOREACH(f, &vsi->mac_list, next) {
7065 			rte_memcpy(&mv_f[i].macaddr,
7066 				&f->mac_info.mac_addr, ETH_ADDR_LEN);
7067 			mv_f[i].filter_type = f->mac_info.filter_type;
7068 			mv_f[i].vlan_id = 0;
7069 			i++;
7070 		}
7071 	} else {
7072 		TAILQ_FOREACH(f, &vsi->mac_list, next) {
7073 			ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
7074 					vsi->vlan_num, &f->mac_info.mac_addr);
7075 			if (ret != I40E_SUCCESS)
7076 				goto DONE;
7077 			for (j = i; j < i + vsi->vlan_num; j++)
7078 				mv_f[j].filter_type = f->mac_info.filter_type;
7079 			i += vsi->vlan_num;
7080 		}
7081 	}
7082 
7083 	ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
7084 DONE:
7085 	rte_free(mv_f);
7086 
7087 	return ret;
7088 }
7089 
7090 int
7091 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
7092 {
7093 	struct i40e_macvlan_filter *mv_f;
7094 	int mac_num;
7095 	int ret = I40E_SUCCESS;
7096 
7097 	if (!vsi || vlan > ETHER_MAX_VLAN_ID)
7098 		return I40E_ERR_PARAM;
7099 
7100 	/* If it's already set, just return */
7101 	if (i40e_find_vlan_filter(vsi,vlan))
7102 		return I40E_SUCCESS;
7103 
7104 	mac_num = vsi->mac_num;
7105 
7106 	if (mac_num == 0) {
7107 		PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
7108 		return I40E_ERR_PARAM;
7109 	}
7110 
7111 	mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
7112 
7113 	if (mv_f == NULL) {
7114 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7115 		return I40E_ERR_NO_MEMORY;
7116 	}
7117 
7118 	ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
7119 
7120 	if (ret != I40E_SUCCESS)
7121 		goto DONE;
7122 
7123 	ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
7124 
7125 	if (ret != I40E_SUCCESS)
7126 		goto DONE;
7127 
7128 	i40e_set_vlan_filter(vsi, vlan, 1);
7129 
7130 	vsi->vlan_num++;
7131 	ret = I40E_SUCCESS;
7132 DONE:
7133 	rte_free(mv_f);
7134 	return ret;
7135 }
7136 
7137 int
7138 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
7139 {
7140 	struct i40e_macvlan_filter *mv_f;
7141 	int mac_num;
7142 	int ret = I40E_SUCCESS;
7143 
7144 	/**
7145 	 * Vlan 0 is the generic filter for untagged packets
7146 	 * and can't be removed.
7147 	 */
7148 	if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
7149 		return I40E_ERR_PARAM;
7150 
7151 	/* If can't find it, just return */
7152 	if (!i40e_find_vlan_filter(vsi, vlan))
7153 		return I40E_ERR_PARAM;
7154 
7155 	mac_num = vsi->mac_num;
7156 
7157 	if (mac_num == 0) {
7158 		PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
7159 		return I40E_ERR_PARAM;
7160 	}
7161 
7162 	mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
7163 
7164 	if (mv_f == NULL) {
7165 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7166 		return I40E_ERR_NO_MEMORY;
7167 	}
7168 
7169 	ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
7170 
7171 	if (ret != I40E_SUCCESS)
7172 		goto DONE;
7173 
7174 	ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
7175 
7176 	if (ret != I40E_SUCCESS)
7177 		goto DONE;
7178 
7179 	/* This is last vlan to remove, replace all mac filter with vlan 0 */
7180 	if (vsi->vlan_num == 1) {
7181 		ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
7182 		if (ret != I40E_SUCCESS)
7183 			goto DONE;
7184 
7185 		ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
7186 		if (ret != I40E_SUCCESS)
7187 			goto DONE;
7188 	}
7189 
7190 	i40e_set_vlan_filter(vsi, vlan, 0);
7191 
7192 	vsi->vlan_num--;
7193 	ret = I40E_SUCCESS;
7194 DONE:
7195 	rte_free(mv_f);
7196 	return ret;
7197 }
7198 
7199 int
7200 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
7201 {
7202 	struct i40e_mac_filter *f;
7203 	struct i40e_macvlan_filter *mv_f;
7204 	int i, vlan_num = 0;
7205 	int ret = I40E_SUCCESS;
7206 
7207 	/* If it's add and we've config it, return */
7208 	f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
7209 	if (f != NULL)
7210 		return I40E_SUCCESS;
7211 	if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
7212 		(mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
7213 
7214 		/**
7215 		 * If vlan_num is 0, that's the first time to add mac,
7216 		 * set mask for vlan_id 0.
7217 		 */
7218 		if (vsi->vlan_num == 0) {
7219 			i40e_set_vlan_filter(vsi, 0, 1);
7220 			vsi->vlan_num = 1;
7221 		}
7222 		vlan_num = vsi->vlan_num;
7223 	} else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
7224 			(mac_filter->filter_type == RTE_MAC_HASH_MATCH))
7225 		vlan_num = 1;
7226 
7227 	mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
7228 	if (mv_f == NULL) {
7229 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7230 		return I40E_ERR_NO_MEMORY;
7231 	}
7232 
7233 	for (i = 0; i < vlan_num; i++) {
7234 		mv_f[i].filter_type = mac_filter->filter_type;
7235 		rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
7236 				ETH_ADDR_LEN);
7237 	}
7238 
7239 	if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
7240 		mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
7241 		ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
7242 					&mac_filter->mac_addr);
7243 		if (ret != I40E_SUCCESS)
7244 			goto DONE;
7245 	}
7246 
7247 	ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
7248 	if (ret != I40E_SUCCESS)
7249 		goto DONE;
7250 
7251 	/* Add the mac addr into mac list */
7252 	f = rte_zmalloc("macv_filter", sizeof(*f), 0);
7253 	if (f == NULL) {
7254 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7255 		ret = I40E_ERR_NO_MEMORY;
7256 		goto DONE;
7257 	}
7258 	rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
7259 			ETH_ADDR_LEN);
7260 	f->mac_info.filter_type = mac_filter->filter_type;
7261 	TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
7262 	vsi->mac_num++;
7263 
7264 	ret = I40E_SUCCESS;
7265 DONE:
7266 	rte_free(mv_f);
7267 
7268 	return ret;
7269 }
7270 
7271 int
7272 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
7273 {
7274 	struct i40e_mac_filter *f;
7275 	struct i40e_macvlan_filter *mv_f;
7276 	int i, vlan_num;
7277 	enum rte_mac_filter_type filter_type;
7278 	int ret = I40E_SUCCESS;
7279 
7280 	/* Can't find it, return an error */
7281 	f = i40e_find_mac_filter(vsi, addr);
7282 	if (f == NULL)
7283 		return I40E_ERR_PARAM;
7284 
7285 	vlan_num = vsi->vlan_num;
7286 	filter_type = f->mac_info.filter_type;
7287 	if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
7288 		filter_type == RTE_MACVLAN_HASH_MATCH) {
7289 		if (vlan_num == 0) {
7290 			PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0");
7291 			return I40E_ERR_PARAM;
7292 		}
7293 	} else if (filter_type == RTE_MAC_PERFECT_MATCH ||
7294 			filter_type == RTE_MAC_HASH_MATCH)
7295 		vlan_num = 1;
7296 
7297 	mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
7298 	if (mv_f == NULL) {
7299 		PMD_DRV_LOG(ERR, "failed to allocate memory");
7300 		return I40E_ERR_NO_MEMORY;
7301 	}
7302 
7303 	for (i = 0; i < vlan_num; i++) {
7304 		mv_f[i].filter_type = filter_type;
7305 		rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
7306 				ETH_ADDR_LEN);
7307 	}
7308 	if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
7309 			filter_type == RTE_MACVLAN_HASH_MATCH) {
7310 		ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
7311 		if (ret != I40E_SUCCESS)
7312 			goto DONE;
7313 	}
7314 
7315 	ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
7316 	if (ret != I40E_SUCCESS)
7317 		goto DONE;
7318 
7319 	/* Remove the mac addr into mac list */
7320 	TAILQ_REMOVE(&vsi->mac_list, f, next);
7321 	rte_free(f);
7322 	vsi->mac_num--;
7323 
7324 	ret = I40E_SUCCESS;
7325 DONE:
7326 	rte_free(mv_f);
7327 	return ret;
7328 }
7329 
7330 /* Configure hash enable flags for RSS */
7331 uint64_t
7332 i40e_config_hena(const struct i40e_adapter *adapter, uint64_t flags)
7333 {
7334 	uint64_t hena = 0;
7335 	int i;
7336 
7337 	if (!flags)
7338 		return hena;
7339 
7340 	for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < I40E_FLOW_TYPE_MAX; i++) {
7341 		if (flags & (1ULL << i))
7342 			hena |= adapter->pctypes_tbl[i];
7343 	}
7344 
7345 	return hena;
7346 }
7347 
7348 /* Parse the hash enable flags */
7349 uint64_t
7350 i40e_parse_hena(const struct i40e_adapter *adapter, uint64_t flags)
7351 {
7352 	uint64_t rss_hf = 0;
7353 
7354 	if (!flags)
7355 		return rss_hf;
7356 	int i;
7357 
7358 	for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < I40E_FLOW_TYPE_MAX; i++) {
7359 		if (flags & adapter->pctypes_tbl[i])
7360 			rss_hf |= (1ULL << i);
7361 	}
7362 	return rss_hf;
7363 }
7364 
7365 /* Disable RSS */
7366 static void
7367 i40e_pf_disable_rss(struct i40e_pf *pf)
7368 {
7369 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7370 
7371 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
7372 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
7373 	I40E_WRITE_FLUSH(hw);
7374 }
7375 
7376 int
7377 i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
7378 {
7379 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
7380 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
7381 	uint16_t key_idx = (vsi->type == I40E_VSI_SRIOV) ?
7382 			   I40E_VFQF_HKEY_MAX_INDEX :
7383 			   I40E_PFQF_HKEY_MAX_INDEX;
7384 	int ret = 0;
7385 
7386 	if (!key || key_len == 0) {
7387 		PMD_DRV_LOG(DEBUG, "No key to be configured");
7388 		return 0;
7389 	} else if (key_len != (key_idx + 1) *
7390 		sizeof(uint32_t)) {
7391 		PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
7392 		return -EINVAL;
7393 	}
7394 
7395 	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
7396 		struct i40e_aqc_get_set_rss_key_data *key_dw =
7397 			(struct i40e_aqc_get_set_rss_key_data *)key;
7398 
7399 		ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
7400 		if (ret)
7401 			PMD_INIT_LOG(ERR, "Failed to configure RSS key via AQ");
7402 	} else {
7403 		uint32_t *hash_key = (uint32_t *)key;
7404 		uint16_t i;
7405 
7406 		if (vsi->type == I40E_VSI_SRIOV) {
7407 			for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
7408 				I40E_WRITE_REG(
7409 					hw,
7410 					I40E_VFQF_HKEY1(i, vsi->user_param),
7411 					hash_key[i]);
7412 
7413 		} else {
7414 			for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
7415 				I40E_WRITE_REG(hw, I40E_PFQF_HKEY(i),
7416 					       hash_key[i]);
7417 		}
7418 		I40E_WRITE_FLUSH(hw);
7419 	}
7420 
7421 	return ret;
7422 }
7423 
7424 static int
7425 i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
7426 {
7427 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
7428 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
7429 	uint32_t reg;
7430 	int ret;
7431 
7432 	if (!key || !key_len)
7433 		return 0;
7434 
7435 	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
7436 		ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
7437 			(struct i40e_aqc_get_set_rss_key_data *)key);
7438 		if (ret) {
7439 			PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
7440 			return ret;
7441 		}
7442 	} else {
7443 		uint32_t *key_dw = (uint32_t *)key;
7444 		uint16_t i;
7445 
7446 		if (vsi->type == I40E_VSI_SRIOV) {
7447 			for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++) {
7448 				reg = I40E_VFQF_HKEY1(i, vsi->user_param);
7449 				key_dw[i] = i40e_read_rx_ctl(hw, reg);
7450 			}
7451 			*key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
7452 				   sizeof(uint32_t);
7453 		} else {
7454 			for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) {
7455 				reg = I40E_PFQF_HKEY(i);
7456 				key_dw[i] = i40e_read_rx_ctl(hw, reg);
7457 			}
7458 			*key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
7459 				   sizeof(uint32_t);
7460 		}
7461 	}
7462 	return 0;
7463 }
7464 
7465 static int
7466 i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf)
7467 {
7468 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7469 	uint64_t hena;
7470 	int ret;
7471 
7472 	ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key,
7473 			       rss_conf->rss_key_len);
7474 	if (ret)
7475 		return ret;
7476 
7477 	hena = i40e_config_hena(pf->adapter, rss_conf->rss_hf);
7478 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
7479 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
7480 	I40E_WRITE_FLUSH(hw);
7481 
7482 	return 0;
7483 }
7484 
7485 static int
7486 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
7487 			 struct rte_eth_rss_conf *rss_conf)
7488 {
7489 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7490 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7491 	uint64_t rss_hf = rss_conf->rss_hf & pf->adapter->flow_types_mask;
7492 	uint64_t hena;
7493 
7494 	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
7495 	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
7496 
7497 	if (!(hena & pf->adapter->pctypes_mask)) { /* RSS disabled */
7498 		if (rss_hf != 0) /* Enable RSS */
7499 			return -EINVAL;
7500 		return 0; /* Nothing to do */
7501 	}
7502 	/* RSS enabled */
7503 	if (rss_hf == 0) /* Disable RSS */
7504 		return -EINVAL;
7505 
7506 	return i40e_hw_rss_hash_set(pf, rss_conf);
7507 }
7508 
7509 static int
7510 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
7511 			   struct rte_eth_rss_conf *rss_conf)
7512 {
7513 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7514 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7515 	uint64_t hena;
7516 	int ret;
7517 
7518 	if (!rss_conf)
7519 		return -EINVAL;
7520 
7521 	ret = i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key,
7522 			 &rss_conf->rss_key_len);
7523 	if (ret)
7524 		return ret;
7525 
7526 	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
7527 	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
7528 	rss_conf->rss_hf = i40e_parse_hena(pf->adapter, hena);
7529 
7530 	return 0;
7531 }
7532 
7533 static int
7534 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
7535 {
7536 	switch (filter_type) {
7537 	case RTE_TUNNEL_FILTER_IMAC_IVLAN:
7538 		*flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
7539 		break;
7540 	case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
7541 		*flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
7542 		break;
7543 	case RTE_TUNNEL_FILTER_IMAC_TENID:
7544 		*flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
7545 		break;
7546 	case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
7547 		*flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
7548 		break;
7549 	case ETH_TUNNEL_FILTER_IMAC:
7550 		*flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
7551 		break;
7552 	case ETH_TUNNEL_FILTER_OIP:
7553 		*flag = I40E_AQC_ADD_CLOUD_FILTER_OIP;
7554 		break;
7555 	case ETH_TUNNEL_FILTER_IIP:
7556 		*flag = I40E_AQC_ADD_CLOUD_FILTER_IIP;
7557 		break;
7558 	default:
7559 		PMD_DRV_LOG(ERR, "invalid tunnel filter type");
7560 		return -EINVAL;
7561 	}
7562 
7563 	return 0;
7564 }
7565 
7566 /* Convert tunnel filter structure */
7567 static int
7568 i40e_tunnel_filter_convert(
7569 	struct i40e_aqc_cloud_filters_element_bb *cld_filter,
7570 	struct i40e_tunnel_filter *tunnel_filter)
7571 {
7572 	ether_addr_copy((struct ether_addr *)&cld_filter->element.outer_mac,
7573 			(struct ether_addr *)&tunnel_filter->input.outer_mac);
7574 	ether_addr_copy((struct ether_addr *)&cld_filter->element.inner_mac,
7575 			(struct ether_addr *)&tunnel_filter->input.inner_mac);
7576 	tunnel_filter->input.inner_vlan = cld_filter->element.inner_vlan;
7577 	if ((rte_le_to_cpu_16(cld_filter->element.flags) &
7578 	     I40E_AQC_ADD_CLOUD_FLAGS_IPV6) ==
7579 	    I40E_AQC_ADD_CLOUD_FLAGS_IPV6)
7580 		tunnel_filter->input.ip_type = I40E_TUNNEL_IPTYPE_IPV6;
7581 	else
7582 		tunnel_filter->input.ip_type = I40E_TUNNEL_IPTYPE_IPV4;
7583 	tunnel_filter->input.flags = cld_filter->element.flags;
7584 	tunnel_filter->input.tenant_id = cld_filter->element.tenant_id;
7585 	tunnel_filter->queue = cld_filter->element.queue_number;
7586 	rte_memcpy(tunnel_filter->input.general_fields,
7587 		   cld_filter->general_fields,
7588 		   sizeof(cld_filter->general_fields));
7589 
7590 	return 0;
7591 }
7592 
7593 /* Check if there exists the tunnel filter */
7594 struct i40e_tunnel_filter *
7595 i40e_sw_tunnel_filter_lookup(struct i40e_tunnel_rule *tunnel_rule,
7596 			     const struct i40e_tunnel_filter_input *input)
7597 {
7598 	int ret;
7599 
7600 	ret = rte_hash_lookup(tunnel_rule->hash_table, (const void *)input);
7601 	if (ret < 0)
7602 		return NULL;
7603 
7604 	return tunnel_rule->hash_map[ret];
7605 }
7606 
7607 /* Add a tunnel filter into the SW list */
7608 static int
7609 i40e_sw_tunnel_filter_insert(struct i40e_pf *pf,
7610 			     struct i40e_tunnel_filter *tunnel_filter)
7611 {
7612 	struct i40e_tunnel_rule *rule = &pf->tunnel;
7613 	int ret;
7614 
7615 	ret = rte_hash_add_key(rule->hash_table, &tunnel_filter->input);
7616 	if (ret < 0) {
7617 		PMD_DRV_LOG(ERR,
7618 			    "Failed to insert tunnel filter to hash table %d!",
7619 			    ret);
7620 		return ret;
7621 	}
7622 	rule->hash_map[ret] = tunnel_filter;
7623 
7624 	TAILQ_INSERT_TAIL(&rule->tunnel_list, tunnel_filter, rules);
7625 
7626 	return 0;
7627 }
7628 
7629 /* Delete a tunnel filter from the SW list */
7630 int
7631 i40e_sw_tunnel_filter_del(struct i40e_pf *pf,
7632 			  struct i40e_tunnel_filter_input *input)
7633 {
7634 	struct i40e_tunnel_rule *rule = &pf->tunnel;
7635 	struct i40e_tunnel_filter *tunnel_filter;
7636 	int ret;
7637 
7638 	ret = rte_hash_del_key(rule->hash_table, input);
7639 	if (ret < 0) {
7640 		PMD_DRV_LOG(ERR,
7641 			    "Failed to delete tunnel filter to hash table %d!",
7642 			    ret);
7643 		return ret;
7644 	}
7645 	tunnel_filter = rule->hash_map[ret];
7646 	rule->hash_map[ret] = NULL;
7647 
7648 	TAILQ_REMOVE(&rule->tunnel_list, tunnel_filter, rules);
7649 	rte_free(tunnel_filter);
7650 
7651 	return 0;
7652 }
7653 
7654 int
7655 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
7656 			struct rte_eth_tunnel_filter_conf *tunnel_filter,
7657 			uint8_t add)
7658 {
7659 	uint16_t ip_type;
7660 	uint32_t ipv4_addr, ipv4_addr_le;
7661 	uint8_t i, tun_type = 0;
7662 	/* internal varialbe to convert ipv6 byte order */
7663 	uint32_t convert_ipv6[4];
7664 	int val, ret = 0;
7665 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7666 	struct i40e_vsi *vsi = pf->main_vsi;
7667 	struct i40e_aqc_cloud_filters_element_bb *cld_filter;
7668 	struct i40e_aqc_cloud_filters_element_bb *pfilter;
7669 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
7670 	struct i40e_tunnel_filter *tunnel, *node;
7671 	struct i40e_tunnel_filter check_filter; /* Check if filter exists */
7672 
7673 	cld_filter = rte_zmalloc("tunnel_filter",
7674 			 sizeof(struct i40e_aqc_add_rm_cloud_filt_elem_ext),
7675 	0);
7676 
7677 	if (NULL == cld_filter) {
7678 		PMD_DRV_LOG(ERR, "Failed to alloc memory.");
7679 		return -ENOMEM;
7680 	}
7681 	pfilter = cld_filter;
7682 
7683 	ether_addr_copy(&tunnel_filter->outer_mac,
7684 			(struct ether_addr *)&pfilter->element.outer_mac);
7685 	ether_addr_copy(&tunnel_filter->inner_mac,
7686 			(struct ether_addr *)&pfilter->element.inner_mac);
7687 
7688 	pfilter->element.inner_vlan =
7689 		rte_cpu_to_le_16(tunnel_filter->inner_vlan);
7690 	if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
7691 		ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
7692 		ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr);
7693 		ipv4_addr_le = rte_cpu_to_le_32(ipv4_addr);
7694 		rte_memcpy(&pfilter->element.ipaddr.v4.data,
7695 				&ipv4_addr_le,
7696 				sizeof(pfilter->element.ipaddr.v4.data));
7697 	} else {
7698 		ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
7699 		for (i = 0; i < 4; i++) {
7700 			convert_ipv6[i] =
7701 			rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i]));
7702 		}
7703 		rte_memcpy(&pfilter->element.ipaddr.v6.data,
7704 			   &convert_ipv6,
7705 			   sizeof(pfilter->element.ipaddr.v6.data));
7706 	}
7707 
7708 	/* check tunneled type */
7709 	switch (tunnel_filter->tunnel_type) {
7710 	case RTE_TUNNEL_TYPE_VXLAN:
7711 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
7712 		break;
7713 	case RTE_TUNNEL_TYPE_NVGRE:
7714 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
7715 		break;
7716 	case RTE_TUNNEL_TYPE_IP_IN_GRE:
7717 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
7718 		break;
7719 	default:
7720 		/* Other tunnel types is not supported. */
7721 		PMD_DRV_LOG(ERR, "tunnel type is not supported.");
7722 		rte_free(cld_filter);
7723 		return -EINVAL;
7724 	}
7725 
7726 	val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
7727 				       &pfilter->element.flags);
7728 	if (val < 0) {
7729 		rte_free(cld_filter);
7730 		return -EINVAL;
7731 	}
7732 
7733 	pfilter->element.flags |= rte_cpu_to_le_16(
7734 		I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
7735 		ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
7736 	pfilter->element.tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
7737 	pfilter->element.queue_number =
7738 		rte_cpu_to_le_16(tunnel_filter->queue_id);
7739 
7740 	/* Check if there is the filter in SW list */
7741 	memset(&check_filter, 0, sizeof(check_filter));
7742 	i40e_tunnel_filter_convert(cld_filter, &check_filter);
7743 	node = i40e_sw_tunnel_filter_lookup(tunnel_rule, &check_filter.input);
7744 	if (add && node) {
7745 		PMD_DRV_LOG(ERR, "Conflict with existing tunnel rules!");
7746 		rte_free(cld_filter);
7747 		return -EINVAL;
7748 	}
7749 
7750 	if (!add && !node) {
7751 		PMD_DRV_LOG(ERR, "There's no corresponding tunnel filter!");
7752 		rte_free(cld_filter);
7753 		return -EINVAL;
7754 	}
7755 
7756 	if (add) {
7757 		ret = i40e_aq_add_cloud_filters(hw,
7758 					vsi->seid, &cld_filter->element, 1);
7759 		if (ret < 0) {
7760 			PMD_DRV_LOG(ERR, "Failed to add a tunnel filter.");
7761 			rte_free(cld_filter);
7762 			return -ENOTSUP;
7763 		}
7764 		tunnel = rte_zmalloc("tunnel_filter", sizeof(*tunnel), 0);
7765 		if (tunnel == NULL) {
7766 			PMD_DRV_LOG(ERR, "Failed to alloc memory.");
7767 			rte_free(cld_filter);
7768 			return -ENOMEM;
7769 		}
7770 
7771 		rte_memcpy(tunnel, &check_filter, sizeof(check_filter));
7772 		ret = i40e_sw_tunnel_filter_insert(pf, tunnel);
7773 		if (ret < 0)
7774 			rte_free(tunnel);
7775 	} else {
7776 		ret = i40e_aq_rem_cloud_filters(hw, vsi->seid,
7777 						   &cld_filter->element, 1);
7778 		if (ret < 0) {
7779 			PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter.");
7780 			rte_free(cld_filter);
7781 			return -ENOTSUP;
7782 		}
7783 		ret = i40e_sw_tunnel_filter_del(pf, &node->input);
7784 	}
7785 
7786 	rte_free(cld_filter);
7787 	return ret;
7788 }
7789 
7790 #define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_TR_WORD0 0x48
7791 #define I40E_TR_VXLAN_GRE_KEY_MASK		0x4
7792 #define I40E_TR_GENEVE_KEY_MASK			0x8
7793 #define I40E_TR_GENERIC_UDP_TUNNEL_MASK		0x40
7794 #define I40E_TR_GRE_KEY_MASK			0x400
7795 #define I40E_TR_GRE_KEY_WITH_XSUM_MASK		0x800
7796 #define I40E_TR_GRE_NO_KEY_MASK			0x8000
7797 
7798 static enum
7799 i40e_status_code i40e_replace_mpls_l1_filter(struct i40e_pf *pf)
7800 {
7801 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
7802 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
7803 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7804 	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
7805 	enum i40e_status_code status = I40E_SUCCESS;
7806 
7807 	if (pf->support_multi_driver) {
7808 		PMD_DRV_LOG(ERR, "Replace l1 filter is not supported.");
7809 		return I40E_NOT_SUPPORTED;
7810 	}
7811 
7812 	memset(&filter_replace, 0,
7813 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
7814 	memset(&filter_replace_buf, 0,
7815 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
7816 
7817 	/* create L1 filter */
7818 	filter_replace.old_filter_type =
7819 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IMAC;
7820 	filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X11;
7821 	filter_replace.tr_bit = 0;
7822 
7823 	/* Prepare the buffer, 3 entries */
7824 	filter_replace_buf.data[0] =
7825 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0;
7826 	filter_replace_buf.data[0] |=
7827 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7828 	filter_replace_buf.data[2] = 0xFF;
7829 	filter_replace_buf.data[3] = 0xFF;
7830 	filter_replace_buf.data[4] =
7831 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1;
7832 	filter_replace_buf.data[4] |=
7833 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7834 	filter_replace_buf.data[7] = 0xF0;
7835 	filter_replace_buf.data[8]
7836 		= I40E_AQC_REPLACE_CLOUD_CMD_INPUT_TR_WORD0;
7837 	filter_replace_buf.data[8] |=
7838 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7839 	filter_replace_buf.data[10] = I40E_TR_VXLAN_GRE_KEY_MASK |
7840 		I40E_TR_GENEVE_KEY_MASK |
7841 		I40E_TR_GENERIC_UDP_TUNNEL_MASK;
7842 	filter_replace_buf.data[11] = (I40E_TR_GRE_KEY_MASK |
7843 		I40E_TR_GRE_KEY_WITH_XSUM_MASK |
7844 		I40E_TR_GRE_NO_KEY_MASK) >> 8;
7845 
7846 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
7847 					       &filter_replace_buf);
7848 	if (!status && (filter_replace.old_filter_type !=
7849 			filter_replace.new_filter_type))
7850 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
7851 			    " original: 0x%x, new: 0x%x",
7852 			    dev->device->name,
7853 			    filter_replace.old_filter_type,
7854 			    filter_replace.new_filter_type);
7855 
7856 	return status;
7857 }
7858 
7859 static enum
7860 i40e_status_code i40e_replace_mpls_cloud_filter(struct i40e_pf *pf)
7861 {
7862 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
7863 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
7864 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7865 	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
7866 	enum i40e_status_code status = I40E_SUCCESS;
7867 
7868 	if (pf->support_multi_driver) {
7869 		PMD_DRV_LOG(ERR, "Replace cloud filter is not supported.");
7870 		return I40E_NOT_SUPPORTED;
7871 	}
7872 
7873 	/* For MPLSoUDP */
7874 	memset(&filter_replace, 0,
7875 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
7876 	memset(&filter_replace_buf, 0,
7877 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
7878 	filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER |
7879 		I40E_AQC_MIRROR_CLOUD_FILTER;
7880 	filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IIP;
7881 	filter_replace.new_filter_type =
7882 		I40E_AQC_ADD_CLOUD_FILTER_0X11;
7883 	/* Prepare the buffer, 2 entries */
7884 	filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG;
7885 	filter_replace_buf.data[0] |=
7886 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7887 	filter_replace_buf.data[4] = I40E_AQC_ADD_L1_FILTER_0X11;
7888 	filter_replace_buf.data[4] |=
7889 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7890 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
7891 					       &filter_replace_buf);
7892 	if (status < 0)
7893 		return status;
7894 	if (filter_replace.old_filter_type !=
7895 	    filter_replace.new_filter_type)
7896 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
7897 			    " original: 0x%x, new: 0x%x",
7898 			    dev->device->name,
7899 			    filter_replace.old_filter_type,
7900 			    filter_replace.new_filter_type);
7901 
7902 	/* For MPLSoGRE */
7903 	memset(&filter_replace, 0,
7904 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
7905 	memset(&filter_replace_buf, 0,
7906 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
7907 
7908 	filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER |
7909 		I40E_AQC_MIRROR_CLOUD_FILTER;
7910 	filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
7911 	filter_replace.new_filter_type =
7912 		I40E_AQC_ADD_CLOUD_FILTER_0X12;
7913 	/* Prepare the buffer, 2 entries */
7914 	filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG;
7915 	filter_replace_buf.data[0] |=
7916 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7917 	filter_replace_buf.data[4] = I40E_AQC_ADD_L1_FILTER_0X11;
7918 	filter_replace_buf.data[4] |=
7919 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7920 
7921 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
7922 					       &filter_replace_buf);
7923 	if (!status && (filter_replace.old_filter_type !=
7924 			filter_replace.new_filter_type))
7925 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
7926 			    " original: 0x%x, new: 0x%x",
7927 			    dev->device->name,
7928 			    filter_replace.old_filter_type,
7929 			    filter_replace.new_filter_type);
7930 
7931 	return status;
7932 }
7933 
7934 static enum i40e_status_code
7935 i40e_replace_gtp_l1_filter(struct i40e_pf *pf)
7936 {
7937 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
7938 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
7939 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7940 	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
7941 	enum i40e_status_code status = I40E_SUCCESS;
7942 
7943 	if (pf->support_multi_driver) {
7944 		PMD_DRV_LOG(ERR, "Replace l1 filter is not supported.");
7945 		return I40E_NOT_SUPPORTED;
7946 	}
7947 
7948 	/* For GTP-C */
7949 	memset(&filter_replace, 0,
7950 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
7951 	memset(&filter_replace_buf, 0,
7952 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
7953 	/* create L1 filter */
7954 	filter_replace.old_filter_type =
7955 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IMAC;
7956 	filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X12;
7957 	filter_replace.tr_bit = I40E_AQC_NEW_TR_22 |
7958 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7959 	/* Prepare the buffer, 2 entries */
7960 	filter_replace_buf.data[0] =
7961 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0;
7962 	filter_replace_buf.data[0] |=
7963 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7964 	filter_replace_buf.data[2] = 0xFF;
7965 	filter_replace_buf.data[3] = 0xFF;
7966 	filter_replace_buf.data[4] =
7967 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1;
7968 	filter_replace_buf.data[4] |=
7969 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7970 	filter_replace_buf.data[6] = 0xFF;
7971 	filter_replace_buf.data[7] = 0xFF;
7972 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
7973 					       &filter_replace_buf);
7974 	if (status < 0)
7975 		return status;
7976 	if (filter_replace.old_filter_type !=
7977 	    filter_replace.new_filter_type)
7978 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
7979 			    " original: 0x%x, new: 0x%x",
7980 			    dev->device->name,
7981 			    filter_replace.old_filter_type,
7982 			    filter_replace.new_filter_type);
7983 
7984 	/* for GTP-U */
7985 	memset(&filter_replace, 0,
7986 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
7987 	memset(&filter_replace_buf, 0,
7988 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
7989 	/* create L1 filter */
7990 	filter_replace.old_filter_type =
7991 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TUNNLE_KEY;
7992 	filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X13;
7993 	filter_replace.tr_bit = I40E_AQC_NEW_TR_21 |
7994 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
7995 	/* Prepare the buffer, 2 entries */
7996 	filter_replace_buf.data[0] =
7997 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0;
7998 	filter_replace_buf.data[0] |=
7999 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8000 	filter_replace_buf.data[2] = 0xFF;
8001 	filter_replace_buf.data[3] = 0xFF;
8002 	filter_replace_buf.data[4] =
8003 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1;
8004 	filter_replace_buf.data[4] |=
8005 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8006 	filter_replace_buf.data[6] = 0xFF;
8007 	filter_replace_buf.data[7] = 0xFF;
8008 
8009 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
8010 					       &filter_replace_buf);
8011 	if (!status && (filter_replace.old_filter_type !=
8012 			filter_replace.new_filter_type))
8013 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
8014 			    " original: 0x%x, new: 0x%x",
8015 			    dev->device->name,
8016 			    filter_replace.old_filter_type,
8017 			    filter_replace.new_filter_type);
8018 
8019 	return status;
8020 }
8021 
8022 static enum
8023 i40e_status_code i40e_replace_gtp_cloud_filter(struct i40e_pf *pf)
8024 {
8025 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
8026 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
8027 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8028 	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
8029 	enum i40e_status_code status = I40E_SUCCESS;
8030 
8031 	if (pf->support_multi_driver) {
8032 		PMD_DRV_LOG(ERR, "Replace cloud filter is not supported.");
8033 		return I40E_NOT_SUPPORTED;
8034 	}
8035 
8036 	/* for GTP-C */
8037 	memset(&filter_replace, 0,
8038 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
8039 	memset(&filter_replace_buf, 0,
8040 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
8041 	filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER;
8042 	filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
8043 	filter_replace.new_filter_type =
8044 		I40E_AQC_ADD_CLOUD_FILTER_0X11;
8045 	/* Prepare the buffer, 2 entries */
8046 	filter_replace_buf.data[0] = I40E_AQC_ADD_L1_FILTER_0X12;
8047 	filter_replace_buf.data[0] |=
8048 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8049 	filter_replace_buf.data[4] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG;
8050 	filter_replace_buf.data[4] |=
8051 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8052 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
8053 					       &filter_replace_buf);
8054 	if (status < 0)
8055 		return status;
8056 	if (filter_replace.old_filter_type !=
8057 	    filter_replace.new_filter_type)
8058 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
8059 			    " original: 0x%x, new: 0x%x",
8060 			    dev->device->name,
8061 			    filter_replace.old_filter_type,
8062 			    filter_replace.new_filter_type);
8063 
8064 	/* for GTP-U */
8065 	memset(&filter_replace, 0,
8066 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
8067 	memset(&filter_replace_buf, 0,
8068 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
8069 	filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER;
8070 	filter_replace.old_filter_type =
8071 		I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
8072 	filter_replace.new_filter_type =
8073 		I40E_AQC_ADD_CLOUD_FILTER_0X12;
8074 	/* Prepare the buffer, 2 entries */
8075 	filter_replace_buf.data[0] = I40E_AQC_ADD_L1_FILTER_0X13;
8076 	filter_replace_buf.data[0] |=
8077 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8078 	filter_replace_buf.data[4] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG;
8079 	filter_replace_buf.data[4] |=
8080 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
8081 
8082 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
8083 					       &filter_replace_buf);
8084 	if (!status && (filter_replace.old_filter_type !=
8085 			filter_replace.new_filter_type))
8086 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
8087 			    " original: 0x%x, new: 0x%x",
8088 			    dev->device->name,
8089 			    filter_replace.old_filter_type,
8090 			    filter_replace.new_filter_type);
8091 
8092 	return status;
8093 }
8094 
8095 int
8096 i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf,
8097 		      struct i40e_tunnel_filter_conf *tunnel_filter,
8098 		      uint8_t add)
8099 {
8100 	uint16_t ip_type;
8101 	uint32_t ipv4_addr, ipv4_addr_le;
8102 	uint8_t i, tun_type = 0;
8103 	/* internal variable to convert ipv6 byte order */
8104 	uint32_t convert_ipv6[4];
8105 	int val, ret = 0;
8106 	struct i40e_pf_vf *vf = NULL;
8107 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8108 	struct i40e_vsi *vsi;
8109 	struct i40e_aqc_cloud_filters_element_bb *cld_filter;
8110 	struct i40e_aqc_cloud_filters_element_bb *pfilter;
8111 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
8112 	struct i40e_tunnel_filter *tunnel, *node;
8113 	struct i40e_tunnel_filter check_filter; /* Check if filter exists */
8114 	uint32_t teid_le;
8115 	bool big_buffer = 0;
8116 
8117 	cld_filter = rte_zmalloc("tunnel_filter",
8118 			 sizeof(struct i40e_aqc_add_rm_cloud_filt_elem_ext),
8119 			 0);
8120 
8121 	if (cld_filter == NULL) {
8122 		PMD_DRV_LOG(ERR, "Failed to alloc memory.");
8123 		return -ENOMEM;
8124 	}
8125 	pfilter = cld_filter;
8126 
8127 	ether_addr_copy(&tunnel_filter->outer_mac,
8128 			(struct ether_addr *)&pfilter->element.outer_mac);
8129 	ether_addr_copy(&tunnel_filter->inner_mac,
8130 			(struct ether_addr *)&pfilter->element.inner_mac);
8131 
8132 	pfilter->element.inner_vlan =
8133 		rte_cpu_to_le_16(tunnel_filter->inner_vlan);
8134 	if (tunnel_filter->ip_type == I40E_TUNNEL_IPTYPE_IPV4) {
8135 		ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
8136 		ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr);
8137 		ipv4_addr_le = rte_cpu_to_le_32(ipv4_addr);
8138 		rte_memcpy(&pfilter->element.ipaddr.v4.data,
8139 				&ipv4_addr_le,
8140 				sizeof(pfilter->element.ipaddr.v4.data));
8141 	} else {
8142 		ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
8143 		for (i = 0; i < 4; i++) {
8144 			convert_ipv6[i] =
8145 			rte_cpu_to_le_32(rte_be_to_cpu_32(
8146 					 tunnel_filter->ip_addr.ipv6_addr[i]));
8147 		}
8148 		rte_memcpy(&pfilter->element.ipaddr.v6.data,
8149 			   &convert_ipv6,
8150 			   sizeof(pfilter->element.ipaddr.v6.data));
8151 	}
8152 
8153 	/* check tunneled type */
8154 	switch (tunnel_filter->tunnel_type) {
8155 	case I40E_TUNNEL_TYPE_VXLAN:
8156 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
8157 		break;
8158 	case I40E_TUNNEL_TYPE_NVGRE:
8159 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
8160 		break;
8161 	case I40E_TUNNEL_TYPE_IP_IN_GRE:
8162 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
8163 		break;
8164 	case I40E_TUNNEL_TYPE_MPLSoUDP:
8165 		if (!pf->mpls_replace_flag) {
8166 			i40e_replace_mpls_l1_filter(pf);
8167 			i40e_replace_mpls_cloud_filter(pf);
8168 			pf->mpls_replace_flag = 1;
8169 		}
8170 		teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id);
8171 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0] =
8172 			teid_le >> 4;
8173 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1] =
8174 			(teid_le & 0xF) << 12;
8175 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2] =
8176 			0x40;
8177 		big_buffer = 1;
8178 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOUDP;
8179 		break;
8180 	case I40E_TUNNEL_TYPE_MPLSoGRE:
8181 		if (!pf->mpls_replace_flag) {
8182 			i40e_replace_mpls_l1_filter(pf);
8183 			i40e_replace_mpls_cloud_filter(pf);
8184 			pf->mpls_replace_flag = 1;
8185 		}
8186 		teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id);
8187 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0] =
8188 			teid_le >> 4;
8189 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1] =
8190 			(teid_le & 0xF) << 12;
8191 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2] =
8192 			0x0;
8193 		big_buffer = 1;
8194 		tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOGRE;
8195 		break;
8196 	case I40E_TUNNEL_TYPE_GTPC:
8197 		if (!pf->gtp_replace_flag) {
8198 			i40e_replace_gtp_l1_filter(pf);
8199 			i40e_replace_gtp_cloud_filter(pf);
8200 			pf->gtp_replace_flag = 1;
8201 		}
8202 		teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id);
8203 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD0] =
8204 			(teid_le >> 16) & 0xFFFF;
8205 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD1] =
8206 			teid_le & 0xFFFF;
8207 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD2] =
8208 			0x0;
8209 		big_buffer = 1;
8210 		break;
8211 	case I40E_TUNNEL_TYPE_GTPU:
8212 		if (!pf->gtp_replace_flag) {
8213 			i40e_replace_gtp_l1_filter(pf);
8214 			i40e_replace_gtp_cloud_filter(pf);
8215 			pf->gtp_replace_flag = 1;
8216 		}
8217 		teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id);
8218 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD0] =
8219 			(teid_le >> 16) & 0xFFFF;
8220 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD1] =
8221 			teid_le & 0xFFFF;
8222 		pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD2] =
8223 			0x0;
8224 		big_buffer = 1;
8225 		break;
8226 	case I40E_TUNNEL_TYPE_QINQ:
8227 		if (!pf->qinq_replace_flag) {
8228 			ret = i40e_cloud_filter_qinq_create(pf);
8229 			if (ret < 0)
8230 				PMD_DRV_LOG(DEBUG,
8231 					    "QinQ tunnel filter already created.");
8232 			pf->qinq_replace_flag = 1;
8233 		}
8234 		/*	Add in the General fields the values of
8235 		 *	the Outer and Inner VLAN
8236 		 *	Big Buffer should be set, see changes in
8237 		 *	i40e_aq_add_cloud_filters
8238 		 */
8239 		pfilter->general_fields[0] = tunnel_filter->inner_vlan;
8240 		pfilter->general_fields[1] = tunnel_filter->outer_vlan;
8241 		big_buffer = 1;
8242 		break;
8243 	default:
8244 		/* Other tunnel types is not supported. */
8245 		PMD_DRV_LOG(ERR, "tunnel type is not supported.");
8246 		rte_free(cld_filter);
8247 		return -EINVAL;
8248 	}
8249 
8250 	if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_MPLSoUDP)
8251 		pfilter->element.flags =
8252 			I40E_AQC_ADD_CLOUD_FILTER_0X11;
8253 	else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_MPLSoGRE)
8254 		pfilter->element.flags =
8255 			I40E_AQC_ADD_CLOUD_FILTER_0X12;
8256 	else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_GTPC)
8257 		pfilter->element.flags =
8258 			I40E_AQC_ADD_CLOUD_FILTER_0X11;
8259 	else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_GTPU)
8260 		pfilter->element.flags =
8261 			I40E_AQC_ADD_CLOUD_FILTER_0X12;
8262 	else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_QINQ)
8263 		pfilter->element.flags |=
8264 			I40E_AQC_ADD_CLOUD_FILTER_0X10;
8265 	else {
8266 		val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
8267 						&pfilter->element.flags);
8268 		if (val < 0) {
8269 			rte_free(cld_filter);
8270 			return -EINVAL;
8271 		}
8272 	}
8273 
8274 	pfilter->element.flags |= rte_cpu_to_le_16(
8275 		I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
8276 		ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
8277 	pfilter->element.tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
8278 	pfilter->element.queue_number =
8279 		rte_cpu_to_le_16(tunnel_filter->queue_id);
8280 
8281 	if (!tunnel_filter->is_to_vf)
8282 		vsi = pf->main_vsi;
8283 	else {
8284 		if (tunnel_filter->vf_id >= pf->vf_num) {
8285 			PMD_DRV_LOG(ERR, "Invalid argument.");
8286 			rte_free(cld_filter);
8287 			return -EINVAL;
8288 		}
8289 		vf = &pf->vfs[tunnel_filter->vf_id];
8290 		vsi = vf->vsi;
8291 	}
8292 
8293 	/* Check if there is the filter in SW list */
8294 	memset(&check_filter, 0, sizeof(check_filter));
8295 	i40e_tunnel_filter_convert(cld_filter, &check_filter);
8296 	check_filter.is_to_vf = tunnel_filter->is_to_vf;
8297 	check_filter.vf_id = tunnel_filter->vf_id;
8298 	node = i40e_sw_tunnel_filter_lookup(tunnel_rule, &check_filter.input);
8299 	if (add && node) {
8300 		PMD_DRV_LOG(ERR, "Conflict with existing tunnel rules!");
8301 		rte_free(cld_filter);
8302 		return -EINVAL;
8303 	}
8304 
8305 	if (!add && !node) {
8306 		PMD_DRV_LOG(ERR, "There's no corresponding tunnel filter!");
8307 		rte_free(cld_filter);
8308 		return -EINVAL;
8309 	}
8310 
8311 	if (add) {
8312 		if (big_buffer)
8313 			ret = i40e_aq_add_cloud_filters_bb(hw,
8314 						   vsi->seid, cld_filter, 1);
8315 		else
8316 			ret = i40e_aq_add_cloud_filters(hw,
8317 					vsi->seid, &cld_filter->element, 1);
8318 		if (ret < 0) {
8319 			PMD_DRV_LOG(ERR, "Failed to add a tunnel filter.");
8320 			rte_free(cld_filter);
8321 			return -ENOTSUP;
8322 		}
8323 		tunnel = rte_zmalloc("tunnel_filter", sizeof(*tunnel), 0);
8324 		if (tunnel == NULL) {
8325 			PMD_DRV_LOG(ERR, "Failed to alloc memory.");
8326 			rte_free(cld_filter);
8327 			return -ENOMEM;
8328 		}
8329 
8330 		rte_memcpy(tunnel, &check_filter, sizeof(check_filter));
8331 		ret = i40e_sw_tunnel_filter_insert(pf, tunnel);
8332 		if (ret < 0)
8333 			rte_free(tunnel);
8334 	} else {
8335 		if (big_buffer)
8336 			ret = i40e_aq_rem_cloud_filters_bb(
8337 				hw, vsi->seid, cld_filter, 1);
8338 		else
8339 			ret = i40e_aq_rem_cloud_filters(hw, vsi->seid,
8340 						&cld_filter->element, 1);
8341 		if (ret < 0) {
8342 			PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter.");
8343 			rte_free(cld_filter);
8344 			return -ENOTSUP;
8345 		}
8346 		ret = i40e_sw_tunnel_filter_del(pf, &node->input);
8347 	}
8348 
8349 	rte_free(cld_filter);
8350 	return ret;
8351 }
8352 
8353 static int
8354 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
8355 {
8356 	uint8_t i;
8357 
8358 	for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
8359 		if (pf->vxlan_ports[i] == port)
8360 			return i;
8361 	}
8362 
8363 	return -1;
8364 }
8365 
8366 static int
8367 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
8368 {
8369 	int  idx, ret;
8370 	uint8_t filter_idx;
8371 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8372 
8373 	idx = i40e_get_vxlan_port_idx(pf, port);
8374 
8375 	/* Check if port already exists */
8376 	if (idx >= 0) {
8377 		PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
8378 		return -EINVAL;
8379 	}
8380 
8381 	/* Now check if there is space to add the new port */
8382 	idx = i40e_get_vxlan_port_idx(pf, 0);
8383 	if (idx < 0) {
8384 		PMD_DRV_LOG(ERR,
8385 			"Maximum number of UDP ports reached, not adding port %d",
8386 			port);
8387 		return -ENOSPC;
8388 	}
8389 
8390 	ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
8391 					&filter_idx, NULL);
8392 	if (ret < 0) {
8393 		PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
8394 		return -1;
8395 	}
8396 
8397 	PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
8398 			 port,  filter_idx);
8399 
8400 	/* New port: add it and mark its index in the bitmap */
8401 	pf->vxlan_ports[idx] = port;
8402 	pf->vxlan_bitmap |= (1 << idx);
8403 
8404 	if (!(pf->flags & I40E_FLAG_VXLAN))
8405 		pf->flags |= I40E_FLAG_VXLAN;
8406 
8407 	return 0;
8408 }
8409 
8410 static int
8411 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
8412 {
8413 	int idx;
8414 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8415 
8416 	if (!(pf->flags & I40E_FLAG_VXLAN)) {
8417 		PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
8418 		return -EINVAL;
8419 	}
8420 
8421 	idx = i40e_get_vxlan_port_idx(pf, port);
8422 
8423 	if (idx < 0) {
8424 		PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
8425 		return -EINVAL;
8426 	}
8427 
8428 	if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
8429 		PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
8430 		return -1;
8431 	}
8432 
8433 	PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
8434 			port, idx);
8435 
8436 	pf->vxlan_ports[idx] = 0;
8437 	pf->vxlan_bitmap &= ~(1 << idx);
8438 
8439 	if (!pf->vxlan_bitmap)
8440 		pf->flags &= ~I40E_FLAG_VXLAN;
8441 
8442 	return 0;
8443 }
8444 
8445 /* Add UDP tunneling port */
8446 static int
8447 i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
8448 			     struct rte_eth_udp_tunnel *udp_tunnel)
8449 {
8450 	int ret = 0;
8451 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8452 
8453 	if (udp_tunnel == NULL)
8454 		return -EINVAL;
8455 
8456 	switch (udp_tunnel->prot_type) {
8457 	case RTE_TUNNEL_TYPE_VXLAN:
8458 		ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
8459 		break;
8460 
8461 	case RTE_TUNNEL_TYPE_GENEVE:
8462 	case RTE_TUNNEL_TYPE_TEREDO:
8463 		PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
8464 		ret = -1;
8465 		break;
8466 
8467 	default:
8468 		PMD_DRV_LOG(ERR, "Invalid tunnel type");
8469 		ret = -1;
8470 		break;
8471 	}
8472 
8473 	return ret;
8474 }
8475 
8476 /* Remove UDP tunneling port */
8477 static int
8478 i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
8479 			     struct rte_eth_udp_tunnel *udp_tunnel)
8480 {
8481 	int ret = 0;
8482 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8483 
8484 	if (udp_tunnel == NULL)
8485 		return -EINVAL;
8486 
8487 	switch (udp_tunnel->prot_type) {
8488 	case RTE_TUNNEL_TYPE_VXLAN:
8489 		ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
8490 		break;
8491 	case RTE_TUNNEL_TYPE_GENEVE:
8492 	case RTE_TUNNEL_TYPE_TEREDO:
8493 		PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
8494 		ret = -1;
8495 		break;
8496 	default:
8497 		PMD_DRV_LOG(ERR, "Invalid tunnel type");
8498 		ret = -1;
8499 		break;
8500 	}
8501 
8502 	return ret;
8503 }
8504 
8505 /* Calculate the maximum number of contiguous PF queues that are configured */
8506 static int
8507 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
8508 {
8509 	struct rte_eth_dev_data *data = pf->dev_data;
8510 	int i, num;
8511 	struct i40e_rx_queue *rxq;
8512 
8513 	num = 0;
8514 	for (i = 0; i < pf->lan_nb_qps; i++) {
8515 		rxq = data->rx_queues[i];
8516 		if (rxq && rxq->q_set)
8517 			num++;
8518 		else
8519 			break;
8520 	}
8521 
8522 	return num;
8523 }
8524 
8525 /* Configure RSS */
8526 static int
8527 i40e_pf_config_rss(struct i40e_pf *pf)
8528 {
8529 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8530 	struct rte_eth_rss_conf rss_conf;
8531 	uint32_t i, lut = 0;
8532 	uint16_t j, num;
8533 
8534 	/*
8535 	 * If both VMDQ and RSS enabled, not all of PF queues are configured.
8536 	 * It's necessary to calculate the actual PF queues that are configured.
8537 	 */
8538 	if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
8539 		num = i40e_pf_calc_configured_queues_num(pf);
8540 	else
8541 		num = pf->dev_data->nb_rx_queues;
8542 
8543 	num = RTE_MIN(num, I40E_MAX_Q_PER_TC);
8544 	PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
8545 			num);
8546 
8547 	if (num == 0) {
8548 		PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
8549 		return -ENOTSUP;
8550 	}
8551 
8552 	if (pf->adapter->rss_reta_updated == 0) {
8553 		for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
8554 			if (j == num)
8555 				j = 0;
8556 			lut = (lut << 8) | (j & ((0x1 <<
8557 				hw->func_caps.rss_table_entry_width) - 1));
8558 			if ((i & 3) == 3)
8559 				I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2),
8560 					       rte_bswap32(lut));
8561 		}
8562 	}
8563 
8564 	rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
8565 	if ((rss_conf.rss_hf & pf->adapter->flow_types_mask) == 0) {
8566 		i40e_pf_disable_rss(pf);
8567 		return 0;
8568 	}
8569 	if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
8570 		(I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
8571 		/* Random default keys */
8572 		static uint32_t rss_key_default[] = {0x6b793944,
8573 			0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
8574 			0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
8575 			0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
8576 
8577 		rss_conf.rss_key = (uint8_t *)rss_key_default;
8578 		rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
8579 							sizeof(uint32_t);
8580 	}
8581 
8582 	return i40e_hw_rss_hash_set(pf, &rss_conf);
8583 }
8584 
8585 static int
8586 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
8587 			       struct rte_eth_tunnel_filter_conf *filter)
8588 {
8589 	if (pf == NULL || filter == NULL) {
8590 		PMD_DRV_LOG(ERR, "Invalid parameter");
8591 		return -EINVAL;
8592 	}
8593 
8594 	if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
8595 		PMD_DRV_LOG(ERR, "Invalid queue ID");
8596 		return -EINVAL;
8597 	}
8598 
8599 	if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
8600 		PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
8601 		return -EINVAL;
8602 	}
8603 
8604 	if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
8605 		(is_zero_ether_addr(&filter->outer_mac))) {
8606 		PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
8607 		return -EINVAL;
8608 	}
8609 
8610 	if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
8611 		(is_zero_ether_addr(&filter->inner_mac))) {
8612 		PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
8613 		return -EINVAL;
8614 	}
8615 
8616 	return 0;
8617 }
8618 
8619 #define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000
8620 #define I40E_GL_PRS_FVBM(_i)     (0x00269760 + ((_i) * 4))
8621 static int
8622 i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
8623 {
8624 	struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
8625 	uint32_t val, reg;
8626 	int ret = -EINVAL;
8627 
8628 	if (pf->support_multi_driver) {
8629 		PMD_DRV_LOG(ERR, "GRE key length configuration is unsupported");
8630 		return -ENOTSUP;
8631 	}
8632 
8633 	val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2));
8634 	PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x", val);
8635 
8636 	if (len == 3) {
8637 		reg = val | I40E_GL_PRS_FVBM_MSK_ENA;
8638 	} else if (len == 4) {
8639 		reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA;
8640 	} else {
8641 		PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len);
8642 		return ret;
8643 	}
8644 
8645 	if (reg != val) {
8646 		ret = i40e_aq_debug_write_global_register(hw,
8647 						   I40E_GL_PRS_FVBM(2),
8648 						   reg, NULL);
8649 		if (ret != 0)
8650 			return ret;
8651 		PMD_DRV_LOG(DEBUG, "Global register 0x%08x is changed "
8652 			    "with value 0x%08x",
8653 			    I40E_GL_PRS_FVBM(2), reg);
8654 	} else {
8655 		ret = 0;
8656 	}
8657 	PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x",
8658 		    I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)));
8659 
8660 	return ret;
8661 }
8662 
8663 static int
8664 i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg)
8665 {
8666 	int ret = -EINVAL;
8667 
8668 	if (!hw || !cfg)
8669 		return -EINVAL;
8670 
8671 	switch (cfg->cfg_type) {
8672 	case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN:
8673 		ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len);
8674 		break;
8675 	default:
8676 		PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type);
8677 		break;
8678 	}
8679 
8680 	return ret;
8681 }
8682 
8683 static int
8684 i40e_filter_ctrl_global_config(struct rte_eth_dev *dev,
8685 			       enum rte_filter_op filter_op,
8686 			       void *arg)
8687 {
8688 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8689 	int ret = I40E_ERR_PARAM;
8690 
8691 	switch (filter_op) {
8692 	case RTE_ETH_FILTER_SET:
8693 		ret = i40e_dev_global_config_set(hw,
8694 			(struct rte_eth_global_cfg *)arg);
8695 		break;
8696 	default:
8697 		PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
8698 		break;
8699 	}
8700 
8701 	return ret;
8702 }
8703 
8704 static int
8705 i40e_tunnel_filter_handle(struct rte_eth_dev *dev,
8706 			  enum rte_filter_op filter_op,
8707 			  void *arg)
8708 {
8709 	struct rte_eth_tunnel_filter_conf *filter;
8710 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8711 	int ret = I40E_SUCCESS;
8712 
8713 	filter = (struct rte_eth_tunnel_filter_conf *)(arg);
8714 
8715 	if (i40e_tunnel_filter_param_check(pf, filter) < 0)
8716 		return I40E_ERR_PARAM;
8717 
8718 	switch (filter_op) {
8719 	case RTE_ETH_FILTER_NOP:
8720 		if (!(pf->flags & I40E_FLAG_VXLAN))
8721 			ret = I40E_NOT_SUPPORTED;
8722 		break;
8723 	case RTE_ETH_FILTER_ADD:
8724 		ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
8725 		break;
8726 	case RTE_ETH_FILTER_DELETE:
8727 		ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
8728 		break;
8729 	default:
8730 		PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
8731 		ret = I40E_ERR_PARAM;
8732 		break;
8733 	}
8734 
8735 	return ret;
8736 }
8737 
8738 static int
8739 i40e_pf_config_mq_rx(struct i40e_pf *pf)
8740 {
8741 	int ret = 0;
8742 	enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
8743 
8744 	/* RSS setup */
8745 	if (mq_mode & ETH_MQ_RX_RSS_FLAG)
8746 		ret = i40e_pf_config_rss(pf);
8747 	else
8748 		i40e_pf_disable_rss(pf);
8749 
8750 	return ret;
8751 }
8752 
8753 /* Get the symmetric hash enable configurations per port */
8754 static void
8755 i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable)
8756 {
8757 	uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
8758 
8759 	*enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0;
8760 }
8761 
8762 /* Set the symmetric hash enable configurations per port */
8763 static void
8764 i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable)
8765 {
8766 	uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
8767 
8768 	if (enable > 0) {
8769 		if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) {
8770 			PMD_DRV_LOG(INFO,
8771 				"Symmetric hash has already been enabled");
8772 			return;
8773 		}
8774 		reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
8775 	} else {
8776 		if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) {
8777 			PMD_DRV_LOG(INFO,
8778 				"Symmetric hash has already been disabled");
8779 			return;
8780 		}
8781 		reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
8782 	}
8783 	i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg);
8784 	I40E_WRITE_FLUSH(hw);
8785 }
8786 
8787 /*
8788  * Get global configurations of hash function type and symmetric hash enable
8789  * per flow type (pctype). Note that global configuration means it affects all
8790  * the ports on the same NIC.
8791  */
8792 static int
8793 i40e_get_hash_filter_global_config(struct i40e_hw *hw,
8794 				   struct rte_eth_hash_global_conf *g_cfg)
8795 {
8796 	struct i40e_adapter *adapter = (struct i40e_adapter *)hw->back;
8797 	uint32_t reg;
8798 	uint16_t i, j;
8799 
8800 	memset(g_cfg, 0, sizeof(*g_cfg));
8801 	reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
8802 	if (reg & I40E_GLQF_CTL_HTOEP_MASK)
8803 		g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
8804 	else
8805 		g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
8806 	PMD_DRV_LOG(DEBUG, "Hash function is %s",
8807 		(reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR");
8808 
8809 	/*
8810 	 * As i40e supports less than 64 flow types, only first 64 bits need to
8811 	 * be checked.
8812 	 */
8813 	for (i = 1; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
8814 		g_cfg->valid_bit_mask[i] = 0ULL;
8815 		g_cfg->sym_hash_enable_mask[i] = 0ULL;
8816 	}
8817 
8818 	g_cfg->valid_bit_mask[0] = adapter->flow_types_mask;
8819 
8820 	for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < UINT64_BIT; i++) {
8821 		if (!adapter->pctypes_tbl[i])
8822 			continue;
8823 		for (j = I40E_FILTER_PCTYPE_INVALID + 1;
8824 		     j < I40E_FILTER_PCTYPE_MAX; j++) {
8825 			if (adapter->pctypes_tbl[i] & (1ULL << j)) {
8826 				reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(j));
8827 				if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK) {
8828 					g_cfg->sym_hash_enable_mask[0] |=
8829 								(1ULL << i);
8830 				}
8831 			}
8832 		}
8833 	}
8834 
8835 	return 0;
8836 }
8837 
8838 static int
8839 i40e_hash_global_config_check(const struct i40e_adapter *adapter,
8840 			      const struct rte_eth_hash_global_conf *g_cfg)
8841 {
8842 	uint32_t i;
8843 	uint64_t mask0, i40e_mask = adapter->flow_types_mask;
8844 
8845 	if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ &&
8846 		g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR &&
8847 		g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
8848 		PMD_DRV_LOG(ERR, "Unsupported hash function type %d",
8849 						g_cfg->hash_func);
8850 		return -EINVAL;
8851 	}
8852 
8853 	/*
8854 	 * As i40e supports less than 64 flow types, only first 64 bits need to
8855 	 * be checked.
8856 	 */
8857 	mask0 = g_cfg->valid_bit_mask[0];
8858 	for (i = 0; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
8859 		if (i == 0) {
8860 			/* Check if any unsupported flow type configured */
8861 			if ((mask0 | i40e_mask) ^ i40e_mask)
8862 				goto mask_err;
8863 		} else {
8864 			if (g_cfg->valid_bit_mask[i])
8865 				goto mask_err;
8866 		}
8867 	}
8868 
8869 	return 0;
8870 
8871 mask_err:
8872 	PMD_DRV_LOG(ERR, "i40e unsupported flow type bit(s) configured");
8873 
8874 	return -EINVAL;
8875 }
8876 
8877 /*
8878  * Set global configurations of hash function type and symmetric hash enable
8879  * per flow type (pctype). Note any modifying global configuration will affect
8880  * all the ports on the same NIC.
8881  */
8882 static int
8883 i40e_set_hash_filter_global_config(struct i40e_hw *hw,
8884 				   struct rte_eth_hash_global_conf *g_cfg)
8885 {
8886 	struct i40e_adapter *adapter = (struct i40e_adapter *)hw->back;
8887 	struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
8888 	int ret;
8889 	uint16_t i, j;
8890 	uint32_t reg;
8891 	uint64_t mask0 = g_cfg->valid_bit_mask[0] & adapter->flow_types_mask;
8892 
8893 	if (pf->support_multi_driver) {
8894 		PMD_DRV_LOG(ERR, "Hash global configuration is not supported.");
8895 		return -ENOTSUP;
8896 	}
8897 
8898 	/* Check the input parameters */
8899 	ret = i40e_hash_global_config_check(adapter, g_cfg);
8900 	if (ret < 0)
8901 		return ret;
8902 
8903 	/*
8904 	 * As i40e supports less than 64 flow types, only first 64 bits need to
8905 	 * be configured.
8906 	 */
8907 	for (i = RTE_ETH_FLOW_UNKNOWN + 1; mask0 && i < UINT64_BIT; i++) {
8908 		if (mask0 & (1UL << i)) {
8909 			reg = (g_cfg->sym_hash_enable_mask[0] & (1ULL << i)) ?
8910 					I40E_GLQF_HSYM_SYMH_ENA_MASK : 0;
8911 
8912 			for (j = I40E_FILTER_PCTYPE_INVALID + 1;
8913 			     j < I40E_FILTER_PCTYPE_MAX; j++) {
8914 				if (adapter->pctypes_tbl[i] & (1ULL << j))
8915 					i40e_write_global_rx_ctl(hw,
8916 							  I40E_GLQF_HSYM(j),
8917 							  reg);
8918 			}
8919 		}
8920 	}
8921 
8922 	reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
8923 	if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
8924 		/* Toeplitz */
8925 		if (reg & I40E_GLQF_CTL_HTOEP_MASK) {
8926 			PMD_DRV_LOG(DEBUG,
8927 				"Hash function already set to Toeplitz");
8928 			goto out;
8929 		}
8930 		reg |= I40E_GLQF_CTL_HTOEP_MASK;
8931 	} else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
8932 		/* Simple XOR */
8933 		if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) {
8934 			PMD_DRV_LOG(DEBUG,
8935 				"Hash function already set to Simple XOR");
8936 			goto out;
8937 		}
8938 		reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
8939 	} else
8940 		/* Use the default, and keep it as it is */
8941 		goto out;
8942 
8943 	i40e_write_global_rx_ctl(hw, I40E_GLQF_CTL, reg);
8944 
8945 out:
8946 	I40E_WRITE_FLUSH(hw);
8947 
8948 	return 0;
8949 }
8950 
8951 /**
8952  * Valid input sets for hash and flow director filters per PCTYPE
8953  */
8954 static uint64_t
8955 i40e_get_valid_input_set(enum i40e_filter_pctype pctype,
8956 		enum rte_filter_type filter)
8957 {
8958 	uint64_t valid;
8959 
8960 	static const uint64_t valid_hash_inset_table[] = {
8961 		[I40E_FILTER_PCTYPE_FRAG_IPV4] =
8962 			I40E_INSET_DMAC | I40E_INSET_SMAC |
8963 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
8964 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC |
8965 			I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS |
8966 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
8967 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
8968 			I40E_INSET_FLEX_PAYLOAD,
8969 		[I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
8970 			I40E_INSET_DMAC | I40E_INSET_SMAC |
8971 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
8972 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
8973 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
8974 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
8975 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
8976 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
8977 			I40E_INSET_FLEX_PAYLOAD,
8978 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
8979 			I40E_INSET_DMAC | I40E_INSET_SMAC |
8980 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
8981 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
8982 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
8983 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
8984 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
8985 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
8986 			I40E_INSET_FLEX_PAYLOAD,
8987 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
8988 			I40E_INSET_DMAC | I40E_INSET_SMAC |
8989 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
8990 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
8991 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
8992 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
8993 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
8994 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
8995 			I40E_INSET_FLEX_PAYLOAD,
8996 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
8997 			I40E_INSET_DMAC | I40E_INSET_SMAC |
8998 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
8999 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
9000 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
9001 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
9002 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9003 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9004 			I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
9005 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
9006 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9007 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9008 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
9009 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
9010 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
9011 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9012 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9013 			I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
9014 		[I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
9015 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9016 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9017 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
9018 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
9019 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
9020 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9021 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9022 			I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
9023 		[I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
9024 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9025 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9026 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
9027 			I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
9028 			I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
9029 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9030 			I40E_INSET_FLEX_PAYLOAD,
9031 		[I40E_FILTER_PCTYPE_FRAG_IPV6] =
9032 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9033 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9034 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9035 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9036 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC |
9037 			I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC |
9038 			I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD,
9039 		[I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
9040 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9041 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9042 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9043 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9044 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9045 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9046 			I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD,
9047 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
9048 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9049 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9050 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9051 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9052 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9053 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9054 			I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
9055 			I40E_INSET_FLEX_PAYLOAD,
9056 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
9057 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9058 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9059 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9060 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9061 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9062 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9063 			I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
9064 			I40E_INSET_FLEX_PAYLOAD,
9065 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
9066 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9067 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9068 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9069 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9070 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9071 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9072 			I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
9073 			I40E_INSET_FLEX_PAYLOAD,
9074 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
9075 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9076 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9077 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9078 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9079 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9080 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9081 			I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
9082 			I40E_INSET_FLEX_PAYLOAD,
9083 		[I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
9084 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9085 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9086 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9087 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9088 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9089 			I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
9090 			I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT |
9091 			I40E_INSET_FLEX_PAYLOAD,
9092 		[I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
9093 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9094 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9095 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
9096 			I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
9097 			I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
9098 			I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID |
9099 			I40E_INSET_FLEX_PAYLOAD,
9100 		[I40E_FILTER_PCTYPE_L2_PAYLOAD] =
9101 			I40E_INSET_DMAC | I40E_INSET_SMAC |
9102 			I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9103 			I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE |
9104 			I40E_INSET_FLEX_PAYLOAD,
9105 	};
9106 
9107 	/**
9108 	 * Flow director supports only fields defined in
9109 	 * union rte_eth_fdir_flow.
9110 	 */
9111 	static const uint64_t valid_fdir_inset_table[] = {
9112 		[I40E_FILTER_PCTYPE_FRAG_IPV4] =
9113 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9114 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9115 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
9116 		I40E_INSET_IPV4_TTL,
9117 		[I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
9118 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9119 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9120 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9121 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9122 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
9123 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9124 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9125 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9126 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9127 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
9128 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9129 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9130 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9131 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9132 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
9133 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9134 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9135 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9136 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9137 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
9138 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9139 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9140 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9141 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9142 		[I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
9143 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9144 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9145 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
9146 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9147 		I40E_INSET_SCTP_VT,
9148 		[I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
9149 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9150 		I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9151 		I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
9152 		I40E_INSET_IPV4_TTL,
9153 		[I40E_FILTER_PCTYPE_FRAG_IPV6] =
9154 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9155 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9156 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
9157 		I40E_INSET_IPV6_HOP_LIMIT,
9158 		[I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
9159 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9160 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9161 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9162 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9163 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
9164 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9165 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9166 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9167 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9168 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
9169 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9170 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9171 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9172 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9173 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
9174 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9175 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9176 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9177 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9178 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
9179 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9180 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9181 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9182 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9183 		[I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
9184 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9185 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9186 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
9187 		I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9188 		I40E_INSET_SCTP_VT,
9189 		[I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
9190 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9191 		I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9192 		I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
9193 		I40E_INSET_IPV6_HOP_LIMIT,
9194 		[I40E_FILTER_PCTYPE_L2_PAYLOAD] =
9195 		I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
9196 		I40E_INSET_LAST_ETHER_TYPE,
9197 	};
9198 
9199 	if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
9200 		return 0;
9201 	if (filter == RTE_ETH_FILTER_HASH)
9202 		valid = valid_hash_inset_table[pctype];
9203 	else
9204 		valid = valid_fdir_inset_table[pctype];
9205 
9206 	return valid;
9207 }
9208 
9209 /**
9210  * Validate if the input set is allowed for a specific PCTYPE
9211  */
9212 int
9213 i40e_validate_input_set(enum i40e_filter_pctype pctype,
9214 		enum rte_filter_type filter, uint64_t inset)
9215 {
9216 	uint64_t valid;
9217 
9218 	valid = i40e_get_valid_input_set(pctype, filter);
9219 	if (inset & (~valid))
9220 		return -EINVAL;
9221 
9222 	return 0;
9223 }
9224 
9225 /* default input set fields combination per pctype */
9226 uint64_t
9227 i40e_get_default_input_set(uint16_t pctype)
9228 {
9229 	static const uint64_t default_inset_table[] = {
9230 		[I40E_FILTER_PCTYPE_FRAG_IPV4] =
9231 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
9232 		[I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
9233 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9234 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9235 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
9236 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9237 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9238 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
9239 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9240 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9241 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
9242 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9243 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9244 		[I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
9245 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9246 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9247 		[I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
9248 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
9249 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9250 			I40E_INSET_SCTP_VT,
9251 		[I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
9252 			I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
9253 		[I40E_FILTER_PCTYPE_FRAG_IPV6] =
9254 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
9255 		[I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
9256 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9257 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9258 		[I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
9259 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9260 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9261 		[I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
9262 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9263 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9264 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
9265 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9266 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9267 		[I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
9268 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9269 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
9270 		[I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
9271 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
9272 			I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
9273 			I40E_INSET_SCTP_VT,
9274 		[I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
9275 			I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
9276 		[I40E_FILTER_PCTYPE_L2_PAYLOAD] =
9277 			I40E_INSET_LAST_ETHER_TYPE,
9278 	};
9279 
9280 	if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
9281 		return 0;
9282 
9283 	return default_inset_table[pctype];
9284 }
9285 
9286 /**
9287  * Parse the input set from index to logical bit masks
9288  */
9289 static int
9290 i40e_parse_input_set(uint64_t *inset,
9291 		     enum i40e_filter_pctype pctype,
9292 		     enum rte_eth_input_set_field *field,
9293 		     uint16_t size)
9294 {
9295 	uint16_t i, j;
9296 	int ret = -EINVAL;
9297 
9298 	static const struct {
9299 		enum rte_eth_input_set_field field;
9300 		uint64_t inset;
9301 	} inset_convert_table[] = {
9302 		{RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE},
9303 		{RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC},
9304 		{RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC},
9305 		{RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER},
9306 		{RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER},
9307 		{RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE},
9308 		{RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC},
9309 		{RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST},
9310 		{RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS},
9311 		{RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO},
9312 		{RTE_ETH_INPUT_SET_L3_IP4_TTL, I40E_INSET_IPV4_TTL},
9313 		{RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC},
9314 		{RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST},
9315 		{RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC},
9316 		{RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER,
9317 			I40E_INSET_IPV6_NEXT_HDR},
9318 		{RTE_ETH_INPUT_SET_L3_IP6_HOP_LIMITS,
9319 			I40E_INSET_IPV6_HOP_LIMIT},
9320 		{RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT},
9321 		{RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT},
9322 		{RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT},
9323 		{RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT},
9324 		{RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT},
9325 		{RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT},
9326 		{RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG,
9327 			I40E_INSET_SCTP_VT},
9328 		{RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC,
9329 			I40E_INSET_TUNNEL_DMAC},
9330 		{RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN,
9331 			I40E_INSET_VLAN_TUNNEL},
9332 		{RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY,
9333 			I40E_INSET_TUNNEL_ID},
9334 		{RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID},
9335 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD,
9336 			I40E_INSET_FLEX_PAYLOAD_W1},
9337 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD,
9338 			I40E_INSET_FLEX_PAYLOAD_W2},
9339 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD,
9340 			I40E_INSET_FLEX_PAYLOAD_W3},
9341 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD,
9342 			I40E_INSET_FLEX_PAYLOAD_W4},
9343 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD,
9344 			I40E_INSET_FLEX_PAYLOAD_W5},
9345 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD,
9346 			I40E_INSET_FLEX_PAYLOAD_W6},
9347 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD,
9348 			I40E_INSET_FLEX_PAYLOAD_W7},
9349 		{RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD,
9350 			I40E_INSET_FLEX_PAYLOAD_W8},
9351 	};
9352 
9353 	if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX)
9354 		return ret;
9355 
9356 	/* Only one item allowed for default or all */
9357 	if (size == 1) {
9358 		if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) {
9359 			*inset = i40e_get_default_input_set(pctype);
9360 			return 0;
9361 		} else if (field[0] == RTE_ETH_INPUT_SET_NONE) {
9362 			*inset = I40E_INSET_NONE;
9363 			return 0;
9364 		}
9365 	}
9366 
9367 	for (i = 0, *inset = 0; i < size; i++) {
9368 		for (j = 0; j < RTE_DIM(inset_convert_table); j++) {
9369 			if (field[i] == inset_convert_table[j].field) {
9370 				*inset |= inset_convert_table[j].inset;
9371 				break;
9372 			}
9373 		}
9374 
9375 		/* It contains unsupported input set, return immediately */
9376 		if (j == RTE_DIM(inset_convert_table))
9377 			return ret;
9378 	}
9379 
9380 	return 0;
9381 }
9382 
9383 /**
9384  * Translate the input set from bit masks to register aware bit masks
9385  * and vice versa
9386  */
9387 uint64_t
9388 i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input)
9389 {
9390 	uint64_t val = 0;
9391 	uint16_t i;
9392 
9393 	struct inset_map {
9394 		uint64_t inset;
9395 		uint64_t inset_reg;
9396 	};
9397 
9398 	static const struct inset_map inset_map_common[] = {
9399 		{I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC},
9400 		{I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC},
9401 		{I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN},
9402 		{I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN},
9403 		{I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE},
9404 		{I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS},
9405 		{I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6},
9406 		{I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6},
9407 		{I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC},
9408 		{I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR},
9409 		{I40E_INSET_IPV6_HOP_LIMIT, I40E_REG_INSET_L3_IP6_HOP_LIMIT},
9410 		{I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT},
9411 		{I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT},
9412 		{I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG},
9413 		{I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
9414 		{I40E_INSET_TUNNEL_DMAC,
9415 			I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC},
9416 		{I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4},
9417 		{I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6},
9418 		{I40E_INSET_TUNNEL_SRC_PORT,
9419 			I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT},
9420 		{I40E_INSET_TUNNEL_DST_PORT,
9421 			I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT},
9422 		{I40E_INSET_VLAN_TUNNEL, I40E_REG_INSET_TUNNEL_VLAN},
9423 		{I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1},
9424 		{I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2},
9425 		{I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3},
9426 		{I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4},
9427 		{I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5},
9428 		{I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6},
9429 		{I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7},
9430 		{I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8},
9431 	};
9432 
9433     /* some different registers map in x722*/
9434 	static const struct inset_map inset_map_diff_x722[] = {
9435 		{I40E_INSET_IPV4_SRC, I40E_X722_REG_INSET_L3_SRC_IP4},
9436 		{I40E_INSET_IPV4_DST, I40E_X722_REG_INSET_L3_DST_IP4},
9437 		{I40E_INSET_IPV4_PROTO, I40E_X722_REG_INSET_L3_IP4_PROTO},
9438 		{I40E_INSET_IPV4_TTL, I40E_X722_REG_INSET_L3_IP4_TTL},
9439 	};
9440 
9441 	static const struct inset_map inset_map_diff_not_x722[] = {
9442 		{I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4},
9443 		{I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4},
9444 		{I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO},
9445 		{I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL},
9446 	};
9447 
9448 	if (input == 0)
9449 		return val;
9450 
9451 	/* Translate input set to register aware inset */
9452 	if (type == I40E_MAC_X722) {
9453 		for (i = 0; i < RTE_DIM(inset_map_diff_x722); i++) {
9454 			if (input & inset_map_diff_x722[i].inset)
9455 				val |= inset_map_diff_x722[i].inset_reg;
9456 		}
9457 	} else {
9458 		for (i = 0; i < RTE_DIM(inset_map_diff_not_x722); i++) {
9459 			if (input & inset_map_diff_not_x722[i].inset)
9460 				val |= inset_map_diff_not_x722[i].inset_reg;
9461 		}
9462 	}
9463 
9464 	for (i = 0; i < RTE_DIM(inset_map_common); i++) {
9465 		if (input & inset_map_common[i].inset)
9466 			val |= inset_map_common[i].inset_reg;
9467 	}
9468 
9469 	return val;
9470 }
9471 
9472 int
9473 i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem)
9474 {
9475 	uint8_t i, idx = 0;
9476 	uint64_t inset_need_mask = inset;
9477 
9478 	static const struct {
9479 		uint64_t inset;
9480 		uint32_t mask;
9481 	} inset_mask_map[] = {
9482 		{I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK},
9483 		{I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL, 0},
9484 		{I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK},
9485 		{I40E_INSET_IPV4_TTL, I40E_INSET_IPv4_TTL_MASK},
9486 		{I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK},
9487 		{I40E_INSET_IPV6_NEXT_HDR | I40E_INSET_IPV6_HOP_LIMIT, 0},
9488 		{I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK},
9489 		{I40E_INSET_IPV6_HOP_LIMIT, I40E_INSET_IPV6_HOP_LIMIT_MASK},
9490 	};
9491 
9492 	if (!inset || !mask || !nb_elem)
9493 		return 0;
9494 
9495 	for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
9496 		/* Clear the inset bit, if no MASK is required,
9497 		 * for example proto + ttl
9498 		 */
9499 		if ((inset & inset_mask_map[i].inset) ==
9500 		     inset_mask_map[i].inset && inset_mask_map[i].mask == 0)
9501 			inset_need_mask &= ~inset_mask_map[i].inset;
9502 		if (!inset_need_mask)
9503 			return 0;
9504 	}
9505 	for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
9506 		if ((inset_need_mask & inset_mask_map[i].inset) ==
9507 		    inset_mask_map[i].inset) {
9508 			if (idx >= nb_elem) {
9509 				PMD_DRV_LOG(ERR, "exceed maximal number of bitmasks");
9510 				return -EINVAL;
9511 			}
9512 			mask[idx] = inset_mask_map[i].mask;
9513 			idx++;
9514 		}
9515 	}
9516 
9517 	return idx;
9518 }
9519 
9520 void
9521 i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
9522 {
9523 	uint32_t reg = i40e_read_rx_ctl(hw, addr);
9524 
9525 	PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x", addr, reg);
9526 	if (reg != val)
9527 		i40e_write_rx_ctl(hw, addr, val);
9528 	PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x", addr,
9529 		    (uint32_t)i40e_read_rx_ctl(hw, addr));
9530 }
9531 
9532 void
9533 i40e_check_write_global_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
9534 {
9535 	uint32_t reg = i40e_read_rx_ctl(hw, addr);
9536 	struct rte_eth_dev *dev;
9537 
9538 	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
9539 	if (reg != val) {
9540 		i40e_write_rx_ctl(hw, addr, val);
9541 		PMD_DRV_LOG(WARNING,
9542 			    "i40e device %s changed global register [0x%08x]."
9543 			    " original: 0x%08x, new: 0x%08x",
9544 			    dev->device->name, addr, reg,
9545 			    (uint32_t)i40e_read_rx_ctl(hw, addr));
9546 	}
9547 }
9548 
9549 static void
9550 i40e_filter_input_set_init(struct i40e_pf *pf)
9551 {
9552 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9553 	enum i40e_filter_pctype pctype;
9554 	uint64_t input_set, inset_reg;
9555 	uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
9556 	int num, i;
9557 	uint16_t flow_type;
9558 
9559 	for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
9560 	     pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
9561 		flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype);
9562 
9563 		if (flow_type == RTE_ETH_FLOW_UNKNOWN)
9564 			continue;
9565 
9566 		input_set = i40e_get_default_input_set(pctype);
9567 
9568 		num = i40e_generate_inset_mask_reg(input_set, mask_reg,
9569 						   I40E_INSET_MASK_NUM_REG);
9570 		if (num < 0)
9571 			return;
9572 		if (pf->support_multi_driver && num > 0) {
9573 			PMD_DRV_LOG(ERR, "Input set setting is not supported.");
9574 			return;
9575 		}
9576 		inset_reg = i40e_translate_input_set_reg(hw->mac.type,
9577 					input_set);
9578 
9579 		i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
9580 				      (uint32_t)(inset_reg & UINT32_MAX));
9581 		i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
9582 				     (uint32_t)((inset_reg >>
9583 				     I40E_32_BIT_WIDTH) & UINT32_MAX));
9584 		if (!pf->support_multi_driver) {
9585 			i40e_check_write_global_reg(hw,
9586 					    I40E_GLQF_HASH_INSET(0, pctype),
9587 					    (uint32_t)(inset_reg & UINT32_MAX));
9588 			i40e_check_write_global_reg(hw,
9589 					     I40E_GLQF_HASH_INSET(1, pctype),
9590 					     (uint32_t)((inset_reg >>
9591 					      I40E_32_BIT_WIDTH) & UINT32_MAX));
9592 
9593 			for (i = 0; i < num; i++) {
9594 				i40e_check_write_global_reg(hw,
9595 						    I40E_GLQF_FD_MSK(i, pctype),
9596 						    mask_reg[i]);
9597 				i40e_check_write_global_reg(hw,
9598 						  I40E_GLQF_HASH_MSK(i, pctype),
9599 						  mask_reg[i]);
9600 			}
9601 			/*clear unused mask registers of the pctype */
9602 			for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) {
9603 				i40e_check_write_global_reg(hw,
9604 						    I40E_GLQF_FD_MSK(i, pctype),
9605 						    0);
9606 				i40e_check_write_global_reg(hw,
9607 						  I40E_GLQF_HASH_MSK(i, pctype),
9608 						  0);
9609 			}
9610 		} else {
9611 			PMD_DRV_LOG(ERR, "Input set setting is not supported.");
9612 		}
9613 		I40E_WRITE_FLUSH(hw);
9614 
9615 		/* store the default input set */
9616 		if (!pf->support_multi_driver)
9617 			pf->hash_input_set[pctype] = input_set;
9618 		pf->fdir.input_set[pctype] = input_set;
9619 	}
9620 }
9621 
9622 int
9623 i40e_hash_filter_inset_select(struct i40e_hw *hw,
9624 			 struct rte_eth_input_set_conf *conf)
9625 {
9626 	struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
9627 	enum i40e_filter_pctype pctype;
9628 	uint64_t input_set, inset_reg = 0;
9629 	uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
9630 	int ret, i, num;
9631 
9632 	if (!conf) {
9633 		PMD_DRV_LOG(ERR, "Invalid pointer");
9634 		return -EFAULT;
9635 	}
9636 	if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
9637 	    conf->op != RTE_ETH_INPUT_SET_ADD) {
9638 		PMD_DRV_LOG(ERR, "Unsupported input set operation");
9639 		return -EINVAL;
9640 	}
9641 
9642 	if (pf->support_multi_driver) {
9643 		PMD_DRV_LOG(ERR, "Hash input set setting is not supported.");
9644 		return -ENOTSUP;
9645 	}
9646 
9647 	pctype = i40e_flowtype_to_pctype(pf->adapter, conf->flow_type);
9648 	if (pctype == I40E_FILTER_PCTYPE_INVALID) {
9649 		PMD_DRV_LOG(ERR, "invalid flow_type input.");
9650 		return -EINVAL;
9651 	}
9652 
9653 	if (hw->mac.type == I40E_MAC_X722) {
9654 		/* get translated pctype value in fd pctype register */
9655 		pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(hw,
9656 			I40E_GLQF_FD_PCTYPES((int)pctype));
9657 	}
9658 
9659 	ret = i40e_parse_input_set(&input_set, pctype, conf->field,
9660 				   conf->inset_size);
9661 	if (ret) {
9662 		PMD_DRV_LOG(ERR, "Failed to parse input set");
9663 		return -EINVAL;
9664 	}
9665 
9666 	if (conf->op == RTE_ETH_INPUT_SET_ADD) {
9667 		/* get inset value in register */
9668 		inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype));
9669 		inset_reg <<= I40E_32_BIT_WIDTH;
9670 		inset_reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype));
9671 		input_set |= pf->hash_input_set[pctype];
9672 	}
9673 	num = i40e_generate_inset_mask_reg(input_set, mask_reg,
9674 					   I40E_INSET_MASK_NUM_REG);
9675 	if (num < 0)
9676 		return -EINVAL;
9677 
9678 	inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set);
9679 
9680 	i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
9681 				    (uint32_t)(inset_reg & UINT32_MAX));
9682 	i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
9683 				    (uint32_t)((inset_reg >>
9684 				    I40E_32_BIT_WIDTH) & UINT32_MAX));
9685 
9686 	for (i = 0; i < num; i++)
9687 		i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
9688 					    mask_reg[i]);
9689 	/*clear unused mask registers of the pctype */
9690 	for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
9691 		i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
9692 					    0);
9693 	I40E_WRITE_FLUSH(hw);
9694 
9695 	pf->hash_input_set[pctype] = input_set;
9696 	return 0;
9697 }
9698 
9699 int
9700 i40e_fdir_filter_inset_select(struct i40e_pf *pf,
9701 			 struct rte_eth_input_set_conf *conf)
9702 {
9703 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9704 	enum i40e_filter_pctype pctype;
9705 	uint64_t input_set, inset_reg = 0;
9706 	uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
9707 	int ret, i, num;
9708 
9709 	if (!hw || !conf) {
9710 		PMD_DRV_LOG(ERR, "Invalid pointer");
9711 		return -EFAULT;
9712 	}
9713 	if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
9714 	    conf->op != RTE_ETH_INPUT_SET_ADD) {
9715 		PMD_DRV_LOG(ERR, "Unsupported input set operation");
9716 		return -EINVAL;
9717 	}
9718 
9719 	pctype = i40e_flowtype_to_pctype(pf->adapter, conf->flow_type);
9720 
9721 	if (pctype == I40E_FILTER_PCTYPE_INVALID) {
9722 		PMD_DRV_LOG(ERR, "invalid flow_type input.");
9723 		return -EINVAL;
9724 	}
9725 
9726 	ret = i40e_parse_input_set(&input_set, pctype, conf->field,
9727 				   conf->inset_size);
9728 	if (ret) {
9729 		PMD_DRV_LOG(ERR, "Failed to parse input set");
9730 		return -EINVAL;
9731 	}
9732 
9733 	/* get inset value in register */
9734 	inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1));
9735 	inset_reg <<= I40E_32_BIT_WIDTH;
9736 	inset_reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0));
9737 
9738 	/* Can not change the inset reg for flex payload for fdir,
9739 	 * it is done by writing I40E_PRTQF_FD_FLXINSET
9740 	 * in i40e_set_flex_mask_on_pctype.
9741 	 */
9742 	if (conf->op == RTE_ETH_INPUT_SET_SELECT)
9743 		inset_reg &= I40E_REG_INSET_FLEX_PAYLOAD_WORDS;
9744 	else
9745 		input_set |= pf->fdir.input_set[pctype];
9746 	num = i40e_generate_inset_mask_reg(input_set, mask_reg,
9747 					   I40E_INSET_MASK_NUM_REG);
9748 	if (num < 0)
9749 		return -EINVAL;
9750 	if (pf->support_multi_driver && num > 0) {
9751 		PMD_DRV_LOG(ERR, "FDIR bit mask is not supported.");
9752 		return -ENOTSUP;
9753 	}
9754 
9755 	inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set);
9756 
9757 	i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
9758 			      (uint32_t)(inset_reg & UINT32_MAX));
9759 	i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
9760 			     (uint32_t)((inset_reg >>
9761 			     I40E_32_BIT_WIDTH) & UINT32_MAX));
9762 
9763 	if (!pf->support_multi_driver) {
9764 		for (i = 0; i < num; i++)
9765 			i40e_check_write_global_reg(hw,
9766 						    I40E_GLQF_FD_MSK(i, pctype),
9767 						    mask_reg[i]);
9768 		/*clear unused mask registers of the pctype */
9769 		for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
9770 			i40e_check_write_global_reg(hw,
9771 						    I40E_GLQF_FD_MSK(i, pctype),
9772 						    0);
9773 	} else {
9774 		PMD_DRV_LOG(ERR, "FDIR bit mask is not supported.");
9775 	}
9776 	I40E_WRITE_FLUSH(hw);
9777 
9778 	pf->fdir.input_set[pctype] = input_set;
9779 	return 0;
9780 }
9781 
9782 static int
9783 i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
9784 {
9785 	int ret = 0;
9786 
9787 	if (!hw || !info) {
9788 		PMD_DRV_LOG(ERR, "Invalid pointer");
9789 		return -EFAULT;
9790 	}
9791 
9792 	switch (info->info_type) {
9793 	case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
9794 		i40e_get_symmetric_hash_enable_per_port(hw,
9795 					&(info->info.enable));
9796 		break;
9797 	case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
9798 		ret = i40e_get_hash_filter_global_config(hw,
9799 				&(info->info.global_conf));
9800 		break;
9801 	default:
9802 		PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
9803 							info->info_type);
9804 		ret = -EINVAL;
9805 		break;
9806 	}
9807 
9808 	return ret;
9809 }
9810 
9811 static int
9812 i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
9813 {
9814 	int ret = 0;
9815 
9816 	if (!hw || !info) {
9817 		PMD_DRV_LOG(ERR, "Invalid pointer");
9818 		return -EFAULT;
9819 	}
9820 
9821 	switch (info->info_type) {
9822 	case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
9823 		i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable);
9824 		break;
9825 	case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
9826 		ret = i40e_set_hash_filter_global_config(hw,
9827 				&(info->info.global_conf));
9828 		break;
9829 	case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
9830 		ret = i40e_hash_filter_inset_select(hw,
9831 					       &(info->info.input_set_conf));
9832 		break;
9833 
9834 	default:
9835 		PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
9836 							info->info_type);
9837 		ret = -EINVAL;
9838 		break;
9839 	}
9840 
9841 	return ret;
9842 }
9843 
9844 /* Operations for hash function */
9845 static int
9846 i40e_hash_filter_ctrl(struct rte_eth_dev *dev,
9847 		      enum rte_filter_op filter_op,
9848 		      void *arg)
9849 {
9850 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9851 	int ret = 0;
9852 
9853 	switch (filter_op) {
9854 	case RTE_ETH_FILTER_NOP:
9855 		break;
9856 	case RTE_ETH_FILTER_GET:
9857 		ret = i40e_hash_filter_get(hw,
9858 			(struct rte_eth_hash_filter_info *)arg);
9859 		break;
9860 	case RTE_ETH_FILTER_SET:
9861 		ret = i40e_hash_filter_set(hw,
9862 			(struct rte_eth_hash_filter_info *)arg);
9863 		break;
9864 	default:
9865 		PMD_DRV_LOG(WARNING, "Filter operation (%d) not supported",
9866 								filter_op);
9867 		ret = -ENOTSUP;
9868 		break;
9869 	}
9870 
9871 	return ret;
9872 }
9873 
9874 /* Convert ethertype filter structure */
9875 static int
9876 i40e_ethertype_filter_convert(const struct rte_eth_ethertype_filter *input,
9877 			      struct i40e_ethertype_filter *filter)
9878 {
9879 	rte_memcpy(&filter->input.mac_addr, &input->mac_addr, ETHER_ADDR_LEN);
9880 	filter->input.ether_type = input->ether_type;
9881 	filter->flags = input->flags;
9882 	filter->queue = input->queue;
9883 
9884 	return 0;
9885 }
9886 
9887 /* Check if there exists the ehtertype filter */
9888 struct i40e_ethertype_filter *
9889 i40e_sw_ethertype_filter_lookup(struct i40e_ethertype_rule *ethertype_rule,
9890 				const struct i40e_ethertype_filter_input *input)
9891 {
9892 	int ret;
9893 
9894 	ret = rte_hash_lookup(ethertype_rule->hash_table, (const void *)input);
9895 	if (ret < 0)
9896 		return NULL;
9897 
9898 	return ethertype_rule->hash_map[ret];
9899 }
9900 
9901 /* Add ethertype filter in SW list */
9902 static int
9903 i40e_sw_ethertype_filter_insert(struct i40e_pf *pf,
9904 				struct i40e_ethertype_filter *filter)
9905 {
9906 	struct i40e_ethertype_rule *rule = &pf->ethertype;
9907 	int ret;
9908 
9909 	ret = rte_hash_add_key(rule->hash_table, &filter->input);
9910 	if (ret < 0) {
9911 		PMD_DRV_LOG(ERR,
9912 			    "Failed to insert ethertype filter"
9913 			    " to hash table %d!",
9914 			    ret);
9915 		return ret;
9916 	}
9917 	rule->hash_map[ret] = filter;
9918 
9919 	TAILQ_INSERT_TAIL(&rule->ethertype_list, filter, rules);
9920 
9921 	return 0;
9922 }
9923 
9924 /* Delete ethertype filter in SW list */
9925 int
9926 i40e_sw_ethertype_filter_del(struct i40e_pf *pf,
9927 			     struct i40e_ethertype_filter_input *input)
9928 {
9929 	struct i40e_ethertype_rule *rule = &pf->ethertype;
9930 	struct i40e_ethertype_filter *filter;
9931 	int ret;
9932 
9933 	ret = rte_hash_del_key(rule->hash_table, input);
9934 	if (ret < 0) {
9935 		PMD_DRV_LOG(ERR,
9936 			    "Failed to delete ethertype filter"
9937 			    " to hash table %d!",
9938 			    ret);
9939 		return ret;
9940 	}
9941 	filter = rule->hash_map[ret];
9942 	rule->hash_map[ret] = NULL;
9943 
9944 	TAILQ_REMOVE(&rule->ethertype_list, filter, rules);
9945 	rte_free(filter);
9946 
9947 	return 0;
9948 }
9949 
9950 /*
9951  * Configure ethertype filter, which can director packet by filtering
9952  * with mac address and ether_type or only ether_type
9953  */
9954 int
9955 i40e_ethertype_filter_set(struct i40e_pf *pf,
9956 			struct rte_eth_ethertype_filter *filter,
9957 			bool add)
9958 {
9959 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9960 	struct i40e_ethertype_rule *ethertype_rule = &pf->ethertype;
9961 	struct i40e_ethertype_filter *ethertype_filter, *node;
9962 	struct i40e_ethertype_filter check_filter;
9963 	struct i40e_control_filter_stats stats;
9964 	uint16_t flags = 0;
9965 	int ret;
9966 
9967 	if (filter->queue >= pf->dev_data->nb_rx_queues) {
9968 		PMD_DRV_LOG(ERR, "Invalid queue ID");
9969 		return -EINVAL;
9970 	}
9971 	if (filter->ether_type == ETHER_TYPE_IPv4 ||
9972 		filter->ether_type == ETHER_TYPE_IPv6) {
9973 		PMD_DRV_LOG(ERR,
9974 			"unsupported ether_type(0x%04x) in control packet filter.",
9975 			filter->ether_type);
9976 		return -EINVAL;
9977 	}
9978 	if (filter->ether_type == ETHER_TYPE_VLAN)
9979 		PMD_DRV_LOG(WARNING,
9980 			"filter vlan ether_type in first tag is not supported.");
9981 
9982 	/* Check if there is the filter in SW list */
9983 	memset(&check_filter, 0, sizeof(check_filter));
9984 	i40e_ethertype_filter_convert(filter, &check_filter);
9985 	node = i40e_sw_ethertype_filter_lookup(ethertype_rule,
9986 					       &check_filter.input);
9987 	if (add && node) {
9988 		PMD_DRV_LOG(ERR, "Conflict with existing ethertype rules!");
9989 		return -EINVAL;
9990 	}
9991 
9992 	if (!add && !node) {
9993 		PMD_DRV_LOG(ERR, "There's no corresponding ethertype filter!");
9994 		return -EINVAL;
9995 	}
9996 
9997 	if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
9998 		flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
9999 	if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
10000 		flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
10001 	flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
10002 
10003 	memset(&stats, 0, sizeof(stats));
10004 	ret = i40e_aq_add_rem_control_packet_filter(hw,
10005 			filter->mac_addr.addr_bytes,
10006 			filter->ether_type, flags,
10007 			pf->main_vsi->seid,
10008 			filter->queue, add, &stats, NULL);
10009 
10010 	PMD_DRV_LOG(INFO,
10011 		"add/rem control packet filter, return %d, mac_etype_used = %u, etype_used = %u, mac_etype_free = %u, etype_free = %u",
10012 		ret, stats.mac_etype_used, stats.etype_used,
10013 		stats.mac_etype_free, stats.etype_free);
10014 	if (ret < 0)
10015 		return -ENOSYS;
10016 
10017 	/* Add or delete a filter in SW list */
10018 	if (add) {
10019 		ethertype_filter = rte_zmalloc("ethertype_filter",
10020 				       sizeof(*ethertype_filter), 0);
10021 		if (ethertype_filter == NULL) {
10022 			PMD_DRV_LOG(ERR, "Failed to alloc memory.");
10023 			return -ENOMEM;
10024 		}
10025 
10026 		rte_memcpy(ethertype_filter, &check_filter,
10027 			   sizeof(check_filter));
10028 		ret = i40e_sw_ethertype_filter_insert(pf, ethertype_filter);
10029 		if (ret < 0)
10030 			rte_free(ethertype_filter);
10031 	} else {
10032 		ret = i40e_sw_ethertype_filter_del(pf, &node->input);
10033 	}
10034 
10035 	return ret;
10036 }
10037 
10038 /*
10039  * Handle operations for ethertype filter.
10040  */
10041 static int
10042 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
10043 				enum rte_filter_op filter_op,
10044 				void *arg)
10045 {
10046 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
10047 	int ret = 0;
10048 
10049 	if (filter_op == RTE_ETH_FILTER_NOP)
10050 		return ret;
10051 
10052 	if (arg == NULL) {
10053 		PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
10054 			    filter_op);
10055 		return -EINVAL;
10056 	}
10057 
10058 	switch (filter_op) {
10059 	case RTE_ETH_FILTER_ADD:
10060 		ret = i40e_ethertype_filter_set(pf,
10061 			(struct rte_eth_ethertype_filter *)arg,
10062 			TRUE);
10063 		break;
10064 	case RTE_ETH_FILTER_DELETE:
10065 		ret = i40e_ethertype_filter_set(pf,
10066 			(struct rte_eth_ethertype_filter *)arg,
10067 			FALSE);
10068 		break;
10069 	default:
10070 		PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
10071 		ret = -ENOSYS;
10072 		break;
10073 	}
10074 	return ret;
10075 }
10076 
10077 static int
10078 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
10079 		     enum rte_filter_type filter_type,
10080 		     enum rte_filter_op filter_op,
10081 		     void *arg)
10082 {
10083 	int ret = 0;
10084 
10085 	if (dev == NULL)
10086 		return -EINVAL;
10087 
10088 	switch (filter_type) {
10089 	case RTE_ETH_FILTER_NONE:
10090 		/* For global configuration */
10091 		ret = i40e_filter_ctrl_global_config(dev, filter_op, arg);
10092 		break;
10093 	case RTE_ETH_FILTER_HASH:
10094 		ret = i40e_hash_filter_ctrl(dev, filter_op, arg);
10095 		break;
10096 	case RTE_ETH_FILTER_MACVLAN:
10097 		ret = i40e_mac_filter_handle(dev, filter_op, arg);
10098 		break;
10099 	case RTE_ETH_FILTER_ETHERTYPE:
10100 		ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
10101 		break;
10102 	case RTE_ETH_FILTER_TUNNEL:
10103 		ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
10104 		break;
10105 	case RTE_ETH_FILTER_FDIR:
10106 		ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
10107 		break;
10108 	case RTE_ETH_FILTER_GENERIC:
10109 		if (filter_op != RTE_ETH_FILTER_GET)
10110 			return -EINVAL;
10111 		*(const void **)arg = &i40e_flow_ops;
10112 		break;
10113 	default:
10114 		PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
10115 							filter_type);
10116 		ret = -EINVAL;
10117 		break;
10118 	}
10119 
10120 	return ret;
10121 }
10122 
10123 /*
10124  * Check and enable Extended Tag.
10125  * Enabling Extended Tag is important for 40G performance.
10126  */
10127 static void
10128 i40e_enable_extended_tag(struct rte_eth_dev *dev)
10129 {
10130 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
10131 	uint32_t buf = 0;
10132 	int ret;
10133 
10134 	ret = rte_pci_read_config(pci_dev, &buf, sizeof(buf),
10135 				      PCI_DEV_CAP_REG);
10136 	if (ret < 0) {
10137 		PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
10138 			    PCI_DEV_CAP_REG);
10139 		return;
10140 	}
10141 	if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) {
10142 		PMD_DRV_LOG(ERR, "Does not support Extended Tag");
10143 		return;
10144 	}
10145 
10146 	buf = 0;
10147 	ret = rte_pci_read_config(pci_dev, &buf, sizeof(buf),
10148 				      PCI_DEV_CTRL_REG);
10149 	if (ret < 0) {
10150 		PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
10151 			    PCI_DEV_CTRL_REG);
10152 		return;
10153 	}
10154 	if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) {
10155 		PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled");
10156 		return;
10157 	}
10158 	buf |= PCI_DEV_CTRL_EXT_TAG_MASK;
10159 	ret = rte_pci_write_config(pci_dev, &buf, sizeof(buf),
10160 				       PCI_DEV_CTRL_REG);
10161 	if (ret < 0) {
10162 		PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x",
10163 			    PCI_DEV_CTRL_REG);
10164 		return;
10165 	}
10166 }
10167 
10168 /*
10169  * As some registers wouldn't be reset unless a global hardware reset,
10170  * hardware initialization is needed to put those registers into an
10171  * expected initial state.
10172  */
10173 static void
10174 i40e_hw_init(struct rte_eth_dev *dev)
10175 {
10176 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10177 
10178 	i40e_enable_extended_tag(dev);
10179 
10180 	/* clear the PF Queue Filter control register */
10181 	i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0);
10182 
10183 	/* Disable symmetric hash per port */
10184 	i40e_set_symmetric_hash_enable_per_port(hw, 0);
10185 }
10186 
10187 /*
10188  * For X722 it is possible to have multiple pctypes mapped to the same flowtype
10189  * however this function will return only one highest pctype index,
10190  * which is not quite correct. This is known problem of i40e driver
10191  * and needs to be fixed later.
10192  */
10193 enum i40e_filter_pctype
10194 i40e_flowtype_to_pctype(const struct i40e_adapter *adapter, uint16_t flow_type)
10195 {
10196 	int i;
10197 	uint64_t pctype_mask;
10198 
10199 	if (flow_type < I40E_FLOW_TYPE_MAX) {
10200 		pctype_mask = adapter->pctypes_tbl[flow_type];
10201 		for (i = I40E_FILTER_PCTYPE_MAX - 1; i > 0; i--) {
10202 			if (pctype_mask & (1ULL << i))
10203 				return (enum i40e_filter_pctype)i;
10204 		}
10205 	}
10206 	return I40E_FILTER_PCTYPE_INVALID;
10207 }
10208 
10209 uint16_t
10210 i40e_pctype_to_flowtype(const struct i40e_adapter *adapter,
10211 			enum i40e_filter_pctype pctype)
10212 {
10213 	uint16_t flowtype;
10214 	uint64_t pctype_mask = 1ULL << pctype;
10215 
10216 	for (flowtype = RTE_ETH_FLOW_UNKNOWN + 1; flowtype < I40E_FLOW_TYPE_MAX;
10217 	     flowtype++) {
10218 		if (adapter->pctypes_tbl[flowtype] & pctype_mask)
10219 			return flowtype;
10220 	}
10221 
10222 	return RTE_ETH_FLOW_UNKNOWN;
10223 }
10224 
10225 /*
10226  * On X710, performance number is far from the expectation on recent firmware
10227  * versions; on XL710, performance number is also far from the expectation on
10228  * recent firmware versions, if promiscuous mode is disabled, or promiscuous
10229  * mode is enabled and port MAC address is equal to the packet destination MAC
10230  * address. The fix for this issue may not be integrated in the following
10231  * firmware version. So the workaround in software driver is needed. It needs
10232  * to modify the initial values of 3 internal only registers for both X710 and
10233  * XL710. Note that the values for X710 or XL710 could be different, and the
10234  * workaround can be removed when it is fixed in firmware in the future.
10235  */
10236 
10237 /* For both X710 and XL710 */
10238 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1	0x10000200
10239 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2	0x203F0200
10240 #define I40E_GL_SWR_PRI_JOIN_MAP_0		0x26CE00
10241 
10242 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200
10243 #define I40E_GL_SWR_PRI_JOIN_MAP_2       0x26CE08
10244 
10245 /* For X722 */
10246 #define I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x20000200
10247 #define I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x013F0200
10248 
10249 /* For X710 */
10250 #define I40E_GL_SWR_PM_UP_THR_EF_VALUE   0x03030303
10251 /* For XL710 */
10252 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
10253 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
10254 
10255 /*
10256  * GL_SWR_PM_UP_THR:
10257  * The value is not impacted from the link speed, its value is set according
10258  * to the total number of ports for a better pipe-monitor configuration.
10259  */
10260 static bool
10261 i40e_get_swr_pm_cfg(struct i40e_hw *hw, uint32_t *value)
10262 {
10263 #define I40E_GL_SWR_PM_EF_DEVICE(dev) \
10264 		.device_id = (dev),   \
10265 		.val = I40E_GL_SWR_PM_UP_THR_EF_VALUE
10266 
10267 #define I40E_GL_SWR_PM_SF_DEVICE(dev) \
10268 		.device_id = (dev),   \
10269 		.val = I40E_GL_SWR_PM_UP_THR_SF_VALUE
10270 
10271 	static const struct {
10272 		uint16_t device_id;
10273 		uint32_t val;
10274 	} swr_pm_table[] = {
10275 		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_SFP_XL710) },
10276 		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_KX_C) },
10277 		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T) },
10278 		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T4) },
10279 
10280 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_KX_B) },
10281 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_A) },
10282 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_B) },
10283 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2) },
10284 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2_A) },
10285 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_B) },
10286 		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_SFP28) },
10287 	};
10288 	uint32_t i;
10289 
10290 	if (value == NULL) {
10291 		PMD_DRV_LOG(ERR, "value is NULL");
10292 		return false;
10293 	}
10294 
10295 	for (i = 0; i < RTE_DIM(swr_pm_table); i++) {
10296 		if (hw->device_id == swr_pm_table[i].device_id) {
10297 			*value = swr_pm_table[i].val;
10298 
10299 			PMD_DRV_LOG(DEBUG, "Device 0x%x with GL_SWR_PM_UP_THR "
10300 				    "value - 0x%08x",
10301 				    hw->device_id, *value);
10302 			return true;
10303 		}
10304 	}
10305 
10306 	return false;
10307 }
10308 
10309 static int
10310 i40e_dev_sync_phy_type(struct i40e_hw *hw)
10311 {
10312 	enum i40e_status_code status;
10313 	struct i40e_aq_get_phy_abilities_resp phy_ab;
10314 	int ret = -ENOTSUP;
10315 	int retries = 0;
10316 
10317 	status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab,
10318 					      NULL);
10319 
10320 	while (status) {
10321 		PMD_INIT_LOG(WARNING, "Failed to sync phy type: status=%d",
10322 			status);
10323 		retries++;
10324 		rte_delay_us(100000);
10325 		if  (retries < 5)
10326 			status = i40e_aq_get_phy_capabilities(hw, false,
10327 					true, &phy_ab, NULL);
10328 		else
10329 			return ret;
10330 	}
10331 	return 0;
10332 }
10333 
10334 static void
10335 i40e_configure_registers(struct i40e_hw *hw)
10336 {
10337 	static struct {
10338 		uint32_t addr;
10339 		uint64_t val;
10340 	} reg_table[] = {
10341 		{I40E_GL_SWR_PRI_JOIN_MAP_0, 0},
10342 		{I40E_GL_SWR_PRI_JOIN_MAP_2, 0},
10343 		{I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */
10344 	};
10345 	uint64_t reg;
10346 	uint32_t i;
10347 	int ret;
10348 
10349 	for (i = 0; i < RTE_DIM(reg_table); i++) {
10350 		if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_0) {
10351 			if (hw->mac.type == I40E_MAC_X722) /* For X722 */
10352 				reg_table[i].val =
10353 					I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE;
10354 			else /* For X710/XL710/XXV710 */
10355 				if (hw->aq.fw_maj_ver < 6)
10356 					reg_table[i].val =
10357 					     I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1;
10358 				else
10359 					reg_table[i].val =
10360 					     I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2;
10361 		}
10362 
10363 		if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_2) {
10364 			if (hw->mac.type == I40E_MAC_X722) /* For X722 */
10365 				reg_table[i].val =
10366 					I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE;
10367 			else /* For X710/XL710/XXV710 */
10368 				reg_table[i].val =
10369 					I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE;
10370 		}
10371 
10372 		if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
10373 			uint32_t cfg_val;
10374 
10375 			if (!i40e_get_swr_pm_cfg(hw, &cfg_val)) {
10376 				PMD_DRV_LOG(DEBUG, "Device 0x%x skips "
10377 					    "GL_SWR_PM_UP_THR value fixup",
10378 					    hw->device_id);
10379 				continue;
10380 			}
10381 
10382 			reg_table[i].val = cfg_val;
10383 		}
10384 
10385 		ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
10386 							&reg, NULL);
10387 		if (ret < 0) {
10388 			PMD_DRV_LOG(ERR, "Failed to read from 0x%"PRIx32,
10389 							reg_table[i].addr);
10390 			break;
10391 		}
10392 		PMD_DRV_LOG(DEBUG, "Read from 0x%"PRIx32": 0x%"PRIx64,
10393 						reg_table[i].addr, reg);
10394 		if (reg == reg_table[i].val)
10395 			continue;
10396 
10397 		ret = i40e_aq_debug_write_register(hw, reg_table[i].addr,
10398 						reg_table[i].val, NULL);
10399 		if (ret < 0) {
10400 			PMD_DRV_LOG(ERR,
10401 				"Failed to write 0x%"PRIx64" to the address of 0x%"PRIx32,
10402 				reg_table[i].val, reg_table[i].addr);
10403 			break;
10404 		}
10405 		PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of "
10406 			"0x%"PRIx32, reg_table[i].val, reg_table[i].addr);
10407 	}
10408 }
10409 
10410 #define I40E_VSI_TSR(_i)            (0x00050800 + ((_i) * 4))
10411 #define I40E_VSI_TSR_QINQ_CONFIG    0xc030
10412 #define I40E_VSI_L2TAGSTXVALID(_i)  (0x00042800 + ((_i) * 4))
10413 #define I40E_VSI_L2TAGSTXVALID_QINQ 0xab
10414 static int
10415 i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi)
10416 {
10417 	uint32_t reg;
10418 	int ret;
10419 
10420 	if (vsi->vsi_id >= I40E_MAX_NUM_VSIS) {
10421 		PMD_DRV_LOG(ERR, "VSI ID exceeds the maximum");
10422 		return -EINVAL;
10423 	}
10424 
10425 	/* Configure for double VLAN RX stripping */
10426 	reg = I40E_READ_REG(hw, I40E_VSI_TSR(vsi->vsi_id));
10427 	if ((reg & I40E_VSI_TSR_QINQ_CONFIG) != I40E_VSI_TSR_QINQ_CONFIG) {
10428 		reg |= I40E_VSI_TSR_QINQ_CONFIG;
10429 		ret = i40e_aq_debug_write_register(hw,
10430 						   I40E_VSI_TSR(vsi->vsi_id),
10431 						   reg, NULL);
10432 		if (ret < 0) {
10433 			PMD_DRV_LOG(ERR, "Failed to update VSI_TSR[%d]",
10434 				    vsi->vsi_id);
10435 			return I40E_ERR_CONFIG;
10436 		}
10437 	}
10438 
10439 	/* Configure for double VLAN TX insertion */
10440 	reg = I40E_READ_REG(hw, I40E_VSI_L2TAGSTXVALID(vsi->vsi_id));
10441 	if ((reg & 0xff) != I40E_VSI_L2TAGSTXVALID_QINQ) {
10442 		reg = I40E_VSI_L2TAGSTXVALID_QINQ;
10443 		ret = i40e_aq_debug_write_register(hw,
10444 						   I40E_VSI_L2TAGSTXVALID(
10445 						   vsi->vsi_id), reg, NULL);
10446 		if (ret < 0) {
10447 			PMD_DRV_LOG(ERR,
10448 				"Failed to update VSI_L2TAGSTXVALID[%d]",
10449 				vsi->vsi_id);
10450 			return I40E_ERR_CONFIG;
10451 		}
10452 	}
10453 
10454 	return 0;
10455 }
10456 
10457 /**
10458  * i40e_aq_add_mirror_rule
10459  * @hw: pointer to the hardware structure
10460  * @seid: VEB seid to add mirror rule to
10461  * @dst_id: destination vsi seid
10462  * @entries: Buffer which contains the entities to be mirrored
10463  * @count: number of entities contained in the buffer
10464  * @rule_id:the rule_id of the rule to be added
10465  *
10466  * Add a mirror rule for a given veb.
10467  *
10468  **/
10469 static enum i40e_status_code
10470 i40e_aq_add_mirror_rule(struct i40e_hw *hw,
10471 			uint16_t seid, uint16_t dst_id,
10472 			uint16_t rule_type, uint16_t *entries,
10473 			uint16_t count, uint16_t *rule_id)
10474 {
10475 	struct i40e_aq_desc desc;
10476 	struct i40e_aqc_add_delete_mirror_rule cmd;
10477 	struct i40e_aqc_add_delete_mirror_rule_completion *resp =
10478 		(struct i40e_aqc_add_delete_mirror_rule_completion *)
10479 		&desc.params.raw;
10480 	uint16_t buff_len;
10481 	enum i40e_status_code status;
10482 
10483 	i40e_fill_default_direct_cmd_desc(&desc,
10484 					  i40e_aqc_opc_add_mirror_rule);
10485 	memset(&cmd, 0, sizeof(cmd));
10486 
10487 	buff_len = sizeof(uint16_t) * count;
10488 	desc.datalen = rte_cpu_to_le_16(buff_len);
10489 	if (buff_len > 0)
10490 		desc.flags |= rte_cpu_to_le_16(
10491 			(uint16_t)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
10492 	cmd.rule_type = rte_cpu_to_le_16(rule_type <<
10493 				I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
10494 	cmd.num_entries = rte_cpu_to_le_16(count);
10495 	cmd.seid = rte_cpu_to_le_16(seid);
10496 	cmd.destination = rte_cpu_to_le_16(dst_id);
10497 
10498 	rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
10499 	status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL);
10500 	PMD_DRV_LOG(INFO,
10501 		"i40e_aq_add_mirror_rule, aq_status %d, rule_id = %u mirror_rules_used = %u, mirror_rules_free = %u,",
10502 		hw->aq.asq_last_status, resp->rule_id,
10503 		resp->mirror_rules_used, resp->mirror_rules_free);
10504 	*rule_id = rte_le_to_cpu_16(resp->rule_id);
10505 
10506 	return status;
10507 }
10508 
10509 /**
10510  * i40e_aq_del_mirror_rule
10511  * @hw: pointer to the hardware structure
10512  * @seid: VEB seid to add mirror rule to
10513  * @entries: Buffer which contains the entities to be mirrored
10514  * @count: number of entities contained in the buffer
10515  * @rule_id:the rule_id of the rule to be delete
10516  *
10517  * Delete a mirror rule for a given veb.
10518  *
10519  **/
10520 static enum i40e_status_code
10521 i40e_aq_del_mirror_rule(struct i40e_hw *hw,
10522 		uint16_t seid, uint16_t rule_type, uint16_t *entries,
10523 		uint16_t count, uint16_t rule_id)
10524 {
10525 	struct i40e_aq_desc desc;
10526 	struct i40e_aqc_add_delete_mirror_rule cmd;
10527 	uint16_t buff_len = 0;
10528 	enum i40e_status_code status;
10529 	void *buff = NULL;
10530 
10531 	i40e_fill_default_direct_cmd_desc(&desc,
10532 					  i40e_aqc_opc_delete_mirror_rule);
10533 	memset(&cmd, 0, sizeof(cmd));
10534 	if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) {
10535 		desc.flags |= rte_cpu_to_le_16((uint16_t)(I40E_AQ_FLAG_BUF |
10536 							  I40E_AQ_FLAG_RD));
10537 		cmd.num_entries = count;
10538 		buff_len = sizeof(uint16_t) * count;
10539 		desc.datalen = rte_cpu_to_le_16(buff_len);
10540 		buff = (void *)entries;
10541 	} else
10542 		/* rule id is filled in destination field for deleting mirror rule */
10543 		cmd.destination = rte_cpu_to_le_16(rule_id);
10544 
10545 	cmd.rule_type = rte_cpu_to_le_16(rule_type <<
10546 				I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
10547 	cmd.seid = rte_cpu_to_le_16(seid);
10548 
10549 	rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
10550 	status = i40e_asq_send_command(hw, &desc, buff, buff_len, NULL);
10551 
10552 	return status;
10553 }
10554 
10555 /**
10556  * i40e_mirror_rule_set
10557  * @dev: pointer to the hardware structure
10558  * @mirror_conf: mirror rule info
10559  * @sw_id: mirror rule's sw_id
10560  * @on: enable/disable
10561  *
10562  * set a mirror rule.
10563  *
10564  **/
10565 static int
10566 i40e_mirror_rule_set(struct rte_eth_dev *dev,
10567 			struct rte_eth_mirror_conf *mirror_conf,
10568 			uint8_t sw_id, uint8_t on)
10569 {
10570 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
10571 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10572 	struct i40e_mirror_rule *it, *mirr_rule = NULL;
10573 	struct i40e_mirror_rule *parent = NULL;
10574 	uint16_t seid, dst_seid, rule_id;
10575 	uint16_t i, j = 0;
10576 	int ret;
10577 
10578 	PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id);
10579 
10580 	if (pf->main_vsi->veb == NULL || pf->vfs == NULL) {
10581 		PMD_DRV_LOG(ERR,
10582 			"mirror rule can not be configured without veb or vfs.");
10583 		return -ENOSYS;
10584 	}
10585 	if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) {
10586 		PMD_DRV_LOG(ERR, "mirror table is full.");
10587 		return -ENOSPC;
10588 	}
10589 	if (mirror_conf->dst_pool > pf->vf_num) {
10590 		PMD_DRV_LOG(ERR, "invalid destination pool %u.",
10591 				 mirror_conf->dst_pool);
10592 		return -EINVAL;
10593 	}
10594 
10595 	seid = pf->main_vsi->veb->seid;
10596 
10597 	TAILQ_FOREACH(it, &pf->mirror_list, rules) {
10598 		if (sw_id <= it->index) {
10599 			mirr_rule = it;
10600 			break;
10601 		}
10602 		parent = it;
10603 	}
10604 	if (mirr_rule && sw_id == mirr_rule->index) {
10605 		if (on) {
10606 			PMD_DRV_LOG(ERR, "mirror rule exists.");
10607 			return -EEXIST;
10608 		} else {
10609 			ret = i40e_aq_del_mirror_rule(hw, seid,
10610 					mirr_rule->rule_type,
10611 					mirr_rule->entries,
10612 					mirr_rule->num_entries, mirr_rule->id);
10613 			if (ret < 0) {
10614 				PMD_DRV_LOG(ERR,
10615 					"failed to remove mirror rule: ret = %d, aq_err = %d.",
10616 					ret, hw->aq.asq_last_status);
10617 				return -ENOSYS;
10618 			}
10619 			TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
10620 			rte_free(mirr_rule);
10621 			pf->nb_mirror_rule--;
10622 			return 0;
10623 		}
10624 	} else if (!on) {
10625 		PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
10626 		return -ENOENT;
10627 	}
10628 
10629 	mirr_rule = rte_zmalloc("i40e_mirror_rule",
10630 				sizeof(struct i40e_mirror_rule) , 0);
10631 	if (!mirr_rule) {
10632 		PMD_DRV_LOG(ERR, "failed to allocate memory");
10633 		return I40E_ERR_NO_MEMORY;
10634 	}
10635 	switch (mirror_conf->rule_type) {
10636 	case ETH_MIRROR_VLAN:
10637 		for (i = 0, j = 0; i < ETH_MIRROR_MAX_VLANS; i++) {
10638 			if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
10639 				mirr_rule->entries[j] =
10640 					mirror_conf->vlan.vlan_id[i];
10641 				j++;
10642 			}
10643 		}
10644 		if (j == 0) {
10645 			PMD_DRV_LOG(ERR, "vlan is not specified.");
10646 			rte_free(mirr_rule);
10647 			return -EINVAL;
10648 		}
10649 		mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_VLAN;
10650 		break;
10651 	case ETH_MIRROR_VIRTUAL_POOL_UP:
10652 	case ETH_MIRROR_VIRTUAL_POOL_DOWN:
10653 		/* check if the specified pool bit is out of range */
10654 		if (mirror_conf->pool_mask > (uint64_t)(1ULL << (pf->vf_num + 1))) {
10655 			PMD_DRV_LOG(ERR, "pool mask is out of range.");
10656 			rte_free(mirr_rule);
10657 			return -EINVAL;
10658 		}
10659 		for (i = 0, j = 0; i < pf->vf_num; i++) {
10660 			if (mirror_conf->pool_mask & (1ULL << i)) {
10661 				mirr_rule->entries[j] = pf->vfs[i].vsi->seid;
10662 				j++;
10663 			}
10664 		}
10665 		if (mirror_conf->pool_mask & (1ULL << pf->vf_num)) {
10666 			/* add pf vsi to entries */
10667 			mirr_rule->entries[j] = pf->main_vsi_seid;
10668 			j++;
10669 		}
10670 		if (j == 0) {
10671 			PMD_DRV_LOG(ERR, "pool is not specified.");
10672 			rte_free(mirr_rule);
10673 			return -EINVAL;
10674 		}
10675 		/* egress and ingress in aq commands means from switch but not port */
10676 		mirr_rule->rule_type =
10677 			(mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) ?
10678 			I40E_AQC_MIRROR_RULE_TYPE_VPORT_EGRESS :
10679 			I40E_AQC_MIRROR_RULE_TYPE_VPORT_INGRESS;
10680 		break;
10681 	case ETH_MIRROR_UPLINK_PORT:
10682 		/* egress and ingress in aq commands means from switch but not port*/
10683 		mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS;
10684 		break;
10685 	case ETH_MIRROR_DOWNLINK_PORT:
10686 		mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS;
10687 		break;
10688 	default:
10689 		PMD_DRV_LOG(ERR, "unsupported mirror type %d.",
10690 			mirror_conf->rule_type);
10691 		rte_free(mirr_rule);
10692 		return -EINVAL;
10693 	}
10694 
10695 	/* If the dst_pool is equal to vf_num, consider it as PF */
10696 	if (mirror_conf->dst_pool == pf->vf_num)
10697 		dst_seid = pf->main_vsi_seid;
10698 	else
10699 		dst_seid = pf->vfs[mirror_conf->dst_pool].vsi->seid;
10700 
10701 	ret = i40e_aq_add_mirror_rule(hw, seid, dst_seid,
10702 				      mirr_rule->rule_type, mirr_rule->entries,
10703 				      j, &rule_id);
10704 	if (ret < 0) {
10705 		PMD_DRV_LOG(ERR,
10706 			"failed to add mirror rule: ret = %d, aq_err = %d.",
10707 			ret, hw->aq.asq_last_status);
10708 		rte_free(mirr_rule);
10709 		return -ENOSYS;
10710 	}
10711 
10712 	mirr_rule->index = sw_id;
10713 	mirr_rule->num_entries = j;
10714 	mirr_rule->id = rule_id;
10715 	mirr_rule->dst_vsi_seid = dst_seid;
10716 
10717 	if (parent)
10718 		TAILQ_INSERT_AFTER(&pf->mirror_list, parent, mirr_rule, rules);
10719 	else
10720 		TAILQ_INSERT_HEAD(&pf->mirror_list, mirr_rule, rules);
10721 
10722 	pf->nb_mirror_rule++;
10723 	return 0;
10724 }
10725 
10726 /**
10727  * i40e_mirror_rule_reset
10728  * @dev: pointer to the device
10729  * @sw_id: mirror rule's sw_id
10730  *
10731  * reset a mirror rule.
10732  *
10733  **/
10734 static int
10735 i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id)
10736 {
10737 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
10738 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10739 	struct i40e_mirror_rule *it, *mirr_rule = NULL;
10740 	uint16_t seid;
10741 	int ret;
10742 
10743 	PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_reset: sw_id = %d.", sw_id);
10744 
10745 	seid = pf->main_vsi->veb->seid;
10746 
10747 	TAILQ_FOREACH(it, &pf->mirror_list, rules) {
10748 		if (sw_id == it->index) {
10749 			mirr_rule = it;
10750 			break;
10751 		}
10752 	}
10753 	if (mirr_rule) {
10754 		ret = i40e_aq_del_mirror_rule(hw, seid,
10755 				mirr_rule->rule_type,
10756 				mirr_rule->entries,
10757 				mirr_rule->num_entries, mirr_rule->id);
10758 		if (ret < 0) {
10759 			PMD_DRV_LOG(ERR,
10760 				"failed to remove mirror rule: status = %d, aq_err = %d.",
10761 				ret, hw->aq.asq_last_status);
10762 			return -ENOSYS;
10763 		}
10764 		TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
10765 		rte_free(mirr_rule);
10766 		pf->nb_mirror_rule--;
10767 	} else {
10768 		PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
10769 		return -ENOENT;
10770 	}
10771 	return 0;
10772 }
10773 
10774 static uint64_t
10775 i40e_read_systime_cyclecounter(struct rte_eth_dev *dev)
10776 {
10777 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10778 	uint64_t systim_cycles;
10779 
10780 	systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L);
10781 	systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H)
10782 			<< 32;
10783 
10784 	return systim_cycles;
10785 }
10786 
10787 static uint64_t
10788 i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index)
10789 {
10790 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10791 	uint64_t rx_tstamp;
10792 
10793 	rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index));
10794 	rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index))
10795 			<< 32;
10796 
10797 	return rx_tstamp;
10798 }
10799 
10800 static uint64_t
10801 i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
10802 {
10803 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10804 	uint64_t tx_tstamp;
10805 
10806 	tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L);
10807 	tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H)
10808 			<< 32;
10809 
10810 	return tx_tstamp;
10811 }
10812 
10813 static void
10814 i40e_start_timecounters(struct rte_eth_dev *dev)
10815 {
10816 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10817 	struct i40e_adapter *adapter = dev->data->dev_private;
10818 	struct rte_eth_link link;
10819 	uint32_t tsync_inc_l;
10820 	uint32_t tsync_inc_h;
10821 
10822 	/* Get current link speed. */
10823 	i40e_dev_link_update(dev, 1);
10824 	rte_eth_linkstatus_get(dev, &link);
10825 
10826 	switch (link.link_speed) {
10827 	case ETH_SPEED_NUM_40G:
10828 	case ETH_SPEED_NUM_25G:
10829 		tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF;
10830 		tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32;
10831 		break;
10832 	case ETH_SPEED_NUM_10G:
10833 		tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF;
10834 		tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32;
10835 		break;
10836 	case ETH_SPEED_NUM_1G:
10837 		tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF;
10838 		tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32;
10839 		break;
10840 	default:
10841 		tsync_inc_l = 0x0;
10842 		tsync_inc_h = 0x0;
10843 	}
10844 
10845 	/* Set the timesync increment value. */
10846 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l);
10847 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h);
10848 
10849 	memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
10850 	memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
10851 	memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
10852 
10853 	adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
10854 	adapter->systime_tc.cc_shift = 0;
10855 	adapter->systime_tc.nsec_mask = 0;
10856 
10857 	adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
10858 	adapter->rx_tstamp_tc.cc_shift = 0;
10859 	adapter->rx_tstamp_tc.nsec_mask = 0;
10860 
10861 	adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
10862 	adapter->tx_tstamp_tc.cc_shift = 0;
10863 	adapter->tx_tstamp_tc.nsec_mask = 0;
10864 }
10865 
10866 static int
10867 i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
10868 {
10869 	struct i40e_adapter *adapter = dev->data->dev_private;
10870 
10871 	adapter->systime_tc.nsec += delta;
10872 	adapter->rx_tstamp_tc.nsec += delta;
10873 	adapter->tx_tstamp_tc.nsec += delta;
10874 
10875 	return 0;
10876 }
10877 
10878 static int
10879 i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
10880 {
10881 	uint64_t ns;
10882 	struct i40e_adapter *adapter = dev->data->dev_private;
10883 
10884 	ns = rte_timespec_to_ns(ts);
10885 
10886 	/* Set the timecounters to a new value. */
10887 	adapter->systime_tc.nsec = ns;
10888 	adapter->rx_tstamp_tc.nsec = ns;
10889 	adapter->tx_tstamp_tc.nsec = ns;
10890 
10891 	return 0;
10892 }
10893 
10894 static int
10895 i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
10896 {
10897 	uint64_t ns, systime_cycles;
10898 	struct i40e_adapter *adapter = dev->data->dev_private;
10899 
10900 	systime_cycles = i40e_read_systime_cyclecounter(dev);
10901 	ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
10902 	*ts = rte_ns_to_timespec(ns);
10903 
10904 	return 0;
10905 }
10906 
10907 static int
10908 i40e_timesync_enable(struct rte_eth_dev *dev)
10909 {
10910 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10911 	uint32_t tsync_ctl_l;
10912 	uint32_t tsync_ctl_h;
10913 
10914 	/* Stop the timesync system time. */
10915 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
10916 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
10917 	/* Reset the timesync system time value. */
10918 	I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0);
10919 	I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0);
10920 
10921 	i40e_start_timecounters(dev);
10922 
10923 	/* Clear timesync registers. */
10924 	I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
10925 	I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H);
10926 	I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0));
10927 	I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1));
10928 	I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2));
10929 	I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3));
10930 
10931 	/* Enable timestamping of PTP packets. */
10932 	tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
10933 	tsync_ctl_l |= I40E_PRTTSYN_TSYNENA;
10934 
10935 	tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
10936 	tsync_ctl_h |= I40E_PRTTSYN_TSYNENA;
10937 	tsync_ctl_h |= I40E_PRTTSYN_TSYNTYPE;
10938 
10939 	I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
10940 	I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
10941 
10942 	return 0;
10943 }
10944 
10945 static int
10946 i40e_timesync_disable(struct rte_eth_dev *dev)
10947 {
10948 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10949 	uint32_t tsync_ctl_l;
10950 	uint32_t tsync_ctl_h;
10951 
10952 	/* Disable timestamping of transmitted PTP packets. */
10953 	tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
10954 	tsync_ctl_l &= ~I40E_PRTTSYN_TSYNENA;
10955 
10956 	tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
10957 	tsync_ctl_h &= ~I40E_PRTTSYN_TSYNENA;
10958 
10959 	I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
10960 	I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
10961 
10962 	/* Reset the timesync increment value. */
10963 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
10964 	I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
10965 
10966 	return 0;
10967 }
10968 
10969 static int
10970 i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
10971 				struct timespec *timestamp, uint32_t flags)
10972 {
10973 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10974 	struct i40e_adapter *adapter = dev->data->dev_private;
10975 	uint32_t sync_status;
10976 	uint32_t index = flags & 0x03;
10977 	uint64_t rx_tstamp_cycles;
10978 	uint64_t ns;
10979 
10980 	sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1);
10981 	if ((sync_status & (1 << index)) == 0)
10982 		return -EINVAL;
10983 
10984 	rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index);
10985 	ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
10986 	*timestamp = rte_ns_to_timespec(ns);
10987 
10988 	return 0;
10989 }
10990 
10991 static int
10992 i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
10993 				struct timespec *timestamp)
10994 {
10995 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10996 	struct i40e_adapter *adapter = dev->data->dev_private;
10997 	uint32_t sync_status;
10998 	uint64_t tx_tstamp_cycles;
10999 	uint64_t ns;
11000 
11001 	sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
11002 	if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0)
11003 		return -EINVAL;
11004 
11005 	tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev);
11006 	ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
11007 	*timestamp = rte_ns_to_timespec(ns);
11008 
11009 	return 0;
11010 }
11011 
11012 /*
11013  * i40e_parse_dcb_configure - parse dcb configure from user
11014  * @dev: the device being configured
11015  * @dcb_cfg: pointer of the result of parse
11016  * @*tc_map: bit map of enabled traffic classes
11017  *
11018  * Returns 0 on success, negative value on failure
11019  */
11020 static int
11021 i40e_parse_dcb_configure(struct rte_eth_dev *dev,
11022 			 struct i40e_dcbx_config *dcb_cfg,
11023 			 uint8_t *tc_map)
11024 {
11025 	struct rte_eth_dcb_rx_conf *dcb_rx_conf;
11026 	uint8_t i, tc_bw, bw_lf;
11027 
11028 	memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config));
11029 
11030 	dcb_rx_conf = &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
11031 	if (dcb_rx_conf->nb_tcs > I40E_MAX_TRAFFIC_CLASS) {
11032 		PMD_INIT_LOG(ERR, "number of tc exceeds max.");
11033 		return -EINVAL;
11034 	}
11035 
11036 	/* assume each tc has the same bw */
11037 	tc_bw = I40E_MAX_PERCENT / dcb_rx_conf->nb_tcs;
11038 	for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
11039 		dcb_cfg->etscfg.tcbwtable[i] = tc_bw;
11040 	/* to ensure the sum of tcbw is equal to 100 */
11041 	bw_lf = I40E_MAX_PERCENT % dcb_rx_conf->nb_tcs;
11042 	for (i = 0; i < bw_lf; i++)
11043 		dcb_cfg->etscfg.tcbwtable[i]++;
11044 
11045 	/* assume each tc has the same Transmission Selection Algorithm */
11046 	for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
11047 		dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS;
11048 
11049 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
11050 		dcb_cfg->etscfg.prioritytable[i] =
11051 				dcb_rx_conf->dcb_tc[i];
11052 
11053 	/* FW needs one App to configure HW */
11054 	dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM;
11055 	dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE;
11056 	dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO;
11057 	dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE;
11058 
11059 	if (dcb_rx_conf->nb_tcs == 0)
11060 		*tc_map = 1; /* tc0 only */
11061 	else
11062 		*tc_map = RTE_LEN2MASK(dcb_rx_conf->nb_tcs, uint8_t);
11063 
11064 	if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
11065 		dcb_cfg->pfc.willing = 0;
11066 		dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
11067 		dcb_cfg->pfc.pfcenable = *tc_map;
11068 	}
11069 	return 0;
11070 }
11071 
11072 
11073 static enum i40e_status_code
11074 i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi,
11075 			      struct i40e_aqc_vsi_properties_data *info,
11076 			      uint8_t enabled_tcmap)
11077 {
11078 	enum i40e_status_code ret;
11079 	int i, total_tc = 0;
11080 	uint16_t qpnum_per_tc, bsf, qp_idx;
11081 	struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
11082 	struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
11083 	uint16_t used_queues;
11084 
11085 	ret = validate_tcmap_parameter(vsi, enabled_tcmap);
11086 	if (ret != I40E_SUCCESS)
11087 		return ret;
11088 
11089 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11090 		if (enabled_tcmap & (1 << i))
11091 			total_tc++;
11092 	}
11093 	if (total_tc == 0)
11094 		total_tc = 1;
11095 	vsi->enabled_tc = enabled_tcmap;
11096 
11097 	/* different VSI has different queues assigned */
11098 	if (vsi->type == I40E_VSI_MAIN)
11099 		used_queues = dev_data->nb_rx_queues -
11100 			pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
11101 	else if (vsi->type == I40E_VSI_VMDQ2)
11102 		used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
11103 	else {
11104 		PMD_INIT_LOG(ERR, "unsupported VSI type.");
11105 		return I40E_ERR_NO_AVAILABLE_VSI;
11106 	}
11107 
11108 	qpnum_per_tc = used_queues / total_tc;
11109 	/* Number of queues per enabled TC */
11110 	if (qpnum_per_tc == 0) {
11111 		PMD_INIT_LOG(ERR, " number of queues is less that tcs.");
11112 		return I40E_ERR_INVALID_QP_ID;
11113 	}
11114 	qpnum_per_tc = RTE_MIN(i40e_align_floor(qpnum_per_tc),
11115 				I40E_MAX_Q_PER_TC);
11116 	bsf = rte_bsf32(qpnum_per_tc);
11117 
11118 	/**
11119 	 * Configure TC and queue mapping parameters, for enabled TC,
11120 	 * allocate qpnum_per_tc queues to this traffic. For disabled TC,
11121 	 * default queue will serve it.
11122 	 */
11123 	qp_idx = 0;
11124 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11125 		if (vsi->enabled_tc & (1 << i)) {
11126 			info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
11127 					I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
11128 				(bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
11129 			qp_idx += qpnum_per_tc;
11130 		} else
11131 			info->tc_mapping[i] = 0;
11132 	}
11133 
11134 	/* Associate queue number with VSI, Keep vsi->nb_qps unchanged */
11135 	if (vsi->type == I40E_VSI_SRIOV) {
11136 		info->mapping_flags |=
11137 			rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
11138 		for (i = 0; i < vsi->nb_qps; i++)
11139 			info->queue_mapping[i] =
11140 				rte_cpu_to_le_16(vsi->base_queue + i);
11141 	} else {
11142 		info->mapping_flags |=
11143 			rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
11144 		info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
11145 	}
11146 	info->valid_sections |=
11147 		rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
11148 
11149 	return I40E_SUCCESS;
11150 }
11151 
11152 /*
11153  * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map
11154  * @veb: VEB to be configured
11155  * @tc_map: enabled TC bitmap
11156  *
11157  * Returns 0 on success, negative value on failure
11158  */
11159 static enum i40e_status_code
11160 i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map)
11161 {
11162 	struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw;
11163 	struct i40e_aqc_query_switching_comp_bw_config_resp bw_query;
11164 	struct i40e_aqc_query_switching_comp_ets_config_resp ets_query;
11165 	struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi);
11166 	enum i40e_status_code ret = I40E_SUCCESS;
11167 	int i;
11168 	uint32_t bw_max;
11169 
11170 	/* Check if enabled_tc is same as existing or new TCs */
11171 	if (veb->enabled_tc == tc_map)
11172 		return ret;
11173 
11174 	/* configure tc bandwidth */
11175 	memset(&veb_bw, 0, sizeof(veb_bw));
11176 	veb_bw.tc_valid_bits = tc_map;
11177 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
11178 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11179 		if (tc_map & BIT_ULL(i))
11180 			veb_bw.tc_bw_share_credits[i] = 1;
11181 	}
11182 	ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid,
11183 						   &veb_bw, NULL);
11184 	if (ret) {
11185 		PMD_INIT_LOG(ERR,
11186 			"AQ command Config switch_comp BW allocation per TC failed = %d",
11187 			hw->aq.asq_last_status);
11188 		return ret;
11189 	}
11190 
11191 	memset(&ets_query, 0, sizeof(ets_query));
11192 	ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
11193 						   &ets_query, NULL);
11194 	if (ret != I40E_SUCCESS) {
11195 		PMD_DRV_LOG(ERR,
11196 			"Failed to get switch_comp ETS configuration %u",
11197 			hw->aq.asq_last_status);
11198 		return ret;
11199 	}
11200 	memset(&bw_query, 0, sizeof(bw_query));
11201 	ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
11202 						  &bw_query, NULL);
11203 	if (ret != I40E_SUCCESS) {
11204 		PMD_DRV_LOG(ERR,
11205 			"Failed to get switch_comp bandwidth configuration %u",
11206 			hw->aq.asq_last_status);
11207 		return ret;
11208 	}
11209 
11210 	/* store and print out BW info */
11211 	veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit);
11212 	veb->bw_info.bw_max = ets_query.tc_bw_max;
11213 	PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit);
11214 	PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max);
11215 	bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) |
11216 		    (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) <<
11217 		     I40E_16_BIT_WIDTH);
11218 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11219 		veb->bw_info.bw_ets_share_credits[i] =
11220 				bw_query.tc_bw_share_credits[i];
11221 		veb->bw_info.bw_ets_credits[i] =
11222 				rte_le_to_cpu_16(bw_query.tc_bw_limits[i]);
11223 		/* 4 bits per TC, 4th bit is reserved */
11224 		veb->bw_info.bw_ets_max[i] =
11225 			(uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
11226 				  RTE_LEN2MASK(3, uint8_t));
11227 		PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i,
11228 			    veb->bw_info.bw_ets_share_credits[i]);
11229 		PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i,
11230 			    veb->bw_info.bw_ets_credits[i]);
11231 		PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i,
11232 			    veb->bw_info.bw_ets_max[i]);
11233 	}
11234 
11235 	veb->enabled_tc = tc_map;
11236 
11237 	return ret;
11238 }
11239 
11240 
11241 /*
11242  * i40e_vsi_config_tc - Configure VSI tc setting for given TC map
11243  * @vsi: VSI to be configured
11244  * @tc_map: enabled TC bitmap
11245  *
11246  * Returns 0 on success, negative value on failure
11247  */
11248 static enum i40e_status_code
11249 i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map)
11250 {
11251 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
11252 	struct i40e_vsi_context ctxt;
11253 	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
11254 	enum i40e_status_code ret = I40E_SUCCESS;
11255 	int i;
11256 
11257 	/* Check if enabled_tc is same as existing or new TCs */
11258 	if (vsi->enabled_tc == tc_map)
11259 		return ret;
11260 
11261 	/* configure tc bandwidth */
11262 	memset(&bw_data, 0, sizeof(bw_data));
11263 	bw_data.tc_valid_bits = tc_map;
11264 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
11265 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11266 		if (tc_map & BIT_ULL(i))
11267 			bw_data.tc_bw_credits[i] = 1;
11268 	}
11269 	ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL);
11270 	if (ret) {
11271 		PMD_INIT_LOG(ERR,
11272 			"AQ command Config VSI BW allocation per TC failed = %d",
11273 			hw->aq.asq_last_status);
11274 		goto out;
11275 	}
11276 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
11277 		vsi->info.qs_handle[i] = bw_data.qs_handles[i];
11278 
11279 	/* Update Queue Pairs Mapping for currently enabled UPs */
11280 	ctxt.seid = vsi->seid;
11281 	ctxt.pf_num = hw->pf_id;
11282 	ctxt.vf_num = 0;
11283 	ctxt.uplink_seid = vsi->uplink_seid;
11284 	ctxt.info = vsi->info;
11285 	i40e_get_cap(hw);
11286 	ret = i40e_vsi_update_queue_mapping(vsi, &ctxt.info, tc_map);
11287 	if (ret)
11288 		goto out;
11289 
11290 	/* Update the VSI after updating the VSI queue-mapping information */
11291 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
11292 	if (ret) {
11293 		PMD_INIT_LOG(ERR, "Failed to configure TC queue mapping = %d",
11294 			hw->aq.asq_last_status);
11295 		goto out;
11296 	}
11297 	/* update the local VSI info with updated queue map */
11298 	rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
11299 					sizeof(vsi->info.tc_mapping));
11300 	rte_memcpy(&vsi->info.queue_mapping,
11301 			&ctxt.info.queue_mapping,
11302 		sizeof(vsi->info.queue_mapping));
11303 	vsi->info.mapping_flags = ctxt.info.mapping_flags;
11304 	vsi->info.valid_sections = 0;
11305 
11306 	/* query and update current VSI BW information */
11307 	ret = i40e_vsi_get_bw_config(vsi);
11308 	if (ret) {
11309 		PMD_INIT_LOG(ERR,
11310 			 "Failed updating vsi bw info, err %s aq_err %s",
11311 			 i40e_stat_str(hw, ret),
11312 			 i40e_aq_str(hw, hw->aq.asq_last_status));
11313 		goto out;
11314 	}
11315 
11316 	vsi->enabled_tc = tc_map;
11317 
11318 out:
11319 	return ret;
11320 }
11321 
11322 /*
11323  * i40e_dcb_hw_configure - program the dcb setting to hw
11324  * @pf: pf the configuration is taken on
11325  * @new_cfg: new configuration
11326  * @tc_map: enabled TC bitmap
11327  *
11328  * Returns 0 on success, negative value on failure
11329  */
11330 static enum i40e_status_code
11331 i40e_dcb_hw_configure(struct i40e_pf *pf,
11332 		      struct i40e_dcbx_config *new_cfg,
11333 		      uint8_t tc_map)
11334 {
11335 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
11336 	struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config;
11337 	struct i40e_vsi *main_vsi = pf->main_vsi;
11338 	struct i40e_vsi_list *vsi_list;
11339 	enum i40e_status_code ret;
11340 	int i;
11341 	uint32_t val;
11342 
11343 	/* Use the FW API if FW > v4.4*/
11344 	if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) ||
11345 	      (hw->aq.fw_maj_ver >= 5))) {
11346 		PMD_INIT_LOG(ERR,
11347 			"FW < v4.4, can not use FW LLDP API to configure DCB");
11348 		return I40E_ERR_FIRMWARE_API_VERSION;
11349 	}
11350 
11351 	/* Check if need reconfiguration */
11352 	if (!memcmp(new_cfg, old_cfg, sizeof(struct i40e_dcbx_config))) {
11353 		PMD_INIT_LOG(ERR, "No Change in DCB Config required.");
11354 		return I40E_SUCCESS;
11355 	}
11356 
11357 	/* Copy the new config to the current config */
11358 	*old_cfg = *new_cfg;
11359 	old_cfg->etsrec = old_cfg->etscfg;
11360 	ret = i40e_set_dcb_config(hw);
11361 	if (ret) {
11362 		PMD_INIT_LOG(ERR, "Set DCB Config failed, err %s aq_err %s",
11363 			 i40e_stat_str(hw, ret),
11364 			 i40e_aq_str(hw, hw->aq.asq_last_status));
11365 		return ret;
11366 	}
11367 	/* set receive Arbiter to RR mode and ETS scheme by default */
11368 	for (i = 0; i <= I40E_PRTDCB_RETSTCC_MAX_INDEX; i++) {
11369 		val = I40E_READ_REG(hw, I40E_PRTDCB_RETSTCC(i));
11370 		val &= ~(I40E_PRTDCB_RETSTCC_BWSHARE_MASK     |
11371 			 I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK |
11372 			 I40E_PRTDCB_RETSTCC_ETSTC_SHIFT);
11373 		val |= ((uint32_t)old_cfg->etscfg.tcbwtable[i] <<
11374 			I40E_PRTDCB_RETSTCC_BWSHARE_SHIFT) &
11375 			 I40E_PRTDCB_RETSTCC_BWSHARE_MASK;
11376 		val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) &
11377 			 I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK;
11378 		val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) &
11379 			 I40E_PRTDCB_RETSTCC_ETSTC_MASK;
11380 		I40E_WRITE_REG(hw, I40E_PRTDCB_RETSTCC(i), val);
11381 	}
11382 	/* get local mib to check whether it is configured correctly */
11383 	/* IEEE mode */
11384 	hw->local_dcbx_config.dcbx_mode = I40E_DCBX_MODE_IEEE;
11385 	/* Get Local DCB Config */
11386 	i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0,
11387 				     &hw->local_dcbx_config);
11388 
11389 	/* if Veb is created, need to update TC of it at first */
11390 	if (main_vsi->veb) {
11391 		ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map);
11392 		if (ret)
11393 			PMD_INIT_LOG(WARNING,
11394 				 "Failed configuring TC for VEB seid=%d",
11395 				 main_vsi->veb->seid);
11396 	}
11397 	/* Update each VSI */
11398 	i40e_vsi_config_tc(main_vsi, tc_map);
11399 	if (main_vsi->veb) {
11400 		TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) {
11401 			/* Beside main VSI and VMDQ VSIs, only enable default
11402 			 * TC for other VSIs
11403 			 */
11404 			if (vsi_list->vsi->type == I40E_VSI_VMDQ2)
11405 				ret = i40e_vsi_config_tc(vsi_list->vsi,
11406 							 tc_map);
11407 			else
11408 				ret = i40e_vsi_config_tc(vsi_list->vsi,
11409 							 I40E_DEFAULT_TCMAP);
11410 			if (ret)
11411 				PMD_INIT_LOG(WARNING,
11412 					"Failed configuring TC for VSI seid=%d",
11413 					vsi_list->vsi->seid);
11414 			/* continue */
11415 		}
11416 	}
11417 	return I40E_SUCCESS;
11418 }
11419 
11420 /*
11421  * i40e_dcb_init_configure - initial dcb config
11422  * @dev: device being configured
11423  * @sw_dcb: indicate whether dcb is sw configured or hw offload
11424  *
11425  * Returns 0 on success, negative value on failure
11426  */
11427 int
11428 i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
11429 {
11430 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
11431 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11432 	int i, ret = 0;
11433 
11434 	if ((pf->flags & I40E_FLAG_DCB) == 0) {
11435 		PMD_INIT_LOG(ERR, "HW doesn't support DCB");
11436 		return -ENOTSUP;
11437 	}
11438 
11439 	/* DCB initialization:
11440 	 * Update DCB configuration from the Firmware and configure
11441 	 * LLDP MIB change event.
11442 	 */
11443 	if (sw_dcb == TRUE) {
11444 		if (i40e_need_stop_lldp(dev)) {
11445 			ret = i40e_aq_stop_lldp(hw, TRUE, NULL);
11446 			if (ret != I40E_SUCCESS)
11447 				PMD_INIT_LOG(DEBUG, "Failed to stop lldp");
11448 		}
11449 
11450 		ret = i40e_init_dcb(hw);
11451 		/* If lldp agent is stopped, the return value from
11452 		 * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
11453 		 * adminq status. Otherwise, it should return success.
11454 		 */
11455 		if ((ret == I40E_SUCCESS) || (ret != I40E_SUCCESS &&
11456 		    hw->aq.asq_last_status == I40E_AQ_RC_EPERM)) {
11457 			memset(&hw->local_dcbx_config, 0,
11458 				sizeof(struct i40e_dcbx_config));
11459 			/* set dcb default configuration */
11460 			hw->local_dcbx_config.etscfg.willing = 0;
11461 			hw->local_dcbx_config.etscfg.maxtcs = 0;
11462 			hw->local_dcbx_config.etscfg.tcbwtable[0] = 100;
11463 			hw->local_dcbx_config.etscfg.tsatable[0] =
11464 						I40E_IEEE_TSA_ETS;
11465 			/* all UPs mapping to TC0 */
11466 			for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
11467 				hw->local_dcbx_config.etscfg.prioritytable[i] = 0;
11468 			hw->local_dcbx_config.etsrec =
11469 				hw->local_dcbx_config.etscfg;
11470 			hw->local_dcbx_config.pfc.willing = 0;
11471 			hw->local_dcbx_config.pfc.pfccap =
11472 						I40E_MAX_TRAFFIC_CLASS;
11473 			/* FW needs one App to configure HW */
11474 			hw->local_dcbx_config.numapps = 1;
11475 			hw->local_dcbx_config.app[0].selector =
11476 						I40E_APP_SEL_ETHTYPE;
11477 			hw->local_dcbx_config.app[0].priority = 3;
11478 			hw->local_dcbx_config.app[0].protocolid =
11479 						I40E_APP_PROTOID_FCOE;
11480 			ret = i40e_set_dcb_config(hw);
11481 			if (ret) {
11482 				PMD_INIT_LOG(ERR,
11483 					"default dcb config fails. err = %d, aq_err = %d.",
11484 					ret, hw->aq.asq_last_status);
11485 				return -ENOSYS;
11486 			}
11487 		} else {
11488 			PMD_INIT_LOG(ERR,
11489 				"DCB initialization in FW fails, err = %d, aq_err = %d.",
11490 				ret, hw->aq.asq_last_status);
11491 			return -ENOTSUP;
11492 		}
11493 	} else {
11494 		ret = i40e_aq_start_lldp(hw, NULL);
11495 		if (ret != I40E_SUCCESS)
11496 			PMD_INIT_LOG(DEBUG, "Failed to start lldp");
11497 
11498 		ret = i40e_init_dcb(hw);
11499 		if (!ret) {
11500 			if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
11501 				PMD_INIT_LOG(ERR,
11502 					"HW doesn't support DCBX offload.");
11503 				return -ENOTSUP;
11504 			}
11505 		} else {
11506 			PMD_INIT_LOG(ERR,
11507 				"DCBX configuration failed, err = %d, aq_err = %d.",
11508 				ret, hw->aq.asq_last_status);
11509 			return -ENOTSUP;
11510 		}
11511 	}
11512 	return 0;
11513 }
11514 
11515 /*
11516  * i40e_dcb_setup - setup dcb related config
11517  * @dev: device being configured
11518  *
11519  * Returns 0 on success, negative value on failure
11520  */
11521 static int
11522 i40e_dcb_setup(struct rte_eth_dev *dev)
11523 {
11524 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
11525 	struct i40e_dcbx_config dcb_cfg;
11526 	uint8_t tc_map = 0;
11527 	int ret = 0;
11528 
11529 	if ((pf->flags & I40E_FLAG_DCB) == 0) {
11530 		PMD_INIT_LOG(ERR, "HW doesn't support DCB");
11531 		return -ENOTSUP;
11532 	}
11533 
11534 	if (pf->vf_num != 0)
11535 		PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis.");
11536 
11537 	ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map);
11538 	if (ret) {
11539 		PMD_INIT_LOG(ERR, "invalid dcb config");
11540 		return -EINVAL;
11541 	}
11542 	ret = i40e_dcb_hw_configure(pf, &dcb_cfg, tc_map);
11543 	if (ret) {
11544 		PMD_INIT_LOG(ERR, "dcb sw configure fails");
11545 		return -ENOSYS;
11546 	}
11547 
11548 	return 0;
11549 }
11550 
11551 static int
11552 i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
11553 		      struct rte_eth_dcb_info *dcb_info)
11554 {
11555 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
11556 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11557 	struct i40e_vsi *vsi = pf->main_vsi;
11558 	struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config;
11559 	uint16_t bsf, tc_mapping;
11560 	int i, j = 0;
11561 
11562 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
11563 		dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1);
11564 	else
11565 		dcb_info->nb_tcs = 1;
11566 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
11567 		dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i];
11568 	for (i = 0; i < dcb_info->nb_tcs; i++)
11569 		dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i];
11570 
11571 	/* get queue mapping if vmdq is disabled */
11572 	if (!pf->nb_cfg_vmdq_vsi) {
11573 		for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11574 			if (!(vsi->enabled_tc & (1 << i)))
11575 				continue;
11576 			tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
11577 			dcb_info->tc_queue.tc_rxq[j][i].base =
11578 				(tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
11579 				I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
11580 			dcb_info->tc_queue.tc_txq[j][i].base =
11581 				dcb_info->tc_queue.tc_rxq[j][i].base;
11582 			bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
11583 				I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
11584 			dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
11585 			dcb_info->tc_queue.tc_txq[j][i].nb_queue =
11586 				dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
11587 		}
11588 		return 0;
11589 	}
11590 
11591 	/* get queue mapping if vmdq is enabled */
11592 	do {
11593 		vsi = pf->vmdq[j].vsi;
11594 		for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
11595 			if (!(vsi->enabled_tc & (1 << i)))
11596 				continue;
11597 			tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
11598 			dcb_info->tc_queue.tc_rxq[j][i].base =
11599 				(tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
11600 				I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
11601 			dcb_info->tc_queue.tc_txq[j][i].base =
11602 				dcb_info->tc_queue.tc_rxq[j][i].base;
11603 			bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
11604 				I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
11605 			dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
11606 			dcb_info->tc_queue.tc_txq[j][i].nb_queue =
11607 				dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
11608 		}
11609 		j++;
11610 	} while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL));
11611 	return 0;
11612 }
11613 
11614 static int
11615 i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
11616 {
11617 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
11618 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
11619 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11620 	uint16_t msix_intr;
11621 
11622 	msix_intr = intr_handle->intr_vec[queue_id];
11623 	if (msix_intr == I40E_MISC_VEC_ID)
11624 		I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
11625 			       I40E_PFINT_DYN_CTL0_INTENA_MASK |
11626 			       I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
11627 			       I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
11628 	else
11629 		I40E_WRITE_REG(hw,
11630 			       I40E_PFINT_DYN_CTLN(msix_intr -
11631 						   I40E_RX_VEC_START),
11632 			       I40E_PFINT_DYN_CTLN_INTENA_MASK |
11633 			       I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
11634 			       I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
11635 
11636 	I40E_WRITE_FLUSH(hw);
11637 	rte_intr_enable(&pci_dev->intr_handle);
11638 
11639 	return 0;
11640 }
11641 
11642 static int
11643 i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
11644 {
11645 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
11646 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
11647 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11648 	uint16_t msix_intr;
11649 
11650 	msix_intr = intr_handle->intr_vec[queue_id];
11651 	if (msix_intr == I40E_MISC_VEC_ID)
11652 		I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
11653 			       I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
11654 	else
11655 		I40E_WRITE_REG(hw,
11656 			       I40E_PFINT_DYN_CTLN(msix_intr -
11657 						   I40E_RX_VEC_START),
11658 			       I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
11659 	I40E_WRITE_FLUSH(hw);
11660 
11661 	return 0;
11662 }
11663 
11664 /**
11665  * This function is used to check if the register is valid.
11666  * Below is the valid registers list for X722 only:
11667  * 0x2b800--0x2bb00
11668  * 0x38700--0x38a00
11669  * 0x3d800--0x3db00
11670  * 0x208e00--0x209000
11671  * 0x20be00--0x20c000
11672  * 0x263c00--0x264000
11673  * 0x265c00--0x266000
11674  */
11675 static inline int i40e_valid_regs(enum i40e_mac_type type, uint32_t reg_offset)
11676 {
11677 	if ((type != I40E_MAC_X722) &&
11678 	    ((reg_offset >= 0x2b800 && reg_offset <= 0x2bb00) ||
11679 	     (reg_offset >= 0x38700 && reg_offset <= 0x38a00) ||
11680 	     (reg_offset >= 0x3d800 && reg_offset <= 0x3db00) ||
11681 	     (reg_offset >= 0x208e00 && reg_offset <= 0x209000) ||
11682 	     (reg_offset >= 0x20be00 && reg_offset <= 0x20c000) ||
11683 	     (reg_offset >= 0x263c00 && reg_offset <= 0x264000) ||
11684 	     (reg_offset >= 0x265c00 && reg_offset <= 0x266000)))
11685 		return 0;
11686 	else
11687 		return 1;
11688 }
11689 
11690 static int i40e_get_regs(struct rte_eth_dev *dev,
11691 			 struct rte_dev_reg_info *regs)
11692 {
11693 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11694 	uint32_t *ptr_data = regs->data;
11695 	uint32_t reg_idx, arr_idx, arr_idx2, reg_offset;
11696 	const struct i40e_reg_info *reg_info;
11697 
11698 	if (ptr_data == NULL) {
11699 		regs->length = I40E_GLGEN_STAT_CLEAR + 4;
11700 		regs->width = sizeof(uint32_t);
11701 		return 0;
11702 	}
11703 
11704 	/* The first few registers have to be read using AQ operations */
11705 	reg_idx = 0;
11706 	while (i40e_regs_adminq[reg_idx].name) {
11707 		reg_info = &i40e_regs_adminq[reg_idx++];
11708 		for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
11709 			for (arr_idx2 = 0;
11710 					arr_idx2 <= reg_info->count2;
11711 					arr_idx2++) {
11712 				reg_offset = arr_idx * reg_info->stride1 +
11713 					arr_idx2 * reg_info->stride2;
11714 				reg_offset += reg_info->base_addr;
11715 				ptr_data[reg_offset >> 2] =
11716 					i40e_read_rx_ctl(hw, reg_offset);
11717 			}
11718 	}
11719 
11720 	/* The remaining registers can be read using primitives */
11721 	reg_idx = 0;
11722 	while (i40e_regs_others[reg_idx].name) {
11723 		reg_info = &i40e_regs_others[reg_idx++];
11724 		for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
11725 			for (arr_idx2 = 0;
11726 					arr_idx2 <= reg_info->count2;
11727 					arr_idx2++) {
11728 				reg_offset = arr_idx * reg_info->stride1 +
11729 					arr_idx2 * reg_info->stride2;
11730 				reg_offset += reg_info->base_addr;
11731 				if (!i40e_valid_regs(hw->mac.type, reg_offset))
11732 					ptr_data[reg_offset >> 2] = 0;
11733 				else
11734 					ptr_data[reg_offset >> 2] =
11735 						I40E_READ_REG(hw, reg_offset);
11736 			}
11737 	}
11738 
11739 	return 0;
11740 }
11741 
11742 static int i40e_get_eeprom_length(struct rte_eth_dev *dev)
11743 {
11744 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11745 
11746 	/* Convert word count to byte count */
11747 	return hw->nvm.sr_size << 1;
11748 }
11749 
11750 static int i40e_get_eeprom(struct rte_eth_dev *dev,
11751 			   struct rte_dev_eeprom_info *eeprom)
11752 {
11753 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11754 	uint16_t *data = eeprom->data;
11755 	uint16_t offset, length, cnt_words;
11756 	int ret_code;
11757 
11758 	offset = eeprom->offset >> 1;
11759 	length = eeprom->length >> 1;
11760 	cnt_words = length;
11761 
11762 	if (offset > hw->nvm.sr_size ||
11763 		offset + length > hw->nvm.sr_size) {
11764 		PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range.");
11765 		return -EINVAL;
11766 	}
11767 
11768 	eeprom->magic = hw->vendor_id | (hw->device_id << 16);
11769 
11770 	ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data);
11771 	if (ret_code != I40E_SUCCESS || cnt_words != length) {
11772 		PMD_DRV_LOG(ERR, "EEPROM read failed.");
11773 		return -EIO;
11774 	}
11775 
11776 	return 0;
11777 }
11778 
11779 static int i40e_get_module_info(struct rte_eth_dev *dev,
11780 				struct rte_eth_dev_module_info *modinfo)
11781 {
11782 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11783 	uint32_t sff8472_comp = 0;
11784 	uint32_t sff8472_swap = 0;
11785 	uint32_t sff8636_rev = 0;
11786 	i40e_status status;
11787 	uint32_t type = 0;
11788 
11789 	/* Check if firmware supports reading module EEPROM. */
11790 	if (!(hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE)) {
11791 		PMD_DRV_LOG(ERR,
11792 			    "Module EEPROM memory read not supported. "
11793 			    "Please update the NVM image.\n");
11794 		return -EINVAL;
11795 	}
11796 
11797 	status = i40e_update_link_info(hw);
11798 	if (status)
11799 		return -EIO;
11800 
11801 	if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_EMPTY) {
11802 		PMD_DRV_LOG(ERR,
11803 			    "Cannot read module EEPROM memory. "
11804 			    "No module connected.\n");
11805 		return -EINVAL;
11806 	}
11807 
11808 	type = hw->phy.link_info.module_type[0];
11809 
11810 	switch (type) {
11811 	case I40E_MODULE_TYPE_SFP:
11812 		status = i40e_aq_get_phy_register(hw,
11813 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
11814 				I40E_I2C_EEPROM_DEV_ADDR, 1,
11815 				I40E_MODULE_SFF_8472_COMP,
11816 				&sff8472_comp, NULL);
11817 		if (status)
11818 			return -EIO;
11819 
11820 		status = i40e_aq_get_phy_register(hw,
11821 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
11822 				I40E_I2C_EEPROM_DEV_ADDR, 1,
11823 				I40E_MODULE_SFF_8472_SWAP,
11824 				&sff8472_swap, NULL);
11825 		if (status)
11826 			return -EIO;
11827 
11828 		/* Check if the module requires address swap to access
11829 		 * the other EEPROM memory page.
11830 		 */
11831 		if (sff8472_swap & I40E_MODULE_SFF_ADDR_MODE) {
11832 			PMD_DRV_LOG(WARNING,
11833 				    "Module address swap to access "
11834 				    "page 0xA2 is not supported.\n");
11835 			modinfo->type = RTE_ETH_MODULE_SFF_8079;
11836 			modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN;
11837 		} else if (sff8472_comp == 0x00) {
11838 			/* Module is not SFF-8472 compliant */
11839 			modinfo->type = RTE_ETH_MODULE_SFF_8079;
11840 			modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8079_LEN;
11841 		} else {
11842 			modinfo->type = RTE_ETH_MODULE_SFF_8472;
11843 			modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN;
11844 		}
11845 		break;
11846 	case I40E_MODULE_TYPE_QSFP_PLUS:
11847 		/* Read from memory page 0. */
11848 		status = i40e_aq_get_phy_register(hw,
11849 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
11850 				0, 1,
11851 				I40E_MODULE_REVISION_ADDR,
11852 				&sff8636_rev, NULL);
11853 		if (status)
11854 			return -EIO;
11855 		/* Determine revision compliance byte */
11856 		if (sff8636_rev > 0x02) {
11857 			/* Module is SFF-8636 compliant */
11858 			modinfo->type = RTE_ETH_MODULE_SFF_8636;
11859 			modinfo->eeprom_len = I40E_MODULE_QSFP_MAX_LEN;
11860 		} else {
11861 			modinfo->type = RTE_ETH_MODULE_SFF_8436;
11862 			modinfo->eeprom_len = I40E_MODULE_QSFP_MAX_LEN;
11863 		}
11864 		break;
11865 	case I40E_MODULE_TYPE_QSFP28:
11866 		modinfo->type = RTE_ETH_MODULE_SFF_8636;
11867 		modinfo->eeprom_len = I40E_MODULE_QSFP_MAX_LEN;
11868 		break;
11869 	default:
11870 		PMD_DRV_LOG(ERR, "Module type unrecognized\n");
11871 		return -EINVAL;
11872 	}
11873 	return 0;
11874 }
11875 
11876 static int i40e_get_module_eeprom(struct rte_eth_dev *dev,
11877 				  struct rte_dev_eeprom_info *info)
11878 {
11879 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11880 	bool is_sfp = false;
11881 	i40e_status status;
11882 	uint8_t *data;
11883 	uint32_t value = 0;
11884 	uint32_t i;
11885 
11886 	if (!info || !info->length || !info->data)
11887 		return -EINVAL;
11888 
11889 	if (hw->phy.link_info.module_type[0] == I40E_MODULE_TYPE_SFP)
11890 		is_sfp = true;
11891 
11892 	data = info->data;
11893 	for (i = 0; i < info->length; i++) {
11894 		u32 offset = i + info->offset;
11895 		u32 addr = is_sfp ? I40E_I2C_EEPROM_DEV_ADDR : 0;
11896 
11897 		/* Check if we need to access the other memory page */
11898 		if (is_sfp) {
11899 			if (offset >= RTE_ETH_MODULE_SFF_8079_LEN) {
11900 				offset -= RTE_ETH_MODULE_SFF_8079_LEN;
11901 				addr = I40E_I2C_EEPROM_DEV_ADDR2;
11902 			}
11903 		} else {
11904 			while (offset >= RTE_ETH_MODULE_SFF_8436_LEN) {
11905 				/* Compute memory page number and offset. */
11906 				offset -= RTE_ETH_MODULE_SFF_8436_LEN / 2;
11907 				addr++;
11908 			}
11909 		}
11910 		status = i40e_aq_get_phy_register(hw,
11911 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
11912 				addr, offset, 1, &value, NULL);
11913 		if (status)
11914 			return -EIO;
11915 		data[i] = (uint8_t)value;
11916 	}
11917 	return 0;
11918 }
11919 
11920 static int i40e_set_default_mac_addr(struct rte_eth_dev *dev,
11921 				     struct ether_addr *mac_addr)
11922 {
11923 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
11924 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
11925 	struct i40e_vsi *vsi = pf->main_vsi;
11926 	struct i40e_mac_filter_info mac_filter;
11927 	struct i40e_mac_filter *f;
11928 	int ret;
11929 
11930 	if (!is_valid_assigned_ether_addr(mac_addr)) {
11931 		PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
11932 		return -EINVAL;
11933 	}
11934 
11935 	TAILQ_FOREACH(f, &vsi->mac_list, next) {
11936 		if (is_same_ether_addr(&pf->dev_addr, &f->mac_info.mac_addr))
11937 			break;
11938 	}
11939 
11940 	if (f == NULL) {
11941 		PMD_DRV_LOG(ERR, "Failed to find filter for default mac");
11942 		return -EIO;
11943 	}
11944 
11945 	mac_filter = f->mac_info;
11946 	ret = i40e_vsi_delete_mac(vsi, &mac_filter.mac_addr);
11947 	if (ret != I40E_SUCCESS) {
11948 		PMD_DRV_LOG(ERR, "Failed to delete mac filter");
11949 		return -EIO;
11950 	}
11951 	memcpy(&mac_filter.mac_addr, mac_addr, ETH_ADDR_LEN);
11952 	ret = i40e_vsi_add_mac(vsi, &mac_filter);
11953 	if (ret != I40E_SUCCESS) {
11954 		PMD_DRV_LOG(ERR, "Failed to add mac filter");
11955 		return -EIO;
11956 	}
11957 	memcpy(&pf->dev_addr, mac_addr, ETH_ADDR_LEN);
11958 
11959 	ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
11960 					mac_addr->addr_bytes, NULL);
11961 	if (ret != I40E_SUCCESS) {
11962 		PMD_DRV_LOG(ERR, "Failed to change mac");
11963 		return -EIO;
11964 	}
11965 
11966 	return 0;
11967 }
11968 
11969 static int
11970 i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
11971 {
11972 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
11973 	struct rte_eth_dev_data *dev_data = pf->dev_data;
11974 	uint32_t frame_size = mtu + I40E_ETH_OVERHEAD;
11975 	int ret = 0;
11976 
11977 	/* check if mtu is within the allowed range */
11978 	if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX))
11979 		return -EINVAL;
11980 
11981 	/* mtu setting is forbidden if port is start */
11982 	if (dev_data->dev_started) {
11983 		PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
11984 			    dev_data->port_id);
11985 		return -EBUSY;
11986 	}
11987 
11988 	if (frame_size > ETHER_MAX_LEN)
11989 		dev_data->dev_conf.rxmode.offloads |=
11990 			DEV_RX_OFFLOAD_JUMBO_FRAME;
11991 	else
11992 		dev_data->dev_conf.rxmode.offloads &=
11993 			~DEV_RX_OFFLOAD_JUMBO_FRAME;
11994 
11995 	dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
11996 
11997 	return ret;
11998 }
11999 
12000 /* Restore ethertype filter */
12001 static void
12002 i40e_ethertype_filter_restore(struct i40e_pf *pf)
12003 {
12004 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
12005 	struct i40e_ethertype_filter_list
12006 		*ethertype_list = &pf->ethertype.ethertype_list;
12007 	struct i40e_ethertype_filter *f;
12008 	struct i40e_control_filter_stats stats;
12009 	uint16_t flags;
12010 
12011 	TAILQ_FOREACH(f, ethertype_list, rules) {
12012 		flags = 0;
12013 		if (!(f->flags & RTE_ETHTYPE_FLAGS_MAC))
12014 			flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
12015 		if (f->flags & RTE_ETHTYPE_FLAGS_DROP)
12016 			flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
12017 		flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
12018 
12019 		memset(&stats, 0, sizeof(stats));
12020 		i40e_aq_add_rem_control_packet_filter(hw,
12021 					    f->input.mac_addr.addr_bytes,
12022 					    f->input.ether_type,
12023 					    flags, pf->main_vsi->seid,
12024 					    f->queue, 1, &stats, NULL);
12025 	}
12026 	PMD_DRV_LOG(INFO, "Ethertype filter:"
12027 		    " mac_etype_used = %u, etype_used = %u,"
12028 		    " mac_etype_free = %u, etype_free = %u",
12029 		    stats.mac_etype_used, stats.etype_used,
12030 		    stats.mac_etype_free, stats.etype_free);
12031 }
12032 
12033 /* Restore tunnel filter */
12034 static void
12035 i40e_tunnel_filter_restore(struct i40e_pf *pf)
12036 {
12037 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
12038 	struct i40e_vsi *vsi;
12039 	struct i40e_pf_vf *vf;
12040 	struct i40e_tunnel_filter_list
12041 		*tunnel_list = &pf->tunnel.tunnel_list;
12042 	struct i40e_tunnel_filter *f;
12043 	struct i40e_aqc_cloud_filters_element_bb cld_filter;
12044 	bool big_buffer = 0;
12045 
12046 	TAILQ_FOREACH(f, tunnel_list, rules) {
12047 		if (!f->is_to_vf)
12048 			vsi = pf->main_vsi;
12049 		else {
12050 			vf = &pf->vfs[f->vf_id];
12051 			vsi = vf->vsi;
12052 		}
12053 		memset(&cld_filter, 0, sizeof(cld_filter));
12054 		ether_addr_copy((struct ether_addr *)&f->input.outer_mac,
12055 			(struct ether_addr *)&cld_filter.element.outer_mac);
12056 		ether_addr_copy((struct ether_addr *)&f->input.inner_mac,
12057 			(struct ether_addr *)&cld_filter.element.inner_mac);
12058 		cld_filter.element.inner_vlan = f->input.inner_vlan;
12059 		cld_filter.element.flags = f->input.flags;
12060 		cld_filter.element.tenant_id = f->input.tenant_id;
12061 		cld_filter.element.queue_number = f->queue;
12062 		rte_memcpy(cld_filter.general_fields,
12063 			   f->input.general_fields,
12064 			   sizeof(f->input.general_fields));
12065 
12066 		if (((f->input.flags &
12067 		     I40E_AQC_ADD_CLOUD_FILTER_0X11) ==
12068 		     I40E_AQC_ADD_CLOUD_FILTER_0X11) ||
12069 		    ((f->input.flags &
12070 		     I40E_AQC_ADD_CLOUD_FILTER_0X12) ==
12071 		     I40E_AQC_ADD_CLOUD_FILTER_0X12) ||
12072 		    ((f->input.flags &
12073 		     I40E_AQC_ADD_CLOUD_FILTER_0X10) ==
12074 		     I40E_AQC_ADD_CLOUD_FILTER_0X10))
12075 			big_buffer = 1;
12076 
12077 		if (big_buffer)
12078 			i40e_aq_add_cloud_filters_bb(hw,
12079 					vsi->seid, &cld_filter, 1);
12080 		else
12081 			i40e_aq_add_cloud_filters(hw, vsi->seid,
12082 						  &cld_filter.element, 1);
12083 	}
12084 }
12085 
12086 /* Restore rss filter */
12087 static inline void
12088 i40e_rss_filter_restore(struct i40e_pf *pf)
12089 {
12090 	struct i40e_rte_flow_rss_conf *conf =
12091 					&pf->rss_info;
12092 	if (conf->conf.queue_num)
12093 		i40e_config_rss_filter(pf, conf, TRUE);
12094 }
12095 
12096 static void
12097 i40e_filter_restore(struct i40e_pf *pf)
12098 {
12099 	i40e_ethertype_filter_restore(pf);
12100 	i40e_tunnel_filter_restore(pf);
12101 	i40e_fdir_filter_restore(pf);
12102 	i40e_rss_filter_restore(pf);
12103 }
12104 
12105 static bool
12106 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv)
12107 {
12108 	if (strcmp(dev->device->driver->name, drv->driver.name))
12109 		return false;
12110 
12111 	return true;
12112 }
12113 
12114 bool
12115 is_i40e_supported(struct rte_eth_dev *dev)
12116 {
12117 	return is_device_supported(dev, &rte_i40e_pmd);
12118 }
12119 
12120 struct i40e_customized_pctype*
12121 i40e_find_customized_pctype(struct i40e_pf *pf, uint8_t index)
12122 {
12123 	int i;
12124 
12125 	for (i = 0; i < I40E_CUSTOMIZED_MAX; i++) {
12126 		if (pf->customized_pctype[i].index == index)
12127 			return &pf->customized_pctype[i];
12128 	}
12129 	return NULL;
12130 }
12131 
12132 static int
12133 i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg,
12134 			      uint32_t pkg_size, uint32_t proto_num,
12135 			      struct rte_pmd_i40e_proto_info *proto,
12136 			      enum rte_pmd_i40e_package_op op)
12137 {
12138 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
12139 	uint32_t pctype_num;
12140 	struct rte_pmd_i40e_ptype_info *pctype;
12141 	uint32_t buff_size;
12142 	struct i40e_customized_pctype *new_pctype = NULL;
12143 	uint8_t proto_id;
12144 	uint8_t pctype_value;
12145 	char name[64];
12146 	uint32_t i, j, n;
12147 	int ret;
12148 
12149 	if (op != RTE_PMD_I40E_PKG_OP_WR_ADD &&
12150 	    op != RTE_PMD_I40E_PKG_OP_WR_DEL) {
12151 		PMD_DRV_LOG(ERR, "Unsupported operation.");
12152 		return -1;
12153 	}
12154 
12155 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12156 				(uint8_t *)&pctype_num, sizeof(pctype_num),
12157 				RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM);
12158 	if (ret) {
12159 		PMD_DRV_LOG(ERR, "Failed to get pctype number");
12160 		return -1;
12161 	}
12162 	if (!pctype_num) {
12163 		PMD_DRV_LOG(INFO, "No new pctype added");
12164 		return -1;
12165 	}
12166 
12167 	buff_size = pctype_num * sizeof(struct rte_pmd_i40e_proto_info);
12168 	pctype = rte_zmalloc("new_pctype", buff_size, 0);
12169 	if (!pctype) {
12170 		PMD_DRV_LOG(ERR, "Failed to allocate memory");
12171 		return -1;
12172 	}
12173 	/* get information about new pctype list */
12174 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12175 					(uint8_t *)pctype, buff_size,
12176 					RTE_PMD_I40E_PKG_INFO_PCTYPE_LIST);
12177 	if (ret) {
12178 		PMD_DRV_LOG(ERR, "Failed to get pctype list");
12179 		rte_free(pctype);
12180 		return -1;
12181 	}
12182 
12183 	/* Update customized pctype. */
12184 	for (i = 0; i < pctype_num; i++) {
12185 		pctype_value = pctype[i].ptype_id;
12186 		memset(name, 0, sizeof(name));
12187 		for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) {
12188 			proto_id = pctype[i].protocols[j];
12189 			if (proto_id == RTE_PMD_I40E_PROTO_UNUSED)
12190 				continue;
12191 			for (n = 0; n < proto_num; n++) {
12192 				if (proto[n].proto_id != proto_id)
12193 					continue;
12194 				strlcat(name, proto[n].name, sizeof(name));
12195 				strlcat(name, "_", sizeof(name));
12196 				break;
12197 			}
12198 		}
12199 		name[strlen(name) - 1] = '\0';
12200 		if (!strcmp(name, "GTPC"))
12201 			new_pctype =
12202 				i40e_find_customized_pctype(pf,
12203 						      I40E_CUSTOMIZED_GTPC);
12204 		else if (!strcmp(name, "GTPU_IPV4"))
12205 			new_pctype =
12206 				i40e_find_customized_pctype(pf,
12207 						   I40E_CUSTOMIZED_GTPU_IPV4);
12208 		else if (!strcmp(name, "GTPU_IPV6"))
12209 			new_pctype =
12210 				i40e_find_customized_pctype(pf,
12211 						   I40E_CUSTOMIZED_GTPU_IPV6);
12212 		else if (!strcmp(name, "GTPU"))
12213 			new_pctype =
12214 				i40e_find_customized_pctype(pf,
12215 						      I40E_CUSTOMIZED_GTPU);
12216 		if (new_pctype) {
12217 			if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) {
12218 				new_pctype->pctype = pctype_value;
12219 				new_pctype->valid = true;
12220 			} else {
12221 				new_pctype->pctype = I40E_FILTER_PCTYPE_INVALID;
12222 				new_pctype->valid = false;
12223 			}
12224 		}
12225 	}
12226 
12227 	rte_free(pctype);
12228 	return 0;
12229 }
12230 
12231 static int
12232 i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg,
12233 			     uint32_t pkg_size, uint32_t proto_num,
12234 			     struct rte_pmd_i40e_proto_info *proto,
12235 			     enum rte_pmd_i40e_package_op op)
12236 {
12237 	struct rte_pmd_i40e_ptype_mapping *ptype_mapping;
12238 	uint16_t port_id = dev->data->port_id;
12239 	uint32_t ptype_num;
12240 	struct rte_pmd_i40e_ptype_info *ptype;
12241 	uint32_t buff_size;
12242 	uint8_t proto_id;
12243 	char name[RTE_PMD_I40E_DDP_NAME_SIZE];
12244 	uint32_t i, j, n;
12245 	bool in_tunnel;
12246 	int ret;
12247 
12248 	if (op != RTE_PMD_I40E_PKG_OP_WR_ADD &&
12249 	    op != RTE_PMD_I40E_PKG_OP_WR_DEL) {
12250 		PMD_DRV_LOG(ERR, "Unsupported operation.");
12251 		return -1;
12252 	}
12253 
12254 	if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) {
12255 		rte_pmd_i40e_ptype_mapping_reset(port_id);
12256 		return 0;
12257 	}
12258 
12259 	/* get information about new ptype num */
12260 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12261 				(uint8_t *)&ptype_num, sizeof(ptype_num),
12262 				RTE_PMD_I40E_PKG_INFO_PTYPE_NUM);
12263 	if (ret) {
12264 		PMD_DRV_LOG(ERR, "Failed to get ptype number");
12265 		return ret;
12266 	}
12267 	if (!ptype_num) {
12268 		PMD_DRV_LOG(INFO, "No new ptype added");
12269 		return -1;
12270 	}
12271 
12272 	buff_size = ptype_num * sizeof(struct rte_pmd_i40e_ptype_info);
12273 	ptype = rte_zmalloc("new_ptype", buff_size, 0);
12274 	if (!ptype) {
12275 		PMD_DRV_LOG(ERR, "Failed to allocate memory");
12276 		return -1;
12277 	}
12278 
12279 	/* get information about new ptype list */
12280 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12281 					(uint8_t *)ptype, buff_size,
12282 					RTE_PMD_I40E_PKG_INFO_PTYPE_LIST);
12283 	if (ret) {
12284 		PMD_DRV_LOG(ERR, "Failed to get ptype list");
12285 		rte_free(ptype);
12286 		return ret;
12287 	}
12288 
12289 	buff_size = ptype_num * sizeof(struct rte_pmd_i40e_ptype_mapping);
12290 	ptype_mapping = rte_zmalloc("ptype_mapping", buff_size, 0);
12291 	if (!ptype_mapping) {
12292 		PMD_DRV_LOG(ERR, "Failed to allocate memory");
12293 		rte_free(ptype);
12294 		return -1;
12295 	}
12296 
12297 	/* Update ptype mapping table. */
12298 	for (i = 0; i < ptype_num; i++) {
12299 		ptype_mapping[i].hw_ptype = ptype[i].ptype_id;
12300 		ptype_mapping[i].sw_ptype = 0;
12301 		in_tunnel = false;
12302 		for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) {
12303 			proto_id = ptype[i].protocols[j];
12304 			if (proto_id == RTE_PMD_I40E_PROTO_UNUSED)
12305 				continue;
12306 			for (n = 0; n < proto_num; n++) {
12307 				if (proto[n].proto_id != proto_id)
12308 					continue;
12309 				memset(name, 0, sizeof(name));
12310 				strcpy(name, proto[n].name);
12311 				if (!strncasecmp(name, "PPPOE", 5))
12312 					ptype_mapping[i].sw_ptype |=
12313 						RTE_PTYPE_L2_ETHER_PPPOE;
12314 				else if (!strncasecmp(name, "IPV4FRAG", 8) &&
12315 					 !in_tunnel) {
12316 					ptype_mapping[i].sw_ptype |=
12317 						RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
12318 					ptype_mapping[i].sw_ptype |=
12319 						RTE_PTYPE_L4_FRAG;
12320 				} else if (!strncasecmp(name, "IPV4FRAG", 8) &&
12321 					   in_tunnel) {
12322 					ptype_mapping[i].sw_ptype |=
12323 					    RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN;
12324 					ptype_mapping[i].sw_ptype |=
12325 						RTE_PTYPE_INNER_L4_FRAG;
12326 				} else if (!strncasecmp(name, "OIPV4", 5)) {
12327 					ptype_mapping[i].sw_ptype |=
12328 						RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
12329 					in_tunnel = true;
12330 				} else if (!strncasecmp(name, "IPV4", 4) &&
12331 					   !in_tunnel)
12332 					ptype_mapping[i].sw_ptype |=
12333 						RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
12334 				else if (!strncasecmp(name, "IPV4", 4) &&
12335 					 in_tunnel)
12336 					ptype_mapping[i].sw_ptype |=
12337 					    RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN;
12338 				else if (!strncasecmp(name, "IPV6FRAG", 8) &&
12339 					 !in_tunnel) {
12340 					ptype_mapping[i].sw_ptype |=
12341 						RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
12342 					ptype_mapping[i].sw_ptype |=
12343 						RTE_PTYPE_L4_FRAG;
12344 				} else if (!strncasecmp(name, "IPV6FRAG", 8) &&
12345 					   in_tunnel) {
12346 					ptype_mapping[i].sw_ptype |=
12347 					    RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN;
12348 					ptype_mapping[i].sw_ptype |=
12349 						RTE_PTYPE_INNER_L4_FRAG;
12350 				} else if (!strncasecmp(name, "OIPV6", 5)) {
12351 					ptype_mapping[i].sw_ptype |=
12352 						RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
12353 					in_tunnel = true;
12354 				} else if (!strncasecmp(name, "IPV6", 4) &&
12355 					   !in_tunnel)
12356 					ptype_mapping[i].sw_ptype |=
12357 						RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
12358 				else if (!strncasecmp(name, "IPV6", 4) &&
12359 					 in_tunnel)
12360 					ptype_mapping[i].sw_ptype |=
12361 					    RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN;
12362 				else if (!strncasecmp(name, "UDP", 3) &&
12363 					 !in_tunnel)
12364 					ptype_mapping[i].sw_ptype |=
12365 						RTE_PTYPE_L4_UDP;
12366 				else if (!strncasecmp(name, "UDP", 3) &&
12367 					 in_tunnel)
12368 					ptype_mapping[i].sw_ptype |=
12369 						RTE_PTYPE_INNER_L4_UDP;
12370 				else if (!strncasecmp(name, "TCP", 3) &&
12371 					 !in_tunnel)
12372 					ptype_mapping[i].sw_ptype |=
12373 						RTE_PTYPE_L4_TCP;
12374 				else if (!strncasecmp(name, "TCP", 3) &&
12375 					 in_tunnel)
12376 					ptype_mapping[i].sw_ptype |=
12377 						RTE_PTYPE_INNER_L4_TCP;
12378 				else if (!strncasecmp(name, "SCTP", 4) &&
12379 					 !in_tunnel)
12380 					ptype_mapping[i].sw_ptype |=
12381 						RTE_PTYPE_L4_SCTP;
12382 				else if (!strncasecmp(name, "SCTP", 4) &&
12383 					 in_tunnel)
12384 					ptype_mapping[i].sw_ptype |=
12385 						RTE_PTYPE_INNER_L4_SCTP;
12386 				else if ((!strncasecmp(name, "ICMP", 4) ||
12387 					  !strncasecmp(name, "ICMPV6", 6)) &&
12388 					 !in_tunnel)
12389 					ptype_mapping[i].sw_ptype |=
12390 						RTE_PTYPE_L4_ICMP;
12391 				else if ((!strncasecmp(name, "ICMP", 4) ||
12392 					  !strncasecmp(name, "ICMPV6", 6)) &&
12393 					 in_tunnel)
12394 					ptype_mapping[i].sw_ptype |=
12395 						RTE_PTYPE_INNER_L4_ICMP;
12396 				else if (!strncasecmp(name, "GTPC", 4)) {
12397 					ptype_mapping[i].sw_ptype |=
12398 						RTE_PTYPE_TUNNEL_GTPC;
12399 					in_tunnel = true;
12400 				} else if (!strncasecmp(name, "GTPU", 4)) {
12401 					ptype_mapping[i].sw_ptype |=
12402 						RTE_PTYPE_TUNNEL_GTPU;
12403 					in_tunnel = true;
12404 				} else if (!strncasecmp(name, "GRENAT", 6)) {
12405 					ptype_mapping[i].sw_ptype |=
12406 						RTE_PTYPE_TUNNEL_GRENAT;
12407 					in_tunnel = true;
12408 				} else if (!strncasecmp(name, "L2TPV2CTL", 9) ||
12409 					   !strncasecmp(name, "L2TPV2", 6)) {
12410 					ptype_mapping[i].sw_ptype |=
12411 						RTE_PTYPE_TUNNEL_L2TP;
12412 					in_tunnel = true;
12413 				}
12414 
12415 				break;
12416 			}
12417 		}
12418 	}
12419 
12420 	ret = rte_pmd_i40e_ptype_mapping_update(port_id, ptype_mapping,
12421 						ptype_num, 0);
12422 	if (ret)
12423 		PMD_DRV_LOG(ERR, "Failed to update mapping table.");
12424 
12425 	rte_free(ptype_mapping);
12426 	rte_free(ptype);
12427 	return ret;
12428 }
12429 
12430 void
12431 i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg,
12432 			    uint32_t pkg_size, enum rte_pmd_i40e_package_op op)
12433 {
12434 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
12435 	uint32_t proto_num;
12436 	struct rte_pmd_i40e_proto_info *proto;
12437 	uint32_t buff_size;
12438 	uint32_t i;
12439 	int ret;
12440 
12441 	if (op != RTE_PMD_I40E_PKG_OP_WR_ADD &&
12442 	    op != RTE_PMD_I40E_PKG_OP_WR_DEL) {
12443 		PMD_DRV_LOG(ERR, "Unsupported operation.");
12444 		return;
12445 	}
12446 
12447 	/* get information about protocol number */
12448 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12449 				       (uint8_t *)&proto_num, sizeof(proto_num),
12450 				       RTE_PMD_I40E_PKG_INFO_PROTOCOL_NUM);
12451 	if (ret) {
12452 		PMD_DRV_LOG(ERR, "Failed to get protocol number");
12453 		return;
12454 	}
12455 	if (!proto_num) {
12456 		PMD_DRV_LOG(INFO, "No new protocol added");
12457 		return;
12458 	}
12459 
12460 	buff_size = proto_num * sizeof(struct rte_pmd_i40e_proto_info);
12461 	proto = rte_zmalloc("new_proto", buff_size, 0);
12462 	if (!proto) {
12463 		PMD_DRV_LOG(ERR, "Failed to allocate memory");
12464 		return;
12465 	}
12466 
12467 	/* get information about protocol list */
12468 	ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size,
12469 					(uint8_t *)proto, buff_size,
12470 					RTE_PMD_I40E_PKG_INFO_PROTOCOL_LIST);
12471 	if (ret) {
12472 		PMD_DRV_LOG(ERR, "Failed to get protocol list");
12473 		rte_free(proto);
12474 		return;
12475 	}
12476 
12477 	/* Check if GTP is supported. */
12478 	for (i = 0; i < proto_num; i++) {
12479 		if (!strncmp(proto[i].name, "GTP", 3)) {
12480 			if (op == RTE_PMD_I40E_PKG_OP_WR_ADD)
12481 				pf->gtp_support = true;
12482 			else
12483 				pf->gtp_support = false;
12484 			break;
12485 		}
12486 	}
12487 
12488 	/* Update customized pctype info */
12489 	ret = i40e_update_customized_pctype(dev, pkg, pkg_size,
12490 					    proto_num, proto, op);
12491 	if (ret)
12492 		PMD_DRV_LOG(INFO, "No pctype is updated.");
12493 
12494 	/* Update customized ptype info */
12495 	ret = i40e_update_customized_ptype(dev, pkg, pkg_size,
12496 					   proto_num, proto, op);
12497 	if (ret)
12498 		PMD_DRV_LOG(INFO, "No ptype is updated.");
12499 
12500 	rte_free(proto);
12501 }
12502 
12503 /* Create a QinQ cloud filter
12504  *
12505  * The Fortville NIC has limited resources for tunnel filters,
12506  * so we can only reuse existing filters.
12507  *
12508  * In step 1 we define which Field Vector fields can be used for
12509  * filter types.
12510  * As we do not have the inner tag defined as a field,
12511  * we have to define it first, by reusing one of L1 entries.
12512  *
12513  * In step 2 we are replacing one of existing filter types with
12514  * a new one for QinQ.
12515  * As we reusing L1 and replacing L2, some of the default filter
12516  * types will disappear,which depends on L1 and L2 entries we reuse.
12517  *
12518  * Step 1: Create L1 filter of outer vlan (12b) + inner vlan (12b)
12519  *
12520  * 1.	Create L1 filter of outer vlan (12b) which will be in use
12521  *		later when we define the cloud filter.
12522  *	a.	Valid_flags.replace_cloud = 0
12523  *	b.	Old_filter = 10 (Stag_Inner_Vlan)
12524  *	c.	New_filter = 0x10
12525  *	d.	TR bit = 0xff (optional, not used here)
12526  *	e.	Buffer – 2 entries:
12527  *		i.	Byte 0 = 8 (outer vlan FV index).
12528  *			Byte 1 = 0 (rsv)
12529  *			Byte 2-3 = 0x0fff
12530  *		ii.	Byte 0 = 37 (inner vlan FV index).
12531  *			Byte 1 =0 (rsv)
12532  *			Byte 2-3 = 0x0fff
12533  *
12534  * Step 2:
12535  * 2.	Create cloud filter using two L1 filters entries: stag and
12536  *		new filter(outer vlan+ inner vlan)
12537  *	a.	Valid_flags.replace_cloud = 1
12538  *	b.	Old_filter = 1 (instead of outer IP)
12539  *	c.	New_filter = 0x10
12540  *	d.	Buffer – 2 entries:
12541  *		i.	Byte 0 = 0x80 | 7 (valid | Stag).
12542  *			Byte 1-3 = 0 (rsv)
12543  *		ii.	Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1)
12544  *			Byte 9-11 = 0 (rsv)
12545  */
12546 static int
12547 i40e_cloud_filter_qinq_create(struct i40e_pf *pf)
12548 {
12549 	int ret = -ENOTSUP;
12550 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
12551 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
12552 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
12553 	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
12554 
12555 	if (pf->support_multi_driver) {
12556 		PMD_DRV_LOG(ERR, "Replace cloud filter is not supported.");
12557 		return ret;
12558 	}
12559 
12560 	/* Init */
12561 	memset(&filter_replace, 0,
12562 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
12563 	memset(&filter_replace_buf, 0,
12564 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
12565 
12566 	/* create L1 filter */
12567 	filter_replace.old_filter_type =
12568 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG_IVLAN;
12569 	filter_replace.new_filter_type = I40E_AQC_ADD_CLOUD_FILTER_0X10;
12570 	filter_replace.tr_bit = 0;
12571 
12572 	/* Prepare the buffer, 2 entries */
12573 	filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_VLAN;
12574 	filter_replace_buf.data[0] |=
12575 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
12576 	/* Field Vector 12b mask */
12577 	filter_replace_buf.data[2] = 0xff;
12578 	filter_replace_buf.data[3] = 0x0f;
12579 	filter_replace_buf.data[4] =
12580 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_INNER_VLAN;
12581 	filter_replace_buf.data[4] |=
12582 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
12583 	/* Field Vector 12b mask */
12584 	filter_replace_buf.data[6] = 0xff;
12585 	filter_replace_buf.data[7] = 0x0f;
12586 	ret = i40e_aq_replace_cloud_filters(hw, &filter_replace,
12587 			&filter_replace_buf);
12588 	if (ret != I40E_SUCCESS)
12589 		return ret;
12590 
12591 	if (filter_replace.old_filter_type !=
12592 	    filter_replace.new_filter_type)
12593 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
12594 			    " original: 0x%x, new: 0x%x",
12595 			    dev->device->name,
12596 			    filter_replace.old_filter_type,
12597 			    filter_replace.new_filter_type);
12598 
12599 	/* Apply the second L2 cloud filter */
12600 	memset(&filter_replace, 0,
12601 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd));
12602 	memset(&filter_replace_buf, 0,
12603 	       sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf));
12604 
12605 	/* create L2 filter, input for L2 filter will be L1 filter  */
12606 	filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER;
12607 	filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_OIP;
12608 	filter_replace.new_filter_type = I40E_AQC_ADD_CLOUD_FILTER_0X10;
12609 
12610 	/* Prepare the buffer, 2 entries */
12611 	filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG;
12612 	filter_replace_buf.data[0] |=
12613 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
12614 	filter_replace_buf.data[4] = I40E_AQC_ADD_CLOUD_FILTER_0X10;
12615 	filter_replace_buf.data[4] |=
12616 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
12617 	ret = i40e_aq_replace_cloud_filters(hw, &filter_replace,
12618 			&filter_replace_buf);
12619 	if (!ret && (filter_replace.old_filter_type !=
12620 		     filter_replace.new_filter_type))
12621 		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
12622 			    " original: 0x%x, new: 0x%x",
12623 			    dev->device->name,
12624 			    filter_replace.old_filter_type,
12625 			    filter_replace.new_filter_type);
12626 
12627 	return ret;
12628 }
12629 
12630 int
12631 i40e_rss_conf_init(struct i40e_rte_flow_rss_conf *out,
12632 		   const struct rte_flow_action_rss *in)
12633 {
12634 	if (in->key_len > RTE_DIM(out->key) ||
12635 	    in->queue_num > RTE_DIM(out->queue))
12636 		return -EINVAL;
12637 	if (!in->key && in->key_len)
12638 		return -EINVAL;
12639 	out->conf = (struct rte_flow_action_rss){
12640 		.func = in->func,
12641 		.level = in->level,
12642 		.types = in->types,
12643 		.key_len = in->key_len,
12644 		.queue_num = in->queue_num,
12645 		.queue = memcpy(out->queue, in->queue,
12646 				sizeof(*in->queue) * in->queue_num),
12647 	};
12648 	if (in->key)
12649 		out->conf.key = memcpy(out->key, in->key, in->key_len);
12650 	return 0;
12651 }
12652 
12653 int
12654 i40e_action_rss_same(const struct rte_flow_action_rss *comp,
12655 		     const struct rte_flow_action_rss *with)
12656 {
12657 	return (comp->func == with->func &&
12658 		comp->level == with->level &&
12659 		comp->types == with->types &&
12660 		comp->key_len == with->key_len &&
12661 		comp->queue_num == with->queue_num &&
12662 		!memcmp(comp->key, with->key, with->key_len) &&
12663 		!memcmp(comp->queue, with->queue,
12664 			sizeof(*with->queue) * with->queue_num));
12665 }
12666 
12667 int
12668 i40e_config_rss_filter(struct i40e_pf *pf,
12669 		struct i40e_rte_flow_rss_conf *conf, bool add)
12670 {
12671 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
12672 	uint32_t i, lut = 0;
12673 	uint16_t j, num;
12674 	struct rte_eth_rss_conf rss_conf = {
12675 		.rss_key = conf->conf.key_len ?
12676 			(void *)(uintptr_t)conf->conf.key : NULL,
12677 		.rss_key_len = conf->conf.key_len,
12678 		.rss_hf = conf->conf.types,
12679 	};
12680 	struct i40e_rte_flow_rss_conf *rss_info = &pf->rss_info;
12681 
12682 	if (!add) {
12683 		if (i40e_action_rss_same(&rss_info->conf, &conf->conf)) {
12684 			i40e_pf_disable_rss(pf);
12685 			memset(rss_info, 0,
12686 				sizeof(struct i40e_rte_flow_rss_conf));
12687 			return 0;
12688 		}
12689 		return -EINVAL;
12690 	}
12691 
12692 	/* If both VMDQ and RSS enabled, not all of PF queues are configured.
12693 	 * It's necessary to calculate the actual PF queues that are configured.
12694 	 */
12695 	if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
12696 		num = i40e_pf_calc_configured_queues_num(pf);
12697 	else
12698 		num = pf->dev_data->nb_rx_queues;
12699 
12700 	num = RTE_MIN(num, conf->conf.queue_num);
12701 	PMD_DRV_LOG(INFO, "Max of contiguous %u PF queues are configured",
12702 			num);
12703 
12704 	if (num == 0) {
12705 		PMD_DRV_LOG(ERR, "No PF queues are configured to enable RSS");
12706 		return -ENOTSUP;
12707 	}
12708 
12709 	/* Fill in redirection table */
12710 	for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
12711 		if (j == num)
12712 			j = 0;
12713 		lut = (lut << 8) | (conf->conf.queue[j] & ((0x1 <<
12714 			hw->func_caps.rss_table_entry_width) - 1));
12715 		if ((i & 3) == 3)
12716 			I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
12717 	}
12718 
12719 	if ((rss_conf.rss_hf & pf->adapter->flow_types_mask) == 0) {
12720 		i40e_pf_disable_rss(pf);
12721 		return 0;
12722 	}
12723 	if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
12724 		(I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
12725 		/* Random default keys */
12726 		static uint32_t rss_key_default[] = {0x6b793944,
12727 			0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
12728 			0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
12729 			0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
12730 
12731 		rss_conf.rss_key = (uint8_t *)rss_key_default;
12732 		rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
12733 							sizeof(uint32_t);
12734 		PMD_DRV_LOG(INFO,
12735 			"No valid RSS key config for i40e, using default\n");
12736 	}
12737 
12738 	i40e_hw_rss_hash_set(pf, &rss_conf);
12739 
12740 	if (i40e_rss_conf_init(rss_info, &conf->conf))
12741 		return -EINVAL;
12742 
12743 	return 0;
12744 }
12745 
12746 RTE_INIT(i40e_init_log)
12747 {
12748 	i40e_logtype_init = rte_log_register("pmd.net.i40e.init");
12749 	if (i40e_logtype_init >= 0)
12750 		rte_log_set_level(i40e_logtype_init, RTE_LOG_NOTICE);
12751 	i40e_logtype_driver = rte_log_register("pmd.net.i40e.driver");
12752 	if (i40e_logtype_driver >= 0)
12753 		rte_log_set_level(i40e_logtype_driver, RTE_LOG_NOTICE);
12754 }
12755 
12756 RTE_PMD_REGISTER_PARAM_STRING(net_i40e,
12757 			      ETH_I40E_FLOATING_VEB_ARG "=1"
12758 			      ETH_I40E_FLOATING_VEB_LIST_ARG "=<string>"
12759 			      ETH_I40E_QUEUE_NUM_PER_VF_ARG "=1|2|4|8|16"
12760 			      ETH_I40E_SUPPORT_MULTI_DRIVER "=1"
12761 			      ETH_I40E_USE_LATEST_VEC "=0|1");
12762