xref: /dpdk/drivers/net/bnxt/bnxt_flow.c (revision b19f366c)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2014-2021 Broadcom
3  * All rights reserved.
4  */
5 
6 #include <sys/queue.h>
7 
8 #include <rte_log.h>
9 #include <rte_malloc.h>
10 #include <rte_flow.h>
11 #include <rte_flow_driver.h>
12 #include <rte_tailq.h>
13 #include <rte_alarm.h>
14 #include <rte_cycles.h>
15 
16 #include "bnxt.h"
17 #include "bnxt_filter.h"
18 #include "bnxt_hwrm.h"
19 #include "bnxt_ring.h"
20 #include "bnxt_rxq.h"
21 #include "bnxt_rxr.h"
22 #include "bnxt_vnic.h"
23 #include "hsi_struct_def_dpdk.h"
24 
25 static int
26 bnxt_flow_args_validate(const struct rte_flow_attr *attr,
27 			const struct rte_flow_item pattern[],
28 			const struct rte_flow_action actions[],
29 			struct rte_flow_error *error)
30 {
31 	if (!pattern) {
32 		rte_flow_error_set(error,
33 				   EINVAL,
34 				   RTE_FLOW_ERROR_TYPE_ITEM_NUM,
35 				   NULL,
36 				   "NULL pattern.");
37 		return -rte_errno;
38 	}
39 
40 	if (!actions) {
41 		rte_flow_error_set(error,
42 				   EINVAL,
43 				   RTE_FLOW_ERROR_TYPE_ACTION_NUM,
44 				   NULL,
45 				   "NULL action.");
46 		return -rte_errno;
47 	}
48 
49 	if (!attr) {
50 		rte_flow_error_set(error,
51 				   EINVAL,
52 				   RTE_FLOW_ERROR_TYPE_ATTR,
53 				   NULL,
54 				   "NULL attribute.");
55 		return -rte_errno;
56 	}
57 
58 	return 0;
59 }
60 
61 static const struct rte_flow_item *
62 bnxt_flow_non_void_item(const struct rte_flow_item *cur)
63 {
64 	while (1) {
65 		if (cur->type != RTE_FLOW_ITEM_TYPE_VOID)
66 			return cur;
67 		cur++;
68 	}
69 }
70 
71 static const struct rte_flow_action *
72 bnxt_flow_non_void_action(const struct rte_flow_action *cur)
73 {
74 	while (1) {
75 		if (cur->type != RTE_FLOW_ACTION_TYPE_VOID)
76 			return cur;
77 		cur++;
78 	}
79 }
80 
81 static int
82 bnxt_filter_type_check(const struct rte_flow_item pattern[],
83 		       struct rte_flow_error *error)
84 {
85 	const struct rte_flow_item *item =
86 		bnxt_flow_non_void_item(pattern);
87 	int use_ntuple = 1;
88 	bool has_vlan = 0;
89 
90 	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
91 		switch (item->type) {
92 		case RTE_FLOW_ITEM_TYPE_ANY:
93 		case RTE_FLOW_ITEM_TYPE_ETH:
94 			use_ntuple = 0;
95 			break;
96 		case RTE_FLOW_ITEM_TYPE_VLAN:
97 			use_ntuple = 0;
98 			has_vlan = 1;
99 			break;
100 		case RTE_FLOW_ITEM_TYPE_IPV4:
101 		case RTE_FLOW_ITEM_TYPE_IPV6:
102 		case RTE_FLOW_ITEM_TYPE_TCP:
103 		case RTE_FLOW_ITEM_TYPE_UDP:
104 			/* FALLTHROUGH */
105 			/* need ntuple match, reset exact match */
106 			use_ntuple |= 1;
107 			break;
108 		default:
109 			PMD_DRV_LOG(DEBUG, "Unknown Flow type\n");
110 			use_ntuple |= 0;
111 		}
112 		item++;
113 	}
114 
115 	if (has_vlan && use_ntuple) {
116 		PMD_DRV_LOG(ERR,
117 			    "VLAN flow cannot use NTUPLE filter\n");
118 		rte_flow_error_set(error, EINVAL,
119 				   RTE_FLOW_ERROR_TYPE_ITEM,
120 				   item,
121 				   "Cannot use VLAN with NTUPLE");
122 		return -rte_errno;
123 	}
124 
125 	return use_ntuple;
126 }
127 
128 static int
129 bnxt_validate_and_parse_flow_type(struct bnxt *bp,
130 				  const struct rte_flow_attr *attr,
131 				  const struct rte_flow_item pattern[],
132 				  struct rte_flow_error *error,
133 				  struct bnxt_filter_info *filter)
134 {
135 	const struct rte_flow_item *item = bnxt_flow_non_void_item(pattern);
136 	const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
137 	const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask;
138 	const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask;
139 	const struct rte_flow_item_tcp *tcp_spec, *tcp_mask;
140 	const struct rte_flow_item_udp *udp_spec, *udp_mask;
141 	const struct rte_flow_item_eth *eth_spec, *eth_mask;
142 	const struct rte_ether_addr *dst, *src;
143 	const struct rte_flow_item_nvgre *nvgre_spec;
144 	const struct rte_flow_item_nvgre *nvgre_mask;
145 	const struct rte_flow_item_gre *gre_spec;
146 	const struct rte_flow_item_gre *gre_mask;
147 	const struct rte_flow_item_vxlan *vxlan_spec;
148 	const struct rte_flow_item_vxlan *vxlan_mask;
149 	uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF};
150 	uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF};
151 	const struct rte_flow_item_vf *vf_spec;
152 	uint32_t tenant_id_be = 0, valid_flags = 0;
153 	bool vni_masked = 0;
154 	bool tni_masked = 0;
155 	uint32_t en_ethertype;
156 	uint8_t inner = 0;
157 	uint32_t vf = 0;
158 	uint32_t en = 0;
159 	int use_ntuple;
160 	int dflt_vnic;
161 
162 	use_ntuple = bnxt_filter_type_check(pattern, error);
163 	if (use_ntuple < 0)
164 		return use_ntuple;
165 	PMD_DRV_LOG(DEBUG, "Use NTUPLE %d\n", use_ntuple);
166 
167 	filter->filter_type = use_ntuple ?
168 		HWRM_CFA_NTUPLE_FILTER : HWRM_CFA_L2_FILTER;
169 	en_ethertype = use_ntuple ?
170 		NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE :
171 		EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE;
172 
173 	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
174 		if (item->last) {
175 			/* last or range is NOT supported as match criteria */
176 			rte_flow_error_set(error, EINVAL,
177 					   RTE_FLOW_ERROR_TYPE_ITEM,
178 					   item,
179 					   "No support for range");
180 			return -rte_errno;
181 		}
182 
183 		switch (item->type) {
184 		case RTE_FLOW_ITEM_TYPE_ANY:
185 			inner =
186 			((const struct rte_flow_item_any *)item->spec)->num > 3;
187 			if (inner)
188 				PMD_DRV_LOG(DEBUG, "Parse inner header\n");
189 			break;
190 		case RTE_FLOW_ITEM_TYPE_ETH:
191 			if (!item->spec)
192 				break;
193 
194 			eth_spec = item->spec;
195 
196 			if (item->mask)
197 				eth_mask = item->mask;
198 			else
199 				eth_mask = &rte_flow_item_eth_mask;
200 
201 			/* Source MAC address mask cannot be partially set.
202 			 * Should be All 0's or all 1's.
203 			 * Destination MAC address mask must not be partially
204 			 * set. Should be all 1's or all 0's.
205 			 */
206 			if ((!rte_is_zero_ether_addr(&eth_mask->src) &&
207 			     !rte_is_broadcast_ether_addr(&eth_mask->src)) ||
208 			    (!rte_is_zero_ether_addr(&eth_mask->dst) &&
209 			     !rte_is_broadcast_ether_addr(&eth_mask->dst))) {
210 				rte_flow_error_set(error,
211 						   EINVAL,
212 						   RTE_FLOW_ERROR_TYPE_ITEM,
213 						   item,
214 						   "MAC_addr mask not valid");
215 				return -rte_errno;
216 			}
217 
218 			/* Mask is not allowed. Only exact matches are */
219 			if (eth_mask->type &&
220 			    eth_mask->type != RTE_BE16(0xffff)) {
221 				rte_flow_error_set(error, EINVAL,
222 						   RTE_FLOW_ERROR_TYPE_ITEM,
223 						   item,
224 						   "ethertype mask not valid");
225 				return -rte_errno;
226 			}
227 
228 			if (rte_is_broadcast_ether_addr(&eth_mask->dst)) {
229 				dst = &eth_spec->dst;
230 				if (!rte_is_valid_assigned_ether_addr(dst)) {
231 					rte_flow_error_set(error,
232 							   EINVAL,
233 							   RTE_FLOW_ERROR_TYPE_ITEM,
234 							   item,
235 							   "DMAC is invalid");
236 					PMD_DRV_LOG(ERR,
237 						    "DMAC is invalid!\n");
238 					return -rte_errno;
239 				}
240 				rte_memcpy(filter->dst_macaddr,
241 					   &eth_spec->dst, RTE_ETHER_ADDR_LEN);
242 				en |= use_ntuple ?
243 					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR :
244 					EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR;
245 				valid_flags |= inner ?
246 					BNXT_FLOW_L2_INNER_DST_VALID_FLAG :
247 					BNXT_FLOW_L2_DST_VALID_FLAG;
248 				filter->priority = attr->priority;
249 				PMD_DRV_LOG(DEBUG,
250 					    "Creating a priority flow\n");
251 			}
252 			if (rte_is_broadcast_ether_addr(&eth_mask->src)) {
253 				src = &eth_spec->src;
254 				if (!rte_is_valid_assigned_ether_addr(src)) {
255 					rte_flow_error_set(error,
256 							   EINVAL,
257 							   RTE_FLOW_ERROR_TYPE_ITEM,
258 							   item,
259 							   "SMAC is invalid");
260 					PMD_DRV_LOG(ERR,
261 						    "SMAC is invalid!\n");
262 					return -rte_errno;
263 				}
264 				rte_memcpy(filter->src_macaddr,
265 					   &eth_spec->src, RTE_ETHER_ADDR_LEN);
266 				en |= use_ntuple ?
267 					NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR :
268 					EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR;
269 				valid_flags |= inner ?
270 					BNXT_FLOW_L2_INNER_SRC_VALID_FLAG :
271 					BNXT_FLOW_L2_SRC_VALID_FLAG;
272 			} /*
273 			   * else {
274 			   *  PMD_DRV_LOG(ERR, "Handle this condition\n");
275 			   * }
276 			   */
277 			if (eth_mask->type) {
278 				filter->ethertype =
279 					rte_be_to_cpu_16(eth_spec->type);
280 				en |= en_ethertype;
281 			}
282 			if (inner)
283 				valid_flags |= BNXT_FLOW_PARSE_INNER_FLAG;
284 
285 			break;
286 		case RTE_FLOW_ITEM_TYPE_VLAN:
287 			vlan_spec = item->spec;
288 
289 			if (item->mask)
290 				vlan_mask = item->mask;
291 			else
292 				vlan_mask = &rte_flow_item_vlan_mask;
293 
294 			if (en & en_ethertype) {
295 				rte_flow_error_set(error, EINVAL,
296 						   RTE_FLOW_ERROR_TYPE_ITEM,
297 						   item,
298 						   "VLAN TPID matching is not"
299 						   " supported");
300 				return -rte_errno;
301 			}
302 			if (vlan_mask->tci &&
303 			    vlan_mask->tci == RTE_BE16(0x0fff)) {
304 				/* Only the VLAN ID can be matched. */
305 				filter->l2_ovlan =
306 					rte_be_to_cpu_16(vlan_spec->tci &
307 							 RTE_BE16(0x0fff));
308 				en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
309 			} else {
310 				rte_flow_error_set(error,
311 						   EINVAL,
312 						   RTE_FLOW_ERROR_TYPE_ITEM,
313 						   item,
314 						   "VLAN mask is invalid");
315 				return -rte_errno;
316 			}
317 			if (vlan_mask->inner_type &&
318 			    vlan_mask->inner_type != RTE_BE16(0xffff)) {
319 				rte_flow_error_set(error, EINVAL,
320 						   RTE_FLOW_ERROR_TYPE_ITEM,
321 						   item,
322 						   "inner ethertype mask not"
323 						   " valid");
324 				return -rte_errno;
325 			}
326 			if (vlan_mask->inner_type) {
327 				filter->ethertype =
328 					rte_be_to_cpu_16(vlan_spec->inner_type);
329 				en |= en_ethertype;
330 			}
331 
332 			break;
333 		case RTE_FLOW_ITEM_TYPE_IPV4:
334 			/* If mask is not involved, we could use EM filters. */
335 			ipv4_spec = item->spec;
336 
337 			if (!item->spec)
338 				break;
339 
340 			if (item->mask)
341 				ipv4_mask = item->mask;
342 			else
343 				ipv4_mask = &rte_flow_item_ipv4_mask;
344 
345 			/* Only IP DST and SRC fields are maskable. */
346 			if (ipv4_mask->hdr.version_ihl ||
347 			    ipv4_mask->hdr.type_of_service ||
348 			    ipv4_mask->hdr.total_length ||
349 			    ipv4_mask->hdr.packet_id ||
350 			    ipv4_mask->hdr.fragment_offset ||
351 			    ipv4_mask->hdr.time_to_live ||
352 			    ipv4_mask->hdr.next_proto_id ||
353 			    ipv4_mask->hdr.hdr_checksum) {
354 				rte_flow_error_set(error,
355 						   EINVAL,
356 						   RTE_FLOW_ERROR_TYPE_ITEM,
357 						   item,
358 						   "Invalid IPv4 mask.");
359 				return -rte_errno;
360 			}
361 
362 			filter->dst_ipaddr[0] = ipv4_spec->hdr.dst_addr;
363 			filter->src_ipaddr[0] = ipv4_spec->hdr.src_addr;
364 
365 			if (use_ntuple)
366 				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
367 					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
368 			else
369 				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
370 					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
371 
372 			if (ipv4_mask->hdr.src_addr) {
373 				filter->src_ipaddr_mask[0] =
374 					ipv4_mask->hdr.src_addr;
375 				en |= !use_ntuple ? 0 :
376 				     NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
377 			}
378 
379 			if (ipv4_mask->hdr.dst_addr) {
380 				filter->dst_ipaddr_mask[0] =
381 					ipv4_mask->hdr.dst_addr;
382 				en |= !use_ntuple ? 0 :
383 				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
384 			}
385 
386 			filter->ip_addr_type = use_ntuple ?
387 			 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 :
388 			 HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
389 
390 			if (ipv4_spec->hdr.next_proto_id) {
391 				filter->ip_protocol =
392 					ipv4_spec->hdr.next_proto_id;
393 				if (use_ntuple)
394 					en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
395 				else
396 					en |= EM_FLOW_ALLOC_INPUT_EN_IP_PROTO;
397 			}
398 			break;
399 		case RTE_FLOW_ITEM_TYPE_IPV6:
400 			ipv6_spec = item->spec;
401 
402 			if (!item->spec)
403 				break;
404 
405 			if (item->mask)
406 				ipv6_mask = item->mask;
407 			else
408 				ipv6_mask = &rte_flow_item_ipv6_mask;
409 
410 			/* Only IP DST and SRC fields are maskable. */
411 			if (ipv6_mask->hdr.vtc_flow ||
412 			    ipv6_mask->hdr.payload_len ||
413 			    ipv6_mask->hdr.proto ||
414 			    ipv6_mask->hdr.hop_limits) {
415 				rte_flow_error_set(error,
416 						   EINVAL,
417 						   RTE_FLOW_ERROR_TYPE_ITEM,
418 						   item,
419 						   "Invalid IPv6 mask.");
420 				return -rte_errno;
421 			}
422 
423 			if (use_ntuple)
424 				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
425 					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
426 			else
427 				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
428 					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
429 
430 			rte_memcpy(filter->src_ipaddr,
431 				   ipv6_spec->hdr.src_addr, 16);
432 			rte_memcpy(filter->dst_ipaddr,
433 				   ipv6_spec->hdr.dst_addr, 16);
434 
435 			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.src_addr,
436 						   16)) {
437 				rte_memcpy(filter->src_ipaddr_mask,
438 					   ipv6_mask->hdr.src_addr, 16);
439 				en |= !use_ntuple ? 0 :
440 				    NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
441 			}
442 
443 			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.dst_addr,
444 						   16)) {
445 				rte_memcpy(filter->dst_ipaddr_mask,
446 					   ipv6_mask->hdr.dst_addr, 16);
447 				en |= !use_ntuple ? 0 :
448 				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
449 			}
450 
451 			filter->ip_addr_type = use_ntuple ?
452 				NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 :
453 				EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
454 			break;
455 		case RTE_FLOW_ITEM_TYPE_TCP:
456 			tcp_spec = item->spec;
457 
458 			if (!item->spec)
459 				break;
460 
461 			if (item->mask)
462 				tcp_mask = item->mask;
463 			else
464 				tcp_mask = &rte_flow_item_tcp_mask;
465 
466 			/* Check TCP mask. Only DST & SRC ports are maskable */
467 			if (tcp_mask->hdr.sent_seq ||
468 			    tcp_mask->hdr.recv_ack ||
469 			    tcp_mask->hdr.data_off ||
470 			    tcp_mask->hdr.tcp_flags ||
471 			    tcp_mask->hdr.rx_win ||
472 			    tcp_mask->hdr.cksum ||
473 			    tcp_mask->hdr.tcp_urp) {
474 				rte_flow_error_set(error,
475 						   EINVAL,
476 						   RTE_FLOW_ERROR_TYPE_ITEM,
477 						   item,
478 						   "Invalid TCP mask");
479 				return -rte_errno;
480 			}
481 
482 			filter->src_port = tcp_spec->hdr.src_port;
483 			filter->dst_port = tcp_spec->hdr.dst_port;
484 
485 			if (use_ntuple)
486 				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
487 					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
488 			else
489 				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
490 					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
491 
492 			if (tcp_mask->hdr.dst_port) {
493 				filter->dst_port_mask = tcp_mask->hdr.dst_port;
494 				en |= !use_ntuple ? 0 :
495 				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
496 			}
497 
498 			if (tcp_mask->hdr.src_port) {
499 				filter->src_port_mask = tcp_mask->hdr.src_port;
500 				en |= !use_ntuple ? 0 :
501 				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
502 			}
503 			break;
504 		case RTE_FLOW_ITEM_TYPE_UDP:
505 			udp_spec = item->spec;
506 
507 			if (!item->spec)
508 				break;
509 
510 			if (item->mask)
511 				udp_mask = item->mask;
512 			else
513 				udp_mask = &rte_flow_item_udp_mask;
514 
515 			if (udp_mask->hdr.dgram_len ||
516 			    udp_mask->hdr.dgram_cksum) {
517 				rte_flow_error_set(error,
518 						   EINVAL,
519 						   RTE_FLOW_ERROR_TYPE_ITEM,
520 						   item,
521 						   "Invalid UDP mask");
522 				return -rte_errno;
523 			}
524 
525 			filter->src_port = udp_spec->hdr.src_port;
526 			filter->dst_port = udp_spec->hdr.dst_port;
527 
528 			if (use_ntuple)
529 				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
530 					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
531 			else
532 				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
533 					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
534 
535 			if (udp_mask->hdr.dst_port) {
536 				filter->dst_port_mask = udp_mask->hdr.dst_port;
537 				en |= !use_ntuple ? 0 :
538 				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
539 			}
540 
541 			if (udp_mask->hdr.src_port) {
542 				filter->src_port_mask = udp_mask->hdr.src_port;
543 				en |= !use_ntuple ? 0 :
544 				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
545 			}
546 			break;
547 		case RTE_FLOW_ITEM_TYPE_VXLAN:
548 			vxlan_spec = item->spec;
549 			vxlan_mask = item->mask;
550 			/* Check if VXLAN item is used to describe protocol.
551 			 * If yes, both spec and mask should be NULL.
552 			 * If no, both spec and mask shouldn't be NULL.
553 			 */
554 			if ((!vxlan_spec && vxlan_mask) ||
555 			    (vxlan_spec && !vxlan_mask)) {
556 				rte_flow_error_set(error,
557 						   EINVAL,
558 						   RTE_FLOW_ERROR_TYPE_ITEM,
559 						   item,
560 						   "Invalid VXLAN item");
561 				return -rte_errno;
562 			}
563 
564 			if (!vxlan_spec && !vxlan_mask) {
565 				filter->tunnel_type =
566 				CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
567 				break;
568 			}
569 
570 			if (vxlan_spec->rsvd1 || vxlan_spec->rsvd0[0] ||
571 			    vxlan_spec->rsvd0[1] || vxlan_spec->rsvd0[2] ||
572 			    vxlan_spec->flags != 0x8) {
573 				rte_flow_error_set(error,
574 						   EINVAL,
575 						   RTE_FLOW_ERROR_TYPE_ITEM,
576 						   item,
577 						   "Invalid VXLAN item");
578 				return -rte_errno;
579 			}
580 
581 			/* Check if VNI is masked. */
582 			if (vxlan_mask != NULL) {
583 				vni_masked =
584 					!!memcmp(vxlan_mask->vni, vni_mask,
585 						 RTE_DIM(vni_mask));
586 				if (vni_masked) {
587 					rte_flow_error_set
588 						(error,
589 						 EINVAL,
590 						 RTE_FLOW_ERROR_TYPE_ITEM,
591 						 item,
592 						 "Invalid VNI mask");
593 					return -rte_errno;
594 				}
595 
596 				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
597 					   vxlan_spec->vni, 3);
598 				filter->vni =
599 					rte_be_to_cpu_32(tenant_id_be);
600 				filter->tunnel_type =
601 				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
602 			}
603 			break;
604 		case RTE_FLOW_ITEM_TYPE_NVGRE:
605 			nvgre_spec = item->spec;
606 			nvgre_mask = item->mask;
607 			/* Check if NVGRE item is used to describe protocol.
608 			 * If yes, both spec and mask should be NULL.
609 			 * If no, both spec and mask shouldn't be NULL.
610 			 */
611 			if ((!nvgre_spec && nvgre_mask) ||
612 			    (nvgre_spec && !nvgre_mask)) {
613 				rte_flow_error_set(error,
614 						   EINVAL,
615 						   RTE_FLOW_ERROR_TYPE_ITEM,
616 						   item,
617 						   "Invalid NVGRE item");
618 				return -rte_errno;
619 			}
620 
621 			if (!nvgre_spec && !nvgre_mask) {
622 				filter->tunnel_type =
623 				CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
624 				break;
625 			}
626 
627 			if (nvgre_spec->c_k_s_rsvd0_ver != 0x2000 ||
628 			    nvgre_spec->protocol != 0x6558) {
629 				rte_flow_error_set(error,
630 						   EINVAL,
631 						   RTE_FLOW_ERROR_TYPE_ITEM,
632 						   item,
633 						   "Invalid NVGRE item");
634 				return -rte_errno;
635 			}
636 
637 			if (nvgre_spec && nvgre_mask) {
638 				tni_masked =
639 					!!memcmp(nvgre_mask->tni, tni_mask,
640 						 RTE_DIM(tni_mask));
641 				if (tni_masked) {
642 					rte_flow_error_set
643 						(error,
644 						 EINVAL,
645 						 RTE_FLOW_ERROR_TYPE_ITEM,
646 						 item,
647 						 "Invalid TNI mask");
648 					return -rte_errno;
649 				}
650 				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
651 					   nvgre_spec->tni, 3);
652 				filter->vni =
653 					rte_be_to_cpu_32(tenant_id_be);
654 				filter->tunnel_type =
655 				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
656 			}
657 			break;
658 
659 		case RTE_FLOW_ITEM_TYPE_GRE:
660 			gre_spec = (const struct rte_flow_item_gre *)item->spec;
661 			gre_mask = (const struct rte_flow_item_gre *)item->mask;
662 
663 			/*
664 			 *Check if GRE item is used to describe protocol.
665 			 * If yes, both spec and mask should be NULL.
666 			 * If no, both spec and mask shouldn't be NULL.
667 			 */
668 			if (!!gre_spec ^ !!gre_mask) {
669 				rte_flow_error_set(error, EINVAL,
670 						   RTE_FLOW_ERROR_TYPE_ITEM,
671 						   item,
672 						   "Invalid GRE item");
673 				return -rte_errno;
674 			}
675 
676 			if (!gre_spec && !gre_mask) {
677 				filter->tunnel_type =
678 				CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE;
679 				break;
680 			}
681 			break;
682 
683 		case RTE_FLOW_ITEM_TYPE_VF:
684 			vf_spec = item->spec;
685 			vf = vf_spec->id;
686 			if (!BNXT_PF(bp)) {
687 				rte_flow_error_set(error,
688 						   EINVAL,
689 						   RTE_FLOW_ERROR_TYPE_ITEM,
690 						   item,
691 						   "Configuring on a VF!");
692 				return -rte_errno;
693 			}
694 
695 			if (vf >= bp->pdev->max_vfs) {
696 				rte_flow_error_set(error,
697 						   EINVAL,
698 						   RTE_FLOW_ERROR_TYPE_ITEM,
699 						   item,
700 						   "Incorrect VF id!");
701 				return -rte_errno;
702 			}
703 
704 			if (!attr->transfer) {
705 				rte_flow_error_set(error,
706 						   ENOTSUP,
707 						   RTE_FLOW_ERROR_TYPE_ITEM,
708 						   item,
709 						   "Matching VF traffic without"
710 						   " affecting it (transfer attribute)"
711 						   " is unsupported");
712 				return -rte_errno;
713 			}
714 
715 			filter->mirror_vnic_id =
716 			dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
717 			if (dflt_vnic < 0) {
718 				/* This simply indicates there's no driver
719 				 * loaded. This is not an error.
720 				 */
721 				rte_flow_error_set
722 					(error,
723 					 EINVAL,
724 					 RTE_FLOW_ERROR_TYPE_ITEM,
725 					 item,
726 					 "Unable to get default VNIC for VF");
727 				return -rte_errno;
728 			}
729 
730 			filter->mirror_vnic_id = dflt_vnic;
731 			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
732 			break;
733 		default:
734 			break;
735 		}
736 		item++;
737 	}
738 	filter->enables = en;
739 	filter->valid_flags = valid_flags;
740 
741 	return 0;
742 }
743 
744 /* Parse attributes */
745 static int
746 bnxt_flow_parse_attr(const struct rte_flow_attr *attr,
747 		     struct rte_flow_error *error)
748 {
749 	/* Must be input direction */
750 	if (!attr->ingress) {
751 		rte_flow_error_set(error,
752 				   EINVAL,
753 				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
754 				   attr,
755 				   "Only support ingress.");
756 		return -rte_errno;
757 	}
758 
759 	/* Not supported */
760 	if (attr->egress) {
761 		rte_flow_error_set(error,
762 				   EINVAL,
763 				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
764 				   attr,
765 				   "No support for egress.");
766 		return -rte_errno;
767 	}
768 
769 	return 0;
770 }
771 
772 static struct bnxt_filter_info *
773 bnxt_find_matching_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf)
774 {
775 	struct bnxt_filter_info *mf, *f0;
776 	struct bnxt_vnic_info *vnic0;
777 	int i;
778 
779 	vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
780 	f0 = STAILQ_FIRST(&vnic0->filter);
781 
782 	/* This flow has same DST MAC as the port/l2 filter. */
783 	if (memcmp(f0->l2_addr, nf->dst_macaddr, RTE_ETHER_ADDR_LEN) == 0)
784 		return f0;
785 
786 	for (i = bp->max_vnics - 1; i >= 0; i--) {
787 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
788 
789 		if (vnic->fw_vnic_id == INVALID_VNIC_ID)
790 			continue;
791 
792 		STAILQ_FOREACH(mf, &vnic->filter, next) {
793 
794 			if (mf->matching_l2_fltr_ptr)
795 				continue;
796 
797 			if (mf->ethertype == nf->ethertype &&
798 			    mf->l2_ovlan == nf->l2_ovlan &&
799 			    mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
800 			    mf->l2_ivlan == nf->l2_ivlan &&
801 			    mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
802 			    !memcmp(mf->src_macaddr, nf->src_macaddr,
803 				    RTE_ETHER_ADDR_LEN) &&
804 			    !memcmp(mf->dst_macaddr, nf->dst_macaddr,
805 				    RTE_ETHER_ADDR_LEN))
806 				return mf;
807 		}
808 	}
809 	return NULL;
810 }
811 
812 static struct bnxt_filter_info *
813 bnxt_create_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf,
814 		      struct bnxt_vnic_info *vnic)
815 {
816 	struct bnxt_filter_info *filter1;
817 	int rc;
818 
819 	/* Alloc new L2 filter.
820 	 * This flow needs MAC filter which does not match any existing
821 	 * L2 filters.
822 	 */
823 	filter1 = bnxt_get_unused_filter(bp);
824 	if (filter1 == NULL)
825 		return NULL;
826 
827 	memcpy(filter1, nf, sizeof(*filter1));
828 
829 	filter1->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_XDP_DISABLE;
830 	filter1->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX;
831 	if (nf->valid_flags & BNXT_FLOW_L2_SRC_VALID_FLAG ||
832 	    nf->valid_flags & BNXT_FLOW_L2_DST_VALID_FLAG) {
833 		filter1->flags |=
834 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
835 		PMD_DRV_LOG(DEBUG, "Create Outer filter\n");
836 	}
837 
838 	if (nf->filter_type == HWRM_CFA_L2_FILTER &&
839 	    (nf->valid_flags & BNXT_FLOW_L2_SRC_VALID_FLAG ||
840 	     nf->valid_flags & BNXT_FLOW_L2_INNER_SRC_VALID_FLAG)) {
841 		PMD_DRV_LOG(DEBUG, "Create L2 filter for SRC MAC\n");
842 		filter1->flags |=
843 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_SOURCE_VALID;
844 		memcpy(filter1->l2_addr, nf->src_macaddr, RTE_ETHER_ADDR_LEN);
845 	} else {
846 		PMD_DRV_LOG(DEBUG, "Create L2 filter for DST MAC\n");
847 		memcpy(filter1->l2_addr, nf->dst_macaddr, RTE_ETHER_ADDR_LEN);
848 	}
849 
850 	if (nf->priority &&
851 	    (nf->valid_flags & BNXT_FLOW_L2_DST_VALID_FLAG ||
852 	     nf->valid_flags & BNXT_FLOW_L2_INNER_DST_VALID_FLAG)) {
853 		/* Tell the FW where to place the filter in the table. */
854 		if (nf->priority > 65535) {
855 			filter1->pri_hint =
856 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_BELOW_FILTER;
857 			/* This will place the filter in TCAM */
858 			filter1->l2_filter_id_hint = (uint64_t)-1;
859 		}
860 	}
861 
862 	if (nf->valid_flags & (BNXT_FLOW_L2_DST_VALID_FLAG |
863 			       BNXT_FLOW_L2_SRC_VALID_FLAG |
864 			       BNXT_FLOW_L2_INNER_SRC_VALID_FLAG |
865 			       BNXT_FLOW_L2_INNER_DST_VALID_FLAG)) {
866 		filter1->enables =
867 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
868 			L2_FILTER_ALLOC_INPUT_EN_L2_ADDR_MASK;
869 		memset(filter1->l2_addr_mask, 0xff, RTE_ETHER_ADDR_LEN);
870 	}
871 
872 	if (nf->valid_flags & BNXT_FLOW_L2_DROP_FLAG) {
873 		filter1->flags |=
874 			HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_DROP;
875 		if (nf->ethertype == RTE_ETHER_TYPE_IPV4) {
876 			/* Num VLANs for drop filter will/should be 0.
877 			 * If the req is memset to 0, then the count will
878 			 * be automatically set to 0.
879 			 */
880 			if (nf->valid_flags & BNXT_FLOW_PARSE_INNER_FLAG) {
881 				filter1->enables |=
882 					L2_FILTER_ALLOC_INPUT_EN_T_NUM_VLANS;
883 			} else {
884 				filter1->enables |=
885 					L2_FILTER_ALLOC_INPUT_EN_NUM_VLANS;
886 				filter1->flags |=
887 				HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
888 			}
889 		}
890 	}
891 
892 	rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id,
893 				     filter1);
894 	if (rc) {
895 		bnxt_free_filter(bp, filter1);
896 		return NULL;
897 	}
898 	return filter1;
899 }
900 
901 struct bnxt_filter_info *
902 bnxt_get_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf,
903 		   struct bnxt_vnic_info *vnic)
904 {
905 	struct bnxt_filter_info *l2_filter = NULL;
906 
907 	l2_filter = bnxt_find_matching_l2_filter(bp, nf);
908 	if (l2_filter) {
909 		l2_filter->l2_ref_cnt++;
910 	} else {
911 		l2_filter = bnxt_create_l2_filter(bp, nf, vnic);
912 		if (l2_filter) {
913 			STAILQ_INSERT_TAIL(&vnic->filter, l2_filter, next);
914 			l2_filter->vnic = vnic;
915 		}
916 	}
917 	nf->matching_l2_fltr_ptr = l2_filter;
918 
919 	return l2_filter;
920 }
921 
922 static int bnxt_vnic_prep(struct bnxt *bp, struct bnxt_vnic_info *vnic)
923 {
924 	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
925 	uint64_t rx_offloads = dev_conf->rxmode.offloads;
926 	int rc;
927 
928 	if (bp->nr_vnics > bp->max_vnics - 1)
929 		return -ENOMEM;
930 
931 	rc = bnxt_vnic_grp_alloc(bp, vnic);
932 	if (rc)
933 		goto ret;
934 
935 	rc = bnxt_hwrm_vnic_alloc(bp, vnic);
936 	if (rc) {
937 		PMD_DRV_LOG(ERR, "HWRM vnic alloc failure rc: %x\n", rc);
938 		goto ret;
939 	}
940 	bp->nr_vnics++;
941 
942 	/* RSS context is required only when there is more than one RSS ring */
943 	if (vnic->rx_queue_cnt > 1) {
944 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 0 /* ctx_idx 0 */);
945 		if (rc) {
946 			PMD_DRV_LOG(ERR,
947 				    "HWRM vnic ctx alloc failure: %x\n", rc);
948 			goto ret;
949 		}
950 	} else {
951 		PMD_DRV_LOG(DEBUG, "No RSS context required\n");
952 	}
953 
954 	if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
955 		vnic->vlan_strip = true;
956 	else
957 		vnic->vlan_strip = false;
958 
959 	rc = bnxt_hwrm_vnic_cfg(bp, vnic);
960 	if (rc)
961 		goto ret;
962 
963 	bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
964 
965 ret:
966 	return rc;
967 }
968 
969 static int match_vnic_rss_cfg(struct bnxt *bp,
970 			      struct bnxt_vnic_info *vnic,
971 			      const struct rte_flow_action_rss *rss)
972 {
973 	unsigned int match = 0, i;
974 
975 	if (vnic->rx_queue_cnt != rss->queue_num)
976 		return -EINVAL;
977 
978 	for (i = 0; i < rss->queue_num; i++) {
979 		if (!bp->rx_queues[rss->queue[i]]->vnic->rx_queue_cnt &&
980 		    !bp->rx_queues[rss->queue[i]]->rx_started)
981 			return -EINVAL;
982 	}
983 
984 	for (i = 0; i < vnic->rx_queue_cnt; i++) {
985 		int j;
986 
987 		for (j = 0; j < vnic->rx_queue_cnt; j++) {
988 			if (bp->grp_info[rss->queue[i]].fw_grp_id ==
989 			    vnic->fw_grp_ids[j])
990 				match++;
991 		}
992 	}
993 
994 	if (match != vnic->rx_queue_cnt) {
995 		PMD_DRV_LOG(ERR,
996 			    "VNIC queue count %d vs queues matched %d\n",
997 			    match, vnic->rx_queue_cnt);
998 		return -EINVAL;
999 	}
1000 
1001 	return 0;
1002 }
1003 
1004 static void
1005 bnxt_update_filter_flags_en(struct bnxt_filter_info *filter,
1006 			    struct bnxt_filter_info *filter1,
1007 			    int use_ntuple)
1008 {
1009 	if (!use_ntuple &&
1010 	    !(filter->valid_flags &
1011 	      ~(BNXT_FLOW_L2_DST_VALID_FLAG |
1012 		BNXT_FLOW_L2_SRC_VALID_FLAG |
1013 		BNXT_FLOW_L2_INNER_SRC_VALID_FLAG |
1014 		BNXT_FLOW_L2_INNER_DST_VALID_FLAG |
1015 		BNXT_FLOW_L2_DROP_FLAG |
1016 		BNXT_FLOW_PARSE_INNER_FLAG))) {
1017 		filter->flags = filter1->flags;
1018 		filter->enables = filter1->enables;
1019 		filter->filter_type = HWRM_CFA_L2_FILTER;
1020 		memcpy(filter->l2_addr, filter1->l2_addr, RTE_ETHER_ADDR_LEN);
1021 		memset(filter->l2_addr_mask, 0xff, RTE_ETHER_ADDR_LEN);
1022 		filter->pri_hint = filter1->pri_hint;
1023 		filter->l2_filter_id_hint = filter1->l2_filter_id_hint;
1024 	}
1025 	filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
1026 	filter->l2_ref_cnt = filter1->l2_ref_cnt;
1027 	filter->flow_id = filter1->flow_id;
1028 	PMD_DRV_LOG(DEBUG,
1029 		"l2_filter: %p fw_l2_filter_id %" PRIx64 " l2_ref_cnt %u\n",
1030 		filter1, filter->fw_l2_filter_id, filter->l2_ref_cnt);
1031 }
1032 
1033 static int
1034 bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
1035 			     const struct rte_flow_item pattern[],
1036 			     const struct rte_flow_action actions[],
1037 			     const struct rte_flow_attr *attr,
1038 			     struct rte_flow_error *error,
1039 			     struct bnxt_filter_info *filter)
1040 {
1041 	const struct rte_flow_action *act =
1042 		bnxt_flow_non_void_action(actions);
1043 	struct bnxt *bp = dev->data->dev_private;
1044 	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
1045 	struct bnxt_vnic_info *vnic = NULL, *vnic0 = NULL;
1046 	const struct rte_flow_action_queue *act_q;
1047 	const struct rte_flow_action_vf *act_vf;
1048 	struct bnxt_filter_info *filter1 = NULL;
1049 	const struct rte_flow_action_rss *rss;
1050 	struct bnxt_rx_queue *rxq = NULL;
1051 	int dflt_vnic, vnic_id;
1052 	unsigned int rss_idx;
1053 	uint32_t vf = 0, i;
1054 	int rc, use_ntuple;
1055 
1056 	rc =
1057 	bnxt_validate_and_parse_flow_type(bp, attr, pattern, error, filter);
1058 	if (rc != 0)
1059 		goto ret;
1060 
1061 	rc = bnxt_flow_parse_attr(attr, error);
1062 	if (rc != 0)
1063 		goto ret;
1064 
1065 	/* Since we support ingress attribute only - right now. */
1066 	if (filter->filter_type == HWRM_CFA_EM_FILTER)
1067 		filter->flags = HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX;
1068 
1069 	use_ntuple = bnxt_filter_type_check(pattern, error);
1070 
1071 start:
1072 	switch (act->type) {
1073 	case RTE_FLOW_ACTION_TYPE_QUEUE:
1074 		/* Allow this flow. Redirect to a VNIC. */
1075 		act_q = (const struct rte_flow_action_queue *)act->conf;
1076 		if (!act_q->index || act_q->index >= bp->rx_nr_rings) {
1077 			rte_flow_error_set(error,
1078 					   EINVAL,
1079 					   RTE_FLOW_ERROR_TYPE_ACTION,
1080 					   act,
1081 					   "Invalid queue ID.");
1082 			rc = -rte_errno;
1083 			goto ret;
1084 		}
1085 		PMD_DRV_LOG(DEBUG, "Queue index %d\n", act_q->index);
1086 
1087 		if (use_ntuple && !BNXT_RFS_NEEDS_VNIC(bp)) {
1088 			filter->flags =
1089 				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DEST_RFS_RING_IDX;
1090 			filter->dst_id = act_q->index;
1091 			goto skip_vnic_alloc;
1092 		}
1093 
1094 		vnic_id = attr->group;
1095 		if (!vnic_id) {
1096 			PMD_DRV_LOG(DEBUG, "Group id is 0\n");
1097 			vnic_id = act_q->index;
1098 		}
1099 
1100 		BNXT_VALID_VNIC_OR_RET(bp, vnic_id);
1101 
1102 		vnic = &bp->vnic_info[vnic_id];
1103 		if (vnic->rx_queue_cnt) {
1104 			if (vnic->start_grp_id != act_q->index) {
1105 				PMD_DRV_LOG(ERR,
1106 					    "VNIC already in use\n");
1107 				rte_flow_error_set(error,
1108 						   EINVAL,
1109 						   RTE_FLOW_ERROR_TYPE_ACTION,
1110 						   act,
1111 						   "VNIC already in use");
1112 				rc = -rte_errno;
1113 				goto ret;
1114 			}
1115 			goto use_vnic;
1116 		}
1117 
1118 		rxq = bp->rx_queues[act_q->index];
1119 
1120 		if (!(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS) && rxq &&
1121 		    vnic->fw_vnic_id != INVALID_HW_RING_ID)
1122 			goto use_vnic;
1123 
1124 		if (!rxq) {
1125 			PMD_DRV_LOG(ERR,
1126 				    "Queue invalid or used with other VNIC\n");
1127 			rte_flow_error_set(error,
1128 					   EINVAL,
1129 					   RTE_FLOW_ERROR_TYPE_ACTION,
1130 					   act,
1131 					   "Queue invalid queue or in use");
1132 			rc = -rte_errno;
1133 			goto ret;
1134 		}
1135 
1136 		rxq->vnic = vnic;
1137 		rxq->rx_started = 1;
1138 		vnic->rx_queue_cnt++;
1139 		vnic->start_grp_id = act_q->index;
1140 		vnic->end_grp_id = act_q->index;
1141 		vnic->func_default = 0;	//This is not a default VNIC.
1142 
1143 		PMD_DRV_LOG(DEBUG, "VNIC found\n");
1144 
1145 		rc = bnxt_vnic_prep(bp, vnic);
1146 		if (rc)  {
1147 			rte_flow_error_set(error,
1148 					   EINVAL,
1149 					   RTE_FLOW_ERROR_TYPE_ACTION,
1150 					   act,
1151 					   "VNIC prep fail");
1152 			rc = -rte_errno;
1153 			goto ret;
1154 		}
1155 
1156 		PMD_DRV_LOG(DEBUG,
1157 			    "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
1158 			    act_q->index, vnic, vnic->fw_grp_ids);
1159 
1160 use_vnic:
1161 		vnic->ff_pool_idx = vnic_id;
1162 		PMD_DRV_LOG(DEBUG,
1163 			    "Setting vnic ff_idx %d\n", vnic->ff_pool_idx);
1164 		filter->dst_id = vnic->fw_vnic_id;
1165 skip_vnic_alloc:
1166 		/* For ntuple filter, create the L2 filter with default VNIC.
1167 		 * The user specified redirect queue will be set while creating
1168 		 * the ntuple filter in hardware.
1169 		 */
1170 		vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
1171 		if (use_ntuple)
1172 			filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
1173 		else
1174 			filter1 = bnxt_get_l2_filter(bp, filter, vnic);
1175 		if (filter1 == NULL) {
1176 			rte_flow_error_set(error,
1177 					   ENOSPC,
1178 					   RTE_FLOW_ERROR_TYPE_ACTION,
1179 					   act,
1180 					   "Filter not available");
1181 			rc = -rte_errno;
1182 			goto ret;
1183 		}
1184 
1185 		PMD_DRV_LOG(DEBUG, "new fltr: %p l2fltr: %p l2_ref_cnt: %d\n",
1186 			    filter, filter1, filter1->l2_ref_cnt);
1187 		bnxt_update_filter_flags_en(filter, filter1, use_ntuple);
1188 		break;
1189 	case RTE_FLOW_ACTION_TYPE_DROP:
1190 		vnic0 = &bp->vnic_info[0];
1191 		filter->dst_id = vnic0->fw_vnic_id;
1192 		filter->valid_flags |= BNXT_FLOW_L2_DROP_FLAG;
1193 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
1194 		if (filter1 == NULL) {
1195 			rte_flow_error_set(error,
1196 					   ENOSPC,
1197 					   RTE_FLOW_ERROR_TYPE_ACTION,
1198 					   act,
1199 					   "Filter not available");
1200 			rc = -rte_errno;
1201 			goto ret;
1202 		}
1203 
1204 		if (filter->filter_type == HWRM_CFA_EM_FILTER)
1205 			filter->flags =
1206 				HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP;
1207 		else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
1208 			filter->flags =
1209 				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
1210 
1211 		bnxt_update_filter_flags_en(filter, filter1, use_ntuple);
1212 		break;
1213 	case RTE_FLOW_ACTION_TYPE_COUNT:
1214 		vnic0 = &bp->vnic_info[0];
1215 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
1216 		if (filter1 == NULL) {
1217 			rte_flow_error_set(error,
1218 					   ENOSPC,
1219 					   RTE_FLOW_ERROR_TYPE_ACTION,
1220 					   act,
1221 					   "New filter not available");
1222 			rc = -rte_errno;
1223 			goto ret;
1224 		}
1225 
1226 		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
1227 		filter->flow_id = filter1->flow_id;
1228 		filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER;
1229 		break;
1230 	case RTE_FLOW_ACTION_TYPE_VF:
1231 		act_vf = (const struct rte_flow_action_vf *)act->conf;
1232 		vf = act_vf->id;
1233 
1234 		if (filter->tunnel_type ==
1235 		    CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN ||
1236 		    filter->tunnel_type ==
1237 		    CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE) {
1238 			/* If issued on a VF, ensure id is 0 and is trusted */
1239 			if (BNXT_VF(bp)) {
1240 				if (!BNXT_VF_IS_TRUSTED(bp) || vf) {
1241 					rte_flow_error_set(error, EINVAL,
1242 						RTE_FLOW_ERROR_TYPE_ACTION,
1243 						act,
1244 						"Incorrect VF");
1245 					rc = -rte_errno;
1246 					goto ret;
1247 				}
1248 			}
1249 
1250 			filter->enables |= filter->tunnel_type;
1251 			filter->filter_type = HWRM_CFA_TUNNEL_REDIRECT_FILTER;
1252 			goto done;
1253 		}
1254 
1255 		if (vf >= bp->pdev->max_vfs) {
1256 			rte_flow_error_set(error,
1257 					   EINVAL,
1258 					   RTE_FLOW_ERROR_TYPE_ACTION,
1259 					   act,
1260 					   "Incorrect VF id!");
1261 			rc = -rte_errno;
1262 			goto ret;
1263 		}
1264 
1265 		filter->mirror_vnic_id =
1266 		dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
1267 		if (dflt_vnic < 0) {
1268 			/* This simply indicates there's no driver loaded.
1269 			 * This is not an error.
1270 			 */
1271 			rte_flow_error_set(error,
1272 					   EINVAL,
1273 					   RTE_FLOW_ERROR_TYPE_ACTION,
1274 					   act,
1275 					   "Unable to get default VNIC for VF");
1276 			rc = -rte_errno;
1277 			goto ret;
1278 		}
1279 
1280 		filter->mirror_vnic_id = dflt_vnic;
1281 		filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
1282 
1283 		vnic0 = &bp->vnic_info[0];
1284 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
1285 		if (filter1 == NULL) {
1286 			rte_flow_error_set(error,
1287 					   ENOSPC,
1288 					   RTE_FLOW_ERROR_TYPE_ACTION,
1289 					   act,
1290 					   "New filter not available");
1291 			rc = -rte_errno;
1292 			goto ret;
1293 		}
1294 
1295 		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
1296 		filter->flow_id = filter1->flow_id;
1297 		break;
1298 	case RTE_FLOW_ACTION_TYPE_RSS:
1299 		rss = (const struct rte_flow_action_rss *)act->conf;
1300 
1301 		vnic_id = attr->group;
1302 
1303 		BNXT_VALID_VNIC_OR_RET(bp, vnic_id);
1304 		vnic = &bp->vnic_info[vnic_id];
1305 
1306 		/* Check if requested RSS config matches RSS config of VNIC
1307 		 * only if it is not a fresh VNIC configuration.
1308 		 * Otherwise the existing VNIC configuration can be used.
1309 		 */
1310 		if (vnic->rx_queue_cnt) {
1311 			rc = match_vnic_rss_cfg(bp, vnic, rss);
1312 			if (rc) {
1313 				PMD_DRV_LOG(ERR,
1314 					    "VNIC and RSS config mismatch\n");
1315 				rte_flow_error_set(error,
1316 						   EINVAL,
1317 						   RTE_FLOW_ERROR_TYPE_ACTION,
1318 						   act,
1319 						   "VNIC and RSS cfg mismatch");
1320 				rc = -rte_errno;
1321 				goto ret;
1322 			}
1323 			goto vnic_found;
1324 		}
1325 
1326 		for (i = 0; i < rss->queue_num; i++) {
1327 			PMD_DRV_LOG(DEBUG, "RSS action Queue %d\n",
1328 				    rss->queue[i]);
1329 
1330 			if (!rss->queue[i] ||
1331 			    rss->queue[i] >= bp->rx_nr_rings ||
1332 			    !bp->rx_queues[rss->queue[i]]) {
1333 				rte_flow_error_set(error,
1334 						   EINVAL,
1335 						   RTE_FLOW_ERROR_TYPE_ACTION,
1336 						   act,
1337 						   "Invalid queue ID for RSS");
1338 				rc = -rte_errno;
1339 				goto ret;
1340 			}
1341 			rxq = bp->rx_queues[rss->queue[i]];
1342 
1343 			if (bp->vnic_info[0].fw_grp_ids[rss->queue[i]] !=
1344 			    INVALID_HW_RING_ID) {
1345 				PMD_DRV_LOG(ERR,
1346 					    "queue active with other VNIC\n");
1347 				rte_flow_error_set(error,
1348 						   EINVAL,
1349 						   RTE_FLOW_ERROR_TYPE_ACTION,
1350 						   act,
1351 						   "Invalid queue ID for RSS");
1352 				rc = -rte_errno;
1353 				goto ret;
1354 			}
1355 
1356 			rxq->vnic = vnic;
1357 			rxq->rx_started = 1;
1358 			vnic->rx_queue_cnt++;
1359 		}
1360 
1361 		vnic->start_grp_id = rss->queue[0];
1362 		vnic->end_grp_id = rss->queue[rss->queue_num - 1];
1363 		vnic->func_default = 0;	//This is not a default VNIC.
1364 
1365 		rc = bnxt_vnic_prep(bp, vnic);
1366 		if (rc) {
1367 			rte_flow_error_set(error,
1368 					   EINVAL,
1369 					   RTE_FLOW_ERROR_TYPE_ACTION,
1370 					   act,
1371 					   "VNIC prep fail");
1372 			rc = -rte_errno;
1373 			goto ret;
1374 		}
1375 
1376 		PMD_DRV_LOG(DEBUG,
1377 			    "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
1378 			    vnic_id, vnic, vnic->fw_grp_ids);
1379 
1380 		vnic->ff_pool_idx = vnic_id;
1381 		PMD_DRV_LOG(DEBUG,
1382 			    "Setting vnic ff_pool_idx %d\n", vnic->ff_pool_idx);
1383 
1384 		/* This can be done only after vnic_grp_alloc is done. */
1385 		for (i = 0; i < vnic->rx_queue_cnt; i++) {
1386 			vnic->fw_grp_ids[i] =
1387 				bp->grp_info[rss->queue[i]].fw_grp_id;
1388 			/* Make sure vnic0 does not use these rings. */
1389 			bp->vnic_info[0].fw_grp_ids[rss->queue[i]] =
1390 				INVALID_HW_RING_ID;
1391 		}
1392 
1393 		for (rss_idx = 0; rss_idx < HW_HASH_INDEX_SIZE; ) {
1394 			for (i = 0; i < vnic->rx_queue_cnt; i++)
1395 				vnic->rss_table[rss_idx++] =
1396 					vnic->fw_grp_ids[i];
1397 		}
1398 
1399 		/* Configure RSS only if the queue count is > 1 */
1400 		if (vnic->rx_queue_cnt > 1) {
1401 			vnic->hash_type =
1402 				bnxt_rte_to_hwrm_hash_types(rss->types);
1403 			vnic->hash_mode =
1404 			bnxt_rte_to_hwrm_hash_level(bp, rss->types, rss->level);
1405 
1406 			if (!rss->key_len) {
1407 				/* If hash key has not been specified,
1408 				 * use random hash key.
1409 				 */
1410 				bnxt_prandom_bytes(vnic->rss_hash_key,
1411 						   HW_HASH_KEY_SIZE);
1412 			} else {
1413 				if (rss->key_len > HW_HASH_KEY_SIZE)
1414 					memcpy(vnic->rss_hash_key,
1415 					       rss->key,
1416 					       HW_HASH_KEY_SIZE);
1417 				else
1418 					memcpy(vnic->rss_hash_key,
1419 					       rss->key,
1420 					       rss->key_len);
1421 			}
1422 			bnxt_hwrm_vnic_rss_cfg(bp, vnic);
1423 		} else {
1424 			PMD_DRV_LOG(DEBUG, "No RSS config required\n");
1425 		}
1426 
1427 vnic_found:
1428 		filter->dst_id = vnic->fw_vnic_id;
1429 		filter1 = bnxt_get_l2_filter(bp, filter, vnic);
1430 		if (filter1 == NULL) {
1431 			rte_flow_error_set(error,
1432 					   ENOSPC,
1433 					   RTE_FLOW_ERROR_TYPE_ACTION,
1434 					   act,
1435 					   "New filter not available");
1436 			rc = -rte_errno;
1437 			goto ret;
1438 		}
1439 
1440 		PMD_DRV_LOG(DEBUG, "L2 filter created\n");
1441 		bnxt_update_filter_flags_en(filter, filter1, use_ntuple);
1442 		break;
1443 	case RTE_FLOW_ACTION_TYPE_MARK:
1444 		if (bp->mark_table == NULL) {
1445 			rte_flow_error_set(error,
1446 					   ENOMEM,
1447 					   RTE_FLOW_ERROR_TYPE_ACTION,
1448 					   act,
1449 					   "Mark table not allocated.");
1450 			rc = -rte_errno;
1451 			goto ret;
1452 		}
1453 
1454 		if (bp->flags & BNXT_FLAG_RX_VECTOR_PKT_MODE) {
1455 			PMD_DRV_LOG(DEBUG,
1456 				    "Disabling vector processing for mark\n");
1457 			bp->eth_dev->rx_pkt_burst = bnxt_recv_pkts;
1458 			bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1459 		}
1460 
1461 		filter->valid_flags |= BNXT_FLOW_MARK_FLAG;
1462 		filter->mark = ((const struct rte_flow_action_mark *)
1463 				act->conf)->id;
1464 		PMD_DRV_LOG(DEBUG, "Mark the flow %d\n", filter->mark);
1465 		break;
1466 	default:
1467 		rte_flow_error_set(error,
1468 				   EINVAL,
1469 				   RTE_FLOW_ERROR_TYPE_ACTION,
1470 				   act,
1471 				   "Invalid action.");
1472 		rc = -rte_errno;
1473 		goto ret;
1474 	}
1475 
1476 done:
1477 	act = bnxt_flow_non_void_action(++act);
1478 	while (act->type != RTE_FLOW_ACTION_TYPE_END)
1479 		goto start;
1480 
1481 	return rc;
1482 ret:
1483 
1484 	if (filter1) {
1485 		bnxt_hwrm_clear_l2_filter(bp, filter1);
1486 		bnxt_free_filter(bp, filter1);
1487 	}
1488 
1489 	if (rte_errno)  {
1490 		if (vnic && STAILQ_EMPTY(&vnic->filter))
1491 			vnic->rx_queue_cnt = 0;
1492 
1493 		if (rxq && !vnic->rx_queue_cnt)
1494 			rxq->vnic = &bp->vnic_info[0];
1495 	}
1496 	return -rte_errno;
1497 }
1498 
1499 static
1500 struct bnxt_vnic_info *find_matching_vnic(struct bnxt *bp,
1501 					  struct bnxt_filter_info *filter)
1502 {
1503 	struct bnxt_vnic_info *vnic = NULL;
1504 	unsigned int i;
1505 
1506 	for (i = 0; i < bp->max_vnics; i++) {
1507 		vnic = &bp->vnic_info[i];
1508 		if (vnic->fw_vnic_id != INVALID_VNIC_ID &&
1509 		    filter->dst_id == vnic->fw_vnic_id) {
1510 			PMD_DRV_LOG(DEBUG, "Found matching VNIC Id %d\n",
1511 				    vnic->ff_pool_idx);
1512 			return vnic;
1513 		}
1514 	}
1515 	return NULL;
1516 }
1517 
1518 static int
1519 bnxt_flow_validate(struct rte_eth_dev *dev,
1520 		   const struct rte_flow_attr *attr,
1521 		   const struct rte_flow_item pattern[],
1522 		   const struct rte_flow_action actions[],
1523 		   struct rte_flow_error *error)
1524 {
1525 	struct bnxt *bp = dev->data->dev_private;
1526 	struct bnxt_vnic_info *vnic = NULL;
1527 	struct bnxt_filter_info *filter;
1528 	int ret = 0;
1529 
1530 	bnxt_acquire_flow_lock(bp);
1531 	ret = bnxt_flow_args_validate(attr, pattern, actions, error);
1532 	if (ret != 0) {
1533 		bnxt_release_flow_lock(bp);
1534 		return ret;
1535 	}
1536 
1537 	filter = bnxt_get_unused_filter(bp);
1538 	if (filter == NULL) {
1539 		PMD_DRV_LOG(ERR, "Not enough resources for a new flow.\n");
1540 		bnxt_release_flow_lock(bp);
1541 		return -ENOMEM;
1542 	}
1543 
1544 	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
1545 					   error, filter);
1546 	if (ret)
1547 		goto exit;
1548 
1549 	vnic = find_matching_vnic(bp, filter);
1550 	if (vnic) {
1551 		if (STAILQ_EMPTY(&vnic->filter)) {
1552 			rte_free(vnic->fw_grp_ids);
1553 			bnxt_hwrm_vnic_ctx_free(bp, vnic);
1554 			bnxt_hwrm_vnic_free(bp, vnic);
1555 			vnic->rx_queue_cnt = 0;
1556 			bp->nr_vnics--;
1557 			PMD_DRV_LOG(DEBUG, "Free VNIC\n");
1558 		}
1559 	}
1560 
1561 	if (filter->filter_type == HWRM_CFA_EM_FILTER)
1562 		bnxt_hwrm_clear_em_filter(bp, filter);
1563 	else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
1564 		bnxt_hwrm_clear_ntuple_filter(bp, filter);
1565 	else
1566 		bnxt_hwrm_clear_l2_filter(bp, filter);
1567 
1568 exit:
1569 	/* No need to hold on to this filter if we are just validating flow */
1570 	bnxt_free_filter(bp, filter);
1571 	bnxt_release_flow_lock(bp);
1572 
1573 	return ret;
1574 }
1575 
1576 static void
1577 bnxt_update_filter(struct bnxt *bp, struct bnxt_filter_info *old_filter,
1578 		   struct bnxt_filter_info *new_filter)
1579 {
1580 	/* Clear the new L2 filter that was created in the previous step in
1581 	 * bnxt_validate_and_parse_flow. For L2 filters, we will use the new
1582 	 * filter which points to the new destination queue and so we clear
1583 	 * the previous L2 filter. For ntuple filters, we are going to reuse
1584 	 * the old L2 filter and create new NTUPLE filter with this new
1585 	 * destination queue subsequently during bnxt_flow_create. So we
1586 	 * decrement the ref cnt of the L2 filter that would've been bumped
1587 	 * up previously in bnxt_validate_and_parse_flow as the old n-tuple
1588 	 * filter that was referencing it will be deleted now.
1589 	 */
1590 	bnxt_hwrm_clear_l2_filter(bp, old_filter);
1591 	if (new_filter->filter_type == HWRM_CFA_L2_FILTER) {
1592 		bnxt_hwrm_set_l2_filter(bp, new_filter->dst_id, new_filter);
1593 	} else {
1594 		if (new_filter->filter_type == HWRM_CFA_EM_FILTER)
1595 			bnxt_hwrm_clear_em_filter(bp, old_filter);
1596 		if (new_filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
1597 			bnxt_hwrm_clear_ntuple_filter(bp, old_filter);
1598 	}
1599 }
1600 
1601 static int
1602 bnxt_match_filter(struct bnxt *bp, struct bnxt_filter_info *nf)
1603 {
1604 	struct bnxt_filter_info *mf;
1605 	struct rte_flow *flow;
1606 	int i;
1607 
1608 	for (i = bp->max_vnics - 1; i >= 0; i--) {
1609 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
1610 
1611 		if (vnic->fw_vnic_id == INVALID_VNIC_ID)
1612 			continue;
1613 
1614 		STAILQ_FOREACH(flow, &vnic->flow_list, next) {
1615 			mf = flow->filter;
1616 
1617 			if (mf->filter_type == nf->filter_type &&
1618 			    mf->flags == nf->flags &&
1619 			    mf->src_port == nf->src_port &&
1620 			    mf->src_port_mask == nf->src_port_mask &&
1621 			    mf->dst_port == nf->dst_port &&
1622 			    mf->dst_port_mask == nf->dst_port_mask &&
1623 			    mf->ip_protocol == nf->ip_protocol &&
1624 			    mf->ip_addr_type == nf->ip_addr_type &&
1625 			    mf->ethertype == nf->ethertype &&
1626 			    mf->vni == nf->vni &&
1627 			    mf->tunnel_type == nf->tunnel_type &&
1628 			    mf->l2_ovlan == nf->l2_ovlan &&
1629 			    mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
1630 			    mf->l2_ivlan == nf->l2_ivlan &&
1631 			    mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
1632 			    !memcmp(mf->l2_addr, nf->l2_addr,
1633 				    RTE_ETHER_ADDR_LEN) &&
1634 			    !memcmp(mf->l2_addr_mask, nf->l2_addr_mask,
1635 				    RTE_ETHER_ADDR_LEN) &&
1636 			    !memcmp(mf->src_macaddr, nf->src_macaddr,
1637 				    RTE_ETHER_ADDR_LEN) &&
1638 			    !memcmp(mf->dst_macaddr, nf->dst_macaddr,
1639 				    RTE_ETHER_ADDR_LEN) &&
1640 			    !memcmp(mf->src_ipaddr, nf->src_ipaddr,
1641 				    sizeof(nf->src_ipaddr)) &&
1642 			    !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask,
1643 				    sizeof(nf->src_ipaddr_mask)) &&
1644 			    !memcmp(mf->dst_ipaddr, nf->dst_ipaddr,
1645 				    sizeof(nf->dst_ipaddr)) &&
1646 			    !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask,
1647 				    sizeof(nf->dst_ipaddr_mask))) {
1648 				if (mf->dst_id == nf->dst_id)
1649 					return -EEXIST;
1650 				/* Free the old filter, update flow
1651 				 * with new filter
1652 				 */
1653 				bnxt_update_filter(bp, mf, nf);
1654 				STAILQ_REMOVE(&vnic->filter, mf,
1655 					      bnxt_filter_info, next);
1656 				STAILQ_INSERT_TAIL(&vnic->filter, nf, next);
1657 				bnxt_free_filter(bp, mf);
1658 				flow->filter = nf;
1659 				return -EXDEV;
1660 			}
1661 		}
1662 	}
1663 	return 0;
1664 }
1665 
1666 static void
1667 bnxt_setup_flow_counter(struct bnxt *bp)
1668 {
1669 	if (bp->fw_cap & BNXT_FW_CAP_ADV_FLOW_COUNTERS &&
1670 	    !(bp->flags & BNXT_FLAG_FC_THREAD) && BNXT_FLOW_XSTATS_EN(bp)) {
1671 		rte_eal_alarm_set(US_PER_S * BNXT_FC_TIMER,
1672 				  bnxt_flow_cnt_alarm_cb,
1673 				  (void *)bp);
1674 		bp->flags |= BNXT_FLAG_FC_THREAD;
1675 	}
1676 }
1677 
1678 void bnxt_flow_cnt_alarm_cb(void *arg)
1679 {
1680 	int rc = 0;
1681 	struct bnxt *bp = arg;
1682 
1683 	if (!bp->flow_stat->rx_fc_out_tbl.va) {
1684 		PMD_DRV_LOG(ERR, "bp->flow_stat->rx_fc_out_tbl.va is NULL?\n");
1685 		bnxt_cancel_fc_thread(bp);
1686 		return;
1687 	}
1688 
1689 	if (!bp->flow_stat->flow_count) {
1690 		bnxt_cancel_fc_thread(bp);
1691 		return;
1692 	}
1693 
1694 	if (!bp->eth_dev->data->dev_started) {
1695 		bnxt_cancel_fc_thread(bp);
1696 		return;
1697 	}
1698 
1699 	rc = bnxt_flow_stats_req(bp);
1700 	if (rc) {
1701 		PMD_DRV_LOG(ERR, "Flow stat alarm not rescheduled.\n");
1702 		return;
1703 	}
1704 
1705 	rte_eal_alarm_set(US_PER_S * BNXT_FC_TIMER,
1706 			  bnxt_flow_cnt_alarm_cb,
1707 			  (void *)bp);
1708 }
1709 
1710 
1711 static struct rte_flow *
1712 bnxt_flow_create(struct rte_eth_dev *dev,
1713 		 const struct rte_flow_attr *attr,
1714 		 const struct rte_flow_item pattern[],
1715 		 const struct rte_flow_action actions[],
1716 		 struct rte_flow_error *error)
1717 {
1718 	struct bnxt *bp = dev->data->dev_private;
1719 	struct bnxt_vnic_info *vnic = NULL;
1720 	struct bnxt_filter_info *filter;
1721 	bool update_flow = false;
1722 	struct rte_flow *flow;
1723 	int ret = 0;
1724 	uint32_t tun_type, flow_id;
1725 
1726 	if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp)) {
1727 		rte_flow_error_set(error, EINVAL,
1728 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1729 				   "Failed to create flow, Not a Trusted VF!");
1730 		return NULL;
1731 	}
1732 
1733 	if (!dev->data->dev_started) {
1734 		rte_flow_error_set(error,
1735 				   EINVAL,
1736 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1737 				   NULL,
1738 				   "Device must be started");
1739 		return NULL;
1740 	}
1741 
1742 	flow = rte_zmalloc("bnxt_flow", sizeof(struct rte_flow), 0);
1743 	if (!flow) {
1744 		rte_flow_error_set(error, ENOMEM,
1745 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1746 				   "Failed to allocate memory");
1747 		return flow;
1748 	}
1749 
1750 	bnxt_acquire_flow_lock(bp);
1751 	ret = bnxt_flow_args_validate(attr, pattern, actions, error);
1752 	if (ret != 0) {
1753 		PMD_DRV_LOG(ERR, "Not a validate flow.\n");
1754 		goto free_flow;
1755 	}
1756 
1757 	filter = bnxt_get_unused_filter(bp);
1758 	if (filter == NULL) {
1759 		rte_flow_error_set(error, ENOSPC,
1760 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1761 				   "Not enough resources for a new flow");
1762 		goto free_flow;
1763 	}
1764 
1765 	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
1766 					   error, filter);
1767 	if (ret != 0)
1768 		goto free_filter;
1769 
1770 	ret = bnxt_match_filter(bp, filter);
1771 	if (ret == -EEXIST) {
1772 		PMD_DRV_LOG(DEBUG, "Flow already exists.\n");
1773 		/* Clear the filter that was created as part of
1774 		 * validate_and_parse_flow() above
1775 		 */
1776 		bnxt_hwrm_clear_l2_filter(bp, filter);
1777 		goto free_filter;
1778 	} else if (ret == -EXDEV) {
1779 		PMD_DRV_LOG(DEBUG, "Flow with same pattern exists\n");
1780 		PMD_DRV_LOG(DEBUG, "Updating with different destination\n");
1781 		update_flow = true;
1782 	}
1783 
1784 	/* If tunnel redirection to a VF/PF is specified then only tunnel_type
1785 	 * is set and enable is set to the tunnel type. Issue hwrm cmd directly
1786 	 * in such a case.
1787 	 */
1788 	if (filter->filter_type == HWRM_CFA_TUNNEL_REDIRECT_FILTER &&
1789 	    filter->enables == filter->tunnel_type) {
1790 		ret = bnxt_hwrm_tunnel_redirect_query(bp, &tun_type);
1791 		if (ret) {
1792 			rte_flow_error_set(error, -ret,
1793 					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1794 					   "Unable to query tunnel to VF");
1795 			goto free_filter;
1796 		}
1797 		if (tun_type == (1U << filter->tunnel_type)) {
1798 			ret =
1799 			bnxt_hwrm_tunnel_redirect_free(bp,
1800 						       filter->tunnel_type);
1801 			if (ret) {
1802 				PMD_DRV_LOG(ERR,
1803 					    "Unable to free existing tunnel\n");
1804 				rte_flow_error_set(error, -ret,
1805 						   RTE_FLOW_ERROR_TYPE_HANDLE,
1806 						   NULL,
1807 						   "Unable to free preexisting "
1808 						   "tunnel on VF");
1809 				goto free_filter;
1810 			}
1811 		}
1812 		ret = bnxt_hwrm_tunnel_redirect(bp, filter->tunnel_type);
1813 		if (ret) {
1814 			rte_flow_error_set(error, -ret,
1815 					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1816 					   "Unable to redirect tunnel to VF");
1817 			goto free_filter;
1818 		}
1819 		vnic = &bp->vnic_info[0];
1820 		goto done;
1821 	}
1822 
1823 	if (filter->filter_type == HWRM_CFA_EM_FILTER) {
1824 		filter->enables |=
1825 			HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
1826 		ret = bnxt_hwrm_set_em_filter(bp, filter->dst_id, filter);
1827 		if (ret != 0) {
1828 			rte_flow_error_set(error, -ret,
1829 					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1830 					   "Failed to create EM filter");
1831 			goto free_filter;
1832 		}
1833 	}
1834 
1835 	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) {
1836 		filter->enables |=
1837 			HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
1838 		ret = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, filter);
1839 		if (ret != 0) {
1840 			rte_flow_error_set(error, -ret,
1841 					   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1842 					   "Failed to create ntuple filter");
1843 			goto free_filter;
1844 		}
1845 	}
1846 
1847 	if (BNXT_RFS_NEEDS_VNIC(bp))
1848 		vnic = find_matching_vnic(bp, filter);
1849 	else
1850 		vnic = BNXT_GET_DEFAULT_VNIC(bp);
1851 done:
1852 	if (!ret || update_flow) {
1853 		flow->filter = filter;
1854 		flow->vnic = vnic;
1855 		if (update_flow) {
1856 			ret = -EXDEV;
1857 			goto free_flow;
1858 		}
1859 
1860 		if (filter->valid_flags & BNXT_FLOW_MARK_FLAG) {
1861 			PMD_DRV_LOG(DEBUG,
1862 				    "Mark action: mark id 0x%x, flow id 0x%x\n",
1863 				    filter->mark, filter->flow_id);
1864 
1865 			/* TCAM and EM should be 16-bit only.
1866 			 * Other modes not supported.
1867 			 */
1868 			flow_id = filter->flow_id & BNXT_FLOW_ID_MASK;
1869 			if (bp->mark_table[flow_id].valid) {
1870 				rte_flow_error_set(error, EEXIST,
1871 						   RTE_FLOW_ERROR_TYPE_HANDLE,
1872 						   NULL,
1873 						   "Flow with mark id exists");
1874 				bnxt_clear_one_vnic_filter(bp, filter);
1875 				goto free_filter;
1876 			}
1877 			bp->mark_table[flow_id].valid = true;
1878 			bp->mark_table[flow_id].mark_id = filter->mark;
1879 		}
1880 
1881 		STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1882 		STAILQ_INSERT_TAIL(&vnic->flow_list, flow, next);
1883 
1884 		if (BNXT_FLOW_XSTATS_EN(bp))
1885 			bp->flow_stat->flow_count++;
1886 		bnxt_release_flow_lock(bp);
1887 		bnxt_setup_flow_counter(bp);
1888 		PMD_DRV_LOG(DEBUG, "Successfully created flow.\n");
1889 		return flow;
1890 	}
1891 
1892 free_filter:
1893 	bnxt_free_filter(bp, filter);
1894 free_flow:
1895 	if (ret == -EEXIST)
1896 		rte_flow_error_set(error, ret,
1897 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1898 				   "Matching Flow exists.");
1899 	else if (ret == -EXDEV)
1900 		rte_flow_error_set(error, 0,
1901 				   RTE_FLOW_ERROR_TYPE_NONE, NULL,
1902 				   "Flow with pattern exists, updating destination queue");
1903 	else if (!rte_errno)
1904 		rte_flow_error_set(error, -ret,
1905 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1906 				   "Failed to create flow.");
1907 	rte_free(flow);
1908 	flow = NULL;
1909 	bnxt_release_flow_lock(bp);
1910 	return flow;
1911 }
1912 
1913 static int bnxt_handle_tunnel_redirect_destroy(struct bnxt *bp,
1914 					       struct bnxt_filter_info *filter,
1915 					       struct rte_flow_error *error)
1916 {
1917 	uint16_t tun_dst_fid;
1918 	uint32_t tun_type;
1919 	int ret = 0;
1920 
1921 	ret = bnxt_hwrm_tunnel_redirect_query(bp, &tun_type);
1922 	if (ret) {
1923 		rte_flow_error_set(error, -ret,
1924 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1925 				   "Unable to query tunnel to VF");
1926 		return ret;
1927 	}
1928 	if (tun_type == (1U << filter->tunnel_type)) {
1929 		ret = bnxt_hwrm_tunnel_redirect_info(bp, filter->tunnel_type,
1930 						     &tun_dst_fid);
1931 		if (ret) {
1932 			rte_flow_error_set(error, -ret,
1933 					   RTE_FLOW_ERROR_TYPE_HANDLE,
1934 					   NULL,
1935 					   "tunnel_redirect info cmd fail");
1936 			return ret;
1937 		}
1938 		PMD_DRV_LOG(INFO, "Pre-existing tunnel fid = %x vf->fid = %x\n",
1939 			    tun_dst_fid + bp->first_vf_id, bp->fw_fid);
1940 
1941 		/* Tunnel doesn't belong to this VF, so don't send HWRM
1942 		 * cmd, just delete the flow from driver
1943 		 */
1944 		if (bp->fw_fid != (tun_dst_fid + bp->first_vf_id))
1945 			PMD_DRV_LOG(ERR,
1946 				    "Tunnel does not belong to this VF, skip hwrm_tunnel_redirect_free\n");
1947 		else
1948 			ret = bnxt_hwrm_tunnel_redirect_free(bp,
1949 							filter->tunnel_type);
1950 	}
1951 	return ret;
1952 }
1953 
1954 static int
1955 _bnxt_flow_destroy(struct bnxt *bp,
1956 		   struct rte_flow *flow,
1957 		    struct rte_flow_error *error)
1958 {
1959 	struct bnxt_filter_info *filter;
1960 	struct bnxt_vnic_info *vnic;
1961 	int ret = 0;
1962 	uint32_t flow_id;
1963 
1964 	filter = flow->filter;
1965 	vnic = flow->vnic;
1966 
1967 	if (filter->filter_type == HWRM_CFA_TUNNEL_REDIRECT_FILTER &&
1968 	    filter->enables == filter->tunnel_type) {
1969 		ret = bnxt_handle_tunnel_redirect_destroy(bp, filter, error);
1970 		if (!ret)
1971 			goto done;
1972 		else
1973 			return ret;
1974 	}
1975 
1976 	ret = bnxt_match_filter(bp, filter);
1977 	if (ret == 0)
1978 		PMD_DRV_LOG(ERR, "Could not find matching flow\n");
1979 
1980 	if (filter->valid_flags & BNXT_FLOW_MARK_FLAG) {
1981 		flow_id = filter->flow_id & BNXT_FLOW_ID_MASK;
1982 		memset(&bp->mark_table[flow_id], 0,
1983 		       sizeof(bp->mark_table[flow_id]));
1984 		filter->flow_id = 0;
1985 	}
1986 
1987 	ret = bnxt_clear_one_vnic_filter(bp, filter);
1988 
1989 done:
1990 	if (!ret) {
1991 		/* If it is a L2 drop filter, when the filter is created,
1992 		 * the FW updates the BC/MC records.
1993 		 * Once this filter is removed, issue the set_rx_mask command
1994 		 * to reset the BC/MC records in the HW to the settings
1995 		 * before the drop counter is created.
1996 		 */
1997 		if (filter->valid_flags & BNXT_FLOW_L2_DROP_FLAG)
1998 			bnxt_set_rx_mask_no_vlan(bp, &bp->vnic_info[0]);
1999 
2000 		STAILQ_REMOVE(&vnic->filter, filter, bnxt_filter_info, next);
2001 		bnxt_free_filter(bp, filter);
2002 		STAILQ_REMOVE(&vnic->flow_list, flow, rte_flow, next);
2003 		rte_free(flow);
2004 		if (BNXT_FLOW_XSTATS_EN(bp))
2005 			bp->flow_stat->flow_count--;
2006 
2007 		/* If this was the last flow associated with this vnic,
2008 		 * switch the queue back to RSS pool.
2009 		 */
2010 		if (vnic && !vnic->func_default &&
2011 		    STAILQ_EMPTY(&vnic->flow_list)) {
2012 			rte_free(vnic->fw_grp_ids);
2013 			if (vnic->rx_queue_cnt > 1)
2014 				bnxt_hwrm_vnic_ctx_free(bp, vnic);
2015 
2016 			bnxt_hwrm_vnic_free(bp, vnic);
2017 			vnic->rx_queue_cnt = 0;
2018 			bp->nr_vnics--;
2019 		}
2020 	} else {
2021 		rte_flow_error_set(error, -ret,
2022 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2023 				   "Failed to destroy flow.");
2024 	}
2025 
2026 	return ret;
2027 }
2028 
2029 static int
2030 bnxt_flow_destroy(struct rte_eth_dev *dev,
2031 		  struct rte_flow *flow,
2032 		  struct rte_flow_error *error)
2033 {
2034 	struct bnxt *bp = dev->data->dev_private;
2035 	int ret = 0;
2036 
2037 	bnxt_acquire_flow_lock(bp);
2038 	if (!flow) {
2039 		rte_flow_error_set(error, EINVAL,
2040 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2041 				   "Invalid flow: failed to destroy flow.");
2042 		bnxt_release_flow_lock(bp);
2043 		return -EINVAL;
2044 	}
2045 
2046 	if (!flow->filter) {
2047 		rte_flow_error_set(error, EINVAL,
2048 				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2049 				   "Invalid flow: failed to destroy flow.");
2050 		bnxt_release_flow_lock(bp);
2051 		return -EINVAL;
2052 	}
2053 	ret = _bnxt_flow_destroy(bp, flow, error);
2054 	bnxt_release_flow_lock(bp);
2055 
2056 	return ret;
2057 }
2058 
2059 void bnxt_cancel_fc_thread(struct bnxt *bp)
2060 {
2061 	bp->flags &= ~BNXT_FLAG_FC_THREAD;
2062 	rte_eal_alarm_cancel(bnxt_flow_cnt_alarm_cb, (void *)bp);
2063 }
2064 
2065 static int
2066 bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
2067 {
2068 	struct bnxt *bp = dev->data->dev_private;
2069 	struct bnxt_vnic_info *vnic;
2070 	struct rte_flow *flow;
2071 	unsigned int i;
2072 	int ret = 0;
2073 
2074 	bnxt_acquire_flow_lock(bp);
2075 	for (i = 0; i < bp->max_vnics; i++) {
2076 		vnic = &bp->vnic_info[i];
2077 		if (vnic && vnic->fw_vnic_id == INVALID_VNIC_ID)
2078 			continue;
2079 
2080 		while (!STAILQ_EMPTY(&vnic->flow_list)) {
2081 			flow = STAILQ_FIRST(&vnic->flow_list);
2082 
2083 			if (!flow->filter)
2084 				continue;
2085 
2086 			ret = _bnxt_flow_destroy(bp, flow, error);
2087 			if (ret)
2088 				break;
2089 		}
2090 	}
2091 
2092 	bnxt_cancel_fc_thread(bp);
2093 	bnxt_release_flow_lock(bp);
2094 
2095 	return ret;
2096 }
2097 
2098 const struct rte_flow_ops bnxt_flow_ops = {
2099 	.validate = bnxt_flow_validate,
2100 	.create = bnxt_flow_create,
2101 	.destroy = bnxt_flow_destroy,
2102 	.flush = bnxt_flow_flush,
2103 };
2104