1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
3 */
4
5 #include <sys/queue.h>
6 #include <stdio.h>
7 #include <errno.h>
8 #include <stdint.h>
9 #include <string.h>
10 #include <unistd.h>
11 #include <stdarg.h>
12
13 #include <rte_debug.h>
14 #include <rte_ether.h>
15 #include <rte_ethdev_driver.h>
16 #include <rte_log.h>
17 #include <rte_malloc.h>
18 #include <rte_eth_ctrl.h>
19 #include <rte_tailq.h>
20 #include <rte_flow_driver.h>
21
22 #include "iavf_log.h"
23 #include "iavf.h"
24 #include "iavf_generic_flow.h"
25
26 #define IAVF_PHINT_NONE 0
27 #define IAVF_PHINT_GTPU BIT_ULL(0)
28 #define IAVF_PHINT_GTPU_EH BIT_ULL(1)
29 #define IAVF_PHINT_GTPU_EH_DWN BIT_ULL(2)
30 #define IAVF_PHINT_GTPU_EH_UP BIT_ULL(3)
31 #define IAVF_PHINT_OUTER_IPV4 BIT_ULL(4)
32 #define IAVF_PHINT_OUTER_IPV6 BIT_ULL(5)
33
34 #define IAVF_PHINT_GTPU_MSK (IAVF_PHINT_GTPU | \
35 IAVF_PHINT_GTPU_EH | \
36 IAVF_PHINT_GTPU_EH_DWN | \
37 IAVF_PHINT_GTPU_EH_UP)
38
39 #define IAVF_PHINT_LAYERS_MSK (IAVF_PHINT_OUTER_IPV4 | \
40 IAVF_PHINT_OUTER_IPV6)
41
42 #define IAVF_GTPU_EH_DWNLINK 0
43 #define IAVF_GTPU_EH_UPLINK 1
44
45 struct iavf_hash_match_type {
46 uint64_t hash_type;
47 struct virtchnl_proto_hdrs *proto_hdrs;
48 uint64_t pattern_hint;
49 };
50
51 struct iavf_rss_meta {
52 struct virtchnl_proto_hdrs proto_hdrs;
53 enum virtchnl_rss_algorithm rss_algorithm;
54 };
55
56 struct iavf_hash_flow_cfg {
57 struct virtchnl_rss_cfg *rss_cfg;
58 bool simple_xor;
59 };
60
61 static int
62 iavf_hash_init(struct iavf_adapter *ad);
63 static int
64 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
65 struct rte_flow_error *error);
66 static int
67 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
68 struct rte_flow_error *error);
69 static void
70 iavf_hash_uninit(struct iavf_adapter *ad);
71 static void
72 iavf_hash_free(struct rte_flow *flow);
73 static int
74 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
75 struct iavf_pattern_match_item *array,
76 uint32_t array_len,
77 const struct rte_flow_item pattern[],
78 const struct rte_flow_action actions[],
79 void **meta,
80 struct rte_flow_error *error);
81
82 #define FIELD_SELECTOR(proto_hdr_field) \
83 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
84 #define BUFF_NOUSED 0
85
86 #define proto_hdr_eth { \
87 VIRTCHNL_PROTO_HDR_ETH, \
88 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
89 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
90
91 #define proto_hdr_svlan { \
92 VIRTCHNL_PROTO_HDR_S_VLAN, \
93 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
94
95 #define proto_hdr_cvlan { \
96 VIRTCHNL_PROTO_HDR_C_VLAN, \
97 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
98
99 #define proto_hdr_ipv4 { \
100 VIRTCHNL_PROTO_HDR_IPV4, \
101 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
102 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
103
104 #define proto_hdr_ipv4_with_prot { \
105 VIRTCHNL_PROTO_HDR_IPV4, \
106 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
107 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | \
108 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), {BUFF_NOUSED} }
109
110 #define proto_hdr_ipv6 { \
111 VIRTCHNL_PROTO_HDR_IPV6, \
112 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
113 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
114
115 #define proto_hdr_ipv6_with_prot { \
116 VIRTCHNL_PROTO_HDR_IPV6, \
117 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
118 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | \
119 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), {BUFF_NOUSED} }
120
121 #define proto_hdr_udp { \
122 VIRTCHNL_PROTO_HDR_UDP, \
123 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
124 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
125
126 #define proto_hdr_tcp { \
127 VIRTCHNL_PROTO_HDR_TCP, \
128 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
129 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
130
131 #define proto_hdr_sctp { \
132 VIRTCHNL_PROTO_HDR_SCTP, \
133 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
134 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
135
136 #define proto_hdr_esp { \
137 VIRTCHNL_PROTO_HDR_ESP, \
138 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
139
140 #define proto_hdr_ah { \
141 VIRTCHNL_PROTO_HDR_AH, \
142 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
143
144 #define proto_hdr_l2tpv3 { \
145 VIRTCHNL_PROTO_HDR_L2TPV3, \
146 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
147
148 #define proto_hdr_pfcp { \
149 VIRTCHNL_PROTO_HDR_PFCP, \
150 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
151
152 #define proto_hdr_gtpc { \
153 VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
154
155 #define TUNNEL_LEVEL_OUTER 0
156 #define TUNNEL_LEVEL_INNER 1
157
158 /* proto_hdrs template */
159 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
160 TUNNEL_LEVEL_OUTER, 4,
161 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
162 };
163
164 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
165 TUNNEL_LEVEL_OUTER, 5,
166 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
167 proto_hdr_ipv4_with_prot,
168 proto_hdr_udp}
169 };
170
171 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
172 TUNNEL_LEVEL_OUTER, 5,
173 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
174 proto_hdr_ipv4_with_prot,
175 proto_hdr_tcp}
176 };
177
178 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
179 TUNNEL_LEVEL_OUTER, 5,
180 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
181 proto_hdr_sctp}
182 };
183
184 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
185 TUNNEL_LEVEL_OUTER, 4,
186 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
187 };
188
189 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
190 TUNNEL_LEVEL_OUTER, 5,
191 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
192 proto_hdr_ipv6_with_prot,
193 proto_hdr_udp}
194 };
195
196 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
197 TUNNEL_LEVEL_OUTER, 5,
198 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
199 proto_hdr_ipv6_with_prot,
200 proto_hdr_tcp}
201 };
202
203 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
204 TUNNEL_LEVEL_OUTER, 5,
205 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
206 proto_hdr_sctp}
207 };
208
209 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
210 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
211 };
212
213 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
214 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
215 };
216
217 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
218 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
219 };
220
221 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
222 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
223 };
224
225 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
226 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
227 };
228
229 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
230 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
231 };
232
233 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
234 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
235 };
236
237 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
238 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
239 };
240
241 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
242 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
243 };
244
245 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
246 TUNNEL_LEVEL_OUTER, 3,
247 {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
248 };
249
250 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
251 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
252 };
253
254 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
255 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
256 };
257
258 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
259 TUNNEL_LEVEL_OUTER, 3,
260 {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
261 };
262
263 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
264 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
265 };
266
267 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
268 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
269 };
270
271 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
272 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
273 };
274
275 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
276 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
277 };
278
279 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
280 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
281 };
282
283 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
284 TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
285 };
286
287 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
288 TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
289 };
290
291 /* rss type super set */
292
293 /* IPv4 outer */
294 #define IAVF_RSS_TYPE_OUTER_IPV4 (ETH_RSS_ETH | ETH_RSS_IPV4)
295 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4 | \
296 ETH_RSS_NONFRAG_IPV4_UDP)
297 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4 | \
298 ETH_RSS_NONFRAG_IPV4_TCP)
299 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4 | \
300 ETH_RSS_NONFRAG_IPV4_SCTP)
301 /* IPv6 outer */
302 #define IAVF_RSS_TYPE_OUTER_IPV6 (ETH_RSS_ETH | ETH_RSS_IPV6)
303 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6 | \
304 ETH_RSS_NONFRAG_IPV6_UDP)
305 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6 | \
306 ETH_RSS_NONFRAG_IPV6_TCP)
307 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6 | \
308 ETH_RSS_NONFRAG_IPV6_SCTP)
309 /* VLAN IPV4 */
310 #define IAVF_RSS_TYPE_VLAN_IPV4 (IAVF_RSS_TYPE_OUTER_IPV4 | \
311 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
312 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
313 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
314 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
315 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
316 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
317 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
318 /* VLAN IPv6 */
319 #define IAVF_RSS_TYPE_VLAN_IPV6 (IAVF_RSS_TYPE_OUTER_IPV6 | \
320 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
321 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
322 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
323 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
324 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
325 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
326 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
327 /* IPv4 inner */
328 #define IAVF_RSS_TYPE_INNER_IPV4 ETH_RSS_IPV4
329 #define IAVF_RSS_TYPE_INNER_IPV4_UDP (ETH_RSS_IPV4 | \
330 ETH_RSS_NONFRAG_IPV4_UDP)
331 #define IAVF_RSS_TYPE_INNER_IPV4_TCP (ETH_RSS_IPV4 | \
332 ETH_RSS_NONFRAG_IPV4_TCP)
333 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP (ETH_RSS_IPV4 | \
334 ETH_RSS_NONFRAG_IPV4_SCTP)
335 /* IPv6 inner */
336 #define IAVF_RSS_TYPE_INNER_IPV6 ETH_RSS_IPV6
337 #define IAVF_RSS_TYPE_INNER_IPV6_UDP (ETH_RSS_IPV6 | \
338 ETH_RSS_NONFRAG_IPV6_UDP)
339 #define IAVF_RSS_TYPE_INNER_IPV6_TCP (ETH_RSS_IPV6 | \
340 ETH_RSS_NONFRAG_IPV6_TCP)
341 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP (ETH_RSS_IPV6 | \
342 ETH_RSS_NONFRAG_IPV6_SCTP)
343 /* GTPU IPv4 */
344 #define IAVF_RSS_TYPE_GTPU_IPV4 (IAVF_RSS_TYPE_INNER_IPV4 | \
345 ETH_RSS_GTPU)
346 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
347 ETH_RSS_GTPU)
348 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
349 ETH_RSS_GTPU)
350 /* GTPU IPv6 */
351 #define IAVF_RSS_TYPE_GTPU_IPV6 (IAVF_RSS_TYPE_INNER_IPV6 | \
352 ETH_RSS_GTPU)
353 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
354 ETH_RSS_GTPU)
355 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
356 ETH_RSS_GTPU)
357 /* ESP, AH, L2TPV3 and PFCP */
358 #define IAVF_RSS_TYPE_IPV4_ESP (ETH_RSS_ESP | ETH_RSS_IPV4)
359 #define IAVF_RSS_TYPE_IPV4_AH (ETH_RSS_AH | ETH_RSS_IPV4)
360 #define IAVF_RSS_TYPE_IPV6_ESP (ETH_RSS_ESP | ETH_RSS_IPV6)
361 #define IAVF_RSS_TYPE_IPV6_AH (ETH_RSS_AH | ETH_RSS_IPV6)
362 #define IAVF_RSS_TYPE_IPV4_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
363 #define IAVF_RSS_TYPE_IPV6_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
364 #define IAVF_RSS_TYPE_IPV4_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV4)
365 #define IAVF_RSS_TYPE_IPV6_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV6)
366
367 /**
368 * Supported pattern for hash.
369 * The first member is pattern item type,
370 * the second member is input set mask,
371 * the third member is virtchnl_proto_hdrs template
372 */
373 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
374 /* IPv4 */
375 {iavf_pattern_eth_ipv4, IAVF_RSS_TYPE_OUTER_IPV4, &outer_ipv4_tmplt},
376 {iavf_pattern_eth_ipv4_udp, IAVF_RSS_TYPE_OUTER_IPV4_UDP, &outer_ipv4_udp_tmplt},
377 {iavf_pattern_eth_ipv4_tcp, IAVF_RSS_TYPE_OUTER_IPV4_TCP, &outer_ipv4_tcp_tmplt},
378 {iavf_pattern_eth_ipv4_sctp, IAVF_RSS_TYPE_OUTER_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
379 {iavf_pattern_eth_vlan_ipv4, IAVF_RSS_TYPE_VLAN_IPV4, &outer_ipv4_tmplt},
380 {iavf_pattern_eth_vlan_ipv4_udp, IAVF_RSS_TYPE_VLAN_IPV4_UDP, &outer_ipv4_udp_tmplt},
381 {iavf_pattern_eth_vlan_ipv4_tcp, IAVF_RSS_TYPE_VLAN_IPV4_TCP, &outer_ipv4_tcp_tmplt},
382 {iavf_pattern_eth_vlan_ipv4_sctp, IAVF_RSS_TYPE_VLAN_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
383 {iavf_pattern_eth_ipv4_gtpu, ETH_RSS_IPV4, &outer_ipv4_udp_tmplt},
384 {iavf_pattern_eth_ipv4_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
385 {iavf_pattern_eth_ipv4_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
386 {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
387 {iavf_pattern_eth_ipv6_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
388 {iavf_pattern_eth_ipv6_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
389 {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
390 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
391 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
392 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
393 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
394 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
395 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
396 {iavf_pattern_eth_ipv4_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_esp_tmplt},
397 {iavf_pattern_eth_ipv4_udp_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_udp_esp_tmplt},
398 {iavf_pattern_eth_ipv4_ah, IAVF_RSS_TYPE_IPV4_AH, &ipv4_ah_tmplt},
399 {iavf_pattern_eth_ipv4_l2tpv3, IAVF_RSS_TYPE_IPV4_L2TPV3, &ipv4_l2tpv3_tmplt},
400 {iavf_pattern_eth_ipv4_pfcp, IAVF_RSS_TYPE_IPV4_PFCP, &ipv4_pfcp_tmplt},
401 {iavf_pattern_eth_ipv4_gtpc, ETH_RSS_IPV4, &ipv4_udp_gtpc_tmplt},
402 /* IPv6 */
403 {iavf_pattern_eth_ipv6, IAVF_RSS_TYPE_OUTER_IPV6, &outer_ipv6_tmplt},
404 {iavf_pattern_eth_ipv6_udp, IAVF_RSS_TYPE_OUTER_IPV6_UDP, &outer_ipv6_udp_tmplt},
405 {iavf_pattern_eth_ipv6_tcp, IAVF_RSS_TYPE_OUTER_IPV6_TCP, &outer_ipv6_tcp_tmplt},
406 {iavf_pattern_eth_ipv6_sctp, IAVF_RSS_TYPE_OUTER_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
407 {iavf_pattern_eth_vlan_ipv6, IAVF_RSS_TYPE_VLAN_IPV6, &outer_ipv6_tmplt},
408 {iavf_pattern_eth_vlan_ipv6_udp, IAVF_RSS_TYPE_VLAN_IPV6_UDP, &outer_ipv6_udp_tmplt},
409 {iavf_pattern_eth_vlan_ipv6_tcp, IAVF_RSS_TYPE_VLAN_IPV6_TCP, &outer_ipv6_tcp_tmplt},
410 {iavf_pattern_eth_vlan_ipv6_sctp, IAVF_RSS_TYPE_VLAN_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
411 {iavf_pattern_eth_ipv6_gtpu, ETH_RSS_IPV6, &outer_ipv6_udp_tmplt},
412 {iavf_pattern_eth_ipv4_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
413 {iavf_pattern_eth_ipv4_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
414 {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
415 {iavf_pattern_eth_ipv6_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
416 {iavf_pattern_eth_ipv6_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
417 {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
418 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
419 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
420 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
421 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
422 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
423 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
424 {iavf_pattern_eth_ipv6_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_esp_tmplt},
425 {iavf_pattern_eth_ipv6_udp_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_udp_esp_tmplt},
426 {iavf_pattern_eth_ipv6_ah, IAVF_RSS_TYPE_IPV6_AH, &ipv6_ah_tmplt},
427 {iavf_pattern_eth_ipv6_l2tpv3, IAVF_RSS_TYPE_IPV6_L2TPV3, &ipv6_l2tpv3_tmplt},
428 {iavf_pattern_eth_ipv6_pfcp, IAVF_RSS_TYPE_IPV6_PFCP, &ipv6_pfcp_tmplt},
429 {iavf_pattern_eth_ipv6_gtpc, ETH_RSS_IPV6, &ipv6_udp_gtpc_tmplt},
430 };
431
432 struct virtchnl_proto_hdrs *iavf_hash_default_hdrs[] = {
433 &inner_ipv4_tmplt,
434 &inner_ipv4_udp_tmplt,
435 &inner_ipv4_tcp_tmplt,
436 &inner_ipv4_sctp_tmplt,
437 &inner_ipv6_tmplt,
438 &inner_ipv6_udp_tmplt,
439 &inner_ipv6_tcp_tmplt,
440 &inner_ipv6_sctp_tmplt,
441 };
442
443 static struct iavf_flow_engine iavf_hash_engine = {
444 .init = iavf_hash_init,
445 .create = iavf_hash_create,
446 .destroy = iavf_hash_destroy,
447 .uninit = iavf_hash_uninit,
448 .free = iavf_hash_free,
449 .type = IAVF_FLOW_ENGINE_HASH,
450 };
451
452 /* Register parser for comms package. */
453 static struct iavf_flow_parser iavf_hash_parser = {
454 .engine = &iavf_hash_engine,
455 .array = iavf_hash_pattern_list,
456 .array_len = RTE_DIM(iavf_hash_pattern_list),
457 .parse_pattern_action = iavf_hash_parse_pattern_action,
458 .stage = IAVF_FLOW_STAGE_RSS,
459 };
460
461 static int
iavf_hash_default_set(struct iavf_adapter * ad,bool add)462 iavf_hash_default_set(struct iavf_adapter *ad, bool add)
463 {
464 struct virtchnl_rss_cfg *rss_cfg;
465 uint16_t i;
466
467 rss_cfg = rte_zmalloc("iavf rss rule",
468 sizeof(struct virtchnl_rss_cfg), 0);
469 if (!rss_cfg)
470 return -ENOMEM;
471
472 for (i = 0; i < RTE_DIM(iavf_hash_default_hdrs); i++) {
473 rss_cfg->proto_hdrs = *iavf_hash_default_hdrs[i];
474 rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
475
476 iavf_add_del_rss_cfg(ad, rss_cfg, add);
477 }
478
479 return 0;
480 }
481
RTE_INIT(iavf_hash_engine_init)482 RTE_INIT(iavf_hash_engine_init)
483 {
484 struct iavf_flow_engine *engine = &iavf_hash_engine;
485
486 iavf_register_flow_engine(engine);
487 }
488
489 static int
iavf_hash_init(struct iavf_adapter * ad)490 iavf_hash_init(struct iavf_adapter *ad)
491 {
492 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
493 struct iavf_flow_parser *parser;
494 int ret;
495
496 if (vf->vf_reset)
497 return -EIO;
498
499 if (!vf->vf_res)
500 return -EINVAL;
501
502 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
503 return -ENOTSUP;
504
505 parser = &iavf_hash_parser;
506
507 ret = iavf_register_parser(parser, ad);
508 if (ret) {
509 PMD_DRV_LOG(ERR, "fail to register hash parser");
510 return ret;
511 }
512
513 ret = iavf_hash_default_set(ad, true);
514 if (ret) {
515 PMD_DRV_LOG(ERR, "fail to set default RSS");
516 iavf_unregister_parser(parser, ad);
517 }
518
519 return ret;
520 }
521
522 static int
iavf_hash_parse_pattern(const struct rte_flow_item pattern[],uint64_t * phint,struct rte_flow_error * error)523 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
524 struct rte_flow_error *error)
525 {
526 const struct rte_flow_item *item = pattern;
527 const struct rte_flow_item_gtp_psc *psc;
528
529 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
530 if (item->last) {
531 rte_flow_error_set(error, EINVAL,
532 RTE_FLOW_ERROR_TYPE_ITEM, item,
533 "Not support range");
534 return -rte_errno;
535 }
536
537 switch (item->type) {
538 case RTE_FLOW_ITEM_TYPE_IPV4:
539 if (!(*phint & IAVF_PHINT_GTPU_MSK))
540 *phint |= IAVF_PHINT_OUTER_IPV4;
541 break;
542 case RTE_FLOW_ITEM_TYPE_IPV6:
543 if (!(*phint & IAVF_PHINT_GTPU_MSK))
544 *phint |= IAVF_PHINT_OUTER_IPV6;
545 break;
546 case RTE_FLOW_ITEM_TYPE_GTPU:
547 *phint |= IAVF_PHINT_GTPU;
548 break;
549 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
550 *phint |= IAVF_PHINT_GTPU_EH;
551 psc = item->spec;
552 if (!psc)
553 break;
554 else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
555 *phint |= IAVF_PHINT_GTPU_EH_UP;
556 else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
557 *phint |= IAVF_PHINT_GTPU_EH_DWN;
558 break;
559 default:
560 break;
561 }
562 }
563
564 return 0;
565 }
566
567 #define REFINE_PROTO_FLD(op, fld) \
568 VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
569 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
570 do { \
571 REFINE_PROTO_FLD(DEL, fld_1); \
572 REFINE_PROTO_FLD(ADD, fld_2); \
573 } while (0)
574
575 /* refine proto hdrs base on l2, l3, l4 rss type */
576 static void
iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs * proto_hdrs,uint64_t rss_type)577 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
578 uint64_t rss_type)
579 {
580 struct virtchnl_proto_hdr *hdr;
581 int i;
582
583 for (i = 0; i < proto_hdrs->count; i++) {
584 hdr = &proto_hdrs->proto_hdr[i];
585 switch (hdr->type) {
586 case VIRTCHNL_PROTO_HDR_ETH:
587 if (!(rss_type & ETH_RSS_ETH))
588 hdr->field_selector = 0;
589 else if (rss_type & ETH_RSS_L2_SRC_ONLY)
590 REFINE_PROTO_FLD(DEL, ETH_DST);
591 else if (rss_type & ETH_RSS_L2_DST_ONLY)
592 REFINE_PROTO_FLD(DEL, ETH_SRC);
593 break;
594 case VIRTCHNL_PROTO_HDR_IPV4:
595 if (rss_type &
596 (ETH_RSS_IPV4 |
597 ETH_RSS_NONFRAG_IPV4_UDP |
598 ETH_RSS_NONFRAG_IPV4_TCP |
599 ETH_RSS_NONFRAG_IPV4_SCTP)) {
600 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
601 REFINE_PROTO_FLD(DEL, IPV4_DST);
602 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
603 REFINE_PROTO_FLD(DEL, IPV4_SRC);
604 } else if (rss_type &
605 (ETH_RSS_L4_SRC_ONLY |
606 ETH_RSS_L4_DST_ONLY)) {
607 REFINE_PROTO_FLD(DEL, IPV4_DST);
608 REFINE_PROTO_FLD(DEL, IPV4_SRC);
609 }
610 } else {
611 hdr->field_selector = 0;
612 }
613 break;
614 case VIRTCHNL_PROTO_HDR_IPV6:
615 if (rss_type &
616 (ETH_RSS_IPV6 |
617 ETH_RSS_NONFRAG_IPV6_UDP |
618 ETH_RSS_NONFRAG_IPV6_TCP |
619 ETH_RSS_NONFRAG_IPV6_SCTP)) {
620 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
621 REFINE_PROTO_FLD(DEL, IPV6_DST);
622 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
623 REFINE_PROTO_FLD(DEL, IPV6_SRC);
624 } else if (rss_type &
625 (ETH_RSS_L4_SRC_ONLY |
626 ETH_RSS_L4_DST_ONLY)) {
627 REFINE_PROTO_FLD(DEL, IPV6_DST);
628 REFINE_PROTO_FLD(DEL, IPV6_SRC);
629 }
630 } else {
631 hdr->field_selector = 0;
632 }
633 if (rss_type & RTE_ETH_RSS_L3_PRE64) {
634 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
635 REPALCE_PROTO_FLD(IPV6_SRC,
636 IPV6_PREFIX64_SRC);
637 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
638 REPALCE_PROTO_FLD(IPV6_DST,
639 IPV6_PREFIX64_DST);
640 }
641 break;
642 case VIRTCHNL_PROTO_HDR_UDP:
643 if (rss_type &
644 (ETH_RSS_NONFRAG_IPV4_UDP |
645 ETH_RSS_NONFRAG_IPV6_UDP)) {
646 if (rss_type & ETH_RSS_L4_SRC_ONLY)
647 REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
648 else if (rss_type & ETH_RSS_L4_DST_ONLY)
649 REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
650 else if (rss_type &
651 (ETH_RSS_L3_SRC_ONLY |
652 ETH_RSS_L3_DST_ONLY))
653 hdr->field_selector = 0;
654 } else {
655 hdr->field_selector = 0;
656 }
657 break;
658 case VIRTCHNL_PROTO_HDR_TCP:
659 if (rss_type &
660 (ETH_RSS_NONFRAG_IPV4_TCP |
661 ETH_RSS_NONFRAG_IPV6_TCP)) {
662 if (rss_type & ETH_RSS_L4_SRC_ONLY)
663 REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
664 else if (rss_type & ETH_RSS_L4_DST_ONLY)
665 REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
666 else if (rss_type &
667 (ETH_RSS_L3_SRC_ONLY |
668 ETH_RSS_L3_DST_ONLY))
669 hdr->field_selector = 0;
670 } else {
671 hdr->field_selector = 0;
672 }
673 break;
674 case VIRTCHNL_PROTO_HDR_SCTP:
675 if (rss_type &
676 (ETH_RSS_NONFRAG_IPV4_SCTP |
677 ETH_RSS_NONFRAG_IPV6_SCTP)) {
678 if (rss_type & ETH_RSS_L4_SRC_ONLY)
679 REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
680 else if (rss_type & ETH_RSS_L4_DST_ONLY)
681 REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
682 else if (rss_type &
683 (ETH_RSS_L3_SRC_ONLY |
684 ETH_RSS_L3_DST_ONLY))
685 hdr->field_selector = 0;
686 } else {
687 hdr->field_selector = 0;
688 }
689 break;
690 case VIRTCHNL_PROTO_HDR_S_VLAN:
691 if (!(rss_type & ETH_RSS_S_VLAN))
692 hdr->field_selector = 0;
693 break;
694 case VIRTCHNL_PROTO_HDR_C_VLAN:
695 if (!(rss_type & ETH_RSS_C_VLAN))
696 hdr->field_selector = 0;
697 break;
698 case VIRTCHNL_PROTO_HDR_L2TPV3:
699 if (!(rss_type & ETH_RSS_L2TPV3))
700 hdr->field_selector = 0;
701 break;
702 case VIRTCHNL_PROTO_HDR_ESP:
703 if (!(rss_type & ETH_RSS_ESP))
704 hdr->field_selector = 0;
705 break;
706 case VIRTCHNL_PROTO_HDR_AH:
707 if (!(rss_type & ETH_RSS_AH))
708 hdr->field_selector = 0;
709 break;
710 case VIRTCHNL_PROTO_HDR_PFCP:
711 if (!(rss_type & ETH_RSS_PFCP))
712 hdr->field_selector = 0;
713 break;
714 default:
715 break;
716 }
717 }
718 }
719
720 /* refine proto hdrs base on gtpu rss type */
721 static void
iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs * proto_hdrs,uint64_t rss_type)722 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
723 uint64_t rss_type)
724 {
725 struct virtchnl_proto_hdr *hdr;
726 int i;
727
728 if (!(rss_type & ETH_RSS_GTPU))
729 return;
730
731 for (i = 0; i < proto_hdrs->count; i++) {
732 hdr = &proto_hdrs->proto_hdr[i];
733 switch (hdr->type) {
734 case VIRTCHNL_PROTO_HDR_GTPU_IP:
735 REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
736 break;
737 default:
738 break;
739 }
740 }
741 }
742
743 static void
iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs * proto_hdrs,uint64_t phint)744 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
745 uint64_t phint)
746 {
747 struct virtchnl_proto_hdr *hdr1;
748 struct virtchnl_proto_hdr *hdr2;
749 int i, shift_count = 1;
750
751 if (!(phint & IAVF_PHINT_GTPU_MSK))
752 return;
753
754 if (phint & IAVF_PHINT_LAYERS_MSK)
755 shift_count++;
756
757 if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
758 /* shift headers layer */
759 for (i = proto_hdrs->count - 1 + shift_count;
760 i > shift_count - 1; i--) {
761 hdr1 = &proto_hdrs->proto_hdr[i];
762 hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
763 *hdr1 = *hdr2;
764 }
765
766 if (shift_count == 1) {
767 /* adding gtpu header at layer 0 */
768 hdr1 = &proto_hdrs->proto_hdr[0];
769 } else {
770 /* adding gtpu header and outer ip header */
771 hdr1 = &proto_hdrs->proto_hdr[1];
772 hdr2 = &proto_hdrs->proto_hdr[0];
773 hdr2->field_selector = 0;
774 proto_hdrs->count++;
775 proto_hdrs->tunnel_level = TUNNEL_LEVEL_OUTER;
776
777 if (phint & IAVF_PHINT_OUTER_IPV4)
778 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
779 else if (phint & IAVF_PHINT_OUTER_IPV6)
780 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
781 }
782 } else {
783 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
784 }
785
786 hdr1->field_selector = 0;
787 proto_hdrs->count++;
788
789 if (phint & IAVF_PHINT_GTPU_EH_DWN)
790 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
791 else if (phint & IAVF_PHINT_GTPU_EH_UP)
792 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
793 else if (phint & IAVF_PHINT_GTPU_EH)
794 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
795 else if (phint & IAVF_PHINT_GTPU)
796 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
797 }
798
iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs * proto_hdrs,uint64_t rss_type,uint64_t phint)799 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
800 uint64_t rss_type, uint64_t phint)
801 {
802 iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
803 iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
804 iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
805 }
806
807 static uint64_t invalid_rss_comb[] = {
808 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
809 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
810 RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
811 RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
812 RTE_ETH_RSS_L3_PRE96
813 };
814
815 struct rss_attr_type {
816 uint64_t attr;
817 uint64_t type;
818 };
819
820 #define VALID_RSS_IPV4_L4 (ETH_RSS_NONFRAG_IPV4_UDP | \
821 ETH_RSS_NONFRAG_IPV4_TCP | \
822 ETH_RSS_NONFRAG_IPV4_SCTP)
823
824 #define VALID_RSS_IPV6_L4 (ETH_RSS_NONFRAG_IPV6_UDP | \
825 ETH_RSS_NONFRAG_IPV6_TCP | \
826 ETH_RSS_NONFRAG_IPV6_SCTP)
827
828 #define VALID_RSS_IPV4 (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
829 #define VALID_RSS_IPV6 (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
830 #define VALID_RSS_L3 (VALID_RSS_IPV4 | VALID_RSS_IPV6)
831 #define VALID_RSS_L4 (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
832
833 #define VALID_RSS_ATTR (ETH_RSS_L3_SRC_ONLY | \
834 ETH_RSS_L3_DST_ONLY | \
835 ETH_RSS_L4_SRC_ONLY | \
836 ETH_RSS_L4_DST_ONLY | \
837 ETH_RSS_L2_SRC_ONLY | \
838 ETH_RSS_L2_DST_ONLY | \
839 RTE_ETH_RSS_L3_PRE64)
840
841 #define INVALID_RSS_ATTR (RTE_ETH_RSS_L3_PRE32 | \
842 RTE_ETH_RSS_L3_PRE40 | \
843 RTE_ETH_RSS_L3_PRE48 | \
844 RTE_ETH_RSS_L3_PRE56 | \
845 RTE_ETH_RSS_L3_PRE96)
846
847 static struct rss_attr_type rss_attr_to_valid_type[] = {
848 {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY, ETH_RSS_ETH},
849 {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY, VALID_RSS_L3},
850 {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY, VALID_RSS_L4},
851 /* current ipv6 prefix only supports prefix 64 bits*/
852 {RTE_ETH_RSS_L3_PRE64, VALID_RSS_IPV6},
853 {INVALID_RSS_ATTR, 0}
854 };
855
856 static bool
iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,uint64_t rss_type,uint64_t allow_rss_type)857 iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
858 uint64_t rss_type, uint64_t allow_rss_type)
859 {
860 uint32_t i;
861
862 /**
863 * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
864 * hash function.
865 */
866 if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
867 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
868 ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
869 return true;
870 }
871
872 /* check invalid combination */
873 for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
874 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
875 return true;
876 }
877
878 /* check invalid RSS attribute */
879 for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
880 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
881
882 if (rat->attr & rss_type && !(rat->type & rss_type))
883 return true;
884 }
885
886 /* check not allowed RSS type */
887 rss_type &= ~VALID_RSS_ATTR;
888
889 return ((rss_type & allow_rss_type) != rss_type);
890 }
891
892 static int
iavf_hash_parse_action(struct iavf_pattern_match_item * match_item,const struct rte_flow_action actions[],uint64_t pattern_hint,void ** meta,struct rte_flow_error * error)893 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
894 const struct rte_flow_action actions[],
895 uint64_t pattern_hint, void **meta,
896 struct rte_flow_error *error)
897 {
898 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
899 struct virtchnl_proto_hdrs *proto_hdrs;
900 enum rte_flow_action_type action_type;
901 const struct rte_flow_action_rss *rss;
902 const struct rte_flow_action *action;
903 uint64_t rss_type;
904
905 /* Supported action is RSS. */
906 for (action = actions; action->type !=
907 RTE_FLOW_ACTION_TYPE_END; action++) {
908 action_type = action->type;
909 switch (action_type) {
910 case RTE_FLOW_ACTION_TYPE_RSS:
911 rss = action->conf;
912 rss_type = rss->types;
913
914 if (rss->func ==
915 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
916 rss_meta->rss_algorithm =
917 VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
918 return rte_flow_error_set(error, ENOTSUP,
919 RTE_FLOW_ERROR_TYPE_ACTION, action,
920 "function simple_xor is not supported");
921 } else if (rss->func ==
922 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
923 rss_meta->rss_algorithm =
924 VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
925 } else {
926 rss_meta->rss_algorithm =
927 VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
928 }
929
930 if (rss->level)
931 return rte_flow_error_set(error, ENOTSUP,
932 RTE_FLOW_ERROR_TYPE_ACTION, action,
933 "a nonzero RSS encapsulation level is not supported");
934
935 if (rss->key_len)
936 return rte_flow_error_set(error, ENOTSUP,
937 RTE_FLOW_ERROR_TYPE_ACTION, action,
938 "a nonzero RSS key_len is not supported");
939
940 if (rss->queue_num)
941 return rte_flow_error_set(error, ENOTSUP,
942 RTE_FLOW_ERROR_TYPE_ACTION, action,
943 "a non-NULL RSS queue is not supported");
944
945 /**
946 * Check simultaneous use of SRC_ONLY and DST_ONLY
947 * of the same level.
948 */
949 rss_type = rte_eth_rss_hf_refine(rss_type);
950
951 if (iavf_any_invalid_rss_type(rss->func, rss_type,
952 match_item->input_set_mask))
953 return rte_flow_error_set(error, ENOTSUP,
954 RTE_FLOW_ERROR_TYPE_ACTION,
955 action, "RSS type not supported");
956 proto_hdrs = match_item->meta;
957 rss_meta->proto_hdrs = *proto_hdrs;
958 iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
959 rss_type, pattern_hint);
960 break;
961
962 case RTE_FLOW_ACTION_TYPE_END:
963 break;
964
965 default:
966 rte_flow_error_set(error, EINVAL,
967 RTE_FLOW_ERROR_TYPE_ACTION, action,
968 "Invalid action.");
969 return -rte_errno;
970 }
971 }
972
973 return 0;
974 }
975
976 static int
iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter * ad,struct iavf_pattern_match_item * array,uint32_t array_len,const struct rte_flow_item pattern[],const struct rte_flow_action actions[],void ** meta,struct rte_flow_error * error)977 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
978 struct iavf_pattern_match_item *array,
979 uint32_t array_len,
980 const struct rte_flow_item pattern[],
981 const struct rte_flow_action actions[],
982 void **meta,
983 struct rte_flow_error *error)
984 {
985 struct iavf_pattern_match_item *pattern_match_item;
986 struct iavf_rss_meta *rss_meta_ptr;
987 uint64_t phint = IAVF_PHINT_NONE;
988 int ret = 0;
989
990 rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
991 if (!rss_meta_ptr) {
992 rte_flow_error_set(error, EINVAL,
993 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
994 "No memory for rss_meta_ptr");
995 return -ENOMEM;
996 }
997
998 /* Check rss supported pattern and find matched pattern. */
999 pattern_match_item =
1000 iavf_search_pattern_match_item(pattern, array, array_len,
1001 error);
1002 if (!pattern_match_item) {
1003 ret = -rte_errno;
1004 goto error;
1005 }
1006
1007 ret = iavf_hash_parse_pattern(pattern, &phint, error);
1008 if (ret)
1009 goto error;
1010
1011 ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
1012 (void **)&rss_meta_ptr, error);
1013
1014 error:
1015 if (!ret && meta)
1016 *meta = rss_meta_ptr;
1017 else
1018 rte_free(rss_meta_ptr);
1019
1020 rte_free(pattern_match_item);
1021
1022 return ret;
1023 }
1024
1025 static int
iavf_hash_create(__rte_unused struct iavf_adapter * ad,__rte_unused struct rte_flow * flow,void * meta,__rte_unused struct rte_flow_error * error)1026 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
1027 __rte_unused struct rte_flow *flow, void *meta,
1028 __rte_unused struct rte_flow_error *error)
1029 {
1030 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1031 struct virtchnl_rss_cfg *rss_cfg;
1032 int ret = 0;
1033
1034 rss_cfg = rte_zmalloc("iavf rss rule",
1035 sizeof(struct virtchnl_rss_cfg), 0);
1036 if (!rss_cfg) {
1037 rte_flow_error_set(error, EINVAL,
1038 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1039 "No memory for rss rule");
1040 return -ENOMEM;
1041 }
1042
1043 rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1044 rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1045
1046 ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1047 if (!ret) {
1048 flow->rule = rss_cfg;
1049 } else {
1050 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1051 rte_flow_error_set(error, -ret,
1052 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1053 "Failed to add rss rule.");
1054 rte_free(rss_cfg);
1055 return -rte_errno;
1056 }
1057
1058 rte_free(meta);
1059
1060 return ret;
1061 }
1062
1063 static int
iavf_hash_destroy(__rte_unused struct iavf_adapter * ad,struct rte_flow * flow,__rte_unused struct rte_flow_error * error)1064 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1065 struct rte_flow *flow,
1066 __rte_unused struct rte_flow_error *error)
1067 {
1068 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1069 struct virtchnl_rss_cfg *rss_cfg;
1070 int ret = 0;
1071
1072 if (vf->vf_reset)
1073 return 0;
1074
1075 rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1076
1077 ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1078 if (ret) {
1079 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1080 rte_flow_error_set(error, -ret,
1081 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1082 "Failed to delete rss rule.");
1083 return -rte_errno;
1084 }
1085 return ret;
1086 }
1087
1088 static void
iavf_hash_uninit(struct iavf_adapter * ad)1089 iavf_hash_uninit(struct iavf_adapter *ad)
1090 {
1091 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1092
1093 if (vf->vf_reset)
1094 return;
1095
1096 if (!vf->vf_res)
1097 return;
1098
1099 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
1100 return;
1101
1102 if (iavf_hash_default_set(ad, false))
1103 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1104
1105 iavf_unregister_parser(&iavf_hash_parser, ad);
1106 }
1107
1108 static void
iavf_hash_free(struct rte_flow * flow)1109 iavf_hash_free(struct rte_flow *flow)
1110 {
1111 rte_free(flow->rule);
1112 }
1113