1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 13 #include <rte_debug.h> 14 #include <rte_ether.h> 15 #include <ethdev_driver.h> 16 #include <rte_log.h> 17 #include <rte_malloc.h> 18 #include <rte_eth_ctrl.h> 19 #include <rte_tailq.h> 20 #include <rte_flow_driver.h> 21 22 #include "ice_logs.h" 23 #include "base/ice_type.h" 24 #include "base/ice_flow.h" 25 #include "ice_ethdev.h" 26 #include "ice_generic_flow.h" 27 28 #define ICE_PHINT_NONE 0 29 #define ICE_PHINT_VLAN BIT_ULL(0) 30 #define ICE_PHINT_PPPOE BIT_ULL(1) 31 #define ICE_PHINT_GTPU BIT_ULL(2) 32 #define ICE_PHINT_GTPU_EH BIT_ULL(3) 33 #define ICE_PHINT_GTPU_EH_DWN BIT_ULL(4) 34 #define ICE_PHINT_GTPU_EH_UP BIT_ULL(5) 35 36 #define ICE_GTPU_EH_DWNLINK 0 37 #define ICE_GTPU_EH_UPLINK 1 38 39 #define ICE_IPV4_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) 40 #define ICE_IPV6_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) 41 42 #define VALID_RSS_IPV4_L4 (ETH_RSS_NONFRAG_IPV4_UDP | \ 43 ETH_RSS_NONFRAG_IPV4_TCP | \ 44 ETH_RSS_NONFRAG_IPV4_SCTP) 45 46 #define VALID_RSS_IPV6_L4 (ETH_RSS_NONFRAG_IPV6_UDP | \ 47 ETH_RSS_NONFRAG_IPV6_TCP | \ 48 ETH_RSS_NONFRAG_IPV6_SCTP) 49 50 #define VALID_RSS_IPV4 (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4) 51 #define VALID_RSS_IPV6 (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4) 52 #define VALID_RSS_L3 (VALID_RSS_IPV4 | VALID_RSS_IPV6) 53 #define VALID_RSS_L4 (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4) 54 55 #define VALID_RSS_ATTR (ETH_RSS_L3_SRC_ONLY | \ 56 ETH_RSS_L3_DST_ONLY | \ 57 ETH_RSS_L4_SRC_ONLY | \ 58 ETH_RSS_L4_DST_ONLY | \ 59 ETH_RSS_L2_SRC_ONLY | \ 60 ETH_RSS_L2_DST_ONLY | \ 61 RTE_ETH_RSS_L3_PRE32 | \ 62 RTE_ETH_RSS_L3_PRE48 | \ 63 RTE_ETH_RSS_L3_PRE64) 64 65 #define INVALID_RSS_ATTR (RTE_ETH_RSS_L3_PRE40 | \ 66 RTE_ETH_RSS_L3_PRE56 | \ 67 RTE_ETH_RSS_L3_PRE96) 68 69 struct ice_rss_meta { 70 uint8_t hash_function; 71 struct ice_rss_hash_cfg cfg; 72 }; 73 74 struct ice_hash_flow_cfg { 75 bool simple_xor; 76 struct ice_rss_cfg rss_cfg; 77 }; 78 79 static int 80 ice_hash_init(struct ice_adapter *ad); 81 82 static int 83 ice_hash_create(struct ice_adapter *ad, 84 struct rte_flow *flow, 85 void *meta, 86 struct rte_flow_error *error); 87 88 static int 89 ice_hash_destroy(struct ice_adapter *ad, 90 struct rte_flow *flow, 91 struct rte_flow_error *error); 92 93 static void 94 ice_hash_uninit(struct ice_adapter *ad); 95 96 static void 97 ice_hash_free(struct rte_flow *flow); 98 99 static int 100 ice_hash_parse_pattern_action(struct ice_adapter *ad, 101 struct ice_pattern_match_item *array, 102 uint32_t array_len, 103 const struct rte_flow_item pattern[], 104 const struct rte_flow_action actions[], 105 void **meta, 106 struct rte_flow_error *error); 107 108 /* Rss configuration template */ 109 struct ice_rss_hash_cfg ipv4_tmplt = { 110 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 111 ICE_FLOW_SEG_HDR_IPV_OTHER, 112 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV4, 113 ICE_RSS_ANY_HEADERS, 114 0 115 }; 116 117 struct ice_rss_hash_cfg ipv4_udp_tmplt = { 118 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 119 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP, 120 ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 121 ICE_RSS_ANY_HEADERS, 122 0 123 }; 124 125 struct ice_rss_hash_cfg ipv4_tcp_tmplt = { 126 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 127 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP, 128 ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 129 ICE_RSS_ANY_HEADERS, 130 0 131 }; 132 133 struct ice_rss_hash_cfg ipv4_sctp_tmplt = { 134 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 135 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP, 136 ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV4 | ICE_IPV4_PROT, 137 ICE_RSS_ANY_HEADERS, 138 0 139 }; 140 141 struct ice_rss_hash_cfg ipv6_tmplt = { 142 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 143 ICE_FLOW_SEG_HDR_IPV_OTHER, 144 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV6, 145 ICE_RSS_ANY_HEADERS, 146 0 147 }; 148 149 struct ice_rss_hash_cfg ipv6_udp_tmplt = { 150 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 151 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP, 152 ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 153 ICE_RSS_ANY_HEADERS, 154 0 155 }; 156 157 struct ice_rss_hash_cfg ipv6_tcp_tmplt = { 158 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 159 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP, 160 ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 161 ICE_RSS_ANY_HEADERS, 162 0 163 }; 164 165 struct ice_rss_hash_cfg ipv6_sctp_tmplt = { 166 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 167 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP, 168 ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV6 | ICE_IPV6_PROT, 169 ICE_RSS_ANY_HEADERS, 170 0 171 }; 172 173 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tmplt = { 174 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, 175 ICE_FLOW_HASH_IPV4, 176 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 177 0 178 }; 179 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_udp_tmplt = { 180 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 181 ICE_FLOW_SEG_HDR_UDP, 182 ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 183 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 184 0 185 }; 186 187 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tcp_tmplt = { 188 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 189 ICE_FLOW_SEG_HDR_TCP, 190 ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 191 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 192 0 193 }; 194 195 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tmplt = { 196 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, 197 ICE_FLOW_HASH_IPV4, 198 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 199 0 200 }; 201 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_udp_tmplt = { 202 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 203 ICE_FLOW_SEG_HDR_UDP, 204 ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 205 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 206 0 207 }; 208 209 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tcp_tmplt = { 210 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 211 ICE_FLOW_SEG_HDR_TCP, 212 ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 213 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 214 0 215 }; 216 217 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tmplt = { 218 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, 219 ICE_FLOW_HASH_IPV6, 220 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 221 0 222 }; 223 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_udp_tmplt = { 224 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 225 ICE_FLOW_SEG_HDR_UDP, 226 ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 227 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 228 0 229 }; 230 231 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tcp_tmplt = { 232 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 233 ICE_FLOW_SEG_HDR_TCP, 234 ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 235 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 236 0 237 }; 238 239 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tmplt = { 240 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, 241 ICE_FLOW_HASH_IPV6, 242 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 243 0 244 }; 245 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_udp_tmplt = { 246 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 247 ICE_FLOW_SEG_HDR_UDP, 248 ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 249 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 250 0 251 }; 252 253 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tcp_tmplt = { 254 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 255 ICE_FLOW_SEG_HDR_TCP, 256 ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 257 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 258 0 259 }; 260 261 struct ice_rss_hash_cfg eth_ipv4_esp_tmplt = { 262 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 263 ICE_FLOW_SEG_HDR_ESP, 264 ICE_FLOW_HASH_ESP_SPI, 265 ICE_RSS_ANY_HEADERS, 266 0 267 }; 268 269 struct ice_rss_hash_cfg eth_ipv4_udp_esp_tmplt = { 270 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 271 ICE_FLOW_SEG_HDR_NAT_T_ESP, 272 ICE_FLOW_HASH_NAT_T_ESP_SPI, 273 ICE_RSS_ANY_HEADERS, 274 0 275 }; 276 277 struct ice_rss_hash_cfg eth_ipv4_ah_tmplt = { 278 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 279 ICE_FLOW_SEG_HDR_AH, 280 ICE_FLOW_HASH_AH_SPI, 281 ICE_RSS_ANY_HEADERS, 282 0 283 }; 284 285 struct ice_rss_hash_cfg eth_ipv4_l2tpv3_tmplt = { 286 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 287 ICE_FLOW_SEG_HDR_L2TPV3, 288 ICE_FLOW_HASH_L2TPV3_SESS_ID, 289 ICE_RSS_ANY_HEADERS, 290 0 291 }; 292 293 struct ice_rss_hash_cfg eth_ipv4_pfcp_tmplt = { 294 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 295 ICE_FLOW_SEG_HDR_PFCP_SESSION, 296 ICE_FLOW_HASH_PFCP_SEID, 297 ICE_RSS_ANY_HEADERS, 298 0 299 }; 300 301 struct ice_rss_hash_cfg eth_ipv6_esp_tmplt = { 302 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 303 ICE_FLOW_SEG_HDR_ESP, 304 ICE_FLOW_HASH_ESP_SPI, 305 ICE_RSS_ANY_HEADERS, 306 0 307 }; 308 309 struct ice_rss_hash_cfg eth_ipv6_udp_esp_tmplt = { 310 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 311 ICE_FLOW_SEG_HDR_NAT_T_ESP, 312 ICE_FLOW_HASH_NAT_T_ESP_SPI, 313 ICE_RSS_ANY_HEADERS, 314 0 315 }; 316 317 struct ice_rss_hash_cfg eth_ipv6_ah_tmplt = { 318 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 319 ICE_FLOW_SEG_HDR_AH, 320 ICE_FLOW_HASH_AH_SPI, 321 ICE_RSS_ANY_HEADERS, 322 0 323 }; 324 325 struct ice_rss_hash_cfg eth_ipv6_l2tpv3_tmplt = { 326 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 327 ICE_FLOW_SEG_HDR_L2TPV3, 328 ICE_FLOW_HASH_L2TPV3_SESS_ID, 329 ICE_RSS_ANY_HEADERS, 330 0 331 }; 332 333 struct ice_rss_hash_cfg eth_ipv6_pfcp_tmplt = { 334 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 335 ICE_FLOW_SEG_HDR_PFCP_SESSION, 336 ICE_FLOW_HASH_PFCP_SEID, 337 ICE_RSS_ANY_HEADERS, 338 0 339 }; 340 341 struct ice_rss_hash_cfg pppoe_tmplt = { 342 ICE_FLOW_SEG_HDR_ETH, 343 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_PPPOE_SESS_ID, 344 ICE_RSS_ANY_HEADERS, 345 0 346 }; 347 348 struct ice_rss_hash_cfg empty_tmplt = { 349 ICE_FLOW_SEG_HDR_NONE, 350 0, 351 ICE_RSS_ANY_HEADERS, 352 0 353 }; 354 355 /* IPv4 */ 356 #define ICE_RSS_TYPE_ETH_IPV4 (ETH_RSS_ETH | ETH_RSS_IPV4) 357 #define ICE_RSS_TYPE_ETH_IPV4_UDP (ICE_RSS_TYPE_ETH_IPV4 | \ 358 ETH_RSS_NONFRAG_IPV4_UDP) 359 #define ICE_RSS_TYPE_ETH_IPV4_TCP (ICE_RSS_TYPE_ETH_IPV4 | \ 360 ETH_RSS_NONFRAG_IPV4_TCP) 361 #define ICE_RSS_TYPE_ETH_IPV4_SCTP (ICE_RSS_TYPE_ETH_IPV4 | \ 362 ETH_RSS_NONFRAG_IPV4_SCTP) 363 #define ICE_RSS_TYPE_IPV4 ETH_RSS_IPV4 364 #define ICE_RSS_TYPE_IPV4_UDP (ETH_RSS_IPV4 | \ 365 ETH_RSS_NONFRAG_IPV4_UDP) 366 #define ICE_RSS_TYPE_IPV4_TCP (ETH_RSS_IPV4 | \ 367 ETH_RSS_NONFRAG_IPV4_TCP) 368 #define ICE_RSS_TYPE_IPV4_SCTP (ETH_RSS_IPV4 | \ 369 ETH_RSS_NONFRAG_IPV4_SCTP) 370 371 /* IPv6 */ 372 #define ICE_RSS_TYPE_ETH_IPV6 (ETH_RSS_ETH | ETH_RSS_IPV6) 373 #define ICE_RSS_TYPE_ETH_IPV6_UDP (ICE_RSS_TYPE_ETH_IPV6 | \ 374 ETH_RSS_NONFRAG_IPV6_UDP) 375 #define ICE_RSS_TYPE_ETH_IPV6_TCP (ICE_RSS_TYPE_ETH_IPV6 | \ 376 ETH_RSS_NONFRAG_IPV6_TCP) 377 #define ICE_RSS_TYPE_ETH_IPV6_SCTP (ICE_RSS_TYPE_ETH_IPV6 | \ 378 ETH_RSS_NONFRAG_IPV6_SCTP) 379 #define ICE_RSS_TYPE_IPV6 ETH_RSS_IPV6 380 #define ICE_RSS_TYPE_IPV6_UDP (ETH_RSS_IPV6 | \ 381 ETH_RSS_NONFRAG_IPV6_UDP) 382 #define ICE_RSS_TYPE_IPV6_TCP (ETH_RSS_IPV6 | \ 383 ETH_RSS_NONFRAG_IPV6_TCP) 384 #define ICE_RSS_TYPE_IPV6_SCTP (ETH_RSS_IPV6 | \ 385 ETH_RSS_NONFRAG_IPV6_SCTP) 386 387 /* VLAN IPV4 */ 388 #define ICE_RSS_TYPE_VLAN_IPV4 (ICE_RSS_TYPE_IPV4 | \ 389 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 390 #define ICE_RSS_TYPE_VLAN_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 391 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 392 #define ICE_RSS_TYPE_VLAN_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 393 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 394 #define ICE_RSS_TYPE_VLAN_IPV4_SCTP (ICE_RSS_TYPE_IPV4_SCTP | \ 395 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 396 /* VLAN IPv6 */ 397 #define ICE_RSS_TYPE_VLAN_IPV6 (ICE_RSS_TYPE_IPV6 | \ 398 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 399 #define ICE_RSS_TYPE_VLAN_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 400 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 401 #define ICE_RSS_TYPE_VLAN_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 402 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 403 #define ICE_RSS_TYPE_VLAN_IPV6_SCTP (ICE_RSS_TYPE_IPV6_SCTP | \ 404 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 405 406 /* GTPU IPv4 */ 407 #define ICE_RSS_TYPE_GTPU_IPV4 (ICE_RSS_TYPE_IPV4 | \ 408 ETH_RSS_GTPU) 409 #define ICE_RSS_TYPE_GTPU_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 410 ETH_RSS_GTPU) 411 #define ICE_RSS_TYPE_GTPU_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 412 ETH_RSS_GTPU) 413 /* GTPU IPv6 */ 414 #define ICE_RSS_TYPE_GTPU_IPV6 (ICE_RSS_TYPE_IPV6 | \ 415 ETH_RSS_GTPU) 416 #define ICE_RSS_TYPE_GTPU_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 417 ETH_RSS_GTPU) 418 #define ICE_RSS_TYPE_GTPU_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 419 ETH_RSS_GTPU) 420 421 /* PPPOE */ 422 #define ICE_RSS_TYPE_PPPOE (ETH_RSS_ETH | ETH_RSS_PPPOE) 423 424 /* PPPOE IPv4 */ 425 #define ICE_RSS_TYPE_PPPOE_IPV4 (ICE_RSS_TYPE_IPV4 | \ 426 ICE_RSS_TYPE_PPPOE) 427 #define ICE_RSS_TYPE_PPPOE_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 428 ICE_RSS_TYPE_PPPOE) 429 #define ICE_RSS_TYPE_PPPOE_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 430 ICE_RSS_TYPE_PPPOE) 431 432 /* PPPOE IPv6 */ 433 #define ICE_RSS_TYPE_PPPOE_IPV6 (ICE_RSS_TYPE_IPV6 | \ 434 ICE_RSS_TYPE_PPPOE) 435 #define ICE_RSS_TYPE_PPPOE_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 436 ICE_RSS_TYPE_PPPOE) 437 #define ICE_RSS_TYPE_PPPOE_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 438 ICE_RSS_TYPE_PPPOE) 439 440 /* ESP, AH, L2TPV3 and PFCP */ 441 #define ICE_RSS_TYPE_IPV4_ESP (ETH_RSS_ESP | ETH_RSS_IPV4) 442 #define ICE_RSS_TYPE_IPV6_ESP (ETH_RSS_ESP | ETH_RSS_IPV6) 443 #define ICE_RSS_TYPE_IPV4_AH (ETH_RSS_AH | ETH_RSS_IPV4) 444 #define ICE_RSS_TYPE_IPV6_AH (ETH_RSS_AH | ETH_RSS_IPV6) 445 #define ICE_RSS_TYPE_IPV4_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV4) 446 #define ICE_RSS_TYPE_IPV6_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV6) 447 #define ICE_RSS_TYPE_IPV4_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV4) 448 #define ICE_RSS_TYPE_IPV6_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV6) 449 450 /** 451 * Supported pattern for hash. 452 * The first member is pattern item type, 453 * the second member is input set mask, 454 * the third member is ice_rss_hash_cfg template. 455 */ 456 static struct ice_pattern_match_item ice_hash_pattern_list[] = { 457 /* IPV4 */ 458 {pattern_eth_ipv4, ICE_RSS_TYPE_ETH_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 459 {pattern_eth_ipv4_udp, ICE_RSS_TYPE_ETH_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 460 {pattern_eth_ipv4_tcp, ICE_RSS_TYPE_ETH_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 461 {pattern_eth_ipv4_sctp, ICE_RSS_TYPE_ETH_IPV4_SCTP, ICE_INSET_NONE, &ipv4_sctp_tmplt}, 462 {pattern_eth_vlan_ipv4, ICE_RSS_TYPE_VLAN_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 463 {pattern_eth_vlan_ipv4_udp, ICE_RSS_TYPE_VLAN_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 464 {pattern_eth_vlan_ipv4_tcp, ICE_RSS_TYPE_VLAN_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 465 {pattern_eth_vlan_ipv4_sctp, ICE_RSS_TYPE_VLAN_IPV4_SCTP, ICE_INSET_NONE, &ipv4_sctp_tmplt}, 466 {pattern_eth_ipv4_gtpu_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt}, 467 {pattern_eth_ipv4_gtpu_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt}, 468 {pattern_eth_ipv4_gtpu_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt}, 469 {pattern_eth_ipv6_gtpu_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt}, 470 {pattern_eth_ipv6_gtpu_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt}, 471 {pattern_eth_ipv6_gtpu_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt}, 472 {pattern_eth_ipv4_gtpu_eh_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt}, 473 {pattern_eth_ipv4_gtpu_eh_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt}, 474 {pattern_eth_ipv4_gtpu_eh_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt}, 475 {pattern_eth_ipv6_gtpu_eh_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt}, 476 {pattern_eth_ipv6_gtpu_eh_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt}, 477 {pattern_eth_ipv6_gtpu_eh_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt}, 478 {pattern_eth_pppoes_ipv4, ICE_RSS_TYPE_PPPOE_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 479 {pattern_eth_pppoes_ipv4_udp, ICE_RSS_TYPE_PPPOE_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 480 {pattern_eth_pppoes_ipv4_tcp, ICE_RSS_TYPE_PPPOE_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 481 {pattern_eth_ipv4_esp, ICE_RSS_TYPE_IPV4_ESP, ICE_INSET_NONE, ð_ipv4_esp_tmplt}, 482 {pattern_eth_ipv4_udp_esp, ICE_RSS_TYPE_IPV4_ESP, ICE_INSET_NONE, ð_ipv4_udp_esp_tmplt}, 483 {pattern_eth_ipv4_ah, ICE_RSS_TYPE_IPV4_AH, ICE_INSET_NONE, ð_ipv4_ah_tmplt}, 484 {pattern_eth_ipv4_l2tp, ICE_RSS_TYPE_IPV4_L2TPV3, ICE_INSET_NONE, ð_ipv4_l2tpv3_tmplt}, 485 {pattern_eth_ipv4_pfcp, ICE_RSS_TYPE_IPV4_PFCP, ICE_INSET_NONE, ð_ipv4_pfcp_tmplt}, 486 /* IPV6 */ 487 {pattern_eth_ipv6, ICE_RSS_TYPE_ETH_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 488 {pattern_eth_ipv6_udp, ICE_RSS_TYPE_ETH_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 489 {pattern_eth_ipv6_tcp, ICE_RSS_TYPE_ETH_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 490 {pattern_eth_ipv6_sctp, ICE_RSS_TYPE_ETH_IPV6_SCTP, ICE_INSET_NONE, &ipv6_sctp_tmplt}, 491 {pattern_eth_vlan_ipv6, ICE_RSS_TYPE_VLAN_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 492 {pattern_eth_vlan_ipv6_udp, ICE_RSS_TYPE_VLAN_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 493 {pattern_eth_vlan_ipv6_tcp, ICE_RSS_TYPE_VLAN_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 494 {pattern_eth_vlan_ipv6_sctp, ICE_RSS_TYPE_VLAN_IPV6_SCTP, ICE_INSET_NONE, &ipv6_sctp_tmplt}, 495 {pattern_eth_ipv4_gtpu_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt}, 496 {pattern_eth_ipv4_gtpu_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt}, 497 {pattern_eth_ipv4_gtpu_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt}, 498 {pattern_eth_ipv6_gtpu_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt}, 499 {pattern_eth_ipv6_gtpu_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt}, 500 {pattern_eth_ipv6_gtpu_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt}, 501 {pattern_eth_ipv4_gtpu_eh_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt}, 502 {pattern_eth_ipv4_gtpu_eh_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt}, 503 {pattern_eth_ipv4_gtpu_eh_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt}, 504 {pattern_eth_ipv6_gtpu_eh_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt}, 505 {pattern_eth_ipv6_gtpu_eh_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt}, 506 {pattern_eth_ipv6_gtpu_eh_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt}, 507 {pattern_eth_pppoes_ipv6, ICE_RSS_TYPE_PPPOE_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 508 {pattern_eth_pppoes_ipv6_udp, ICE_RSS_TYPE_PPPOE_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 509 {pattern_eth_pppoes_ipv6_tcp, ICE_RSS_TYPE_PPPOE_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 510 {pattern_eth_ipv6_esp, ICE_RSS_TYPE_IPV6_ESP, ICE_INSET_NONE, ð_ipv6_esp_tmplt}, 511 {pattern_eth_ipv6_udp_esp, ICE_RSS_TYPE_IPV6_ESP, ICE_INSET_NONE, ð_ipv6_udp_esp_tmplt}, 512 {pattern_eth_ipv6_ah, ICE_RSS_TYPE_IPV6_AH, ICE_INSET_NONE, ð_ipv6_ah_tmplt}, 513 {pattern_eth_ipv6_l2tp, ICE_RSS_TYPE_IPV6_L2TPV3, ICE_INSET_NONE, ð_ipv6_l2tpv3_tmplt}, 514 {pattern_eth_ipv6_pfcp, ICE_RSS_TYPE_IPV6_PFCP, ICE_INSET_NONE, ð_ipv6_pfcp_tmplt}, 515 /* PPPOE */ 516 {pattern_eth_pppoes, ICE_RSS_TYPE_PPPOE, ICE_INSET_NONE, &pppoe_tmplt}, 517 /* EMPTY */ 518 {pattern_empty, ICE_INSET_NONE, ICE_INSET_NONE, &empty_tmplt}, 519 }; 520 521 static struct ice_flow_engine ice_hash_engine = { 522 .init = ice_hash_init, 523 .create = ice_hash_create, 524 .destroy = ice_hash_destroy, 525 .uninit = ice_hash_uninit, 526 .free = ice_hash_free, 527 .type = ICE_FLOW_ENGINE_HASH, 528 }; 529 530 /* Register parser for os package. */ 531 static struct ice_flow_parser ice_hash_parser = { 532 .engine = &ice_hash_engine, 533 .array = ice_hash_pattern_list, 534 .array_len = RTE_DIM(ice_hash_pattern_list), 535 .parse_pattern_action = ice_hash_parse_pattern_action, 536 .stage = ICE_FLOW_STAGE_RSS, 537 }; 538 539 RTE_INIT(ice_hash_engine_init) 540 { 541 struct ice_flow_engine *engine = &ice_hash_engine; 542 ice_register_flow_engine(engine); 543 } 544 545 static int 546 ice_hash_init(struct ice_adapter *ad) 547 { 548 struct ice_flow_parser *parser = NULL; 549 550 if (ad->hw.dcf_enabled) 551 return 0; 552 553 parser = &ice_hash_parser; 554 555 return ice_register_parser(parser, ad); 556 } 557 558 static int 559 ice_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint, 560 struct rte_flow_error *error) 561 { 562 const struct rte_flow_item *item = pattern; 563 const struct rte_flow_item_gtp_psc *psc; 564 565 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 566 if (item->last) { 567 rte_flow_error_set(error, EINVAL, 568 RTE_FLOW_ERROR_TYPE_ITEM, item, 569 "Not support range"); 570 return -rte_errno; 571 } 572 573 switch (item->type) { 574 case RTE_FLOW_ITEM_TYPE_VLAN: 575 *phint |= ICE_PHINT_VLAN; 576 break; 577 case RTE_FLOW_ITEM_TYPE_PPPOES: 578 *phint |= ICE_PHINT_PPPOE; 579 break; 580 case RTE_FLOW_ITEM_TYPE_GTPU: 581 *phint |= ICE_PHINT_GTPU; 582 break; 583 case RTE_FLOW_ITEM_TYPE_GTP_PSC: 584 *phint |= ICE_PHINT_GTPU_EH; 585 psc = item->spec; 586 if (!psc) 587 break; 588 else if (psc->pdu_type == ICE_GTPU_EH_UPLINK) 589 *phint |= ICE_PHINT_GTPU_EH_UP; 590 else if (psc->pdu_type == ICE_GTPU_EH_DWNLINK) 591 *phint |= ICE_PHINT_GTPU_EH_DWN; 592 break; 593 default: 594 break; 595 } 596 } 597 598 return 0; 599 } 600 601 static void 602 ice_refine_hash_cfg_l234(struct ice_rss_hash_cfg *hash_cfg, 603 uint64_t rss_type) 604 { 605 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 606 uint64_t *hash_flds = &hash_cfg->hash_flds; 607 608 if (*addl_hdrs & ICE_FLOW_SEG_HDR_ETH) { 609 if (!(rss_type & ETH_RSS_ETH)) 610 *hash_flds &= ~ICE_FLOW_HASH_ETH; 611 if (rss_type & ETH_RSS_L2_SRC_ONLY) 612 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA)); 613 else if (rss_type & ETH_RSS_L2_DST_ONLY) 614 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)); 615 *addl_hdrs &= ~ICE_FLOW_SEG_HDR_ETH; 616 } 617 618 if (*addl_hdrs & ICE_FLOW_SEG_HDR_VLAN) { 619 if (rss_type & ETH_RSS_C_VLAN) 620 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN); 621 else if (rss_type & ETH_RSS_S_VLAN) 622 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN); 623 } 624 625 if (*addl_hdrs & ICE_FLOW_SEG_HDR_PPPOE) { 626 if (!(rss_type & ETH_RSS_PPPOE)) 627 *hash_flds &= ~ICE_FLOW_HASH_PPPOE_SESS_ID; 628 } 629 630 if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV4) { 631 if (rss_type & 632 (ETH_RSS_IPV4 | 633 ETH_RSS_NONFRAG_IPV4_UDP | 634 ETH_RSS_NONFRAG_IPV4_TCP | 635 ETH_RSS_NONFRAG_IPV4_SCTP)) { 636 if (rss_type & ETH_RSS_L3_SRC_ONLY) 637 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)); 638 else if (rss_type & ETH_RSS_L3_DST_ONLY) 639 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)); 640 else if (rss_type & 641 (ETH_RSS_L4_SRC_ONLY | 642 ETH_RSS_L4_DST_ONLY)) 643 *hash_flds &= ~ICE_FLOW_HASH_IPV4; 644 } else { 645 *hash_flds &= ~ICE_FLOW_HASH_IPV4; 646 } 647 } 648 649 if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV6) { 650 if (rss_type & 651 (ETH_RSS_IPV6 | 652 ETH_RSS_NONFRAG_IPV6_UDP | 653 ETH_RSS_NONFRAG_IPV6_TCP | 654 ETH_RSS_NONFRAG_IPV6_SCTP)) { 655 if (rss_type & ETH_RSS_L3_SRC_ONLY) 656 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 657 else if (rss_type & ETH_RSS_L3_DST_ONLY) 658 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 659 else if (rss_type & 660 (ETH_RSS_L4_SRC_ONLY | 661 ETH_RSS_L4_DST_ONLY)) 662 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 663 } else { 664 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 665 } 666 667 if (rss_type & RTE_ETH_RSS_L3_PRE32) { 668 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 669 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 670 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_SA)); 671 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 672 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 673 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_DA)); 674 } else { 675 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 676 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE32; 677 } 678 } 679 if (rss_type & RTE_ETH_RSS_L3_PRE48) { 680 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 681 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 682 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_SA)); 683 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 684 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 685 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_DA)); 686 } else { 687 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 688 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE48; 689 } 690 } 691 if (rss_type & RTE_ETH_RSS_L3_PRE64) { 692 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 693 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 694 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_SA)); 695 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 696 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 697 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_DA)); 698 } else { 699 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 700 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE64; 701 } 702 } 703 } 704 705 if (*addl_hdrs & ICE_FLOW_SEG_HDR_UDP) { 706 if (rss_type & 707 (ETH_RSS_NONFRAG_IPV4_UDP | 708 ETH_RSS_NONFRAG_IPV6_UDP)) { 709 if (rss_type & ETH_RSS_L4_SRC_ONLY) 710 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)); 711 else if (rss_type & ETH_RSS_L4_DST_ONLY) 712 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)); 713 else if (rss_type & 714 (ETH_RSS_L3_SRC_ONLY | 715 ETH_RSS_L3_DST_ONLY)) 716 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT; 717 } else { 718 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT; 719 } 720 } 721 722 if (*addl_hdrs & ICE_FLOW_SEG_HDR_TCP) { 723 if (rss_type & 724 (ETH_RSS_NONFRAG_IPV4_TCP | 725 ETH_RSS_NONFRAG_IPV6_TCP)) { 726 if (rss_type & ETH_RSS_L4_SRC_ONLY) 727 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)); 728 else if (rss_type & ETH_RSS_L4_DST_ONLY) 729 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)); 730 else if (rss_type & 731 (ETH_RSS_L3_SRC_ONLY | 732 ETH_RSS_L3_DST_ONLY)) 733 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT; 734 } else { 735 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT; 736 } 737 } 738 739 if (*addl_hdrs & ICE_FLOW_SEG_HDR_SCTP) { 740 if (rss_type & 741 (ETH_RSS_NONFRAG_IPV4_SCTP | 742 ETH_RSS_NONFRAG_IPV6_SCTP)) { 743 if (rss_type & ETH_RSS_L4_SRC_ONLY) 744 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)); 745 else if (rss_type & ETH_RSS_L4_DST_ONLY) 746 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)); 747 else if (rss_type & 748 (ETH_RSS_L3_SRC_ONLY | 749 ETH_RSS_L3_DST_ONLY)) 750 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT; 751 } else { 752 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT; 753 } 754 } 755 756 if (*addl_hdrs & ICE_FLOW_SEG_HDR_L2TPV3) { 757 if (!(rss_type & ETH_RSS_L2TPV3)) 758 *hash_flds &= ~ICE_FLOW_HASH_L2TPV3_SESS_ID; 759 } 760 761 if (*addl_hdrs & ICE_FLOW_SEG_HDR_ESP) { 762 if (!(rss_type & ETH_RSS_ESP)) 763 *hash_flds &= ~ICE_FLOW_HASH_ESP_SPI; 764 } 765 766 if (*addl_hdrs & ICE_FLOW_SEG_HDR_AH) { 767 if (!(rss_type & ETH_RSS_AH)) 768 *hash_flds &= ~ICE_FLOW_HASH_AH_SPI; 769 } 770 771 if (*addl_hdrs & ICE_FLOW_SEG_HDR_PFCP_SESSION) { 772 if (!(rss_type & ETH_RSS_PFCP)) 773 *hash_flds &= ~ICE_FLOW_HASH_PFCP_SEID; 774 } 775 } 776 777 static void 778 ice_refine_proto_hdrs_by_pattern(struct ice_rss_hash_cfg *hash_cfg, 779 uint64_t phint) 780 { 781 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 782 if (phint & ICE_PHINT_VLAN) 783 *addl_hdrs |= ICE_FLOW_SEG_HDR_VLAN; 784 785 if (phint & ICE_PHINT_PPPOE) 786 *addl_hdrs |= ICE_FLOW_SEG_HDR_PPPOE; 787 788 if (phint & ICE_PHINT_GTPU_EH_DWN) 789 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_DWN; 790 else if (phint & ICE_PHINT_GTPU_EH_UP) 791 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_UP; 792 else if (phint & ICE_PHINT_GTPU_EH) 793 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_EH; 794 else if (phint & ICE_PHINT_GTPU) 795 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_IP; 796 } 797 798 static void 799 ice_refine_hash_cfg_gtpu(struct ice_rss_hash_cfg *hash_cfg, 800 uint64_t rss_type) 801 { 802 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 803 uint64_t *hash_flds = &hash_cfg->hash_flds; 804 805 /* update hash field for gtpu eh/gtpu dwn/gtpu up. */ 806 if (!(rss_type & ETH_RSS_GTPU)) 807 return; 808 809 if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_DWN) 810 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_DWN_TEID); 811 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_UP) 812 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_UP_TEID); 813 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_EH) 814 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_EH_TEID); 815 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_IP) 816 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID); 817 } 818 819 static void ice_refine_hash_cfg(struct ice_rss_hash_cfg *hash_cfg, 820 uint64_t rss_type, uint64_t phint) 821 { 822 ice_refine_proto_hdrs_by_pattern(hash_cfg, phint); 823 ice_refine_hash_cfg_l234(hash_cfg, rss_type); 824 ice_refine_hash_cfg_gtpu(hash_cfg, rss_type); 825 } 826 827 static uint64_t invalid_rss_comb[] = { 828 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP, 829 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP, 830 RTE_ETH_RSS_L3_PRE40 | 831 RTE_ETH_RSS_L3_PRE56 | 832 RTE_ETH_RSS_L3_PRE96 833 }; 834 835 struct rss_attr_type { 836 uint64_t attr; 837 uint64_t type; 838 }; 839 840 static struct rss_attr_type rss_attr_to_valid_type[] = { 841 {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY, ETH_RSS_ETH}, 842 {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY, VALID_RSS_L3}, 843 {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY, VALID_RSS_L4}, 844 /* current ipv6 prefix only supports prefix 64 bits*/ 845 {RTE_ETH_RSS_L3_PRE32, VALID_RSS_IPV6}, 846 {RTE_ETH_RSS_L3_PRE48, VALID_RSS_IPV6}, 847 {RTE_ETH_RSS_L3_PRE64, VALID_RSS_IPV6}, 848 {INVALID_RSS_ATTR, 0} 849 }; 850 851 static bool 852 ice_any_invalid_rss_type(enum rte_eth_hash_function rss_func, 853 uint64_t rss_type, uint64_t allow_rss_type) 854 { 855 uint32_t i; 856 857 /** 858 * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ 859 * hash function. 860 */ 861 if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 862 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY | 863 ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) 864 return true; 865 866 if (!(rss_type & 867 (ETH_RSS_IPV4 | ETH_RSS_IPV6 | 868 ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP | 869 ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP | 870 ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP))) 871 return true; 872 } 873 874 /* check invalid combination */ 875 for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) { 876 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1) 877 return true; 878 } 879 880 /* check invalid RSS attribute */ 881 for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) { 882 struct rss_attr_type *rat = &rss_attr_to_valid_type[i]; 883 884 if (rat->attr & rss_type && !(rat->type & rss_type)) 885 return true; 886 } 887 888 /* check not allowed RSS type */ 889 rss_type &= ~VALID_RSS_ATTR; 890 891 return ((rss_type & allow_rss_type) != rss_type); 892 } 893 894 static int 895 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item, 896 const struct rte_flow_action actions[], 897 uint64_t pattern_hint, void **meta, 898 struct rte_flow_error *error) 899 { 900 struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)*meta; 901 struct ice_rss_hash_cfg *cfg = pattern_match_item->meta; 902 enum rte_flow_action_type action_type; 903 const struct rte_flow_action_rss *rss; 904 const struct rte_flow_action *action; 905 uint64_t rss_type; 906 907 /* Supported action is RSS. */ 908 for (action = actions; action->type != 909 RTE_FLOW_ACTION_TYPE_END; action++) { 910 action_type = action->type; 911 switch (action_type) { 912 case RTE_FLOW_ACTION_TYPE_RSS: 913 rss = action->conf; 914 rss_type = rss->types; 915 916 /* Check hash function and save it to rss_meta. */ 917 if (pattern_match_item->pattern_list != 918 pattern_empty && rss->func == 919 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { 920 return rte_flow_error_set(error, ENOTSUP, 921 RTE_FLOW_ERROR_TYPE_ACTION, action, 922 "Not supported flow"); 923 } else if (rss->func == 924 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){ 925 rss_meta->hash_function = 926 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR; 927 return 0; 928 } else if (rss->func == 929 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 930 rss_meta->hash_function = 931 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ; 932 cfg->symm = true; 933 } 934 935 if (rss->level) 936 return rte_flow_error_set(error, ENOTSUP, 937 RTE_FLOW_ERROR_TYPE_ACTION, action, 938 "a nonzero RSS encapsulation level is not supported"); 939 940 if (rss->key_len) 941 return rte_flow_error_set(error, ENOTSUP, 942 RTE_FLOW_ERROR_TYPE_ACTION, action, 943 "a nonzero RSS key_len is not supported"); 944 945 if (rss->queue) 946 return rte_flow_error_set(error, ENOTSUP, 947 RTE_FLOW_ERROR_TYPE_ACTION, action, 948 "a non-NULL RSS queue is not supported"); 949 950 /** 951 * Check simultaneous use of SRC_ONLY and DST_ONLY 952 * of the same level. 953 */ 954 rss_type = rte_eth_rss_hf_refine(rss_type); 955 956 if (ice_any_invalid_rss_type(rss->func, rss_type, 957 pattern_match_item->input_set_mask_o)) 958 return rte_flow_error_set(error, ENOTSUP, 959 RTE_FLOW_ERROR_TYPE_ACTION, 960 action, "RSS type not supported"); 961 962 rss_meta->cfg = *cfg; 963 ice_refine_hash_cfg(&rss_meta->cfg, 964 rss_type, pattern_hint); 965 break; 966 case RTE_FLOW_ACTION_TYPE_END: 967 break; 968 969 default: 970 rte_flow_error_set(error, EINVAL, 971 RTE_FLOW_ERROR_TYPE_ACTION, action, 972 "Invalid action."); 973 return -rte_errno; 974 } 975 } 976 977 return 0; 978 } 979 980 static int 981 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad, 982 struct ice_pattern_match_item *array, 983 uint32_t array_len, 984 const struct rte_flow_item pattern[], 985 const struct rte_flow_action actions[], 986 void **meta, 987 struct rte_flow_error *error) 988 { 989 int ret = 0; 990 struct ice_pattern_match_item *pattern_match_item; 991 struct ice_rss_meta *rss_meta_ptr; 992 uint64_t phint = ICE_PHINT_NONE; 993 994 rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0); 995 if (!rss_meta_ptr) { 996 rte_flow_error_set(error, EINVAL, 997 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 998 "No memory for rss_meta_ptr"); 999 return -ENOMEM; 1000 } 1001 1002 /* Check rss supported pattern and find matched pattern. */ 1003 pattern_match_item = ice_search_pattern_match_item(ad, pattern, array, 1004 array_len, error); 1005 if (!pattern_match_item) { 1006 ret = -rte_errno; 1007 goto error; 1008 } 1009 1010 ret = ice_hash_parse_pattern(pattern, &phint, error); 1011 if (ret) 1012 goto error; 1013 1014 /* Check rss action. */ 1015 ret = ice_hash_parse_action(pattern_match_item, actions, phint, 1016 (void **)&rss_meta_ptr, error); 1017 1018 error: 1019 if (!ret && meta) 1020 *meta = rss_meta_ptr; 1021 else 1022 rte_free(rss_meta_ptr); 1023 rte_free(pattern_match_item); 1024 1025 return ret; 1026 } 1027 1028 static int 1029 ice_hash_create(struct ice_adapter *ad, 1030 struct rte_flow *flow, 1031 void *meta, 1032 struct rte_flow_error *error) 1033 { 1034 struct ice_pf *pf = &ad->pf; 1035 struct ice_hw *hw = ICE_PF_TO_HW(pf); 1036 struct ice_vsi *vsi = pf->main_vsi; 1037 int ret; 1038 uint32_t reg; 1039 struct ice_hash_flow_cfg *filter_ptr; 1040 struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)meta; 1041 uint8_t hash_function = rss_meta->hash_function; 1042 1043 filter_ptr = rte_zmalloc("ice_rss_filter", 1044 sizeof(struct ice_hash_flow_cfg), 0); 1045 if (!filter_ptr) { 1046 rte_flow_error_set(error, EINVAL, 1047 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1048 "No memory for filter_ptr"); 1049 return -ENOMEM; 1050 } 1051 1052 if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { 1053 /* Enable registers for simple_xor hash function. */ 1054 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id)); 1055 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) | 1056 (2 << VSIQF_HASH_CTL_HASH_SCHEME_S); 1057 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg); 1058 1059 filter_ptr->simple_xor = 1; 1060 1061 goto out; 1062 } else { 1063 memcpy(&filter_ptr->rss_cfg.hash, &rss_meta->cfg, 1064 sizeof(struct ice_rss_hash_cfg)); 1065 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, 1066 &filter_ptr->rss_cfg.hash); 1067 if (ret) { 1068 rte_flow_error_set(error, EINVAL, 1069 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1070 "rss flow create fail"); 1071 goto error; 1072 } 1073 } 1074 1075 out: 1076 flow->rule = filter_ptr; 1077 rte_free(meta); 1078 return 0; 1079 1080 error: 1081 rte_free(filter_ptr); 1082 rte_free(meta); 1083 return -rte_errno; 1084 } 1085 1086 static int 1087 ice_hash_destroy(struct ice_adapter *ad, 1088 struct rte_flow *flow, 1089 struct rte_flow_error *error) 1090 { 1091 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad); 1092 struct ice_hw *hw = ICE_PF_TO_HW(pf); 1093 struct ice_vsi *vsi = pf->main_vsi; 1094 int ret; 1095 uint32_t reg; 1096 struct ice_hash_flow_cfg *filter_ptr; 1097 1098 filter_ptr = (struct ice_hash_flow_cfg *)flow->rule; 1099 1100 if (filter_ptr->simple_xor == 1) { 1101 /* Return to symmetric_toeplitz state. */ 1102 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id)); 1103 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) | 1104 (1 << VSIQF_HASH_CTL_HASH_SCHEME_S); 1105 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg); 1106 } else { 1107 ret = ice_rem_rss_cfg_wrap(pf, vsi->idx, 1108 &filter_ptr->rss_cfg.hash); 1109 /* Fixme: Ignore the error if a rule does not exist. 1110 * Currently a rule for inputset change or symm turn on/off 1111 * will overwrite an exist rule, while application still 1112 * have 2 rte_flow handles. 1113 **/ 1114 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) { 1115 rte_flow_error_set(error, EINVAL, 1116 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1117 "rss flow destroy fail"); 1118 goto error; 1119 } 1120 } 1121 1122 rte_free(filter_ptr); 1123 return 0; 1124 1125 error: 1126 rte_free(filter_ptr); 1127 return -rte_errno; 1128 } 1129 1130 static void 1131 ice_hash_uninit(struct ice_adapter *ad) 1132 { 1133 if (ad->hw.dcf_enabled) 1134 return; 1135 1136 ice_unregister_parser(&ice_hash_parser, ad); 1137 } 1138 1139 static void 1140 ice_hash_free(struct rte_flow *flow) 1141 { 1142 rte_free(flow->rule); 1143 } 1144