1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 13 #include <rte_debug.h> 14 #include <rte_ether.h> 15 #include <ethdev_driver.h> 16 #include <rte_log.h> 17 #include <rte_malloc.h> 18 #include <rte_eth_ctrl.h> 19 #include <rte_tailq.h> 20 #include <rte_flow_driver.h> 21 22 #include "ice_logs.h" 23 #include "base/ice_type.h" 24 #include "base/ice_flow.h" 25 #include "ice_ethdev.h" 26 #include "ice_generic_flow.h" 27 28 #define ICE_PHINT_NONE 0 29 #define ICE_PHINT_VLAN BIT_ULL(0) 30 #define ICE_PHINT_PPPOE BIT_ULL(1) 31 #define ICE_PHINT_GTPU BIT_ULL(2) 32 #define ICE_PHINT_GTPU_EH BIT_ULL(3) 33 #define ICE_PHINT_GTPU_EH_DWN BIT_ULL(4) 34 #define ICE_PHINT_GTPU_EH_UP BIT_ULL(5) 35 36 #define ICE_GTPU_EH_DWNLINK 0 37 #define ICE_GTPU_EH_UPLINK 1 38 39 #define ICE_IPV4_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) 40 #define ICE_IPV6_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) 41 42 #define VALID_RSS_IPV4_L4 (ETH_RSS_NONFRAG_IPV4_UDP | \ 43 ETH_RSS_NONFRAG_IPV4_TCP | \ 44 ETH_RSS_NONFRAG_IPV4_SCTP) 45 46 #define VALID_RSS_IPV6_L4 (ETH_RSS_NONFRAG_IPV6_UDP | \ 47 ETH_RSS_NONFRAG_IPV6_TCP | \ 48 ETH_RSS_NONFRAG_IPV6_SCTP) 49 50 #define VALID_RSS_IPV4 (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4) 51 #define VALID_RSS_IPV6 (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4) 52 #define VALID_RSS_L3 (VALID_RSS_IPV4 | VALID_RSS_IPV6) 53 #define VALID_RSS_L4 (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4) 54 55 #define VALID_RSS_ATTR (ETH_RSS_L3_SRC_ONLY | \ 56 ETH_RSS_L3_DST_ONLY | \ 57 ETH_RSS_L4_SRC_ONLY | \ 58 ETH_RSS_L4_DST_ONLY | \ 59 ETH_RSS_L2_SRC_ONLY | \ 60 ETH_RSS_L2_DST_ONLY | \ 61 RTE_ETH_RSS_L3_PRE32 | \ 62 RTE_ETH_RSS_L3_PRE48 | \ 63 RTE_ETH_RSS_L3_PRE64) 64 65 #define INVALID_RSS_ATTR (RTE_ETH_RSS_L3_PRE40 | \ 66 RTE_ETH_RSS_L3_PRE56 | \ 67 RTE_ETH_RSS_L3_PRE96) 68 69 struct ice_rss_meta { 70 uint8_t hash_function; 71 struct ice_rss_hash_cfg cfg; 72 }; 73 74 struct ice_hash_flow_cfg { 75 bool simple_xor; 76 struct ice_rss_cfg rss_cfg; 77 }; 78 79 static int 80 ice_hash_init(struct ice_adapter *ad); 81 82 static int 83 ice_hash_create(struct ice_adapter *ad, 84 struct rte_flow *flow, 85 void *meta, 86 struct rte_flow_error *error); 87 88 static int 89 ice_hash_destroy(struct ice_adapter *ad, 90 struct rte_flow *flow, 91 struct rte_flow_error *error); 92 93 static void 94 ice_hash_uninit(struct ice_adapter *ad); 95 96 static void 97 ice_hash_free(struct rte_flow *flow); 98 99 static int 100 ice_hash_parse_pattern_action(struct ice_adapter *ad, 101 struct ice_pattern_match_item *array, 102 uint32_t array_len, 103 const struct rte_flow_item pattern[], 104 const struct rte_flow_action actions[], 105 uint32_t priority, 106 void **meta, 107 struct rte_flow_error *error); 108 109 /* Rss configuration template */ 110 struct ice_rss_hash_cfg ipv4_tmplt = { 111 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 112 ICE_FLOW_SEG_HDR_IPV_OTHER, 113 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV4, 114 ICE_RSS_OUTER_HEADERS, 115 0 116 }; 117 118 struct ice_rss_hash_cfg ipv4_udp_tmplt = { 119 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 120 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP, 121 ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 122 ICE_RSS_OUTER_HEADERS, 123 0 124 }; 125 126 struct ice_rss_hash_cfg ipv4_tcp_tmplt = { 127 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 128 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP, 129 ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 130 ICE_RSS_OUTER_HEADERS, 131 0 132 }; 133 134 struct ice_rss_hash_cfg ipv4_sctp_tmplt = { 135 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 | 136 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP, 137 ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV4 | ICE_IPV4_PROT, 138 ICE_RSS_OUTER_HEADERS, 139 0 140 }; 141 142 struct ice_rss_hash_cfg ipv6_tmplt = { 143 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 144 ICE_FLOW_SEG_HDR_IPV_OTHER, 145 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV6, 146 ICE_RSS_OUTER_HEADERS, 147 0 148 }; 149 150 struct ice_rss_hash_cfg ipv6_udp_tmplt = { 151 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 152 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP, 153 ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 154 ICE_RSS_OUTER_HEADERS, 155 0 156 }; 157 158 struct ice_rss_hash_cfg ipv6_tcp_tmplt = { 159 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 160 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP, 161 ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 162 ICE_RSS_OUTER_HEADERS, 163 0 164 }; 165 166 struct ice_rss_hash_cfg ipv6_sctp_tmplt = { 167 ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 | 168 ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP, 169 ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV6 | ICE_IPV6_PROT, 170 ICE_RSS_OUTER_HEADERS, 171 0 172 }; 173 174 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tmplt = { 175 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, 176 ICE_FLOW_HASH_IPV4, 177 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 178 0 179 }; 180 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_udp_tmplt = { 181 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 182 ICE_FLOW_SEG_HDR_UDP, 183 ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 184 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 185 0 186 }; 187 188 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tcp_tmplt = { 189 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 190 ICE_FLOW_SEG_HDR_TCP, 191 ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 192 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 193 0 194 }; 195 196 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tmplt = { 197 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, 198 ICE_FLOW_HASH_IPV4, 199 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 200 0 201 }; 202 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_udp_tmplt = { 203 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 204 ICE_FLOW_SEG_HDR_UDP, 205 ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT, 206 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 207 0 208 }; 209 210 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tcp_tmplt = { 211 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 212 ICE_FLOW_SEG_HDR_TCP, 213 ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT, 214 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 215 0 216 }; 217 218 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tmplt = { 219 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, 220 ICE_FLOW_HASH_IPV6, 221 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 222 0 223 }; 224 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_udp_tmplt = { 225 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 226 ICE_FLOW_SEG_HDR_UDP, 227 ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 228 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 229 0 230 }; 231 232 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tcp_tmplt = { 233 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 234 ICE_FLOW_SEG_HDR_TCP, 235 ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 236 ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, 237 0 238 }; 239 240 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tmplt = { 241 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, 242 ICE_FLOW_HASH_IPV6, 243 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 244 0 245 }; 246 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_udp_tmplt = { 247 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 248 ICE_FLOW_SEG_HDR_UDP, 249 ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT, 250 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 251 0 252 }; 253 254 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tcp_tmplt = { 255 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 256 ICE_FLOW_SEG_HDR_TCP, 257 ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT, 258 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, 259 0 260 }; 261 262 struct ice_rss_hash_cfg eth_ipv4_esp_tmplt = { 263 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 264 ICE_FLOW_SEG_HDR_ESP, 265 ICE_FLOW_HASH_ESP_SPI, 266 ICE_RSS_OUTER_HEADERS, 267 0 268 }; 269 270 struct ice_rss_hash_cfg eth_ipv4_udp_esp_tmplt = { 271 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 272 ICE_FLOW_SEG_HDR_NAT_T_ESP, 273 ICE_FLOW_HASH_NAT_T_ESP_SPI, 274 ICE_RSS_OUTER_HEADERS, 275 0 276 }; 277 278 struct ice_rss_hash_cfg eth_ipv4_ah_tmplt = { 279 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 280 ICE_FLOW_SEG_HDR_AH, 281 ICE_FLOW_HASH_AH_SPI, 282 ICE_RSS_OUTER_HEADERS, 283 0 284 }; 285 286 struct ice_rss_hash_cfg eth_ipv4_l2tpv3_tmplt = { 287 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 288 ICE_FLOW_SEG_HDR_L2TPV3, 289 ICE_FLOW_HASH_L2TPV3_SESS_ID, 290 ICE_RSS_OUTER_HEADERS, 291 0 292 }; 293 294 struct ice_rss_hash_cfg eth_ipv4_pfcp_tmplt = { 295 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER | 296 ICE_FLOW_SEG_HDR_PFCP_SESSION, 297 ICE_FLOW_HASH_PFCP_SEID, 298 ICE_RSS_OUTER_HEADERS, 299 0 300 }; 301 302 struct ice_rss_hash_cfg eth_ipv6_esp_tmplt = { 303 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 304 ICE_FLOW_SEG_HDR_ESP, 305 ICE_FLOW_HASH_ESP_SPI, 306 ICE_RSS_OUTER_HEADERS, 307 0 308 }; 309 310 struct ice_rss_hash_cfg eth_ipv6_udp_esp_tmplt = { 311 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 312 ICE_FLOW_SEG_HDR_NAT_T_ESP, 313 ICE_FLOW_HASH_NAT_T_ESP_SPI, 314 ICE_RSS_OUTER_HEADERS, 315 0 316 }; 317 318 struct ice_rss_hash_cfg eth_ipv6_ah_tmplt = { 319 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 320 ICE_FLOW_SEG_HDR_AH, 321 ICE_FLOW_HASH_AH_SPI, 322 ICE_RSS_OUTER_HEADERS, 323 0 324 }; 325 326 struct ice_rss_hash_cfg eth_ipv6_l2tpv3_tmplt = { 327 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 328 ICE_FLOW_SEG_HDR_L2TPV3, 329 ICE_FLOW_HASH_L2TPV3_SESS_ID, 330 ICE_RSS_OUTER_HEADERS, 331 0 332 }; 333 334 struct ice_rss_hash_cfg eth_ipv6_pfcp_tmplt = { 335 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER | 336 ICE_FLOW_SEG_HDR_PFCP_SESSION, 337 ICE_FLOW_HASH_PFCP_SEID, 338 ICE_RSS_OUTER_HEADERS, 339 0 340 }; 341 342 struct ice_rss_hash_cfg pppoe_tmplt = { 343 ICE_FLOW_SEG_HDR_ETH, 344 ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_PPPOE_SESS_ID, 345 ICE_RSS_OUTER_HEADERS, 346 0 347 }; 348 349 struct ice_rss_hash_cfg empty_tmplt = { 350 ICE_FLOW_SEG_HDR_NONE, 351 0, 352 ICE_RSS_ANY_HEADERS, 353 0 354 }; 355 356 /* IPv4 */ 357 #define ICE_RSS_TYPE_ETH_IPV4 (ETH_RSS_ETH | ETH_RSS_IPV4) 358 #define ICE_RSS_TYPE_ETH_IPV4_UDP (ICE_RSS_TYPE_ETH_IPV4 | \ 359 ETH_RSS_NONFRAG_IPV4_UDP) 360 #define ICE_RSS_TYPE_ETH_IPV4_TCP (ICE_RSS_TYPE_ETH_IPV4 | \ 361 ETH_RSS_NONFRAG_IPV4_TCP) 362 #define ICE_RSS_TYPE_ETH_IPV4_SCTP (ICE_RSS_TYPE_ETH_IPV4 | \ 363 ETH_RSS_NONFRAG_IPV4_SCTP) 364 #define ICE_RSS_TYPE_IPV4 ETH_RSS_IPV4 365 #define ICE_RSS_TYPE_IPV4_UDP (ETH_RSS_IPV4 | \ 366 ETH_RSS_NONFRAG_IPV4_UDP) 367 #define ICE_RSS_TYPE_IPV4_TCP (ETH_RSS_IPV4 | \ 368 ETH_RSS_NONFRAG_IPV4_TCP) 369 #define ICE_RSS_TYPE_IPV4_SCTP (ETH_RSS_IPV4 | \ 370 ETH_RSS_NONFRAG_IPV4_SCTP) 371 372 /* IPv6 */ 373 #define ICE_RSS_TYPE_ETH_IPV6 (ETH_RSS_ETH | ETH_RSS_IPV6) 374 #define ICE_RSS_TYPE_ETH_IPV6_UDP (ICE_RSS_TYPE_ETH_IPV6 | \ 375 ETH_RSS_NONFRAG_IPV6_UDP) 376 #define ICE_RSS_TYPE_ETH_IPV6_TCP (ICE_RSS_TYPE_ETH_IPV6 | \ 377 ETH_RSS_NONFRAG_IPV6_TCP) 378 #define ICE_RSS_TYPE_ETH_IPV6_SCTP (ICE_RSS_TYPE_ETH_IPV6 | \ 379 ETH_RSS_NONFRAG_IPV6_SCTP) 380 #define ICE_RSS_TYPE_IPV6 ETH_RSS_IPV6 381 #define ICE_RSS_TYPE_IPV6_UDP (ETH_RSS_IPV6 | \ 382 ETH_RSS_NONFRAG_IPV6_UDP) 383 #define ICE_RSS_TYPE_IPV6_TCP (ETH_RSS_IPV6 | \ 384 ETH_RSS_NONFRAG_IPV6_TCP) 385 #define ICE_RSS_TYPE_IPV6_SCTP (ETH_RSS_IPV6 | \ 386 ETH_RSS_NONFRAG_IPV6_SCTP) 387 388 /* VLAN IPV4 */ 389 #define ICE_RSS_TYPE_VLAN_IPV4 (ICE_RSS_TYPE_IPV4 | \ 390 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 391 #define ICE_RSS_TYPE_VLAN_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 392 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 393 #define ICE_RSS_TYPE_VLAN_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 394 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 395 #define ICE_RSS_TYPE_VLAN_IPV4_SCTP (ICE_RSS_TYPE_IPV4_SCTP | \ 396 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 397 /* VLAN IPv6 */ 398 #define ICE_RSS_TYPE_VLAN_IPV6 (ICE_RSS_TYPE_IPV6 | \ 399 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 400 #define ICE_RSS_TYPE_VLAN_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 401 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 402 #define ICE_RSS_TYPE_VLAN_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 403 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 404 #define ICE_RSS_TYPE_VLAN_IPV6_SCTP (ICE_RSS_TYPE_IPV6_SCTP | \ 405 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN) 406 407 /* GTPU IPv4 */ 408 #define ICE_RSS_TYPE_GTPU_IPV4 (ICE_RSS_TYPE_IPV4 | \ 409 ETH_RSS_GTPU) 410 #define ICE_RSS_TYPE_GTPU_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 411 ETH_RSS_GTPU) 412 #define ICE_RSS_TYPE_GTPU_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 413 ETH_RSS_GTPU) 414 /* GTPU IPv6 */ 415 #define ICE_RSS_TYPE_GTPU_IPV6 (ICE_RSS_TYPE_IPV6 | \ 416 ETH_RSS_GTPU) 417 #define ICE_RSS_TYPE_GTPU_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 418 ETH_RSS_GTPU) 419 #define ICE_RSS_TYPE_GTPU_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 420 ETH_RSS_GTPU) 421 422 /* PPPOE */ 423 #define ICE_RSS_TYPE_PPPOE (ETH_RSS_ETH | ETH_RSS_PPPOE) 424 425 /* PPPOE IPv4 */ 426 #define ICE_RSS_TYPE_PPPOE_IPV4 (ICE_RSS_TYPE_IPV4 | \ 427 ICE_RSS_TYPE_PPPOE) 428 #define ICE_RSS_TYPE_PPPOE_IPV4_UDP (ICE_RSS_TYPE_IPV4_UDP | \ 429 ICE_RSS_TYPE_PPPOE) 430 #define ICE_RSS_TYPE_PPPOE_IPV4_TCP (ICE_RSS_TYPE_IPV4_TCP | \ 431 ICE_RSS_TYPE_PPPOE) 432 433 /* PPPOE IPv6 */ 434 #define ICE_RSS_TYPE_PPPOE_IPV6 (ICE_RSS_TYPE_IPV6 | \ 435 ICE_RSS_TYPE_PPPOE) 436 #define ICE_RSS_TYPE_PPPOE_IPV6_UDP (ICE_RSS_TYPE_IPV6_UDP | \ 437 ICE_RSS_TYPE_PPPOE) 438 #define ICE_RSS_TYPE_PPPOE_IPV6_TCP (ICE_RSS_TYPE_IPV6_TCP | \ 439 ICE_RSS_TYPE_PPPOE) 440 441 /* ESP, AH, L2TPV3 and PFCP */ 442 #define ICE_RSS_TYPE_IPV4_ESP (ETH_RSS_ESP | ETH_RSS_IPV4) 443 #define ICE_RSS_TYPE_IPV6_ESP (ETH_RSS_ESP | ETH_RSS_IPV6) 444 #define ICE_RSS_TYPE_IPV4_AH (ETH_RSS_AH | ETH_RSS_IPV4) 445 #define ICE_RSS_TYPE_IPV6_AH (ETH_RSS_AH | ETH_RSS_IPV6) 446 #define ICE_RSS_TYPE_IPV4_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV4) 447 #define ICE_RSS_TYPE_IPV6_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV6) 448 #define ICE_RSS_TYPE_IPV4_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV4) 449 #define ICE_RSS_TYPE_IPV6_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV6) 450 451 /** 452 * Supported pattern for hash. 453 * The first member is pattern item type, 454 * the second member is input set mask, 455 * the third member is ice_rss_hash_cfg template. 456 */ 457 static struct ice_pattern_match_item ice_hash_pattern_list[] = { 458 /* IPV4 */ 459 {pattern_eth_ipv4, ICE_RSS_TYPE_ETH_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 460 {pattern_eth_ipv4_udp, ICE_RSS_TYPE_ETH_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 461 {pattern_eth_ipv4_tcp, ICE_RSS_TYPE_ETH_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 462 {pattern_eth_ipv4_sctp, ICE_RSS_TYPE_ETH_IPV4_SCTP, ICE_INSET_NONE, &ipv4_sctp_tmplt}, 463 {pattern_eth_vlan_ipv4, ICE_RSS_TYPE_VLAN_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 464 {pattern_eth_vlan_ipv4_udp, ICE_RSS_TYPE_VLAN_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 465 {pattern_eth_vlan_ipv4_tcp, ICE_RSS_TYPE_VLAN_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 466 {pattern_eth_vlan_ipv4_sctp, ICE_RSS_TYPE_VLAN_IPV4_SCTP, ICE_INSET_NONE, &ipv4_sctp_tmplt}, 467 {pattern_eth_ipv4_gtpu_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt}, 468 {pattern_eth_ipv4_gtpu_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt}, 469 {pattern_eth_ipv4_gtpu_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt}, 470 {pattern_eth_ipv6_gtpu_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt}, 471 {pattern_eth_ipv6_gtpu_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt}, 472 {pattern_eth_ipv6_gtpu_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt}, 473 {pattern_eth_ipv4_gtpu_eh_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt}, 474 {pattern_eth_ipv4_gtpu_eh_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt}, 475 {pattern_eth_ipv4_gtpu_eh_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt}, 476 {pattern_eth_ipv6_gtpu_eh_ipv4, ICE_RSS_TYPE_GTPU_IPV4, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt}, 477 {pattern_eth_ipv6_gtpu_eh_ipv4_udp, ICE_RSS_TYPE_GTPU_IPV4_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt}, 478 {pattern_eth_ipv6_gtpu_eh_ipv4_tcp, ICE_RSS_TYPE_GTPU_IPV4_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt}, 479 {pattern_eth_pppoes_ipv4, ICE_RSS_TYPE_PPPOE_IPV4, ICE_INSET_NONE, &ipv4_tmplt}, 480 {pattern_eth_pppoes_ipv4_udp, ICE_RSS_TYPE_PPPOE_IPV4_UDP, ICE_INSET_NONE, &ipv4_udp_tmplt}, 481 {pattern_eth_pppoes_ipv4_tcp, ICE_RSS_TYPE_PPPOE_IPV4_TCP, ICE_INSET_NONE, &ipv4_tcp_tmplt}, 482 {pattern_eth_ipv4_esp, ICE_RSS_TYPE_IPV4_ESP, ICE_INSET_NONE, ð_ipv4_esp_tmplt}, 483 {pattern_eth_ipv4_udp_esp, ICE_RSS_TYPE_IPV4_ESP, ICE_INSET_NONE, ð_ipv4_udp_esp_tmplt}, 484 {pattern_eth_ipv4_ah, ICE_RSS_TYPE_IPV4_AH, ICE_INSET_NONE, ð_ipv4_ah_tmplt}, 485 {pattern_eth_ipv4_l2tp, ICE_RSS_TYPE_IPV4_L2TPV3, ICE_INSET_NONE, ð_ipv4_l2tpv3_tmplt}, 486 {pattern_eth_ipv4_pfcp, ICE_RSS_TYPE_IPV4_PFCP, ICE_INSET_NONE, ð_ipv4_pfcp_tmplt}, 487 /* IPV6 */ 488 {pattern_eth_ipv6, ICE_RSS_TYPE_ETH_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 489 {pattern_eth_ipv6_udp, ICE_RSS_TYPE_ETH_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 490 {pattern_eth_ipv6_tcp, ICE_RSS_TYPE_ETH_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 491 {pattern_eth_ipv6_sctp, ICE_RSS_TYPE_ETH_IPV6_SCTP, ICE_INSET_NONE, &ipv6_sctp_tmplt}, 492 {pattern_eth_vlan_ipv6, ICE_RSS_TYPE_VLAN_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 493 {pattern_eth_vlan_ipv6_udp, ICE_RSS_TYPE_VLAN_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 494 {pattern_eth_vlan_ipv6_tcp, ICE_RSS_TYPE_VLAN_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 495 {pattern_eth_vlan_ipv6_sctp, ICE_RSS_TYPE_VLAN_IPV6_SCTP, ICE_INSET_NONE, &ipv6_sctp_tmplt}, 496 {pattern_eth_ipv4_gtpu_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt}, 497 {pattern_eth_ipv4_gtpu_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt}, 498 {pattern_eth_ipv4_gtpu_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt}, 499 {pattern_eth_ipv6_gtpu_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt}, 500 {pattern_eth_ipv6_gtpu_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt}, 501 {pattern_eth_ipv6_gtpu_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt}, 502 {pattern_eth_ipv4_gtpu_eh_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt}, 503 {pattern_eth_ipv4_gtpu_eh_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt}, 504 {pattern_eth_ipv4_gtpu_eh_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt}, 505 {pattern_eth_ipv6_gtpu_eh_ipv6, ICE_RSS_TYPE_GTPU_IPV6, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt}, 506 {pattern_eth_ipv6_gtpu_eh_ipv6_udp, ICE_RSS_TYPE_GTPU_IPV6_UDP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt}, 507 {pattern_eth_ipv6_gtpu_eh_ipv6_tcp, ICE_RSS_TYPE_GTPU_IPV6_TCP, ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt}, 508 {pattern_eth_pppoes_ipv6, ICE_RSS_TYPE_PPPOE_IPV6, ICE_INSET_NONE, &ipv6_tmplt}, 509 {pattern_eth_pppoes_ipv6_udp, ICE_RSS_TYPE_PPPOE_IPV6_UDP, ICE_INSET_NONE, &ipv6_udp_tmplt}, 510 {pattern_eth_pppoes_ipv6_tcp, ICE_RSS_TYPE_PPPOE_IPV6_TCP, ICE_INSET_NONE, &ipv6_tcp_tmplt}, 511 {pattern_eth_ipv6_esp, ICE_RSS_TYPE_IPV6_ESP, ICE_INSET_NONE, ð_ipv6_esp_tmplt}, 512 {pattern_eth_ipv6_udp_esp, ICE_RSS_TYPE_IPV6_ESP, ICE_INSET_NONE, ð_ipv6_udp_esp_tmplt}, 513 {pattern_eth_ipv6_ah, ICE_RSS_TYPE_IPV6_AH, ICE_INSET_NONE, ð_ipv6_ah_tmplt}, 514 {pattern_eth_ipv6_l2tp, ICE_RSS_TYPE_IPV6_L2TPV3, ICE_INSET_NONE, ð_ipv6_l2tpv3_tmplt}, 515 {pattern_eth_ipv6_pfcp, ICE_RSS_TYPE_IPV6_PFCP, ICE_INSET_NONE, ð_ipv6_pfcp_tmplt}, 516 /* PPPOE */ 517 {pattern_eth_pppoes, ICE_RSS_TYPE_PPPOE, ICE_INSET_NONE, &pppoe_tmplt}, 518 /* EMPTY */ 519 {pattern_empty, ICE_INSET_NONE, ICE_INSET_NONE, &empty_tmplt}, 520 }; 521 522 static struct ice_flow_engine ice_hash_engine = { 523 .init = ice_hash_init, 524 .create = ice_hash_create, 525 .destroy = ice_hash_destroy, 526 .uninit = ice_hash_uninit, 527 .free = ice_hash_free, 528 .type = ICE_FLOW_ENGINE_HASH, 529 }; 530 531 /* Register parser for os package. */ 532 static struct ice_flow_parser ice_hash_parser = { 533 .engine = &ice_hash_engine, 534 .array = ice_hash_pattern_list, 535 .array_len = RTE_DIM(ice_hash_pattern_list), 536 .parse_pattern_action = ice_hash_parse_pattern_action, 537 .stage = ICE_FLOW_STAGE_RSS, 538 }; 539 540 RTE_INIT(ice_hash_engine_init) 541 { 542 struct ice_flow_engine *engine = &ice_hash_engine; 543 ice_register_flow_engine(engine); 544 } 545 546 static int 547 ice_hash_init(struct ice_adapter *ad) 548 { 549 struct ice_flow_parser *parser = NULL; 550 551 if (ad->hw.dcf_enabled) 552 return 0; 553 554 parser = &ice_hash_parser; 555 556 return ice_register_parser(parser, ad); 557 } 558 559 static int 560 ice_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint, 561 struct rte_flow_error *error) 562 { 563 const struct rte_flow_item *item = pattern; 564 const struct rte_flow_item_gtp_psc *psc; 565 566 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 567 if (item->last) { 568 rte_flow_error_set(error, EINVAL, 569 RTE_FLOW_ERROR_TYPE_ITEM, item, 570 "Not support range"); 571 return -rte_errno; 572 } 573 574 switch (item->type) { 575 case RTE_FLOW_ITEM_TYPE_VLAN: 576 *phint |= ICE_PHINT_VLAN; 577 break; 578 case RTE_FLOW_ITEM_TYPE_PPPOES: 579 *phint |= ICE_PHINT_PPPOE; 580 break; 581 case RTE_FLOW_ITEM_TYPE_GTPU: 582 *phint |= ICE_PHINT_GTPU; 583 break; 584 case RTE_FLOW_ITEM_TYPE_GTP_PSC: 585 *phint |= ICE_PHINT_GTPU_EH; 586 psc = item->spec; 587 if (!psc) 588 break; 589 else if (psc->pdu_type == ICE_GTPU_EH_UPLINK) 590 *phint |= ICE_PHINT_GTPU_EH_UP; 591 else if (psc->pdu_type == ICE_GTPU_EH_DWNLINK) 592 *phint |= ICE_PHINT_GTPU_EH_DWN; 593 break; 594 default: 595 break; 596 } 597 } 598 599 return 0; 600 } 601 602 static void 603 ice_refine_hash_cfg_l234(struct ice_rss_hash_cfg *hash_cfg, 604 uint64_t rss_type) 605 { 606 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 607 uint64_t *hash_flds = &hash_cfg->hash_flds; 608 609 if (*addl_hdrs & ICE_FLOW_SEG_HDR_ETH) { 610 if (!(rss_type & ETH_RSS_ETH)) 611 *hash_flds &= ~ICE_FLOW_HASH_ETH; 612 if (rss_type & ETH_RSS_L2_SRC_ONLY) 613 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA)); 614 else if (rss_type & ETH_RSS_L2_DST_ONLY) 615 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)); 616 *addl_hdrs &= ~ICE_FLOW_SEG_HDR_ETH; 617 } 618 619 if (*addl_hdrs & ICE_FLOW_SEG_HDR_VLAN) { 620 if (rss_type & ETH_RSS_C_VLAN) 621 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN); 622 else if (rss_type & ETH_RSS_S_VLAN) 623 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN); 624 } 625 626 if (*addl_hdrs & ICE_FLOW_SEG_HDR_PPPOE) { 627 if (!(rss_type & ETH_RSS_PPPOE)) 628 *hash_flds &= ~ICE_FLOW_HASH_PPPOE_SESS_ID; 629 } 630 631 if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV4) { 632 if (rss_type & 633 (ETH_RSS_IPV4 | 634 ETH_RSS_NONFRAG_IPV4_UDP | 635 ETH_RSS_NONFRAG_IPV4_TCP | 636 ETH_RSS_NONFRAG_IPV4_SCTP)) { 637 if (rss_type & ETH_RSS_L3_SRC_ONLY) 638 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)); 639 else if (rss_type & ETH_RSS_L3_DST_ONLY) 640 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)); 641 else if (rss_type & 642 (ETH_RSS_L4_SRC_ONLY | 643 ETH_RSS_L4_DST_ONLY)) 644 *hash_flds &= ~ICE_FLOW_HASH_IPV4; 645 } else { 646 *hash_flds &= ~ICE_FLOW_HASH_IPV4; 647 } 648 } 649 650 if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV6) { 651 if (rss_type & 652 (ETH_RSS_IPV6 | 653 ETH_RSS_NONFRAG_IPV6_UDP | 654 ETH_RSS_NONFRAG_IPV6_TCP | 655 ETH_RSS_NONFRAG_IPV6_SCTP)) { 656 if (rss_type & ETH_RSS_L3_SRC_ONLY) 657 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 658 else if (rss_type & ETH_RSS_L3_DST_ONLY) 659 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 660 else if (rss_type & 661 (ETH_RSS_L4_SRC_ONLY | 662 ETH_RSS_L4_DST_ONLY)) 663 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 664 } else { 665 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 666 } 667 668 if (rss_type & RTE_ETH_RSS_L3_PRE32) { 669 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 670 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 671 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_SA)); 672 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 673 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 674 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_DA)); 675 } else { 676 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 677 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE32; 678 } 679 } 680 if (rss_type & RTE_ETH_RSS_L3_PRE48) { 681 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 682 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 683 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_SA)); 684 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 685 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 686 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_DA)); 687 } else { 688 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 689 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE48; 690 } 691 } 692 if (rss_type & RTE_ETH_RSS_L3_PRE64) { 693 if (rss_type & ETH_RSS_L3_SRC_ONLY) { 694 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)); 695 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_SA)); 696 } else if (rss_type & ETH_RSS_L3_DST_ONLY) { 697 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)); 698 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_DA)); 699 } else { 700 *hash_flds &= ~ICE_FLOW_HASH_IPV6; 701 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE64; 702 } 703 } 704 } 705 706 if (*addl_hdrs & ICE_FLOW_SEG_HDR_UDP) { 707 if (rss_type & 708 (ETH_RSS_NONFRAG_IPV4_UDP | 709 ETH_RSS_NONFRAG_IPV6_UDP)) { 710 if (rss_type & ETH_RSS_L4_SRC_ONLY) 711 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)); 712 else if (rss_type & ETH_RSS_L4_DST_ONLY) 713 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)); 714 else if (rss_type & 715 (ETH_RSS_L3_SRC_ONLY | 716 ETH_RSS_L3_DST_ONLY)) 717 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT; 718 } else { 719 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT; 720 } 721 } 722 723 if (*addl_hdrs & ICE_FLOW_SEG_HDR_TCP) { 724 if (rss_type & 725 (ETH_RSS_NONFRAG_IPV4_TCP | 726 ETH_RSS_NONFRAG_IPV6_TCP)) { 727 if (rss_type & ETH_RSS_L4_SRC_ONLY) 728 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)); 729 else if (rss_type & ETH_RSS_L4_DST_ONLY) 730 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)); 731 else if (rss_type & 732 (ETH_RSS_L3_SRC_ONLY | 733 ETH_RSS_L3_DST_ONLY)) 734 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT; 735 } else { 736 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT; 737 } 738 } 739 740 if (*addl_hdrs & ICE_FLOW_SEG_HDR_SCTP) { 741 if (rss_type & 742 (ETH_RSS_NONFRAG_IPV4_SCTP | 743 ETH_RSS_NONFRAG_IPV6_SCTP)) { 744 if (rss_type & ETH_RSS_L4_SRC_ONLY) 745 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)); 746 else if (rss_type & ETH_RSS_L4_DST_ONLY) 747 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)); 748 else if (rss_type & 749 (ETH_RSS_L3_SRC_ONLY | 750 ETH_RSS_L3_DST_ONLY)) 751 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT; 752 } else { 753 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT; 754 } 755 } 756 757 if (*addl_hdrs & ICE_FLOW_SEG_HDR_L2TPV3) { 758 if (!(rss_type & ETH_RSS_L2TPV3)) 759 *hash_flds &= ~ICE_FLOW_HASH_L2TPV3_SESS_ID; 760 } 761 762 if (*addl_hdrs & ICE_FLOW_SEG_HDR_ESP) { 763 if (!(rss_type & ETH_RSS_ESP)) 764 *hash_flds &= ~ICE_FLOW_HASH_ESP_SPI; 765 } 766 767 if (*addl_hdrs & ICE_FLOW_SEG_HDR_AH) { 768 if (!(rss_type & ETH_RSS_AH)) 769 *hash_flds &= ~ICE_FLOW_HASH_AH_SPI; 770 } 771 772 if (*addl_hdrs & ICE_FLOW_SEG_HDR_PFCP_SESSION) { 773 if (!(rss_type & ETH_RSS_PFCP)) 774 *hash_flds &= ~ICE_FLOW_HASH_PFCP_SEID; 775 } 776 } 777 778 static void 779 ice_refine_proto_hdrs_by_pattern(struct ice_rss_hash_cfg *hash_cfg, 780 uint64_t phint) 781 { 782 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 783 if (phint & ICE_PHINT_VLAN) 784 *addl_hdrs |= ICE_FLOW_SEG_HDR_VLAN; 785 786 if (phint & ICE_PHINT_PPPOE) 787 *addl_hdrs |= ICE_FLOW_SEG_HDR_PPPOE; 788 789 if (phint & ICE_PHINT_GTPU_EH_DWN) 790 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_DWN; 791 else if (phint & ICE_PHINT_GTPU_EH_UP) 792 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_UP; 793 else if (phint & ICE_PHINT_GTPU_EH) 794 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_EH; 795 else if (phint & ICE_PHINT_GTPU) 796 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_IP; 797 } 798 799 static void 800 ice_refine_hash_cfg_gtpu(struct ice_rss_hash_cfg *hash_cfg, 801 uint64_t rss_type) 802 { 803 uint32_t *addl_hdrs = &hash_cfg->addl_hdrs; 804 uint64_t *hash_flds = &hash_cfg->hash_flds; 805 806 /* update hash field for gtpu eh/gtpu dwn/gtpu up. */ 807 if (!(rss_type & ETH_RSS_GTPU)) 808 return; 809 810 if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_DWN) 811 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_DWN_TEID); 812 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_UP) 813 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_UP_TEID); 814 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_EH) 815 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_EH_TEID); 816 else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_IP) 817 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID); 818 } 819 820 static void ice_refine_hash_cfg(struct ice_rss_hash_cfg *hash_cfg, 821 uint64_t rss_type, uint64_t phint) 822 { 823 ice_refine_proto_hdrs_by_pattern(hash_cfg, phint); 824 ice_refine_hash_cfg_l234(hash_cfg, rss_type); 825 ice_refine_hash_cfg_gtpu(hash_cfg, rss_type); 826 } 827 828 static uint64_t invalid_rss_comb[] = { 829 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP, 830 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP, 831 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP, 832 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP, 833 RTE_ETH_RSS_L3_PRE40 | 834 RTE_ETH_RSS_L3_PRE56 | 835 RTE_ETH_RSS_L3_PRE96 836 }; 837 838 struct rss_attr_type { 839 uint64_t attr; 840 uint64_t type; 841 }; 842 843 static struct rss_attr_type rss_attr_to_valid_type[] = { 844 {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY, ETH_RSS_ETH}, 845 {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY, VALID_RSS_L3}, 846 {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY, VALID_RSS_L4}, 847 /* current ipv6 prefix only supports prefix 64 bits*/ 848 {RTE_ETH_RSS_L3_PRE32, VALID_RSS_IPV6}, 849 {RTE_ETH_RSS_L3_PRE48, VALID_RSS_IPV6}, 850 {RTE_ETH_RSS_L3_PRE64, VALID_RSS_IPV6}, 851 {INVALID_RSS_ATTR, 0} 852 }; 853 854 static bool 855 ice_any_invalid_rss_type(enum rte_eth_hash_function rss_func, 856 uint64_t rss_type, uint64_t allow_rss_type) 857 { 858 uint32_t i; 859 860 /** 861 * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ 862 * hash function. 863 */ 864 if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 865 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY | 866 ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) 867 return true; 868 869 if (!(rss_type & 870 (ETH_RSS_IPV4 | ETH_RSS_IPV6 | 871 ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP | 872 ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP | 873 ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP))) 874 return true; 875 } 876 877 /* check invalid combination */ 878 for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) { 879 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1) 880 return true; 881 } 882 883 /* check invalid RSS attribute */ 884 for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) { 885 struct rss_attr_type *rat = &rss_attr_to_valid_type[i]; 886 887 if (rat->attr & rss_type && !(rat->type & rss_type)) 888 return true; 889 } 890 891 /* check not allowed RSS type */ 892 rss_type &= ~VALID_RSS_ATTR; 893 894 return ((rss_type & allow_rss_type) != rss_type); 895 } 896 897 static int 898 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item, 899 const struct rte_flow_action actions[], 900 uint64_t pattern_hint, void **meta, 901 struct rte_flow_error *error) 902 { 903 struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)*meta; 904 struct ice_rss_hash_cfg *cfg = pattern_match_item->meta; 905 enum rte_flow_action_type action_type; 906 const struct rte_flow_action_rss *rss; 907 const struct rte_flow_action *action; 908 uint64_t rss_type; 909 910 /* Supported action is RSS. */ 911 for (action = actions; action->type != 912 RTE_FLOW_ACTION_TYPE_END; action++) { 913 action_type = action->type; 914 switch (action_type) { 915 case RTE_FLOW_ACTION_TYPE_RSS: 916 rss = action->conf; 917 rss_type = rss->types; 918 919 /* Check hash function and save it to rss_meta. */ 920 if (pattern_match_item->pattern_list != 921 pattern_empty && rss->func == 922 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { 923 return rte_flow_error_set(error, ENOTSUP, 924 RTE_FLOW_ERROR_TYPE_ACTION, action, 925 "Not supported flow"); 926 } else if (rss->func == 927 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){ 928 rss_meta->hash_function = 929 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR; 930 return 0; 931 } else if (rss->func == 932 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 933 rss_meta->hash_function = 934 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ; 935 cfg->symm = true; 936 } 937 938 if (rss->level) 939 return rte_flow_error_set(error, ENOTSUP, 940 RTE_FLOW_ERROR_TYPE_ACTION, action, 941 "a nonzero RSS encapsulation level is not supported"); 942 943 if (rss->key_len) 944 return rte_flow_error_set(error, ENOTSUP, 945 RTE_FLOW_ERROR_TYPE_ACTION, action, 946 "a nonzero RSS key_len is not supported"); 947 948 if (rss->queue) 949 return rte_flow_error_set(error, ENOTSUP, 950 RTE_FLOW_ERROR_TYPE_ACTION, action, 951 "a non-NULL RSS queue is not supported"); 952 953 /** 954 * Check simultaneous use of SRC_ONLY and DST_ONLY 955 * of the same level. 956 */ 957 rss_type = rte_eth_rss_hf_refine(rss_type); 958 959 if (ice_any_invalid_rss_type(rss->func, rss_type, 960 pattern_match_item->input_set_mask_o)) 961 return rte_flow_error_set(error, ENOTSUP, 962 RTE_FLOW_ERROR_TYPE_ACTION, 963 action, "RSS type not supported"); 964 965 rss_meta->cfg = *cfg; 966 ice_refine_hash_cfg(&rss_meta->cfg, 967 rss_type, pattern_hint); 968 break; 969 case RTE_FLOW_ACTION_TYPE_END: 970 break; 971 972 default: 973 rte_flow_error_set(error, EINVAL, 974 RTE_FLOW_ERROR_TYPE_ACTION, action, 975 "Invalid action."); 976 return -rte_errno; 977 } 978 } 979 980 return 0; 981 } 982 983 static int 984 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad, 985 struct ice_pattern_match_item *array, 986 uint32_t array_len, 987 const struct rte_flow_item pattern[], 988 const struct rte_flow_action actions[], 989 uint32_t priority __rte_unused, 990 void **meta, 991 struct rte_flow_error *error) 992 { 993 int ret = 0; 994 struct ice_pattern_match_item *pattern_match_item; 995 struct ice_rss_meta *rss_meta_ptr; 996 uint64_t phint = ICE_PHINT_NONE; 997 998 rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0); 999 if (!rss_meta_ptr) { 1000 rte_flow_error_set(error, EINVAL, 1001 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1002 "No memory for rss_meta_ptr"); 1003 return -ENOMEM; 1004 } 1005 1006 /* Check rss supported pattern and find matched pattern. */ 1007 pattern_match_item = ice_search_pattern_match_item(ad, pattern, array, 1008 array_len, error); 1009 if (!pattern_match_item) { 1010 ret = -rte_errno; 1011 goto error; 1012 } 1013 1014 ret = ice_hash_parse_pattern(pattern, &phint, error); 1015 if (ret) 1016 goto error; 1017 1018 /* Check rss action. */ 1019 ret = ice_hash_parse_action(pattern_match_item, actions, phint, 1020 (void **)&rss_meta_ptr, error); 1021 1022 error: 1023 if (!ret && meta) 1024 *meta = rss_meta_ptr; 1025 else 1026 rte_free(rss_meta_ptr); 1027 rte_free(pattern_match_item); 1028 1029 return ret; 1030 } 1031 1032 static int 1033 ice_hash_create(struct ice_adapter *ad, 1034 struct rte_flow *flow, 1035 void *meta, 1036 struct rte_flow_error *error) 1037 { 1038 struct ice_pf *pf = &ad->pf; 1039 struct ice_hw *hw = ICE_PF_TO_HW(pf); 1040 struct ice_vsi *vsi = pf->main_vsi; 1041 int ret; 1042 uint32_t reg; 1043 struct ice_hash_flow_cfg *filter_ptr; 1044 struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)meta; 1045 uint8_t hash_function = rss_meta->hash_function; 1046 1047 filter_ptr = rte_zmalloc("ice_rss_filter", 1048 sizeof(struct ice_hash_flow_cfg), 0); 1049 if (!filter_ptr) { 1050 rte_flow_error_set(error, EINVAL, 1051 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1052 "No memory for filter_ptr"); 1053 return -ENOMEM; 1054 } 1055 1056 if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { 1057 /* Enable registers for simple_xor hash function. */ 1058 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id)); 1059 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) | 1060 (2 << VSIQF_HASH_CTL_HASH_SCHEME_S); 1061 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg); 1062 1063 filter_ptr->simple_xor = 1; 1064 1065 goto out; 1066 } else { 1067 memcpy(&filter_ptr->rss_cfg.hash, &rss_meta->cfg, 1068 sizeof(struct ice_rss_hash_cfg)); 1069 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, 1070 &filter_ptr->rss_cfg.hash); 1071 if (ret) { 1072 rte_flow_error_set(error, EINVAL, 1073 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1074 "rss flow create fail"); 1075 goto error; 1076 } 1077 } 1078 1079 out: 1080 flow->rule = filter_ptr; 1081 rte_free(meta); 1082 return 0; 1083 1084 error: 1085 rte_free(filter_ptr); 1086 rte_free(meta); 1087 return -rte_errno; 1088 } 1089 1090 static int 1091 ice_hash_destroy(struct ice_adapter *ad, 1092 struct rte_flow *flow, 1093 struct rte_flow_error *error) 1094 { 1095 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad); 1096 struct ice_hw *hw = ICE_PF_TO_HW(pf); 1097 struct ice_vsi *vsi = pf->main_vsi; 1098 int ret; 1099 uint32_t reg; 1100 struct ice_hash_flow_cfg *filter_ptr; 1101 1102 filter_ptr = (struct ice_hash_flow_cfg *)flow->rule; 1103 1104 if (filter_ptr->simple_xor == 1) { 1105 /* Return to symmetric_toeplitz state. */ 1106 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id)); 1107 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) | 1108 (1 << VSIQF_HASH_CTL_HASH_SCHEME_S); 1109 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg); 1110 } else { 1111 ret = ice_rem_rss_cfg_wrap(pf, vsi->idx, 1112 &filter_ptr->rss_cfg.hash); 1113 /* Fixme: Ignore the error if a rule does not exist. 1114 * Currently a rule for inputset change or symm turn on/off 1115 * will overwrite an exist rule, while application still 1116 * have 2 rte_flow handles. 1117 **/ 1118 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) { 1119 rte_flow_error_set(error, EINVAL, 1120 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 1121 "rss flow destroy fail"); 1122 goto error; 1123 } 1124 } 1125 1126 rte_free(filter_ptr); 1127 return 0; 1128 1129 error: 1130 rte_free(filter_ptr); 1131 return -rte_errno; 1132 } 1133 1134 static void 1135 ice_hash_uninit(struct ice_adapter *ad) 1136 { 1137 if (ad->hw.dcf_enabled) 1138 return; 1139 1140 ice_unregister_parser(&ice_hash_parser, ad); 1141 } 1142 1143 static void 1144 ice_hash_free(struct rte_flow *flow) 1145 { 1146 rte_free(flow->rule); 1147 } 1148