1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <assert.h> 11 12 #include <rte_malloc.h> 13 #include <rte_tailq.h> 14 #include "base/i40e_prototype.h" 15 #include "i40e_logs.h" 16 #include "i40e_ethdev.h" 17 #include "i40e_hash.h" 18 19 #ifndef BIT 20 #define BIT(n) (1UL << (n)) 21 #endif 22 23 #ifndef BIT_ULL 24 #define BIT_ULL(n) (1ULL << (n)) 25 #endif 26 27 /* Pattern item headers */ 28 #define I40E_HASH_HDR_ETH 0x01ULL 29 #define I40E_HASH_HDR_IPV4 0x10ULL 30 #define I40E_HASH_HDR_IPV6 0x20ULL 31 #define I40E_HASH_HDR_TCP 0x100ULL 32 #define I40E_HASH_HDR_UDP 0x200ULL 33 #define I40E_HASH_HDR_SCTP 0x400ULL 34 #define I40E_HASH_HDR_ESP 0x10000ULL 35 #define I40E_HASH_HDR_L2TPV3 0x20000ULL 36 #define I40E_HASH_HDR_AH 0x40000ULL 37 #define I40E_HASH_HDR_GTPC 0x100000ULL 38 #define I40E_HASH_HDR_GTPU 0x200000ULL 39 40 #define I40E_HASH_HDR_INNER_SHIFT 32 41 #define I40E_HASH_HDR_IPV4_INNER (I40E_HASH_HDR_IPV4 << \ 42 I40E_HASH_HDR_INNER_SHIFT) 43 #define I40E_HASH_HDR_IPV6_INNER (I40E_HASH_HDR_IPV6 << \ 44 I40E_HASH_HDR_INNER_SHIFT) 45 46 /* ETH */ 47 #define I40E_PHINT_ETH I40E_HASH_HDR_ETH 48 49 /* IPv4 */ 50 #define I40E_PHINT_IPV4 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4) 51 #define I40E_PHINT_IPV4_TCP (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP) 52 #define I40E_PHINT_IPV4_UDP (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP) 53 #define I40E_PHINT_IPV4_SCTP (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP) 54 55 /* IPv6 */ 56 #define I40E_PHINT_IPV6 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6) 57 #define I40E_PHINT_IPV6_TCP (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP) 58 #define I40E_PHINT_IPV6_UDP (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP) 59 #define I40E_PHINT_IPV6_SCTP (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP) 60 61 /* ESP */ 62 #define I40E_PHINT_IPV4_ESP (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP) 63 #define I40E_PHINT_IPV6_ESP (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP) 64 #define I40E_PHINT_IPV4_UDP_ESP (I40E_PHINT_IPV4_UDP | \ 65 I40E_HASH_HDR_ESP) 66 #define I40E_PHINT_IPV6_UDP_ESP (I40E_PHINT_IPV6_UDP | \ 67 I40E_HASH_HDR_ESP) 68 69 /* GTPC */ 70 #define I40E_PHINT_IPV4_GTPC (I40E_PHINT_IPV4_UDP | \ 71 I40E_HASH_HDR_GTPC) 72 #define I40E_PHINT_IPV6_GTPC (I40E_PHINT_IPV6_UDP | \ 73 I40E_HASH_HDR_GTPC) 74 75 /* GTPU */ 76 #define I40E_PHINT_IPV4_GTPU (I40E_PHINT_IPV4_UDP | \ 77 I40E_HASH_HDR_GTPU) 78 #define I40E_PHINT_IPV4_GTPU_IPV4 (I40E_PHINT_IPV4_GTPU | \ 79 I40E_HASH_HDR_IPV4_INNER) 80 #define I40E_PHINT_IPV4_GTPU_IPV6 (I40E_PHINT_IPV4_GTPU | \ 81 I40E_HASH_HDR_IPV6_INNER) 82 #define I40E_PHINT_IPV6_GTPU (I40E_PHINT_IPV6_UDP | \ 83 I40E_HASH_HDR_GTPU) 84 #define I40E_PHINT_IPV6_GTPU_IPV4 (I40E_PHINT_IPV6_GTPU | \ 85 I40E_HASH_HDR_IPV4_INNER) 86 #define I40E_PHINT_IPV6_GTPU_IPV6 (I40E_PHINT_IPV6_GTPU | \ 87 I40E_HASH_HDR_IPV6_INNER) 88 89 /* L2TPV3 */ 90 #define I40E_PHINT_IPV4_L2TPV3 (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3) 91 #define I40E_PHINT_IPV6_L2TPV3 (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3) 92 93 /* AH */ 94 #define I40E_PHINT_IPV4_AH (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH) 95 #define I40E_PHINT_IPV6_AH (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH) 96 97 /* Structure of mapping RSS type to input set */ 98 struct i40e_hash_map_rss_inset { 99 uint64_t rss_type; 100 uint64_t inset; 101 }; 102 103 const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = { 104 /* IPv4 */ 105 { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST }, 106 { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST }, 107 108 { ETH_RSS_NONFRAG_IPV4_OTHER, 109 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST }, 110 111 { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | 112 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT }, 113 114 { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | 115 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT }, 116 117 { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | 118 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT }, 119 120 /* IPv6 */ 121 { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST }, 122 { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST }, 123 124 { ETH_RSS_NONFRAG_IPV6_OTHER, 125 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST }, 126 127 { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | 128 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT }, 129 130 { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | 131 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT }, 132 133 { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | 134 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT }, 135 136 /* Port */ 137 { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT }, 138 139 /* Ether */ 140 { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE }, 141 { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC }, 142 143 /* VLAN */ 144 { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER }, 145 { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER }, 146 }; 147 148 #define I40E_HASH_VOID_NEXT_ALLOW BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH) 149 150 #define I40E_HASH_ETH_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \ 151 BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \ 152 BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN)) 153 154 #define I40E_HASH_IP_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \ 155 BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \ 156 BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \ 157 BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \ 158 BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\ 159 BIT_ULL(RTE_FLOW_ITEM_TYPE_AH)) 160 161 #define I40E_HASH_UDP_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \ 162 BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC)) 163 164 #define I40E_HASH_GTPU_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \ 165 BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6)) 166 167 static const uint64_t pattern_next_allow_items[] = { 168 [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW, 169 [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW, 170 [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW, 171 [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IP_NEXT_ALLOW, 172 [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW, 173 [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW, 174 }; 175 176 static const uint64_t pattern_item_header[] = { 177 [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH, 178 [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4, 179 [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6, 180 [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP, 181 [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP, 182 [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP, 183 [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP, 184 [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC, 185 [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU, 186 [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3, 187 [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH, 188 }; 189 190 /* Structure of matched pattern */ 191 struct i40e_hash_match_pattern { 192 uint64_t pattern_type; 193 uint64_t rss_mask; /* Supported RSS type for this pattern */ 194 bool custom_pctype_flag;/* true for custom packet type */ 195 uint8_t pctype; 196 }; 197 198 #define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \ 199 pattern, rss_mask, false, pctype } 200 201 #define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \ 202 pattern, rss_mask, true, cus_pctype } 203 204 #define I40E_HASH_L2_RSS_MASK (ETH_RSS_VLAN | ETH_RSS_ETH | \ 205 ETH_RSS_L2_SRC_ONLY | \ 206 ETH_RSS_L2_DST_ONLY) 207 208 #define I40E_HASH_L23_RSS_MASK (I40E_HASH_L2_RSS_MASK | \ 209 ETH_RSS_L3_SRC_ONLY | \ 210 ETH_RSS_L3_DST_ONLY) 211 212 #define I40E_HASH_IPV4_L23_RSS_MASK (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK) 213 #define I40E_HASH_IPV6_L23_RSS_MASK (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK) 214 215 #define I40E_HASH_L234_RSS_MASK (I40E_HASH_L23_RSS_MASK | \ 216 ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \ 217 ETH_RSS_L4_DST_ONLY) 218 219 #define I40E_HASH_IPV4_L234_RSS_MASK (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4) 220 #define I40E_HASH_IPV6_L234_RSS_MASK (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6) 221 222 #define I40E_HASH_L4_TYPES (ETH_RSS_NONFRAG_IPV4_TCP | \ 223 ETH_RSS_NONFRAG_IPV4_UDP | \ 224 ETH_RSS_NONFRAG_IPV4_SCTP | \ 225 ETH_RSS_NONFRAG_IPV6_TCP | \ 226 ETH_RSS_NONFRAG_IPV6_UDP | \ 227 ETH_RSS_NONFRAG_IPV6_SCTP) 228 229 /* Current supported patterns and RSS types. 230 * All items that have the same pattern types are together. 231 */ 232 static const struct i40e_hash_match_pattern match_patterns[] = { 233 /* Ether */ 234 I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH, 235 ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK, 236 I40E_FILTER_PCTYPE_L2_PAYLOAD), 237 238 /* IPv4 */ 239 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4, 240 ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK, 241 I40E_FILTER_PCTYPE_FRAG_IPV4), 242 243 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4, 244 ETH_RSS_NONFRAG_IPV4_OTHER | 245 I40E_HASH_IPV4_L23_RSS_MASK, 246 I40E_FILTER_PCTYPE_NONF_IPV4_OTHER), 247 248 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP, 249 ETH_RSS_NONFRAG_IPV4_TCP | 250 I40E_HASH_IPV4_L234_RSS_MASK, 251 I40E_FILTER_PCTYPE_NONF_IPV4_TCP), 252 253 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP, 254 ETH_RSS_NONFRAG_IPV4_UDP | 255 I40E_HASH_IPV4_L234_RSS_MASK, 256 I40E_FILTER_PCTYPE_NONF_IPV4_UDP), 257 258 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP, 259 ETH_RSS_NONFRAG_IPV4_SCTP | 260 I40E_HASH_IPV4_L234_RSS_MASK, 261 I40E_FILTER_PCTYPE_NONF_IPV4_SCTP), 262 263 /* IPv6 */ 264 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6, 265 ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK, 266 I40E_FILTER_PCTYPE_FRAG_IPV6), 267 268 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6, 269 ETH_RSS_NONFRAG_IPV6_OTHER | 270 I40E_HASH_IPV6_L23_RSS_MASK, 271 I40E_FILTER_PCTYPE_NONF_IPV6_OTHER), 272 273 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP, 274 ETH_RSS_NONFRAG_IPV6_TCP | 275 I40E_HASH_IPV6_L234_RSS_MASK, 276 I40E_FILTER_PCTYPE_NONF_IPV6_TCP), 277 278 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP, 279 ETH_RSS_NONFRAG_IPV6_UDP | 280 I40E_HASH_IPV6_L234_RSS_MASK, 281 I40E_FILTER_PCTYPE_NONF_IPV6_UDP), 282 283 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP, 284 ETH_RSS_NONFRAG_IPV6_SCTP | 285 I40E_HASH_IPV6_L234_RSS_MASK, 286 I40E_FILTER_PCTYPE_NONF_IPV6_SCTP), 287 288 /* ESP */ 289 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP, 290 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4), 291 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP, 292 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6), 293 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP, 294 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP), 295 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP, 296 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP), 297 298 /* GTPC */ 299 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC, 300 I40E_HASH_IPV4_L234_RSS_MASK, 301 I40E_CUSTOMIZED_GTPC), 302 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC, 303 I40E_HASH_IPV6_L234_RSS_MASK, 304 I40E_CUSTOMIZED_GTPC), 305 306 /* GTPU */ 307 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU, 308 I40E_HASH_IPV4_L234_RSS_MASK, 309 I40E_CUSTOMIZED_GTPU), 310 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4, 311 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4), 312 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6, 313 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6), 314 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU, 315 I40E_HASH_IPV6_L234_RSS_MASK, 316 I40E_CUSTOMIZED_GTPU), 317 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4, 318 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4), 319 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6, 320 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6), 321 322 /* L2TPV3 */ 323 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3, 324 ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3), 325 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3, 326 ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3), 327 328 /* AH */ 329 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH, 330 I40E_CUSTOMIZED_AH_IPV4), 331 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH, 332 I40E_CUSTOMIZED_AH_IPV6), 333 }; 334 335 static int 336 i40e_hash_get_pattern_type(const struct rte_flow_item pattern[], 337 uint64_t *pattern_types, 338 struct rte_flow_error *error) 339 { 340 const char *message = "Pattern not supported"; 341 enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID; 342 enum rte_flow_item_type last_item_type = prev_item_type; 343 uint64_t item_hdr, pattern_hdrs = 0; 344 bool inner_flag = false; 345 int vlan_count = 0; 346 347 for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) { 348 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID) 349 continue; 350 351 if (pattern->mask || pattern->spec || pattern->last) { 352 message = "Header info should not be specified"; 353 goto not_sup; 354 } 355 356 /* Check the previous item allows this sub-item. */ 357 if (prev_item_type >= (enum rte_flow_item_type) 358 RTE_DIM(pattern_next_allow_items) || 359 !(pattern_next_allow_items[prev_item_type] & 360 BIT_ULL(pattern->type))) 361 goto not_sup; 362 363 /* For VLAN item, it does no matter about to pattern type 364 * recognition. So just count the number of VLAN and do not 365 * change the value of variable `prev_item_type`. 366 */ 367 last_item_type = pattern->type; 368 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) { 369 if (vlan_count >= 2) 370 goto not_sup; 371 vlan_count++; 372 continue; 373 } 374 375 prev_item_type = last_item_type; 376 assert(last_item_type < (enum rte_flow_item_type) 377 RTE_DIM(pattern_item_header)); 378 item_hdr = pattern_item_header[last_item_type]; 379 assert(item_hdr); 380 381 if (inner_flag) { 382 item_hdr <<= I40E_HASH_HDR_INNER_SHIFT; 383 384 /* Inner layer should not have GTPU item */ 385 if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) 386 goto not_sup; 387 } else { 388 if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) { 389 inner_flag = true; 390 vlan_count = 0; 391 } 392 } 393 394 if (item_hdr & pattern_hdrs) 395 goto not_sup; 396 397 pattern_hdrs |= item_hdr; 398 } 399 400 if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) { 401 *pattern_types = pattern_hdrs; 402 return 0; 403 } 404 405 not_sup: 406 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 407 pattern, message); 408 } 409 410 static uint64_t 411 i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype) 412 { 413 uint64_t pctypes = 0; 414 415 switch (match_pctype) { 416 case I40E_FILTER_PCTYPE_NONF_IPV4_TCP: 417 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); 418 break; 419 420 case I40E_FILTER_PCTYPE_NONF_IPV4_UDP: 421 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | 422 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP); 423 break; 424 425 case I40E_FILTER_PCTYPE_NONF_IPV6_TCP: 426 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); 427 break; 428 429 case I40E_FILTER_PCTYPE_NONF_IPV6_UDP: 430 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | 431 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP); 432 break; 433 } 434 435 return pctypes; 436 } 437 438 static int 439 i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf, 440 struct rte_flow_error *error) 441 { 442 if (rss_conf->inset & 443 (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC | 444 I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT)) 445 return rte_flow_error_set(error, ENOTSUP, 446 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 447 NULL, 448 "Only support external destination IP"); 449 450 if (rss_conf->inset & I40E_INSET_IPV4_DST) 451 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) | 452 I40E_INSET_TUNNEL_IPV4_DST; 453 454 if (rss_conf->inset & I40E_INSET_IPV6_DST) 455 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) | 456 I40E_INSET_TUNNEL_IPV6_DST; 457 458 return 0; 459 } 460 461 static int 462 i40e_hash_get_pctypes(const struct rte_eth_dev *dev, 463 const struct i40e_hash_match_pattern *match, 464 struct i40e_rte_flow_rss_conf *rss_conf, 465 struct rte_flow_error *error) 466 { 467 if (match->custom_pctype_flag) { 468 struct i40e_pf *pf; 469 struct i40e_customized_pctype *custom_type; 470 471 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 472 custom_type = i40e_find_customized_pctype(pf, match->pctype); 473 if (!custom_type || !custom_type->valid) 474 return rte_flow_error_set(error, ENOTSUP, 475 RTE_FLOW_ERROR_TYPE_ITEM, 476 NULL, "PCTYPE not supported"); 477 478 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype); 479 480 if (match->pctype == I40E_CUSTOMIZED_GTPU || 481 match->pctype == I40E_CUSTOMIZED_GTPC) 482 return i40e_hash_translate_gtp_inset(rss_conf, error); 483 } else { 484 struct i40e_hw *hw = 485 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 486 uint64_t types; 487 488 rss_conf->config_pctypes |= BIT_ULL(match->pctype); 489 if (hw->mac.type == I40E_MAC_X722) { 490 types = i40e_hash_get_x722_ext_pctypes(match->pctype); 491 rss_conf->config_pctypes |= types; 492 } 493 } 494 495 return 0; 496 } 497 498 static int 499 i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev, 500 const struct rte_flow_item pattern[], 501 const struct rte_flow_action_rss *rss_act, 502 struct i40e_rte_flow_rss_conf *rss_conf, 503 struct rte_flow_error *error) 504 { 505 uint64_t pattern_types = 0; 506 bool match_flag = false; 507 int i, ret; 508 509 ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error); 510 if (ret) 511 return ret; 512 513 for (i = 0; i < (int)RTE_DIM(match_patterns); i++) { 514 const struct i40e_hash_match_pattern *match = 515 &match_patterns[i]; 516 517 /* Check pattern types match. All items that have the same 518 * pattern types are together, so if the pattern types match 519 * previous item but they doesn't match current item, it means 520 * the pattern types do not match all remain items. 521 */ 522 if (pattern_types != match->pattern_type) { 523 if (match_flag) 524 break; 525 continue; 526 } 527 match_flag = true; 528 529 /* Check RSS types match */ 530 if (!(rss_act->types & ~match->rss_mask)) { 531 ret = i40e_hash_get_pctypes(dev, match, 532 rss_conf, error); 533 if (ret) 534 return ret; 535 } 536 } 537 538 if (rss_conf->config_pctypes) 539 return 0; 540 541 if (match_flag) 542 return rte_flow_error_set(error, ENOTSUP, 543 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 544 NULL, "RSS types not supported"); 545 546 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, 547 NULL, "Pattern not supported"); 548 } 549 550 static uint64_t 551 i40e_hash_get_inset(uint64_t rss_types) 552 { 553 uint64_t mask, inset = 0; 554 int i; 555 556 for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) { 557 if (rss_types & i40e_hash_rss_inset[i].rss_type) 558 inset |= i40e_hash_rss_inset[i].inset; 559 } 560 561 if (!inset) 562 return 0; 563 564 /* If SRC_ONLY and DST_ONLY of the same level are used simultaneously, 565 * it is the same case as none of them are added. 566 */ 567 mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY); 568 if (mask == ETH_RSS_L2_SRC_ONLY) 569 inset &= ~I40E_INSET_DMAC; 570 else if (mask == ETH_RSS_L2_DST_ONLY) 571 inset &= ~I40E_INSET_SMAC; 572 573 mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY); 574 if (mask == ETH_RSS_L3_SRC_ONLY) 575 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST); 576 else if (mask == ETH_RSS_L3_DST_ONLY) 577 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC); 578 579 mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY); 580 if (mask == ETH_RSS_L4_SRC_ONLY) 581 inset &= ~I40E_INSET_DST_PORT; 582 else if (mask == ETH_RSS_L4_DST_ONLY) 583 inset &= ~I40E_INSET_SRC_PORT; 584 585 if (rss_types & I40E_HASH_L4_TYPES) { 586 uint64_t l3_mask = rss_types & 587 (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY); 588 uint64_t l4_mask = rss_types & 589 (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY); 590 591 if (l3_mask && !l4_mask) 592 inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT); 593 else if (!l3_mask && l4_mask) 594 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST | 595 I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC); 596 } 597 598 return inset; 599 } 600 601 static int 602 i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func) 603 { 604 struct i40e_pf *pf; 605 uint32_t reg; 606 uint8_t symmetric = 0; 607 608 reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL); 609 610 if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { 611 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) 612 goto set_symmetric; 613 614 reg &= ~I40E_GLQF_CTL_HTOEP_MASK; 615 } else { 616 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) 617 symmetric = 1; 618 619 if (reg & I40E_GLQF_CTL_HTOEP_MASK) 620 goto set_symmetric; 621 622 reg |= I40E_GLQF_CTL_HTOEP_MASK; 623 } 624 625 pf = &((struct i40e_adapter *)hw->back)->pf; 626 if (pf->support_multi_driver) { 627 PMD_DRV_LOG(ERR, 628 "Modify hash function is not permitted when multi-driver enabled"); 629 return -EPERM; 630 } 631 632 PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func); 633 i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg); 634 I40E_WRITE_FLUSH(hw); 635 636 set_symmetric: 637 i40e_set_symmetric_hash_enable_per_port(hw, symmetric); 638 return 0; 639 } 640 641 static int 642 i40e_hash_config_pctype_symmetric(struct i40e_hw *hw, 643 uint32_t pctype, 644 bool symmetric) 645 { 646 struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf; 647 uint32_t reg; 648 649 /* For X722, get translated pctype in fd pctype register */ 650 if (hw->mac.type == I40E_MAC_X722) 651 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype)); 652 653 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype)); 654 if (symmetric) { 655 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK) 656 return 0; 657 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK; 658 } else { 659 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)) 660 return 0; 661 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK; 662 } 663 664 if (pf->support_multi_driver) { 665 PMD_DRV_LOG(ERR, 666 "Enable/Disable symmetric hash is not permitted when multi-driver enabled"); 667 return -EPERM; 668 } 669 670 i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg); 671 I40E_WRITE_FLUSH(hw); 672 return 0; 673 } 674 675 static void 676 i40e_hash_enable_pctype(struct i40e_hw *hw, 677 uint32_t pctype, bool enable) 678 { 679 uint32_t reg, reg_val, mask; 680 681 if (pctype < 32) { 682 mask = BIT(pctype); 683 reg = I40E_PFQF_HENA(0); 684 } else { 685 mask = BIT(pctype - 32); 686 reg = I40E_PFQF_HENA(1); 687 } 688 689 reg_val = i40e_read_rx_ctl(hw, reg); 690 691 if (enable) { 692 if (reg_val & mask) 693 return; 694 695 reg_val |= mask; 696 } else { 697 if (!(reg_val & mask)) 698 return; 699 700 reg_val &= ~mask; 701 } 702 703 i40e_write_rx_ctl(hw, reg, reg_val); 704 I40E_WRITE_FLUSH(hw); 705 } 706 707 static int 708 i40e_hash_config_pctype(struct i40e_hw *hw, 709 struct i40e_rte_flow_rss_conf *rss_conf, 710 uint32_t pctype) 711 { 712 uint64_t rss_types = rss_conf->conf.types; 713 int ret; 714 715 if (rss_types == 0) { 716 i40e_hash_enable_pctype(hw, pctype, false); 717 return 0; 718 } 719 720 if (rss_conf->inset) { 721 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false); 722 if (ret) 723 return ret; 724 } 725 726 i40e_hash_enable_pctype(hw, pctype, true); 727 return 0; 728 } 729 730 static int 731 i40e_hash_config_region(struct i40e_pf *pf, 732 const struct i40e_rte_flow_rss_conf *rss_conf) 733 { 734 struct i40e_hw *hw = &pf->adapter->hw; 735 struct rte_eth_dev *dev = &rte_eth_devices[pf->dev_data->port_id]; 736 struct i40e_queue_region_info *regions = pf->queue_region.region; 737 uint32_t num = pf->queue_region.queue_region_number; 738 uint32_t i, region_id_mask = 0; 739 740 /* Use a 32 bit variable to represent all regions */ 741 RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31); 742 743 /* Re-configure the region if it existed */ 744 for (i = 0; i < num; i++) { 745 if (rss_conf->region_queue_start == 746 regions[i].queue_start_index && 747 rss_conf->region_queue_num == regions[i].queue_num) { 748 uint32_t j; 749 750 for (j = 0; j < regions[i].user_priority_num; j++) { 751 if (regions[i].user_priority[j] == 752 rss_conf->region_priority) 753 return 0; 754 } 755 756 if (j >= I40E_MAX_USER_PRIORITY) { 757 PMD_DRV_LOG(ERR, 758 "Priority number exceed the maximum %d", 759 I40E_MAX_USER_PRIORITY); 760 return -ENOSPC; 761 } 762 763 regions[i].user_priority[j] = rss_conf->region_priority; 764 regions[i].user_priority_num++; 765 return i40e_flush_queue_region_all_conf(dev, hw, pf, 1); 766 } 767 768 region_id_mask |= BIT(regions[i].region_id); 769 } 770 771 if (num > I40E_REGION_MAX_INDEX) { 772 PMD_DRV_LOG(ERR, "Queue region resource used up"); 773 return -ENOSPC; 774 } 775 776 /* Add a new region */ 777 778 pf->queue_region.queue_region_number++; 779 memset(®ions[num], 0, sizeof(regions[0])); 780 781 regions[num].region_id = rte_bsf32(~region_id_mask); 782 regions[num].queue_num = rss_conf->region_queue_num; 783 regions[num].queue_start_index = rss_conf->region_queue_start; 784 regions[num].user_priority[0] = rss_conf->region_priority; 785 regions[num].user_priority_num = 1; 786 787 return i40e_flush_queue_region_all_conf(dev, hw, pf, 1); 788 } 789 790 static int 791 i40e_hash_config(struct i40e_pf *pf, 792 struct i40e_rte_flow_rss_conf *rss_conf) 793 { 794 struct rte_flow_action_rss *rss_info = &rss_conf->conf; 795 struct i40e_hw *hw = &pf->adapter->hw; 796 uint64_t pctypes; 797 int ret; 798 799 if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) { 800 ret = i40e_hash_config_func(hw, rss_info->func); 801 if (ret) 802 return ret; 803 804 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 805 rss_conf->misc_reset_flags |= 806 I40E_HASH_FLOW_RESET_FLAG_FUNC; 807 } 808 809 if (rss_conf->region_queue_num > 0) { 810 ret = i40e_hash_config_region(pf, rss_conf); 811 if (ret) 812 return ret; 813 814 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION; 815 } 816 817 if (rss_info->key_len > 0) { 818 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key, 819 rss_info->key_len); 820 if (ret) 821 return ret; 822 823 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY; 824 } 825 826 /* Update lookup table */ 827 if (rss_info->queue_num > 0) { 828 uint8_t lut[ETH_RSS_RETA_SIZE_512]; 829 uint32_t i, j = 0; 830 831 for (i = 0; i < hw->func_caps.rss_table_size; i++) { 832 lut[i] = (uint8_t)rss_info->queue[j]; 833 j = (j == rss_info->queue_num - 1) ? 0 : (j + 1); 834 } 835 836 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i); 837 if (ret) 838 return ret; 839 840 pf->hash_enabled_queues = 0; 841 for (i = 0; i < rss_info->queue_num; i++) 842 pf->hash_enabled_queues |= BIT_ULL(lut[i]); 843 844 pf->adapter->rss_reta_updated = 0; 845 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE; 846 } 847 848 /* The codes behind configure the input sets and symmetric hash 849 * function of the packet types and enable hash on them. 850 */ 851 pctypes = rss_conf->config_pctypes; 852 if (!pctypes) 853 return 0; 854 855 /* For first flow that will enable hash on any packet type, we clean 856 * the RSS sets that by legacy configuration commands and parameters. 857 */ 858 if (!pf->hash_filter_enabled) { 859 i40e_pf_disable_rss(pf); 860 pf->hash_filter_enabled = true; 861 } 862 863 do { 864 uint32_t idx = rte_bsf64(pctypes); 865 uint64_t bit = BIT_ULL(idx); 866 867 if (rss_conf->symmetric_enable) { 868 ret = i40e_hash_config_pctype_symmetric(hw, idx, true); 869 if (ret) 870 return ret; 871 872 rss_conf->reset_symmetric_pctypes |= bit; 873 } 874 875 ret = i40e_hash_config_pctype(hw, rss_conf, idx); 876 if (ret) 877 return ret; 878 879 rss_conf->reset_config_pctypes |= bit; 880 pctypes &= ~bit; 881 } while (pctypes); 882 883 return 0; 884 } 885 886 static void 887 i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act, 888 struct i40e_rte_flow_rss_conf *rss_conf) 889 { 890 const uint8_t *key = rss_act->key; 891 892 if (!key || rss_act->key_len != sizeof(rss_conf->key)) { 893 const uint32_t rss_key_default[] = {0x6b793944, 894 0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8, 895 0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605, 896 0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581}; 897 898 if (rss_act->key_len != sizeof(rss_conf->key)) 899 PMD_DRV_LOG(WARNING, 900 "RSS key length invalid, must be %u bytes, now set key to default", 901 (uint32_t)sizeof(rss_conf->key)); 902 903 memcpy(rss_conf->key, rss_key_default, sizeof(rss_conf->key)); 904 } else { 905 memcpy(rss_conf->key, key, sizeof(rss_conf->key)); 906 } 907 908 rss_conf->conf.key = rss_conf->key; 909 rss_conf->conf.key_len = sizeof(rss_conf->key); 910 } 911 912 static int 913 i40e_hash_parse_queues(const struct rte_eth_dev *dev, 914 const struct rte_flow_action_rss *rss_act, 915 struct i40e_rte_flow_rss_conf *rss_conf, 916 struct rte_flow_error *error) 917 { 918 struct i40e_pf *pf; 919 struct i40e_hw *hw; 920 uint16_t i; 921 int max_queue; 922 923 hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 924 if (!rss_act->queue_num || 925 rss_act->queue_num > hw->func_caps.rss_table_size) 926 return rte_flow_error_set(error, EINVAL, 927 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 928 NULL, "Invalid RSS queue number"); 929 930 if (rss_act->key_len) 931 PMD_DRV_LOG(WARNING, 932 "RSS key is ignored when queues specified"); 933 934 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 935 if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) 936 max_queue = i40e_pf_calc_configured_queues_num(pf); 937 else 938 max_queue = pf->dev_data->nb_rx_queues; 939 940 max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC); 941 942 for (i = 0; i < rss_act->queue_num; i++) { 943 if ((int)rss_act->queue[i] >= max_queue) 944 break; 945 } 946 947 if (i < rss_act->queue_num) 948 return rte_flow_error_set(error, EINVAL, 949 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 950 NULL, "Invalid RSS queues"); 951 952 memcpy(rss_conf->queue, rss_act->queue, 953 rss_act->queue_num * sizeof(rss_conf->queue[0])); 954 rss_conf->conf.queue = rss_conf->queue; 955 rss_conf->conf.queue_num = rss_act->queue_num; 956 return 0; 957 } 958 959 static int 960 i40e_hash_parse_queue_region(const struct rte_eth_dev *dev, 961 const struct rte_flow_item pattern[], 962 const struct rte_flow_action_rss *rss_act, 963 struct i40e_rte_flow_rss_conf *rss_conf, 964 struct rte_flow_error *error) 965 { 966 struct i40e_pf *pf; 967 const struct rte_flow_item_vlan *vlan_spec, *vlan_mask; 968 uint64_t hash_queues; 969 uint32_t i; 970 971 if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END) 972 return rte_flow_error_set(error, ENOTSUP, 973 RTE_FLOW_ERROR_TYPE_ITEM_NUM, 974 &pattern[1], 975 "Pattern not supported."); 976 977 vlan_spec = pattern->spec; 978 vlan_mask = pattern->mask; 979 if (!vlan_spec || !vlan_mask || 980 (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7) 981 return rte_flow_error_set(error, EINVAL, 982 RTE_FLOW_ERROR_TYPE_ITEM, pattern, 983 "Pattern error."); 984 985 if (!rss_act->queue) 986 return rte_flow_error_set(error, EINVAL, 987 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 988 NULL, "Queues not specified"); 989 990 if (rss_act->key_len) 991 PMD_DRV_LOG(WARNING, 992 "RSS key is ignored when configure queue region"); 993 994 /* Use a 64 bit variable to represent all queues in a region. */ 995 RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64); 996 997 if (!rss_act->queue_num || 998 !rte_is_power_of_2(rss_act->queue_num) || 999 rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC) 1000 return rte_flow_error_set(error, EINVAL, 1001 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1002 NULL, "Queue number error"); 1003 1004 for (i = 1; i < rss_act->queue_num; i++) { 1005 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i]) 1006 break; 1007 } 1008 1009 if (i < rss_act->queue_num) 1010 return rte_flow_error_set(error, EINVAL, 1011 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1012 NULL, 1013 "Queues must be incremented continuously"); 1014 1015 /* Map all queues to bits of uint64_t */ 1016 hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) & 1017 ~(BIT_ULL(rss_act->queue[0]) - 1); 1018 1019 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1020 if (hash_queues & ~pf->hash_enabled_queues) 1021 return rte_flow_error_set(error, EINVAL, 1022 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1023 NULL, "Some queues are not in LUT"); 1024 1025 rss_conf->region_queue_num = (uint8_t)rss_act->queue_num; 1026 rss_conf->region_queue_start = rss_act->queue[0]; 1027 rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13; 1028 return 0; 1029 } 1030 1031 static int 1032 i40e_hash_parse_global_conf(const struct rte_eth_dev *dev, 1033 const struct rte_flow_item pattern[], 1034 const struct rte_flow_action_rss *rss_act, 1035 struct i40e_rte_flow_rss_conf *rss_conf, 1036 struct rte_flow_error *error) 1037 { 1038 if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) 1039 return rte_flow_error_set(error, EINVAL, 1040 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1041 NULL, 1042 "Symmetric function should be set with pattern types"); 1043 1044 rss_conf->conf.func = rss_act->func; 1045 1046 if (rss_act->types) 1047 PMD_DRV_LOG(WARNING, 1048 "RSS types are ignored when no pattern specified"); 1049 1050 if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN) 1051 return i40e_hash_parse_queue_region(dev, pattern, rss_act, 1052 rss_conf, error); 1053 1054 if (rss_act->queue) 1055 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error); 1056 1057 if (rss_act->key_len) { 1058 i40e_hash_parse_key(rss_act, rss_conf); 1059 return 0; 1060 } 1061 1062 if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT) 1063 PMD_DRV_LOG(WARNING, "Nothing change"); 1064 return 0; 1065 } 1066 1067 static bool 1068 i40e_hash_validate_rss_types(uint64_t rss_types) 1069 { 1070 uint64_t type, mask; 1071 1072 /* Validate L2 */ 1073 type = ETH_RSS_ETH & rss_types; 1074 mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types; 1075 if (!type && mask) 1076 return false; 1077 1078 /* Validate L3 */ 1079 type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | 1080 ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 | 1081 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types; 1082 mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types; 1083 if (!type && mask) 1084 return false; 1085 1086 /* Validate L4 */ 1087 type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types; 1088 mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types; 1089 if (!type && mask) 1090 return false; 1091 1092 return true; 1093 } 1094 1095 static int 1096 i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev, 1097 const struct rte_flow_item pattern[], 1098 const struct rte_flow_action_rss *rss_act, 1099 struct i40e_rte_flow_rss_conf *rss_conf, 1100 struct rte_flow_error *error) 1101 { 1102 if (rss_act->queue) 1103 return rte_flow_error_set(error, EINVAL, 1104 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1105 NULL, 1106 "RSS Queues not supported when pattern specified"); 1107 1108 switch (rss_act->func) { 1109 case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ: 1110 rss_conf->symmetric_enable = true; 1111 break; 1112 case RTE_ETH_HASH_FUNCTION_DEFAULT: 1113 case RTE_ETH_HASH_FUNCTION_TOEPLITZ: 1114 case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR: 1115 break; 1116 default: 1117 return rte_flow_error_set(error, EINVAL, 1118 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1119 NULL, 1120 "RSS hash function not supported " 1121 "when pattern specified"); 1122 } 1123 1124 if (!i40e_hash_validate_rss_types(rss_act->types)) 1125 return rte_flow_error_set(error, EINVAL, 1126 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1127 NULL, "RSS types are invalid"); 1128 1129 if (rss_act->key_len) 1130 i40e_hash_parse_key(rss_act, rss_conf); 1131 1132 rss_conf->conf.func = rss_act->func; 1133 rss_conf->conf.types = rss_act->types; 1134 rss_conf->inset = i40e_hash_get_inset(rss_act->types); 1135 1136 return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act, 1137 rss_conf, error); 1138 } 1139 1140 int 1141 i40e_hash_parse(const struct rte_eth_dev *dev, 1142 const struct rte_flow_item pattern[], 1143 const struct rte_flow_action actions[], 1144 struct i40e_rte_flow_rss_conf *rss_conf, 1145 struct rte_flow_error *error) 1146 { 1147 const struct rte_flow_action_rss *rss_act; 1148 1149 if (actions[1].type != RTE_FLOW_ACTION_TYPE_END) 1150 return rte_flow_error_set(error, EINVAL, 1151 RTE_FLOW_ERROR_TYPE_ACTION, 1152 &actions[1], 1153 "Only support one action for RSS."); 1154 1155 rss_act = (const struct rte_flow_action_rss *)actions[0].conf; 1156 if (rss_act->level) 1157 return rte_flow_error_set(error, ENOTSUP, 1158 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1159 actions, 1160 "RSS level is not supported"); 1161 1162 while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID) 1163 pattern++; 1164 1165 if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END || 1166 pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN) 1167 return i40e_hash_parse_global_conf(dev, pattern, rss_act, 1168 rss_conf, error); 1169 1170 return i40e_hash_parse_pattern_act(dev, pattern, rss_act, 1171 rss_conf, error); 1172 } 1173 1174 static void 1175 i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf, 1176 struct i40e_rte_flow_rss_conf *conf) 1177 { 1178 uint32_t reset_flags = conf->misc_reset_flags; 1179 1180 conf->misc_reset_flags &= ~ref_conf->misc_reset_flags; 1181 1182 if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) && 1183 (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) && 1184 (conf->region_queue_start != ref_conf->region_queue_start || 1185 conf->region_queue_num != ref_conf->region_queue_num)) 1186 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION; 1187 1188 conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes; 1189 conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes; 1190 } 1191 1192 int 1193 i40e_hash_filter_restore(struct i40e_pf *pf) 1194 { 1195 struct i40e_rss_filter *filter; 1196 int ret; 1197 1198 TAILQ_FOREACH(filter, &pf->rss_config_list, next) { 1199 struct i40e_rte_flow_rss_conf *rss_conf = 1200 &filter->rss_filter_info; 1201 struct i40e_rss_filter *prev; 1202 1203 rss_conf->misc_reset_flags = 0; 1204 rss_conf->reset_config_pctypes = 0; 1205 rss_conf->reset_symmetric_pctypes = 0; 1206 1207 ret = i40e_hash_config(pf, rss_conf); 1208 if (ret) { 1209 pf->hash_filter_enabled = 0; 1210 i40e_pf_disable_rss(pf); 1211 PMD_DRV_LOG(ERR, 1212 "Re-configure RSS failed, RSS has been disabled"); 1213 return ret; 1214 } 1215 1216 /* Invalid previous RSS filter */ 1217 TAILQ_FOREACH(prev, &pf->rss_config_list, next) { 1218 if (prev == filter) 1219 break; 1220 i40e_invalid_rss_filter(rss_conf, 1221 &prev->rss_filter_info); 1222 } 1223 } 1224 1225 return 0; 1226 } 1227 1228 int 1229 i40e_hash_filter_create(struct i40e_pf *pf, 1230 struct i40e_rte_flow_rss_conf *rss_conf) 1231 { 1232 struct i40e_rss_filter *filter, *prev; 1233 struct i40e_rte_flow_rss_conf *new_conf; 1234 int ret; 1235 1236 filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0); 1237 if (!filter) { 1238 PMD_DRV_LOG(ERR, "Failed to allocate memory."); 1239 return -ENOMEM; 1240 } 1241 1242 new_conf = &filter->rss_filter_info; 1243 1244 memcpy(new_conf, rss_conf, sizeof(*new_conf)); 1245 if (new_conf->conf.queue_num) 1246 new_conf->conf.queue = new_conf->queue; 1247 if (new_conf->conf.key_len) 1248 new_conf->conf.key = new_conf->key; 1249 1250 ret = i40e_hash_config(pf, new_conf); 1251 if (ret) { 1252 rte_free(filter); 1253 if (i40e_pf_config_rss(pf)) 1254 return ret; 1255 1256 (void)i40e_hash_filter_restore(pf); 1257 return ret; 1258 } 1259 1260 /* Invalid previous RSS filter */ 1261 TAILQ_FOREACH(prev, &pf->rss_config_list, next) 1262 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info); 1263 1264 TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next); 1265 return 0; 1266 } 1267 1268 static int 1269 i40e_hash_reset_conf(struct i40e_pf *pf, 1270 struct i40e_rte_flow_rss_conf *rss_conf) 1271 { 1272 struct i40e_hw *hw = &pf->adapter->hw; 1273 struct rte_eth_dev *dev; 1274 uint64_t inset; 1275 uint32_t idx; 1276 int ret; 1277 1278 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) { 1279 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ); 1280 if (ret) 1281 return ret; 1282 1283 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC; 1284 } 1285 1286 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) { 1287 dev = &rte_eth_devices[pf->dev_data->port_id]; 1288 ret = i40e_flush_queue_region_all_conf(dev, hw, pf, 0); 1289 if (ret) 1290 return ret; 1291 1292 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION; 1293 } 1294 1295 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) { 1296 ret = i40e_pf_reset_rss_key(pf); 1297 if (ret) 1298 return ret; 1299 1300 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY; 1301 } 1302 1303 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) { 1304 if (!pf->adapter->rss_reta_updated) { 1305 ret = i40e_pf_reset_rss_reta(pf); 1306 if (ret) 1307 return ret; 1308 } 1309 1310 pf->hash_enabled_queues = 0; 1311 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE; 1312 } 1313 1314 while (rss_conf->reset_config_pctypes) { 1315 idx = rte_bsf64(rss_conf->reset_config_pctypes); 1316 1317 i40e_hash_enable_pctype(hw, idx, false); 1318 inset = i40e_get_default_input_set(idx); 1319 if (inset) { 1320 ret = i40e_set_hash_inset(hw, inset, idx, false); 1321 if (ret) 1322 return ret; 1323 } 1324 1325 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx); 1326 } 1327 1328 while (rss_conf->reset_symmetric_pctypes) { 1329 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes); 1330 1331 ret = i40e_hash_config_pctype_symmetric(hw, idx, false); 1332 if (ret) 1333 return ret; 1334 1335 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx); 1336 } 1337 1338 return 0; 1339 } 1340 1341 int 1342 i40e_hash_filter_destroy(struct i40e_pf *pf, 1343 const struct i40e_rss_filter *rss_filter) 1344 { 1345 struct i40e_rss_filter *filter; 1346 int ret; 1347 1348 TAILQ_FOREACH(filter, &pf->rss_config_list, next) { 1349 if (rss_filter == filter) { 1350 ret = i40e_hash_reset_conf(pf, 1351 &filter->rss_filter_info); 1352 if (ret) 1353 return ret; 1354 1355 TAILQ_REMOVE(&pf->rss_config_list, filter, next); 1356 rte_free(filter); 1357 return 0; 1358 } 1359 } 1360 1361 return -ENOENT; 1362 } 1363 1364 int 1365 i40e_hash_filter_flush(struct i40e_pf *pf) 1366 { 1367 struct rte_flow *flow, *next; 1368 1369 TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) { 1370 if (flow->filter_type != RTE_ETH_FILTER_HASH) 1371 continue; 1372 1373 if (flow->rule) { 1374 struct i40e_rss_filter *filter = flow->rule; 1375 int ret; 1376 1377 ret = i40e_hash_reset_conf(pf, 1378 &filter->rss_filter_info); 1379 if (ret) 1380 return ret; 1381 1382 TAILQ_REMOVE(&pf->rss_config_list, filter, next); 1383 rte_free(filter); 1384 } 1385 1386 TAILQ_REMOVE(&pf->flow_list, flow, node); 1387 rte_free(flow); 1388 } 1389 1390 assert(!pf->rss_config_list.tqh_first); 1391 return 0; 1392 } 1393