1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2019 Vladimir Medvedkin <[email protected]>
3 */
4
5 #ifndef _RTE_THASH_H
6 #define _RTE_THASH_H
7
8 /**
9 * @file
10 *
11 * toeplitz hash functions.
12 */
13
14 #ifdef __cplusplus
15 extern "C" {
16 #endif
17
18 /**
19 * Software implementation of the Toeplitz hash function used by RSS.
20 * Can be used either for packet distribution on single queue NIC
21 * or for simulating of RSS computation on specific NIC (for example
22 * after GRE header decapsulating)
23 */
24
25 #include <stdint.h>
26 #include <rte_byteorder.h>
27 #include <rte_config.h>
28 #include <rte_ip.h>
29 #include <rte_common.h>
30
31 #if defined(RTE_ARCH_X86) || defined(__ARM_NEON)
32 #include <rte_vect.h>
33 #endif
34
35 #ifdef RTE_ARCH_X86
36 /* Byte swap mask used for converting IPv6 address
37 * 4-byte chunks to CPU byte order
38 */
39 static const __m128i rte_thash_ipv6_bswap_mask = {
40 0x0405060700010203ULL, 0x0C0D0E0F08090A0BULL};
41 #endif
42
43 /**
44 * length in dwords of input tuple to
45 * calculate hash of ipv4 header only
46 */
47 #define RTE_THASH_V4_L3_LEN ((sizeof(struct rte_ipv4_tuple) - \
48 sizeof(((struct rte_ipv4_tuple *)0)->sctp_tag)) / 4)
49
50 /**
51 * length in dwords of input tuple to
52 * calculate hash of ipv4 header +
53 * transport header
54 */
55 #define RTE_THASH_V4_L4_LEN ((sizeof(struct rte_ipv4_tuple)) / 4)
56
57 /**
58 * length in dwords of input tuple to
59 * calculate hash of ipv6 header only
60 */
61 #define RTE_THASH_V6_L3_LEN ((sizeof(struct rte_ipv6_tuple) - \
62 sizeof(((struct rte_ipv6_tuple *)0)->sctp_tag)) / 4)
63
64 /**
65 * length in dwords of input tuple to
66 * calculate hash of ipv6 header +
67 * transport header
68 */
69 #define RTE_THASH_V6_L4_LEN ((sizeof(struct rte_ipv6_tuple)) / 4)
70
71 /**
72 * IPv4 tuple
73 * addresses and ports/sctp_tag have to be CPU byte order
74 */
75 struct rte_ipv4_tuple {
76 uint32_t src_addr;
77 uint32_t dst_addr;
78 RTE_STD_C11
79 union {
80 struct {
81 uint16_t dport;
82 uint16_t sport;
83 };
84 uint32_t sctp_tag;
85 };
86 };
87
88 /**
89 * IPv6 tuple
90 * Addresses have to be filled by rte_thash_load_v6_addr()
91 * ports/sctp_tag have to be CPU byte order
92 */
93 struct rte_ipv6_tuple {
94 uint8_t src_addr[16];
95 uint8_t dst_addr[16];
96 RTE_STD_C11
97 union {
98 struct {
99 uint16_t dport;
100 uint16_t sport;
101 };
102 uint32_t sctp_tag;
103 };
104 };
105
106 union rte_thash_tuple {
107 struct rte_ipv4_tuple v4;
108 struct rte_ipv6_tuple v6;
109 #ifdef RTE_ARCH_X86
110 } __rte_aligned(XMM_SIZE);
111 #else
112 };
113 #endif
114
115 /**
116 * Prepare special converted key to use with rte_softrss_be()
117 * @param orig
118 * pointer to original RSS key
119 * @param targ
120 * pointer to target RSS key
121 * @param len
122 * RSS key length
123 */
124 static inline void
rte_convert_rss_key(const uint32_t * orig,uint32_t * targ,int len)125 rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len)
126 {
127 int i;
128
129 for (i = 0; i < (len >> 2); i++)
130 targ[i] = rte_be_to_cpu_32(orig[i]);
131 }
132
133 /**
134 * Prepare and load IPv6 addresses (src and dst)
135 * into target tuple
136 * @param orig
137 * Pointer to ipv6 header of the original packet
138 * @param targ
139 * Pointer to rte_ipv6_tuple structure
140 */
141 static inline void
rte_thash_load_v6_addrs(const struct rte_ipv6_hdr * orig,union rte_thash_tuple * targ)142 rte_thash_load_v6_addrs(const struct rte_ipv6_hdr *orig,
143 union rte_thash_tuple *targ)
144 {
145 #ifdef RTE_ARCH_X86
146 __m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr);
147 *(__m128i *)targ->v6.src_addr =
148 _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
149 ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr);
150 *(__m128i *)targ->v6.dst_addr =
151 _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
152 #elif defined(__ARM_NEON)
153 uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr);
154 vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6));
155 ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr);
156 vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6));
157 #else
158 int i;
159 for (i = 0; i < 4; i++) {
160 *((uint32_t *)targ->v6.src_addr + i) =
161 rte_be_to_cpu_32(*((const uint32_t *)orig->src_addr + i));
162 *((uint32_t *)targ->v6.dst_addr + i) =
163 rte_be_to_cpu_32(*((const uint32_t *)orig->dst_addr + i));
164 }
165 #endif
166 }
167
168 /**
169 * Generic implementation. Can be used with original rss_key
170 * @param input_tuple
171 * Pointer to input tuple
172 * @param input_len
173 * Length of input_tuple in 4-bytes chunks
174 * @param rss_key
175 * Pointer to RSS hash key.
176 * @return
177 * Calculated hash value.
178 */
179 static inline uint32_t
rte_softrss(uint32_t * input_tuple,uint32_t input_len,const uint8_t * rss_key)180 rte_softrss(uint32_t *input_tuple, uint32_t input_len,
181 const uint8_t *rss_key)
182 {
183 uint32_t i, j, map, ret = 0;
184
185 for (j = 0; j < input_len; j++) {
186 for (map = input_tuple[j]; map; map &= (map - 1)) {
187 i = rte_bsf32(map);
188 ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << (31 - i) |
189 (uint32_t)((uint64_t)(rte_cpu_to_be_32(((const uint32_t *)rss_key)[j + 1])) >>
190 (i + 1));
191 }
192 }
193 return ret;
194 }
195
196 /**
197 * Optimized implementation.
198 * If you want the calculated hash value matches NIC RSS value
199 * you have to use special converted key with rte_convert_rss_key() fn.
200 * @param input_tuple
201 * Pointer to input tuple
202 * @param input_len
203 * Length of input_tuple in 4-bytes chunks
204 * @param *rss_key
205 * Pointer to RSS hash key.
206 * @return
207 * Calculated hash value.
208 */
209 static inline uint32_t
rte_softrss_be(uint32_t * input_tuple,uint32_t input_len,const uint8_t * rss_key)210 rte_softrss_be(uint32_t *input_tuple, uint32_t input_len,
211 const uint8_t *rss_key)
212 {
213 uint32_t i, j, map, ret = 0;
214
215 for (j = 0; j < input_len; j++) {
216 for (map = input_tuple[j]; map; map &= (map - 1)) {
217 i = rte_bsf32(map);
218 ret ^= ((const uint32_t *)rss_key)[j] << (31 - i) |
219 (uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (i + 1));
220 }
221 }
222 return ret;
223 }
224
225 #ifdef __cplusplus
226 }
227 #endif
228
229 #endif /* _RTE_THASH_H */
230