1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2018 Intel Corporation
3 */
4
5 #ifndef __INCLUDE_RTE_TABLE_HASH_FUNC_H__
6 #define __INCLUDE_RTE_TABLE_HASH_FUNC_H__
7
8 #ifdef __cplusplus
9 extern "C" {
10 #endif
11
12 #include <stdint.h>
13
14 #include <rte_compat.h>
15 #include <rte_common.h>
16
17 __rte_experimental
18 static inline uint64_t
rte_crc32_u64_generic(uint64_t crc,uint64_t value)19 rte_crc32_u64_generic(uint64_t crc, uint64_t value)
20 {
21 int i;
22
23 crc = (crc & 0xFFFFFFFFLLU) ^ value;
24 for (i = 63; i >= 0; i--) {
25 uint64_t mask;
26
27 mask = -(crc & 1LLU);
28 crc = (crc >> 1LLU) ^ (0x82F63B78LLU & mask);
29 }
30
31 return crc;
32 }
33
34 #if defined(RTE_ARCH_X86_64)
35
36 #include <x86intrin.h>
37
38 static inline uint64_t
rte_crc32_u64(uint64_t crc,uint64_t v)39 rte_crc32_u64(uint64_t crc, uint64_t v)
40 {
41 return _mm_crc32_u64(crc, v);
42 }
43
44 #elif defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32)
45 #include "rte_table_hash_func_arm64.h"
46 #else
47
48 static inline uint64_t
rte_crc32_u64(uint64_t crc,uint64_t v)49 rte_crc32_u64(uint64_t crc, uint64_t v)
50 {
51 return rte_crc32_u64_generic(crc, v);
52 }
53
54 #endif
55
56 __rte_experimental
57 static inline uint64_t
rte_table_hash_crc_key8(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)58 rte_table_hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
59 uint64_t seed)
60 {
61 uint64_t *k = (uint64_t *)key;
62 uint64_t *m = (uint64_t *)mask;
63 uint64_t crc0;
64
65 crc0 = rte_crc32_u64(seed, k[0] & m[0]);
66
67 return crc0;
68 }
69
70 __rte_experimental
71 static inline uint64_t
rte_table_hash_crc_key16(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)72 rte_table_hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
73 uint64_t seed)
74 {
75 uint64_t *k = (uint64_t *)key;
76 uint64_t *m = (uint64_t *)mask;
77 uint64_t k0, crc0, crc1;
78
79 k0 = k[0] & m[0];
80
81 crc0 = rte_crc32_u64(k0, seed);
82 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
83
84 crc0 ^= crc1;
85
86 return crc0;
87 }
88
89 __rte_experimental
90 static inline uint64_t
rte_table_hash_crc_key24(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)91 rte_table_hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
92 uint64_t seed)
93 {
94 uint64_t *k = (uint64_t *)key;
95 uint64_t *m = (uint64_t *)mask;
96 uint64_t k0, k2, crc0, crc1;
97
98 k0 = k[0] & m[0];
99 k2 = k[2] & m[2];
100
101 crc0 = rte_crc32_u64(k0, seed);
102 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
103
104 crc0 = rte_crc32_u64(crc0, k2);
105
106 crc0 ^= crc1;
107
108 return crc0;
109 }
110
111 __rte_experimental
112 static inline uint64_t
rte_table_hash_crc_key32(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)113 rte_table_hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
114 uint64_t seed)
115 {
116 uint64_t *k = (uint64_t *)key;
117 uint64_t *m = (uint64_t *)mask;
118 uint64_t k0, k2, crc0, crc1, crc2, crc3;
119
120 k0 = k[0] & m[0];
121 k2 = k[2] & m[2];
122
123 crc0 = rte_crc32_u64(k0, seed);
124 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
125
126 crc2 = rte_crc32_u64(k2, k[3] & m[3]);
127 crc3 = k2 >> 32;
128
129 crc0 = rte_crc32_u64(crc0, crc1);
130 crc1 = rte_crc32_u64(crc2, crc3);
131
132 crc0 ^= crc1;
133
134 return crc0;
135 }
136
137 __rte_experimental
138 static inline uint64_t
rte_table_hash_crc_key40(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)139 rte_table_hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
140 uint64_t seed)
141 {
142 uint64_t *k = (uint64_t *)key;
143 uint64_t *m = (uint64_t *)mask;
144 uint64_t k0, k2, crc0, crc1, crc2, crc3;
145
146 k0 = k[0] & m[0];
147 k2 = k[2] & m[2];
148
149 crc0 = rte_crc32_u64(k0, seed);
150 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
151
152 crc2 = rte_crc32_u64(k2, k[3] & m[3]);
153 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]);
154
155 crc0 = rte_crc32_u64(crc0, crc1);
156 crc1 = rte_crc32_u64(crc2, crc3);
157
158 crc0 ^= crc1;
159
160 return crc0;
161 }
162
163 __rte_experimental
164 static inline uint64_t
rte_table_hash_crc_key48(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)165 rte_table_hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
166 uint64_t seed)
167 {
168 uint64_t *k = (uint64_t *)key;
169 uint64_t *m = (uint64_t *)mask;
170 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3;
171
172 k0 = k[0] & m[0];
173 k2 = k[2] & m[2];
174 k5 = k[5] & m[5];
175
176 crc0 = rte_crc32_u64(k0, seed);
177 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
178
179 crc2 = rte_crc32_u64(k2, k[3] & m[3]);
180 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]);
181
182 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2);
183 crc1 = rte_crc32_u64(crc3, k5);
184
185 crc0 ^= crc1;
186
187 return crc0;
188 }
189
190 __rte_experimental
191 static inline uint64_t
rte_table_hash_crc_key56(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)192 rte_table_hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
193 uint64_t seed)
194 {
195 uint64_t *k = (uint64_t *)key;
196 uint64_t *m = (uint64_t *)mask;
197 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
198
199 k0 = k[0] & m[0];
200 k2 = k[2] & m[2];
201 k5 = k[5] & m[5];
202
203 crc0 = rte_crc32_u64(k0, seed);
204 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
205
206 crc2 = rte_crc32_u64(k2, k[3] & m[3]);
207 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]);
208
209 crc4 = rte_crc32_u64(k5, k[6] & m[6]);
210 crc5 = k5 >> 32;
211
212 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2);
213 crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5);
214
215 crc0 ^= crc1;
216
217 return crc0;
218 }
219
220 __rte_experimental
221 static inline uint64_t
rte_table_hash_crc_key64(void * key,void * mask,__rte_unused uint32_t key_size,uint64_t seed)222 rte_table_hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
223 uint64_t seed)
224 {
225 uint64_t *k = (uint64_t *)key;
226 uint64_t *m = (uint64_t *)mask;
227 uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
228
229 k0 = k[0] & m[0];
230 k2 = k[2] & m[2];
231 k5 = k[5] & m[5];
232
233 crc0 = rte_crc32_u64(k0, seed);
234 crc1 = rte_crc32_u64(k0 >> 32, k[1] & m[1]);
235
236 crc2 = rte_crc32_u64(k2, k[3] & m[3]);
237 crc3 = rte_crc32_u64(k2 >> 32, k[4] & m[4]);
238
239 crc4 = rte_crc32_u64(k5, k[6] & m[6]);
240 crc5 = rte_crc32_u64(k5 >> 32, k[7] & m[7]);
241
242 crc0 = rte_crc32_u64(crc0, (crc1 << 32) ^ crc2);
243 crc1 = rte_crc32_u64(crc3, (crc4 << 32) ^ crc5);
244
245 crc0 ^= crc1;
246
247 return crc0;
248 }
249
250 #ifdef __cplusplus
251 }
252 #endif
253
254 #endif
255