1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015 Cavium, Inc
3 */
4
5 #ifndef _RTE_CRC_ARM64_H_
6 #define _RTE_CRC_ARM64_H_
7
8 /**
9 * @file
10 *
11 * RTE CRC arm64 Hash
12 */
13
14 #ifdef __cplusplus
15 extern "C" {
16 #endif
17
18 #include <stdint.h>
19 #include <rte_cpuflags.h>
20 #include <rte_branch_prediction.h>
21 #include <rte_common.h>
22
23 static inline uint32_t
crc32c_arm64_u8(uint8_t data,uint32_t init_val)24 crc32c_arm64_u8(uint8_t data, uint32_t init_val)
25 {
26 __asm__ volatile(
27 "crc32cb %w[crc], %w[crc], %w[value]"
28 : [crc] "+r" (init_val)
29 : [value] "r" (data));
30 return init_val;
31 }
32
33 static inline uint32_t
crc32c_arm64_u16(uint16_t data,uint32_t init_val)34 crc32c_arm64_u16(uint16_t data, uint32_t init_val)
35 {
36 __asm__ volatile(
37 "crc32ch %w[crc], %w[crc], %w[value]"
38 : [crc] "+r" (init_val)
39 : [value] "r" (data));
40 return init_val;
41 }
42
43 static inline uint32_t
crc32c_arm64_u32(uint32_t data,uint32_t init_val)44 crc32c_arm64_u32(uint32_t data, uint32_t init_val)
45 {
46 __asm__ volatile(
47 "crc32cw %w[crc], %w[crc], %w[value]"
48 : [crc] "+r" (init_val)
49 : [value] "r" (data));
50 return init_val;
51 }
52
53 static inline uint32_t
crc32c_arm64_u64(uint64_t data,uint32_t init_val)54 crc32c_arm64_u64(uint64_t data, uint32_t init_val)
55 {
56 __asm__ volatile(
57 "crc32cx %w[crc], %w[crc], %x[value]"
58 : [crc] "+r" (init_val)
59 : [value] "r" (data));
60 return init_val;
61 }
62
63 /**
64 * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash
65 * calculation.
66 *
67 * @param alg
68 * An OR of following flags:
69 * - (CRC32_SW) Don't use arm64 crc intrinsics
70 * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available
71 *
72 */
73 static inline void
rte_hash_crc_set_alg(uint8_t alg)74 rte_hash_crc_set_alg(uint8_t alg)
75 {
76 switch (alg) {
77 case CRC32_ARM64:
78 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
79 alg = CRC32_SW;
80 /* fall-through */
81 case CRC32_SW:
82 crc32_alg = alg;
83 /* fall-through */
84 default:
85 break;
86 }
87 }
88
89 /* Setting the best available algorithm */
RTE_INIT(rte_hash_crc_init_alg)90 RTE_INIT(rte_hash_crc_init_alg)
91 {
92 rte_hash_crc_set_alg(CRC32_ARM64);
93 }
94
95 /**
96 * Use single crc32 instruction to perform a hash on a 1 byte value.
97 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
98 * not supported
99 *
100 * @param data
101 * Data to perform hash on.
102 * @param init_val
103 * Value to initialise hash generator.
104 * @return
105 * 32bit calculated hash value.
106 */
107 static inline uint32_t
rte_hash_crc_1byte(uint8_t data,uint32_t init_val)108 rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
109 {
110 if (likely(crc32_alg & CRC32_ARM64))
111 return crc32c_arm64_u8(data, init_val);
112
113 return crc32c_1byte(data, init_val);
114 }
115
116 /**
117 * Use single crc32 instruction to perform a hash on a 2 bytes value.
118 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
119 * not supported
120 *
121 * @param data
122 * Data to perform hash on.
123 * @param init_val
124 * Value to initialise hash generator.
125 * @return
126 * 32bit calculated hash value.
127 */
128 static inline uint32_t
rte_hash_crc_2byte(uint16_t data,uint32_t init_val)129 rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
130 {
131 if (likely(crc32_alg & CRC32_ARM64))
132 return crc32c_arm64_u16(data, init_val);
133
134 return crc32c_2bytes(data, init_val);
135 }
136
137 /**
138 * Use single crc32 instruction to perform a hash on a 4 byte value.
139 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
140 * not supported
141 *
142 * @param data
143 * Data to perform hash on.
144 * @param init_val
145 * Value to initialise hash generator.
146 * @return
147 * 32bit calculated hash value.
148 */
149 static inline uint32_t
rte_hash_crc_4byte(uint32_t data,uint32_t init_val)150 rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
151 {
152 if (likely(crc32_alg & CRC32_ARM64))
153 return crc32c_arm64_u32(data, init_val);
154
155 return crc32c_1word(data, init_val);
156 }
157
158 /**
159 * Use single crc32 instruction to perform a hash on a 8 byte value.
160 * Fall back to software crc32 implementation in case arm64 crc intrinsics is
161 * not supported
162 *
163 * @param data
164 * Data to perform hash on.
165 * @param init_val
166 * Value to initialise hash generator.
167 * @return
168 * 32bit calculated hash value.
169 */
170 static inline uint32_t
rte_hash_crc_8byte(uint64_t data,uint32_t init_val)171 rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
172 {
173 if (likely(crc32_alg == CRC32_ARM64))
174 return crc32c_arm64_u64(data, init_val);
175
176 return crc32c_2words(data, init_val);
177 }
178
179 #ifdef __cplusplus
180 }
181 #endif
182
183 #endif /* _RTE_CRC_ARM64_H_ */
184