1 // Copyright (c) 2018, Arm Limited and affiliates. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #include "util/crc32c_arm64.h"
7
8 #if defined(__linux__) && defined(HAVE_ARM64_CRC)
9
10 #include <asm/hwcap.h>
11 #ifdef ROCKSDB_AUXV_GETAUXVAL_PRESENT
12 #include <sys/auxv.h>
13 #endif
14 #ifndef HWCAP_CRC32
15 #define HWCAP_CRC32 (1 << 7)
16 #endif
17
18 #ifdef HAVE_ARM64_CRYPTO
19 /* unfolding to compute 8 * 3 = 24 bytes parallelly */
20 #define CRC32C24BYTES(ITR) \
21 crc1 = crc32c_u64(crc1, *(buf64 + BLK_LENGTH + (ITR))); \
22 crc2 = crc32c_u64(crc2, *(buf64 + BLK_LENGTH * 2 + (ITR))); \
23 crc0 = crc32c_u64(crc0, *(buf64 + (ITR)));
24
25 /* unfolding to compute 24 * 7 = 168 bytes parallelly */
26 #define CRC32C7X24BYTES(ITR) \
27 do { \
28 CRC32C24BYTES((ITR)*7 + 0) \
29 CRC32C24BYTES((ITR)*7 + 1) \
30 CRC32C24BYTES((ITR)*7 + 2) \
31 CRC32C24BYTES((ITR)*7 + 3) \
32 CRC32C24BYTES((ITR)*7 + 4) \
33 CRC32C24BYTES((ITR)*7 + 5) \
34 CRC32C24BYTES((ITR)*7 + 6) \
35 } while (0)
36 #endif
37
crc32c_runtime_check(void)38 uint32_t crc32c_runtime_check(void) {
39 #ifdef ROCKSDB_AUXV_GETAUXVAL_PRESENT
40 uint64_t auxv = getauxval(AT_HWCAP);
41 return (auxv & HWCAP_CRC32) != 0;
42 #else
43 return 0;
44 #endif
45 }
46
crc32c_arm64(uint32_t crc,unsigned char const * data,unsigned len)47 uint32_t crc32c_arm64(uint32_t crc, unsigned char const *data,
48 unsigned len) {
49 const uint8_t *buf8;
50 const uint64_t *buf64 = (uint64_t *)data;
51 int length = (int)len;
52 crc ^= 0xffffffff;
53
54 #ifdef HAVE_ARM64_CRYPTO
55 /* Crc32c Parallel computation
56 * Algorithm comes from Intel whitepaper:
57 * crc-iscsi-polynomial-crc32-instruction-paper
58 *
59 * Input data is divided into three equal-sized blocks
60 * Three parallel blocks (crc0, crc1, crc2) for 1024 Bytes
61 * One Block: 42(BLK_LENGTH) * 8(step length: crc32c_u64) bytes
62 */
63 #define BLK_LENGTH 42
64 while (length >= 1024) {
65 uint64_t t0, t1;
66 uint32_t crc0 = 0, crc1 = 0, crc2 = 0;
67
68 /* Parallel Param:
69 * k0 = CRC32(x ^ (42 * 8 * 8 * 2 - 1));
70 * k1 = CRC32(x ^ (42 * 8 * 8 - 1));
71 */
72 uint32_t k0 = 0xe417f38a, k1 = 0x8f158014;
73
74 /* Prefetch data for following block to avoid cache miss */
75 PREF1KL1((uint8_t *)buf64, 1024);
76
77 /* First 8 byte for better pipelining */
78 crc0 = crc32c_u64(crc, *buf64++);
79
80 /* 3 blocks crc32c parallel computation
81 * Macro unfolding to compute parallelly
82 * 168 * 6 = 1008 (bytes)
83 */
84 CRC32C7X24BYTES(0);
85 CRC32C7X24BYTES(1);
86 CRC32C7X24BYTES(2);
87 CRC32C7X24BYTES(3);
88 CRC32C7X24BYTES(4);
89 CRC32C7X24BYTES(5);
90 buf64 += (BLK_LENGTH * 3);
91
92 /* Last 8 bytes */
93 crc = crc32c_u64(crc2, *buf64++);
94
95 t0 = (uint64_t)vmull_p64(crc0, k0);
96 t1 = (uint64_t)vmull_p64(crc1, k1);
97
98 /* Merge (crc0, crc1, crc2) -> crc */
99 crc1 = crc32c_u64(0, t1);
100 crc ^= crc1;
101 crc0 = crc32c_u64(0, t0);
102 crc ^= crc0;
103
104 length -= 1024;
105 }
106
107 if (length == 0) return crc ^ (0xffffffffU);
108 #endif
109 buf8 = (const uint8_t *)buf64;
110 while (length >= 8) {
111 crc = crc32c_u64(crc, *(const uint64_t *)buf8);
112 buf8 += 8;
113 length -= 8;
114 }
115
116 /* The following is more efficient than the straight loop */
117 if (length >= 4) {
118 crc = crc32c_u32(crc, *(const uint32_t *)buf8);
119 buf8 += 4;
120 length -= 4;
121 }
122
123 if (length >= 2) {
124 crc = crc32c_u16(crc, *(const uint16_t *)buf8);
125 buf8 += 2;
126 length -= 2;
127 }
128
129 if (length >= 1) crc = crc32c_u8(crc, *buf8);
130
131 crc ^= 0xffffffff;
132 return crc;
133 }
134
135 #endif
136