1 //===-- Elementary operations for aarch64 --------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H 10 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H 11 12 #include "src/__support/architectures.h" 13 14 #if defined(LLVM_LIBC_ARCH_AARCH64) 15 16 #include <src/string/memory_utils/elements.h> 17 #include <stddef.h> // size_t 18 #include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t 19 20 #ifdef __ARM_NEON 21 #include <arm_neon.h> 22 #endif 23 24 namespace __llvm_libc { 25 namespace aarch64_memset { 26 #ifdef __ARM_NEON 27 struct Splat8 { 28 static constexpr size_t kSize = 8; 29 static void SplatSet(char *dst, const unsigned char value) { 30 vst1_u8((uint8_t *)dst, vdup_n_u8(value)); 31 } 32 }; 33 34 struct Splat16 { 35 static constexpr size_t kSize = 16; 36 static void SplatSet(char *dst, const unsigned char value) { 37 vst1q_u8((uint8_t *)dst, vdupq_n_u8(value)); 38 } 39 }; 40 41 using _8 = Splat8; 42 using _16 = Splat16; 43 #else 44 using _8 = __llvm_libc::scalar::_8; 45 using _16 = Repeated<_8, 2>; 46 #endif // __ARM_NEON 47 48 using _1 = __llvm_libc::scalar::_1; 49 using _2 = __llvm_libc::scalar::_2; 50 using _3 = __llvm_libc::scalar::_3; 51 using _4 = __llvm_libc::scalar::_4; 52 using _32 = Chained<_16, _16>; 53 using _64 = Chained<_32, _32>; 54 55 struct ZVA { 56 static constexpr size_t kSize = 64; 57 static void SplatSet(char *dst, const unsigned char value) { 58 asm("dc zva, %[dst]" : : [dst] "r"(dst) : "memory"); 59 } 60 }; 61 62 inline static bool AArch64ZVA(char *dst, size_t count) { 63 uint64_t zva_val; 64 asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val)); 65 if ((zva_val & 31) != 4) 66 return false; 67 SplatSet<Align<_64, Arg::_1>::Then<Loop<ZVA, _64>>>(dst, 0, count); 68 return true; 69 } 70 71 } // namespace aarch64_memset 72 73 namespace aarch64 { 74 75 using _1 = __llvm_libc::scalar::_1; 76 using _2 = __llvm_libc::scalar::_2; 77 using _3 = __llvm_libc::scalar::_3; 78 using _4 = __llvm_libc::scalar::_4; 79 using _8 = __llvm_libc::scalar::_8; 80 using _16 = __llvm_libc::scalar::_16; 81 82 #ifdef __ARM_NEON 83 struct N32 { 84 static constexpr size_t kSize = 32; 85 static bool Equals(const char *lhs, const char *rhs) { 86 uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs); 87 uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs); 88 uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16)); 89 uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16)); 90 uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1)); 91 uint64_t res = 92 vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0); 93 return res == 0; 94 } 95 static int ThreeWayCompare(const char *lhs, const char *rhs) { 96 uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs); 97 uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs); 98 uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16)); 99 uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16)); 100 uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1)); 101 uint64_t res = 102 vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0); 103 if (res == 0) 104 return 0; 105 size_t index = (__builtin_ctzl(res) >> 3) << 2; 106 uint32_t l = *((const uint32_t *)(lhs + index)); 107 uint32_t r = *((const uint32_t *)(rhs + index)); 108 return __llvm_libc::scalar::_4::ScalarThreeWayCompare(l, r); 109 } 110 }; 111 112 using _32 = N32; 113 using _64 = Repeated<_32, 2>; 114 #else 115 using _32 = __llvm_libc::scalar::_32; 116 using _64 = __llvm_libc::scalar::_64; 117 #endif // __ARM_NEON 118 119 } // namespace aarch64 120 } // namespace __llvm_libc 121 122 #endif // defined(LLVM_LIBC_ARCH_AARCH64) 123 124 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H 125