1 //===-- memtag.h ------------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef SCUDO_MEMTAG_H_
10 #define SCUDO_MEMTAG_H_
11
12 #include "internal_defs.h"
13
14 #if SCUDO_LINUX
15 #include <sys/auxv.h>
16 #include <sys/prctl.h>
17 #endif
18
19 namespace scudo {
20
21 #if (__clang_major__ >= 12 && defined(__aarch64__)) || defined(SCUDO_FUZZ)
22
23 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
24 // tagging. Not all operating systems enable TBI, so we only claim architectural
25 // support for memory tagging if the operating system enables TBI.
26 #if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI)
archSupportsMemoryTagging()27 inline constexpr bool archSupportsMemoryTagging() { return true; }
28 #else
29 inline constexpr bool archSupportsMemoryTagging() { return false; }
30 #endif
31
archMemoryTagGranuleSize()32 inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
33
untagPointer(uptr Ptr)34 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
35
extractTag(uptr Ptr)36 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
37
38 #else
39
40 inline constexpr bool archSupportsMemoryTagging() { return false; }
41
42 inline uptr archMemoryTagGranuleSize() {
43 UNREACHABLE("memory tagging not supported");
44 }
45
46 inline uptr untagPointer(uptr Ptr) {
47 (void)Ptr;
48 UNREACHABLE("memory tagging not supported");
49 }
50
51 inline uint8_t extractTag(uptr Ptr) {
52 (void)Ptr;
53 UNREACHABLE("memory tagging not supported");
54 }
55
56 #endif
57
58 #if __clang_major__ >= 12 && defined(__aarch64__)
59
60 #if SCUDO_LINUX
61
systemSupportsMemoryTagging()62 inline bool systemSupportsMemoryTagging() {
63 #ifndef HWCAP2_MTE
64 #define HWCAP2_MTE (1 << 18)
65 #endif
66 return getauxval(AT_HWCAP2) & HWCAP2_MTE;
67 }
68
systemDetectsMemoryTagFaultsTestOnly()69 inline bool systemDetectsMemoryTagFaultsTestOnly() {
70 #ifndef PR_SET_TAGGED_ADDR_CTRL
71 #define PR_SET_TAGGED_ADDR_CTRL 54
72 #endif
73 #ifndef PR_GET_TAGGED_ADDR_CTRL
74 #define PR_GET_TAGGED_ADDR_CTRL 56
75 #endif
76 #ifndef PR_TAGGED_ADDR_ENABLE
77 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
78 #endif
79 #ifndef PR_MTE_TCF_SHIFT
80 #define PR_MTE_TCF_SHIFT 1
81 #endif
82 #ifndef PR_MTE_TAG_SHIFT
83 #define PR_MTE_TAG_SHIFT 3
84 #endif
85 #ifndef PR_MTE_TCF_NONE
86 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
87 #endif
88 #ifndef PR_MTE_TCF_SYNC
89 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
90 #endif
91 #ifndef PR_MTE_TCF_MASK
92 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
93 #endif
94 return (static_cast<unsigned long>(
95 prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &
96 PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
97 }
98
enableSystemMemoryTaggingTestOnly()99 inline void enableSystemMemoryTaggingTestOnly() {
100 prctl(PR_SET_TAGGED_ADDR_CTRL,
101 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
102 0, 0, 0);
103 }
104
105 #else // !SCUDO_LINUX
106
systemSupportsMemoryTagging()107 inline bool systemSupportsMemoryTagging() { return false; }
108
systemDetectsMemoryTagFaultsTestOnly()109 inline bool systemDetectsMemoryTagFaultsTestOnly() {
110 UNREACHABLE("memory tagging not supported");
111 }
112
enableSystemMemoryTaggingTestOnly()113 inline void enableSystemMemoryTaggingTestOnly() {
114 UNREACHABLE("memory tagging not supported");
115 }
116
117 #endif // SCUDO_LINUX
118
119 class ScopedDisableMemoryTagChecks {
120 uptr PrevTCO;
121
122 public:
ScopedDisableMemoryTagChecks()123 ScopedDisableMemoryTagChecks() {
124 __asm__ __volatile__(
125 R"(
126 .arch_extension memtag
127 mrs %0, tco
128 msr tco, #1
129 )"
130 : "=r"(PrevTCO));
131 }
132
~ScopedDisableMemoryTagChecks()133 ~ScopedDisableMemoryTagChecks() {
134 __asm__ __volatile__(
135 R"(
136 .arch_extension memtag
137 msr tco, %0
138 )"
139 :
140 : "r"(PrevTCO));
141 }
142 };
143
selectRandomTag(uptr Ptr,uptr ExcludeMask)144 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
145 ExcludeMask |= 1; // Always exclude Tag 0.
146 uptr TaggedPtr;
147 __asm__ __volatile__(
148 R"(
149 .arch_extension memtag
150 irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
151 )"
152 : [TaggedPtr] "=r"(TaggedPtr)
153 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
154 return TaggedPtr;
155 }
156
addFixedTag(uptr Ptr,uptr Tag)157 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
158 DCHECK_LT(Tag, 16);
159 DCHECK_EQ(untagPointer(Ptr), Ptr);
160 return Ptr | (Tag << 56);
161 }
162
storeTags(uptr Begin,uptr End)163 inline uptr storeTags(uptr Begin, uptr End) {
164 DCHECK_EQ(0, Begin % 16);
165 uptr LineSize, Next, Tmp;
166 __asm__ __volatile__(
167 R"(
168 .arch_extension memtag
169
170 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
171 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
172 // indicates that the DC instructions are unavailable.
173 DCZID .req %[Tmp]
174 mrs DCZID, dczid_el0
175 tbnz DCZID, #4, 3f
176 and DCZID, DCZID, #15
177 mov %[LineSize], #4
178 lsl %[LineSize], %[LineSize], DCZID
179 .unreq DCZID
180
181 // Our main loop doesn't handle the case where we don't need to perform any
182 // DC GZVA operations. If the size of our tagged region is less than
183 // twice the cache line size, bail out to the slow path since it's not
184 // guaranteed that we'll be able to do a DC GZVA.
185 Size .req %[Tmp]
186 sub Size, %[End], %[Cur]
187 cmp Size, %[LineSize], lsl #1
188 b.lt 3f
189 .unreq Size
190
191 LineMask .req %[Tmp]
192 sub LineMask, %[LineSize], #1
193
194 // STZG until the start of the next cache line.
195 orr %[Next], %[Cur], LineMask
196 1:
197 stzg %[Cur], [%[Cur]], #16
198 cmp %[Cur], %[Next]
199 b.lt 1b
200
201 // DC GZVA cache lines until we have no more full cache lines.
202 bic %[Next], %[End], LineMask
203 .unreq LineMask
204 2:
205 dc gzva, %[Cur]
206 add %[Cur], %[Cur], %[LineSize]
207 cmp %[Cur], %[Next]
208 b.lt 2b
209
210 // STZG until the end of the tagged region. This loop is also used to handle
211 // slow path cases.
212 3:
213 cmp %[Cur], %[End]
214 b.ge 4f
215 stzg %[Cur], [%[Cur]], #16
216 b 3b
217
218 4:
219 )"
220 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
221 [Tmp] "=&r"(Tmp)
222 : [End] "r"(End)
223 : "memory");
224 DCHECK_EQ(0, Begin % 16);
225 return Begin;
226 }
227
storeTag(uptr Ptr)228 inline void storeTag(uptr Ptr) {
229 DCHECK_EQ(0, Ptr % 16);
230 __asm__ __volatile__(R"(
231 .arch_extension memtag
232 stg %0, [%0]
233 )"
234 :
235 : "r"(Ptr)
236 : "memory");
237 }
238
loadTag(uptr Ptr)239 inline uptr loadTag(uptr Ptr) {
240 DCHECK_EQ(0, Ptr % 16);
241 uptr TaggedPtr = Ptr;
242 __asm__ __volatile__(
243 R"(
244 .arch_extension memtag
245 ldg %0, [%0]
246 )"
247 : "+r"(TaggedPtr)
248 :
249 : "memory");
250 return TaggedPtr;
251 }
252
253 #else
254
systemSupportsMemoryTagging()255 inline bool systemSupportsMemoryTagging() {
256 UNREACHABLE("memory tagging not supported");
257 }
258
systemDetectsMemoryTagFaultsTestOnly()259 inline bool systemDetectsMemoryTagFaultsTestOnly() {
260 UNREACHABLE("memory tagging not supported");
261 }
262
enableSystemMemoryTaggingTestOnly()263 inline void enableSystemMemoryTaggingTestOnly() {
264 UNREACHABLE("memory tagging not supported");
265 }
266
267 struct ScopedDisableMemoryTagChecks {
ScopedDisableMemoryTagChecksScopedDisableMemoryTagChecks268 ScopedDisableMemoryTagChecks() {}
269 };
270
selectRandomTag(uptr Ptr,uptr ExcludeMask)271 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
272 (void)Ptr;
273 (void)ExcludeMask;
274 UNREACHABLE("memory tagging not supported");
275 }
276
addFixedTag(uptr Ptr,uptr Tag)277 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
278 (void)Ptr;
279 (void)Tag;
280 UNREACHABLE("memory tagging not supported");
281 }
282
storeTags(uptr Begin,uptr End)283 inline uptr storeTags(uptr Begin, uptr End) {
284 (void)Begin;
285 (void)End;
286 UNREACHABLE("memory tagging not supported");
287 }
288
storeTag(uptr Ptr)289 inline void storeTag(uptr Ptr) {
290 (void)Ptr;
291 UNREACHABLE("memory tagging not supported");
292 }
293
loadTag(uptr Ptr)294 inline uptr loadTag(uptr Ptr) {
295 (void)Ptr;
296 UNREACHABLE("memory tagging not supported");
297 }
298
299 #endif
300
setRandomTag(void * Ptr,uptr Size,uptr ExcludeMask,uptr * TaggedBegin,uptr * TaggedEnd)301 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
302 uptr *TaggedBegin, uptr *TaggedEnd) {
303 *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
304 *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
305 }
306
untagPointer(void * Ptr)307 inline void *untagPointer(void *Ptr) {
308 return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
309 }
310
loadTag(void * Ptr)311 inline void *loadTag(void *Ptr) {
312 return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
313 }
314
addFixedTag(void * Ptr,uptr Tag)315 inline void *addFixedTag(void *Ptr, uptr Tag) {
316 return reinterpret_cast<void *>(
317 addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
318 }
319
320 template <typename Config>
allocatorSupportsMemoryTagging()321 inline constexpr bool allocatorSupportsMemoryTagging() {
322 return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging &&
323 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
324 }
325
326 } // namespace scudo
327
328 #endif
329