1 //===- bolt/runtime/common.h ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #if !defined(__APPLE__)
10
11 #include <cstddef>
12 #include <cstdint>
13
14 #include "config.h"
15
16 #ifdef HAVE_ELF_H
17 #include <elf.h>
18 #endif
19
20 #else
21
22 typedef __SIZE_TYPE__ size_t;
23 #define __SSIZE_TYPE__ \
24 __typeof__(_Generic((__SIZE_TYPE__)0, unsigned long long int \
25 : (long long int)0, unsigned long int \
26 : (long int)0, unsigned int \
27 : (int)0, unsigned short \
28 : (short)0, unsigned char \
29 : (signed char)0))
30 typedef __SSIZE_TYPE__ ssize_t;
31
32 typedef unsigned long long uint64_t;
33 typedef unsigned uint32_t;
34 typedef unsigned char uint8_t;
35
36 typedef long long int64_t;
37 typedef int int32_t;
38
39 #endif
40
41 // Save all registers while keeping 16B stack alignment
42 #define SAVE_ALL \
43 "push %%rax\n" \
44 "push %%rbx\n" \
45 "push %%rcx\n" \
46 "push %%rdx\n" \
47 "push %%rdi\n" \
48 "push %%rsi\n" \
49 "push %%rbp\n" \
50 "push %%r8\n" \
51 "push %%r9\n" \
52 "push %%r10\n" \
53 "push %%r11\n" \
54 "push %%r12\n" \
55 "push %%r13\n" \
56 "push %%r14\n" \
57 "push %%r15\n" \
58 "sub $8, %%rsp\n"
59
60 // Mirrors SAVE_ALL
61 #define RESTORE_ALL \
62 "add $8, %%rsp\n" \
63 "pop %%r15\n" \
64 "pop %%r14\n" \
65 "pop %%r13\n" \
66 "pop %%r12\n" \
67 "pop %%r11\n" \
68 "pop %%r10\n" \
69 "pop %%r9\n" \
70 "pop %%r8\n" \
71 "pop %%rbp\n" \
72 "pop %%rsi\n" \
73 "pop %%rdi\n" \
74 "pop %%rdx\n" \
75 "pop %%rcx\n" \
76 "pop %%rbx\n" \
77 "pop %%rax\n"
78
79 // Functions that are required by freestanding environment. Compiler may
80 // generate calls to these implicitly.
81 extern "C" {
memcpy(void * Dest,const void * Src,size_t Len)82 void *memcpy(void *Dest, const void *Src, size_t Len) {
83 uint8_t *d = static_cast<uint8_t *>(Dest);
84 const uint8_t *s = static_cast<const uint8_t *>(Src);
85 while (Len--)
86 *d++ = *s++;
87 return Dest;
88 }
89
memmove(void * Dest,const void * Src,size_t Len)90 void *memmove(void *Dest, const void *Src, size_t Len) {
91 uint8_t *d = static_cast<uint8_t *>(Dest);
92 const uint8_t *s = static_cast<const uint8_t *>(Src);
93 if (d < s) {
94 while (Len--)
95 *d++ = *s++;
96 } else {
97 s += Len - 1;
98 d += Len - 1;
99 while (Len--)
100 *d-- = *s--;
101 }
102
103 return Dest;
104 }
105
memset(void * Buf,int C,size_t Size)106 void *memset(void *Buf, int C, size_t Size) {
107 char *S = (char *)Buf;
108 for (size_t I = 0; I < Size; ++I)
109 *S++ = C;
110 return Buf;
111 }
112
memcmp(const void * s1,const void * s2,size_t n)113 int memcmp(const void *s1, const void *s2, size_t n) {
114 const uint8_t *c1 = static_cast<const uint8_t *>(s1);
115 const uint8_t *c2 = static_cast<const uint8_t *>(s2);
116 for (; n--; c1++, c2++) {
117 if (*c1 != *c2)
118 return *c1 < *c2 ? -1 : 1;
119 }
120 return 0;
121 }
122 } // extern "C"
123
124 // Anonymous namespace covering everything but our library entry point
125 namespace {
126
127 constexpr uint32_t BufSize = 10240;
128
129 #define _STRINGIFY(x) #x
130 #define STRINGIFY(x) _STRINGIFY(x)
131
__read(uint64_t fd,const void * buf,uint64_t count)132 uint64_t __read(uint64_t fd, const void *buf, uint64_t count) {
133 uint64_t ret;
134 #if defined(__APPLE__)
135 #define READ_SYSCALL 0x2000003
136 #else
137 #define READ_SYSCALL 0
138 #endif
139 __asm__ __volatile__("movq $" STRINGIFY(READ_SYSCALL) ", %%rax\n"
140 "syscall\n"
141 : "=a"(ret)
142 : "D"(fd), "S"(buf), "d"(count)
143 : "cc", "rcx", "r11", "memory");
144 return ret;
145 }
146
__write(uint64_t fd,const void * buf,uint64_t count)147 uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
148 uint64_t ret;
149 #if defined(__APPLE__)
150 #define WRITE_SYSCALL 0x2000004
151 #else
152 #define WRITE_SYSCALL 1
153 #endif
154 __asm__ __volatile__("movq $" STRINGIFY(WRITE_SYSCALL) ", %%rax\n"
155 "syscall\n"
156 : "=a"(ret)
157 : "D"(fd), "S"(buf), "d"(count)
158 : "cc", "rcx", "r11", "memory");
159 return ret;
160 }
161
__mmap(uint64_t addr,uint64_t size,uint64_t prot,uint64_t flags,uint64_t fd,uint64_t offset)162 void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags,
163 uint64_t fd, uint64_t offset) {
164 #if defined(__APPLE__)
165 #define MMAP_SYSCALL 0x20000c5
166 #else
167 #define MMAP_SYSCALL 9
168 #endif
169 void *ret;
170 register uint64_t r8 asm("r8") = fd;
171 register uint64_t r9 asm("r9") = offset;
172 register uint64_t r10 asm("r10") = flags;
173 __asm__ __volatile__("movq $" STRINGIFY(MMAP_SYSCALL) ", %%rax\n"
174 "syscall\n"
175 : "=a"(ret)
176 : "D"(addr), "S"(size), "d"(prot), "r"(r10), "r"(r8),
177 "r"(r9)
178 : "cc", "rcx", "r11", "memory");
179 return ret;
180 }
181
__munmap(void * addr,uint64_t size)182 uint64_t __munmap(void *addr, uint64_t size) {
183 #if defined(__APPLE__)
184 #define MUNMAP_SYSCALL 0x2000049
185 #else
186 #define MUNMAP_SYSCALL 11
187 #endif
188 uint64_t ret;
189 __asm__ __volatile__("movq $" STRINGIFY(MUNMAP_SYSCALL) ", %%rax\n"
190 "syscall\n"
191 : "=a"(ret)
192 : "D"(addr), "S"(size)
193 : "cc", "rcx", "r11", "memory");
194 return ret;
195 }
196
197 #define SIG_BLOCK 0
198 #define SIG_UNBLOCK 1
199 #define SIG_SETMASK 2
200
201 static const uint64_t MaskAllSignals[] = {-1ULL};
202
__sigprocmask(int how,const void * set,void * oldset)203 uint64_t __sigprocmask(int how, const void *set, void *oldset) {
204 #if defined(__APPLE__)
205 #define SIGPROCMASK_SYSCALL 0x2000030
206 #else
207 #define SIGPROCMASK_SYSCALL 14
208 #endif
209 uint64_t ret;
210 register long r10 asm("r10") = sizeof(uint64_t);
211 __asm__ __volatile__("movq $" STRINGIFY(SIGPROCMASK_SYSCALL) ", %%rax\n"
212 "syscall\n"
213 : "=a"(ret)
214 : "D"(how), "S"(set), "d"(oldset), "r"(r10)
215 : "cc", "rcx", "r11", "memory");
216 return ret;
217 }
218
__exit(uint64_t code)219 uint64_t __exit(uint64_t code) {
220 #if defined(__APPLE__)
221 #define EXIT_SYSCALL 0x2000001
222 #else
223 #define EXIT_SYSCALL 231
224 #endif
225 uint64_t ret;
226 __asm__ __volatile__("movq $" STRINGIFY(EXIT_SYSCALL) ", %%rax\n"
227 "syscall\n"
228 : "=a"(ret)
229 : "D"(code)
230 : "cc", "rcx", "r11", "memory");
231 return ret;
232 }
233
234 // Helper functions for writing strings to the .fdata file. We intentionally
235 // avoid using libc names to make it clear it is our impl.
236
237 /// Write number Num using Base to the buffer in OutBuf, returns a pointer to
238 /// the end of the string.
intToStr(char * OutBuf,uint64_t Num,uint32_t Base)239 char *intToStr(char *OutBuf, uint64_t Num, uint32_t Base) {
240 const char *Chars = "0123456789abcdef";
241 char Buf[21];
242 char *Ptr = Buf;
243 while (Num) {
244 *Ptr++ = *(Chars + (Num % Base));
245 Num /= Base;
246 }
247 if (Ptr == Buf) {
248 *OutBuf++ = '0';
249 return OutBuf;
250 }
251 while (Ptr != Buf)
252 *OutBuf++ = *--Ptr;
253
254 return OutBuf;
255 }
256
257 /// Copy Str to OutBuf, returns a pointer to the end of the copied string
258 char *strCopy(char *OutBuf, const char *Str, int32_t Size = BufSize) {
259 while (*Str) {
260 *OutBuf++ = *Str++;
261 if (--Size <= 0)
262 return OutBuf;
263 }
264 return OutBuf;
265 }
266
267 /// Compare two strings, at most Num bytes.
strnCmp(const char * Str1,const char * Str2,size_t Num)268 int strnCmp(const char *Str1, const char *Str2, size_t Num) {
269 while (Num && *Str1 && (*Str1 == *Str2)) {
270 Num--;
271 Str1++;
272 Str2++;
273 }
274 if (Num == 0)
275 return 0;
276 return *(unsigned char *)Str1 - *(unsigned char *)Str2;
277 }
278
strLen(const char * Str)279 uint32_t strLen(const char *Str) {
280 uint32_t Size = 0;
281 while (*Str++)
282 ++Size;
283 return Size;
284 }
285
reportNumber(const char * Msg,uint64_t Num,uint32_t Base)286 void reportNumber(const char *Msg, uint64_t Num, uint32_t Base) {
287 char Buf[BufSize];
288 char *Ptr = Buf;
289 Ptr = strCopy(Ptr, Msg, BufSize - 23);
290 Ptr = intToStr(Ptr, Num, Base);
291 Ptr = strCopy(Ptr, "\n");
292 __write(2, Buf, Ptr - Buf);
293 }
294
report(const char * Msg)295 void report(const char *Msg) { __write(2, Msg, strLen(Msg)); }
296
297 unsigned long hexToLong(const char *Str, char Terminator = '\0') {
298 unsigned long Res = 0;
299 while (*Str != Terminator) {
300 Res <<= 4;
301 if ('0' <= *Str && *Str <= '9')
302 Res += *Str++ - '0';
303 else if ('a' <= *Str && *Str <= 'f')
304 Res += *Str++ - 'a' + 10;
305 else if ('A' <= *Str && *Str <= 'F')
306 Res += *Str++ - 'A' + 10;
307 else
308 return 0;
309 }
310 return Res;
311 }
312
313 #if !defined(__APPLE__)
314 // We use a stack-allocated buffer for string manipulation in many pieces of
315 // this code, including the code that prints each line of the fdata file. This
316 // buffer needs to accomodate large function names, but shouldn't be arbitrarily
317 // large (dynamically allocated) for simplicity of our memory space usage.
318
319 // Declare some syscall wrappers we use throughout this code to avoid linking
320 // against system libc.
__open(const char * pathname,uint64_t flags,uint64_t mode)321 uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) {
322 uint64_t ret;
323 __asm__ __volatile__("movq $2, %%rax\n"
324 "syscall"
325 : "=a"(ret)
326 : "D"(pathname), "S"(flags), "d"(mode)
327 : "cc", "rcx", "r11", "memory");
328 return ret;
329 }
330
331 struct dirent {
332 unsigned long d_ino; /* Inode number */
333 unsigned long d_off; /* Offset to next linux_dirent */
334 unsigned short d_reclen; /* Length of this linux_dirent */
335 char d_name[]; /* Filename (null-terminated) */
336 /* length is actually (d_reclen - 2 -
337 offsetof(struct linux_dirent, d_name)) */
338 };
339
__getdents(unsigned int fd,dirent * dirp,size_t count)340 long __getdents(unsigned int fd, dirent *dirp, size_t count) {
341 long ret;
342 __asm__ __volatile__("movq $78, %%rax\n"
343 "syscall"
344 : "=a"(ret)
345 : "D"(fd), "S"(dirp), "d"(count)
346 : "cc", "rcx", "r11", "memory");
347 return ret;
348 }
349
__readlink(const char * pathname,char * buf,size_t bufsize)350 uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) {
351 uint64_t ret;
352 __asm__ __volatile__("movq $89, %%rax\n"
353 "syscall"
354 : "=a"(ret)
355 : "D"(pathname), "S"(buf), "d"(bufsize)
356 : "cc", "rcx", "r11", "memory");
357 return ret;
358 }
359
__lseek(uint64_t fd,uint64_t pos,uint64_t whence)360 uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
361 uint64_t ret;
362 __asm__ __volatile__("movq $8, %%rax\n"
363 "syscall\n"
364 : "=a"(ret)
365 : "D"(fd), "S"(pos), "d"(whence)
366 : "cc", "rcx", "r11", "memory");
367 return ret;
368 }
369
__close(uint64_t fd)370 int __close(uint64_t fd) {
371 uint64_t ret;
372 __asm__ __volatile__("movq $3, %%rax\n"
373 "syscall\n"
374 : "=a"(ret)
375 : "D"(fd)
376 : "cc", "rcx", "r11", "memory");
377 return ret;
378 }
379
__madvise(void * addr,size_t length,int advice)380 int __madvise(void *addr, size_t length, int advice) {
381 int ret;
382 __asm__ __volatile__("movq $28, %%rax\n"
383 "syscall\n"
384 : "=a"(ret)
385 : "D"(addr), "S"(length), "d"(advice)
386 : "cc", "rcx", "r11", "memory");
387 return ret;
388 }
389
390 struct timespec {
391 uint64_t tv_sec; /* seconds */
392 uint64_t tv_nsec; /* nanoseconds */
393 };
394
__nanosleep(const timespec * req,timespec * rem)395 uint64_t __nanosleep(const timespec *req, timespec *rem) {
396 uint64_t ret;
397 __asm__ __volatile__("movq $35, %%rax\n"
398 "syscall\n"
399 : "=a"(ret)
400 : "D"(req), "S"(rem)
401 : "cc", "rcx", "r11", "memory");
402 return ret;
403 }
404
__fork()405 int64_t __fork() {
406 uint64_t ret;
407 __asm__ __volatile__("movq $57, %%rax\n"
408 "syscall\n"
409 : "=a"(ret)
410 :
411 : "cc", "rcx", "r11", "memory");
412 return ret;
413 }
414
__mprotect(void * addr,size_t len,int prot)415 int __mprotect(void *addr, size_t len, int prot) {
416 int ret;
417 __asm__ __volatile__("movq $10, %%rax\n"
418 "syscall\n"
419 : "=a"(ret)
420 : "D"(addr), "S"(len), "d"(prot)
421 : "cc", "rcx", "r11", "memory");
422 return ret;
423 }
424
__getpid()425 uint64_t __getpid() {
426 uint64_t ret;
427 __asm__ __volatile__("movq $39, %%rax\n"
428 "syscall\n"
429 : "=a"(ret)
430 :
431 : "cc", "rcx", "r11", "memory");
432 return ret;
433 }
434
__getppid()435 uint64_t __getppid() {
436 uint64_t ret;
437 __asm__ __volatile__("movq $110, %%rax\n"
438 "syscall\n"
439 : "=a"(ret)
440 :
441 : "cc", "rcx", "r11", "memory");
442 return ret;
443 }
444
__setpgid(uint64_t pid,uint64_t pgid)445 int __setpgid(uint64_t pid, uint64_t pgid) {
446 int ret;
447 __asm__ __volatile__("movq $109, %%rax\n"
448 "syscall\n"
449 : "=a"(ret)
450 : "D"(pid), "S"(pgid)
451 : "cc", "rcx", "r11", "memory");
452 return ret;
453 }
454
__getpgid(uint64_t pid)455 uint64_t __getpgid(uint64_t pid) {
456 uint64_t ret;
457 __asm__ __volatile__("movq $121, %%rax\n"
458 "syscall\n"
459 : "=a"(ret)
460 : "D"(pid)
461 : "cc", "rcx", "r11", "memory");
462 return ret;
463 }
464
__kill(uint64_t pid,int sig)465 int __kill(uint64_t pid, int sig) {
466 int ret;
467 __asm__ __volatile__("movq $62, %%rax\n"
468 "syscall\n"
469 : "=a"(ret)
470 : "D"(pid), "S"(sig)
471 : "cc", "rcx", "r11", "memory");
472 return ret;
473 }
474
__fsync(int fd)475 int __fsync(int fd) {
476 int ret;
477 __asm__ __volatile__("movq $74, %%rax\n"
478 "syscall\n"
479 : "=a"(ret)
480 : "D"(fd)
481 : "cc", "rcx", "r11", "memory");
482 return ret;
483 }
484
485 #endif
486
reportError(const char * Msg,uint64_t Size)487 void reportError(const char *Msg, uint64_t Size) {
488 __write(2, Msg, Size);
489 __exit(1);
490 }
491
assert(bool Assertion,const char * Msg)492 void assert(bool Assertion, const char *Msg) {
493 if (Assertion)
494 return;
495 char Buf[BufSize];
496 char *Ptr = Buf;
497 Ptr = strCopy(Ptr, "Assertion failed: ");
498 Ptr = strCopy(Ptr, Msg, BufSize - 40);
499 Ptr = strCopy(Ptr, "\n");
500 reportError(Buf, Ptr - Buf);
501 }
502
503 class Mutex {
504 volatile bool InUse{false};
505
506 public:
acquire()507 bool acquire() { return !__atomic_test_and_set(&InUse, __ATOMIC_ACQUIRE); }
release()508 void release() { __atomic_clear(&InUse, __ATOMIC_RELEASE); }
509 };
510
511 /// RAII wrapper for Mutex
512 class Lock {
513 Mutex &M;
514 uint64_t SignalMask[1] = {};
515
516 public:
Lock(Mutex & M)517 Lock(Mutex &M) : M(M) {
518 __sigprocmask(SIG_BLOCK, MaskAllSignals, SignalMask);
519 while (!M.acquire()) {
520 }
521 }
522
~Lock()523 ~Lock() {
524 M.release();
525 __sigprocmask(SIG_SETMASK, SignalMask, nullptr);
526 }
527 };
528
529 /// RAII wrapper for Mutex
530 class TryLock {
531 Mutex &M;
532 bool Locked = false;
533
534 public:
TryLock(Mutex & M)535 TryLock(Mutex &M) : M(M) {
536 int Retry = 100;
537 while (--Retry && !M.acquire())
538 ;
539 if (Retry)
540 Locked = true;
541 }
isLocked()542 bool isLocked() { return Locked; }
543
~TryLock()544 ~TryLock() {
545 if (isLocked())
546 M.release();
547 }
548 };
549
alignTo(uint64_t Value,uint64_t Align)550 inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
551 return (Value + Align - 1) / Align * Align;
552 }
553
554 } // anonymous namespace
555