1 //===- bolt/runtime/common.h ------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #if !defined(__APPLE__) 10 11 #include <cstddef> 12 #include <cstdint> 13 14 #include "config.h" 15 16 #ifdef HAVE_ELF_H 17 #include <elf.h> 18 #endif 19 20 #else 21 22 typedef __SIZE_TYPE__ size_t; 23 #define __SSIZE_TYPE__ \ 24 __typeof__(_Generic((__SIZE_TYPE__)0, unsigned long long int \ 25 : (long long int)0, unsigned long int \ 26 : (long int)0, unsigned int \ 27 : (int)0, unsigned short \ 28 : (short)0, unsigned char \ 29 : (signed char)0)) 30 typedef __SSIZE_TYPE__ ssize_t; 31 32 typedef unsigned long long uint64_t; 33 typedef unsigned uint32_t; 34 typedef unsigned char uint8_t; 35 36 typedef long long int64_t; 37 typedef int int32_t; 38 39 #endif 40 41 // Save all registers while keeping 16B stack alignment 42 #define SAVE_ALL \ 43 "push %%rax\n" \ 44 "push %%rbx\n" \ 45 "push %%rcx\n" \ 46 "push %%rdx\n" \ 47 "push %%rdi\n" \ 48 "push %%rsi\n" \ 49 "push %%rbp\n" \ 50 "push %%r8\n" \ 51 "push %%r9\n" \ 52 "push %%r10\n" \ 53 "push %%r11\n" \ 54 "push %%r12\n" \ 55 "push %%r13\n" \ 56 "push %%r14\n" \ 57 "push %%r15\n" \ 58 "sub $8, %%rsp\n" 59 60 // Mirrors SAVE_ALL 61 #define RESTORE_ALL \ 62 "add $8, %%rsp\n" \ 63 "pop %%r15\n" \ 64 "pop %%r14\n" \ 65 "pop %%r13\n" \ 66 "pop %%r12\n" \ 67 "pop %%r11\n" \ 68 "pop %%r10\n" \ 69 "pop %%r9\n" \ 70 "pop %%r8\n" \ 71 "pop %%rbp\n" \ 72 "pop %%rsi\n" \ 73 "pop %%rdi\n" \ 74 "pop %%rdx\n" \ 75 "pop %%rcx\n" \ 76 "pop %%rbx\n" \ 77 "pop %%rax\n" 78 79 // Functions that are required by freestanding environment. Compiler may 80 // generate calls to these implicitly. 81 extern "C" { 82 void *memcpy(void *Dest, const void *Src, size_t Len) { 83 uint8_t *d = static_cast<uint8_t *>(Dest); 84 const uint8_t *s = static_cast<const uint8_t *>(Src); 85 while (Len--) 86 *d++ = *s++; 87 return Dest; 88 } 89 90 void *memmove(void *Dest, const void *Src, size_t Len) { 91 uint8_t *d = static_cast<uint8_t *>(Dest); 92 const uint8_t *s = static_cast<const uint8_t *>(Src); 93 if (d < s) { 94 while (Len--) 95 *d++ = *s++; 96 } else { 97 s += Len - 1; 98 d += Len - 1; 99 while (Len--) 100 *d-- = *s--; 101 } 102 103 return Dest; 104 } 105 106 void *memset(void *Buf, int C, size_t Size) { 107 char *S = (char *)Buf; 108 for (size_t I = 0; I < Size; ++I) 109 *S++ = C; 110 return Buf; 111 } 112 113 int memcmp(const void *s1, const void *s2, size_t n) { 114 const uint8_t *c1 = static_cast<const uint8_t *>(s1); 115 const uint8_t *c2 = static_cast<const uint8_t *>(s2); 116 for (; n--; c1++, c2++) { 117 if (*c1 != *c2) 118 return *c1 < *c2 ? -1 : 1; 119 } 120 return 0; 121 } 122 } // extern "C" 123 124 // Anonymous namespace covering everything but our library entry point 125 namespace { 126 127 constexpr uint32_t BufSize = 10240; 128 129 #define _STRINGIFY(x) #x 130 #define STRINGIFY(x) _STRINGIFY(x) 131 132 uint64_t __read(uint64_t fd, const void *buf, uint64_t count) { 133 uint64_t ret; 134 #if defined(__APPLE__) 135 #define READ_SYSCALL 0x2000003 136 #else 137 #define READ_SYSCALL 0 138 #endif 139 __asm__ __volatile__("movq $" STRINGIFY(READ_SYSCALL) ", %%rax\n" 140 "syscall\n" 141 : "=a"(ret) 142 : "D"(fd), "S"(buf), "d"(count) 143 : "cc", "rcx", "r11", "memory"); 144 return ret; 145 } 146 147 uint64_t __write(uint64_t fd, const void *buf, uint64_t count) { 148 uint64_t ret; 149 #if defined(__APPLE__) 150 #define WRITE_SYSCALL 0x2000004 151 #else 152 #define WRITE_SYSCALL 1 153 #endif 154 __asm__ __volatile__("movq $" STRINGIFY(WRITE_SYSCALL) ", %%rax\n" 155 "syscall\n" 156 : "=a"(ret) 157 : "D"(fd), "S"(buf), "d"(count) 158 : "cc", "rcx", "r11", "memory"); 159 return ret; 160 } 161 162 void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags, 163 uint64_t fd, uint64_t offset) { 164 #if defined(__APPLE__) 165 #define MMAP_SYSCALL 0x20000c5 166 #else 167 #define MMAP_SYSCALL 9 168 #endif 169 void *ret; 170 register uint64_t r8 asm("r8") = fd; 171 register uint64_t r9 asm("r9") = offset; 172 register uint64_t r10 asm("r10") = flags; 173 __asm__ __volatile__("movq $" STRINGIFY(MMAP_SYSCALL) ", %%rax\n" 174 "syscall\n" 175 : "=a"(ret) 176 : "D"(addr), "S"(size), "d"(prot), "r"(r10), "r"(r8), 177 "r"(r9) 178 : "cc", "rcx", "r11", "memory"); 179 return ret; 180 } 181 182 uint64_t __munmap(void *addr, uint64_t size) { 183 #if defined(__APPLE__) 184 #define MUNMAP_SYSCALL 0x2000049 185 #else 186 #define MUNMAP_SYSCALL 11 187 #endif 188 uint64_t ret; 189 __asm__ __volatile__("movq $" STRINGIFY(MUNMAP_SYSCALL) ", %%rax\n" 190 "syscall\n" 191 : "=a"(ret) 192 : "D"(addr), "S"(size) 193 : "cc", "rcx", "r11", "memory"); 194 return ret; 195 } 196 197 #define SIG_BLOCK 0 198 #define SIG_UNBLOCK 1 199 #define SIG_SETMASK 2 200 201 static const uint64_t MaskAllSignals[] = {-1ULL}; 202 203 uint64_t __sigprocmask(int how, const void *set, void *oldset) { 204 #if defined(__APPLE__) 205 #define SIGPROCMASK_SYSCALL 0x2000030 206 #else 207 #define SIGPROCMASK_SYSCALL 14 208 #endif 209 uint64_t ret; 210 register long r10 asm("r10") = sizeof(uint64_t); 211 __asm__ __volatile__("movq $" STRINGIFY(SIGPROCMASK_SYSCALL) ", %%rax\n" 212 "syscall\n" 213 : "=a"(ret) 214 : "D"(how), "S"(set), "d"(oldset), "r"(r10) 215 : "cc", "rcx", "r11", "memory"); 216 return ret; 217 } 218 219 uint64_t __exit(uint64_t code) { 220 #if defined(__APPLE__) 221 #define EXIT_SYSCALL 0x2000001 222 #else 223 #define EXIT_SYSCALL 231 224 #endif 225 uint64_t ret; 226 __asm__ __volatile__("movq $" STRINGIFY(EXIT_SYSCALL) ", %%rax\n" 227 "syscall\n" 228 : "=a"(ret) 229 : "D"(code) 230 : "cc", "rcx", "r11", "memory"); 231 return ret; 232 } 233 234 // Helper functions for writing strings to the .fdata file. We intentionally 235 // avoid using libc names to make it clear it is our impl. 236 237 /// Write number Num using Base to the buffer in OutBuf, returns a pointer to 238 /// the end of the string. 239 char *intToStr(char *OutBuf, uint64_t Num, uint32_t Base) { 240 const char *Chars = "0123456789abcdef"; 241 char Buf[21]; 242 char *Ptr = Buf; 243 while (Num) { 244 *Ptr++ = *(Chars + (Num % Base)); 245 Num /= Base; 246 } 247 if (Ptr == Buf) { 248 *OutBuf++ = '0'; 249 return OutBuf; 250 } 251 while (Ptr != Buf) 252 *OutBuf++ = *--Ptr; 253 254 return OutBuf; 255 } 256 257 /// Copy Str to OutBuf, returns a pointer to the end of the copied string 258 char *strCopy(char *OutBuf, const char *Str, int32_t Size = BufSize) { 259 while (*Str) { 260 *OutBuf++ = *Str++; 261 if (--Size <= 0) 262 return OutBuf; 263 } 264 return OutBuf; 265 } 266 267 /// Compare two strings, at most Num bytes. 268 int strnCmp(const char *Str1, const char *Str2, size_t Num) { 269 while (Num && *Str1 && (*Str1 == *Str2)) { 270 Num--; 271 Str1++; 272 Str2++; 273 } 274 if (Num == 0) 275 return 0; 276 return *(unsigned char *)Str1 - *(unsigned char *)Str2; 277 } 278 279 uint32_t strLen(const char *Str) { 280 uint32_t Size = 0; 281 while (*Str++) 282 ++Size; 283 return Size; 284 } 285 286 void reportNumber(const char *Msg, uint64_t Num, uint32_t Base) { 287 char Buf[BufSize]; 288 char *Ptr = Buf; 289 Ptr = strCopy(Ptr, Msg, BufSize - 23); 290 Ptr = intToStr(Ptr, Num, Base); 291 Ptr = strCopy(Ptr, "\n"); 292 __write(2, Buf, Ptr - Buf); 293 } 294 295 void report(const char *Msg) { __write(2, Msg, strLen(Msg)); } 296 297 unsigned long hexToLong(const char *Str, char Terminator = '\0') { 298 unsigned long Res = 0; 299 while (*Str != Terminator) { 300 Res <<= 4; 301 if ('0' <= *Str && *Str <= '9') 302 Res += *Str++ - '0'; 303 else if ('a' <= *Str && *Str <= 'f') 304 Res += *Str++ - 'a' + 10; 305 else if ('A' <= *Str && *Str <= 'F') 306 Res += *Str++ - 'A' + 10; 307 else 308 return 0; 309 } 310 return Res; 311 } 312 313 #if !defined(__APPLE__) 314 // We use a stack-allocated buffer for string manipulation in many pieces of 315 // this code, including the code that prints each line of the fdata file. This 316 // buffer needs to accomodate large function names, but shouldn't be arbitrarily 317 // large (dynamically allocated) for simplicity of our memory space usage. 318 319 // Declare some syscall wrappers we use throughout this code to avoid linking 320 // against system libc. 321 uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) { 322 uint64_t ret; 323 __asm__ __volatile__("movq $2, %%rax\n" 324 "syscall" 325 : "=a"(ret) 326 : "D"(pathname), "S"(flags), "d"(mode) 327 : "cc", "rcx", "r11", "memory"); 328 return ret; 329 } 330 331 struct dirent { 332 unsigned long d_ino; /* Inode number */ 333 unsigned long d_off; /* Offset to next linux_dirent */ 334 unsigned short d_reclen; /* Length of this linux_dirent */ 335 char d_name[]; /* Filename (null-terminated) */ 336 /* length is actually (d_reclen - 2 - 337 offsetof(struct linux_dirent, d_name)) */ 338 }; 339 340 long __getdents(unsigned int fd, dirent *dirp, size_t count) { 341 long ret; 342 __asm__ __volatile__("movq $78, %%rax\n" 343 "syscall" 344 : "=a"(ret) 345 : "D"(fd), "S"(dirp), "d"(count) 346 : "cc", "rcx", "r11", "memory"); 347 return ret; 348 } 349 350 uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) { 351 uint64_t ret; 352 __asm__ __volatile__("movq $89, %%rax\n" 353 "syscall" 354 : "=a"(ret) 355 : "D"(pathname), "S"(buf), "d"(bufsize) 356 : "cc", "rcx", "r11", "memory"); 357 return ret; 358 } 359 360 uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) { 361 uint64_t ret; 362 __asm__ __volatile__("movq $8, %%rax\n" 363 "syscall\n" 364 : "=a"(ret) 365 : "D"(fd), "S"(pos), "d"(whence) 366 : "cc", "rcx", "r11", "memory"); 367 return ret; 368 } 369 370 int __close(uint64_t fd) { 371 uint64_t ret; 372 __asm__ __volatile__("movq $3, %%rax\n" 373 "syscall\n" 374 : "=a"(ret) 375 : "D"(fd) 376 : "cc", "rcx", "r11", "memory"); 377 return ret; 378 } 379 380 int __madvise(void *addr, size_t length, int advice) { 381 int ret; 382 __asm__ __volatile__("movq $28, %%rax\n" 383 "syscall\n" 384 : "=a"(ret) 385 : "D"(addr), "S"(length), "d"(advice) 386 : "cc", "rcx", "r11", "memory"); 387 return ret; 388 } 389 390 struct timespec { 391 uint64_t tv_sec; /* seconds */ 392 uint64_t tv_nsec; /* nanoseconds */ 393 }; 394 395 uint64_t __nanosleep(const timespec *req, timespec *rem) { 396 uint64_t ret; 397 __asm__ __volatile__("movq $35, %%rax\n" 398 "syscall\n" 399 : "=a"(ret) 400 : "D"(req), "S"(rem) 401 : "cc", "rcx", "r11", "memory"); 402 return ret; 403 } 404 405 int64_t __fork() { 406 uint64_t ret; 407 __asm__ __volatile__("movq $57, %%rax\n" 408 "syscall\n" 409 : "=a"(ret) 410 : 411 : "cc", "rcx", "r11", "memory"); 412 return ret; 413 } 414 415 int __mprotect(void *addr, size_t len, int prot) { 416 int ret; 417 __asm__ __volatile__("movq $10, %%rax\n" 418 "syscall\n" 419 : "=a"(ret) 420 : "D"(addr), "S"(len), "d"(prot) 421 : "cc", "rcx", "r11", "memory"); 422 return ret; 423 } 424 425 uint64_t __getpid() { 426 uint64_t ret; 427 __asm__ __volatile__("movq $39, %%rax\n" 428 "syscall\n" 429 : "=a"(ret) 430 : 431 : "cc", "rcx", "r11", "memory"); 432 return ret; 433 } 434 435 uint64_t __getppid() { 436 uint64_t ret; 437 __asm__ __volatile__("movq $110, %%rax\n" 438 "syscall\n" 439 : "=a"(ret) 440 : 441 : "cc", "rcx", "r11", "memory"); 442 return ret; 443 } 444 445 int __setpgid(uint64_t pid, uint64_t pgid) { 446 int ret; 447 __asm__ __volatile__("movq $109, %%rax\n" 448 "syscall\n" 449 : "=a"(ret) 450 : "D"(pid), "S"(pgid) 451 : "cc", "rcx", "r11", "memory"); 452 return ret; 453 } 454 455 uint64_t __getpgid(uint64_t pid) { 456 uint64_t ret; 457 __asm__ __volatile__("movq $121, %%rax\n" 458 "syscall\n" 459 : "=a"(ret) 460 : "D"(pid) 461 : "cc", "rcx", "r11", "memory"); 462 return ret; 463 } 464 465 int __kill(uint64_t pid, int sig) { 466 int ret; 467 __asm__ __volatile__("movq $62, %%rax\n" 468 "syscall\n" 469 : "=a"(ret) 470 : "D"(pid), "S"(sig) 471 : "cc", "rcx", "r11", "memory"); 472 return ret; 473 } 474 475 int __fsync(int fd) { 476 int ret; 477 __asm__ __volatile__("movq $74, %%rax\n" 478 "syscall\n" 479 : "=a"(ret) 480 : "D"(fd) 481 : "cc", "rcx", "r11", "memory"); 482 return ret; 483 } 484 485 #endif 486 487 void reportError(const char *Msg, uint64_t Size) { 488 __write(2, Msg, Size); 489 __exit(1); 490 } 491 492 void assert(bool Assertion, const char *Msg) { 493 if (Assertion) 494 return; 495 char Buf[BufSize]; 496 char *Ptr = Buf; 497 Ptr = strCopy(Ptr, "Assertion failed: "); 498 Ptr = strCopy(Ptr, Msg, BufSize - 40); 499 Ptr = strCopy(Ptr, "\n"); 500 reportError(Buf, Ptr - Buf); 501 } 502 503 class Mutex { 504 volatile bool InUse{false}; 505 506 public: 507 bool acquire() { return !__atomic_test_and_set(&InUse, __ATOMIC_ACQUIRE); } 508 void release() { __atomic_clear(&InUse, __ATOMIC_RELEASE); } 509 }; 510 511 /// RAII wrapper for Mutex 512 class Lock { 513 Mutex &M; 514 uint64_t SignalMask[1] = {}; 515 516 public: 517 Lock(Mutex &M) : M(M) { 518 __sigprocmask(SIG_BLOCK, MaskAllSignals, SignalMask); 519 while (!M.acquire()) { 520 } 521 } 522 523 ~Lock() { 524 M.release(); 525 __sigprocmask(SIG_SETMASK, SignalMask, nullptr); 526 } 527 }; 528 529 /// RAII wrapper for Mutex 530 class TryLock { 531 Mutex &M; 532 bool Locked = false; 533 534 public: 535 TryLock(Mutex &M) : M(M) { 536 int Retry = 100; 537 while (--Retry && !M.acquire()) 538 ; 539 if (Retry) 540 Locked = true; 541 } 542 bool isLocked() { return Locked; } 543 544 ~TryLock() { 545 if (isLocked()) 546 M.release(); 547 } 548 }; 549 550 inline uint64_t alignTo(uint64_t Value, uint64_t Align) { 551 return (Value + Align - 1) / Align * Align; 552 } 553 554 } // anonymous namespace 555