1 //===- bolt/runtime/common.h ------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #if !defined(__APPLE__) 10 11 #include <cstddef> 12 #include <cstdint> 13 14 #include "config.h" 15 16 #ifdef HAVE_ELF_H 17 #include <elf.h> 18 #endif 19 20 #else 21 22 typedef __SIZE_TYPE__ size_t; 23 #define __SSIZE_TYPE__ \ 24 __typeof__(_Generic((__SIZE_TYPE__)0, unsigned long long int \ 25 : (long long int)0, unsigned long int \ 26 : (long int)0, unsigned int \ 27 : (int)0, unsigned short \ 28 : (short)0, unsigned char \ 29 : (signed char)0)) 30 typedef __SSIZE_TYPE__ ssize_t; 31 32 typedef unsigned long long uint64_t; 33 typedef unsigned uint32_t; 34 typedef unsigned char uint8_t; 35 36 typedef long long int64_t; 37 typedef int int32_t; 38 39 #endif 40 41 // Save all registers while keeping 16B stack alignment 42 #define SAVE_ALL \ 43 "push %%rax\n" \ 44 "push %%rbx\n" \ 45 "push %%rcx\n" \ 46 "push %%rdx\n" \ 47 "push %%rdi\n" \ 48 "push %%rsi\n" \ 49 "push %%rbp\n" \ 50 "push %%r8\n" \ 51 "push %%r9\n" \ 52 "push %%r10\n" \ 53 "push %%r11\n" \ 54 "push %%r12\n" \ 55 "push %%r13\n" \ 56 "push %%r14\n" \ 57 "push %%r15\n" \ 58 "sub $8, %%rsp\n" 59 60 // Mirrors SAVE_ALL 61 #define RESTORE_ALL \ 62 "add $8, %%rsp\n" \ 63 "pop %%r15\n" \ 64 "pop %%r14\n" \ 65 "pop %%r13\n" \ 66 "pop %%r12\n" \ 67 "pop %%r11\n" \ 68 "pop %%r10\n" \ 69 "pop %%r9\n" \ 70 "pop %%r8\n" \ 71 "pop %%rbp\n" \ 72 "pop %%rsi\n" \ 73 "pop %%rdi\n" \ 74 "pop %%rdx\n" \ 75 "pop %%rcx\n" \ 76 "pop %%rbx\n" \ 77 "pop %%rax\n" 78 79 // Anonymous namespace covering everything but our library entry point 80 namespace { 81 82 constexpr uint32_t BufSize = 10240; 83 84 #define _STRINGIFY(x) #x 85 #define STRINGIFY(x) _STRINGIFY(x) 86 87 uint64_t __read(uint64_t fd, const void *buf, uint64_t count) { 88 uint64_t ret; 89 #if defined(__APPLE__) 90 #define READ_SYSCALL 0x2000003 91 #else 92 #define READ_SYSCALL 0 93 #endif 94 __asm__ __volatile__("movq $" STRINGIFY(READ_SYSCALL) ", %%rax\n" 95 "syscall\n" 96 : "=a"(ret) 97 : "D"(fd), "S"(buf), "d"(count) 98 : "cc", "rcx", "r11", "memory"); 99 return ret; 100 } 101 102 uint64_t __write(uint64_t fd, const void *buf, uint64_t count) { 103 uint64_t ret; 104 #if defined(__APPLE__) 105 #define WRITE_SYSCALL 0x2000004 106 #else 107 #define WRITE_SYSCALL 1 108 #endif 109 __asm__ __volatile__("movq $" STRINGIFY(WRITE_SYSCALL) ", %%rax\n" 110 "syscall\n" 111 : "=a"(ret) 112 : "D"(fd), "S"(buf), "d"(count) 113 : "cc", "rcx", "r11", "memory"); 114 return ret; 115 } 116 117 void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags, 118 uint64_t fd, uint64_t offset) { 119 #if defined(__APPLE__) 120 #define MMAP_SYSCALL 0x20000c5 121 #else 122 #define MMAP_SYSCALL 9 123 #endif 124 void *ret; 125 register uint64_t r8 asm("r8") = fd; 126 register uint64_t r9 asm("r9") = offset; 127 register uint64_t r10 asm("r10") = flags; 128 __asm__ __volatile__("movq $" STRINGIFY(MMAP_SYSCALL) ", %%rax\n" 129 "syscall\n" 130 : "=a"(ret) 131 : "D"(addr), "S"(size), "d"(prot), "r"(r10), "r"(r8), 132 "r"(r9) 133 : "cc", "rcx", "r11", "memory"); 134 return ret; 135 } 136 137 uint64_t __munmap(void *addr, uint64_t size) { 138 #if defined(__APPLE__) 139 #define MUNMAP_SYSCALL 0x2000049 140 #else 141 #define MUNMAP_SYSCALL 11 142 #endif 143 uint64_t ret; 144 __asm__ __volatile__("movq $" STRINGIFY(MUNMAP_SYSCALL) ", %%rax\n" 145 "syscall\n" 146 : "=a"(ret) 147 : "D"(addr), "S"(size) 148 : "cc", "rcx", "r11", "memory"); 149 return ret; 150 } 151 152 #define SIG_BLOCK 0 153 #define SIG_UNBLOCK 1 154 #define SIG_SETMASK 2 155 156 static const uint64_t MaskAllSignals[] = {-1ULL}; 157 158 uint64_t __sigprocmask(int how, const void *set, void *oldset) { 159 #if defined(__APPLE__) 160 #define SIGPROCMASK_SYSCALL 0x2000030 161 #else 162 #define SIGPROCMASK_SYSCALL 14 163 #endif 164 uint64_t ret; 165 register long r10 asm("r10") = sizeof(uint64_t); 166 __asm__ __volatile__("movq $" STRINGIFY(SIGPROCMASK_SYSCALL) ", %%rax\n" 167 "syscall\n" 168 : "=a"(ret) 169 : "D"(how), "S"(set), "d"(oldset), "r"(r10) 170 : "cc", "rcx", "r11", "memory"); 171 return ret; 172 } 173 174 uint64_t __exit(uint64_t code) { 175 #if defined(__APPLE__) 176 #define EXIT_SYSCALL 0x2000001 177 #else 178 #define EXIT_SYSCALL 231 179 #endif 180 uint64_t ret; 181 __asm__ __volatile__("movq $" STRINGIFY(EXIT_SYSCALL) ", %%rax\n" 182 "syscall\n" 183 : "=a"(ret) 184 : "D"(code) 185 : "cc", "rcx", "r11", "memory"); 186 return ret; 187 } 188 189 // Helper functions for writing strings to the .fdata file. We intentionally 190 // avoid using libc names (lowercase memset) to make it clear it is our impl. 191 192 /// Write number Num using Base to the buffer in OutBuf, returns a pointer to 193 /// the end of the string. 194 char *intToStr(char *OutBuf, uint64_t Num, uint32_t Base) { 195 const char *Chars = "0123456789abcdef"; 196 char Buf[21]; 197 char *Ptr = Buf; 198 while (Num) { 199 *Ptr++ = *(Chars + (Num % Base)); 200 Num /= Base; 201 } 202 if (Ptr == Buf) { 203 *OutBuf++ = '0'; 204 return OutBuf; 205 } 206 while (Ptr != Buf) 207 *OutBuf++ = *--Ptr; 208 209 return OutBuf; 210 } 211 212 /// Copy Str to OutBuf, returns a pointer to the end of the copied string 213 char *strCopy(char *OutBuf, const char *Str, int32_t Size = BufSize) { 214 while (*Str) { 215 *OutBuf++ = *Str++; 216 if (--Size <= 0) 217 return OutBuf; 218 } 219 return OutBuf; 220 } 221 222 /// Compare two strings, at most Num bytes. 223 int strnCmp(const char *Str1, const char *Str2, size_t Num) { 224 while (Num && *Str1 && (*Str1 == *Str2)) { 225 Num--; 226 Str1++; 227 Str2++; 228 } 229 if (Num == 0) 230 return 0; 231 return *(unsigned char *)Str1 - *(unsigned char *)Str2; 232 } 233 234 void memSet(char *Buf, char C, uint32_t Size) { 235 for (int I = 0; I < Size; ++I) 236 *Buf++ = C; 237 } 238 239 void *memCpy(void *Dest, const void *Src, size_t Len) { 240 char *d = static_cast<char *>(Dest); 241 const char *s = static_cast<const char *>(Src); 242 while (Len--) 243 *d++ = *s++; 244 return Dest; 245 } 246 247 uint32_t strLen(const char *Str) { 248 uint32_t Size = 0; 249 while (*Str++) 250 ++Size; 251 return Size; 252 } 253 254 void reportNumber(const char *Msg, uint64_t Num, uint32_t Base) { 255 char Buf[BufSize]; 256 char *Ptr = Buf; 257 Ptr = strCopy(Ptr, Msg, BufSize - 23); 258 Ptr = intToStr(Ptr, Num, Base); 259 Ptr = strCopy(Ptr, "\n"); 260 __write(2, Buf, Ptr - Buf); 261 } 262 263 void report(const char *Msg) { __write(2, Msg, strLen(Msg)); } 264 265 unsigned long hexToLong(const char *Str, char Terminator = '\0') { 266 unsigned long Res = 0; 267 while (*Str != Terminator) { 268 Res <<= 4; 269 if ('0' <= *Str && *Str <= '9') 270 Res += *Str++ - '0'; 271 else if ('a' <= *Str && *Str <= 'f') 272 Res += *Str++ - 'a' + 10; 273 else if ('A' <= *Str && *Str <= 'F') 274 Res += *Str++ - 'A' + 10; 275 else 276 return 0; 277 } 278 return Res; 279 } 280 281 #if !defined(__APPLE__) 282 // We use a stack-allocated buffer for string manipulation in many pieces of 283 // this code, including the code that prints each line of the fdata file. This 284 // buffer needs to accomodate large function names, but shouldn't be arbitrarily 285 // large (dynamically allocated) for simplicity of our memory space usage. 286 287 // Declare some syscall wrappers we use throughout this code to avoid linking 288 // against system libc. 289 uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) { 290 uint64_t ret; 291 __asm__ __volatile__("movq $2, %%rax\n" 292 "syscall" 293 : "=a"(ret) 294 : "D"(pathname), "S"(flags), "d"(mode) 295 : "cc", "rcx", "r11", "memory"); 296 return ret; 297 } 298 299 struct dirent { 300 unsigned long d_ino; /* Inode number */ 301 unsigned long d_off; /* Offset to next linux_dirent */ 302 unsigned short d_reclen; /* Length of this linux_dirent */ 303 char d_name[]; /* Filename (null-terminated) */ 304 /* length is actually (d_reclen - 2 - 305 offsetof(struct linux_dirent, d_name)) */ 306 }; 307 308 long __getdents(unsigned int fd, dirent *dirp, size_t count) { 309 long ret; 310 __asm__ __volatile__("movq $78, %%rax\n" 311 "syscall" 312 : "=a"(ret) 313 : "D"(fd), "S"(dirp), "d"(count) 314 : "cc", "rcx", "r11", "memory"); 315 return ret; 316 } 317 318 uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) { 319 uint64_t ret; 320 __asm__ __volatile__("movq $89, %%rax\n" 321 "syscall" 322 : "=a"(ret) 323 : "D"(pathname), "S"(buf), "d"(bufsize) 324 : "cc", "rcx", "r11", "memory"); 325 return ret; 326 } 327 328 uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) { 329 uint64_t ret; 330 __asm__ __volatile__("movq $8, %%rax\n" 331 "syscall\n" 332 : "=a"(ret) 333 : "D"(fd), "S"(pos), "d"(whence) 334 : "cc", "rcx", "r11", "memory"); 335 return ret; 336 } 337 338 int __close(uint64_t fd) { 339 uint64_t ret; 340 __asm__ __volatile__("movq $3, %%rax\n" 341 "syscall\n" 342 : "=a"(ret) 343 : "D"(fd) 344 : "cc", "rcx", "r11", "memory"); 345 return ret; 346 } 347 348 int __madvise(void *addr, size_t length, int advice) { 349 int ret; 350 __asm__ __volatile__("movq $28, %%rax\n" 351 "syscall\n" 352 : "=a"(ret) 353 : "D"(addr), "S"(length), "d"(advice) 354 : "cc", "rcx", "r11", "memory"); 355 return ret; 356 } 357 358 struct timespec { 359 uint64_t tv_sec; /* seconds */ 360 uint64_t tv_nsec; /* nanoseconds */ 361 }; 362 363 uint64_t __nanosleep(const timespec *req, timespec *rem) { 364 uint64_t ret; 365 __asm__ __volatile__("movq $35, %%rax\n" 366 "syscall\n" 367 : "=a"(ret) 368 : "D"(req), "S"(rem) 369 : "cc", "rcx", "r11", "memory"); 370 return ret; 371 } 372 373 int64_t __fork() { 374 uint64_t ret; 375 __asm__ __volatile__("movq $57, %%rax\n" 376 "syscall\n" 377 : "=a"(ret) 378 : 379 : "cc", "rcx", "r11", "memory"); 380 return ret; 381 } 382 383 int __mprotect(void *addr, size_t len, int prot) { 384 int ret; 385 __asm__ __volatile__("movq $10, %%rax\n" 386 "syscall\n" 387 : "=a"(ret) 388 : "D"(addr), "S"(len), "d"(prot) 389 : "cc", "rcx", "r11", "memory"); 390 return ret; 391 } 392 393 uint64_t __getpid() { 394 uint64_t ret; 395 __asm__ __volatile__("movq $39, %%rax\n" 396 "syscall\n" 397 : "=a"(ret) 398 : 399 : "cc", "rcx", "r11", "memory"); 400 return ret; 401 } 402 403 uint64_t __getppid() { 404 uint64_t ret; 405 __asm__ __volatile__("movq $110, %%rax\n" 406 "syscall\n" 407 : "=a"(ret) 408 : 409 : "cc", "rcx", "r11", "memory"); 410 return ret; 411 } 412 413 int __setpgid(uint64_t pid, uint64_t pgid) { 414 int ret; 415 __asm__ __volatile__("movq $109, %%rax\n" 416 "syscall\n" 417 : "=a"(ret) 418 : "D"(pid), "S"(pgid) 419 : "cc", "rcx", "r11", "memory"); 420 return ret; 421 } 422 423 uint64_t __getpgid(uint64_t pid) { 424 uint64_t ret; 425 __asm__ __volatile__("movq $121, %%rax\n" 426 "syscall\n" 427 : "=a"(ret) 428 : "D"(pid) 429 : "cc", "rcx", "r11", "memory"); 430 return ret; 431 } 432 433 int __kill(uint64_t pid, int sig) { 434 int ret; 435 __asm__ __volatile__("movq $62, %%rax\n" 436 "syscall\n" 437 : "=a"(ret) 438 : "D"(pid), "S"(sig) 439 : "cc", "rcx", "r11", "memory"); 440 return ret; 441 } 442 443 int __fsync(int fd) { 444 int ret; 445 __asm__ __volatile__("movq $74, %%rax\n" 446 "syscall\n" 447 : "=a"(ret) 448 : "D"(fd) 449 : "cc", "rcx", "r11", "memory"); 450 return ret; 451 } 452 453 #endif 454 455 void reportError(const char *Msg, uint64_t Size) { 456 __write(2, Msg, Size); 457 __exit(1); 458 } 459 460 void assert(bool Assertion, const char *Msg) { 461 if (Assertion) 462 return; 463 char Buf[BufSize]; 464 char *Ptr = Buf; 465 Ptr = strCopy(Ptr, "Assertion failed: "); 466 Ptr = strCopy(Ptr, Msg, BufSize - 40); 467 Ptr = strCopy(Ptr, "\n"); 468 reportError(Buf, Ptr - Buf); 469 } 470 471 /// 1B mutex accessed by lock xchg 472 class Mutex { 473 volatile bool InUse{false}; 474 475 public: 476 bool acquire() { 477 bool Result = true; 478 asm volatile("lock; xchg %0, %1" : "+m"(InUse), "=r"(Result) : : "cc"); 479 return !Result; 480 } 481 void release() { InUse = false; } 482 }; 483 484 /// RAII wrapper for Mutex 485 class Lock { 486 Mutex &M; 487 uint64_t SignalMask[1] = {}; 488 489 public: 490 Lock(Mutex &M) : M(M) { 491 __sigprocmask(SIG_BLOCK, MaskAllSignals, SignalMask); 492 while (!M.acquire()) { 493 } 494 } 495 496 ~Lock() { 497 M.release(); 498 __sigprocmask(SIG_SETMASK, SignalMask, nullptr); 499 } 500 }; 501 502 /// RAII wrapper for Mutex 503 class TryLock { 504 Mutex &M; 505 bool Locked = false; 506 507 public: 508 TryLock(Mutex &M) : M(M) { 509 int Retry = 100; 510 while (--Retry && !M.acquire()) 511 ; 512 if (Retry) 513 Locked = true; 514 } 515 bool isLocked() { return Locked; } 516 517 ~TryLock() { 518 if (isLocked()) 519 M.release(); 520 } 521 }; 522 523 inline uint64_t alignTo(uint64_t Value, uint64_t Align) { 524 return (Value + Align - 1) / Align * Align; 525 } 526 527 } // anonymous namespace 528