1 //===- bolt/runtime/hugify.cpp --------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #if defined (__x86_64__)
10 #if !defined(__APPLE__)
11
12 #include "common.h"
13 #include <sys/mman.h>
14
15 // Enables a very verbose logging to stderr useful when debugging
16 //#define ENABLE_DEBUG
17
18 // Function pointers to init routines in the binary, so we can resume
19 // regular execution of the function that we hooked.
20 extern void (*__bolt_hugify_init_ptr)();
21
22 // The __hot_start and __hot_end symbols set by Bolt. We use them to figure
23 // out the rage for marking huge pages.
24 extern uint64_t __hot_start;
25 extern uint64_t __hot_end;
26
27 #ifdef MADV_HUGEPAGE
28 /// Check whether the kernel supports THP via corresponding sysfs entry.
has_pagecache_thp_support()29 static bool has_pagecache_thp_support() {
30 char buf[256] = {0};
31 const char *madviseStr = "always [madvise] never";
32
33 int fd = __open("/sys/kernel/mm/transparent_hugepage/enabled",
34 0 /* O_RDONLY */, 0);
35 if (fd < 0)
36 return false;
37
38 size_t res = __read(fd, buf, 256);
39 if (res < 0)
40 return false;
41
42 int cmp = strnCmp(buf, madviseStr, strLen(madviseStr));
43 return cmp == 0;
44 }
45
hugify_for_old_kernel(uint8_t * from,uint8_t * to)46 static void hugify_for_old_kernel(uint8_t *from, uint8_t *to) {
47 size_t size = to - from;
48
49 uint8_t *mem = reinterpret_cast<uint8_t *>(
50 __mmap(0, size, 0x3 /* PROT_READ | PROT_WRITE*/,
51 0x22 /* MAP_PRIVATE | MAP_ANONYMOUS*/, -1, 0));
52
53 if (mem == (void *)MAP_FAILED) {
54 char msg[] = "Could not allocate memory for text move\n";
55 reportError(msg, sizeof(msg));
56 }
57 #ifdef ENABLE_DEBUG
58 reportNumber("Allocated temporary space: ", (uint64_t)mem, 16);
59 #endif
60
61 // Copy the hot code to a temproary location.
62 memcpy(mem, from, size);
63
64 // Maps out the existing hot code.
65 if (__mmap(reinterpret_cast<uint64_t>(from), size,
66 PROT_READ | PROT_WRITE | PROT_EXEC,
67 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1,
68 0) == (void *)MAP_FAILED) {
69 char msg[] = "failed to mmap memory for large page move terminating\n";
70 reportError(msg, sizeof(msg));
71 }
72
73 // Mark the hot code page to be huge page.
74 if (__madvise(from, size, MADV_HUGEPAGE) == -1) {
75 char msg[] = "failed to allocate large page\n";
76 reportError(msg, sizeof(msg));
77 }
78
79 // Copy the hot code back.
80 memcpy(from, mem, size);
81
82 // Change permission back to read-only, ignore failure
83 __mprotect(from, size, PROT_READ | PROT_EXEC);
84
85 __munmap(mem, size);
86 }
87 #endif
88
__bolt_hugify_self_impl()89 extern "C" void __bolt_hugify_self_impl() {
90 #ifdef MADV_HUGEPAGE
91 uint8_t *hotStart = (uint8_t *)&__hot_start;
92 uint8_t *hotEnd = (uint8_t *)&__hot_end;
93 // Make sure the start and end are aligned with huge page address
94 const size_t hugePageBytes = 2L * 1024 * 1024;
95 uint8_t *from = hotStart - ((intptr_t)hotStart & (hugePageBytes - 1));
96 uint8_t *to = hotEnd + (hugePageBytes - 1);
97 to -= (intptr_t)to & (hugePageBytes - 1);
98
99 #ifdef ENABLE_DEBUG
100 reportNumber("[hugify] hot start: ", (uint64_t)hotStart, 16);
101 reportNumber("[hugify] hot end: ", (uint64_t)hotEnd, 16);
102 reportNumber("[hugify] aligned huge page from: ", (uint64_t)from, 16);
103 reportNumber("[hugify] aligned huge page to: ", (uint64_t)to, 16);
104 #endif
105
106 if (!has_pagecache_thp_support()) {
107 hugify_for_old_kernel(from, to);
108 return;
109 }
110
111 if (__madvise(from, (to - from), MADV_HUGEPAGE) == -1) {
112 char msg[] = "failed to allocate large page\n";
113 // TODO: allow user to control the failure behavior.
114 reportError(msg, sizeof(msg));
115 }
116 #endif
117 }
118
119 /// This is hooking ELF's entry, it needs to save all machine state.
__bolt_hugify_self()120 extern "C" __attribute((naked)) void __bolt_hugify_self() {
121 __asm__ __volatile__(SAVE_ALL
122 "call __bolt_hugify_self_impl\n"
123 RESTORE_ALL
124 "jmp *__bolt_hugify_init_ptr(%%rip)\n"
125 :::);
126 }
127
128 #endif
129 #endif
130