1 //===-- hugify.cpp ----------------------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 // This file contains code that is linked to the final binary with a function
8 // that is called at program entry to put hot code into a huge page.
9 //
10 //===----------------------------------------------------------------------===//
11 
12 #if !defined(__APPLE__)
13 
14 #include "common.h"
15 #include <sys/mman.h>
16 
17 // Enables a very verbose logging to stderr useful when debugging
18 //#define ENABLE_DEBUG
19 
20 // Function pointers to init routines in the binary, so we can resume
21 // regular execution of the function that we hooked.
22 extern void (*__bolt_hugify_init_ptr)();
23 
24 // The __hot_start and __hot_end symbols set by Bolt. We use them to figure
25 // out the rage for marking huge pages.
26 extern uint64_t __hot_start;
27 extern uint64_t __hot_end;
28 
29 #ifdef MADV_HUGEPAGE
30 /// Starting from character at \p buf, find the longest consecutive sequence
31 /// of digits (0-9) and convert it to uint32_t. The converted value
32 /// is put into \p ret. \p end marks the end of the buffer to avoid buffer
33 /// overflow. The function \returns whether a valid uint32_t value is found.
34 /// \p buf will be updated to the next character right after the digits.
35 static bool scanUInt32(const char *&buf, const char *end, uint32_t &ret) {
36   uint64_t result = 0;
37   const char *oldBuf = buf;
38   while (buf < end && ((*buf) >= '0' && (*buf) <= '9')) {
39     result = result * 10 + (*buf) - '0';
40     ++buf;
41   }
42   if (oldBuf != buf && result <= 0xFFFFFFFFu) {
43     ret = static_cast<uint32_t>(result);
44     return true;
45   }
46   return false;
47 }
48 
49 /// Check whether the kernel supports THP by checking the kernel version.
50 /// Only fb kernel 5.2 and latter supports it.
51 static bool has_pagecache_thp_support() {
52   struct utsname u;
53   int ret = __uname(&u);
54   if (ret) {
55     return false;
56   }
57 
58   const char *buf = u.release;
59 #ifdef ENABLE_DEBUG
60   report("[hugify] uname release: ");
61   report(buf);
62   report("\n");
63 #endif
64   const char *end = buf + strLen(buf);
65   uint32_t nums[5];
66   char delims[4][5] = {".", ".", "-", "_fbk"};
67   // release should be in the format: %d.%d.%d-%d_fbk%d
68   // they represent: major, minor, release, build, fbk.
69   for (int i = 0; i < 5; ++i) {
70     if (!scanUInt32(buf, end, nums[i])) {
71       return false;
72     }
73     if (i < 4) {
74       const char *ptr = delims[i];
75       while (*ptr != '\0') {
76         if (*ptr != *buf) {
77           return false;
78         }
79         ++ptr;
80         ++buf;
81       }
82     }
83   }
84   if (nums[0] > 5) {
85     // Major is > 5.
86     return true;
87   }
88   if (nums[0] < 5) {
89     // Major is < 5.
90     return false;
91   }
92   // minor > 2 || fbk >= 5.
93   return nums[1] > 2 || nums[4] >= 5;
94 }
95 
96 static void hugify_for_old_kernel(uint8_t *from, uint8_t *to) {
97   size_t size = to - from;
98 
99   uint8_t *mem = reinterpret_cast<uint8_t *>(
100       __mmap(0, size, 0x3 /* PROT_READ | PROT_WRITE*/,
101              0x22 /* MAP_PRIVATE | MAP_ANONYMOUS*/, -1, 0));
102 
103   if (mem == (void *)MAP_FAILED) {
104     char msg[] = "Could not allocate memory for text move\n";
105     reportError(msg, sizeof(msg));
106   }
107 #ifdef ENABLE_DEBUG
108   reportNumber("Allocated temporary space: ", (uint64_t)mem, 16);
109 #endif
110 
111   // Copy the hot code to a temproary location.
112   memCpy(mem, from, size);
113 
114   // Maps out the existing hot code.
115   if (__mmap(reinterpret_cast<uint64_t>(from), size,
116              PROT_READ | PROT_WRITE | PROT_EXEC,
117              MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1,
118              0) == (void *)MAP_FAILED) {
119     char msg[] = "failed to mmap memory for large page move terminating\n";
120     reportError(msg, sizeof(msg));
121   }
122 
123   // Mark the hot code page to be huge page.
124   if (__madvise(from, size, MADV_HUGEPAGE) == -1) {
125     char msg[] = "failed to allocate large page\n";
126     reportError(msg, sizeof(msg));
127   }
128 
129   // Copy the hot code back.
130   memCpy(from, mem, size);
131 
132   // Change permission back to read-only, ignore failure
133   __mprotect(from, size, PROT_READ | PROT_EXEC);
134 
135   __munmap(mem, size);
136 }
137 #endif
138 
139 extern "C" void __bolt_hugify_self_impl() {
140 #ifdef MADV_HUGEPAGE
141   uint8_t *hotStart = (uint8_t *)&__hot_start;
142   uint8_t *hotEnd = (uint8_t *)&__hot_end;
143   // Make sure the start and end are aligned with huge page address
144   const size_t hugePageBytes = 2L * 1024 * 1024;
145   uint8_t *from = hotStart - ((intptr_t)hotStart & (hugePageBytes - 1));
146   uint8_t *to = hotEnd + (hugePageBytes - 1);
147   to -= (intptr_t)to & (hugePageBytes - 1);
148 
149 #ifdef ENABLE_DEBUG
150   reportNumber("[hugify] hot start: ", (uint64_t)hotStart, 16);
151   reportNumber("[hugify] hot end: ", (uint64_t)hotEnd, 16);
152   reportNumber("[hugify] aligned huge page from: ", (uint64_t)from, 16);
153   reportNumber("[hugify] aligned huge page to: ", (uint64_t)to, 16);
154 #endif
155 
156   if (!has_pagecache_thp_support()) {
157     hugify_for_old_kernel(from, to);
158     return;
159   }
160 
161   if (__madvise(from, (to - from), MADV_HUGEPAGE) == -1) {
162     char msg[] = "failed to allocate large page\n";
163     // TODO: allow user to control the failure behavior.
164     reportError(msg, sizeof(msg));
165   }
166 #endif
167 }
168 
169 /// This is hooking ELF's entry, it needs to save all machine state.
170 extern "C" __attribute((naked)) void __bolt_hugify_self() {
171   __asm__ __volatile__(SAVE_ALL
172                        "call __bolt_hugify_self_impl\n"
173                        RESTORE_ALL
174                        "jmp *__bolt_hugify_init_ptr(%%rip)\n"
175                        :::);
176 }
177 
178 #endif
179