1 //===- bolt/runtime/hugify.cpp --------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #if defined (__x86_64__)
10 #if !defined(__APPLE__)
11 
12 #include "common.h"
13 #include <sys/mman.h>
14 
15 // Enables a very verbose logging to stderr useful when debugging
16 //#define ENABLE_DEBUG
17 
18 // Function pointers to init routines in the binary, so we can resume
19 // regular execution of the function that we hooked.
20 extern void (*__bolt_hugify_init_ptr)();
21 
22 // The __hot_start and __hot_end symbols set by Bolt. We use them to figure
23 // out the rage for marking huge pages.
24 extern uint64_t __hot_start;
25 extern uint64_t __hot_end;
26 
27 #ifdef MADV_HUGEPAGE
28 /// Check whether the kernel supports THP via corresponding sysfs entry.
29 static bool has_pagecache_thp_support() {
30   char buf[256] = {0};
31   const char *madviseStr = "always [madvise] never";
32 
33   int fd = __open("/sys/kernel/mm/transparent_hugepage/enabled",
34                   0 /* O_RDONLY */, 0);
35   if (fd < 0)
36     return false;
37 
38   size_t res = __read(fd, buf, 256);
39   if (res < 0)
40     return false;
41 
42   int cmp = strnCmp(buf, madviseStr, strLen(madviseStr));
43   return cmp == 0;
44 }
45 
46 static void hugify_for_old_kernel(uint8_t *from, uint8_t *to) {
47   size_t size = to - from;
48 
49   uint8_t *mem = reinterpret_cast<uint8_t *>(
50       __mmap(0, size, 0x3 /* PROT_READ | PROT_WRITE*/,
51              0x22 /* MAP_PRIVATE | MAP_ANONYMOUS*/, -1, 0));
52 
53   if (mem == (void *)MAP_FAILED) {
54     char msg[] = "Could not allocate memory for text move\n";
55     reportError(msg, sizeof(msg));
56   }
57 #ifdef ENABLE_DEBUG
58   reportNumber("Allocated temporary space: ", (uint64_t)mem, 16);
59 #endif
60 
61   // Copy the hot code to a temproary location.
62   memCpy(mem, from, size);
63 
64   // Maps out the existing hot code.
65   if (__mmap(reinterpret_cast<uint64_t>(from), size,
66              PROT_READ | PROT_WRITE | PROT_EXEC,
67              MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1,
68              0) == (void *)MAP_FAILED) {
69     char msg[] = "failed to mmap memory for large page move terminating\n";
70     reportError(msg, sizeof(msg));
71   }
72 
73   // Mark the hot code page to be huge page.
74   if (__madvise(from, size, MADV_HUGEPAGE) == -1) {
75     char msg[] = "failed to allocate large page\n";
76     reportError(msg, sizeof(msg));
77   }
78 
79   // Copy the hot code back.
80   memCpy(from, mem, size);
81 
82   // Change permission back to read-only, ignore failure
83   __mprotect(from, size, PROT_READ | PROT_EXEC);
84 
85   __munmap(mem, size);
86 }
87 #endif
88 
89 extern "C" void __bolt_hugify_self_impl() {
90 #ifdef MADV_HUGEPAGE
91   uint8_t *hotStart = (uint8_t *)&__hot_start;
92   uint8_t *hotEnd = (uint8_t *)&__hot_end;
93   // Make sure the start and end are aligned with huge page address
94   const size_t hugePageBytes = 2L * 1024 * 1024;
95   uint8_t *from = hotStart - ((intptr_t)hotStart & (hugePageBytes - 1));
96   uint8_t *to = hotEnd + (hugePageBytes - 1);
97   to -= (intptr_t)to & (hugePageBytes - 1);
98 
99 #ifdef ENABLE_DEBUG
100   reportNumber("[hugify] hot start: ", (uint64_t)hotStart, 16);
101   reportNumber("[hugify] hot end: ", (uint64_t)hotEnd, 16);
102   reportNumber("[hugify] aligned huge page from: ", (uint64_t)from, 16);
103   reportNumber("[hugify] aligned huge page to: ", (uint64_t)to, 16);
104 #endif
105 
106   if (!has_pagecache_thp_support()) {
107     hugify_for_old_kernel(from, to);
108     return;
109   }
110 
111   if (__madvise(from, (to - from), MADV_HUGEPAGE) == -1) {
112     char msg[] = "failed to allocate large page\n";
113     // TODO: allow user to control the failure behavior.
114     reportError(msg, sizeof(msg));
115   }
116 #endif
117 }
118 
119 /// This is hooking ELF's entry, it needs to save all machine state.
120 extern "C" __attribute((naked)) void __bolt_hugify_self() {
121   __asm__ __volatile__(SAVE_ALL
122                        "call __bolt_hugify_self_impl\n"
123                        RESTORE_ALL
124                        "jmp *__bolt_hugify_init_ptr(%%rip)\n"
125                        :::);
126 }
127 
128 #endif
129 #endif
130