llvm-project
177 строк · 5.4 Кб
1//===- bolt/runtime/hugify.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8
9#if defined (__x86_64__) && !defined(__APPLE__)
10
11#include "common.h"
12
13#pragma GCC visibility push(hidden)
14
15// Enables a very verbose logging to stderr useful when debugging
16// #define ENABLE_DEBUG
17
18#ifdef ENABLE_DEBUG
19#define DEBUG(X) \
20{ X; }
21#else
22#define DEBUG(X) \
23{}
24#endif
25
26// Function constains trampoline to _start,
27// so we can resume regular execution of the function that we hooked.
28extern void __bolt_hugify_start_program();
29
30// The __hot_start and __hot_end symbols set by Bolt. We use them to figure
31// out the rage for marking huge pages.
32extern uint64_t __hot_start;
33extern uint64_t __hot_end;
34
35static void getKernelVersion(uint32_t *Val) {
36// release should be in the format: %d.%d.%d
37// major, minor, release
38struct UtsNameTy UtsName;
39int Ret = __uname(&UtsName);
40const char *Buf = UtsName.release;
41const char *End = Buf + strLen(Buf);
42const char Delims[2][2] = {".", "."};
43
44for (int i = 0; i < 3; ++i) {
45if (!scanUInt32(Buf, End, Val[i])) {
46return;
47}
48if (i < sizeof(Delims) / sizeof(Delims[0])) {
49const char *Ptr = Delims[i];
50while (*Ptr != '\0') {
51if (*Ptr != *Buf) {
52return;
53}
54++Ptr;
55++Buf;
56}
57}
58}
59}
60
61/// Check whether the kernel supports THP via corresponding sysfs entry.
62/// thp works only starting from 5.10
63static bool hasPagecacheTHPSupport() {
64char Buf[64];
65
66int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
670 /* O_RDONLY */, 0);
68if (FD < 0)
69return false;
70
71memset(Buf, 0, sizeof(Buf));
72const size_t Res = __read(FD, Buf, sizeof(Buf));
73if (Res < 0)
74return false;
75
76if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]"))
77return false;
78
79struct KernelVersionTy {
80uint32_t major;
81uint32_t minor;
82uint32_t release;
83};
84
85KernelVersionTy KernelVersion;
86
87getKernelVersion((uint32_t *)&KernelVersion);
88if (KernelVersion.major >= 5 && KernelVersion.minor >= 10)
89return true;
90
91return false;
92}
93
94static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
95const size_t Size = To - From;
96
97uint8_t *Mem = reinterpret_cast<uint8_t *>(
98__mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
990x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
100
101if (Mem == ((void *)-1) /* MAP_FAILED */) {
102char Msg[] = "[hugify] could not allocate memory for text move\n";
103reportError(Msg, sizeof(Msg));
104}
105
106DEBUG(reportNumber("[hugify] allocated temporary address: ", (uint64_t)Mem,
10716);)
108DEBUG(reportNumber("[hugify] allocated size: ", (uint64_t)Size, 16);)
109
110// Copy the hot code to a temporary location.
111memcpy(Mem, From, Size);
112
113__prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
114// Maps out the existing hot code.
115if (__mmap(reinterpret_cast<uint64_t>(From), Size,
1160x3 /* PROT_READ | PROT_WRITE */,
1170x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
1180) == ((void *)-1) /*MAP_FAILED*/) {
119char Msg[] =
120"[hugify] failed to mmap memory for large page move terminating\n";
121reportError(Msg, sizeof(Msg));
122}
123
124// Mark the hot code page to be huge page.
125if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
126char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
127reportError(Msg, sizeof(Msg));
128}
129
130// Copy the hot code back.
131memcpy(From, Mem, Size);
132
133// Change permission back to read-only, ignore failure
134__mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
135
136__munmap(Mem, Size);
137}
138
139extern "C" void __bolt_hugify_self_impl() {
140uint8_t *HotStart = (uint8_t *)&__hot_start;
141uint8_t *HotEnd = (uint8_t *)&__hot_end;
142// Make sure the start and end are aligned with huge page address
143const size_t HugePageBytes = 2L * 1024 * 1024;
144uint8_t *From = HotStart - ((intptr_t)HotStart & (HugePageBytes - 1));
145uint8_t *To = HotEnd + (HugePageBytes - 1);
146To -= (intptr_t)To & (HugePageBytes - 1);
147
148DEBUG(reportNumber("[hugify] hot start: ", (uint64_t)HotStart, 16);)
149DEBUG(reportNumber("[hugify] hot end: ", (uint64_t)HotEnd, 16);)
150DEBUG(reportNumber("[hugify] aligned huge page from: ", (uint64_t)From, 16);)
151DEBUG(reportNumber("[hugify] aligned huge page to: ", (uint64_t)To, 16);)
152
153if (!hasPagecacheTHPSupport()) {
154DEBUG(report(
155"[hugify] workaround with memory alignment for kernel < 5.10\n");)
156hugifyForOldKernel(From, To);
157return;
158}
159
160if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
161char Msg[] = "[hugify] failed to allocate large page\n";
162// TODO: allow user to control the failure behavior.
163reportError(Msg, sizeof(Msg));
164}
165}
166
167/// This is hooking ELF's entry, it needs to save all machine state.
168extern "C" __attribute((naked)) void __bolt_hugify_self() {
169#if defined(__x86_64__)
170__asm__ __volatile__(SAVE_ALL "call __bolt_hugify_self_impl\n" RESTORE_ALL
171"jmp __bolt_hugify_start_program\n" ::
172:);
173#else
174exit(1);
175#endif
176}
177#endif
178