llvm-project

1
//===- bolt/runtime/hugify.cpp -------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===---------------------------------------------------------------------===//
8

9
#if defined (__x86_64__) && !defined(__APPLE__)
10

11
#include "common.h"
12

13
#pragma GCC visibility push(hidden)
14

15
// Enables a very verbose logging to stderr useful when debugging
16
// #define ENABLE_DEBUG
17

18
#ifdef ENABLE_DEBUG
19
#define DEBUG(X)                                                               \
20
  { X; }
21
#else
22
#define DEBUG(X)                                                               \
23
  {}
24
#endif
25

26
// Function constains trampoline to _start,
27
// so we can resume regular execution of the function that we hooked.
28
extern void __bolt_hugify_start_program();
29

30
// The __hot_start and __hot_end symbols set by Bolt. We use them to figure
31
// out the rage for marking huge pages.
32
extern uint64_t __hot_start;
33
extern uint64_t __hot_end;
34

35
static void getKernelVersion(uint32_t *Val) {
36
  // release should be in the format: %d.%d.%d
37
  // major, minor, release
38
  struct UtsNameTy UtsName;
39
  int Ret = __uname(&UtsName);
40
  const char *Buf = UtsName.release;
41
  const char *End = Buf + strLen(Buf);
42
  const char Delims[2][2] = {".", "."};
43

44
  for (int i = 0; i < 3; ++i) {
45
    if (!scanUInt32(Buf, End, Val[i])) {
46
      return;
47
    }
48
    if (i < sizeof(Delims) / sizeof(Delims[0])) {
49
      const char *Ptr = Delims[i];
50
      while (*Ptr != '\0') {
51
        if (*Ptr != *Buf) {
52
          return;
53
        }
54
        ++Ptr;
55
        ++Buf;
56
      }
57
    }
58
  }
59
}
60

61
/// Check whether the kernel supports THP via corresponding sysfs entry.
62
/// thp works only starting from 5.10
63
static bool hasPagecacheTHPSupport() {
64
  char Buf[64];
65

66
  int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
67
                  0 /* O_RDONLY */, 0);
68
  if (FD < 0)
69
    return false;
70

71
  memset(Buf, 0, sizeof(Buf));
72
  const size_t Res = __read(FD, Buf, sizeof(Buf));
73
  if (Res < 0)
74
    return false;
75

76
  if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]"))
77
    return false;
78

79
  struct KernelVersionTy {
80
    uint32_t major;
81
    uint32_t minor;
82
    uint32_t release;
83
  };
84

85
  KernelVersionTy KernelVersion;
86

87
  getKernelVersion((uint32_t *)&KernelVersion);
88
  if (KernelVersion.major >= 5 && KernelVersion.minor >= 10)
89
    return true;
90

91
  return false;
92
}
93

94
static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
95
  const size_t Size = To - From;
96

97
  uint8_t *Mem = reinterpret_cast<uint8_t *>(
98
      __mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
99
             0x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
100

101
  if (Mem == ((void *)-1) /* MAP_FAILED */) {
102
    char Msg[] = "[hugify] could not allocate memory for text move\n";
103
    reportError(Msg, sizeof(Msg));
104
  }
105

106
  DEBUG(reportNumber("[hugify] allocated temporary address: ", (uint64_t)Mem,
107
                     16);)
108
  DEBUG(reportNumber("[hugify] allocated size: ", (uint64_t)Size, 16);)
109

110
  // Copy the hot code to a temporary location.
111
  memcpy(Mem, From, Size);
112

113
  __prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
114
  // Maps out the existing hot code.
115
  if (__mmap(reinterpret_cast<uint64_t>(From), Size,
116
             0x3 /* PROT_READ | PROT_WRITE */,
117
             0x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
118
             0) == ((void *)-1) /*MAP_FAILED*/) {
119
    char Msg[] =
120
        "[hugify] failed to mmap memory for large page move terminating\n";
121
    reportError(Msg, sizeof(Msg));
122
  }
123

124
  // Mark the hot code page to be huge page.
125
  if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
126
    char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
127
    reportError(Msg, sizeof(Msg));
128
  }
129

130
  // Copy the hot code back.
131
  memcpy(From, Mem, Size);
132

133
  // Change permission back to read-only, ignore failure
134
  __mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
135

136
  __munmap(Mem, Size);
137
}
138

139
extern "C" void __bolt_hugify_self_impl() {
140
  uint8_t *HotStart = (uint8_t *)&__hot_start;
141
  uint8_t *HotEnd = (uint8_t *)&__hot_end;
142
  // Make sure the start and end are aligned with huge page address
143
  const size_t HugePageBytes = 2L * 1024 * 1024;
144
  uint8_t *From = HotStart - ((intptr_t)HotStart & (HugePageBytes - 1));
145
  uint8_t *To = HotEnd + (HugePageBytes - 1);
146
  To -= (intptr_t)To & (HugePageBytes - 1);
147

148
  DEBUG(reportNumber("[hugify] hot start: ", (uint64_t)HotStart, 16);)
149
  DEBUG(reportNumber("[hugify] hot end: ", (uint64_t)HotEnd, 16);)
150
  DEBUG(reportNumber("[hugify] aligned huge page from: ", (uint64_t)From, 16);)
151
  DEBUG(reportNumber("[hugify] aligned huge page to: ", (uint64_t)To, 16);)
152

153
  if (!hasPagecacheTHPSupport()) {
154
    DEBUG(report(
155
              "[hugify] workaround with memory alignment for kernel < 5.10\n");)
156
    hugifyForOldKernel(From, To);
157
    return;
158
  }
159

160
  if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
161
    char Msg[] = "[hugify] failed to allocate large page\n";
162
    // TODO: allow user to control the failure behavior.
163
    reportError(Msg, sizeof(Msg));
164
  }
165
}
166

167
/// This is hooking ELF's entry, it needs to save all machine state.
168
extern "C" __attribute((naked)) void __bolt_hugify_self() {
169
#if defined(__x86_64__)
170
  __asm__ __volatile__(SAVE_ALL "call __bolt_hugify_self_impl\n" RESTORE_ALL
171
                                "jmp __bolt_hugify_start_program\n" ::
172
                                    :);
173
#else
174
  exit(1);
175
#endif
176
}
177
#endif
178