bolt/runtime/hugify.cpp

   1 //===- bolt/runtime/hugify.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===---------------------------------------------------------------------===//
   8
   9 #if defined (__x86_64__) && !defined(__APPLE__)
  10
  11 #include "common.h"
  12
  13 #pragma GCC visibility push(hidden)
  14
  15 // Enables a very verbose logging to stderr useful when debugging
  16 // #define ENABLE_DEBUG
  17
  18 #ifdef ENABLE_DEBUG
  19 #define DEBUG(X)                                                               \
  20   { X; }
  21 #else
  22 #define DEBUG(X)                                                               \
  23   {}
  24 #endif
  25
  26 // Function constains trampoline to _start,
  27 // so we can resume regular execution of the function that we hooked.
  28 extern void __bolt_hugify_start_program();
  29
  30 // The __hot_start and __hot_end symbols set by Bolt. We use them to figure
  31 // out the rage for marking huge pages.
  32 extern uint64_t __hot_start;
  33 extern uint64_t __hot_end;
  34
  35 static void getKernelVersion(uint32_t *Val) {
  36   // release should be in the format: %d.%d.%d
  37   // major, minor, release
  38   struct UtsNameTy UtsName;
  39   int Ret = __uname(&UtsName);
  40   const char *Buf = UtsName.release;
  41   const char *End = Buf + strLen(Buf);
  42   const char Delims[2][2] = {".", "."};
  43
  44   for (int i = 0; i < 3; ++i) {
  45     if (!scanUInt32(Buf, End, Val[i])) {
  46       return;
  47     }
  48     if (i < sizeof(Delims) / sizeof(Delims[0])) {
  49       const char *Ptr = Delims[i];
  50       while (*Ptr != '\0') {
  51         if (*Ptr != *Buf) {
  52           return;
  53         }
  54         ++Ptr;
  55         ++Buf;
  56       }
  57     }
  58   }
  59 }
  60
  61 /// Check whether the kernel supports THP via corresponding sysfs entry.
  62 /// thp works only starting from 5.10
  63 static bool hasPagecacheTHPSupport() {
  64   char Buf[64];
  65
  66   int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
  67                   0 /* O_RDONLY */, 0);
  68   if (FD < 0)
  69     return false;
  70
  71   memset(Buf, 0, sizeof(Buf));
  72   const size_t Res = __read(FD, Buf, sizeof(Buf));
  73   if (Res < 0)
  74     return false;
  75
  76   if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]"))
  77     return false;
  78
  79   struct KernelVersionTy {
  80     uint32_t major;
  81     uint32_t minor;
  82     uint32_t release;
  83   };
  84
  85   KernelVersionTy KernelVersion;
  86
  87   getKernelVersion((uint32_t *)&KernelVersion);
  88   if (KernelVersion.major >= 5 && KernelVersion.minor >= 10)
  89     return true;
  90
  91   return false;
  92 }
  93
  94 static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
  95   const size_t Size = To - From;
  96
  97   uint8_t *Mem = reinterpret_cast<uint8_t *>(
  98       __mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
  99              0x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
 100
 101   if (Mem == ((void *)-1) /* MAP_FAILED */) {
 102     char Msg[] = "[hugify] could not allocate memory for text move\n";
 103     reportError(Msg, sizeof(Msg));
 104   }
 105
 106   DEBUG(reportNumber("[hugify] allocated temporary address: ", (uint64_t)Mem,
 107                      16);)
 108   DEBUG(reportNumber("[hugify] allocated size: ", (uint64_t)Size, 16);)
 109
 110   // Copy the hot code to a temporary location.
 111   memcpy(Mem, From, Size);
 112
 113   __prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
 114   // Maps out the existing hot code.
 115   if (__mmap(reinterpret_cast<uint64_t>(From), Size,
 116              0x3 /* PROT_READ | PROT_WRITE */,
 117              0x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
 118              0) == ((void *)-1) /*MAP_FAILED*/) {
 119     char Msg[] =
 120         "[hugify] failed to mmap memory for large page move terminating\n";
 121     reportError(Msg, sizeof(Msg));
 122   }
 123
 124   // Mark the hot code page to be huge page.
 125   if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
 126     char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
 127     reportError(Msg, sizeof(Msg));
 128   }
 129
 130   // Copy the hot code back.
 131   memcpy(From, Mem, Size);
 132
 133   // Change permission back to read-only, ignore failure
 134   __mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
 135
 136   __munmap(Mem, Size);
 137 }
 138
 139 extern "C" void __bolt_hugify_self_impl() {
 140   uint8_t *HotStart = (uint8_t *)&__hot_start;
 141   uint8_t *HotEnd = (uint8_t *)&__hot_end;
 142   // Make sure the start and end are aligned with huge page address
 143   const size_t HugePageBytes = 2L * 1024 * 1024;
 144   uint8_t *From = HotStart - ((intptr_t)HotStart & (HugePageBytes - 1));
 145   uint8_t *To = HotEnd + (HugePageBytes - 1);
 146   To -= (intptr_t)To & (HugePageBytes - 1);
 147
 148   DEBUG(reportNumber("[hugify] hot start: ", (uint64_t)HotStart, 16);)
 149   DEBUG(reportNumber("[hugify] hot end: ", (uint64_t)HotEnd, 16);)
 150   DEBUG(reportNumber("[hugify] aligned huge page from: ", (uint64_t)From, 16);)
 151   DEBUG(reportNumber("[hugify] aligned huge page to: ", (uint64_t)To, 16);)
 152
 153   if (!hasPagecacheTHPSupport()) {
 154     DEBUG(report(
 155               "[hugify] workaround with memory alignment for kernel < 5.10\n");)
 156     hugifyForOldKernel(From, To);
 157     return;
 158   }
 159
 160   if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
 161     char Msg[] = "[hugify] failed to allocate large page\n";
 162     // TODO: allow user to control the failure behavior.
 163     reportError(Msg, sizeof(Msg));
 164   }
 165 }
 166
 167 /// This is hooking ELF's entry, it needs to save all machine state.
 168 extern "C" __attribute((naked)) void __bolt_hugify_self() {
 169 #if defined(__x86_64__)
 170   __asm__ __volatile__(SAVE_ALL "call __bolt_hugify_self_impl\n" RESTORE_ALL
 171                                 "jmp __bolt_hugify_start_program\n" ::
 172                                     :);
 173 #else
 174   exit(1);
 175 #endif
 176 }
 177 #endif