llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp

   1 //===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
  10 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
  11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
  12 #include "llvm/Support/Process.h"
  13 #include "llvm/Support/WindowsError.h"
  14 #include <sstream>
  15
  16 #if defined(LLVM_ON_UNIX)
  17 #include <errno.h>
  18 #include <fcntl.h>
  19 #include <sys/mman.h>
  20 #if defined(__MVS__)
  21 #include "llvm/Support/BLAKE3.h"
  22 #include <sys/shm.h>
  23 #endif
  24 #include <unistd.h>
  25 #endif
  26
  27 namespace llvm {
  28 namespace orc {
  29 namespace rt_bootstrap {
  30
  31 #if defined(_WIN32)
  32 static DWORD getWindowsProtectionFlags(MemProt MP) {
  33   if (MP == MemProt::Read)
  34     return PAGE_READONLY;
  35   if (MP == MemProt::Write ||
  36       MP == (MemProt::Write | MemProt::Read)) {
  37     // Note: PAGE_WRITE is not supported by VirtualProtect
  38     return PAGE_READWRITE;
  39   }
  40   if (MP == (MemProt::Read | MemProt::Exec))
  41     return PAGE_EXECUTE_READ;
  42   if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
  43     return PAGE_EXECUTE_READWRITE;
  44   if (MP == MemProt::Exec)
  45     return PAGE_EXECUTE;
  46
  47   return PAGE_NOACCESS;
  48 }
  49 #endif
  50
  51 Expected<std::pair<ExecutorAddr, std::string>>
  52 ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
  53 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
  54
  55 #if defined(LLVM_ON_UNIX)
  56
  57   std::string SharedMemoryName;
  58   {
  59     std::stringstream SharedMemoryNameStream;
  60     SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
  61                            << (++SharedMemoryCount);
  62     SharedMemoryName = SharedMemoryNameStream.str();
  63   }
  64
  65 #if defined(__MVS__)
  66   ArrayRef<uint8_t> Data(
  67       reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
  68       SharedMemoryName.size());
  69   auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
  70   key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
  71   int SharedMemoryId =
  72       shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
  73   if (SharedMemoryId < 0)
  74     return errorCodeToError(errnoAsErrorCode());
  75
  76   void *Addr = shmat(SharedMemoryId, nullptr, 0);
  77   if (Addr == reinterpret_cast<void *>(-1))
  78     return errorCodeToError(errnoAsErrorCode());
  79 #else
  80   int SharedMemoryFile =
  81       shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
  82   if (SharedMemoryFile < 0)
  83     return errorCodeToError(errnoAsErrorCode());
  84
  85   // by default size is 0
  86   if (ftruncate(SharedMemoryFile, Size) < 0)
  87     return errorCodeToError(errnoAsErrorCode());
  88
  89   void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
  90   if (Addr == MAP_FAILED)
  91     return errorCodeToError(errnoAsErrorCode());
  92
  93   close(SharedMemoryFile);
  94 #endif
  95
  96 #elif defined(_WIN32)
  97
  98   std::string SharedMemoryName;
  99   {
 100     std::stringstream SharedMemoryNameStream;
 101     SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
 102                            << (++SharedMemoryCount);
 103     SharedMemoryName = SharedMemoryNameStream.str();
 104   }
 105
 106   std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
 107                                     SharedMemoryName.end());
 108   HANDLE SharedMemoryFile = CreateFileMappingW(
 109       INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
 110       Size & 0xffffffff, WideSharedMemoryName.c_str());
 111   if (!SharedMemoryFile)
 112     return errorCodeToError(mapWindowsError(GetLastError()));
 113
 114   void *Addr = MapViewOfFile(SharedMemoryFile,
 115                              FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
 116   if (!Addr) {
 117     CloseHandle(SharedMemoryFile);
 118     return errorCodeToError(mapWindowsError(GetLastError()));
 119   }
 120
 121 #endif
 122
 123   {
 124     std::lock_guard<std::mutex> Lock(Mutex);
 125     Reservations[Addr].Size = Size;
 126 #if defined(_WIN32)
 127     Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
 128 #endif
 129   }
 130
 131   return std::make_pair(ExecutorAddr::fromPtr(Addr),
 132                         std::move(SharedMemoryName));
 133 #else
 134   return make_error<StringError>(
 135       "SharedMemoryMapper is not supported on this platform yet",
 136       inconvertibleErrorCode());
 137 #endif
 138 }
 139
 140 Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
 141     ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
 142 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 143
 144   ExecutorAddr MinAddr(~0ULL);
 145
 146   // Contents are already in place
 147   for (auto &Segment : FR.Segments) {
 148     if (Segment.Addr < MinAddr)
 149       MinAddr = Segment.Addr;
 150
 151 #if defined(LLVM_ON_UNIX)
 152
 153 #if defined(__MVS__)
 154       // TODO Is it possible to change the protection level?
 155 #else
 156     int NativeProt = 0;
 157     if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
 158       NativeProt |= PROT_READ;
 159     if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
 160       NativeProt |= PROT_WRITE;
 161     if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
 162       NativeProt |= PROT_EXEC;
 163
 164     if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
 165       return errorCodeToError(errnoAsErrorCode());
 166 #endif
 167
 168 #elif defined(_WIN32)
 169
 170     DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
 171
 172     if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
 173                         &NativeProt))
 174       return errorCodeToError(mapWindowsError(GetLastError()));
 175
 176 #endif
 177
 178     if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
 179       sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
 180                                               Segment.Size);
 181   }
 182
 183   // Run finalization actions and get deinitlization action list.
 184   auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);
 185   if (!DeinitializeActions) {
 186     return DeinitializeActions.takeError();
 187   }
 188
 189   {
 190     std::lock_guard<std::mutex> Lock(Mutex);
 191     Allocations[MinAddr].DeinitializationActions =
 192         std::move(*DeinitializeActions);
 193     Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
 194   }
 195
 196   return MinAddr;
 197
 198 #else
 199   return make_error<StringError>(
 200       "SharedMemoryMapper is not supported on this platform yet",
 201       inconvertibleErrorCode());
 202 #endif
 203 }
 204
 205 Error ExecutorSharedMemoryMapperService::deinitialize(
 206     const std::vector<ExecutorAddr> &Bases) {
 207   Error AllErr = Error::success();
 208
 209   {
 210     std::lock_guard<std::mutex> Lock(Mutex);
 211
 212     for (auto Base : llvm::reverse(Bases)) {
 213       if (Error Err = shared::runDeallocActions(
 214               Allocations[Base].DeinitializationActions)) {
 215         AllErr = joinErrors(std::move(AllErr), std::move(Err));
 216       }
 217
 218       // Remove the allocation from the allocation list of its reservation
 219       for (auto &Reservation : Reservations) {
 220         auto AllocationIt = llvm::find(Reservation.second.Allocations, Base);
 221         if (AllocationIt != Reservation.second.Allocations.end()) {
 222           Reservation.second.Allocations.erase(AllocationIt);
 223           break;
 224         }
 225       }
 226
 227       Allocations.erase(Base);
 228     }
 229   }
 230
 231   return AllErr;
 232 }
 233
 234 Error ExecutorSharedMemoryMapperService::release(
 235     const std::vector<ExecutorAddr> &Bases) {
 236 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 237   Error Err = Error::success();
 238
 239   for (auto Base : Bases) {
 240     std::vector<ExecutorAddr> AllocAddrs;
 241     size_t Size;
 242
 243 #if defined(_WIN32)
 244     HANDLE SharedMemoryFile;
 245 #endif
 246
 247     {
 248       std::lock_guard<std::mutex> Lock(Mutex);
 249       auto &R = Reservations[Base.toPtr<void *>()];
 250       Size = R.Size;
 251
 252 #if defined(_WIN32)
 253       SharedMemoryFile = R.SharedMemoryFile;
 254 #endif
 255
 256       AllocAddrs.swap(R.Allocations);
 257     }
 258
 259     // deinitialize sub allocations
 260     if (Error E = deinitialize(AllocAddrs))
 261       Err = joinErrors(std::move(Err), std::move(E));
 262
 263 #if defined(LLVM_ON_UNIX)
 264
 265 #if defined(__MVS__)
 266     (void)Size;
 267
 268     if (shmdt(Base.toPtr<void *>()) < 0)
 269       Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
 270 #else
 271     if (munmap(Base.toPtr<void *>(), Size) != 0)
 272       Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
 273 #endif
 274
 275 #elif defined(_WIN32)
 276     (void)Size;
 277
 278     if (!UnmapViewOfFile(Base.toPtr<void *>()))
 279       Err = joinErrors(std::move(Err),
 280                        errorCodeToError(mapWindowsError(GetLastError())));
 281
 282     CloseHandle(SharedMemoryFile);
 283
 284 #endif
 285
 286     std::lock_guard<std::mutex> Lock(Mutex);
 287     Reservations.erase(Base.toPtr<void *>());
 288   }
 289
 290   return Err;
 291 #else
 292   return make_error<StringError>(
 293       "SharedMemoryMapper is not supported on this platform yet",
 294       inconvertibleErrorCode());
 295 #endif
 296 }
 297
 298 Error ExecutorSharedMemoryMapperService::shutdown() {
 299   if (Reservations.empty())
 300     return Error::success();
 301
 302   std::vector<ExecutorAddr> ReservationAddrs;
 303   ReservationAddrs.reserve(Reservations.size());
 304   for (const auto &R : Reservations)
 305     ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
 306
 307   return release(std::move(ReservationAddrs));
 308 }
 309
 310 void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
 311     StringMap<ExecutorAddr> &M) {
 312   M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
 313       ExecutorAddr::fromPtr(this);
 314   M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
 315       ExecutorAddr::fromPtr(&reserveWrapper);
 316   M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
 317       ExecutorAddr::fromPtr(&initializeWrapper);
 318   M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
 319       ExecutorAddr::fromPtr(&deinitializeWrapper);
 320   M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
 321       ExecutorAddr::fromPtr(&releaseWrapper);
 322 }
 323
 324 llvm::orc::shared::CWrapperFunctionResult
 325 ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
 326                                                   size_t ArgSize) {
 327   return shared::WrapperFunction<
 328              rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
 329       handle(ArgData, ArgSize,
 330              shared::makeMethodWrapperHandler(
 331                  &ExecutorSharedMemoryMapperService::reserve))
 332           .release();
 333 }
 334
 335 llvm::orc::shared::CWrapperFunctionResult
 336 ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
 337                                                      size_t ArgSize) {
 338   return shared::WrapperFunction<
 339              rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
 340       handle(ArgData, ArgSize,
 341              shared::makeMethodWrapperHandler(
 342                  &ExecutorSharedMemoryMapperService::initialize))
 343           .release();
 344 }
 345
 346 llvm::orc::shared::CWrapperFunctionResult
 347 ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
 348                                                        size_t ArgSize) {
 349   return shared::WrapperFunction<
 350              rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
 351       handle(ArgData, ArgSize,
 352              shared::makeMethodWrapperHandler(
 353                  &ExecutorSharedMemoryMapperService::deinitialize))
 354           .release();
 355 }
 356
 357 llvm::orc::shared::CWrapperFunctionResult
 358 ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
 359                                                   size_t ArgSize) {
 360   return shared::WrapperFunction<
 361              rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
 362       handle(ArgData, ArgSize,
 363              shared::makeMethodWrapperHandler(
 364                  &ExecutorSharedMemoryMapperService::release))
 365           .release();
 366 }
 367
 368 } // namespace rt_bootstrap
 369 } // end namespace orc
 370 } // end namespace llvm