llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp

   1 //===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
  10
  11 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
  12 #include "llvm/Support/WindowsError.h"
  13
  14 #include <algorithm>
  15
  16 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
  17 #include <fcntl.h>
  18 #include <sys/mman.h>
  19 #include <unistd.h>
  20 #elif defined(_WIN32)
  21 #include <windows.h>
  22 #endif
  23
  24 namespace llvm {
  25 namespace orc {
  26
  27 MemoryMapper::~MemoryMapper() {}
  28
  29 InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize)
  30     : PageSize(PageSize) {}
  31
  32 Expected<std::unique_ptr<InProcessMemoryMapper>>
  33 InProcessMemoryMapper::Create() {
  34   auto PageSize = sys::Process::getPageSize();
  35   if (!PageSize)
  36     return PageSize.takeError();
  37   return std::make_unique<InProcessMemoryMapper>(*PageSize);
  38 }
  39
  40 void InProcessMemoryMapper::reserve(size_t NumBytes,
  41                                     OnReservedFunction OnReserved) {
  42   std::error_code EC;
  43   auto MB = sys::Memory::allocateMappedMemory(
  44       NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
  45
  46   if (EC)
  47     return OnReserved(errorCodeToError(EC));
  48
  49   {
  50     std::lock_guard<std::mutex> Lock(Mutex);
  51     Reservations[MB.base()].Size = MB.allocatedSize();
  52   }
  53
  54   OnReserved(
  55       ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize()));
  56 }
  57
  58 char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
  59   return Addr.toPtr<char *>();
  60 }
  61
  62 void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
  63                                        OnInitializedFunction OnInitialized) {
  64   ExecutorAddr MinAddr(~0ULL);
  65   ExecutorAddr MaxAddr(0);
  66
  67   // FIXME: Release finalize lifetime segments.
  68   for (auto &Segment : AI.Segments) {
  69     auto Base = AI.MappingBase + Segment.Offset;
  70     auto Size = Segment.ContentSize + Segment.ZeroFillSize;
  71
  72     if (Base < MinAddr)
  73       MinAddr = Base;
  74
  75     if (Base + Size > MaxAddr)
  76       MaxAddr = Base + Size;
  77
  78     std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0,
  79                 Segment.ZeroFillSize);
  80
  81     if (auto EC = sys::Memory::protectMappedMemory(
  82             {Base.toPtr<void *>(), Size},
  83             toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) {
  84       return OnInitialized(errorCodeToError(EC));
  85     }
  86     if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
  87       sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size);
  88   }
  89
  90   auto DeinitializeActions = shared::runFinalizeActions(AI.Actions);
  91   if (!DeinitializeActions)
  92     return OnInitialized(DeinitializeActions.takeError());
  93
  94   {
  95     std::lock_guard<std::mutex> Lock(Mutex);
  96
  97     // This is the maximum range whose permission have been possibly modified
  98     Allocations[MinAddr].Size = MaxAddr - MinAddr;
  99     Allocations[MinAddr].DeinitializationActions =
 100         std::move(*DeinitializeActions);
 101     Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr);
 102   }
 103
 104   OnInitialized(MinAddr);
 105 }
 106
 107 void InProcessMemoryMapper::deinitialize(
 108     ArrayRef<ExecutorAddr> Bases,
 109     MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
 110   Error AllErr = Error::success();
 111
 112   {
 113     std::lock_guard<std::mutex> Lock(Mutex);
 114
 115     for (auto Base : llvm::reverse(Bases)) {
 116
 117       if (Error Err = shared::runDeallocActions(
 118               Allocations[Base].DeinitializationActions)) {
 119         AllErr = joinErrors(std::move(AllErr), std::move(Err));
 120       }
 121
 122       // Reset protections to read/write so the area can be reused
 123       if (auto EC = sys::Memory::protectMappedMemory(
 124               {Base.toPtr<void *>(), Allocations[Base].Size},
 125               sys::Memory::ProtectionFlags::MF_READ |
 126                   sys::Memory::ProtectionFlags::MF_WRITE)) {
 127         AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC));
 128       }
 129
 130       Allocations.erase(Base);
 131     }
 132   }
 133
 134   OnDeinitialized(std::move(AllErr));
 135 }
 136
 137 void InProcessMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
 138                                     OnReleasedFunction OnReleased) {
 139   Error Err = Error::success();
 140
 141   for (auto Base : Bases) {
 142     std::vector<ExecutorAddr> AllocAddrs;
 143     size_t Size;
 144     {
 145       std::lock_guard<std::mutex> Lock(Mutex);
 146       auto &R = Reservations[Base.toPtr<void *>()];
 147       Size = R.Size;
 148       AllocAddrs.swap(R.Allocations);
 149     }
 150
 151     // deinitialize sub allocations
 152     std::promise<MSVCPError> P;
 153     auto F = P.get_future();
 154     deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
 155     if (Error E = F.get()) {
 156       Err = joinErrors(std::move(Err), std::move(E));
 157     }
 158
 159     // free the memory
 160     auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
 161
 162     auto EC = sys::Memory::releaseMappedMemory(MB);
 163     if (EC) {
 164       Err = joinErrors(std::move(Err), errorCodeToError(EC));
 165     }
 166
 167     std::lock_guard<std::mutex> Lock(Mutex);
 168     Reservations.erase(Base.toPtr<void *>());
 169   }
 170
 171   OnReleased(std::move(Err));
 172 }
 173
 174 InProcessMemoryMapper::~InProcessMemoryMapper() {
 175   std::vector<ExecutorAddr> ReservationAddrs;
 176   {
 177     std::lock_guard<std::mutex> Lock(Mutex);
 178
 179     ReservationAddrs.reserve(Reservations.size());
 180     for (const auto &R : Reservations) {
 181       ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
 182     }
 183   }
 184
 185   std::promise<MSVCPError> P;
 186   auto F = P.get_future();
 187   release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
 188   cantFail(F.get());
 189 }
 190
 191 // SharedMemoryMapper
 192
 193 SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC,
 194                                        SymbolAddrs SAs, size_t PageSize)
 195     : EPC(EPC), SAs(SAs), PageSize(PageSize) {
 196 #if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
 197   llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
 198 #endif
 199 }
 200
 201 Expected<std::unique_ptr<SharedMemoryMapper>>
 202 SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) {
 203 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 204   auto PageSize = sys::Process::getPageSize();
 205   if (!PageSize)
 206     return PageSize.takeError();
 207
 208   return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize);
 209 #else
 210   return make_error<StringError>(
 211       "SharedMemoryMapper is not supported on this platform yet",
 212       inconvertibleErrorCode());
 213 #endif
 214 }
 215
 216 void SharedMemoryMapper::reserve(size_t NumBytes,
 217                                  OnReservedFunction OnReserved) {
 218 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 219
 220   EPC.callSPSWrapperAsync<
 221       rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
 222       SAs.Reserve,
 223       [this, NumBytes, OnReserved = std::move(OnReserved)](
 224           Error SerializationErr,
 225           Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
 226         if (SerializationErr) {
 227           cantFail(Result.takeError());
 228           return OnReserved(std::move(SerializationErr));
 229         }
 230
 231         if (!Result)
 232           return OnReserved(Result.takeError());
 233
 234         ExecutorAddr RemoteAddr;
 235         std::string SharedMemoryName;
 236         std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
 237
 238         void *LocalAddr = nullptr;
 239
 240 #if defined(LLVM_ON_UNIX)
 241
 242         int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
 243         if (SharedMemoryFile < 0) {
 244           return OnReserved(errorCodeToError(
 245               std::error_code(errno, std::generic_category())));
 246         }
 247
 248         // this prevents other processes from accessing it by name
 249         shm_unlink(SharedMemoryName.c_str());
 250
 251         LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
 252                          SharedMemoryFile, 0);
 253         if (LocalAddr == MAP_FAILED) {
 254           return OnReserved(errorCodeToError(
 255               std::error_code(errno, std::generic_category())));
 256         }
 257
 258         close(SharedMemoryFile);
 259
 260 #elif defined(_WIN32)
 261
 262         std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
 263                                           SharedMemoryName.end());
 264         HANDLE SharedMemoryFile = OpenFileMappingW(
 265             FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
 266         if (!SharedMemoryFile)
 267           return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
 268
 269         LocalAddr =
 270             MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
 271         if (!LocalAddr) {
 272           CloseHandle(SharedMemoryFile);
 273           return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
 274         }
 275
 276         CloseHandle(SharedMemoryFile);
 277
 278 #endif
 279         {
 280           std::lock_guard<std::mutex> Lock(Mutex);
 281           Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
 282         }
 283
 284         OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
 285       },
 286       SAs.Instance, static_cast<uint64_t>(NumBytes));
 287
 288 #else
 289   OnReserved(make_error<StringError>(
 290       "SharedMemoryMapper is not supported on this platform yet",
 291       inconvertibleErrorCode()));
 292 #endif
 293 }
 294
 295 char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
 296   auto R = Reservations.upper_bound(Addr);
 297   assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
 298   R--;
 299
 300   ExecutorAddrDiff Offset = Addr - R->first;
 301
 302   return static_cast<char *>(R->second.LocalAddr) + Offset;
 303 }
 304
 305 void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
 306                                     OnInitializedFunction OnInitialized) {
 307   auto Reservation = Reservations.upper_bound(AI.MappingBase);
 308   assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
 309   Reservation--;
 310
 311   auto AllocationOffset = AI.MappingBase - Reservation->first;
 312
 313   tpctypes::SharedMemoryFinalizeRequest FR;
 314
 315   AI.Actions.swap(FR.Actions);
 316
 317   FR.Segments.reserve(AI.Segments.size());
 318
 319   for (auto Segment : AI.Segments) {
 320     char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
 321                  AllocationOffset + Segment.Offset;
 322     std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
 323
 324     tpctypes::SharedMemorySegFinalizeRequest SegReq;
 325     SegReq.RAG = {Segment.AG.getMemProt(),
 326                   Segment.AG.getMemLifetime() == MemLifetime::Finalize};
 327     SegReq.Addr = AI.MappingBase + Segment.Offset;
 328     SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
 329
 330     FR.Segments.push_back(SegReq);
 331   }
 332
 333   EPC.callSPSWrapperAsync<
 334       rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
 335       SAs.Initialize,
 336       [OnInitialized = std::move(OnInitialized)](
 337           Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
 338         if (SerializationErr) {
 339           cantFail(Result.takeError());
 340           return OnInitialized(std::move(SerializationErr));
 341         }
 342
 343         OnInitialized(std::move(Result));
 344       },
 345       SAs.Instance, Reservation->first, std::move(FR));
 346 }
 347
 348 void SharedMemoryMapper::deinitialize(
 349     ArrayRef<ExecutorAddr> Allocations,
 350     MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
 351   EPC.callSPSWrapperAsync<
 352       rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
 353       SAs.Deinitialize,
 354       [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
 355                                                      Error Result) mutable {
 356         if (SerializationErr) {
 357           cantFail(std::move(Result));
 358           return OnDeinitialized(std::move(SerializationErr));
 359         }
 360
 361         OnDeinitialized(std::move(Result));
 362       },
 363       SAs.Instance, Allocations);
 364 }
 365
 366 void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
 367                                  OnReleasedFunction OnReleased) {
 368 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 369   Error Err = Error::success();
 370
 371   {
 372     std::lock_guard<std::mutex> Lock(Mutex);
 373
 374     for (auto Base : Bases) {
 375
 376 #if defined(LLVM_ON_UNIX)
 377
 378       if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
 379         Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
 380                                              errno, std::generic_category())));
 381
 382 #elif defined(_WIN32)
 383
 384       if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
 385         Err = joinErrors(std::move(Err),
 386                          errorCodeToError(mapWindowsError(GetLastError())));
 387
 388 #endif
 389
 390       Reservations.erase(Base);
 391     }
 392   }
 393
 394   EPC.callSPSWrapperAsync<
 395       rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
 396       SAs.Release,
 397       [OnReleased = std::move(OnReleased),
 398        Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
 399         if (SerializationErr) {
 400           cantFail(std::move(Result));
 401           return OnReleased(
 402               joinErrors(std::move(Err), std::move(SerializationErr)));
 403         }
 404
 405         return OnReleased(joinErrors(std::move(Err), std::move(Result)));
 406       },
 407       SAs.Instance, Bases);
 408 #else
 409   OnReleased(make_error<StringError>(
 410       "SharedMemoryMapper is not supported on this platform yet",
 411       inconvertibleErrorCode()));
 412 #endif
 413 }
 414
 415 SharedMemoryMapper::~SharedMemoryMapper() {
 416   std::lock_guard<std::mutex> Lock(Mutex);
 417   for (const auto &R : Reservations) {
 418
 419 #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
 420
 421     munmap(R.second.LocalAddr, R.second.Size);
 422
 423 #elif defined(_WIN32)
 424
 425     UnmapViewOfFile(R.second.LocalAddr);
 426
 427 #else
 428
 429     (void)R;
 430
 431 #endif
 432   }
 433 }
 434
 435 } // namespace orc
 436
 437 } // namespace llvm