openmp/libomptarget/src/device.cpp

   1 //===--------- device.cpp - Target independent OpenMP target RTL ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Functionality for managing devices that are handled by RTL plugins.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "device.h"
  14 #include "private.h"
  15 #include "rtl.h"
  16
  17 #include <cassert>
  18 #include <climits>
  19 #include <cstdio>
  20 #include <string>
  21
  22 DeviceTy::DeviceTy(RTLInfoTy *RTL)
  23     : DeviceID(-1), RTL(RTL), RTLDeviceID(-1), IsInit(false), InitFlag(),
  24       HasPendingGlobals(false), HostDataToTargetMap(), PendingCtorsDtors(),
  25       ShadowPtrMap(), DataMapMtx(), PendingGlobalsMtx(), ShadowMtx() {}
  26
  27 DeviceTy::~DeviceTy() {
  28   if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))
  29     return;
  30
  31   ident_t loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"};
  32   dumpTargetPointerMappings(&loc, *this);
  33 }
  34
  35 int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
  36   DataMapMtx.lock();
  37
  38   // Check if entry exists
  39   auto search = HostDataToTargetMap.find(HstPtrBeginTy{(uintptr_t)HstPtrBegin});
  40   if (search != HostDataToTargetMap.end()) {
  41     // Mapping already exists
  42     bool isValid = search->HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
  43                    search->TgtPtrBegin == (uintptr_t)TgtPtrBegin;
  44     DataMapMtx.unlock();
  45     if (isValid) {
  46       DP("Attempt to re-associate the same device ptr+offset with the same "
  47          "host ptr, nothing to do\n");
  48       return OFFLOAD_SUCCESS;
  49     } else {
  50       REPORT("Not allowed to re-associate a different device ptr+offset with "
  51              "the same host ptr\n");
  52       return OFFLOAD_FAIL;
  53     }
  54   }
  55
  56   // Mapping does not exist, allocate it with refCount=INF
  57   const HostDataToTargetTy &newEntry =
  58       *HostDataToTargetMap
  59            .emplace(
  60                /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
  61                /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
  62                /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
  63                /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
  64                /*UseHoldRefCount=*/false, /*Name=*/nullptr,
  65                /*IsRefCountINF=*/true)
  66            .first;
  67   DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
  68      ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
  69      "HoldRefCount=%s\n",
  70      DPxPTR(newEntry.HstPtrBase), DPxPTR(newEntry.HstPtrBegin),
  71      DPxPTR(newEntry.HstPtrEnd), DPxPTR(newEntry.TgtPtrBegin),
  72      newEntry.dynRefCountToStr().c_str(), newEntry.holdRefCountToStr().c_str());
  73   (void)newEntry;
  74
  75   DataMapMtx.unlock();
  76
  77   return OFFLOAD_SUCCESS;
  78 }
  79
  80 int DeviceTy::disassociatePtr(void *HstPtrBegin) {
  81   DataMapMtx.lock();
  82
  83   auto search = HostDataToTargetMap.find(HstPtrBeginTy{(uintptr_t)HstPtrBegin});
  84   if (search != HostDataToTargetMap.end()) {
  85     // Mapping exists
  86     if (search->getHoldRefCount()) {
  87       // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
  88       // "It is an error to call acc_unmap_data if the structured reference
  89       // count for the pointer is not zero."
  90       REPORT("Trying to disassociate a pointer with a non-zero hold reference "
  91              "count\n");
  92     } else if (search->isDynRefCountInf()) {
  93       DP("Association found, removing it\n");
  94       void *Event = search->getEvent();
  95       if (Event)
  96         destroyEvent(Event);
  97       HostDataToTargetMap.erase(search);
  98       DataMapMtx.unlock();
  99       return OFFLOAD_SUCCESS;
 100     } else {
 101       REPORT("Trying to disassociate a pointer which was not mapped via "
 102              "omp_target_associate_ptr\n");
 103     }
 104   } else {
 105     REPORT("Association not found\n");
 106   }
 107
 108   // Mapping not found
 109   DataMapMtx.unlock();
 110   return OFFLOAD_FAIL;
 111 }
 112
 113 LookupResult DeviceTy::lookupMapping(void *HstPtrBegin, int64_t Size) {
 114   uintptr_t hp = (uintptr_t)HstPtrBegin;
 115   LookupResult lr;
 116
 117   DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
 118      DPxPTR(hp), Size);
 119
 120   if (HostDataToTargetMap.empty())
 121     return lr;
 122
 123   auto upper = HostDataToTargetMap.upper_bound(hp);
 124   // check the left bin
 125   if (upper != HostDataToTargetMap.begin()) {
 126     lr.Entry = std::prev(upper);
 127     auto &HT = *lr.Entry;
 128     // Is it contained?
 129     lr.Flags.IsContained = hp >= HT.HstPtrBegin && hp < HT.HstPtrEnd &&
 130                            (hp + Size) <= HT.HstPtrEnd;
 131     // Does it extend beyond the mapped region?
 132     lr.Flags.ExtendsAfter = hp < HT.HstPtrEnd && (hp + Size) > HT.HstPtrEnd;
 133   }
 134
 135   // check the right bin
 136   if (!(lr.Flags.IsContained || lr.Flags.ExtendsAfter) &&
 137       upper != HostDataToTargetMap.end()) {
 138     lr.Entry = upper;
 139     auto &HT = *lr.Entry;
 140     // Does it extend into an already mapped region?
 141     lr.Flags.ExtendsBefore =
 142         hp < HT.HstPtrBegin && (hp + Size) > HT.HstPtrBegin;
 143     // Does it extend beyond the mapped region?
 144     lr.Flags.ExtendsAfter = hp < HT.HstPtrEnd && (hp + Size) > HT.HstPtrEnd;
 145   }
 146
 147   if (lr.Flags.ExtendsBefore) {
 148     DP("WARNING: Pointer is not mapped but section extends into already "
 149        "mapped data\n");
 150   }
 151   if (lr.Flags.ExtendsAfter) {
 152     DP("WARNING: Pointer is already mapped but section extends beyond mapped "
 153        "region\n");
 154   }
 155
 156   return lr;
 157 }
 158
 159 TargetPointerResultTy
 160 DeviceTy::getTargetPointer(void *HstPtrBegin, void *HstPtrBase, int64_t Size,
 161                            map_var_info_t HstPtrName, bool HasFlagTo,
 162                            bool HasFlagAlways, bool IsImplicit,
 163                            bool UpdateRefCount, bool HasCloseModifier,
 164                            bool HasPresentModifier, bool HasHoldModifier,
 165                            AsyncInfoTy &AsyncInfo) {
 166   void *TargetPointer = nullptr;
 167   bool IsHostPtr = false;
 168   bool IsNew = false;
 169
 170   DataMapMtx.lock();
 171
 172   LookupResult LR = lookupMapping(HstPtrBegin, Size);
 173   auto Entry = LR.Entry;
 174
 175   // Check if the pointer is contained.
 176   // If a variable is mapped to the device manually by the user - which would
 177   // lead to the IsContained flag to be true - then we must ensure that the
 178   // device address is returned even under unified memory conditions.
 179   if (LR.Flags.IsContained ||
 180       ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
 181     auto &HT = *LR.Entry;
 182     const char *RefCountAction;
 183     assert(HT.getTotalRefCount() > 0 && "expected existing RefCount > 0");
 184     if (UpdateRefCount) {
 185       // After this, RefCount > 1.
 186       HT.incRefCount(HasHoldModifier);
 187       RefCountAction = " (incremented)";
 188     } else {
 189       // It might have been allocated with the parent, but it's still new.
 190       IsNew = HT.getTotalRefCount() == 1;
 191       RefCountAction = " (update suppressed)";
 192     }
 193     const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
 194     const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
 195     uintptr_t Ptr = HT.TgtPtrBegin + ((uintptr_t)HstPtrBegin - HT.HstPtrBegin);
 196     INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
 197          "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
 198          ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
 199          (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
 200          Size, HT.dynRefCountToStr().c_str(), DynRefCountAction,
 201          HT.holdRefCountToStr().c_str(), HoldRefCountAction,
 202          (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
 203     TargetPointer = (void *)Ptr;
 204   } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
 205     // Explicit extension of mapped data - not allowed.
 206     MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
 207             " (%" PRId64
 208             " bytes), but device allocation maps to host at " DPxMOD
 209             " (%" PRId64 " bytes)",
 210             DPxPTR(HstPtrBegin), Size, DPxPTR(Entry->HstPtrBegin),
 211             Entry->HstPtrEnd - Entry->HstPtrBegin);
 212     if (HasPresentModifier)
 213       MESSAGE("device mapping required by 'present' map type modifier does not "
 214               "exist for host address " DPxMOD " (%" PRId64 " bytes)",
 215               DPxPTR(HstPtrBegin), Size);
 216   } else if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY &&
 217              !HasCloseModifier) {
 218     // If unified shared memory is active, implicitly mapped variables that are
 219     // not privatized use host address. Any explicitly mapped variables also use
 220     // host address where correctness is not impeded. In all other cases maps
 221     // are respected.
 222     // In addition to the mapping rules above, the close map modifier forces the
 223     // mapping of the variable to the device.
 224     if (Size) {
 225       DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
 226          "memory\n",
 227          DPxPTR((uintptr_t)HstPtrBegin), Size);
 228       IsHostPtr = true;
 229       TargetPointer = HstPtrBegin;
 230     }
 231   } else if (HasPresentModifier) {
 232     DP("Mapping required by 'present' map type modifier does not exist for "
 233        "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
 234        DPxPTR(HstPtrBegin), Size);
 235     MESSAGE("device mapping required by 'present' map type modifier does not "
 236             "exist for host address " DPxMOD " (%" PRId64 " bytes)",
 237             DPxPTR(HstPtrBegin), Size);
 238   } else if (Size) {
 239     // If it is not contained and Size > 0, we should create a new entry for it.
 240     IsNew = true;
 241     uintptr_t Ptr = (uintptr_t)allocData(Size, HstPtrBegin);
 242     Entry = HostDataToTargetMap
 243                 .emplace((uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
 244                          (uintptr_t)HstPtrBegin + Size, Ptr, HasHoldModifier,
 245                          HstPtrName)
 246                 .first;
 247     INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
 248          "Creating new map entry with "
 249          "HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", Size=%ld, "
 250          "DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
 251          DPxPTR(HstPtrBegin), DPxPTR(Ptr), Size,
 252          Entry->dynRefCountToStr().c_str(), Entry->holdRefCountToStr().c_str(),
 253          (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
 254     TargetPointer = (void *)Ptr;
 255   }
 256
 257   // If the target pointer is valid, and we need to transfer data, issue the
 258   // data transfer.
 259   if (TargetPointer && !IsHostPtr && HasFlagTo && (IsNew || HasFlagAlways)) {
 260     // Lock the entry before releasing the mapping table lock such that another
 261     // thread that could issue data movement will get the right result.
 262     Entry->lock();
 263     // Release the mapping table lock right after the entry is locked.
 264     DataMapMtx.unlock();
 265
 266     DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
 267        DPxPTR(HstPtrBegin), DPxPTR(TargetPointer));
 268
 269     int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo);
 270     if (Ret != OFFLOAD_SUCCESS) {
 271       Entry->unlock();
 272       REPORT("Copying data to device failed.\n");
 273       // We will also return nullptr if the data movement fails because that
 274       // pointer points to a corrupted memory region so it doesn't make any
 275       // sense to continue to use it.
 276       TargetPointer = nullptr;
 277     }
 278
 279     void *Event = Entry->getEvent();
 280     bool NeedNewEvent = Event == nullptr;
 281     if (NeedNewEvent && createEvent(&Event) != OFFLOAD_SUCCESS) {
 282       Entry->unlock();
 283       REPORT("Failed to create event\n");
 284       return {{false /* IsNewEntry */, false /* IsHostPointer */},
 285               {} /* MapTableEntry */,
 286               nullptr /* TargetPointer */};
 287     }
 288     // We cannot assume the event should not be nullptr because we don't
 289     // know if the target support event. But if a target doesn't,
 290     // recordEvent should always return success.
 291     Ret = recordEvent(Event, AsyncInfo);
 292     if (Ret != OFFLOAD_SUCCESS) {
 293       Entry->unlock();
 294       REPORT("Failed to set dependence on event " DPxMOD "\n", DPxPTR(Event));
 295       return {{false /* IsNewEntry */, false /* IsHostPointer */},
 296               {} /* MapTableEntry */,
 297               nullptr /* TargetPointer */};
 298     }
 299     if (NeedNewEvent)
 300       Entry->setEvent(Event);
 301     // We're done with the entry. Release the entry.
 302     Entry->unlock();
 303   } else {
 304     // Release the mapping table lock directly.
 305     DataMapMtx.unlock();
 306     // If not a host pointer and no present modifier, we need to wait for the
 307     // event if it exists.
 308     // Note: Entry might be nullptr because of zero length array section.
 309     if (Entry != HostDataToTargetListTy::iterator() && !IsHostPtr &&
 310         !HasPresentModifier) {
 311       Entry->lock();
 312       void *Event = Entry->getEvent();
 313       if (Event) {
 314         int Ret = waitEvent(Event, AsyncInfo);
 315         Entry->unlock();
 316         if (Ret != OFFLOAD_SUCCESS) {
 317           // If it fails to wait for the event, we need to return nullptr in
 318           // case of any data race.
 319           REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
 320           return {{false /* IsNewEntry */, false /* IsHostPointer */},
 321                   {} /* MapTableEntry */,
 322                   nullptr /* TargetPointer */};
 323         }
 324       } else {
 325         Entry->unlock();
 326       }
 327     }
 328   }
 329
 330   return {{IsNew, IsHostPtr}, Entry, TargetPointer};
 331 }
 332
 333 // Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
 334 // Return the target pointer begin (where the data will be moved).
 335 // Decrement the reference counter if called from targetDataEnd.
 336 void *DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast,
 337                                bool UpdateRefCount, bool UseHoldRefCount,
 338                                bool &IsHostPtr, bool MustContain,
 339                                bool ForceDelete) {
 340   void *rc = NULL;
 341   IsHostPtr = false;
 342   IsLast = false;
 343   DataMapMtx.lock();
 344   LookupResult lr = lookupMapping(HstPtrBegin, Size);
 345
 346   if (lr.Flags.IsContained ||
 347       (!MustContain && (lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter))) {
 348     auto &HT = *lr.Entry;
 349     // We do not zero the total reference count here.  deallocTgtPtr does that
 350     // atomically with removing the mapping.  Otherwise, before this thread
 351     // removed the mapping in deallocTgtPtr, another thread could retrieve the
 352     // mapping, increment and decrement back to zero, and then both threads
 353     // would try to remove the mapping, resulting in a double free.
 354     IsLast = HT.decShouldRemove(UseHoldRefCount, ForceDelete);
 355     const char *RefCountAction;
 356     if (!UpdateRefCount) {
 357       RefCountAction = " (update suppressed)";
 358     } else if (ForceDelete) {
 359       HT.resetRefCount(UseHoldRefCount);
 360       assert(IsLast == HT.decShouldRemove(UseHoldRefCount) &&
 361              "expected correct IsLast prediction for reset");
 362       if (IsLast)
 363         RefCountAction = " (reset, deferred final decrement)";
 364       else {
 365         HT.decRefCount(UseHoldRefCount);
 366         RefCountAction = " (reset)";
 367       }
 368     } else if (IsLast) {
 369       RefCountAction = " (deferred final decrement)";
 370     } else {
 371       HT.decRefCount(UseHoldRefCount);
 372       RefCountAction = " (decremented)";
 373     }
 374     const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
 375     const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
 376     uintptr_t tp = HT.TgtPtrBegin + ((uintptr_t)HstPtrBegin - HT.HstPtrBegin);
 377     INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
 378          "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
 379          "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
 380          DPxPTR(HstPtrBegin), DPxPTR(tp), Size, HT.dynRefCountToStr().c_str(),
 381          DynRefCountAction, HT.holdRefCountToStr().c_str(), HoldRefCountAction);
 382     rc = (void *)tp;
 383   } else if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) {
 384     // If the value isn't found in the mapping and unified shared memory
 385     // is on then it means we have stumbled upon a value which we need to
 386     // use directly from the host.
 387     DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
 388        "memory\n",
 389        DPxPTR((uintptr_t)HstPtrBegin), Size);
 390     IsHostPtr = true;
 391     rc = HstPtrBegin;
 392   }
 393
 394   DataMapMtx.unlock();
 395   return rc;
 396 }
 397
 398 // Return the target pointer begin (where the data will be moved).
 399 // Lock-free version called when loading global symbols from the fat binary.
 400 void *DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size) {
 401   uintptr_t hp = (uintptr_t)HstPtrBegin;
 402   LookupResult lr = lookupMapping(HstPtrBegin, Size);
 403   if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) {
 404     auto &HT = *lr.Entry;
 405     uintptr_t tp = HT.TgtPtrBegin + (hp - HT.HstPtrBegin);
 406     return (void *)tp;
 407   }
 408
 409   return NULL;
 410 }
 411
 412 int DeviceTy::deallocTgtPtr(void *HstPtrBegin, int64_t Size,
 413                             bool HasHoldModifier) {
 414   // Check if the pointer is contained in any sub-nodes.
 415   int Ret = OFFLOAD_SUCCESS;
 416   DataMapMtx.lock();
 417   LookupResult lr = lookupMapping(HstPtrBegin, Size);
 418   if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) {
 419     auto &HT = *lr.Entry;
 420     if (HT.decRefCount(HasHoldModifier) == 0) {
 421       DP("Deleting tgt data " DPxMOD " of size %" PRId64 "\n",
 422          DPxPTR(HT.TgtPtrBegin), Size);
 423       deleteData((void *)HT.TgtPtrBegin);
 424       INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
 425            "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
 426            ", Size=%" PRId64 ", Name=%s\n",
 427            DPxPTR(HT.HstPtrBegin), DPxPTR(HT.TgtPtrBegin), Size,
 428            (HT.HstPtrName) ? getNameFromMapping(HT.HstPtrName).c_str()
 429                            : "unknown");
 430       void *Event = lr.Entry->getEvent();
 431       HostDataToTargetMap.erase(lr.Entry);
 432       if (Event && destroyEvent(Event) != OFFLOAD_SUCCESS) {
 433         REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
 434         Ret = OFFLOAD_FAIL;
 435       }
 436     }
 437   } else {
 438     REPORT("Section to delete (hst addr " DPxMOD ") does not exist in the"
 439            " allocated memory\n",
 440            DPxPTR(HstPtrBegin));
 441     Ret = OFFLOAD_FAIL;
 442   }
 443
 444   DataMapMtx.unlock();
 445   return Ret;
 446 }
 447
 448 /// Init device, should not be called directly.
 449 void DeviceTy::init() {
 450   // Make call to init_requires if it exists for this plugin.
 451   if (RTL->init_requires)
 452     RTL->init_requires(PM->RTLs.RequiresFlags);
 453   int32_t Ret = RTL->init_device(RTLDeviceID);
 454   if (Ret != OFFLOAD_SUCCESS)
 455     return;
 456
 457   IsInit = true;
 458 }
 459
 460 /// Thread-safe method to initialize the device only once.
 461 int32_t DeviceTy::initOnce() {
 462   std::call_once(InitFlag, &DeviceTy::init, this);
 463
 464   // At this point, if IsInit is true, then either this thread or some other
 465   // thread in the past successfully initialized the device, so we can return
 466   // OFFLOAD_SUCCESS. If this thread executed init() via call_once() and it
 467   // failed, return OFFLOAD_FAIL. If call_once did not invoke init(), it means
 468   // that some other thread already attempted to execute init() and if IsInit
 469   // is still false, return OFFLOAD_FAIL.
 470   if (IsInit)
 471     return OFFLOAD_SUCCESS;
 472   else
 473     return OFFLOAD_FAIL;
 474 }
 475
 476 // Load binary to device.
 477 __tgt_target_table *DeviceTy::load_binary(void *Img) {
 478   RTL->Mtx.lock();
 479   __tgt_target_table *rc = RTL->load_binary(RTLDeviceID, Img);
 480   RTL->Mtx.unlock();
 481   return rc;
 482 }
 483
 484 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
 485   return RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
 486 }
 487
 488 int32_t DeviceTy::deleteData(void *TgtPtrBegin) {
 489   return RTL->data_delete(RTLDeviceID, TgtPtrBegin);
 490 }
 491
 492 // Submit data to device
 493 int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
 494                              AsyncInfoTy &AsyncInfo) {
 495   if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
 496     LookupResult LR = lookupMapping(HstPtrBegin, Size);
 497     auto *HT = &*LR.Entry;
 498
 499     INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceID,
 500          "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
 501          ", Size=%" PRId64 ", Name=%s\n",
 502          DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin), Size,
 503          (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
 504                                 : "unknown");
 505   }
 506
 507   if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize)
 508     return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size);
 509   else
 510     return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
 511                                   AsyncInfo);
 512 }
 513
 514 // Retrieve data from device
 515 int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
 516                                int64_t Size, AsyncInfoTy &AsyncInfo) {
 517   if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
 518     LookupResult LR = lookupMapping(HstPtrBegin, Size);
 519     auto *HT = &*LR.Entry;
 520     INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceID,
 521          "Copying data from device to host, TgtPtr=" DPxMOD ", HstPtr=" DPxMOD
 522          ", Size=%" PRId64 ", Name=%s\n",
 523          DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin), Size,
 524          (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
 525                                 : "unknown");
 526   }
 527
 528   if (!RTL->data_retrieve_async || !RTL->synchronize)
 529     return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size);
 530   else
 531     return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
 532                                     AsyncInfo);
 533 }
 534
 535 // Copy data from current device to destination device directly
 536 int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
 537                                int64_t Size, AsyncInfoTy &AsyncInfo) {
 538   if (!AsyncInfo || !RTL->data_exchange_async || !RTL->synchronize) {
 539     assert(RTL->data_exchange && "RTL->data_exchange is nullptr");
 540     return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
 541                               Size);
 542   } else
 543     return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
 544                                     DstPtr, Size, AsyncInfo);
 545 }
 546
 547 // Run region on device
 548 int32_t DeviceTy::runRegion(void *TgtEntryPtr, void **TgtVarsPtr,
 549                             ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
 550                             AsyncInfoTy &AsyncInfo) {
 551   if (!RTL->run_region || !RTL->synchronize)
 552     return RTL->run_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,
 553                            TgtVarsSize);
 554   else
 555     return RTL->run_region_async(RTLDeviceID, TgtEntryPtr, TgtVarsPtr,
 556                                  TgtOffsets, TgtVarsSize, AsyncInfo);
 557 }
 558
 559 // Run region on device
 560 bool DeviceTy::printDeviceInfo(int32_t RTLDevId) {
 561   if (!RTL->print_device_info)
 562     return false;
 563   RTL->print_device_info(RTLDevId);
 564   return true;
 565 }
 566
 567 // Run team region on device.
 568 int32_t DeviceTy::runTeamRegion(void *TgtEntryPtr, void **TgtVarsPtr,
 569                                 ptrdiff_t *TgtOffsets, int32_t TgtVarsSize,
 570                                 int32_t NumTeams, int32_t ThreadLimit,
 571                                 uint64_t LoopTripCount,
 572                                 AsyncInfoTy &AsyncInfo) {
 573   if (!RTL->run_team_region_async || !RTL->synchronize)
 574     return RTL->run_team_region(RTLDeviceID, TgtEntryPtr, TgtVarsPtr,
 575                                 TgtOffsets, TgtVarsSize, NumTeams, ThreadLimit,
 576                                 LoopTripCount);
 577   else
 578     return RTL->run_team_region_async(RTLDeviceID, TgtEntryPtr, TgtVarsPtr,
 579                                       TgtOffsets, TgtVarsSize, NumTeams,
 580                                       ThreadLimit, LoopTripCount, AsyncInfo);
 581 }
 582
 583 // Whether data can be copied to DstDevice directly
 584 bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) {
 585   if (RTL != DstDevice.RTL || !RTL->is_data_exchangable)
 586     return false;
 587
 588   if (RTL->is_data_exchangable(RTLDeviceID, DstDevice.RTLDeviceID))
 589     return (RTL->data_exchange != nullptr) ||
 590            (RTL->data_exchange_async != nullptr);
 591
 592   return false;
 593 }
 594
 595 int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) {
 596   if (RTL->synchronize)
 597     return RTL->synchronize(RTLDeviceID, AsyncInfo);
 598   return OFFLOAD_SUCCESS;
 599 }
 600
 601 int32_t DeviceTy::createEvent(void **Event) {
 602   if (RTL->create_event)
 603     return RTL->create_event(RTLDeviceID, Event);
 604
 605   return OFFLOAD_SUCCESS;
 606 }
 607
 608 int32_t DeviceTy::recordEvent(void *Event, AsyncInfoTy &AsyncInfo) {
 609   if (RTL->record_event)
 610     return RTL->record_event(RTLDeviceID, Event, AsyncInfo);
 611
 612   return OFFLOAD_SUCCESS;
 613 }
 614
 615 int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) {
 616   if (RTL->wait_event)
 617     return RTL->wait_event(RTLDeviceID, Event, AsyncInfo);
 618
 619   return OFFLOAD_SUCCESS;
 620 }
 621
 622 int32_t DeviceTy::syncEvent(void *Event) {
 623   if (RTL->sync_event)
 624     return RTL->sync_event(RTLDeviceID, Event);
 625
 626   return OFFLOAD_SUCCESS;
 627 }
 628
 629 int32_t DeviceTy::destroyEvent(void *Event) {
 630   if (RTL->create_event)
 631     return RTL->destroy_event(RTLDeviceID, Event);
 632
 633   return OFFLOAD_SUCCESS;
 634 }
 635
 636 /// Check whether a device has an associated RTL and initialize it if it's not
 637 /// already initialized.
 638 bool device_is_ready(int device_num) {
 639   DP("Checking whether device %d is ready.\n", device_num);
 640   // Devices.size() can only change while registering a new
 641   // library, so try to acquire the lock of RTLs' mutex.
 642   PM->RTLsMtx.lock();
 643   size_t DevicesSize = PM->Devices.size();
 644   PM->RTLsMtx.unlock();
 645   if (DevicesSize <= (size_t)device_num) {
 646     DP("Device ID  %d does not have a matching RTL\n", device_num);
 647     return false;
 648   }
 649
 650   // Get device info
 651   DeviceTy &Device = *PM->Devices[device_num];
 652
 653   DP("Is the device %d (local ID %d) initialized? %d\n", device_num,
 654      Device.RTLDeviceID, Device.IsInit);
 655
 656   // Init the device if not done before
 657   if (!Device.IsInit && Device.initOnce() != OFFLOAD_SUCCESS) {
 658     DP("Failed to init device %d\n", device_num);
 659     return false;
 660   }
 661
 662   DP("Device %d is ready to use.\n", device_num);
 663
 664   return true;
 665 }