openmp/libomptarget/src/rtl.cpp

   1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Functionality for handling RTL plugins.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "rtl.h"
  14 #include "device.h"
  15 #include "private.h"
  16
  17 #include <cassert>
  18 #include <cstdlib>
  19 #include <cstring>
  20 #include <dlfcn.h>
  21 #include <mutex>
  22 #include <string>
  23
  24 // List of all plugins that can support offloading.
  25 static const char *RTLNames[] = {
  26     /* PowerPC target       */ "libomptarget.rtl.ppc64.so",
  27     /* x86_64 target        */ "libomptarget.rtl.x86_64.so",
  28     /* CUDA target          */ "libomptarget.rtl.cuda.so",
  29     /* AArch64 target       */ "libomptarget.rtl.aarch64.so",
  30     /* SX-Aurora VE target  */ "libomptarget.rtl.ve.so",
  31     /* AMDGPU target        */ "libomptarget.rtl.amdgpu.so",
  32     /* Remote target        */ "libomptarget.rtl.rpc.so",
  33 };
  34
  35 PluginManager *PM;
  36
  37 #if OMPTARGET_PROFILE_ENABLED
  38 static char *ProfileTraceFile = nullptr;
  39 #endif
  40
  41 __attribute__((constructor(101))) void init() {
  42   DP("Init target library!\n");
  43   PM = new PluginManager();
  44
  45 #ifdef OMPTARGET_PROFILE_ENABLED
  46   ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
  47   // TODO: add a configuration option for time granularity
  48   if (ProfileTraceFile)
  49     llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
  50 #endif
  51 }
  52
  53 __attribute__((destructor(101))) void deinit() {
  54   DP("Deinit target library!\n");
  55   delete PM;
  56
  57 #ifdef OMPTARGET_PROFILE_ENABLED
  58   if (ProfileTraceFile) {
  59     // TODO: add env var for file output
  60     if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-"))
  61       fprintf(stderr, "Error writing out the time trace\n");
  62
  63     llvm::timeTraceProfilerCleanup();
  64   }
  65 #endif
  66 }
  67
  68 void RTLsTy::LoadRTLs() {
  69   // Parse environment variable OMP_TARGET_OFFLOAD (if set)
  70   PM->TargetOffloadPolicy =
  71       (kmp_target_offload_kind_t)__kmpc_get_target_offload();
  72   if (PM->TargetOffloadPolicy == tgt_disabled) {
  73     return;
  74   }
  75
  76   DP("Loading RTLs...\n");
  77
  78   // Attempt to open all the plugins and, if they exist, check if the interface
  79   // is correct and if they are supporting any devices.
  80   for (auto *Name : RTLNames) {
  81     DP("Loading library '%s'...\n", Name);
  82     void *dynlib_handle = dlopen(Name, RTLD_NOW);
  83
  84     if (!dynlib_handle) {
  85       // Library does not exist or cannot be found.
  86       DP("Unable to load library '%s': %s!\n", Name, dlerror());
  87       continue;
  88     }
  89
  90     DP("Successfully loaded library '%s'!\n", Name);
  91
  92     AllRTLs.emplace_back();
  93
  94     // Retrieve the RTL information from the runtime library.
  95     RTLInfoTy &R = AllRTLs.back();
  96
  97     bool ValidPlugin = true;
  98
  99     if (!(*((void **)&R.is_valid_binary) =
 100               dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary")))
 101       ValidPlugin = false;
 102     if (!(*((void **)&R.number_of_devices) =
 103               dlsym(dynlib_handle, "__tgt_rtl_number_of_devices")))
 104       ValidPlugin = false;
 105     if (!(*((void **)&R.init_device) =
 106               dlsym(dynlib_handle, "__tgt_rtl_init_device")))
 107       ValidPlugin = false;
 108     if (!(*((void **)&R.load_binary) =
 109               dlsym(dynlib_handle, "__tgt_rtl_load_binary")))
 110       ValidPlugin = false;
 111     if (!(*((void **)&R.data_alloc) =
 112               dlsym(dynlib_handle, "__tgt_rtl_data_alloc")))
 113       ValidPlugin = false;
 114     if (!(*((void **)&R.data_submit) =
 115               dlsym(dynlib_handle, "__tgt_rtl_data_submit")))
 116       ValidPlugin = false;
 117     if (!(*((void **)&R.data_retrieve) =
 118               dlsym(dynlib_handle, "__tgt_rtl_data_retrieve")))
 119       ValidPlugin = false;
 120     if (!(*((void **)&R.data_delete) =
 121               dlsym(dynlib_handle, "__tgt_rtl_data_delete")))
 122       ValidPlugin = false;
 123     if (!(*((void **)&R.run_region) =
 124               dlsym(dynlib_handle, "__tgt_rtl_run_target_region")))
 125       ValidPlugin = false;
 126     if (!(*((void **)&R.run_team_region) =
 127               dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region")))
 128       ValidPlugin = false;
 129
 130     // Invalid plugin
 131     if (!ValidPlugin) {
 132       DP("Invalid plugin as necessary interface is not found.\n");
 133       AllRTLs.pop_back();
 134       continue;
 135     }
 136
 137     // No devices are supported by this RTL?
 138     if (!(R.NumberOfDevices = R.number_of_devices())) {
 139       // The RTL is invalid! Will pop the object from the RTLs list.
 140       DP("No devices supported in this RTL\n");
 141       AllRTLs.pop_back();
 142       continue;
 143     }
 144
 145     R.LibraryHandler = dynlib_handle;
 146
 147 #ifdef OMPTARGET_DEBUG
 148     R.RTLName = Name;
 149 #endif
 150
 151     DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(),
 152        R.NumberOfDevices);
 153
 154     // Optional functions
 155     *((void **)&R.init_requires) =
 156         dlsym(dynlib_handle, "__tgt_rtl_init_requires");
 157     *((void **)&R.data_submit_async) =
 158         dlsym(dynlib_handle, "__tgt_rtl_data_submit_async");
 159     *((void **)&R.data_retrieve_async) =
 160         dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async");
 161     *((void **)&R.run_region_async) =
 162         dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async");
 163     *((void **)&R.run_team_region_async) =
 164         dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async");
 165     *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize");
 166     *((void **)&R.data_exchange) =
 167         dlsym(dynlib_handle, "__tgt_rtl_data_exchange");
 168     *((void **)&R.data_exchange_async) =
 169         dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async");
 170     *((void **)&R.is_data_exchangable) =
 171         dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable");
 172     *((void **)&R.register_lib) =
 173         dlsym(dynlib_handle, "__tgt_rtl_register_lib");
 174     *((void **)&R.unregister_lib) =
 175         dlsym(dynlib_handle, "__tgt_rtl_unregister_lib");
 176     *((void **)&R.supports_empty_images) =
 177         dlsym(dynlib_handle, "__tgt_rtl_supports_empty_images");
 178     *((void **)&R.set_info_flag) =
 179         dlsym(dynlib_handle, "__tgt_rtl_set_info_flag");
 180     *((void **)&R.print_device_info) =
 181         dlsym(dynlib_handle, "__tgt_rtl_print_device_info");
 182     *((void **)&R.create_event) =
 183         dlsym(dynlib_handle, "__tgt_rtl_create_event");
 184     *((void **)&R.record_event) =
 185         dlsym(dynlib_handle, "__tgt_rtl_record_event");
 186     *((void **)&R.wait_event) = dlsym(dynlib_handle, "__tgt_rtl_wait_event");
 187     *((void **)&R.sync_event) = dlsym(dynlib_handle, "__tgt_rtl_sync_event");
 188     *((void **)&R.destroy_event) =
 189         dlsym(dynlib_handle, "__tgt_rtl_destroy_event");
 190   }
 191
 192   DP("RTLs loaded!\n");
 193
 194   return;
 195 }
 196
 197 ////////////////////////////////////////////////////////////////////////////////
 198 // Functionality for registering libs
 199
 200 static void RegisterImageIntoTranslationTable(TranslationTable &TT,
 201                                               RTLInfoTy &RTL,
 202                                               __tgt_device_image *image) {
 203
 204   // same size, as when we increase one, we also increase the other.
 205   assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
 206          "We should have as many images as we have tables!");
 207
 208   // Resize the Targets Table and Images to accommodate the new targets if
 209   // required
 210   unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices;
 211
 212   if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
 213     TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
 214     TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
 215   }
 216
 217   // Register the image in all devices for this target type.
 218   for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) {
 219     // If we are changing the image we are also invalidating the target table.
 220     if (TT.TargetsImages[RTL.Idx + i] != image) {
 221       TT.TargetsImages[RTL.Idx + i] = image;
 222       TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table.
 223     }
 224   }
 225 }
 226
 227 ////////////////////////////////////////////////////////////////////////////////
 228 // Functionality for registering Ctors/Dtors
 229
 230 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc,
 231                                              __tgt_device_image *img,
 232                                              RTLInfoTy *RTL) {
 233
 234   for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) {
 235     DeviceTy &Device = *PM->Devices[RTL->Idx + i];
 236     Device.PendingGlobalsMtx.lock();
 237     Device.HasPendingGlobals = true;
 238     for (__tgt_offload_entry *entry = img->EntriesBegin;
 239          entry != img->EntriesEnd; ++entry) {
 240       if (entry->flags & OMP_DECLARE_TARGET_CTOR) {
 241         DP("Adding ctor " DPxMOD " to the pending list.\n",
 242            DPxPTR(entry->addr));
 243         Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr);
 244       } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) {
 245         // Dtors are pushed in reverse order so they are executed from end
 246         // to beginning when unregistering the library!
 247         DP("Adding dtor " DPxMOD " to the pending list.\n",
 248            DPxPTR(entry->addr));
 249         Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr);
 250       }
 251
 252       if (entry->flags & OMP_DECLARE_TARGET_LINK) {
 253         DP("The \"link\" attribute is not yet supported!\n");
 254       }
 255     }
 256     Device.PendingGlobalsMtx.unlock();
 257   }
 258 }
 259
 260 void RTLsTy::RegisterRequires(int64_t flags) {
 261   // TODO: add more elaborate check.
 262   // Minimal check: only set requires flags if previous value
 263   // is undefined. This ensures that only the first call to this
 264   // function will set the requires flags. All subsequent calls
 265   // will be checked for compatibility.
 266   assert(flags != OMP_REQ_UNDEFINED &&
 267          "illegal undefined flag for requires directive!");
 268   if (RequiresFlags == OMP_REQ_UNDEFINED) {
 269     RequiresFlags = flags;
 270     return;
 271   }
 272
 273   // If multiple compilation units are present enforce
 274   // consistency across all of them for require clauses:
 275   //  - reverse_offload
 276   //  - unified_address
 277   //  - unified_shared_memory
 278   if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) !=
 279       (flags & OMP_REQ_REVERSE_OFFLOAD)) {
 280     FATAL_MESSAGE0(
 281         1, "'#pragma omp requires reverse_offload' not used consistently!");
 282   }
 283   if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) !=
 284       (flags & OMP_REQ_UNIFIED_ADDRESS)) {
 285     FATAL_MESSAGE0(
 286         1, "'#pragma omp requires unified_address' not used consistently!");
 287   }
 288   if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) !=
 289       (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) {
 290     FATAL_MESSAGE0(
 291         1,
 292         "'#pragma omp requires unified_shared_memory' not used consistently!");
 293   }
 294
 295   // TODO: insert any other missing checks
 296
 297   DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n",
 298      flags, RequiresFlags);
 299 }
 300
 301 void RTLsTy::initRTLonce(RTLInfoTy &R) {
 302   // If this RTL is not already in use, initialize it.
 303   if (!R.isUsed && R.NumberOfDevices != 0) {
 304     // Initialize the device information for the RTL we are about to use.
 305     const size_t Start = PM->Devices.size();
 306     PM->Devices.reserve(Start + R.NumberOfDevices);
 307     for (int32_t device_id = 0; device_id < R.NumberOfDevices; device_id++) {
 308       PM->Devices.push_back(std::make_unique<DeviceTy>(&R));
 309       // global device ID
 310       PM->Devices[Start + device_id]->DeviceID = Start + device_id;
 311       // RTL local device ID
 312       PM->Devices[Start + device_id]->RTLDeviceID = device_id;
 313     }
 314
 315     // Initialize the index of this RTL and save it in the used RTLs.
 316     R.Idx = (UsedRTLs.empty())
 317                 ? 0
 318                 : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices;
 319     assert((size_t)R.Idx == Start &&
 320            "RTL index should equal the number of devices used so far.");
 321     R.isUsed = true;
 322     UsedRTLs.push_back(&R);
 323
 324     DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx);
 325   }
 326 }
 327
 328 void RTLsTy::initAllRTLs() {
 329   for (auto &R : AllRTLs)
 330     initRTLonce(R);
 331 }
 332
 333 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
 334   PM->RTLsMtx.lock();
 335   // Register the images with the RTLs that understand them, if any.
 336   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
 337     // Obtain the image.
 338     __tgt_device_image *img = &desc->DeviceImages[i];
 339
 340     RTLInfoTy *FoundRTL = nullptr;
 341
 342     // Scan the RTLs that have associated images until we find one that supports
 343     // the current image.
 344     for (auto &R : AllRTLs) {
 345       if (!R.is_valid_binary(img)) {
 346         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
 347            DPxPTR(img->ImageStart), R.RTLName.c_str());
 348         continue;
 349       }
 350
 351       DP("Image " DPxMOD " is compatible with RTL %s!\n",
 352          DPxPTR(img->ImageStart), R.RTLName.c_str());
 353
 354       initRTLonce(R);
 355
 356       // Initialize (if necessary) translation table for this library.
 357       PM->TrlTblMtx.lock();
 358       if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) {
 359         PM->HostEntriesBeginRegistrationOrder.push_back(desc->HostEntriesBegin);
 360         TranslationTable &TransTable =
 361             (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
 362         TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin;
 363         TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd;
 364       }
 365
 366       // Retrieve translation table for this library.
 367       TranslationTable &TransTable =
 368           (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
 369
 370       DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(img->ImageStart),
 371          R.RTLName.c_str());
 372       RegisterImageIntoTranslationTable(TransTable, R, img);
 373       PM->TrlTblMtx.unlock();
 374       FoundRTL = &R;
 375
 376       // Load ctors/dtors for static objects
 377       RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL);
 378
 379       // if an RTL was found we are done - proceed to register the next image
 380       break;
 381     }
 382
 383     if (!FoundRTL) {
 384       DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart));
 385     }
 386   }
 387   PM->RTLsMtx.unlock();
 388
 389   DP("Done registering entries!\n");
 390 }
 391
 392 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) {
 393   DP("Unloading target library!\n");
 394
 395   PM->RTLsMtx.lock();
 396   // Find which RTL understands each image, if any.
 397   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
 398     // Obtain the image.
 399     __tgt_device_image *img = &desc->DeviceImages[i];
 400
 401     RTLInfoTy *FoundRTL = NULL;
 402
 403     // Scan the RTLs that have associated images until we find one that supports
 404     // the current image. We only need to scan RTLs that are already being used.
 405     for (auto *R : UsedRTLs) {
 406
 407       assert(R->isUsed && "Expecting used RTLs.");
 408
 409       if (!R->is_valid_binary(img)) {
 410         DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n",
 411            DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
 412         continue;
 413       }
 414
 415       DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n",
 416          DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
 417
 418       FoundRTL = R;
 419
 420       // Execute dtors for static objects if the device has been used, i.e.
 421       // if its PendingCtors list has been emptied.
 422       for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) {
 423         DeviceTy &Device = *PM->Devices[FoundRTL->Idx + i];
 424         Device.PendingGlobalsMtx.lock();
 425         if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) {
 426           AsyncInfoTy AsyncInfo(Device);
 427           for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) {
 428             int rc = target(nullptr, Device, dtor, 0, nullptr, nullptr, nullptr,
 429                             nullptr, nullptr, nullptr, 1, 1, true /*team*/,
 430                             AsyncInfo);
 431             if (rc != OFFLOAD_SUCCESS) {
 432               DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor));
 433             }
 434           }
 435           // Remove this library's entry from PendingCtorsDtors
 436           Device.PendingCtorsDtors.erase(desc);
 437           // All constructors have been issued, wait for them now.
 438           if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS)
 439             DP("Failed synchronizing destructors kernels.\n");
 440         }
 441         Device.PendingGlobalsMtx.unlock();
 442       }
 443
 444       DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
 445          DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
 446
 447       break;
 448     }
 449
 450     // if no RTL was found proceed to unregister the next image
 451     if (!FoundRTL) {
 452       DP("No RTLs in use support the image " DPxMOD "!\n",
 453          DPxPTR(img->ImageStart));
 454     }
 455   }
 456   PM->RTLsMtx.unlock();
 457   DP("Done unregistering images!\n");
 458
 459   // Remove entries from PM->HostPtrToTableMap
 460   PM->TblMapMtx.lock();
 461   for (__tgt_offload_entry *cur = desc->HostEntriesBegin;
 462        cur < desc->HostEntriesEnd; ++cur) {
 463     PM->HostPtrToTableMap.erase(cur->addr);
 464   }
 465
 466   // Remove translation table for this descriptor.
 467   auto TransTable =
 468       PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin);
 469   if (TransTable != PM->HostEntriesBeginToTransTable.end()) {
 470     DP("Removing translation table for descriptor " DPxMOD "\n",
 471        DPxPTR(desc->HostEntriesBegin));
 472     PM->HostEntriesBeginToTransTable.erase(TransTable);
 473   } else {
 474     DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
 475        "it has been already removed.\n",
 476        DPxPTR(desc->HostEntriesBegin));
 477   }
 478
 479   PM->TblMapMtx.unlock();
 480
 481   // TODO: Remove RTL and the devices it manages if it's not used anymore?
 482   // TODO: Write some RTL->unload_image(...) function?
 483
 484   DP("Done unregistering library!\n");
 485 }