openmp/libomptarget/src/OmptCallback.cpp

   1 //===-- OmptCallback.cpp - Target independent OpenMP target RTL --- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Implementation of OMPT callback interfaces for target independent layer
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifdef OMPT_SUPPORT
  14
  15 #include "llvm/Support/DynamicLibrary.h"
  16
  17 #include <cstdlib>
  18 #include <cstring>
  19 #include <memory>
  20
  21 #include "Debug.h"
  22 #include "OmptCallback.h"
  23 #include "OmptConnector.h"
  24 #include "OmptInterface.h"
  25
  26 #undef DEBUG_PREFIX
  27 #define DEBUG_PREFIX "OMPT"
  28
  29 using namespace llvm::omp::target::ompt;
  30
  31 // Define OMPT callback functions (bound to actual callbacks later on)
  32 #define defineOmptCallback(Name, Type, Code)                                   \
  33   Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
  34 FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback)
  35 FOREACH_OMPT_EMI_EVENT(defineOmptCallback)
  36 #undef defineOmptCallback
  37
  38 /// Forward declaration
  39 class LibomptargetRtlFinalizer;
  40
  41 /// Object that will maintain the RTL finalizer from the plugin
  42 LibomptargetRtlFinalizer *LibraryFinalizer = nullptr;
  43
  44 thread_local Interface llvm::omp::target::ompt::RegionInterface;
  45
  46 bool llvm::omp::target::ompt::Initialized = false;
  47
  48 ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr;
  49 ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr;
  50 ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr;
  51 ompt_get_task_data_t ompt_get_task_data_fn = nullptr;
  52
  53 /// Unique correlation id
  54 static std::atomic<uint64_t> IdCounter(1);
  55
  56 /// Used to create a new correlation id
  57 static uint64_t createId() { return IdCounter.fetch_add(1); }
  58
  59 /// Create a new correlation id and update the operations id
  60 static uint64_t createOpId() {
  61   uint64_t NewId = createId();
  62   RegionInterface.setHostOpId(NewId);
  63   return NewId;
  64 }
  65
  66 /// Create a new correlation id and update the target region id
  67 static uint64_t createRegionId() {
  68   uint64_t NewId = createId();
  69   RegionInterface.setTargetDataValue(NewId);
  70   return NewId;
  71 }
  72
  73 void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
  74                                      void **TgtPtrBegin, size_t Size,
  75                                      void *Code) {
  76   beginTargetDataOperation();
  77   if (ompt_callback_target_data_op_emi_fn) {
  78     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
  79     // callback
  80     ompt_callback_target_data_op_emi_fn(
  81         ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId,
  82         ompt_target_data_alloc, HstPtrBegin,
  83         /* SrcDeviceNum */ omp_get_initial_device(), *TgtPtrBegin,
  84         /* TgtDeviceNum */ DeviceId, Size, Code);
  85   } else if (ompt_callback_target_data_op_fn) {
  86     // HostOpId is set by the runtime
  87     HostOpId = createOpId();
  88     // Invoke the tool supplied data op callback
  89     ompt_callback_target_data_op_fn(
  90         TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin,
  91         /* SrcDeviceNum */ omp_get_initial_device(), *TgtPtrBegin,
  92         /* TgtDeviceNum */ DeviceId, Size, Code);
  93   }
  94 }
  95
  96 void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
  97                                    void **TgtPtrBegin, size_t Size,
  98                                    void *Code) {
  99   // Only EMI callback handles end scope
 100   if (ompt_callback_target_data_op_emi_fn) {
 101     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 102     // callback
 103     ompt_callback_target_data_op_emi_fn(
 104         ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId,
 105         ompt_target_data_alloc, HstPtrBegin,
 106         /* SrcDeviceNum */ omp_get_initial_device(), *TgtPtrBegin,
 107         /* TgtDeviceNum */ DeviceId, Size, Code);
 108   }
 109   endTargetDataOperation();
 110 }
 111
 112 void Interface::beginTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin,
 113                                       void *HstPtrBegin, size_t Size,
 114                                       void *Code) {
 115   beginTargetDataOperation();
 116   if (ompt_callback_target_data_op_emi_fn) {
 117     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 118     // callback
 119     ompt_callback_target_data_op_emi_fn(
 120         ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId,
 121         ompt_target_data_transfer_to_device, HstPtrBegin,
 122         /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId,
 123         Size, Code);
 124   } else if (ompt_callback_target_data_op_fn) {
 125     // HostOpId is set by the runtime
 126     HostOpId = createOpId();
 127     // Invoke the tool supplied data op callback
 128     ompt_callback_target_data_op_fn(
 129         TargetData.value, HostOpId, ompt_target_data_transfer_to_device,
 130         HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin,
 131         DeviceId, Size, Code);
 132   }
 133 }
 134
 135 void Interface::endTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin,
 136                                     void *HstPtrBegin, size_t Size,
 137                                     void *Code) {
 138   // Only EMI callback handles end scope
 139   if (ompt_callback_target_data_op_emi_fn) {
 140     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 141     // callback
 142     ompt_callback_target_data_op_emi_fn(
 143         ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId,
 144         ompt_target_data_transfer_to_device, HstPtrBegin,
 145         /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId,
 146         Size, Code);
 147   }
 148   endTargetDataOperation();
 149 }
 150
 151 void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
 152                                       void *Code) {
 153   beginTargetDataOperation();
 154   if (ompt_callback_target_data_op_emi_fn) {
 155     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 156     // callback
 157     ompt_callback_target_data_op_emi_fn(
 158         ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId,
 159         ompt_target_data_delete, TgtPtrBegin, DeviceId,
 160         /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code);
 161   } else if (ompt_callback_target_data_op_fn) {
 162     // HostOpId is set by the runtime
 163     HostOpId = createOpId();
 164     // Invoke the tool supplied data op callback
 165     ompt_callback_target_data_op_fn(TargetData.value, HostOpId,
 166                                     ompt_target_data_delete, TgtPtrBegin,
 167                                     DeviceId, /* TgtPtrBegin */ nullptr,
 168                                     /* TgtDeviceNum */ -1, /* Bytes */ 0, Code);
 169   }
 170 }
 171
 172 void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin,
 173                                     void *Code) {
 174   // Only EMI callback handles end scope
 175   if (ompt_callback_target_data_op_emi_fn) {
 176     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 177     // callback
 178     ompt_callback_target_data_op_emi_fn(
 179         ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId,
 180         ompt_target_data_delete, TgtPtrBegin, DeviceId,
 181         /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code);
 182   }
 183   endTargetDataOperation();
 184 }
 185
 186 void Interface::beginTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin,
 187                                         void *TgtPtrBegin, size_t Size,
 188                                         void *Code) {
 189   beginTargetDataOperation();
 190   if (ompt_callback_target_data_op_emi_fn) {
 191     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 192     // callback
 193     ompt_callback_target_data_op_emi_fn(
 194         ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId,
 195         ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId,
 196         HstPtrBegin,
 197         /* TgtDeviceNum */ omp_get_initial_device(), Size, Code);
 198   } else if (ompt_callback_target_data_op_fn) {
 199     // HostOpId is set by the runtime
 200     HostOpId = createOpId();
 201     // Invoke the tool supplied data op callback
 202     ompt_callback_target_data_op_fn(
 203         TargetData.value, HostOpId, ompt_target_data_transfer_from_device,
 204         TgtPtrBegin, DeviceId, HstPtrBegin,
 205         /* TgtDeviceNum */ omp_get_initial_device(), Size, Code);
 206   }
 207 }
 208
 209 void Interface::endTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin,
 210                                       void *TgtPtrBegin, size_t Size,
 211                                       void *Code) {
 212   // Only EMI callback handles end scope
 213   if (ompt_callback_target_data_op_emi_fn) {
 214     // HostOpId will be set by the tool. Invoke the tool supplied data op EMI
 215     // callback
 216     ompt_callback_target_data_op_emi_fn(
 217         ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId,
 218         ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId,
 219         HstPtrBegin,
 220         /* TgtDeviceNum */ omp_get_initial_device(), Size, Code);
 221   }
 222   endTargetDataOperation();
 223 }
 224
 225 void Interface::beginTargetSubmit(unsigned int numTeams) {
 226   if (ompt_callback_target_submit_emi_fn) {
 227     // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
 228     // callback
 229     ompt_callback_target_submit_emi_fn(ompt_scope_begin, &TargetData, &HostOpId,
 230                                        numTeams);
 231   } else if (ompt_callback_target_submit_fn) {
 232     // HostOpId is set by the runtime
 233     HostOpId = createOpId();
 234     ompt_callback_target_submit_fn(TargetData.value, HostOpId, numTeams);
 235   }
 236 }
 237
 238 void Interface::endTargetSubmit(unsigned int numTeams) {
 239   // Only EMI callback handles end scope
 240   if (ompt_callback_target_submit_emi_fn) {
 241     // HostOpId is set by the tool. Invoke the tool supplied target submit EMI
 242     // callback
 243     ompt_callback_target_submit_emi_fn(ompt_scope_end, &TargetData, &HostOpId,
 244                                        numTeams);
 245   }
 246 }
 247
 248 void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) {
 249   beginTargetRegion();
 250   if (ompt_callback_target_emi_fn) {
 251     // Invoke the tool supplied target EMI callback
 252     ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_begin,
 253                                 DeviceId, TaskData, TargetTaskData, &TargetData,
 254                                 Code);
 255   } else if (ompt_callback_target_fn) {
 256     // Invoke the tool supplied target callback
 257     ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_begin, DeviceId,
 258                             TaskData, TargetData.value, Code);
 259   }
 260 }
 261
 262 void Interface::endTargetDataEnter(int64_t DeviceId, void *Code) {
 263   if (ompt_callback_target_emi_fn) {
 264     // Invoke the tool supplied target EMI callback
 265     ompt_callback_target_emi_fn(ompt_target_enter_data, ompt_scope_end,
 266                                 DeviceId, TaskData, TargetTaskData, &TargetData,
 267                                 Code);
 268   } else if (ompt_callback_target_fn) {
 269     // Invoke the tool supplied target callback
 270     ompt_callback_target_fn(ompt_target_enter_data, ompt_scope_end, DeviceId,
 271                             TaskData, TargetData.value, Code);
 272   }
 273   endTargetRegion();
 274 }
 275
 276 void Interface::beginTargetDataExit(int64_t DeviceId, void *Code) {
 277   beginTargetRegion();
 278   if (ompt_callback_target_emi_fn) {
 279     // Invoke the tool supplied target EMI callback
 280     ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_begin,
 281                                 DeviceId, TaskData, TargetTaskData, &TargetData,
 282                                 Code);
 283   } else if (ompt_callback_target_fn) {
 284     TargetData.value = createRegionId();
 285     // Invoke the tool supplied target callback
 286     ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_begin, DeviceId,
 287                             TaskData, TargetData.value, Code);
 288   }
 289 }
 290
 291 void Interface::endTargetDataExit(int64_t DeviceId, void *Code) {
 292   if (ompt_callback_target_emi_fn) {
 293     // Invoke the tool supplied target EMI callback
 294     ompt_callback_target_emi_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
 295                                 TaskData, TargetTaskData, &TargetData, Code);
 296   } else if (ompt_callback_target_fn) {
 297     // Invoke the tool supplied target callback
 298     ompt_callback_target_fn(ompt_target_exit_data, ompt_scope_end, DeviceId,
 299                             TaskData, TargetData.value, Code);
 300   }
 301   endTargetRegion();
 302 }
 303
 304 void Interface::beginTargetUpdate(int64_t DeviceId, void *Code) {
 305   beginTargetRegion();
 306   if (ompt_callback_target_emi_fn) {
 307     // Invoke the tool supplied target EMI callback
 308     ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_begin, DeviceId,
 309                                 TaskData, TargetTaskData, &TargetData, Code);
 310   } else if (ompt_callback_target_fn) {
 311     TargetData.value = createRegionId();
 312     // Invoke the tool supplied target callback
 313     ompt_callback_target_fn(ompt_target_update, ompt_scope_begin, DeviceId,
 314                             TaskData, TargetData.value, Code);
 315   }
 316 }
 317
 318 void Interface::endTargetUpdate(int64_t DeviceId, void *Code) {
 319   if (ompt_callback_target_emi_fn) {
 320     // Invoke the tool supplied target EMI callback
 321     ompt_callback_target_emi_fn(ompt_target_update, ompt_scope_end, DeviceId,
 322                                 TaskData, TargetTaskData, &TargetData, Code);
 323   } else if (ompt_callback_target_fn) {
 324     // Invoke the tool supplied target callback
 325     ompt_callback_target_fn(ompt_target_update, ompt_scope_end, DeviceId,
 326                             TaskData, TargetData.value, Code);
 327   }
 328   endTargetRegion();
 329 }
 330
 331 void Interface::beginTarget(int64_t DeviceId, void *Code) {
 332   beginTargetRegion();
 333   if (ompt_callback_target_emi_fn) {
 334     // Invoke the tool supplied target EMI callback
 335     ompt_callback_target_emi_fn(ompt_target, ompt_scope_begin, DeviceId,
 336                                 TaskData, TargetTaskData, &TargetData, Code);
 337   } else if (ompt_callback_target_fn) {
 338     TargetData.value = createRegionId();
 339     // Invoke the tool supplied target callback
 340     ompt_callback_target_fn(ompt_target, ompt_scope_begin, DeviceId, TaskData,
 341                             TargetData.value, Code);
 342   }
 343 }
 344
 345 void Interface::endTarget(int64_t DeviceId, void *Code) {
 346   if (ompt_callback_target_emi_fn) {
 347     // Invoke the tool supplied target EMI callback
 348     ompt_callback_target_emi_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
 349                                 TargetTaskData, &TargetData, Code);
 350   } else if (ompt_callback_target_fn) {
 351     // Invoke the tool supplied target callback
 352     ompt_callback_target_fn(ompt_target, ompt_scope_end, DeviceId, TaskData,
 353                             TargetData.value, Code);
 354   }
 355   endTargetRegion();
 356 }
 357
 358 void Interface::beginTargetDataOperation() {
 359   DP("in ompt_target_region_begin (TargetRegionOpId = %lu)\n",
 360      TargetData.value);
 361 }
 362
 363 void Interface::endTargetDataOperation() {
 364   DP("in ompt_target_region_end (TargetRegionOpId = %lu)\n", TargetData.value);
 365 }
 366
 367 void Interface::beginTargetRegion() {
 368   // Set up task state
 369   assert(ompt_get_task_data_fn && "Calling a null task data function");
 370   TaskData = ompt_get_task_data_fn();
 371   // Set up target task state
 372   assert(ompt_get_target_task_data_fn &&
 373          "Calling a null target task data function");
 374   TargetTaskData = ompt_get_target_task_data_fn();
 375   // Target state will be set later
 376   TargetData = ompt_data_none;
 377 }
 378
 379 void Interface::endTargetRegion() {
 380   TaskData = 0;
 381   TargetTaskData = 0;
 382   TargetData = ompt_data_none;
 383 }
 384
 385 /// Used to maintain the finalization functions that are received
 386 /// from the plugins during connect.
 387 /// Note: Currently, there are no plugin-specific finalizations, so each plugin
 388 /// will call the same (empty) function.
 389 class LibomptargetRtlFinalizer {
 390 public:
 391   LibomptargetRtlFinalizer() {}
 392
 393   void registerRtl(ompt_finalize_t FinalizationFunction) {
 394     if (FinalizationFunction) {
 395       RtlFinalizationFunctions.emplace_back(FinalizationFunction);
 396     }
 397   }
 398
 399   void finalize() {
 400     for (auto FinalizationFunction : RtlFinalizationFunctions)
 401       FinalizationFunction(/* tool_data */ nullptr);
 402     RtlFinalizationFunctions.clear();
 403   }
 404
 405 private:
 406   llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions;
 407 };
 408
 409 int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup,
 410                                                int initial_device_num,
 411                                                ompt_data_t *tool_data) {
 412   DP("Executing initializeLibrary (libomp)\n");
 413 #define bindOmptFunctionName(OmptFunction, DestinationFunction)                \
 414   DestinationFunction = (OmptFunction##_t)lookup(#OmptFunction);               \
 415   DP("initializeLibrary (libomp) bound %s=%p\n", #DestinationFunction,         \
 416      ((void *)(uint64_t)DestinationFunction));
 417
 418   bindOmptFunctionName(ompt_get_callback, lookupCallbackByCode);
 419   bindOmptFunctionName(ompt_get_task_data, ompt_get_task_data_fn);
 420   bindOmptFunctionName(ompt_get_target_task_data, ompt_get_target_task_data_fn);
 421 #undef bindOmptFunctionName
 422
 423   // Store pointer of 'ompt_libomp_target_fn_lookup' for use by libomptarget
 424   lookupCallbackByName = lookup;
 425
 426   assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null");
 427   assert(lookupCallbackByName && "lookupCallbackByName should be non-null");
 428   assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null");
 429   assert(ompt_get_target_task_data_fn &&
 430          "ompt_get_target_task_data_fn should be non-null");
 431   assert(LibraryFinalizer == nullptr &&
 432          "LibraryFinalizer should not be initialized yet");
 433
 434   LibraryFinalizer = new LibomptargetRtlFinalizer();
 435
 436   Initialized = true;
 437
 438   return 0;
 439 }
 440
 441 void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
 442   DP("Executing finalizeLibrary (libomp)\n");
 443   // Before disabling OMPT, call the (plugin) finalizations that were registered
 444   // with this library
 445   LibraryFinalizer->finalize();
 446   delete LibraryFinalizer;
 447   Initialized = false;
 448 }
 449
 450 void llvm::omp::target::ompt::connectLibrary() {
 451   DP("Entering connectLibrary (libomp)\n");
 452   // Connect with libomp
 453   static OmptLibraryConnectorTy LibompConnector("libomp");
 454   static ompt_start_tool_result_t OmptResult;
 455
 456   // Initialize OmptResult with the init and fini functions that will be
 457   // called by the connector
 458   OmptResult.initialize = ompt::initializeLibrary;
 459   OmptResult.finalize = ompt::finalizeLibrary;
 460   OmptResult.tool_data.value = 0;
 461
 462   // Now call connect that causes the above init/fini functions to be called
 463   LibompConnector.connect(&OmptResult);
 464
 465 #define bindOmptCallback(Name, Type, Code)                                     \
 466   if (lookupCallbackByCode)                                                    \
 467     lookupCallbackByCode(                                                      \
 468         (ompt_callbacks_t)(Code),                                              \
 469         (ompt_callback_t *)&(llvm::omp::target::ompt::Name##_fn));
 470   FOREACH_OMPT_NOEMI_EVENT(bindOmptCallback)
 471   FOREACH_OMPT_EMI_EVENT(bindOmptCallback)
 472 #undef bindOmptCallback
 473
 474   DP("Exiting connectLibrary (libomp)\n");
 475 }
 476
 477 extern "C" {
 478 /// Used for connecting libomptarget with a plugin
 479 void ompt_libomptarget_connect(ompt_start_tool_result_t *result) {
 480   DP("Enter ompt_libomptarget_connect\n");
 481   if (Initialized && result && LibraryFinalizer) {
 482     // Cache each fini function, so that they can be invoked on exit
 483     LibraryFinalizer->registerRtl(result->finalize);
 484     // Invoke the provided init function with the lookup function maintained
 485     // in this library so that callbacks maintained by this library are
 486     // retrieved.
 487     result->initialize(lookupCallbackByName,
 488                        /* initial_device_num */ 0, /* tool_data */ nullptr);
 489   }
 490   DP("Leave ompt_libomptarget_connect\n");
 491 }
 492 }
 493 #else
 494 extern "C" {
 495 /// Dummy definition when OMPT is disabled
 496 void ompt_libomptarget_connect() {}
 497 }
 498 #endif // OMPT_SUPPORT