openmp/libomptarget/src/api.cpp

   1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Implementation of OpenMP API interface functions.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "device.h"
  14 #include "omptarget.h"
  15 #include "private.h"
  16 #include "rtl.h"
  17
  18 #include <climits>
  19 #include <cstdlib>
  20 #include <cstring>
  21
  22 EXTERN int omp_get_num_devices(void) {
  23   TIMESCOPE();
  24   PM->RTLsMtx.lock();
  25   size_t DevicesSize = PM->Devices.size();
  26   PM->RTLsMtx.unlock();
  27
  28   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
  29
  30   return DevicesSize;
  31 }
  32
  33 EXTERN int omp_get_initial_device(void) {
  34   TIMESCOPE();
  35   int hostDevice = omp_get_num_devices();
  36   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
  37   return hostDevice;
  38 }
  39
  40 EXTERN void *omp_target_alloc(size_t size, int device_num) {
  41   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEFAULT, __func__);
  42 }
  43
  44 EXTERN void *llvm_omp_target_alloc_device(size_t size, int device_num) {
  45   return targetAllocExplicit(size, device_num, TARGET_ALLOC_DEVICE, __func__);
  46 }
  47
  48 EXTERN void *llvm_omp_target_alloc_host(size_t size, int device_num) {
  49   return targetAllocExplicit(size, device_num, TARGET_ALLOC_HOST, __func__);
  50 }
  51
  52 EXTERN void *llvm_omp_target_alloc_shared(size_t size, int device_num) {
  53   return targetAllocExplicit(size, device_num, TARGET_ALLOC_SHARED, __func__);
  54 }
  55
  56 EXTERN void *llvm_omp_get_dynamic_shared() { return nullptr; }
  57
  58 EXTERN void omp_target_free(void *device_ptr, int device_num) {
  59   TIMESCOPE();
  60   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
  61      device_num, DPxPTR(device_ptr));
  62
  63   if (!device_ptr) {
  64     DP("Call to omp_target_free with NULL ptr\n");
  65     return;
  66   }
  67
  68   if (device_num == omp_get_initial_device()) {
  69     free(device_ptr);
  70     DP("omp_target_free deallocated host ptr\n");
  71     return;
  72   }
  73
  74   if (!device_is_ready(device_num)) {
  75     DP("omp_target_free returns, nothing to do\n");
  76     return;
  77   }
  78
  79   PM->Devices[device_num]->deleteData(device_ptr);
  80   DP("omp_target_free deallocated device ptr\n");
  81 }
  82
  83 EXTERN int omp_target_is_present(const void *ptr, int device_num) {
  84   TIMESCOPE();
  85   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
  86      device_num, DPxPTR(ptr));
  87
  88   if (!ptr) {
  89     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
  90     return false;
  91   }
  92
  93   if (device_num == omp_get_initial_device()) {
  94     DP("Call to omp_target_is_present on host, returning true\n");
  95     return true;
  96   }
  97
  98   PM->RTLsMtx.lock();
  99   size_t DevicesSize = PM->Devices.size();
 100   PM->RTLsMtx.unlock();
 101   if (DevicesSize <= (size_t)device_num) {
 102     DP("Call to omp_target_is_present with invalid device ID, returning "
 103        "false\n");
 104     return false;
 105   }
 106
 107   DeviceTy &Device = *PM->Devices[device_num];
 108   bool IsLast; // not used
 109   bool IsHostPtr;
 110   void *TgtPtr = Device.getTgtPtrBegin(const_cast<void *>(ptr), 0, IsLast,
 111                                        /*UpdateRefCount=*/false,
 112                                        /*UseHoldRefCount=*/false, IsHostPtr);
 113   int rc = (TgtPtr != NULL);
 114   // Under unified memory the host pointer can be returned by the
 115   // getTgtPtrBegin() function which means that there is no device
 116   // corresponding point for ptr. This function should return false
 117   // in that situation.
 118   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
 119     rc = !IsHostPtr;
 120   DP("Call to omp_target_is_present returns %d\n", rc);
 121   return rc;
 122 }
 123
 124 EXTERN int omp_target_memcpy(void *dst, const void *src, size_t length,
 125                              size_t dst_offset, size_t src_offset,
 126                              int dst_device, int src_device) {
 127   TIMESCOPE();
 128   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
 129      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
 130      "src offset %zu, length %zu\n",
 131      dst_device, src_device, DPxPTR(dst), DPxPTR(src), dst_offset, src_offset,
 132      length);
 133
 134   if (!dst || !src || length <= 0) {
 135     if (length == 0) {
 136       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
 137       return OFFLOAD_SUCCESS;
 138     }
 139
 140     REPORT("Call to omp_target_memcpy with invalid arguments\n");
 141     return OFFLOAD_FAIL;
 142   }
 143
 144   if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
 145     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
 146     return OFFLOAD_FAIL;
 147   }
 148
 149   if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
 150     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
 151     return OFFLOAD_FAIL;
 152   }
 153
 154   int rc = OFFLOAD_SUCCESS;
 155   void *srcAddr = (char *)const_cast<void *>(src) + src_offset;
 156   void *dstAddr = (char *)dst + dst_offset;
 157
 158   if (src_device == omp_get_initial_device() &&
 159       dst_device == omp_get_initial_device()) {
 160     DP("copy from host to host\n");
 161     const void *p = memcpy(dstAddr, srcAddr, length);
 162     if (p == NULL)
 163       rc = OFFLOAD_FAIL;
 164   } else if (src_device == omp_get_initial_device()) {
 165     DP("copy from host to device\n");
 166     DeviceTy &DstDev = *PM->Devices[dst_device];
 167     AsyncInfoTy AsyncInfo(DstDev);
 168     rc = DstDev.submitData(dstAddr, srcAddr, length, AsyncInfo);
 169   } else if (dst_device == omp_get_initial_device()) {
 170     DP("copy from device to host\n");
 171     DeviceTy &SrcDev = *PM->Devices[src_device];
 172     AsyncInfoTy AsyncInfo(SrcDev);
 173     rc = SrcDev.retrieveData(dstAddr, srcAddr, length, AsyncInfo);
 174   } else {
 175     DP("copy from device to device\n");
 176     DeviceTy &SrcDev = *PM->Devices[src_device];
 177     DeviceTy &DstDev = *PM->Devices[dst_device];
 178     // First try to use D2D memcpy which is more efficient. If fails, fall back
 179     // to unefficient way.
 180     if (SrcDev.isDataExchangable(DstDev)) {
 181       AsyncInfoTy AsyncInfo(SrcDev);
 182       rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, AsyncInfo);
 183       if (rc == OFFLOAD_SUCCESS)
 184         return OFFLOAD_SUCCESS;
 185     }
 186
 187     void *buffer = malloc(length);
 188     {
 189       AsyncInfoTy AsyncInfo(SrcDev);
 190       rc = SrcDev.retrieveData(buffer, srcAddr, length, AsyncInfo);
 191     }
 192     if (rc == OFFLOAD_SUCCESS) {
 193       AsyncInfoTy AsyncInfo(SrcDev);
 194       rc = DstDev.submitData(dstAddr, buffer, length, AsyncInfo);
 195     }
 196     free(buffer);
 197   }
 198
 199   DP("omp_target_memcpy returns %d\n", rc);
 200   return rc;
 201 }
 202
 203 EXTERN int omp_target_memcpy_rect(
 204     void *dst, const void *src, size_t element_size, int num_dims,
 205     const size_t *volume, const size_t *dst_offsets, const size_t *src_offsets,
 206     const size_t *dst_dimensions, const size_t *src_dimensions, int dst_device,
 207     int src_device) {
 208   TIMESCOPE();
 209   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
 210      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
 211      "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
 212      "volume " DPxMOD ", element size %zu, num_dims %d\n",
 213      dst_device, src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
 214      DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
 215      DPxPTR(volume), element_size, num_dims);
 216
 217   if (!(dst || src)) {
 218     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
 219        INT_MAX);
 220     return INT_MAX;
 221   }
 222
 223   if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
 224       !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
 225     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
 226     return OFFLOAD_FAIL;
 227   }
 228
 229   int rc;
 230   if (num_dims == 1) {
 231     rc = omp_target_memcpy(
 232         dst, src, element_size * volume[0], element_size * dst_offsets[0],
 233         element_size * src_offsets[0], dst_device, src_device);
 234   } else {
 235     size_t dst_slice_size = element_size;
 236     size_t src_slice_size = element_size;
 237     for (int i = 1; i < num_dims; ++i) {
 238       dst_slice_size *= dst_dimensions[i];
 239       src_slice_size *= src_dimensions[i];
 240     }
 241
 242     size_t dst_off = dst_offsets[0] * dst_slice_size;
 243     size_t src_off = src_offsets[0] * src_slice_size;
 244     for (size_t i = 0; i < volume[0]; ++i) {
 245       rc = omp_target_memcpy_rect(
 246           (char *)dst + dst_off + dst_slice_size * i,
 247           (char *)const_cast<void *>(src) + src_off + src_slice_size * i,
 248           element_size, num_dims - 1, volume + 1, dst_offsets + 1,
 249           src_offsets + 1, dst_dimensions + 1, src_dimensions + 1, dst_device,
 250           src_device);
 251
 252       if (rc) {
 253         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
 254         return rc;
 255       }
 256     }
 257   }
 258
 259   DP("omp_target_memcpy_rect returns %d\n", rc);
 260   return rc;
 261 }
 262
 263 EXTERN int omp_target_associate_ptr(const void *host_ptr,
 264                                     const void *device_ptr, size_t size,
 265                                     size_t device_offset, int device_num) {
 266   TIMESCOPE();
 267   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
 268      "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
 269      DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
 270
 271   if (!host_ptr || !device_ptr || size <= 0) {
 272     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
 273     return OFFLOAD_FAIL;
 274   }
 275
 276   if (device_num == omp_get_initial_device()) {
 277     REPORT("omp_target_associate_ptr: no association possible on the host\n");
 278     return OFFLOAD_FAIL;
 279   }
 280
 281   if (!device_is_ready(device_num)) {
 282     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
 283     return OFFLOAD_FAIL;
 284   }
 285
 286   DeviceTy &Device = *PM->Devices[device_num];
 287   void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
 288   int rc = Device.associatePtr(const_cast<void *>(host_ptr),
 289                                const_cast<void *>(device_addr), size);
 290   DP("omp_target_associate_ptr returns %d\n", rc);
 291   return rc;
 292 }
 293
 294 EXTERN int omp_target_disassociate_ptr(const void *host_ptr, int device_num) {
 295   TIMESCOPE();
 296   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
 297      "device_num %d\n",
 298      DPxPTR(host_ptr), device_num);
 299
 300   if (!host_ptr) {
 301     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
 302     return OFFLOAD_FAIL;
 303   }
 304
 305   if (device_num == omp_get_initial_device()) {
 306     REPORT(
 307         "omp_target_disassociate_ptr: no association possible on the host\n");
 308     return OFFLOAD_FAIL;
 309   }
 310
 311   if (!device_is_ready(device_num)) {
 312     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
 313     return OFFLOAD_FAIL;
 314   }
 315
 316   DeviceTy &Device = *PM->Devices[device_num];
 317   int rc = Device.disassociatePtr(const_cast<void *>(host_ptr));
 318   DP("omp_target_disassociate_ptr returns %d\n", rc);
 319   return rc;
 320 }