flang/runtime/copy.cpp

   1 //===-- runtime/copy.cpp -------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "copy.h"
  10 #include "stack.h"
  11 #include "terminator.h"
  12 #include "type-info.h"
  13 #include "flang/Runtime/allocatable.h"
  14 #include "flang/Runtime/descriptor.h"
  15 #include <cstring>
  16
  17 namespace Fortran::runtime {
  18 namespace {
  19 using StaticDescTy = StaticDescriptor<maxRank, true, 0>;
  20
  21 // A structure describing the data copy that needs to be done
  22 // from one descriptor to another. It is a helper structure
  23 // for CopyElement.
  24 struct CopyDescriptor {
  25   // A constructor specifying all members explicitly.
  26   // The toAt and fromAt specify subscript storages that might be
  27   // external to CopyElement, and cannot be modified.
  28   // The copy descriptor only establishes toAtPtr_ and fromAtPtr_
  29   // pointers to point to these storages.
  30   RT_API_ATTRS CopyDescriptor(const Descriptor &to, const SubscriptValue toAt[],
  31       const Descriptor &from, const SubscriptValue fromAt[],
  32       std::size_t elements, bool usesStaticDescriptors = false)
  33       : to_(to), from_(from), elements_(elements),
  34         usesStaticDescriptors_(usesStaticDescriptors) {
  35     toAtPtr_ = toAt;
  36     fromAtPtr_ = fromAt;
  37   }
  38   // The number of elements to copy is initialized from the to descriptor.
  39   // The current element subscripts are initialized from the lower bounds
  40   // of the to and from descriptors.
  41   RT_API_ATTRS CopyDescriptor(const Descriptor &to, const Descriptor &from,
  42       bool usesStaticDescriptors = false)
  43       : to_(to), from_(from), elements_(to.Elements()),
  44         usesStaticDescriptors_(usesStaticDescriptors) {
  45     to.GetLowerBounds(toAt_);
  46     from.GetLowerBounds(fromAt_);
  47   }
  48
  49   // Increment the toAt_ and fromAt_ subscripts to the next
  50   // element.
  51   RT_API_ATTRS void IncrementSubscripts(Terminator &terminator) {
  52     // This method must not be called for copy descriptors
  53     // using external non-modifiable subscript storage.
  54     RUNTIME_CHECK(terminator, toAt_ == toAtPtr_ && fromAt_ == fromAtPtr_);
  55     to_.IncrementSubscripts(toAt_);
  56     from_.IncrementSubscripts(fromAt_);
  57   }
  58
  59   // Descriptor of the destination.
  60   const Descriptor &to_;
  61   // A subscript specifying the current element position to copy to.
  62   SubscriptValue toAt_[maxRank];
  63   // A pointer to the storage of the 'to' subscript.
  64   // It may point to toAt_ or to an external non-modifiable
  65   // subscript storage.
  66   const SubscriptValue *toAtPtr_{toAt_};
  67   // Descriptor of the source.
  68   const Descriptor &from_;
  69   // A subscript specifying the current element position to copy from.
  70   SubscriptValue fromAt_[maxRank];
  71   // A pointer to the storage of the 'from' subscript.
  72   // It may point to fromAt_ or to an external non-modifiable
  73   // subscript storage.
  74   const SubscriptValue *fromAtPtr_{fromAt_};
  75   // Number of elements left to copy.
  76   std::size_t elements_;
  77   // Must be true, if the to and from descriptors are allocated
  78   // by the CopyElement runtime. The allocated memory belongs
  79   // to a separate stack that needs to be popped in correspondence
  80   // with popping such a CopyDescriptor node.
  81   bool usesStaticDescriptors_;
  82 };
  83
  84 // A pair of StaticDescTy elements.
  85 struct StaticDescriptorsPair {
  86   StaticDescTy to;
  87   StaticDescTy from;
  88 };
  89 } // namespace
  90
  91 RT_OFFLOAD_API_GROUP_BEGIN
  92
  93 RT_API_ATTRS void CopyElement(const Descriptor &to, const SubscriptValue toAt[],
  94     const Descriptor &from, const SubscriptValue fromAt[],
  95     Terminator &terminator) {
  96   if (!to.Addendum()) {
  97     // Avoid the overhead of creating the work stacks below
  98     // for the simple non-derived type cases, because the overhead
  99     // might be noticeable over the total amount of work that
 100     // needs to be done for the copy.
 101     char *toPtr{to.Element<char>(toAt)};
 102     char *fromPtr{from.Element<char>(fromAt)};
 103     RUNTIME_CHECK(terminator, to.ElementBytes() == from.ElementBytes());
 104     std::memcpy(toPtr, fromPtr, to.ElementBytes());
 105     return;
 106   }
 107
 108 #if !defined(RT_DEVICE_COMPILATION)
 109   constexpr unsigned copyStackReserve{16};
 110   constexpr unsigned descriptorStackReserve{6};
 111 #else
 112   // Always use dynamic allocation on the device to avoid
 113   // big stack sizes. This may be tuned as needed.
 114   constexpr unsigned copyStackReserve{0};
 115   constexpr unsigned descriptorStackReserve{0};
 116 #endif
 117   // Keep a stack of CopyDescriptor's to avoid recursive calls.
 118   Stack<CopyDescriptor, copyStackReserve> copyStack{terminator};
 119   // Keep a separate stack of StaticDescTy pairs. These descriptors
 120   // may be used for representing copies of Component::Genre::Data
 121   // components (since they do not have their descriptors allocated
 122   // in memory).
 123   Stack<StaticDescriptorsPair, descriptorStackReserve> descriptorsStack{
 124       terminator};
 125   copyStack.emplace(to, toAt, from, fromAt, /*elements=*/std::size_t{1});
 126
 127   while (!copyStack.empty()) {
 128     CopyDescriptor &currentCopy{copyStack.top()};
 129     std::size_t &elements{currentCopy.elements_};
 130     if (elements == 0) {
 131       // This copy has been exhausted.
 132       if (currentCopy.usesStaticDescriptors_) {
 133         // Pop the static descriptors, if they were used
 134         // for the current copy.
 135         descriptorsStack.pop();
 136       }
 137       copyStack.pop();
 138       continue;
 139     }
 140     const Descriptor &curTo{currentCopy.to_};
 141     const SubscriptValue *curToAt{currentCopy.toAtPtr_};
 142     const Descriptor &curFrom{currentCopy.from_};
 143     const SubscriptValue *curFromAt{currentCopy.fromAtPtr_};
 144     char *toPtr{curTo.Element<char>(curToAt)};
 145     char *fromPtr{curFrom.Element<char>(curFromAt)};
 146     RUNTIME_CHECK(terminator, curTo.ElementBytes() == curFrom.ElementBytes());
 147     // TODO: the memcpy can be optimized when both to and from are contiguous.
 148     // Moreover, if we came here from an Component::Genre::Data component,
 149     // all the per-element copies are redundant, because the parent
 150     // has already been copied as a whole.
 151     std::memcpy(toPtr, fromPtr, curTo.ElementBytes());
 152     --elements;
 153     if (elements != 0) {
 154       currentCopy.IncrementSubscripts(terminator);
 155     }
 156
 157     // Deep copy allocatable and automatic components if any.
 158     if (const auto *addendum{curTo.Addendum()}) {
 159       if (const auto *derived{addendum->derivedType()};
 160           derived && !derived->noDestructionNeeded()) {
 161         RUNTIME_CHECK(terminator,
 162             curFrom.Addendum() && derived == curFrom.Addendum()->derivedType());
 163         const Descriptor &componentDesc{derived->component()};
 164         const typeInfo::Component *component{
 165             componentDesc.OffsetElement<typeInfo::Component>()};
 166         std::size_t nComponents{componentDesc.Elements()};
 167         for (std::size_t j{0}; j < nComponents; ++j, ++component) {
 168           if (component->genre() == typeInfo::Component::Genre::Allocatable ||
 169               component->genre() == typeInfo::Component::Genre::Automatic) {
 170             Descriptor &toDesc{
 171                 *reinterpret_cast<Descriptor *>(toPtr + component->offset())};
 172             if (toDesc.raw().base_addr != nullptr) {
 173               toDesc.set_base_addr(nullptr);
 174               RUNTIME_CHECK(terminator, toDesc.Allocate() == CFI_SUCCESS);
 175               const Descriptor &fromDesc{*reinterpret_cast<const Descriptor *>(
 176                   fromPtr + component->offset())};
 177               copyStack.emplace(toDesc, fromDesc);
 178             }
 179           } else if (component->genre() == typeInfo::Component::Genre::Data &&
 180               component->derivedType() &&
 181               !component->derivedType()->noDestructionNeeded()) {
 182             SubscriptValue extents[maxRank];
 183             const typeInfo::Value *bounds{component->bounds()};
 184             std::size_t elements{1};
 185             for (int dim{0}; dim < component->rank(); ++dim) {
 186               typeInfo::TypeParameterValue lb{
 187                   bounds[2 * dim].GetValue(&curTo).value_or(0)};
 188               typeInfo::TypeParameterValue ub{
 189                   bounds[2 * dim + 1].GetValue(&curTo).value_or(0)};
 190               extents[dim] = ub >= lb ? ub - lb + 1 : 0;
 191               elements *= extents[dim];
 192             }
 193             if (elements != 0) {
 194               const typeInfo::DerivedType &compType{*component->derivedType()};
 195               // Place a pair of static descriptors onto the descriptors stack.
 196               descriptorsStack.emplace();
 197               StaticDescriptorsPair &descs{descriptorsStack.top()};
 198               Descriptor &toCompDesc{descs.to.descriptor()};
 199               toCompDesc.Establish(compType, toPtr + component->offset(),
 200                   component->rank(), extents);
 201               Descriptor &fromCompDesc{descs.from.descriptor()};
 202               fromCompDesc.Establish(compType, fromPtr + component->offset(),
 203                   component->rank(), extents);
 204               copyStack.emplace(toCompDesc, fromCompDesc,
 205                   /*usesStaticDescriptors=*/true);
 206             }
 207           }
 208         }
 209       }
 210     }
 211   }
 212 }
 213 RT_OFFLOAD_API_GROUP_END
 214 } // namespace Fortran::runtime