src/gromacs/simd/simd_memory.h

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 2017,2018,2019, by the GROMACS development team, led by
   5  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   6  * and including many others, as listed in the AUTHORS file in the
   7  * top-level source directory and at http://www.gromacs.org.
   8  *
   9  * GROMACS is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public License
  11  * as published by the Free Software Foundation; either version 2.1
  12  * of the License, or (at your option) any later version.
  13  *
  14  * GROMACS is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with GROMACS; if not, see
  21  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  23  *
  24  * If you want to redistribute modifications to GROMACS, please
  25  * consider that scientific software is very special. Version
  26  * control is crucial - bugs must be traceable. We will be happy to
  27  * consider code for inclusion in the official distribution, but
  28  * derived work must not be called official GROMACS. Details are found
  29  * in the README & COPYING files - if they are missing, get the
  30  * official version at http://www.gromacs.org.
  31  *
  32  * To help us fund GROMACS development, we humbly ask that you cite
  33  * the research papers on the package. Check out http://www.gromacs.org.
  34  */
  35 /*! \libinternal \file
  36  * \brief Declares SimdArrayRef
  37  *
  38  * \author Roland Schulz <roland.schulz@intel.com>
  39  * \inlibraryapi
  40  * \ingroup module_simd
  41  */
  42 #ifndef GMX_SIMD_SIMD_MEMORY_H
  43 #define GMX_SIMD_SIMD_MEMORY_H
  44
  45 #include "gromacs/utility/arrayref.h"
  46 #include "gromacs/utility/gmxassert.h"
  47
  48 namespace gmx
  49 {
  50 namespace internal
  51 {
  52
  53 template<typename T>
  54 class SimdReference
  55 {
  56     private:
  57         using non_const_T = std::remove_const_t<T>;
  58         using pointer     = typename SimdTraits<T>::type*;
  59     public:
  60         //! \brief Constructor
  61         explicit SimdReference(pointer m) : m_(m) {}
  62         //! \brief Conversion method that will execute load
  63         operator non_const_T() const { return load<non_const_T>(m_); }
  64         //! \brief Assignment operator that will execute store
  65         SimdReference operator=(T o) // NOLINT(misc-unconventional-assign-operator,cppcoreguidelines-c-copy-assignment-signature)
  66         {
  67             store(m_, o);
  68             return *this;
  69         }
  70         //! \brief Addition assignment operator that will execute load+store
  71         SimdReference operator+=(T o)
  72         {
  73             store(m_, load<non_const_T>(m_) + o);
  74             return *this;
  75         }
  76         //! \brief Subtraction assignment operator that will execute load+store
  77         SimdReference operator-=(T o)
  78         {
  79             store(m_, load<non_const_T>(m_) - o);
  80             return *this;
  81         }
  82         //! \brief Multiplication assignment operator that will execute load+store
  83         SimdReference operator*=(T o)
  84         {
  85             store(m_, load<non_const_T>(m_) * o);
  86             return *this;
  87         }
  88     private:
  89         pointer const m_; //!< The pointer used to load memory
  90 };
  91
  92 template<typename T>
  93 class SimdIterator
  94 {
  95     public:
  96         //! Type for representing size of the container.
  97         using size_type         = size_t;
  98         //! Type for representing difference between two container indices.
  99         using difference_type   = std::ptrdiff_t;
 100         //! Type of values stored in the container.
 101         using value_type        = T;
 102         //! Pointer to a container element.
 103         using pointer           = typename SimdTraits<T>::type*;
 104         //! Reference to a container element.
 105         using reference         = internal::SimdReference<T>;
 106
 107         explicit SimdIterator(pointer p = 0) : p_(p)
 108         {
 109             GMX_ASSERT((reinterpret_cast<size_type>(p)/sizeof(*p))%simdWidth == 0,
 110                        "Trying to create aligned iterator for non aligned address.");
 111         }
 112         SimdIterator &operator++()
 113         {
 114             p_ += simdWidth;
 115             return *this;
 116         }
 117         SimdIterator operator++(int)
 118         {
 119             SimdIterator retval = *this;
 120             ++(*this);
 121             return retval;
 122         }
 123         SimdIterator &operator--()
 124         {
 125             p_ -= simdWidth;
 126             return *this;
 127         }
 128         SimdIterator operator--(int)
 129         {
 130             SimdIterator retval = *this;
 131             --(*this);
 132             return retval;
 133         }
 134         SimdIterator &operator+=(difference_type d)
 135         {
 136             p_ += simdWidth * d;
 137             return *this;
 138         }
 139         SimdIterator &operator-=(difference_type d)
 140         {
 141             p_ -= simdWidth * d;
 142             return *this;
 143         }
 144         SimdIterator operator+(difference_type d) { return SimdIterator(p_ + simdWidth*d); }
 145         SimdIterator operator-(difference_type d) { return SimdIterator(p_ - simdWidth*d); }
 146
 147         difference_type operator-(SimdIterator o) { return (p_ - o.p_)/simdWidth; }
 148
 149         bool operator==(SimdIterator other) const { return p_ == other.p_; }
 150         bool operator!=(SimdIterator other) const { return p_ != other.p_; }
 151         bool operator< (SimdIterator other) const { return p_ <  other.p_; }
 152         bool operator> (SimdIterator other) const { return p_ >  other.p_; }
 153         bool operator<=(SimdIterator other) const { return p_ <= other.p_; }
 154         bool operator>=(SimdIterator other) const { return p_ >= other.p_; }
 155
 156         reference operator*() const { return reference(p_); }
 157     private:
 158         pointer              p_;
 159         static constexpr int simdWidth = SimdTraits<T>::width;
 160 };
 161
 162 /*! \internal
 163  * \brief STL-like container for aligned SIMD type. Used as ArrayRef<SimdReal>.
 164  *
 165  * Should provide the same interface as ArrayRef. Any missing functions should be
 166  * added as needed. The pointer type (used e.g. for initialization) is a real
 167  * pointer. The reference type (used e.g. for operator[] and iterator dereference)
 168  * is SimdReference which executes the aligned load/store as is appropriate. For
 169  * both iterator and element access, the access happens in blocks of SIMD width.
 170  * Meaning that a[1] refers to the 2nd SIMD vector and thus reals 8-15 for 8-wide
 171  * SIMD. The starting address has to be aligned and the length has to be multiple
 172  * of the SIMD width.
 173  *
 174  * \tparam T SIMD type (e.g. SimdReal)
 175  */
 176 template<typename T>
 177 class SimdArrayRef
 178 {
 179     public:
 180         //! Type for representing size of the container.
 181         using size_type         = size_t;
 182         //! Type for representing difference between two container indices.
 183         using difference_type   = std::ptrdiff_t;
 184         //! Type of values stored in the container.
 185         using value_type        = T;
 186         //! Pointer to a container element.
 187         using pointer           = typename SimdTraits<T>::type*;
 188         //! Reference to a container element.
 189         using reference         = internal::SimdReference<T>;
 190         //! Iterator type for the container.
 191         using iterator          = SimdIterator<T>;
 192         //! Standard reverse iterator.
 193         using reverse_iterator  = std::reverse_iterator<iterator>;
 194
 195         //! \copydoc ArrayRef::ArrayRef(pointer, pointer)
 196         SimdArrayRef(pointer begin, pointer end)
 197             : begin_(begin), end_(end)
 198         {
 199             GMX_ASSERT(end >= begin, "Invalid range");
 200             GMX_ASSERT((reinterpret_cast<size_type>(begin)/sizeof(*begin))%simdWidth == 0,
 201                        "Aligned ArrayRef requires aligned starting address");
 202             GMX_ASSERT((reinterpret_cast<size_type>(end)/sizeof(*end))%simdWidth == 0,
 203                        "Size of ArrayRef needs to be divisible by type size");
 204         }
 205         //! \copydoc ArrayRef::ArrayRef(U)
 206         template<typename U,
 207                  typename = std::enable_if_t<
 208                          std::is_convertible<typename std::remove_reference_t<U>::pointer,
 209                                              pointer>::value> >
 210         SimdArrayRef(U &&o) : begin_(reinterpret_cast<pointer>(o.data())),
 211                               end_(reinterpret_cast<pointer>(o.data()+o.size())) {}
 212         //reinterpret_cast is only needed for const conversion of SimdArrayRef itself.
 213         //All other containers have type(o.data())==U::pointer (the cast does nothing).
 214
 215         //! Returns the size of the container.
 216         size_type size() const { return (end_-begin_)/simdWidth; }
 217         //! Whether the container is empty.
 218         bool empty() const { return begin_ == end_; }
 219         //! Returns an iterator to the beginning of the container.
 220         iterator begin() const { return iterator(begin_); }
 221         //! Returns an iterator to the end of the container.
 222         iterator end() const { return iterator(end_); }
 223
 224         //! Access container element.
 225         reference operator[](size_type n)
 226         {
 227             return reference(begin_+n*simdWidth);
 228         }
 229
 230         //! Returns the first element in the container.
 231         reference front() const { return reference(begin_); }
 232         //! Returns the first element in the container.
 233         reference back() const { return reference(end_ - simdWidth); }
 234     private:
 235         static constexpr int simdWidth = SimdTraits<T>::width;
 236         using pack_type = typename SimdTraits<T>::type[simdWidth];
 237         // Private because dereferencing return value is undefined behavior (strict aliasing rule)
 238         // Only use is conversion constructor above which immediately casts it back.
 239         // Return type is not "pointer" because then data()+size() would be ill defined.
 240         // Has to be pack_type and not value_type in case
 241         // sizeof(value_type)/sizeof(pointer)!=simdWidth (e.g. int32 for double SSE2).
 242         pack_type* data() const { return reinterpret_cast<pack_type*>(begin_); }
 243
 244         template<typename U>
 245         friend class SimdArrayRef;
 246
 247         pointer const        begin_;
 248         pointer const        end_;
 249 };
 250
 251 }   //namespace internal
 252
 253 /* Specialize ArraryRef<SimdReal>
 254  * So far only an aligned version is implemented. The constructor verifies that
 255  * a ArrayRef<SimdReal> is constructed only for properly aligned data.
 256  */
 257 #if GMX_SIMD_HAVE_FLOAT
 258 template<>
 259 class ArrayRef<SimdFloat> : public internal::SimdArrayRef<SimdFloat>
 260 {
 261     using Base = internal::SimdArrayRef<SimdFloat>;
 262     using Base::Base;
 263 };
 264 template<>
 265 class ArrayRef<const SimdFloat> : public internal::SimdArrayRef<const SimdFloat>
 266 {
 267     using Base = internal::SimdArrayRef<const SimdFloat>;
 268     using Base::Base;
 269 };
 270 #endif
 271 #if GMX_SIMD_HAVE_DOUBLE
 272 template<>
 273 class ArrayRef<SimdDouble> : public internal::SimdArrayRef<SimdDouble>
 274 {
 275     using Base = internal::SimdArrayRef<SimdDouble>;
 276     using Base::Base;
 277 };
 278 template<>
 279 class ArrayRef<const SimdDouble> : public internal::SimdArrayRef<const SimdDouble>
 280 {
 281     using Base = internal::SimdArrayRef<const SimdDouble>;
 282     using Base::Base;
 283 };
 284 #endif
 285
 286 } // namespace gmx
 287
 288 #endif