libc/src/string/memory_utils/memmove_implementations.h

   1 //===-- Memmove implementation ----------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H
  10 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H
  11
  12 #include "src/__support/common.h"
  13 #include "src/__support/macros/optimization.h"
  14 #include "src/string/memory_utils/op_aarch64.h"
  15 #include "src/string/memory_utils/op_builtin.h"
  16 #include "src/string/memory_utils/op_generic.h"
  17 #include "src/string/memory_utils/op_x86.h"
  18 #include <stddef.h> // size_t, ptrdiff_t
  19
  20 namespace __llvm_libc {
  21
  22 [[maybe_unused]] LIBC_INLINE void
  23 inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) {
  24   if ((count == 0) || (dst == src))
  25     return;
  26   if (dst < src) {
  27     LIBC_LOOP_NOUNROLL
  28     for (size_t offset = 0; offset < count; ++offset)
  29       builtin::Memcpy<1>::block(dst + offset, src + offset);
  30   } else {
  31     LIBC_LOOP_NOUNROLL
  32     for (ptrdiff_t offset = count - 1; offset >= 0; --offset)
  33       builtin::Memcpy<1>::block(dst + offset, src + offset);
  34   }
  35 }
  36
  37 LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) {
  38 #if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
  39 #if defined(LIBC_TARGET_ARCH_IS_X86)
  40 #if defined(__AVX512F__)
  41   using uint128_t = uint8x16_t;
  42   using uint256_t = uint8x32_t;
  43   using uint512_t = uint8x64_t;
  44 #elif defined(__AVX__)
  45   using uint128_t = uint8x16_t;
  46   using uint256_t = uint8x32_t;
  47   using uint512_t = cpp::array<uint8x32_t, 2>;
  48 #elif defined(__SSE2__)
  49   using uint128_t = uint8x16_t;
  50   using uint256_t = cpp::array<uint8x16_t, 2>;
  51   using uint512_t = cpp::array<uint8x16_t, 4>;
  52 #else
  53   using uint128_t = cpp::array<uint64_t, 2>;
  54   using uint256_t = cpp::array<uint64_t, 4>;
  55   using uint512_t = cpp::array<uint64_t, 8>;
  56 #endif
  57 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
  58   static_assert(aarch64::kNeon, "aarch64 supports vector types");
  59   using uint128_t = uint8x16_t;
  60   using uint256_t = uint8x32_t;
  61   using uint512_t = uint8x64_t;
  62 #endif
  63   if (count == 0)
  64     return;
  65   if (count == 1)
  66     return generic::Memmove<uint8_t>::block(dst, src);
  67   if (count <= 4)
  68     return generic::Memmove<uint16_t>::head_tail(dst, src, count);
  69   if (count <= 8)
  70     return generic::Memmove<uint32_t>::head_tail(dst, src, count);
  71   if (count <= 16)
  72     return generic::Memmove<uint64_t>::head_tail(dst, src, count);
  73   if (count <= 32)
  74     return generic::Memmove<uint128_t>::head_tail(dst, src, count);
  75   if (count <= 64)
  76     return generic::Memmove<uint256_t>::head_tail(dst, src, count);
  77   if (count <= 128)
  78     return generic::Memmove<uint512_t>::head_tail(dst, src, count);
  79   if (dst < src) {
  80     generic::Memmove<uint256_t>::align_forward<Arg::Src>(dst, src, count);
  81     return generic::Memmove<uint512_t>::loop_and_tail_forward(dst, src, count);
  82   } else {
  83     generic::Memmove<uint256_t>::align_backward<Arg::Src>(dst, src, count);
  84     return generic::Memmove<uint512_t>::loop_and_tail_backward(dst, src, count);
  85   }
  86 #else
  87   return inline_memmove_embedded_tiny(dst, src, count);
  88 #endif
  89 }
  90
  91 LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) {
  92   inline_memmove(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src),
  93                  count);
  94 }
  95
  96 } // namespace __llvm_libc
  97
  98 #endif /* LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H */