libc/src/string/memory_utils/memcpy_implementations.h

   1 //===-- Memcpy implementation -----------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
  10 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
  11
  12 #include "src/__support/macros/config.h"       // LIBC_INLINE
  13 #include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
  14 #include "src/__support/macros/properties/architectures.h"
  15 #include "src/string/memory_utils/op_builtin.h"
  16 #include "src/string/memory_utils/utils.h"
  17
  18 #include <stddef.h> // size_t
  19
  20 #if defined(LIBC_TARGET_ARCH_IS_X86)
  21 #include "src/string/memory_utils/x86_64/memcpy_implementations.h"
  22 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
  23 #include "src/string/memory_utils/aarch64/memcpy_implementations.h"
  24 #endif
  25
  26 namespace __llvm_libc {
  27
  28 [[maybe_unused]] LIBC_INLINE void
  29 inline_memcpy_byte_per_byte(Ptr dst, CPtr src, size_t offset, size_t count) {
  30   LIBC_LOOP_NOUNROLL
  31   for (; offset < count; ++offset)
  32     dst[offset] = src[offset];
  33 }
  34
  35 [[maybe_unused]] LIBC_INLINE void
  36 inline_memcpy_aligned_access_32bit(Ptr __restrict dst, CPtr __restrict src,
  37                                    size_t count) {
  38   constexpr size_t kAlign = sizeof(uint32_t);
  39   if (count <= 2 * kAlign)
  40     return inline_memcpy_byte_per_byte(dst, src, 0, count);
  41   size_t bytes_to_dst_align = distance_to_align_up<kAlign>(dst);
  42   inline_memcpy_byte_per_byte(dst, src, 0, bytes_to_dst_align);
  43   size_t offset = bytes_to_dst_align;
  44   size_t src_alignment = distance_to_align_down<kAlign>(src + offset);
  45   for (; offset < count - kAlign; offset += kAlign) {
  46     uint32_t value;
  47     if (src_alignment == 0)
  48       value = load32_aligned<uint32_t>(src, offset);
  49     else if (src_alignment == 2)
  50       value = load32_aligned<uint16_t, uint16_t>(src, offset);
  51     else
  52       value = load32_aligned<uint8_t, uint16_t, uint8_t>(src, offset);
  53     store32_aligned<uint32_t>(value, dst, offset);
  54   }
  55   // remainder
  56   inline_memcpy_byte_per_byte(dst, src, offset, count);
  57 }
  58
  59 [[maybe_unused]] LIBC_INLINE void
  60 inline_memcpy_aligned_access_64bit(Ptr __restrict dst, CPtr __restrict src,
  61                                    size_t count) {
  62   constexpr size_t kAlign = sizeof(uint64_t);
  63   if (count <= 2 * kAlign)
  64     return inline_memcpy_byte_per_byte(dst, src, 0, count);
  65   size_t bytes_to_dst_align = distance_to_align_up<kAlign>(dst);
  66   inline_memcpy_byte_per_byte(dst, src, 0, bytes_to_dst_align);
  67   size_t offset = bytes_to_dst_align;
  68   size_t src_alignment = distance_to_align_down<kAlign>(src + offset);
  69   for (; offset < count - kAlign; offset += kAlign) {
  70     uint64_t value;
  71     if (src_alignment == 0)
  72       value = load64_aligned<uint64_t>(src, offset);
  73     else if (src_alignment == 4)
  74       value = load64_aligned<uint32_t, uint32_t>(src, offset);
  75     else if (src_alignment == 2)
  76       value =
  77           load64_aligned<uint16_t, uint16_t, uint16_t, uint16_t>(src, offset);
  78     else
  79       value = load64_aligned<uint8_t, uint16_t, uint16_t, uint16_t, uint8_t>(
  80           src, offset);
  81     store64_aligned<uint64_t>(value, dst, offset);
  82   }
  83   // remainder
  84   inline_memcpy_byte_per_byte(dst, src, offset, count);
  85 }
  86
  87 LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
  88                                size_t count) {
  89   using namespace __llvm_libc::builtin;
  90 #if defined(LIBC_COPT_MEMCPY_USE_EMBEDDED_TINY)
  91   return inline_memcpy_byte_per_byte(dst, src, 0, count);
  92 #elif defined(LIBC_TARGET_ARCH_IS_X86)
  93   return inline_memcpy_x86_maybe_interpose_repmovsb(dst, src, count);
  94 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
  95   return inline_memcpy_aarch64(dst, src, count);
  96 #elif defined(LIBC_TARGET_ARCH_IS_RISCV64)
  97   return inline_memcpy_aligned_access_64bit(dst, src, count);
  98 #elif defined(LIBC_TARGET_ARCH_IS_RISCV32)
  99   return inline_memcpy_aligned_access_32bit(dst, src, count);
 100 #else
 101   return inline_memcpy_byte_per_byte(dst, src, 0, count);
 102 #endif
 103 }
 104
 105 LIBC_INLINE void inline_memcpy(void *__restrict dst, const void *__restrict src,
 106                                size_t count) {
 107   inline_memcpy(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src), count);
 108 }
 109
 110 } // namespace __llvm_libc
 111
 112 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H