libc/src/string/memory_utils/x86_64/memcpy_implementations.h

   1 //===-- Memcpy implementation for x86_64 ------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 #ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
   9 #define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
  10
  11 #include "src/__support/macros/config.h"       // LIBC_INLINE
  12 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
  13 #include "src/string/memory_utils/op_builtin.h"
  14 #include "src/string/memory_utils/op_x86.h"
  15 #include "src/string/memory_utils/utils.h"
  16
  17 #include <stddef.h> // size_t
  18
  19 namespace __llvm_libc {
  20
  21 [[maybe_unused]] LIBC_INLINE void
  22 inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
  23   if (count == 0)
  24     return;
  25   if (count == 1)
  26     return builtin::Memcpy<1>::block(dst, src);
  27   if (count == 2)
  28     return builtin::Memcpy<2>::block(dst, src);
  29   if (count == 3)
  30     return builtin::Memcpy<3>::block(dst, src);
  31   if (count == 4)
  32     return builtin::Memcpy<4>::block(dst, src);
  33   if (count < 8)
  34     return builtin::Memcpy<4>::head_tail(dst, src, count);
  35   if (count < 16)
  36     return builtin::Memcpy<8>::head_tail(dst, src, count);
  37   if (count < 32)
  38     return builtin::Memcpy<16>::head_tail(dst, src, count);
  39   if (count < 64)
  40     return builtin::Memcpy<32>::head_tail(dst, src, count);
  41   if (count < 128)
  42     return builtin::Memcpy<64>::head_tail(dst, src, count);
  43   if (x86::kAvx && count < 256)
  44     return builtin::Memcpy<128>::head_tail(dst, src, count);
  45   builtin::Memcpy<32>::block(dst, src);
  46   align_to_next_boundary<32, Arg::Dst>(dst, src, count);
  47   static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
  48   return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
  49 }
  50
  51 [[maybe_unused]] LIBC_INLINE void
  52 inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
  53                                            CPtr __restrict src, size_t count) {
  54   // Whether to use rep;movsb exclusively, not at all, or only above a certain
  55   // threshold.
  56 #ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
  57 #define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
  58 #endif
  59
  60 #ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
  61 #error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
  62 #endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
  63
  64 #ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
  65 #error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
  66 #endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
  67
  68   static constexpr size_t kRepMovsbThreshold =
  69       LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
  70   if constexpr (kRepMovsbThreshold == 0) {
  71     return x86::Memcpy::repmovsb(dst, src, count);
  72   } else if constexpr (kRepMovsbThreshold == size_t(-1)) {
  73     return inline_memcpy_x86(dst, src, count);
  74   } else {
  75     if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
  76       return x86::Memcpy::repmovsb(dst, src, count);
  77     else
  78       return inline_memcpy_x86(dst, src, count);
  79   }
  80 }
  81
  82 } // namespace __llvm_libc
  83
  84 #endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H