libc/src/string/memory_utils/memcpy_utils.h

   1 //===-- Memcpy utils --------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H
  10 #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H
  11
  12 #include "src/string/memory_utils/utils.h"
  13 #include <stddef.h> // size_t
  14
  15 // __builtin_memcpy_inline guarantees to never call external functions.
  16 // Unfortunately it is not widely available.
  17 #ifdef __clang__
  18 #if __has_builtin(__builtin_memcpy_inline)
  19 #define USE_BUILTIN_MEMCPY_INLINE
  20 #endif
  21 #elif defined(__GNUC__)
  22 #define USE_BUILTIN_MEMCPY
  23 #endif
  24
  25 namespace __llvm_libc {
  26
  27 // This is useful for testing.
  28 #if defined(LLVM_LIBC_MEMCPY_MONITOR)
  29 extern "C" void LLVM_LIBC_MEMCPY_MONITOR(char *__restrict,
  30                                          const char *__restrict, size_t);
  31 #endif
  32
  33 // Copies `kBlockSize` bytes from `src` to `dst`.
  34 template <size_t kBlockSize>
  35 static void CopyBlock(char *__restrict dst, const char *__restrict src) {
  36 #if defined(LLVM_LIBC_MEMCPY_MONITOR)
  37   LLVM_LIBC_MEMCPY_MONITOR(dst, src, kBlockSize);
  38 #elif defined(USE_BUILTIN_MEMCPY_INLINE)
  39   __builtin_memcpy_inline(dst, src, kBlockSize);
  40 #elif defined(USE_BUILTIN_MEMCPY)
  41   __builtin_memcpy(dst, src, kBlockSize);
  42 #else
  43   for (size_t i = 0; i < kBlockSize; ++i)
  44     dst[i] = src[i];
  45 #endif
  46 }
  47
  48 // Copies `kBlockSize` bytes from `src + count - kBlockSize` to
  49 // `dst + count - kBlockSize`.
  50 // Precondition: `count >= kBlockSize`.
  51 template <size_t kBlockSize>
  52 static void CopyLastBlock(char *__restrict dst, const char *__restrict src,
  53                           size_t count) {
  54   const size_t offset = count - kBlockSize;
  55   CopyBlock<kBlockSize>(dst + offset, src + offset);
  56 }
  57
  58 // Copies `kBlockSize` bytes twice with an overlap between the two.
  59 //
  60 // [1234567812345678123]
  61 // [__XXXXXXXXXXXXXX___]
  62 // [__XXXXXXXX_________]
  63 // [________XXXXXXXX___]
  64 //
  65 // Precondition: `count >= kBlockSize && count <= kBlockSize`.
  66 template <size_t kBlockSize>
  67 static void CopyBlockOverlap(char *__restrict dst, const char *__restrict src,
  68                              size_t count) {
  69   CopyBlock<kBlockSize>(dst, src);
  70   CopyLastBlock<kBlockSize>(dst, src, count);
  71 }
  72
  73 // Copies `count` bytes by blocks of `kBlockSize` bytes.
  74 // Copies at the start and end of the buffer are unaligned.
  75 // Copies in the middle of the buffer are aligned to `kBlockSize`.
  76 //
  77 // e.g. with
  78 // [12345678123456781234567812345678]
  79 // [__XXXXXXXXXXXXXXXXXXXXXXXXXXX___]
  80 // [__XXXXXXXX______________________]
  81 // [________XXXXXXXX________________]
  82 // [________________XXXXXXXX________]
  83 // [_____________________XXXXXXXX___]
  84 //
  85 // Precondition: `count > 2 * kBlockSize` for efficiency.
  86 //               `count >= kBlockSize` for correctness.
  87 template <size_t kBlockSize>
  88 static void CopyAlignedBlocks(char *__restrict dst, const char *__restrict src,
  89                               size_t count) {
  90   CopyBlock<kBlockSize>(dst, src); // Copy first block
  91
  92   // Copy aligned blocks
  93   size_t offset = kBlockSize - offset_from_last_aligned<kBlockSize>(dst);
  94   for (; offset + kBlockSize < count; offset += kBlockSize)
  95     CopyBlock<kBlockSize>(dst + offset, src + offset);
  96
  97   CopyLastBlock<kBlockSize>(dst, src, count); // Copy last block
  98 }
  99
 100 } // namespace __llvm_libc
 101
 102 #endif //  LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_UTILS_H