libc/src/string/memory_utils/memset_implementations.h

   1 //===-- Implementation of memset and bzero --------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
  10 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
  11
  12 #include "src/__support/common.h"
  13 #include "src/__support/macros/optimization.h"
  14 #include "src/__support/macros/properties/architectures.h"
  15 #include "src/string/memory_utils/op_aarch64.h"
  16 #include "src/string/memory_utils/op_builtin.h"
  17 #include "src/string/memory_utils/op_generic.h"
  18 #include "src/string/memory_utils/op_x86.h"
  19 #include "src/string/memory_utils/utils.h"
  20
  21 #include <stddef.h> // size_t
  22
  23 namespace __llvm_libc {
  24
  25 [[maybe_unused]] LIBC_INLINE static void
  26 inline_memset_byte_per_byte(Ptr dst, size_t offset, uint8_t value,
  27                             size_t count) {
  28   LIBC_LOOP_NOUNROLL
  29   for (; offset < count; ++offset)
  30     generic::Memset<uint8_t>::block(dst + offset, value);
  31 }
  32
  33 [[maybe_unused]] LIBC_INLINE static void
  34 inline_memset_aligned_access_32bit(Ptr dst, uint8_t value, size_t count) {
  35   constexpr size_t kAlign = sizeof(uint32_t);
  36   if (count <= 2 * kAlign)
  37     return inline_memset_byte_per_byte(dst, 0, value, count);
  38   size_t bytes_to_dst_align = distance_to_align_up<kAlign>(dst);
  39   inline_memset_byte_per_byte(dst, 0, value, bytes_to_dst_align);
  40   size_t offset = bytes_to_dst_align;
  41   for (; offset < count - kAlign; offset += kAlign)
  42     store32_aligned<uint32_t>(generic::splat<uint32_t>(value), dst, offset);
  43   inline_memset_byte_per_byte(dst, offset, value, count);
  44 }
  45
  46 [[maybe_unused]] LIBC_INLINE static void
  47 inline_memset_aligned_access_64bit(Ptr dst, uint8_t value, size_t count) {
  48   constexpr size_t kAlign = sizeof(uint64_t);
  49   if (count <= 2 * kAlign)
  50     return inline_memset_byte_per_byte(dst, 0, value, count);
  51   size_t bytes_to_dst_align = distance_to_align_up<kAlign>(dst);
  52   inline_memset_byte_per_byte(dst, 0, value, bytes_to_dst_align);
  53   size_t offset = bytes_to_dst_align;
  54   for (; offset < count - kAlign; offset += kAlign)
  55     store64_aligned<uint64_t>(generic::splat<uint64_t>(value), dst, offset);
  56   inline_memset_byte_per_byte(dst, offset, value, count);
  57 }
  58
  59 #if defined(LIBC_TARGET_ARCH_IS_X86)
  60 [[maybe_unused]] LIBC_INLINE static void
  61 inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
  62 #if defined(__AVX512F__)
  63   using uint128_t = uint8x16_t;
  64   using uint256_t = uint8x32_t;
  65   using uint512_t = uint8x64_t;
  66 #elif defined(__AVX__)
  67   using uint128_t = uint8x16_t;
  68   using uint256_t = uint8x32_t;
  69   using uint512_t = cpp::array<uint8x32_t, 2>;
  70 #elif defined(__SSE2__)
  71   using uint128_t = uint8x16_t;
  72   using uint256_t = cpp::array<uint8x16_t, 2>;
  73   using uint512_t = cpp::array<uint8x16_t, 4>;
  74 #else
  75   using uint128_t = cpp::array<uint64_t, 2>;
  76   using uint256_t = cpp::array<uint64_t, 4>;
  77   using uint512_t = cpp::array<uint64_t, 8>;
  78 #endif
  79
  80   if (count == 0)
  81     return;
  82   if (count == 1)
  83     return generic::Memset<uint8_t>::block(dst, value);
  84   if (count == 2)
  85     return generic::Memset<uint16_t>::block(dst, value);
  86   if (count == 3)
  87     return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
  88   if (count <= 8)
  89     return generic::Memset<uint32_t>::head_tail(dst, value, count);
  90   if (count <= 16)
  91     return generic::Memset<uint64_t>::head_tail(dst, value, count);
  92   if (count <= 32)
  93     return generic::Memset<uint128_t>::head_tail(dst, value, count);
  94   if (count <= 64)
  95     return generic::Memset<uint256_t>::head_tail(dst, value, count);
  96   if (count <= 128)
  97     return generic::Memset<uint512_t>::head_tail(dst, value, count);
  98   // Aligned loop
  99   generic::Memset<uint256_t>::block(dst, value);
 100   align_to_next_boundary<32>(dst, count);
 101   return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
 102 }
 103 #endif // defined(LIBC_TARGET_ARCH_IS_X86)
 104
 105 #if defined(LIBC_TARGET_ARCH_IS_AARCH64)
 106 [[maybe_unused]] LIBC_INLINE static void
 107 inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
 108   static_assert(aarch64::kNeon, "aarch64 supports vector types");
 109   using uint128_t = uint8x16_t;
 110   using uint256_t = uint8x32_t;
 111   using uint512_t = uint8x64_t;
 112   if (count == 0)
 113     return;
 114   if (count <= 3) {
 115     generic::Memset<uint8_t>::block(dst, value);
 116     if (count > 1)
 117       generic::Memset<uint16_t>::tail(dst, value, count);
 118     return;
 119   }
 120   if (count <= 8)
 121     return generic::Memset<uint32_t>::head_tail(dst, value, count);
 122   if (count <= 16)
 123     return generic::Memset<uint64_t>::head_tail(dst, value, count);
 124   if (count <= 32)
 125     return generic::Memset<uint128_t>::head_tail(dst, value, count);
 126   if (count <= (32 + 64)) {
 127     generic::Memset<uint256_t>::block(dst, value);
 128     if (count <= 64)
 129       return generic::Memset<uint256_t>::tail(dst, value, count);
 130     generic::Memset<uint256_t>::block(dst + 32, value);
 131     generic::Memset<uint256_t>::tail(dst, value, count);
 132     return;
 133   }
 134   if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
 135     generic::Memset<uint512_t>::block(dst, 0);
 136     align_to_next_boundary<64>(dst, count);
 137     return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
 138   } else {
 139     generic::Memset<uint128_t>::block(dst, value);
 140     align_to_next_boundary<16>(dst, count);
 141     return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
 142   }
 143 }
 144 #endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)
 145
 146 LIBC_INLINE static void inline_memset(Ptr dst, uint8_t value, size_t count) {
 147 #if defined(LIBC_TARGET_ARCH_IS_X86)
 148   return inline_memset_x86(dst, value, count);
 149 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
 150   return inline_memset_aarch64(dst, value, count);
 151 #elif defined(LIBC_TARGET_ARCH_IS_RISCV64)
 152   return inline_memset_aligned_access_64bit(dst, value, count);
 153 #elif defined(LIBC_TARGET_ARCH_IS_RISCV32)
 154   return inline_memset_aligned_access_32bit(dst, value, count);
 155 #else
 156   return inline_memset_byte_per_byte(dst, 0, value, count);
 157 #endif
 158 }
 159
 160 LIBC_INLINE static void inline_memset(void *dst, uint8_t value, size_t count) {
 161   inline_memset(reinterpret_cast<Ptr>(dst), value, count);
 162 }
 163
 164 } // namespace __llvm_libc
 165
 166 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H