libc/test/src/string/memory_utils/op_tests.cpp

   1 //===-- Unittests for op_ files -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "memory_check_utils.h"
  10 #include "src/string/memory_utils/op_aarch64.h"
  11 #include "src/string/memory_utils/op_builtin.h"
  12 #include "src/string/memory_utils/op_generic.h" // LLVM_LIBC_HAS_UINT64
  13 #include "src/string/memory_utils/op_riscv.h"
  14 #include "src/string/memory_utils/op_x86.h"
  15 #include "test/UnitTest/Test.h"
  16
  17 namespace LIBC_NAMESPACE {
  18
  19 template <typename T> struct has_head_tail {
  20   template <typename C> static char sfinae(decltype(&C::head_tail));
  21   template <typename C> static uint16_t sfinae(...);
  22   static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
  23 };
  24
  25 template <typename T> struct has_loop_and_tail {
  26   template <typename C> static char sfinae(decltype(&C::loop_and_tail));
  27   template <typename C> static uint16_t sfinae(...);
  28   static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
  29 };
  30
  31 // Allocates two Buffer and extracts two spans out of them, one
  32 // aligned and one misaligned. Tests are run on both spans.
  33 struct Buffers {
  34   Buffers(size_t size)
  35       : aligned_buffer(size, Aligned::YES),
  36         misaligned_buffer(size, Aligned::NO) {}
  37
  38   // Returns two spans of 'size' bytes. The first is aligned on
  39   // Buffer::kAlign and the second one is unaligned.
  40   cpp::array<cpp::span<char>, 2> spans() {
  41     return {aligned_buffer.span(), misaligned_buffer.span()};
  42   }
  43
  44   Buffer aligned_buffer;
  45   Buffer misaligned_buffer;
  46 };
  47
  48 using MemcpyImplementations = testing::TypeList<
  49 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
  50     builtin::Memcpy<1>,  //
  51     builtin::Memcpy<2>,  //
  52     builtin::Memcpy<3>,  //
  53     builtin::Memcpy<4>,  //
  54     builtin::Memcpy<8>,  //
  55     builtin::Memcpy<16>, //
  56     builtin::Memcpy<32>, //
  57     builtin::Memcpy<64>
  58 #endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
  59     >;
  60
  61 // Convenient helper to turn a span into cpp::byte *.
  62 static inline cpp::byte *as_byte(cpp::span<char> span) {
  63   return reinterpret_cast<cpp::byte *>(span.data());
  64 }
  65
  66 // Adapt CheckMemcpy signature to op implementation signatures.
  67 template <auto FnImpl>
  68 void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
  69   FnImpl(as_byte(dst), as_byte(src), size);
  70 }
  71 template <size_t Size, auto FnImpl>
  72 void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
  73   FnImpl(as_byte(dst), as_byte(src));
  74 }
  75
  76 TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {
  77   using Impl = ParamType;
  78   constexpr size_t kSize = Impl::SIZE;
  79   { // Test block operation
  80     static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;
  81     Buffers SrcBuffer(kSize);
  82     Buffers DstBuffer(kSize);
  83     for (auto src : SrcBuffer.spans()) {
  84       Randomize(src);
  85       for (auto dst : DstBuffer.spans()) {
  86         ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));
  87       }
  88     }
  89   }
  90   { // Test head tail operations from kSize to 2 * kSize.
  91     static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;
  92     Buffer SrcBuffer(2 * kSize);
  93     Buffer DstBuffer(2 * kSize);
  94     Randomize(SrcBuffer.span());
  95     for (size_t size = kSize; size < 2 * kSize; ++size) {
  96       auto src = SrcBuffer.span().subspan(0, size);
  97       auto dst = DstBuffer.span().subspan(0, size);
  98       ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));
  99     }
 100   }
 101   { // Test loop operations from kSize to 3 * kSize.
 102     if constexpr (kSize > 1) {
 103       static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;
 104       Buffer SrcBuffer(3 * kSize);
 105       Buffer DstBuffer(3 * kSize);
 106       Randomize(SrcBuffer.span());
 107       for (size_t size = kSize; size < 3 * kSize; ++size) {
 108         auto src = SrcBuffer.span().subspan(0, size);
 109         auto dst = DstBuffer.span().subspan(0, size);
 110         ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));
 111       }
 112     }
 113   }
 114 }
 115
 116 using MemsetImplementations = testing::TypeList<
 117 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
 118     builtin::Memset<1>,  //
 119     builtin::Memset<2>,  //
 120     builtin::Memset<3>,  //
 121     builtin::Memset<4>,  //
 122     builtin::Memset<8>,  //
 123     builtin::Memset<16>, //
 124     builtin::Memset<32>, //
 125     builtin::Memset<64>,
 126 #endif
 127 #ifdef LLVM_LIBC_HAS_UINT64
 128     generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,
 129 #endif
 130 #ifdef __AVX512F__
 131     generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,
 132 #endif
 133 #ifdef __AVX__
 134     generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,
 135 #endif
 136 #ifdef __SSE2__
 137     generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,
 138 #endif
 139     generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //
 140     generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //
 141     generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>,   //
 142     generic::MemsetSequence<uint8_t, uint8_t>,                           //
 143     generic::MemsetSequence<uint16_t, uint8_t>,                          //
 144     generic::MemsetSequence<uint32_t, uint16_t, uint8_t>                 //
 145     >;
 146
 147 // Adapt CheckMemset signature to op implementation signatures.
 148 template <auto FnImpl>
 149 void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
 150   FnImpl(as_byte(dst), value, size);
 151 }
 152 template <size_t Size, auto FnImpl>
 153 void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
 154   FnImpl(as_byte(dst), value);
 155 }
 156
 157 TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
 158   using Impl = ParamType;
 159   constexpr size_t kSize = Impl::SIZE;
 160   { // Test block operation
 161     static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;
 162     Buffers DstBuffer(kSize);
 163     for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {
 164       for (auto dst : DstBuffer.spans()) {
 165         ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));
 166       }
 167     }
 168   }
 169   if constexpr (has_head_tail<Impl>::value) {
 170     // Test head tail operations from kSize to 2 * kSize.
 171     static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
 172     Buffer DstBuffer(2 * kSize);
 173     for (size_t size = kSize; size < 2 * kSize; ++size) {
 174       const char value = size % 10;
 175       auto dst = DstBuffer.span().subspan(0, size);
 176       ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
 177     }
 178   }
 179   if constexpr (has_loop_and_tail<Impl>::value) {
 180     // Test loop operations from kSize to 3 * kSize.
 181     if constexpr (kSize > 1) {
 182       static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
 183       Buffer DstBuffer(3 * kSize);
 184       for (size_t size = kSize; size < 3 * kSize; ++size) {
 185         const char value = size % 10;
 186         auto dst = DstBuffer.span().subspan(0, size);
 187         ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
 188       }
 189     }
 190   }
 191 }
 192
 193 using BcmpImplementations = testing::TypeList<
 194 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 195 #ifdef __SSE4_1__
 196     generic::Bcmp<__m128i>,
 197 #endif // __SSE4_1__
 198 #ifdef __AVX2__
 199     generic::Bcmp<__m256i>,
 200 #endif // __AVX2__
 201 #ifdef __AVX512BW__
 202     generic::Bcmp<__m512i>,
 203 #endif // __AVX512BW__
 204
 205 #endif // LIBC_TARGET_ARCH_IS_X86_64
 206 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 207     aarch64::Bcmp<16>, //
 208     aarch64::Bcmp<32>,
 209 #endif
 210 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
 211     generic::Bcmp<uint16_t>,
 212     generic::Bcmp<uint32_t>, //
 213 #ifdef LLVM_LIBC_HAS_UINT64
 214     generic::Bcmp<uint64_t>,
 215 #endif // LLVM_LIBC_HAS_UINT64
 216     generic::BcmpSequence<uint16_t, uint8_t>,
 217     generic::BcmpSequence<uint32_t, uint8_t>,  //
 218     generic::BcmpSequence<uint32_t, uint16_t>, //
 219     generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,
 220 #endif // LIBC_TARGET_ARCH_IS_ARM
 221     generic::BcmpSequence<uint8_t, uint8_t>,
 222     generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //
 223     generic::Bcmp<uint8_t>>;
 224
 225 // Adapt CheckBcmp signature to op implementation signatures.
 226 template <auto FnImpl>
 227 int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
 228   return (int)FnImpl(as_byte(p1), as_byte(p2), size);
 229 }
 230 template <size_t Size, auto FnImpl>
 231 int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
 232   return (int)FnImpl(as_byte(p1), as_byte(p2));
 233 }
 234
 235 TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {
 236   using Impl = ParamType;
 237   constexpr size_t kSize = Impl::SIZE;
 238   { // Test block operation
 239     static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
 240     Buffers Buffer1(kSize);
 241     Buffers Buffer2(kSize);
 242     for (auto span1 : Buffer1.spans()) {
 243       Randomize(span1);
 244       for (auto span2 : Buffer2.spans())
 245         ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));
 246     }
 247   }
 248   if constexpr (has_head_tail<Impl>::value) {
 249     // Test head tail operations from kSize to 2 * kSize.
 250     static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
 251     Buffer Buffer1(2 * kSize);
 252     Buffer Buffer2(2 * kSize);
 253     Randomize(Buffer1.span());
 254     for (size_t size = kSize; size < 2 * kSize; ++size) {
 255       auto span1 = Buffer1.span().subspan(0, size);
 256       auto span2 = Buffer2.span().subspan(0, size);
 257       ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));
 258     }
 259   }
 260   if constexpr (has_loop_and_tail<Impl>::value) {
 261     // Test loop operations from kSize to 3 * kSize.
 262     if constexpr (kSize > 1) {
 263       static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
 264       Buffer Buffer1(3 * kSize);
 265       Buffer Buffer2(3 * kSize);
 266       Randomize(Buffer1.span());
 267       for (size_t size = kSize; size < 3 * kSize; ++size) {
 268         auto span1 = Buffer1.span().subspan(0, size);
 269         auto span2 = Buffer2.span().subspan(0, size);
 270         ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));
 271       }
 272     }
 273   }
 274 }
 275
 276 using MemcmpImplementations = testing::TypeList<
 277 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 278 #ifdef __SSE2__
 279     generic::Memcmp<__m128i>, //
 280 #endif
 281 #ifdef __AVX2__
 282     generic::Memcmp<__m256i>, //
 283 #endif
 284 #ifdef __AVX512BW__
 285     generic::Memcmp<__m512i>, //
 286 #endif
 287 #endif // LIBC_TARGET_ARCH_IS_X86_64
 288 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 289     generic::Memcmp<uint8x16_t>, //
 290     generic::Memcmp<uint8x16x2_t>,
 291 #endif
 292 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
 293     generic::Memcmp<uint16_t>,
 294     generic::Memcmp<uint32_t>, //
 295 #ifdef LLVM_LIBC_HAS_UINT64
 296     generic::Memcmp<uint64_t>,
 297 #endif // LLVM_LIBC_HAS_UINT64
 298     generic::MemcmpSequence<uint16_t, uint8_t>,
 299     generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //
 300 #endif // LIBC_TARGET_ARCH_IS_ARM
 301     generic::MemcmpSequence<uint8_t, uint8_t>,
 302     generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,
 303     generic::Memcmp<uint8_t>>;
 304
 305 TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {
 306   using Impl = ParamType;
 307   constexpr size_t kSize = Impl::SIZE;
 308   { // Test block operation
 309     static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
 310     Buffers Buffer1(kSize);
 311     Buffers Buffer2(kSize);
 312     for (auto span1 : Buffer1.spans()) {
 313       Randomize(span1);
 314       for (auto span2 : Buffer2.spans())
 315         ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));
 316     }
 317   }
 318   if constexpr (has_head_tail<Impl>::value) {
 319     // Test head tail operations from kSize to 2 * kSize.
 320     static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
 321     Buffer Buffer1(2 * kSize);
 322     Buffer Buffer2(2 * kSize);
 323     Randomize(Buffer1.span());
 324     for (size_t size = kSize; size < 2 * kSize; ++size) {
 325       auto span1 = Buffer1.span().subspan(0, size);
 326       auto span2 = Buffer2.span().subspan(0, size);
 327       ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));
 328     }
 329   }
 330   if constexpr (has_loop_and_tail<Impl>::value) {
 331     // Test loop operations from kSize to 3 * kSize.
 332     if constexpr (kSize > 1) {
 333       static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
 334       Buffer Buffer1(3 * kSize);
 335       Buffer Buffer2(3 * kSize);
 336       Randomize(Buffer1.span());
 337       for (size_t size = kSize; size < 3 * kSize; ++size) {
 338         auto span1 = Buffer1.span().subspan(0, size);
 339         auto span2 = Buffer2.span().subspan(0, size);
 340         ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));
 341       }
 342     }
 343   }
 344 }
 345
 346 } // namespace LIBC_NAMESPACE