libc/test/src/string/memory_utils/op_tests.cpp

   1 //===-- Unittests for op_ files -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "memory_check_utils.h"
  10 #include "src/__support/macros/config.h"
  11 #include "src/__support/macros/properties/os.h"
  12 #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64
  13 #include "src/string/memory_utils/op_aarch64.h"
  14 #include "src/string/memory_utils/op_builtin.h"
  15 #include "src/string/memory_utils/op_generic.h"
  16 #include "src/string/memory_utils/op_riscv.h"
  17 #include "src/string/memory_utils/op_x86.h"
  18 #include "test/UnitTest/Test.h"
  19
  20 namespace LIBC_NAMESPACE_DECL {
  21
  22 template <typename T> struct has_head_tail {
  23   template <typename C> static char sfinae(decltype(&C::head_tail));
  24   template <typename C> static uint16_t sfinae(...);
  25   static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
  26 };
  27
  28 template <typename T> struct has_loop_and_tail {
  29   template <typename C> static char sfinae(decltype(&C::loop_and_tail));
  30   template <typename C> static uint16_t sfinae(...);
  31   static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
  32 };
  33
  34 // Allocates two Buffer and extracts two spans out of them, one
  35 // aligned and one misaligned. Tests are run on both spans.
  36 struct Buffers {
  37   Buffers(size_t size)
  38       : aligned_buffer(size, Aligned::YES),
  39         misaligned_buffer(size, Aligned::NO) {}
  40
  41   // Returns two spans of 'size' bytes. The first is aligned on
  42   // Buffer::kAlign and the second one is unaligned.
  43   cpp::array<cpp::span<char>, 2> spans() {
  44     return {aligned_buffer.span(), misaligned_buffer.span()};
  45   }
  46
  47   Buffer aligned_buffer;
  48   Buffer misaligned_buffer;
  49 };
  50
  51 using MemcpyImplementations = testing::TypeList<
  52 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
  53     builtin::Memcpy<1>,  //
  54     builtin::Memcpy<2>,  //
  55     builtin::Memcpy<3>,  //
  56     builtin::Memcpy<4>,  //
  57     builtin::Memcpy<8>,  //
  58     builtin::Memcpy<16>, //
  59     builtin::Memcpy<32>, //
  60     builtin::Memcpy<64>
  61 #endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
  62     >;
  63
  64 // Convenient helper to turn a span into cpp::byte *.
  65 static inline cpp::byte *as_byte(cpp::span<char> span) {
  66   return reinterpret_cast<cpp::byte *>(span.data());
  67 }
  68
  69 // Adapt CheckMemcpy signature to op implementation signatures.
  70 template <auto FnImpl>
  71 void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
  72   FnImpl(as_byte(dst), as_byte(src), size);
  73 }
  74 template <size_t Size, auto FnImpl>
  75 void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
  76   FnImpl(as_byte(dst), as_byte(src));
  77 }
  78
  79 TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {
  80   using Impl = ParamType;
  81   constexpr size_t kSize = Impl::SIZE;
  82   { // Test block operation
  83     static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;
  84     Buffers SrcBuffer(kSize);
  85     Buffers DstBuffer(kSize);
  86     for (auto src : SrcBuffer.spans()) {
  87       Randomize(src);
  88       for (auto dst : DstBuffer.spans()) {
  89         ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));
  90       }
  91     }
  92   }
  93   { // Test head tail operations from kSize to 2 * kSize.
  94     static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;
  95     Buffer SrcBuffer(2 * kSize);
  96     Buffer DstBuffer(2 * kSize);
  97     Randomize(SrcBuffer.span());
  98     for (size_t size = kSize; size < 2 * kSize; ++size) {
  99       auto src = SrcBuffer.span().subspan(0, size);
 100       auto dst = DstBuffer.span().subspan(0, size);
 101       ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));
 102     }
 103   }
 104   { // Test loop operations from kSize to 3 * kSize.
 105     if constexpr (kSize > 1) {
 106       static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;
 107       Buffer SrcBuffer(3 * kSize);
 108       Buffer DstBuffer(3 * kSize);
 109       Randomize(SrcBuffer.span());
 110       for (size_t size = kSize; size < 3 * kSize; ++size) {
 111         auto src = SrcBuffer.span().subspan(0, size);
 112         auto dst = DstBuffer.span().subspan(0, size);
 113         ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));
 114       }
 115     }
 116   }
 117 }
 118
 119 using MemsetImplementations = testing::TypeList<
 120 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
 121     builtin::Memset<1>,  //
 122     builtin::Memset<2>,  //
 123     builtin::Memset<3>,  //
 124     builtin::Memset<4>,  //
 125     builtin::Memset<8>,  //
 126     builtin::Memset<16>, //
 127     builtin::Memset<32>, //
 128     builtin::Memset<64>,
 129 #endif
 130 #ifdef LIBC_TYPES_HAS_INT64
 131     generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,
 132 #endif // LIBC_TYPES_HAS_INT64
 133 #ifdef __AVX512F__
 134     generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,
 135 #endif
 136 #ifdef __AVX__
 137     generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,
 138 #endif
 139 #ifdef __SSE2__
 140     generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,
 141 #endif
 142     generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //
 143     generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //
 144     generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>,   //
 145     generic::MemsetSequence<uint8_t, uint8_t>,                           //
 146     generic::MemsetSequence<uint16_t, uint8_t>,                          //
 147     generic::MemsetSequence<uint32_t, uint16_t, uint8_t>                 //
 148     >;
 149
 150 // Adapt CheckMemset signature to op implementation signatures.
 151 template <auto FnImpl>
 152 void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
 153   FnImpl(as_byte(dst), value, size);
 154 }
 155 template <size_t Size, auto FnImpl>
 156 void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
 157   FnImpl(as_byte(dst), value);
 158 }
 159
 160 TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
 161   using Impl = ParamType;
 162   constexpr size_t kSize = Impl::SIZE;
 163   { // Test block operation
 164     static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;
 165     Buffers DstBuffer(kSize);
 166     for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {
 167       for (auto dst : DstBuffer.spans()) {
 168         ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));
 169       }
 170     }
 171   }
 172   if constexpr (has_head_tail<Impl>::value) {
 173     // Test head tail operations from kSize to 2 * kSize.
 174     static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
 175     Buffer DstBuffer(2 * kSize);
 176     for (size_t size = kSize; size < 2 * kSize; ++size) {
 177       const char value = size % 10;
 178       auto dst = DstBuffer.span().subspan(0, size);
 179       ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
 180     }
 181   }
 182   if constexpr (has_loop_and_tail<Impl>::value) {
 183     // Test loop operations from kSize to 3 * kSize.
 184     if constexpr (kSize > 1) {
 185       static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
 186       Buffer DstBuffer(3 * kSize);
 187       for (size_t size = kSize; size < 3 * kSize; ++size) {
 188         const char value = size % 10;
 189         auto dst = DstBuffer.span().subspan(0, size);
 190         ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
 191       }
 192     }
 193   }
 194 }
 195
 196 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 197 // Prevent GCC warning due to ignored __aligned__ attributes when passing x86
 198 // SIMD types as template arguments.
 199 #pragma GCC diagnostic push
 200 #pragma GCC diagnostic ignored "-Wignored-attributes"
 201 #endif // LIBC_TARGET_ARCH_IS_X86_64
 202
 203 using BcmpImplementations = testing::TypeList<
 204 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 205 #ifdef __SSE4_1__
 206     generic::Bcmp<__m128i>,
 207 #endif // __SSE4_1__
 208 #ifdef __AVX2__
 209     generic::Bcmp<__m256i>,
 210 #endif // __AVX2__
 211 #ifdef __AVX512BW__
 212     generic::Bcmp<__m512i>,
 213 #endif // __AVX512BW__
 214
 215 #endif // LIBC_TARGET_ARCH_IS_X86_64
 216 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 217     aarch64::Bcmp<16>, //
 218     aarch64::Bcmp<32>,
 219 #endif
 220 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
 221     generic::Bcmp<uint16_t>,
 222     generic::Bcmp<uint32_t>, //
 223 #ifdef LIBC_TYPES_HAS_INT64
 224     generic::Bcmp<uint64_t>,
 225 #endif // LIBC_TYPES_HAS_INT64
 226     generic::BcmpSequence<uint16_t, uint8_t>,
 227     generic::BcmpSequence<uint32_t, uint8_t>,  //
 228     generic::BcmpSequence<uint32_t, uint16_t>, //
 229     generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,
 230 #endif // LIBC_TARGET_ARCH_IS_ARM
 231     generic::BcmpSequence<uint8_t, uint8_t>,
 232     generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //
 233     generic::Bcmp<uint8_t>>;
 234
 235 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 236 #pragma GCC diagnostic pop
 237 #endif // LIBC_TARGET_ARCH_IS_X86_64
 238
 239 // Adapt CheckBcmp signature to op implementation signatures.
 240 template <auto FnImpl>
 241 int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
 242   return (int)FnImpl(as_byte(p1), as_byte(p2), size);
 243 }
 244 template <size_t Size, auto FnImpl>
 245 int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
 246   return (int)FnImpl(as_byte(p1), as_byte(p2));
 247 }
 248
 249 TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {
 250   using Impl = ParamType;
 251   constexpr size_t kSize = Impl::SIZE;
 252   { // Test block operation
 253     static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
 254     Buffers Buffer1(kSize);
 255     Buffers Buffer2(kSize);
 256     for (auto span1 : Buffer1.spans()) {
 257       Randomize(span1);
 258       for (auto span2 : Buffer2.spans())
 259         ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));
 260     }
 261   }
 262   if constexpr (has_head_tail<Impl>::value) {
 263     // Test head tail operations from kSize to 2 * kSize.
 264     static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
 265     Buffer Buffer1(2 * kSize);
 266     Buffer Buffer2(2 * kSize);
 267     Randomize(Buffer1.span());
 268     for (size_t size = kSize; size < 2 * kSize; ++size) {
 269       auto span1 = Buffer1.span().subspan(0, size);
 270       auto span2 = Buffer2.span().subspan(0, size);
 271       ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));
 272     }
 273   }
 274   if constexpr (has_loop_and_tail<Impl>::value) {
 275     // Test loop operations from kSize to 3 * kSize.
 276     if constexpr (kSize > 1) {
 277       static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
 278       Buffer Buffer1(3 * kSize);
 279       Buffer Buffer2(3 * kSize);
 280       Randomize(Buffer1.span());
 281       for (size_t size = kSize; size < 3 * kSize; ++size) {
 282         auto span1 = Buffer1.span().subspan(0, size);
 283         auto span2 = Buffer2.span().subspan(0, size);
 284         ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));
 285       }
 286     }
 287   }
 288 }
 289
 290 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 291 // Prevent GCC warning due to ignored __aligned__ attributes when passing x86
 292 // SIMD types as template arguments.
 293 #pragma GCC diagnostic push
 294 #pragma GCC diagnostic ignored "-Wignored-attributes"
 295 #endif // LIBC_TARGET_ARCH_IS_X86_64
 296
 297 using MemcmpImplementations = testing::TypeList<
 298 #if defined(LIBC_TARGET_ARCH_IS_X86_64) && !defined(LIBC_TARGET_OS_IS_WINDOWS)
 299 #ifdef __SSE2__
 300     generic::Memcmp<__m128i>, //
 301 #endif
 302 #ifdef __AVX2__
 303     generic::Memcmp<__m256i>, //
 304 #endif
 305 #ifdef __AVX512BW__
 306     generic::Memcmp<__m512i>, //
 307 #endif
 308 #endif // LIBC_TARGET_ARCH_IS_X86_64
 309 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 310     generic::Memcmp<uint8x16_t>, //
 311     generic::Memcmp<uint8x16x2_t>,
 312 #endif
 313 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
 314     generic::Memcmp<uint16_t>,
 315     generic::Memcmp<uint32_t>, //
 316 #ifdef LIBC_TYPES_HAS_INT64
 317     generic::Memcmp<uint64_t>,
 318 #endif // LIBC_TYPES_HAS_INT64
 319     generic::MemcmpSequence<uint16_t, uint8_t>,
 320     generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //
 321 #endif // LIBC_TARGET_ARCH_IS_ARM
 322     generic::MemcmpSequence<uint8_t, uint8_t>,
 323     generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,
 324     generic::Memcmp<uint8_t>>;
 325
 326 #ifdef LIBC_TARGET_ARCH_IS_X86_64
 327 #pragma GCC diagnostic pop
 328 #endif // LIBC_TARGET_ARCH_IS_X86_64
 329
 330 TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {
 331   using Impl = ParamType;
 332   constexpr size_t kSize = Impl::SIZE;
 333   { // Test block operation
 334     static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
 335     Buffers Buffer1(kSize);
 336     Buffers Buffer2(kSize);
 337     for (auto span1 : Buffer1.spans()) {
 338       Randomize(span1);
 339       for (auto span2 : Buffer2.spans())
 340         ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));
 341     }
 342   }
 343   if constexpr (has_head_tail<Impl>::value) {
 344     // Test head tail operations from kSize to 2 * kSize.
 345     static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
 346     Buffer Buffer1(2 * kSize);
 347     Buffer Buffer2(2 * kSize);
 348     Randomize(Buffer1.span());
 349     for (size_t size = kSize; size < 2 * kSize; ++size) {
 350       auto span1 = Buffer1.span().subspan(0, size);
 351       auto span2 = Buffer2.span().subspan(0, size);
 352       ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));
 353     }
 354   }
 355   if constexpr (has_loop_and_tail<Impl>::value) {
 356     // Test loop operations from kSize to 3 * kSize.
 357     if constexpr (kSize > 1) {
 358       static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
 359       Buffer Buffer1(3 * kSize);
 360       Buffer Buffer2(3 * kSize);
 361       Randomize(Buffer1.span());
 362       for (size_t size = kSize; size < 3 * kSize; ++size) {
 363         auto span1 = Buffer1.span().subspan(0, size);
 364         auto span2 = Buffer2.span().subspan(0, size);
 365         ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));
 366       }
 367     }
 368   }
 369 }
 370
 371 } // namespace LIBC_NAMESPACE_DECL