libc/src/string/memory_utils/aarch64/memcmp_implementations.h

   1 //===-- Memcmp implementation for aarch64 -----------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 #ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H
   9 #define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H
  10
  11 #include "src/__support/macros/config.h"       // LIBC_INLINE
  12 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
  13 #include "src/string/memory_utils/op_aarch64.h"
  14 #include "src/string/memory_utils/op_generic.h"
  15 #include "src/string/memory_utils/utils.h" // MemcmpReturnType
  16
  17 namespace __llvm_libc {
  18
  19 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  20 inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
  21   if (LIBC_UNLIKELY(count >= 384)) {
  22     if (auto value = generic::Memcmp<16>::block(p1, p2))
  23       return value;
  24     align_to_next_boundary<16, Arg::P1>(p1, p2, count);
  25   }
  26   return generic::Memcmp<16>::loop_and_tail(p1, p2, count);
  27 }
  28
  29 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  30 inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
  31   if (LIBC_UNLIKELY(count >= 128)) { // [128, ∞]
  32     if (auto value = generic::Memcmp<16>::block(p1, p2))
  33       return value;
  34     align_to_next_boundary<16, Arg::P1>(p1, p2, count);
  35     return generic::Memcmp<32>::loop_and_tail(p1, p2, count);
  36   }
  37   if (generic::Bcmp<16>::block(p1, p2)) // [16, 16]
  38     return generic::Memcmp<16>::block(p1, p2);
  39   if (count < 32) // [17, 31]
  40     return generic::Memcmp<16>::tail(p1, p2, count);
  41   if (generic::Bcmp<16>::block(p1 + 16, p2 + 16)) // [32, 32]
  42     return generic::Memcmp<16>::block(p1 + 16, p2 + 16);
  43   if (count < 64) // [33, 63]
  44     return generic::Memcmp<32>::tail(p1, p2, count);
  45   // [64, 127]
  46   return generic::Memcmp<16>::loop_and_tail(p1 + 32, p2 + 32, count - 32);
  47 }
  48
  49 LIBC_INLINE MemcmpReturnType inline_memcmp_aarch64(CPtr p1, CPtr p2,
  50                                                    size_t count) {
  51   if (count == 0)
  52     return MemcmpReturnType::ZERO();
  53   if (count == 1)
  54     return generic::Memcmp<1>::block(p1, p2);
  55   if (count == 2)
  56     return generic::Memcmp<2>::block(p1, p2);
  57   if (count == 3)
  58     return generic::Memcmp<3>::block(p1, p2);
  59   if (count <= 8)
  60     return generic::Memcmp<4>::head_tail(p1, p2, count);
  61   if (count <= 16)
  62     return generic::Memcmp<8>::head_tail(p1, p2, count);
  63   if constexpr (aarch64::kNeon)
  64     return inline_memcmp_aarch64_neon_gt16(p1, p2, count);
  65   else
  66     return inline_memcmp_generic_gt16(p1, p2, count);
  67 }
  68 } // namespace __llvm_libc
  69
  70 #endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H