libc/src/string/memory_utils/x86_64/memcmp_implementations.h

   1 //===-- Memcmp implementation for x86_64 ------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 #ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H
   9 #define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H
  10
  11 #include "src/__support/macros/config.h"       // LIBC_INLINE
  12 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
  13 #include "src/string/memory_utils/op_generic.h"
  14 #include "src/string/memory_utils/op_x86.h"
  15 #include "src/string/memory_utils/utils.h" // MemcmpReturnType
  16
  17 namespace __llvm_libc {
  18
  19 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  20 inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
  21   if (LIBC_UNLIKELY(count >= 384)) {
  22     if (auto value = generic::Memcmp<16>::block(p1, p2))
  23       return value;
  24     align_to_next_boundary<16, Arg::P1>(p1, p2, count);
  25   }
  26   return generic::Memcmp<16>::loop_and_tail(p1, p2, count);
  27 }
  28
  29 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  30 inline_memcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) {
  31   if (LIBC_UNLIKELY(count >= 384)) {
  32     if (auto value = x86::sse2::Memcmp<16>::block(p1, p2))
  33       return value;
  34     align_to_next_boundary<16, Arg::P1>(p1, p2, count);
  35   }
  36   return x86::sse2::Memcmp<16>::loop_and_tail(p1, p2, count);
  37 }
  38
  39 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  40 inline_memcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) {
  41   if (count <= 32)
  42     return x86::sse2::Memcmp<16>::head_tail(p1, p2, count);
  43   if (count <= 64)
  44     return x86::avx2::Memcmp<32>::head_tail(p1, p2, count);
  45   if (count <= 128)
  46     return x86::avx2::Memcmp<64>::head_tail(p1, p2, count);
  47   if (LIBC_UNLIKELY(count >= 384)) {
  48     if (auto value = x86::avx2::Memcmp<32>::block(p1, p2))
  49       return value;
  50     align_to_next_boundary<32, Arg::P1>(p1, p2, count);
  51   }
  52   return x86::avx2::Memcmp<32>::loop_and_tail(p1, p2, count);
  53 }
  54
  55 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
  56 inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
  57   if (count <= 32)
  58     return x86::sse2::Memcmp<16>::head_tail(p1, p2, count);
  59   if (count <= 64)
  60     return x86::avx2::Memcmp<32>::head_tail(p1, p2, count);
  61   if (count <= 128)
  62     return x86::avx512bw::Memcmp<64>::head_tail(p1, p2, count);
  63   if (LIBC_UNLIKELY(count >= 384)) {
  64     if (auto value = x86::avx512bw::Memcmp<64>::block(p1, p2))
  65       return value;
  66     align_to_next_boundary<64, Arg::P1>(p1, p2, count);
  67   }
  68   return x86::avx512bw::Memcmp<64>::loop_and_tail(p1, p2, count);
  69 }
  70
  71 LIBC_INLINE MemcmpReturnType inline_memcmp_x86(CPtr p1, CPtr p2, size_t count) {
  72
  73   if (count == 0)
  74     return MemcmpReturnType::ZERO();
  75   if (count == 1)
  76     return generic::Memcmp<1>::block(p1, p2);
  77   if (count == 2)
  78     return generic::Memcmp<2>::block(p1, p2);
  79   if (count == 3)
  80     return generic::Memcmp<3>::block(p1, p2);
  81   if (count <= 8)
  82     return generic::Memcmp<4>::head_tail(p1, p2, count);
  83   if (count <= 16)
  84     return generic::Memcmp<8>::head_tail(p1, p2, count);
  85   if constexpr (x86::kAvx512BW)
  86     return inline_memcmp_x86_avx512bw_gt16(p1, p2, count);
  87   else if constexpr (x86::kAvx2)
  88     return inline_memcmp_x86_avx2_gt16(p1, p2, count);
  89   else if constexpr (x86::kSse2)
  90     return inline_memcmp_x86_sse2_gt16(p1, p2, count);
  91   else
  92     return inline_memcmp_generic_gt16(p1, p2, count);
  93 }
  94 } // namespace __llvm_libc
  95
  96 #endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCMP_IMPLEMENTATIONS_H