libc/src/__support/FPUtil/x86_64/FMA.h

   1 //===-- x86_64 implementations of the fma function --------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H
  10 #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H
  11
  12 #include "src/__support/macros/properties/architectures.h"
  13 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
  14
  15 #if !defined(LIBC_TARGET_ARCH_IS_X86_64)
  16 #error "Invalid include"
  17 #endif
  18
  19 #if !defined(LIBC_TARGET_CPU_HAS_FMA)
  20 #error "FMA instructions are not supported"
  21 #endif
  22
  23 #include "src/__support/CPP/type_traits.h"
  24 #include <immintrin.h>
  25
  26 namespace __llvm_libc {
  27 namespace fputil {
  28
  29 template <typename T>
  30 LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
  31   float result;
  32   __m128 xmm = _mm_load_ss(&x);           // NOLINT
  33   __m128 ymm = _mm_load_ss(&y);           // NOLINT
  34   __m128 zmm = _mm_load_ss(&z);           // NOLINT
  35   __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
  36   _mm_store_ss(&result, r);               // NOLINT
  37   return result;
  38 }
  39
  40 template <typename T>
  41 LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
  42   double result;
  43   __m128d xmm = _mm_load_sd(&x);           // NOLINT
  44   __m128d ymm = _mm_load_sd(&y);           // NOLINT
  45   __m128d zmm = _mm_load_sd(&z);           // NOLINT
  46   __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
  47   _mm_store_sd(&result, r);                // NOLINT
  48   return result;
  49 }
  50
  51 } // namespace fputil
  52 } // namespace __llvm_libc
  53
  54 #endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H