src/port/pg_popcount_avx512_choose.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_popcount_avx512_choose.c
   4  *    Test whether we can use the AVX-512 pg_popcount() implementation.
   5  *
   6  * Copyright (c) 2024, PostgreSQL Global Development Group
   7  *
   8  * IDENTIFICATION
   9  *    src/port/pg_popcount_avx512_choose.c
  10  *
  11  *-------------------------------------------------------------------------
  12  */
  13 #include "c.h"
  14
  15 #if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT)
  16 #include <cpuid.h>
  17 #endif
  18
  19 #ifdef HAVE_XSAVE_INTRINSICS
  20 #include <immintrin.h>
  21 #endif
  22
  23 #if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX)
  24 #include <intrin.h>
  25 #endif
  26
  27 #include "port/pg_bitutils.h"
  28
  29 /*
  30  * It's probably unlikely that TRY_POPCNT_FAST won't be set if we are able to
  31  * use AVX-512 intrinsics, but we check it anyway to be sure.  We piggy-back on
  32  * the function pointers that are only used when TRY_POPCNT_FAST is set.
  33  */
  34 #ifdef TRY_POPCNT_FAST
  35
  36 /*
  37  * Does CPUID say there's support for XSAVE instructions?
  38  */
  39 static inline bool
  40 xsave_available(void)
  41 {
  42         unsigned int exx[4] = {0, 0, 0, 0};
  43
  44 #if defined(HAVE__GET_CPUID)
  45         __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
  46 #elif defined(HAVE__CPUID)
  47         __cpuid(exx, 1);
  48 #else
  49 #error cpuid instruction not available
  50 #endif
  51         return (exx[2] & (1 << 27)) != 0;       /* osxsave */
  52 }
  53
  54 /*
  55  * Does XGETBV say the ZMM registers are enabled?
  56  *
  57  * NB: Caller is responsible for verifying that xsave_available() returns true
  58  * before calling this.
  59  */
  60 static inline bool
  61 zmm_regs_available(void)
  62 {
  63 #ifdef HAVE_XSAVE_INTRINSICS
  64         return (_xgetbv(0) & 0xe6) == 0xe6;
  65 #else
  66         return false;
  67 #endif
  68 }
  69
  70 /*
  71  * Does CPUID say there's support for AVX-512 popcount and byte-and-word
  72  * instructions?
  73  */
  74 static inline bool
  75 avx512_popcnt_available(void)
  76 {
  77         unsigned int exx[4] = {0, 0, 0, 0};
  78
  79 #if defined(HAVE__GET_CPUID_COUNT)
  80         __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]);
  81 #elif defined(HAVE__CPUIDEX)
  82         __cpuidex(exx, 7, 0);
  83 #else
  84 #error cpuid instruction not available
  85 #endif
  86         return (exx[2] & (1 << 14)) != 0 && /* avx512-vpopcntdq */
  87                 (exx[1] & (1 << 30)) != 0;      /* avx512-bw */
  88 }
  89
  90 /*
  91  * Returns true if the CPU supports the instructions required for the AVX-512
  92  * pg_popcount() implementation.
  93  */
  94 bool
  95 pg_popcount_avx512_available(void)
  96 {
  97         return xsave_available() &&
  98                 zmm_regs_available() &&
  99                 avx512_popcnt_available();
 100 }
 101
 102 #endif                                                  /* TRY_POPCNT_FAST */