clang/lib/Headers/avx512vlbf16intrin.h

   1 /*===--------- avx512vlbf16intrin.h - AVX512_BF16 intrinsics ---------------===
   2  *
   3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4  * See https://llvm.org/LICENSE.txt for license information.
   5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6  *
   7  *===-----------------------------------------------------------------------===
   8  */
   9 #ifndef __IMMINTRIN_H
  10 #error "Never use <avx512vlbf16intrin.h> directly; include <immintrin.h> instead."
  11 #endif
  12
  13 #ifdef __SSE2__
  14
  15 #ifndef __AVX512VLBF16INTRIN_H
  16 #define __AVX512VLBF16INTRIN_H
  17
  18 #define __DEFAULT_FN_ATTRS128                                                  \
  19   __attribute__((__always_inline__, __nodebug__,                               \
  20                  __target__("avx512vl,avx512bf16,no-evex512"),                 \
  21                  __min_vector_width__(128)))
  22 #define __DEFAULT_FN_ATTRS256                                                  \
  23   __attribute__((__always_inline__, __nodebug__,                               \
  24                  __target__("avx512vl,avx512bf16,no-evex512"),                 \
  25                  __min_vector_width__(256)))
  26
  27 /// Convert Two Packed Single Data to One Packed BF16 Data.
  28 ///
  29 /// \headerfile <x86intrin.h>
  30 ///
  31 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
  32 ///
  33 /// \param __A
  34 ///    A 128-bit vector of [4 x float].
  35 /// \param __B
  36 ///    A 128-bit vector of [4 x float].
  37 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
  38 ///    conversion of __B, and higher 64 bits come from conversion of __A.
  39 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
  40 _mm_cvtne2ps_pbh(__m128 __A, __m128 __B) {
  41   return (__m128bh)__builtin_ia32_cvtne2ps2bf16_128((__v4sf) __A,
  42                                                     (__v4sf) __B);
  43 }
  44
  45 /// Convert Two Packed Single Data to One Packed BF16 Data.
  46 ///
  47 /// \headerfile <x86intrin.h>
  48 ///
  49 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
  50 ///
  51 /// \param __A
  52 ///    A 128-bit vector of [4 x float].
  53 /// \param __B
  54 ///    A 128-bit vector of [4 x float].
  55 /// \param __W
  56 ///    A 128-bit vector of [8 x bfloat].
  57 /// \param __U
  58 ///    A 8-bit mask value specifying what is chosen for each element.
  59 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
  60 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
  61 ///    conversion of __B, and higher 64 bits come from conversion of __A.
  62 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
  63 _mm_mask_cvtne2ps_pbh(__m128bh __W, __mmask8 __U, __m128 __A, __m128 __B) {
  64   return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U,
  65                                              (__v8bf)_mm_cvtne2ps_pbh(__A, __B),
  66                                              (__v8bf)__W);
  67 }
  68
  69 /// Convert Two Packed Single Data to One Packed BF16 Data.
  70 ///
  71 /// \headerfile <x86intrin.h>
  72 ///
  73 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
  74 ///
  75 /// \param __A
  76 ///    A 128-bit vector of [4 x float].
  77 /// \param __B
  78 ///    A 128-bit vector of [4 x float].
  79 /// \param __U
  80 ///    A 8-bit mask value specifying what is chosen for each element.
  81 ///    A 1 means conversion of __A or __B. A 0 means element is zero.
  82 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
  83 ///    conversion of __B, and higher 64 bits come from conversion of __A.
  84 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
  85 _mm_maskz_cvtne2ps_pbh(__mmask8 __U, __m128 __A, __m128 __B) {
  86   return (__m128bh)__builtin_ia32_selectpbf_128((__mmask8)__U,
  87                                              (__v8bf)_mm_cvtne2ps_pbh(__A, __B),
  88                                              (__v8bf)_mm_setzero_si128());
  89 }
  90
  91 /// Convert Two Packed Single Data to One Packed BF16 Data.
  92 ///
  93 /// \headerfile <x86intrin.h>
  94 ///
  95 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
  96 ///
  97 /// \param __A
  98 ///    A 256-bit vector of [8 x float].
  99 /// \param __B
 100 ///    A 256-bit vector of [8 x float].
 101 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
 102 ///    conversion of __B, and higher 128 bits come from conversion of __A.
 103 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 104 _mm256_cvtne2ps_pbh(__m256 __A, __m256 __B) {
 105   return (__m256bh)__builtin_ia32_cvtne2ps2bf16_256((__v8sf) __A,
 106                                                     (__v8sf) __B);
 107 }
 108
 109 /// Convert Two Packed Single Data to One Packed BF16 Data.
 110 ///
 111 /// \headerfile <x86intrin.h>
 112 ///
 113 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
 114 ///
 115 /// \param __A
 116 ///    A 256-bit vector of [8 x float].
 117 /// \param __B
 118 ///    A 256-bit vector of [8 x float].
 119 /// \param __W
 120 ///    A 256-bit vector of [16 x bfloat].
 121 /// \param __U
 122 ///    A 16-bit mask value specifying what is chosen for each element.
 123 ///    A 1 means conversion of __A or __B. A 0 means element from __W.
 124 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
 125 ///    conversion of __B, and higher 128 bits come from conversion of __A.
 126 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 127 _mm256_mask_cvtne2ps_pbh(__m256bh __W, __mmask16 __U, __m256 __A, __m256 __B) {
 128   return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
 129                                          (__v16bf)_mm256_cvtne2ps_pbh(__A, __B),
 130                                          (__v16bf)__W);
 131 }
 132
 133 /// Convert Two Packed Single Data to One Packed BF16 Data.
 134 ///
 135 /// \headerfile <x86intrin.h>
 136 ///
 137 /// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions.
 138 ///
 139 /// \param __A
 140 ///    A 256-bit vector of [8 x float].
 141 /// \param __B
 142 ///    A 256-bit vector of [8 x float].
 143 /// \param __U
 144 ///    A 16-bit mask value specifying what is chosen for each element.
 145 ///    A 1 means conversion of __A or __B. A 0 means element is zero.
 146 /// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from
 147 ///    conversion of __B, and higher 128 bits come from conversion of __A.
 148 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 149 _mm256_maskz_cvtne2ps_pbh(__mmask16 __U, __m256 __A, __m256 __B) {
 150   return (__m256bh)__builtin_ia32_selectpbf_256((__mmask16)__U,
 151                                          (__v16bf)_mm256_cvtne2ps_pbh(__A, __B),
 152                                          (__v16bf)_mm256_setzero_si256());
 153 }
 154
 155 /// Convert Packed Single Data to Packed BF16 Data.
 156 ///
 157 /// \headerfile <x86intrin.h>
 158 ///
 159 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 160 ///
 161 /// \param __A
 162 ///    A 128-bit vector of [4 x float].
 163 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
 164 ///    conversion of __A, and higher 64 bits are 0.
 165 #define _mm_cvtneps_pbh(A)                                                     \
 166   ((__m128bh)__builtin_ia32_vcvtneps2bf16128((__v4sf)(A)))
 167
 168 /// Convert Packed Single Data to Packed BF16 Data.
 169 ///
 170 /// \headerfile <x86intrin.h>
 171 ///
 172 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 173 ///
 174 /// \param __A
 175 ///    A 128-bit vector of [4 x float].
 176 /// \param __W
 177 ///    A 128-bit vector of [8 x bfloat].
 178 /// \param __U
 179 ///    A 4-bit mask value specifying what is chosen for each element.
 180 ///    A 1 means conversion of __A. A 0 means element from __W.
 181 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
 182 ///    conversion of __A, and higher 64 bits are 0.
 183 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
 184 _mm_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m128 __A) {
 185   return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
 186                                                         (__v8bf)__W,
 187                                                         (__mmask8)__U);
 188 }
 189
 190 /// Convert Packed Single Data to Packed BF16 Data.
 191 ///
 192 /// \headerfile <x86intrin.h>
 193 ///
 194 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 195 ///
 196 /// \param __A
 197 ///    A 128-bit vector of [4 x float].
 198 /// \param __U
 199 ///    A 4-bit mask value specifying what is chosen for each element.
 200 ///    A 1 means conversion of __A. A 0 means element is zero.
 201 /// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from
 202 ///    conversion of __A, and higher 64 bits are 0.
 203 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
 204 _mm_maskz_cvtneps_pbh(__mmask8 __U, __m128 __A) {
 205   return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A,
 206                                                     (__v8bf)_mm_setzero_si128(),
 207                                                     (__mmask8)__U);
 208 }
 209
 210 /// Convert Packed Single Data to Packed BF16 Data.
 211 ///
 212 /// \headerfile <x86intrin.h>
 213 ///
 214 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 215 ///
 216 /// \param __A
 217 ///    A 256-bit vector of [8 x float].
 218 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
 219 #define _mm256_cvtneps_pbh(A)                                                  \
 220   ((__m128bh)__builtin_ia32_vcvtneps2bf16256((__v8sf)(A)))
 221
 222 /// Convert Packed Single Data to Packed BF16 Data.
 223 ///
 224 /// \headerfile <x86intrin.h>
 225 ///
 226 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 227 ///
 228 /// \param __A
 229 ///    A 256-bit vector of [8 x float].
 230 /// \param __W
 231 ///    A 256-bit vector of [8 x bfloat].
 232 /// \param __U
 233 ///    A 8-bit mask value specifying what is chosen for each element.
 234 ///    A 1 means conversion of __A. A 0 means element from __W.
 235 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
 236 static __inline__ __m128bh __DEFAULT_FN_ATTRS256
 237 _mm256_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m256 __A) {
 238   return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
 239                                                         (__v8bf)__W,
 240                                                         (__mmask8)__U);
 241 }
 242
 243 /// Convert Packed Single Data to Packed BF16 Data.
 244 ///
 245 /// \headerfile <x86intrin.h>
 246 ///
 247 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 248 ///
 249 /// \param __A
 250 ///    A 256-bit vector of [8 x float].
 251 /// \param __U
 252 ///    A 8-bit mask value specifying what is chosen for each element.
 253 ///    A 1 means conversion of __A. A 0 means element is zero.
 254 /// \returns A 128-bit vector of [8 x bfloat] comes from conversion of __A.
 255 static __inline__ __m128bh __DEFAULT_FN_ATTRS256
 256 _mm256_maskz_cvtneps_pbh(__mmask8 __U, __m256 __A) {
 257   return (__m128bh)__builtin_ia32_cvtneps2bf16_256_mask((__v8sf)__A,
 258                                                     (__v8bf)_mm_setzero_si128(),
 259                                                     (__mmask8)__U);
 260 }
 261
 262 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 263 ///
 264 /// \headerfile <x86intrin.h>
 265 ///
 266 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 267 ///
 268 /// \param __A
 269 ///    A 128-bit vector of [8 x bfloat].
 270 /// \param __B
 271 ///    A 128-bit vector of [8 x bfloat].
 272 /// \param __D
 273 ///    A 128-bit vector of [4 x float].
 274 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
 275 ///  __A, __B and __D
 276 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 277 _mm_dpbf16_ps(__m128 __D, __m128bh __A, __m128bh __B) {
 278   return (__m128)__builtin_ia32_dpbf16ps_128((__v4sf)__D,
 279                                              (__v8bf)__A,
 280                                              (__v8bf)__B);
 281 }
 282
 283 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 284 ///
 285 /// \headerfile <x86intrin.h>
 286 ///
 287 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 288 ///
 289 /// \param __A
 290 ///    A 128-bit vector of [8 x bfloat].
 291 /// \param __B
 292 ///    A 128-bit vector of [8 x bfloat].
 293 /// \param __D
 294 ///    A 128-bit vector of [4 x float].
 295 /// \param __U
 296 ///    A 8-bit mask value specifying what is chosen for each element.
 297 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
 298 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
 299 ///  __A, __B and __D
 300 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 301 _mm_mask_dpbf16_ps(__m128 __D, __mmask8 __U, __m128bh __A, __m128bh __B) {
 302   return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
 303                                            (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
 304                                            (__v4sf)__D);
 305 }
 306
 307 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 308 ///
 309 /// \headerfile <x86intrin.h>
 310 ///
 311 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 312 ///
 313 /// \param __A
 314 ///    A 128-bit vector of [8 x bfloat].
 315 /// \param __B
 316 ///    A 128-bit vector of [8 x bfloat].
 317 /// \param __D
 318 ///    A 128-bit vector of [4 x float].
 319 /// \param __U
 320 ///    A 8-bit mask value specifying what is chosen for each element.
 321 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
 322 /// \returns A 128-bit vector of [4 x float] comes from  Dot Product of
 323 ///  __A, __B and __D
 324 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 325 _mm_maskz_dpbf16_ps(__mmask8 __U, __m128 __D, __m128bh __A, __m128bh __B) {
 326   return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
 327                                            (__v4sf)_mm_dpbf16_ps(__D, __A, __B),
 328                                            (__v4sf)_mm_setzero_si128());
 329 }
 330
 331 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 332 ///
 333 /// \headerfile <x86intrin.h>
 334 ///
 335 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 336 ///
 337 /// \param __A
 338 ///    A 256-bit vector of [16 x bfloat].
 339 /// \param __B
 340 ///    A 256-bit vector of [16 x bfloat].
 341 /// \param __D
 342 ///    A 256-bit vector of [8 x float].
 343 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
 344 ///  __A, __B and __D
 345 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 346 _mm256_dpbf16_ps(__m256 __D, __m256bh __A, __m256bh __B) {
 347   return (__m256)__builtin_ia32_dpbf16ps_256((__v8sf)__D,
 348                                              (__v16bf)__A,
 349                                              (__v16bf)__B);
 350 }
 351
 352 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 353 ///
 354 /// \headerfile <x86intrin.h>
 355 ///
 356 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 357 ///
 358 /// \param __A
 359 ///    A 256-bit vector of [16 x bfloat].
 360 /// \param __B
 361 ///    A 256-bit vector of [16 x bfloat].
 362 /// \param __D
 363 ///    A 256-bit vector of [8 x float].
 364 /// \param __U
 365 ///    A 16-bit mask value specifying what is chosen for each element.
 366 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means __D.
 367 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
 368 ///  __A, __B and __D
 369 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 370 _mm256_mask_dpbf16_ps(__m256 __D, __mmask8 __U, __m256bh __A, __m256bh __B) {
 371   return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
 372                                         (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
 373                                         (__v8sf)__D);
 374 }
 375
 376 /// Dot Product of BF16 Pairs Accumulated into Packed Single Precision.
 377 ///
 378 /// \headerfile <x86intrin.h>
 379 ///
 380 /// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions.
 381 ///
 382 /// \param __A
 383 ///    A 256-bit vector of [16 x bfloat].
 384 /// \param __B
 385 ///    A 256-bit vector of [16 x bfloat].
 386 /// \param __D
 387 ///    A 256-bit vector of [8 x float].
 388 /// \param __U
 389 ///    A 8-bit mask value specifying what is chosen for each element.
 390 ///    A 1 means __A and __B's dot product accumulated with __D. A 0 means 0.
 391 /// \returns A 256-bit vector of [8 x float] comes from  Dot Product of
 392 ///  __A, __B and __D
 393 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 394 _mm256_maskz_dpbf16_ps(__mmask8 __U, __m256 __D, __m256bh __A, __m256bh __B) {
 395   return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
 396                                         (__v8sf)_mm256_dpbf16_ps(__D, __A, __B),
 397                                         (__v8sf)_mm256_setzero_si256());
 398 }
 399
 400 /// Convert One Single float Data to One BF16 Data.
 401 ///
 402 /// \headerfile <x86intrin.h>
 403 ///
 404 /// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions.
 405 ///
 406 /// \param __A
 407 ///    A float data.
 408 /// \returns A bf16 data whose sign field and exponent field keep unchanged,
 409 ///    and fraction field is truncated to 7 bits.
 410 static __inline__ __bf16 __DEFAULT_FN_ATTRS128 _mm_cvtness_sbh(float __A) {
 411   __v4sf __V = {__A, 0, 0, 0};
 412   __v8bf __R = __builtin_ia32_cvtneps2bf16_128_mask(
 413       (__v4sf)__V, (__v8bf)_mm_undefined_si128(), (__mmask8)-1);
 414   return (__bf16)__R[0];
 415 }
 416
 417 /// Convert Packed BF16 Data to Packed float Data.
 418 ///
 419 /// \headerfile <x86intrin.h>
 420 ///
 421 /// \param __A
 422 ///    A 128-bit vector of [4 x bfloat].
 423 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
 424 static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_cvtpbh_ps(__m128bh __A) {
 425   return _mm_castsi128_ps(
 426       (__m128i)_mm_slli_epi32((__m128i)_mm_cvtepi16_epi32((__m128i)__A), 16));
 427 }
 428
 429 /// Convert Packed BF16 Data to Packed float Data.
 430 ///
 431 /// \headerfile <x86intrin.h>
 432 ///
 433 /// \param __A
 434 ///    A 128-bit vector of [8 x bfloat].
 435 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
 436 static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
 437   return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
 438       (__m256i)_mm256_cvtepi16_epi32((__m128i)__A), 16));
 439 }
 440
 441 /// Convert Packed BF16 Data to Packed float Data using zeroing mask.
 442 ///
 443 /// \headerfile <x86intrin.h>
 444 ///
 445 /// \param __U
 446 ///    A 4-bit mask. Elements are zeroed out when the corresponding mask
 447 ///    bit is not set.
 448 /// \param __A
 449 ///    A 128-bit vector of [4 x bfloat].
 450 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
 451 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 452 _mm_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
 453   return _mm_castsi128_ps((__m128i)_mm_slli_epi32(
 454       (__m128i)_mm_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
 455 }
 456
 457 /// Convert Packed BF16 Data to Packed float Data using zeroing mask.
 458 ///
 459 /// \headerfile <x86intrin.h>
 460 ///
 461 /// \param __U
 462 ///    A 8-bit mask. Elements are zeroed out when the corresponding mask
 463 ///    bit is not set.
 464 /// \param __A
 465 ///    A 128-bit vector of [8 x bfloat].
 466 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
 467 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 468 _mm256_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
 469   return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
 470       (__m256i)_mm256_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
 471 }
 472
 473 /// Convert Packed BF16 Data to Packed float Data using merging mask.
 474 ///
 475 /// \headerfile <x86intrin.h>
 476 ///
 477 /// \param __S
 478 ///    A 128-bit vector of [4 x float]. Elements are copied from __S when
 479 ///     the corresponding mask bit is not set.
 480 /// \param __U
 481 ///    A 4-bit mask. Elements are zeroed out when the corresponding mask
 482 ///    bit is not set.
 483 /// \param __A
 484 ///    A 128-bit vector of [4 x bfloat].
 485 /// \returns A 128-bit vector of [4 x float] come from conversion of __A
 486 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 487 _mm_mask_cvtpbh_ps(__m128 __S, __mmask8 __U, __m128bh __A) {
 488   return _mm_castsi128_ps((__m128i)_mm_mask_slli_epi32(
 489       (__m128i)__S, (__mmask8)__U, (__m128i)_mm_cvtepi16_epi32((__m128i)__A),
 490       16));
 491 }
 492
 493 /// Convert Packed BF16 Data to Packed float Data using merging mask.
 494 ///
 495 /// \headerfile <x86intrin.h>
 496 ///
 497 /// \param __S
 498 ///    A 256-bit vector of [8 x float]. Elements are copied from __S when
 499 ///     the corresponding mask bit is not set.
 500 /// \param __U
 501 ///    A 8-bit mask. Elements are zeroed out when the corresponding mask
 502 ///    bit is not set.
 503 /// \param __A
 504 ///    A 128-bit vector of [8 x bfloat].
 505 /// \returns A 256-bit vector of [8 x float] come from conversion of __A
 506 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 507 _mm256_mask_cvtpbh_ps(__m256 __S, __mmask8 __U, __m128bh __A) {
 508   return _mm256_castsi256_ps((__m256i)_mm256_mask_slli_epi32(
 509       (__m256i)__S, (__mmask8)__U, (__m256i)_mm256_cvtepi16_epi32((__m128i)__A),
 510       16));
 511 }
 512
 513 #undef __DEFAULT_FN_ATTRS128
 514 #undef __DEFAULT_FN_ATTRS256
 515
 516 #endif
 517 #endif