1 /* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
3 This file is part of GCC.
5 GCC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
10 GCC is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GCC; see the file COPYING. If not, write to
17 the Free Software Foundation, 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA. */
20 /* As a special exception, if you include this header file into source
21 files compiled by GCC, this header file does not by itself cause
22 the resulting executable to be covered by the GNU General Public
23 License. This exception does not however invalidate any other
24 reasons why the executable file might be covered by the GNU General
27 /* Implemented from the specification included in the Intel C++ Compiler
28 User Guide and Reference, version 8.0. */
30 #ifndef _EMMINTRIN_H_INCLUDED
31 #define _EMMINTRIN_H_INCLUDED
34 #include <xmmintrin.h>
37 typedef int __v2df
__attribute__ ((mode (V2DF
)));
38 typedef int __v2di
__attribute__ ((mode (V2DI
)));
39 typedef int __v4si
__attribute__ ((mode (V4SI
)));
40 typedef int __v8hi
__attribute__ ((mode (V8HI
)));
41 typedef int __v16qi
__attribute__ ((mode (V16QI
)));
43 /* Create a selector for use with the SHUFPD instruction. */
44 #define _MM_SHUFFLE2(fp1,fp0) \
45 (((fp1) << 1) | (fp0))
47 #define __m128i __v2di
48 #define __m128d __v2df
50 /* Create a vector with element 0 as *P and the rest zero. */
51 static __inline __m128d
52 _mm_load_sd (double const *__P
)
54 return (__m128d
) __builtin_ia32_loadsd (__P
);
57 /* Create a vector with all two elements equal to *P. */
58 static __inline __m128d
59 _mm_load1_pd (double const *__P
)
61 __v2df __tmp
= __builtin_ia32_loadsd (__P
);
62 return (__m128d
) __builtin_ia32_shufpd (__tmp
, __tmp
, _MM_SHUFFLE2 (0,0));
65 static __inline __m128d
66 _mm_load_pd1 (double const *__P
)
68 return _mm_load1_pd (__P
);
71 /* Load two DPFP values from P. The address must be 16-byte aligned. */
72 static __inline __m128d
73 _mm_load_pd (double const *__P
)
75 return (__m128d
) __builtin_ia32_loadapd (__P
);
78 /* Load two DPFP values from P. The address need not be 16-byte aligned. */
79 static __inline __m128d
80 _mm_loadu_pd (double const *__P
)
82 return (__m128d
) __builtin_ia32_loadupd (__P
);
85 /* Load two DPFP values in reverse order. The address must be aligned. */
86 static __inline __m128d
87 _mm_loadr_pd (double const *__P
)
89 __v2df __tmp
= __builtin_ia32_loadapd (__P
);
90 return (__m128d
) __builtin_ia32_shufpd (__tmp
, __tmp
, _MM_SHUFFLE2 (0,1));
93 /* Create a vector with element 0 as F and the rest zero. */
94 static __inline __m128d
95 _mm_set_sd (double __F
)
97 return (__m128d
) __builtin_ia32_loadsd (&__F
);
100 /* Create a vector with all two elements equal to F. */
101 static __inline __m128d
102 _mm_set1_pd (double __F
)
104 __v2df __tmp
= __builtin_ia32_loadsd (&__F
);
105 return (__m128d
) __builtin_ia32_shufpd (__tmp
, __tmp
, _MM_SHUFFLE2 (0,0));
108 static __inline __m128d
109 _mm_set_pd1 (double __F
)
111 return _mm_set1_pd (__F
);
114 /* Create the vector [Z Y]. */
115 static __inline __m128d
116 _mm_set_pd (double __Z
, double __Y
)
118 return (__v2df
) {__Y
, __Z
};
121 /* Create the vector [Y Z]. */
122 static __inline __m128d
123 _mm_setr_pd (double __Z
, double __Y
)
125 return _mm_set_pd (__Y
, __Z
);
128 /* Create a vector of zeros. */
129 static __inline __m128d
130 _mm_setzero_pd (void)
132 return (__m128d
) __builtin_ia32_setzeropd ();
135 /* Stores the lower DPFP value. */
137 _mm_store_sd (double *__P
, __m128d __A
)
139 __builtin_ia32_storesd (__P
, (__v2df
)__A
);
142 /* Store the lower DPFP value across two words. */
144 _mm_store1_pd (double *__P
, __m128d __A
)
146 __v2df __va
= (__v2df
)__A
;
147 __v2df __tmp
= __builtin_ia32_shufpd (__va
, __va
, _MM_SHUFFLE2 (0,0));
148 __builtin_ia32_storeapd (__P
, __tmp
);
152 _mm_store_pd1 (double *__P
, __m128d __A
)
154 _mm_store1_pd (__P
, __A
);
157 /* Store two DPFP values. The address must be 16-byte aligned. */
159 _mm_store_pd (double *__P
, __m128d __A
)
161 __builtin_ia32_storeapd (__P
, (__v2df
)__A
);
164 /* Store two DPFP values. The address need not be 16-byte aligned. */
166 _mm_storeu_pd (double *__P
, __m128d __A
)
168 __builtin_ia32_storeupd (__P
, (__v2df
)__A
);
171 /* Store two DPFP values in reverse order. The address must be aligned. */
173 _mm_storer_pd (double *__P
, __m128d __A
)
175 __v2df __va
= (__v2df
)__A
;
176 __v2df __tmp
= __builtin_ia32_shufpd (__va
, __va
, _MM_SHUFFLE2 (0,1));
177 __builtin_ia32_storeapd (__P
, __tmp
);
180 /* Sets the low DPFP value of A from the low value of B. */
181 static __inline __m128d
182 _mm_move_sd (__m128d __A
, __m128d __B
)
184 return (__m128d
) __builtin_ia32_movsd ((__v2df
)__A
, (__v2df
)__B
);
188 static __inline __m128d
189 _mm_add_pd (__m128d __A
, __m128d __B
)
191 return (__m128d
)__builtin_ia32_addpd ((__v2df
)__A
, (__v2df
)__B
);
194 static __inline __m128d
195 _mm_add_sd (__m128d __A
, __m128d __B
)
197 return (__m128d
)__builtin_ia32_addsd ((__v2df
)__A
, (__v2df
)__B
);
200 static __inline __m128d
201 _mm_sub_pd (__m128d __A
, __m128d __B
)
203 return (__m128d
)__builtin_ia32_subpd ((__v2df
)__A
, (__v2df
)__B
);
206 static __inline __m128d
207 _mm_sub_sd (__m128d __A
, __m128d __B
)
209 return (__m128d
)__builtin_ia32_subsd ((__v2df
)__A
, (__v2df
)__B
);
212 static __inline __m128d
213 _mm_mul_pd (__m128d __A
, __m128d __B
)
215 return (__m128d
)__builtin_ia32_mulpd ((__v2df
)__A
, (__v2df
)__B
);
218 static __inline __m128d
219 _mm_mul_sd (__m128d __A
, __m128d __B
)
221 return (__m128d
)__builtin_ia32_mulsd ((__v2df
)__A
, (__v2df
)__B
);
224 static __inline __m128d
225 _mm_div_pd (__m128d __A
, __m128d __B
)
227 return (__m128d
)__builtin_ia32_divpd ((__v2df
)__A
, (__v2df
)__B
);
230 static __inline __m128d
231 _mm_div_sd (__m128d __A
, __m128d __B
)
233 return (__m128d
)__builtin_ia32_divsd ((__v2df
)__A
, (__v2df
)__B
);
236 static __inline __m128d
237 _mm_sqrt_pd (__m128d __A
)
239 return (__m128d
)__builtin_ia32_sqrtpd ((__v2df
)__A
);
242 /* Return pair {sqrt (A[0), B[1]}. */
243 static __inline __m128d
244 _mm_sqrt_sd (__m128d __A
, __m128d __B
)
246 __v2df __tmp
= __builtin_ia32_movsd ((__v2df
)__A
, (__v2df
)__B
);
247 return (__m128d
)__builtin_ia32_sqrtsd ((__v2df
)__tmp
);
250 static __inline __m128d
251 _mm_min_pd (__m128d __A
, __m128d __B
)
253 return (__m128d
)__builtin_ia32_minpd ((__v2df
)__A
, (__v2df
)__B
);
256 static __inline __m128d
257 _mm_min_sd (__m128d __A
, __m128d __B
)
259 return (__m128d
)__builtin_ia32_minsd ((__v2df
)__A
, (__v2df
)__B
);
262 static __inline __m128d
263 _mm_max_pd (__m128d __A
, __m128d __B
)
265 return (__m128d
)__builtin_ia32_maxpd ((__v2df
)__A
, (__v2df
)__B
);
268 static __inline __m128d
269 _mm_max_sd (__m128d __A
, __m128d __B
)
271 return (__m128d
)__builtin_ia32_maxsd ((__v2df
)__A
, (__v2df
)__B
);
274 static __inline __m128d
275 _mm_and_pd (__m128d __A
, __m128d __B
)
277 return (__m128d
)__builtin_ia32_andpd ((__v2df
)__A
, (__v2df
)__B
);
280 static __inline __m128d
281 _mm_andnot_pd (__m128d __A
, __m128d __B
)
283 return (__m128d
)__builtin_ia32_andnpd ((__v2df
)__A
, (__v2df
)__B
);
286 static __inline __m128d
287 _mm_or_pd (__m128d __A
, __m128d __B
)
289 return (__m128d
)__builtin_ia32_orpd ((__v2df
)__A
, (__v2df
)__B
);
292 static __inline __m128d
293 _mm_xor_pd (__m128d __A
, __m128d __B
)
295 return (__m128d
)__builtin_ia32_xorpd ((__v2df
)__A
, (__v2df
)__B
);
298 static __inline __m128d
299 _mm_cmpeq_pd (__m128d __A
, __m128d __B
)
301 return (__m128d
)__builtin_ia32_cmpeqpd ((__v2df
)__A
, (__v2df
)__B
);
304 static __inline __m128d
305 _mm_cmplt_pd (__m128d __A
, __m128d __B
)
307 return (__m128d
)__builtin_ia32_cmpltpd ((__v2df
)__A
, (__v2df
)__B
);
310 static __inline __m128d
311 _mm_cmple_pd (__m128d __A
, __m128d __B
)
313 return (__m128d
)__builtin_ia32_cmplepd ((__v2df
)__A
, (__v2df
)__B
);
316 static __inline __m128d
317 _mm_cmpgt_pd (__m128d __A
, __m128d __B
)
319 return (__m128d
)__builtin_ia32_cmpgtpd ((__v2df
)__A
, (__v2df
)__B
);
322 static __inline __m128d
323 _mm_cmpge_pd (__m128d __A
, __m128d __B
)
325 return (__m128d
)__builtin_ia32_cmpgepd ((__v2df
)__A
, (__v2df
)__B
);
328 static __inline __m128d
329 _mm_cmpneq_pd (__m128d __A
, __m128d __B
)
331 return (__m128d
)__builtin_ia32_cmpneqpd ((__v2df
)__A
, (__v2df
)__B
);
334 static __inline __m128d
335 _mm_cmpnlt_pd (__m128d __A
, __m128d __B
)
337 return (__m128d
)__builtin_ia32_cmpnltpd ((__v2df
)__A
, (__v2df
)__B
);
340 static __inline __m128d
341 _mm_cmpnle_pd (__m128d __A
, __m128d __B
)
343 return (__m128d
)__builtin_ia32_cmpnlepd ((__v2df
)__A
, (__v2df
)__B
);
346 static __inline __m128d
347 _mm_cmpngt_pd (__m128d __A
, __m128d __B
)
349 return (__m128d
)__builtin_ia32_cmpngtpd ((__v2df
)__A
, (__v2df
)__B
);
352 static __inline __m128d
353 _mm_cmpnge_pd (__m128d __A
, __m128d __B
)
355 return (__m128d
)__builtin_ia32_cmpngepd ((__v2df
)__A
, (__v2df
)__B
);
358 static __inline __m128d
359 _mm_cmpord_pd (__m128d __A
, __m128d __B
)
361 return (__m128d
)__builtin_ia32_cmpordpd ((__v2df
)__A
, (__v2df
)__B
);
364 static __inline __m128d
365 _mm_cmpunord_pd (__m128d __A
, __m128d __B
)
367 return (__m128d
)__builtin_ia32_cmpunordpd ((__v2df
)__A
, (__v2df
)__B
);
370 static __inline __m128d
371 _mm_cmpeq_sd (__m128d __A
, __m128d __B
)
373 return (__m128d
)__builtin_ia32_cmpeqsd ((__v2df
)__A
, (__v2df
)__B
);
376 static __inline __m128d
377 _mm_cmplt_sd (__m128d __A
, __m128d __B
)
379 return (__m128d
)__builtin_ia32_cmpltsd ((__v2df
)__A
, (__v2df
)__B
);
382 static __inline __m128d
383 _mm_cmple_sd (__m128d __A
, __m128d __B
)
385 return (__m128d
)__builtin_ia32_cmplesd ((__v2df
)__A
, (__v2df
)__B
);
388 static __inline __m128d
389 _mm_cmpgt_sd (__m128d __A
, __m128d __B
)
391 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
393 __builtin_ia32_cmpltsd ((__v2df
) __B
,
398 static __inline __m128d
399 _mm_cmpge_sd (__m128d __A
, __m128d __B
)
401 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
403 __builtin_ia32_cmplesd ((__v2df
) __B
,
408 static __inline __m128d
409 _mm_cmpneq_sd (__m128d __A
, __m128d __B
)
411 return (__m128d
)__builtin_ia32_cmpneqsd ((__v2df
)__A
, (__v2df
)__B
);
414 static __inline __m128d
415 _mm_cmpnlt_sd (__m128d __A
, __m128d __B
)
417 return (__m128d
)__builtin_ia32_cmpnltsd ((__v2df
)__A
, (__v2df
)__B
);
420 static __inline __m128d
421 _mm_cmpnle_sd (__m128d __A
, __m128d __B
)
423 return (__m128d
)__builtin_ia32_cmpnlesd ((__v2df
)__A
, (__v2df
)__B
);
426 static __inline __m128d
427 _mm_cmpngt_sd (__m128d __A
, __m128d __B
)
429 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
431 __builtin_ia32_cmpnltsd ((__v2df
) __B
,
436 static __inline __m128d
437 _mm_cmpnge_sd (__m128d __A
, __m128d __B
)
439 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
441 __builtin_ia32_cmpnlesd ((__v2df
) __B
,
446 static __inline __m128d
447 _mm_cmpord_sd (__m128d __A
, __m128d __B
)
449 return (__m128d
)__builtin_ia32_cmpordsd ((__v2df
)__A
, (__v2df
)__B
);
452 static __inline __m128d
453 _mm_cmpunord_sd (__m128d __A
, __m128d __B
)
455 return (__m128d
)__builtin_ia32_cmpunordsd ((__v2df
)__A
, (__v2df
)__B
);
459 _mm_comieq_sd (__m128d __A
, __m128d __B
)
461 return __builtin_ia32_comisdeq ((__v2df
)__A
, (__v2df
)__B
);
465 _mm_comilt_sd (__m128d __A
, __m128d __B
)
467 return __builtin_ia32_comisdlt ((__v2df
)__A
, (__v2df
)__B
);
471 _mm_comile_sd (__m128d __A
, __m128d __B
)
473 return __builtin_ia32_comisdle ((__v2df
)__A
, (__v2df
)__B
);
477 _mm_comigt_sd (__m128d __A
, __m128d __B
)
479 return __builtin_ia32_comisdgt ((__v2df
)__A
, (__v2df
)__B
);
483 _mm_comige_sd (__m128d __A
, __m128d __B
)
485 return __builtin_ia32_comisdge ((__v2df
)__A
, (__v2df
)__B
);
489 _mm_comineq_sd (__m128d __A
, __m128d __B
)
491 return __builtin_ia32_comisdneq ((__v2df
)__A
, (__v2df
)__B
);
495 _mm_ucomieq_sd (__m128d __A
, __m128d __B
)
497 return __builtin_ia32_ucomisdeq ((__v2df
)__A
, (__v2df
)__B
);
501 _mm_ucomilt_sd (__m128d __A
, __m128d __B
)
503 return __builtin_ia32_ucomisdlt ((__v2df
)__A
, (__v2df
)__B
);
507 _mm_ucomile_sd (__m128d __A
, __m128d __B
)
509 return __builtin_ia32_ucomisdle ((__v2df
)__A
, (__v2df
)__B
);
513 _mm_ucomigt_sd (__m128d __A
, __m128d __B
)
515 return __builtin_ia32_ucomisdgt ((__v2df
)__A
, (__v2df
)__B
);
519 _mm_ucomige_sd (__m128d __A
, __m128d __B
)
521 return __builtin_ia32_ucomisdge ((__v2df
)__A
, (__v2df
)__B
);
525 _mm_ucomineq_sd (__m128d __A
, __m128d __B
)
527 return __builtin_ia32_ucomisdneq ((__v2df
)__A
, (__v2df
)__B
);
530 /* Create a vector with element 0 as *P and the rest zero. */
532 static __inline __m128i
533 _mm_load_si128 (__m128i
const *__P
)
535 return (__m128i
) __builtin_ia32_loaddqa ((char const *)__P
);
538 static __inline __m128i
539 _mm_loadu_si128 (__m128i
const *__P
)
541 return (__m128i
) __builtin_ia32_loaddqu ((char const *)__P
);
544 static __inline __m128i
545 _mm_loadl_epi64 (__m128i
const *__P
)
547 return (__m128i
) __builtin_ia32_movq2dq (*(unsigned long long *)__P
);
551 _mm_store_si128 (__m128i
*__P
, __m128i __B
)
553 __builtin_ia32_storedqa ((char *)__P
, (__v16qi
)__B
);
557 _mm_storeu_si128 (__m128i
*__P
, __m128i __B
)
559 __builtin_ia32_storedqu ((char *)__P
, (__v16qi
)__B
);
563 _mm_storel_epi64 (__m128i
*__P
, __m128i __B
)
565 *(long long *)__P
= __builtin_ia32_movdq2q ((__v2di
)__B
);
568 static __inline __m64
569 _mm_movepi64_pi64 (__m128i __B
)
571 return (__m64
) __builtin_ia32_movdq2q ((__v2di
)__B
);
574 static __inline __m128i
575 _mm_move_epi64 (__m128i __A
)
577 return (__m128i
) __builtin_ia32_movq ((__v2di
)__A
);
580 /* Create a vector of zeros. */
581 static __inline __m128i
582 _mm_setzero_si128 (void)
584 return (__m128i
) __builtin_ia32_setzero128 ();
587 static __inline __m128i
588 _mm_set_epi64 (__m64 __A
, __m64 __B
)
590 __v2di __tmp
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
591 __v2di __tmp2
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__B
);
592 return (__m128i
)__builtin_ia32_punpcklqdq128 (__tmp2
, __tmp
);
595 /* Create the vector [Z Y X W]. */
596 static __inline __m128i
597 _mm_set_epi32 (int __Z
, int __Y
, int __X
, int __W
)
613 /* Create the vector [Z Y]. */
614 static __inline __m128i
615 _mm_set_epi64x (long long __Z
, long long __Y
)
629 /* Create the vector [S T U V Z Y X W]. */
630 static __inline __m128i
631 _mm_set_epi16 (short __Z
, short __Y
, short __X
, short __W
,
632 short __V
, short __U
, short __T
, short __S
)
651 /* Create the vector [S T U V Z Y X W]. */
652 static __inline __m128i
653 _mm_set_epi8 (char __Z
, char __Y
, char __X
, char __W
,
654 char __V
, char __U
, char __T
, char __S
,
655 char __Z1
, char __Y1
, char __X1
, char __W1
,
656 char __V1
, char __U1
, char __T1
, char __S1
)
683 static __inline __m128i
684 _mm_set1_epi64 (__m64 __A
)
686 __v2di __tmp
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
687 return (__m128i
)__builtin_ia32_punpcklqdq128 (__tmp
, __tmp
);
690 static __inline __m128i
691 _mm_set1_epi32 (int __A
)
693 __v4si __tmp
= (__v4si
)__builtin_ia32_loadd (&__A
);
694 return (__m128i
) __builtin_ia32_pshufd ((__v4si
)__tmp
, _MM_SHUFFLE (0,0,0,0));
698 static __inline __m128i
699 _mm_set1_epi64x (long long __A
)
701 __v2di __tmp
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
702 return (__m128i
) __builtin_ia32_shufpd ((__v2df
)__tmp
, (__v2df
)__tmp
, _MM_SHUFFLE2 (0,0));
706 static __inline __m128i
707 _mm_set1_epi16 (short __A
)
709 int __Acopy
= (unsigned short)__A
;
710 __v4si __tmp
= (__v4si
)__builtin_ia32_loadd (&__Acopy
);
711 __tmp
= (__v4si
)__builtin_ia32_punpcklwd128 ((__v8hi
)__tmp
, (__v8hi
)__tmp
);
712 return (__m128i
) __builtin_ia32_pshufd ((__v4si
)__tmp
, _MM_SHUFFLE (0,0,0,0));
715 static __inline __m128i
716 _mm_set1_epi8 (char __A
)
718 int __Acopy
= (unsigned char)__A
;
719 __v4si __tmp
= (__v4si
)__builtin_ia32_loadd (&__Acopy
);
720 __tmp
= (__v4si
)__builtin_ia32_punpcklbw128 ((__v16qi
)__tmp
, (__v16qi
)__tmp
);
721 __tmp
= (__v4si
)__builtin_ia32_punpcklbw128 ((__v16qi
)__tmp
, (__v16qi
)__tmp
);
722 return (__m128i
) __builtin_ia32_pshufd ((__v4si
)__tmp
, _MM_SHUFFLE (0,0,0,0));
725 static __inline __m128i
726 _mm_setr_epi64 (__m64 __A
, __m64 __B
)
728 __v2di __tmp
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
729 __v2di __tmp2
= (__v2di
)__builtin_ia32_movq2dq ((unsigned long long)__B
);
730 return (__m128i
)__builtin_ia32_punpcklqdq128 (__tmp
, __tmp2
);
733 /* Create the vector [Z Y X W]. */
734 static __inline __m128i
735 _mm_setr_epi32 (int __W
, int __X
, int __Y
, int __Z
)
749 /* Create the vector [S T U V Z Y X W]. */
750 static __inline __m128i
751 _mm_setr_epi16 (short __S
, short __T
, short __U
, short __V
,
752 short __W
, short __X
, short __Y
, short __Z
)
771 /* Create the vector [S T U V Z Y X W]. */
772 static __inline __m128i
773 _mm_setr_epi8 (char __S1
, char __T1
, char __U1
, char __V1
,
774 char __W1
, char __X1
, char __Y1
, char __Z1
,
775 char __S
, char __T
, char __U
, char __V
,
776 char __W
, char __X
, char __Y
, char __Z
)
803 static __inline __m128d
804 _mm_cvtepi32_pd (__m128i __A
)
806 return (__m128d
)__builtin_ia32_cvtdq2pd ((__v4si
) __A
);
809 static __inline __m128
810 _mm_cvtepi32_ps (__m128i __A
)
812 return (__m128
)__builtin_ia32_cvtdq2ps ((__v4si
) __A
);
815 static __inline __m128i
816 _mm_cvtpd_epi32 (__m128d __A
)
818 return (__m128i
)__builtin_ia32_cvtpd2dq ((__v2df
) __A
);
821 static __inline __m64
822 _mm_cvtpd_pi32 (__m128d __A
)
824 return (__m64
)__builtin_ia32_cvtpd2pi ((__v2df
) __A
);
827 static __inline __m128
828 _mm_cvtpd_ps (__m128d __A
)
830 return (__m128
)__builtin_ia32_cvtpd2ps ((__v2df
) __A
);
833 static __inline __m128i
834 _mm_cvttpd_epi32 (__m128d __A
)
836 return (__m128i
)__builtin_ia32_cvttpd2dq ((__v2df
) __A
);
839 static __inline __m64
840 _mm_cvttpd_pi32 (__m128d __A
)
842 return (__m64
)__builtin_ia32_cvttpd2pi ((__v2df
) __A
);
845 static __inline __m128d
846 _mm_cvtpi32_pd (__m64 __A
)
848 return (__m128d
)__builtin_ia32_cvtpi2pd ((__v2si
) __A
);
851 static __inline __m128i
852 _mm_cvtps_epi32 (__m128 __A
)
854 return (__m128i
)__builtin_ia32_cvtps2dq ((__v4sf
) __A
);
857 static __inline __m128i
858 _mm_cvttps_epi32 (__m128 __A
)
860 return (__m128i
)__builtin_ia32_cvttps2dq ((__v4sf
) __A
);
863 static __inline __m128d
864 _mm_cvtps_pd (__m128 __A
)
866 return (__m128d
)__builtin_ia32_cvtps2pd ((__v4sf
) __A
);
870 _mm_cvtsd_si32 (__m128d __A
)
872 return __builtin_ia32_cvtsd2si ((__v2df
) __A
);
876 static __inline
long long
877 _mm_cvtsd_si64x (__m128d __A
)
879 return __builtin_ia32_cvtsd2si64 ((__v2df
) __A
);
884 _mm_cvttsd_si32 (__m128d __A
)
886 return __builtin_ia32_cvttsd2si ((__v2df
) __A
);
890 static __inline
long long
891 _mm_cvttsd_si64x (__m128d __A
)
893 return __builtin_ia32_cvttsd2si64 ((__v2df
) __A
);
897 static __inline __m128
898 _mm_cvtsd_ss (__m128 __A
, __m128d __B
)
900 return (__m128
)__builtin_ia32_cvtsd2ss ((__v4sf
) __A
, (__v2df
) __B
);
903 static __inline __m128d
904 _mm_cvtsi32_sd (__m128d __A
, int __B
)
906 return (__m128d
)__builtin_ia32_cvtsi2sd ((__v2df
) __A
, __B
);
910 static __inline __m128d
911 _mm_cvtsi64x_sd (__m128d __A
, long long __B
)
913 return (__m128d
)__builtin_ia32_cvtsi642sd ((__v2df
) __A
, __B
);
917 static __inline __m128d
918 _mm_cvtss_sd (__m128d __A
, __m128 __B
)
920 return (__m128d
)__builtin_ia32_cvtss2sd ((__v2df
) __A
, (__v4sf
)__B
);
923 #define _mm_shuffle_pd(__A, __B, __C) ((__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, (__C)))
925 static __inline __m128d
926 _mm_unpackhi_pd (__m128d __A
, __m128d __B
)
928 return (__m128d
)__builtin_ia32_unpckhpd ((__v2df
)__A
, (__v2df
)__B
);
931 static __inline __m128d
932 _mm_unpacklo_pd (__m128d __A
, __m128d __B
)
934 return (__m128d
)__builtin_ia32_unpcklpd ((__v2df
)__A
, (__v2df
)__B
);
937 static __inline __m128d
938 _mm_loadh_pd (__m128d __A
, double const *__B
)
940 return (__m128d
)__builtin_ia32_loadhpd ((__v2df
)__A
, (__v2si
*)__B
);
944 _mm_storeh_pd (double *__A
, __m128d __B
)
946 __builtin_ia32_storehpd ((__v2si
*)__A
, (__v2df
)__B
);
949 static __inline __m128d
950 _mm_loadl_pd (__m128d __A
, double const *__B
)
952 return (__m128d
)__builtin_ia32_loadlpd ((__v2df
)__A
, (__v2si
*)__B
);
956 _mm_storel_pd (double *__A
, __m128d __B
)
958 __builtin_ia32_storelpd ((__v2si
*)__A
, (__v2df
)__B
);
962 _mm_movemask_pd (__m128d __A
)
964 return __builtin_ia32_movmskpd ((__v2df
)__A
);
967 static __inline __m128i
968 _mm_packs_epi16 (__m128i __A
, __m128i __B
)
970 return (__m128i
)__builtin_ia32_packsswb128 ((__v8hi
)__A
, (__v8hi
)__B
);
973 static __inline __m128i
974 _mm_packs_epi32 (__m128i __A
, __m128i __B
)
976 return (__m128i
)__builtin_ia32_packssdw128 ((__v4si
)__A
, (__v4si
)__B
);
979 static __inline __m128i
980 _mm_packus_epi16 (__m128i __A
, __m128i __B
)
982 return (__m128i
)__builtin_ia32_packuswb128 ((__v8hi
)__A
, (__v8hi
)__B
);
985 static __inline __m128i
986 _mm_unpackhi_epi8 (__m128i __A
, __m128i __B
)
988 return (__m128i
)__builtin_ia32_punpckhbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
991 static __inline __m128i
992 _mm_unpackhi_epi16 (__m128i __A
, __m128i __B
)
994 return (__m128i
)__builtin_ia32_punpckhwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
997 static __inline __m128i
998 _mm_unpackhi_epi32 (__m128i __A
, __m128i __B
)
1000 return (__m128i
)__builtin_ia32_punpckhdq128 ((__v4si
)__A
, (__v4si
)__B
);
1003 static __inline __m128i
1004 _mm_unpackhi_epi64 (__m128i __A
, __m128i __B
)
1006 return (__m128i
)__builtin_ia32_punpckhqdq128 ((__v2di
)__A
, (__v2di
)__B
);
1009 static __inline __m128i
1010 _mm_unpacklo_epi8 (__m128i __A
, __m128i __B
)
1012 return (__m128i
)__builtin_ia32_punpcklbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
1015 static __inline __m128i
1016 _mm_unpacklo_epi16 (__m128i __A
, __m128i __B
)
1018 return (__m128i
)__builtin_ia32_punpcklwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
1021 static __inline __m128i
1022 _mm_unpacklo_epi32 (__m128i __A
, __m128i __B
)
1024 return (__m128i
)__builtin_ia32_punpckldq128 ((__v4si
)__A
, (__v4si
)__B
);
1027 static __inline __m128i
1028 _mm_unpacklo_epi64 (__m128i __A
, __m128i __B
)
1030 return (__m128i
)__builtin_ia32_punpcklqdq128 ((__v2di
)__A
, (__v2di
)__B
);
1033 static __inline __m128i
1034 _mm_add_epi8 (__m128i __A
, __m128i __B
)
1036 return (__m128i
)__builtin_ia32_paddb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1039 static __inline __m128i
1040 _mm_add_epi16 (__m128i __A
, __m128i __B
)
1042 return (__m128i
)__builtin_ia32_paddw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1045 static __inline __m128i
1046 _mm_add_epi32 (__m128i __A
, __m128i __B
)
1048 return (__m128i
)__builtin_ia32_paddd128 ((__v4si
)__A
, (__v4si
)__B
);
1051 static __inline __m128i
1052 _mm_add_epi64 (__m128i __A
, __m128i __B
)
1054 return (__m128i
)__builtin_ia32_paddq128 ((__v2di
)__A
, (__v2di
)__B
);
1057 static __inline __m128i
1058 _mm_adds_epi8 (__m128i __A
, __m128i __B
)
1060 return (__m128i
)__builtin_ia32_paddsb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1063 static __inline __m128i
1064 _mm_adds_epi16 (__m128i __A
, __m128i __B
)
1066 return (__m128i
)__builtin_ia32_paddsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1069 static __inline __m128i
1070 _mm_adds_epu8 (__m128i __A
, __m128i __B
)
1072 return (__m128i
)__builtin_ia32_paddusb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1075 static __inline __m128i
1076 _mm_adds_epu16 (__m128i __A
, __m128i __B
)
1078 return (__m128i
)__builtin_ia32_paddusw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1081 static __inline __m128i
1082 _mm_sub_epi8 (__m128i __A
, __m128i __B
)
1084 return (__m128i
)__builtin_ia32_psubb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1087 static __inline __m128i
1088 _mm_sub_epi16 (__m128i __A
, __m128i __B
)
1090 return (__m128i
)__builtin_ia32_psubw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1093 static __inline __m128i
1094 _mm_sub_epi32 (__m128i __A
, __m128i __B
)
1096 return (__m128i
)__builtin_ia32_psubd128 ((__v4si
)__A
, (__v4si
)__B
);
1099 static __inline __m128i
1100 _mm_sub_epi64 (__m128i __A
, __m128i __B
)
1102 return (__m128i
)__builtin_ia32_psubq128 ((__v2di
)__A
, (__v2di
)__B
);
1105 static __inline __m128i
1106 _mm_subs_epi8 (__m128i __A
, __m128i __B
)
1108 return (__m128i
)__builtin_ia32_psubsb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1111 static __inline __m128i
1112 _mm_subs_epi16 (__m128i __A
, __m128i __B
)
1114 return (__m128i
)__builtin_ia32_psubsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1117 static __inline __m128i
1118 _mm_subs_epu8 (__m128i __A
, __m128i __B
)
1120 return (__m128i
)__builtin_ia32_psubusb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1123 static __inline __m128i
1124 _mm_subs_epu16 (__m128i __A
, __m128i __B
)
1126 return (__m128i
)__builtin_ia32_psubusw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1129 static __inline __m128i
1130 _mm_madd_epi16 (__m128i __A
, __m128i __B
)
1132 return (__m128i
)__builtin_ia32_pmaddwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
1135 static __inline __m128i
1136 _mm_mulhi_epi16 (__m128i __A
, __m128i __B
)
1138 return (__m128i
)__builtin_ia32_pmulhw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1141 static __inline __m128i
1142 _mm_mullo_epi16 (__m128i __A
, __m128i __B
)
1144 return (__m128i
)__builtin_ia32_pmullw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1147 static __inline __m64
1148 _mm_mul_su32 (__m64 __A
, __m64 __B
)
1150 return (__m64
)__builtin_ia32_pmuludq ((__v2si
)__A
, (__v2si
)__B
);
1153 static __inline __m128i
1154 _mm_mul_epu32 (__m128i __A
, __m128i __B
)
1156 return (__m128i
)__builtin_ia32_pmuludq128 ((__v4si
)__A
, (__v4si
)__B
);
1159 static __inline __m128i
1160 _mm_sll_epi16 (__m128i __A
, __m128i __B
)
1162 return (__m128i
)__builtin_ia32_psllw128 ((__v8hi
)__A
, (__v2di
)__B
);
1165 static __inline __m128i
1166 _mm_sll_epi32 (__m128i __A
, __m128i __B
)
1168 return (__m128i
)__builtin_ia32_pslld128 ((__v4si
)__A
, (__v2di
)__B
);
1171 static __inline __m128i
1172 _mm_sll_epi64 (__m128i __A
, __m128i __B
)
1174 return (__m128i
)__builtin_ia32_psllq128 ((__v2di
)__A
, (__v2di
)__B
);
1177 static __inline __m128i
1178 _mm_sra_epi16 (__m128i __A
, __m128i __B
)
1180 return (__m128i
)__builtin_ia32_psraw128 ((__v8hi
)__A
, (__v2di
)__B
);
1183 static __inline __m128i
1184 _mm_sra_epi32 (__m128i __A
, __m128i __B
)
1186 return (__m128i
)__builtin_ia32_psrad128 ((__v4si
)__A
, (__v2di
)__B
);
1189 static __inline __m128i
1190 _mm_srl_epi16 (__m128i __A
, __m128i __B
)
1192 return (__m128i
)__builtin_ia32_psrlw128 ((__v8hi
)__A
, (__v2di
)__B
);
1195 static __inline __m128i
1196 _mm_srl_epi32 (__m128i __A
, __m128i __B
)
1198 return (__m128i
)__builtin_ia32_psrld128 ((__v4si
)__A
, (__v2di
)__B
);
1201 static __inline __m128i
1202 _mm_srl_epi64 (__m128i __A
, __m128i __B
)
1204 return (__m128i
)__builtin_ia32_psrlq128 ((__v2di
)__A
, (__v2di
)__B
);
1207 static __inline __m128i
1208 _mm_slli_epi16 (__m128i __A
, int __B
)
1210 return (__m128i
)__builtin_ia32_psllwi128 ((__v8hi
)__A
, __B
);
1213 static __inline __m128i
1214 _mm_slli_epi32 (__m128i __A
, int __B
)
1216 return (__m128i
)__builtin_ia32_pslldi128 ((__v4si
)__A
, __B
);
1219 static __inline __m128i
1220 _mm_slli_epi64 (__m128i __A
, int __B
)
1222 return (__m128i
)__builtin_ia32_psllqi128 ((__v2di
)__A
, __B
);
1225 static __inline __m128i
1226 _mm_srai_epi16 (__m128i __A
, int __B
)
1228 return (__m128i
)__builtin_ia32_psrawi128 ((__v8hi
)__A
, __B
);
1231 static __inline __m128i
1232 _mm_srai_epi32 (__m128i __A
, int __B
)
1234 return (__m128i
)__builtin_ia32_psradi128 ((__v4si
)__A
, __B
);
1238 static __m128i
__attribute__((__always_inline__
))
1239 _mm_srli_si128 (__m128i __A
, const int __B
)
1241 return ((__m128i
)__builtin_ia32_psrldqi128 (__A
, __B
))
1244 static __m128i
__attribute__((__always_inline__
))
1245 _mm_srli_si128 (__m128i __A
, const int __B
)
1247 return ((__m128i
)__builtin_ia32_pslldqi128 (__A
, __B
))
1250 #define _mm_srli_si128(__A, __B) ((__m128i)__builtin_ia32_psrldqi128 (__A, __B))
1251 #define _mm_slli_si128(__A, __B) ((__m128i)__builtin_ia32_pslldqi128 (__A, __B))
1253 static __inline __m128i
1254 _mm_srli_epi16 (__m128i __A
, int __B
)
1256 return (__m128i
)__builtin_ia32_psrlwi128 ((__v8hi
)__A
, __B
);
1259 static __inline __m128i
1260 _mm_srli_epi32 (__m128i __A
, int __B
)
1262 return (__m128i
)__builtin_ia32_psrldi128 ((__v4si
)__A
, __B
);
1265 static __inline __m128i
1266 _mm_srli_epi64 (__m128i __A
, int __B
)
1268 return (__m128i
)__builtin_ia32_psrlqi128 ((__v2di
)__A
, __B
);
1271 static __inline __m128i
1272 _mm_and_si128 (__m128i __A
, __m128i __B
)
1274 return (__m128i
)__builtin_ia32_pand128 ((__v2di
)__A
, (__v2di
)__B
);
1277 static __inline __m128i
1278 _mm_andnot_si128 (__m128i __A
, __m128i __B
)
1280 return (__m128i
)__builtin_ia32_pandn128 ((__v2di
)__A
, (__v2di
)__B
);
1283 static __inline __m128i
1284 _mm_or_si128 (__m128i __A
, __m128i __B
)
1286 return (__m128i
)__builtin_ia32_por128 ((__v2di
)__A
, (__v2di
)__B
);
1289 static __inline __m128i
1290 _mm_xor_si128 (__m128i __A
, __m128i __B
)
1292 return (__m128i
)__builtin_ia32_pxor128 ((__v2di
)__A
, (__v2di
)__B
);
1295 static __inline __m128i
1296 _mm_cmpeq_epi8 (__m128i __A
, __m128i __B
)
1298 return (__m128i
)__builtin_ia32_pcmpeqb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1301 static __inline __m128i
1302 _mm_cmpeq_epi16 (__m128i __A
, __m128i __B
)
1304 return (__m128i
)__builtin_ia32_pcmpeqw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1307 static __inline __m128i
1308 _mm_cmpeq_epi32 (__m128i __A
, __m128i __B
)
1310 return (__m128i
)__builtin_ia32_pcmpeqd128 ((__v4si
)__A
, (__v4si
)__B
);
1313 static __inline __m128i
1314 _mm_cmplt_epi8 (__m128i __A
, __m128i __B
)
1316 return (__m128i
)__builtin_ia32_pcmpgtb128 ((__v16qi
)__B
, (__v16qi
)__A
);
1319 static __inline __m128i
1320 _mm_cmplt_epi16 (__m128i __A
, __m128i __B
)
1322 return (__m128i
)__builtin_ia32_pcmpgtw128 ((__v8hi
)__B
, (__v8hi
)__A
);
1325 static __inline __m128i
1326 _mm_cmplt_epi32 (__m128i __A
, __m128i __B
)
1328 return (__m128i
)__builtin_ia32_pcmpgtd128 ((__v4si
)__B
, (__v4si
)__A
);
1331 static __inline __m128i
1332 _mm_cmpgt_epi8 (__m128i __A
, __m128i __B
)
1334 return (__m128i
)__builtin_ia32_pcmpgtb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1337 static __inline __m128i
1338 _mm_cmpgt_epi16 (__m128i __A
, __m128i __B
)
1340 return (__m128i
)__builtin_ia32_pcmpgtw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1343 static __inline __m128i
1344 _mm_cmpgt_epi32 (__m128i __A
, __m128i __B
)
1346 return (__m128i
)__builtin_ia32_pcmpgtd128 ((__v4si
)__A
, (__v4si
)__B
);
1349 #define _mm_extract_epi16(__A, __B) __builtin_ia32_pextrw128 ((__v8hi)__A, __B)
1351 #define _mm_insert_epi16(__A, __B, __C) ((__m128i)__builtin_ia32_pinsrw128 ((__v8hi)__A, __B, __C))
1353 static __inline __m128i
1354 _mm_max_epi16 (__m128i __A
, __m128i __B
)
1356 return (__m128i
)__builtin_ia32_pmaxsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1359 static __inline __m128i
1360 _mm_max_epu8 (__m128i __A
, __m128i __B
)
1362 return (__m128i
)__builtin_ia32_pmaxub128 ((__v16qi
)__A
, (__v16qi
)__B
);
1365 static __inline __m128i
1366 _mm_min_epi16 (__m128i __A
, __m128i __B
)
1368 return (__m128i
)__builtin_ia32_pminsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1371 static __inline __m128i
1372 _mm_min_epu8 (__m128i __A
, __m128i __B
)
1374 return (__m128i
)__builtin_ia32_pminub128 ((__v16qi
)__A
, (__v16qi
)__B
);
1378 _mm_movemask_epi8 (__m128i __A
)
1380 return __builtin_ia32_pmovmskb128 ((__v16qi
)__A
);
1383 static __inline __m128i
1384 _mm_mulhi_epu16 (__m128i __A
, __m128i __B
)
1386 return (__m128i
)__builtin_ia32_pmulhuw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1389 #define _mm_shufflehi_epi16(__A, __B) ((__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __B))
1390 #define _mm_shufflelo_epi16(__A, __B) ((__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __B))
1391 #define _mm_shuffle_epi32(__A, __B) ((__m128i)__builtin_ia32_pshufd ((__v4si)__A, __B))
1393 static __inline
void
1394 _mm_maskmoveu_si128 (__m128i __A
, __m128i __B
, char *__C
)
1396 __builtin_ia32_maskmovdqu ((__v16qi
)__A
, (__v16qi
)__B
, __C
);
1399 static __inline __m128i
1400 _mm_avg_epu8 (__m128i __A
, __m128i __B
)
1402 return (__m128i
)__builtin_ia32_pavgb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1405 static __inline __m128i
1406 _mm_avg_epu16 (__m128i __A
, __m128i __B
)
1408 return (__m128i
)__builtin_ia32_pavgw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1411 static __inline __m128i
1412 _mm_sad_epu8 (__m128i __A
, __m128i __B
)
1414 return (__m128i
)__builtin_ia32_psadbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
1417 static __inline
void
1418 _mm_stream_si32 (int *__A
, int __B
)
1420 __builtin_ia32_movnti (__A
, __B
);
1423 static __inline
void
1424 _mm_stream_si128 (__m128i
*__A
, __m128i __B
)
1426 __builtin_ia32_movntdq ((__v2di
*)__A
, (__v2di
)__B
);
1429 static __inline
void
1430 _mm_stream_pd (double *__A
, __m128d __B
)
1432 __builtin_ia32_movntpd (__A
, (__v2df
)__B
);
1435 static __inline __m128i
1436 _mm_movpi64_epi64 (__m64 __A
)
1438 return (__m128i
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
1441 static __inline
void
1442 _mm_clflush (void const *__A
)
1444 return __builtin_ia32_clflush (__A
);
1447 static __inline
void
1450 __builtin_ia32_lfence ();
1453 static __inline
void
1456 __builtin_ia32_mfence ();
1459 static __inline __m128i
1460 _mm_cvtsi32_si128 (int __A
)
1462 return (__m128i
) __builtin_ia32_loadd (&__A
);
1466 static __inline __m128i
1467 _mm_cvtsi64x_si128 (long long __A
)
1469 return (__m128i
) __builtin_ia32_movq2dq (__A
);
1474 _mm_cvtsi128_si32 (__m128i __A
)
1477 __builtin_ia32_stored (&__tmp
, (__v4si
)__A
);
1482 static __inline
long long
1483 _mm_cvtsi128_si64x (__m128i __A
)
1485 return __builtin_ia32_movdq2q ((__v2di
)__A
);
1489 #endif /* __SSE2__ */
1491 #endif /* _EMMINTRIN_H_INCLUDED */
