guest agent: remove uneeded dependencies
[qemu/mdroth.git] / target-arm / neon_helper.c
blobb51e35a30fe18f23c3a4f5b5bdfd6d61fce6381e
1 /*
2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licensed under the GNU GPL v2.
8 */
9 #include <stdlib.h>
10 #include <stdio.h>
12 #include "cpu.h"
13 #include "exec-all.h"
14 #include "helper.h"
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
21 #define NEON_TYPE1(name, type) \
22 typedef struct \
23 { \
24 type v1; \
25 } neon_##name;
26 #ifdef HOST_WORDS_BIGENDIAN
27 #define NEON_TYPE2(name, type) \
28 typedef struct \
29 { \
30 type v2; \
31 type v1; \
32 } neon_##name;
33 #define NEON_TYPE4(name, type) \
34 typedef struct \
35 { \
36 type v4; \
37 type v3; \
38 type v2; \
39 type v1; \
40 } neon_##name;
41 #else
42 #define NEON_TYPE2(name, type) \
43 typedef struct \
44 { \
45 type v1; \
46 type v2; \
47 } neon_##name;
48 #define NEON_TYPE4(name, type) \
49 typedef struct \
50 { \
51 type v1; \
52 type v2; \
53 type v3; \
54 type v4; \
55 } neon_##name;
56 #endif
58 NEON_TYPE4(s8, int8_t)
59 NEON_TYPE4(u8, uint8_t)
60 NEON_TYPE2(s16, int16_t)
61 NEON_TYPE2(u16, uint16_t)
62 NEON_TYPE1(s32, int32_t)
63 NEON_TYPE1(u32, uint32_t)
64 #undef NEON_TYPE4
65 #undef NEON_TYPE2
66 #undef NEON_TYPE1
68 /* Copy from a uint32_t to a vector structure type. */
69 #define NEON_UNPACK(vtype, dest, val) do { \
70 union { \
71 vtype v; \
72 uint32_t i; \
73 } conv_u; \
74 conv_u.i = (val); \
75 dest = conv_u.v; \
76 } while(0)
78 /* Copy from a vector structure type to a uint32_t. */
79 #define NEON_PACK(vtype, dest, val) do { \
80 union { \
81 vtype v; \
82 uint32_t i; \
83 } conv_u; \
84 conv_u.v = (val); \
85 dest = conv_u.i; \
86 } while(0)
88 #define NEON_DO1 \
89 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
90 #define NEON_DO2 \
91 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
92 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
93 #define NEON_DO4 \
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
96 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
97 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
99 #define NEON_VOP_BODY(vtype, n) \
101 uint32_t res; \
102 vtype vsrc1; \
103 vtype vsrc2; \
104 vtype vdest; \
105 NEON_UNPACK(vtype, vsrc1, arg1); \
106 NEON_UNPACK(vtype, vsrc2, arg2); \
107 NEON_DO##n; \
108 NEON_PACK(vtype, res, vdest); \
109 return res; \
112 #define NEON_VOP(name, vtype, n) \
113 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
114 NEON_VOP_BODY(vtype, n)
116 #define NEON_VOP_ENV(name, vtype, n) \
117 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
118 NEON_VOP_BODY(vtype, n)
120 /* Pairwise operations. */
121 /* For 32-bit elements each segment only contains a single element, so
122 the elementwise and pairwise operations are the same. */
123 #define NEON_PDO2 \
124 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
125 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
126 #define NEON_PDO4 \
127 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
128 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
129 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
130 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
132 #define NEON_POP(name, vtype, n) \
133 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
135 uint32_t res; \
136 vtype vsrc1; \
137 vtype vsrc2; \
138 vtype vdest; \
139 NEON_UNPACK(vtype, vsrc1, arg1); \
140 NEON_UNPACK(vtype, vsrc2, arg2); \
141 NEON_PDO##n; \
142 NEON_PACK(vtype, res, vdest); \
143 return res; \
146 /* Unary operators. */
147 #define NEON_VOP1(name, vtype, n) \
148 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
150 vtype vsrc1; \
151 vtype vdest; \
152 NEON_UNPACK(vtype, vsrc1, arg); \
153 NEON_DO##n; \
154 NEON_PACK(vtype, arg, vdest); \
155 return arg; \
159 #define NEON_USAT(dest, src1, src2, type) do { \
160 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
161 if (tmp != (type)tmp) { \
162 SET_QC(); \
163 dest = ~0; \
164 } else { \
165 dest = tmp; \
166 }} while(0)
167 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
168 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
169 #undef NEON_FN
170 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
171 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
172 #undef NEON_FN
173 #undef NEON_USAT
175 uint32_t HELPER(neon_qadd_u32)(CPUState *env, uint32_t a, uint32_t b)
177 uint32_t res = a + b;
178 if (res < a) {
179 SET_QC();
180 res = ~0;
182 return res;
185 uint64_t HELPER(neon_qadd_u64)(CPUState *env, uint64_t src1, uint64_t src2)
187 uint64_t res;
189 res = src1 + src2;
190 if (res < src1) {
191 SET_QC();
192 res = ~(uint64_t)0;
194 return res;
197 #define NEON_SSAT(dest, src1, src2, type) do { \
198 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
199 if (tmp != (type)tmp) { \
200 SET_QC(); \
201 if (src2 > 0) { \
202 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
203 } else { \
204 tmp = 1 << (sizeof(type) * 8 - 1); \
207 dest = tmp; \
208 } while(0)
209 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
210 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
211 #undef NEON_FN
212 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
213 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
214 #undef NEON_FN
215 #undef NEON_SSAT
217 uint32_t HELPER(neon_qadd_s32)(CPUState *env, uint32_t a, uint32_t b)
219 uint32_t res = a + b;
220 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
221 SET_QC();
222 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
224 return res;
227 uint64_t HELPER(neon_qadd_s64)(CPUState *env, uint64_t src1, uint64_t src2)
229 uint64_t res;
231 res = src1 + src2;
232 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
233 SET_QC();
234 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
236 return res;
239 #define NEON_USAT(dest, src1, src2, type) do { \
240 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
241 if (tmp != (type)tmp) { \
242 SET_QC(); \
243 dest = 0; \
244 } else { \
245 dest = tmp; \
246 }} while(0)
247 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
248 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
249 #undef NEON_FN
250 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
251 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
252 #undef NEON_FN
253 #undef NEON_USAT
255 uint32_t HELPER(neon_qsub_u32)(CPUState *env, uint32_t a, uint32_t b)
257 uint32_t res = a - b;
258 if (res > a) {
259 SET_QC();
260 res = 0;
262 return res;
265 uint64_t HELPER(neon_qsub_u64)(CPUState *env, uint64_t src1, uint64_t src2)
267 uint64_t res;
269 if (src1 < src2) {
270 SET_QC();
271 res = 0;
272 } else {
273 res = src1 - src2;
275 return res;
278 #define NEON_SSAT(dest, src1, src2, type) do { \
279 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
280 if (tmp != (type)tmp) { \
281 SET_QC(); \
282 if (src2 < 0) { \
283 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
284 } else { \
285 tmp = 1 << (sizeof(type) * 8 - 1); \
288 dest = tmp; \
289 } while(0)
290 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
291 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
292 #undef NEON_FN
293 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
294 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
295 #undef NEON_FN
296 #undef NEON_SSAT
298 uint32_t HELPER(neon_qsub_s32)(CPUState *env, uint32_t a, uint32_t b)
300 uint32_t res = a - b;
301 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
302 SET_QC();
303 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
305 return res;
308 uint64_t HELPER(neon_qsub_s64)(CPUState *env, uint64_t src1, uint64_t src2)
310 uint64_t res;
312 res = src1 - src2;
313 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
314 SET_QC();
315 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
317 return res;
320 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
321 NEON_VOP(hadd_s8, neon_s8, 4)
322 NEON_VOP(hadd_u8, neon_u8, 4)
323 NEON_VOP(hadd_s16, neon_s16, 2)
324 NEON_VOP(hadd_u16, neon_u16, 2)
325 #undef NEON_FN
327 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
329 int32_t dest;
331 dest = (src1 >> 1) + (src2 >> 1);
332 if (src1 & src2 & 1)
333 dest++;
334 return dest;
337 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
339 uint32_t dest;
341 dest = (src1 >> 1) + (src2 >> 1);
342 if (src1 & src2 & 1)
343 dest++;
344 return dest;
347 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
348 NEON_VOP(rhadd_s8, neon_s8, 4)
349 NEON_VOP(rhadd_u8, neon_u8, 4)
350 NEON_VOP(rhadd_s16, neon_s16, 2)
351 NEON_VOP(rhadd_u16, neon_u16, 2)
352 #undef NEON_FN
354 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
356 int32_t dest;
358 dest = (src1 >> 1) + (src2 >> 1);
359 if ((src1 | src2) & 1)
360 dest++;
361 return dest;
364 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
366 uint32_t dest;
368 dest = (src1 >> 1) + (src2 >> 1);
369 if ((src1 | src2) & 1)
370 dest++;
371 return dest;
374 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
375 NEON_VOP(hsub_s8, neon_s8, 4)
376 NEON_VOP(hsub_u8, neon_u8, 4)
377 NEON_VOP(hsub_s16, neon_s16, 2)
378 NEON_VOP(hsub_u16, neon_u16, 2)
379 #undef NEON_FN
381 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
383 int32_t dest;
385 dest = (src1 >> 1) - (src2 >> 1);
386 if ((~src1) & src2 & 1)
387 dest--;
388 return dest;
391 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
393 uint32_t dest;
395 dest = (src1 >> 1) - (src2 >> 1);
396 if ((~src1) & src2 & 1)
397 dest--;
398 return dest;
401 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
402 NEON_VOP(cgt_s8, neon_s8, 4)
403 NEON_VOP(cgt_u8, neon_u8, 4)
404 NEON_VOP(cgt_s16, neon_s16, 2)
405 NEON_VOP(cgt_u16, neon_u16, 2)
406 NEON_VOP(cgt_s32, neon_s32, 1)
407 NEON_VOP(cgt_u32, neon_u32, 1)
408 #undef NEON_FN
410 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
411 NEON_VOP(cge_s8, neon_s8, 4)
412 NEON_VOP(cge_u8, neon_u8, 4)
413 NEON_VOP(cge_s16, neon_s16, 2)
414 NEON_VOP(cge_u16, neon_u16, 2)
415 NEON_VOP(cge_s32, neon_s32, 1)
416 NEON_VOP(cge_u32, neon_u32, 1)
417 #undef NEON_FN
419 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
420 NEON_VOP(min_s8, neon_s8, 4)
421 NEON_VOP(min_u8, neon_u8, 4)
422 NEON_VOP(min_s16, neon_s16, 2)
423 NEON_VOP(min_u16, neon_u16, 2)
424 NEON_VOP(min_s32, neon_s32, 1)
425 NEON_VOP(min_u32, neon_u32, 1)
426 NEON_POP(pmin_s8, neon_s8, 4)
427 NEON_POP(pmin_u8, neon_u8, 4)
428 NEON_POP(pmin_s16, neon_s16, 2)
429 NEON_POP(pmin_u16, neon_u16, 2)
430 #undef NEON_FN
432 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
433 NEON_VOP(max_s8, neon_s8, 4)
434 NEON_VOP(max_u8, neon_u8, 4)
435 NEON_VOP(max_s16, neon_s16, 2)
436 NEON_VOP(max_u16, neon_u16, 2)
437 NEON_VOP(max_s32, neon_s32, 1)
438 NEON_VOP(max_u32, neon_u32, 1)
439 NEON_POP(pmax_s8, neon_s8, 4)
440 NEON_POP(pmax_u8, neon_u8, 4)
441 NEON_POP(pmax_s16, neon_s16, 2)
442 NEON_POP(pmax_u16, neon_u16, 2)
443 #undef NEON_FN
445 #define NEON_FN(dest, src1, src2) \
446 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
447 NEON_VOP(abd_s8, neon_s8, 4)
448 NEON_VOP(abd_u8, neon_u8, 4)
449 NEON_VOP(abd_s16, neon_s16, 2)
450 NEON_VOP(abd_u16, neon_u16, 2)
451 NEON_VOP(abd_s32, neon_s32, 1)
452 NEON_VOP(abd_u32, neon_u32, 1)
453 #undef NEON_FN
455 #define NEON_FN(dest, src1, src2) do { \
456 int8_t tmp; \
457 tmp = (int8_t)src2; \
458 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
459 tmp <= -(ssize_t)sizeof(src1) * 8) { \
460 dest = 0; \
461 } else if (tmp < 0) { \
462 dest = src1 >> -tmp; \
463 } else { \
464 dest = src1 << tmp; \
465 }} while (0)
466 NEON_VOP(shl_u8, neon_u8, 4)
467 NEON_VOP(shl_u16, neon_u16, 2)
468 NEON_VOP(shl_u32, neon_u32, 1)
469 #undef NEON_FN
471 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
473 int8_t shift = (int8_t)shiftop;
474 if (shift >= 64 || shift <= -64) {
475 val = 0;
476 } else if (shift < 0) {
477 val >>= -shift;
478 } else {
479 val <<= shift;
481 return val;
484 #define NEON_FN(dest, src1, src2) do { \
485 int8_t tmp; \
486 tmp = (int8_t)src2; \
487 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
488 dest = 0; \
489 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
490 dest = src1 >> (sizeof(src1) * 8 - 1); \
491 } else if (tmp < 0) { \
492 dest = src1 >> -tmp; \
493 } else { \
494 dest = src1 << tmp; \
495 }} while (0)
496 NEON_VOP(shl_s8, neon_s8, 4)
497 NEON_VOP(shl_s16, neon_s16, 2)
498 NEON_VOP(shl_s32, neon_s32, 1)
499 #undef NEON_FN
501 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
503 int8_t shift = (int8_t)shiftop;
504 int64_t val = valop;
505 if (shift >= 64) {
506 val = 0;
507 } else if (shift <= -64) {
508 val >>= 63;
509 } else if (shift < 0) {
510 val >>= -shift;
511 } else {
512 val <<= shift;
514 return val;
517 #define NEON_FN(dest, src1, src2) do { \
518 int8_t tmp; \
519 tmp = (int8_t)src2; \
520 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
521 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
522 dest = 0; \
523 } else if (tmp < 0) { \
524 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
525 } else { \
526 dest = src1 << tmp; \
527 }} while (0)
528 NEON_VOP(rshl_s8, neon_s8, 4)
529 NEON_VOP(rshl_s16, neon_s16, 2)
530 #undef NEON_FN
532 /* The addition of the rounding constant may overflow, so we use an
533 * intermediate 64 bits accumulator. */
534 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
536 int32_t dest;
537 int32_t val = (int32_t)valop;
538 int8_t shift = (int8_t)shiftop;
539 if ((shift >= 32) || (shift <= -32)) {
540 dest = 0;
541 } else if (shift < 0) {
542 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
543 dest = big_dest >> -shift;
544 } else {
545 dest = val << shift;
547 return dest;
550 /* Handling addition overflow with 64 bits inputs values is more
551 * tricky than with 32 bits values. */
552 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
554 int8_t shift = (int8_t)shiftop;
555 int64_t val = valop;
556 if ((shift >= 64) || (shift <= -64)) {
557 val = 0;
558 } else if (shift < 0) {
559 val >>= (-shift - 1);
560 if (val == INT64_MAX) {
561 /* In this case, it means that the rounding constant is 1,
562 * and the addition would overflow. Return the actual
563 * result directly. */
564 val = 0x4000000000000000LL;
565 } else {
566 val++;
567 val >>= 1;
569 } else {
570 val <<= shift;
572 return val;
575 #define NEON_FN(dest, src1, src2) do { \
576 int8_t tmp; \
577 tmp = (int8_t)src2; \
578 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
579 tmp < -(ssize_t)sizeof(src1) * 8) { \
580 dest = 0; \
581 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
582 dest = src1 >> (-tmp - 1); \
583 } else if (tmp < 0) { \
584 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
585 } else { \
586 dest = src1 << tmp; \
587 }} while (0)
588 NEON_VOP(rshl_u8, neon_u8, 4)
589 NEON_VOP(rshl_u16, neon_u16, 2)
590 #undef NEON_FN
592 /* The addition of the rounding constant may overflow, so we use an
593 * intermediate 64 bits accumulator. */
594 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
596 uint32_t dest;
597 int8_t shift = (int8_t)shiftop;
598 if (shift >= 32 || shift < -32) {
599 dest = 0;
600 } else if (shift == -32) {
601 dest = val >> 31;
602 } else if (shift < 0) {
603 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
604 dest = big_dest >> -shift;
605 } else {
606 dest = val << shift;
608 return dest;
611 /* Handling addition overflow with 64 bits inputs values is more
612 * tricky than with 32 bits values. */
613 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
615 int8_t shift = (uint8_t)shiftop;
616 if (shift >= 64 || shift < -64) {
617 val = 0;
618 } else if (shift == -64) {
619 /* Rounding a 1-bit result just preserves that bit. */
620 val >>= 63;
621 } else if (shift < 0) {
622 val >>= (-shift - 1);
623 if (val == UINT64_MAX) {
624 /* In this case, it means that the rounding constant is 1,
625 * and the addition would overflow. Return the actual
626 * result directly. */
627 val = 0x8000000000000000ULL;
628 } else {
629 val++;
630 val >>= 1;
632 } else {
633 val <<= shift;
635 return val;
638 #define NEON_FN(dest, src1, src2) do { \
639 int8_t tmp; \
640 tmp = (int8_t)src2; \
641 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
642 if (src1) { \
643 SET_QC(); \
644 dest = ~0; \
645 } else { \
646 dest = 0; \
648 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
649 dest = 0; \
650 } else if (tmp < 0) { \
651 dest = src1 >> -tmp; \
652 } else { \
653 dest = src1 << tmp; \
654 if ((dest >> tmp) != src1) { \
655 SET_QC(); \
656 dest = ~0; \
658 }} while (0)
659 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
660 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
661 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
662 #undef NEON_FN
664 uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
666 int8_t shift = (int8_t)shiftop;
667 if (shift >= 64) {
668 if (val) {
669 val = ~(uint64_t)0;
670 SET_QC();
672 } else if (shift <= -64) {
673 val = 0;
674 } else if (shift < 0) {
675 val >>= -shift;
676 } else {
677 uint64_t tmp = val;
678 val <<= shift;
679 if ((val >> shift) != tmp) {
680 SET_QC();
681 val = ~(uint64_t)0;
684 return val;
687 #define NEON_FN(dest, src1, src2) do { \
688 int8_t tmp; \
689 tmp = (int8_t)src2; \
690 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
691 if (src1) { \
692 SET_QC(); \
693 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
694 if (src1 > 0) { \
695 dest--; \
697 } else { \
698 dest = src1; \
700 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
701 dest = src1 >> 31; \
702 } else if (tmp < 0) { \
703 dest = src1 >> -tmp; \
704 } else { \
705 dest = src1 << tmp; \
706 if ((dest >> tmp) != src1) { \
707 SET_QC(); \
708 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
709 if (src1 > 0) { \
710 dest--; \
713 }} while (0)
714 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
715 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
716 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
717 #undef NEON_FN
719 uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
721 int8_t shift = (uint8_t)shiftop;
722 int64_t val = valop;
723 if (shift >= 64) {
724 if (val) {
725 SET_QC();
726 val = (val >> 63) ^ ~SIGNBIT64;
728 } else if (shift <= -64) {
729 val >>= 63;
730 } else if (shift < 0) {
731 val >>= -shift;
732 } else {
733 int64_t tmp = val;
734 val <<= shift;
735 if ((val >> shift) != tmp) {
736 SET_QC();
737 val = (tmp >> 63) ^ ~SIGNBIT64;
740 return val;
743 #define NEON_FN(dest, src1, src2) do { \
744 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
745 SET_QC(); \
746 dest = 0; \
747 } else { \
748 int8_t tmp; \
749 tmp = (int8_t)src2; \
750 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
751 if (src1) { \
752 SET_QC(); \
753 dest = ~0; \
754 } else { \
755 dest = 0; \
757 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
758 dest = 0; \
759 } else if (tmp < 0) { \
760 dest = src1 >> -tmp; \
761 } else { \
762 dest = src1 << tmp; \
763 if ((dest >> tmp) != src1) { \
764 SET_QC(); \
765 dest = ~0; \
768 }} while (0)
769 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
770 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
771 #undef NEON_FN
773 uint32_t HELPER(neon_qshlu_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
775 if ((int32_t)valop < 0) {
776 SET_QC();
777 return 0;
779 return helper_neon_qshl_u32(env, valop, shiftop);
782 uint64_t HELPER(neon_qshlu_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
784 if ((int64_t)valop < 0) {
785 SET_QC();
786 return 0;
788 return helper_neon_qshl_u64(env, valop, shiftop);
791 /* FIXME: This is wrong. */
792 #define NEON_FN(dest, src1, src2) do { \
793 int8_t tmp; \
794 tmp = (int8_t)src2; \
795 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
796 if (src1) { \
797 SET_QC(); \
798 dest = ~0; \
799 } else { \
800 dest = 0; \
802 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
803 dest = 0; \
804 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
805 dest = src1 >> (sizeof(src1) * 8 - 1); \
806 } else if (tmp < 0) { \
807 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
808 } else { \
809 dest = src1 << tmp; \
810 if ((dest >> tmp) != src1) { \
811 SET_QC(); \
812 dest = ~0; \
814 }} while (0)
815 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
816 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
817 #undef NEON_FN
819 /* The addition of the rounding constant may overflow, so we use an
820 * intermediate 64 bits accumulator. */
821 uint32_t HELPER(neon_qrshl_u32)(CPUState *env, uint32_t val, uint32_t shiftop)
823 uint32_t dest;
824 int8_t shift = (int8_t)shiftop;
825 if (shift >= 32) {
826 if (val) {
827 SET_QC();
828 dest = ~0;
829 } else {
830 dest = 0;
832 } else if (shift < -32) {
833 dest = 0;
834 } else if (shift == -32) {
835 dest = val >> 31;
836 } else if (shift < 0) {
837 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
838 dest = big_dest >> -shift;
839 } else {
840 dest = val << shift;
841 if ((dest >> shift) != val) {
842 SET_QC();
843 dest = ~0;
846 return dest;
849 /* Handling addition overflow with 64 bits inputs values is more
850 * tricky than with 32 bits values. */
851 uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
853 int8_t shift = (int8_t)shiftop;
854 if (shift >= 64) {
855 if (val) {
856 SET_QC();
857 val = ~0;
859 } else if (shift < -64) {
860 val = 0;
861 } else if (shift == -64) {
862 val >>= 63;
863 } else if (shift < 0) {
864 val >>= (-shift - 1);
865 if (val == UINT64_MAX) {
866 /* In this case, it means that the rounding constant is 1,
867 * and the addition would overflow. Return the actual
868 * result directly. */
869 val = 0x8000000000000000ULL;
870 } else {
871 val++;
872 val >>= 1;
874 } else { \
875 uint64_t tmp = val;
876 val <<= shift;
877 if ((val >> shift) != tmp) {
878 SET_QC();
879 val = ~0;
882 return val;
885 #define NEON_FN(dest, src1, src2) do { \
886 int8_t tmp; \
887 tmp = (int8_t)src2; \
888 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
889 if (src1) { \
890 SET_QC(); \
891 dest = (1 << (sizeof(src1) * 8 - 1)); \
892 if (src1 > 0) { \
893 dest--; \
895 } else { \
896 dest = 0; \
898 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
899 dest = 0; \
900 } else if (tmp < 0) { \
901 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
902 } else { \
903 dest = src1 << tmp; \
904 if ((dest >> tmp) != src1) { \
905 SET_QC(); \
906 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
907 if (src1 > 0) { \
908 dest--; \
911 }} while (0)
912 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
913 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
914 #undef NEON_FN
916 /* The addition of the rounding constant may overflow, so we use an
917 * intermediate 64 bits accumulator. */
918 uint32_t HELPER(neon_qrshl_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
920 int32_t dest;
921 int32_t val = (int32_t)valop;
922 int8_t shift = (int8_t)shiftop;
923 if (shift >= 32) {
924 if (val) {
925 SET_QC();
926 dest = (val >> 31) ^ ~SIGNBIT;
927 } else {
928 dest = 0;
930 } else if (shift <= -32) {
931 dest = 0;
932 } else if (shift < 0) {
933 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
934 dest = big_dest >> -shift;
935 } else {
936 dest = val << shift;
937 if ((dest >> shift) != val) {
938 SET_QC();
939 dest = (val >> 31) ^ ~SIGNBIT;
942 return dest;
945 /* Handling addition overflow with 64 bits inputs values is more
946 * tricky than with 32 bits values. */
947 uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
949 int8_t shift = (uint8_t)shiftop;
950 int64_t val = valop;
952 if (shift >= 64) {
953 if (val) {
954 SET_QC();
955 val = (val >> 63) ^ ~SIGNBIT64;
957 } else if (shift <= -64) {
958 val = 0;
959 } else if (shift < 0) {
960 val >>= (-shift - 1);
961 if (val == INT64_MAX) {
962 /* In this case, it means that the rounding constant is 1,
963 * and the addition would overflow. Return the actual
964 * result directly. */
965 val = 0x4000000000000000ULL;
966 } else {
967 val++;
968 val >>= 1;
970 } else {
971 int64_t tmp = val;
972 val <<= shift;
973 if ((val >> shift) != tmp) {
974 SET_QC();
975 val = (tmp >> 63) ^ ~SIGNBIT64;
978 return val;
981 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
983 uint32_t mask;
984 mask = (a ^ b) & 0x80808080u;
985 a &= ~0x80808080u;
986 b &= ~0x80808080u;
987 return (a + b) ^ mask;
990 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
992 uint32_t mask;
993 mask = (a ^ b) & 0x80008000u;
994 a &= ~0x80008000u;
995 b &= ~0x80008000u;
996 return (a + b) ^ mask;
999 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1000 NEON_POP(padd_u8, neon_u8, 4)
1001 NEON_POP(padd_u16, neon_u16, 2)
1002 #undef NEON_FN
1004 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1005 NEON_VOP(sub_u8, neon_u8, 4)
1006 NEON_VOP(sub_u16, neon_u16, 2)
1007 #undef NEON_FN
1009 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1010 NEON_VOP(mul_u8, neon_u8, 4)
1011 NEON_VOP(mul_u16, neon_u16, 2)
1012 #undef NEON_FN
1014 /* Polynomial multiplication is like integer multiplication except the
1015 partial products are XORed, not added. */
1016 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1018 uint32_t mask;
1019 uint32_t result;
1020 result = 0;
1021 while (op1) {
1022 mask = 0;
1023 if (op1 & 1)
1024 mask |= 0xff;
1025 if (op1 & (1 << 8))
1026 mask |= (0xff << 8);
1027 if (op1 & (1 << 16))
1028 mask |= (0xff << 16);
1029 if (op1 & (1 << 24))
1030 mask |= (0xff << 24);
1031 result ^= op2 & mask;
1032 op1 = (op1 >> 1) & 0x7f7f7f7f;
1033 op2 = (op2 << 1) & 0xfefefefe;
1035 return result;
1038 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1040 uint64_t result = 0;
1041 uint64_t mask;
1042 uint64_t op2ex = op2;
1043 op2ex = (op2ex & 0xff) |
1044 ((op2ex & 0xff00) << 8) |
1045 ((op2ex & 0xff0000) << 16) |
1046 ((op2ex & 0xff000000) << 24);
1047 while (op1) {
1048 mask = 0;
1049 if (op1 & 1) {
1050 mask |= 0xffff;
1052 if (op1 & (1 << 8)) {
1053 mask |= (0xffffU << 16);
1055 if (op1 & (1 << 16)) {
1056 mask |= (0xffffULL << 32);
1058 if (op1 & (1 << 24)) {
1059 mask |= (0xffffULL << 48);
1061 result ^= op2ex & mask;
1062 op1 = (op1 >> 1) & 0x7f7f7f7f;
1063 op2ex <<= 1;
1065 return result;
1068 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1069 NEON_VOP(tst_u8, neon_u8, 4)
1070 NEON_VOP(tst_u16, neon_u16, 2)
1071 NEON_VOP(tst_u32, neon_u32, 1)
1072 #undef NEON_FN
1074 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1075 NEON_VOP(ceq_u8, neon_u8, 4)
1076 NEON_VOP(ceq_u16, neon_u16, 2)
1077 NEON_VOP(ceq_u32, neon_u32, 1)
1078 #undef NEON_FN
1080 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1081 NEON_VOP1(abs_s8, neon_s8, 4)
1082 NEON_VOP1(abs_s16, neon_s16, 2)
1083 #undef NEON_FN
1085 /* Count Leading Sign/Zero Bits. */
1086 static inline int do_clz8(uint8_t x)
1088 int n;
1089 for (n = 8; x; n--)
1090 x >>= 1;
1091 return n;
1094 static inline int do_clz16(uint16_t x)
1096 int n;
1097 for (n = 16; x; n--)
1098 x >>= 1;
1099 return n;
1102 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1103 NEON_VOP1(clz_u8, neon_u8, 4)
1104 #undef NEON_FN
1106 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1107 NEON_VOP1(clz_u16, neon_u16, 2)
1108 #undef NEON_FN
1110 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1111 NEON_VOP1(cls_s8, neon_s8, 4)
1112 #undef NEON_FN
1114 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1115 NEON_VOP1(cls_s16, neon_s16, 2)
1116 #undef NEON_FN
1118 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1120 int count;
1121 if ((int32_t)x < 0)
1122 x = ~x;
1123 for (count = 32; x; count--)
1124 x = x >> 1;
1125 return count - 1;
1128 /* Bit count. */
1129 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1131 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1132 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1133 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1134 return x;
1137 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1138 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1139 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1140 SET_QC(); \
1141 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1142 } else { \
1143 tmp <<= 1; \
1145 if (round) { \
1146 int32_t old = tmp; \
1147 tmp += 1 << 15; \
1148 if ((int32_t)tmp < old) { \
1149 SET_QC(); \
1150 tmp = SIGNBIT - 1; \
1153 dest = tmp >> 16; \
1154 } while(0)
1155 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1156 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1157 #undef NEON_FN
1158 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1159 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1160 #undef NEON_FN
1161 #undef NEON_QDMULH16
1163 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1164 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1165 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1166 SET_QC(); \
1167 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1168 } else { \
1169 tmp <<= 1; \
1171 if (round) { \
1172 int64_t old = tmp; \
1173 tmp += (int64_t)1 << 31; \
1174 if ((int64_t)tmp < old) { \
1175 SET_QC(); \
1176 tmp = SIGNBIT64 - 1; \
1179 dest = tmp >> 32; \
1180 } while(0)
1181 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1182 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1183 #undef NEON_FN
1184 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1185 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1186 #undef NEON_FN
1187 #undef NEON_QDMULH32
1189 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1191 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1192 | ((x >> 24) & 0xff000000u);
1195 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1197 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1200 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1202 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1203 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1206 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1208 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1211 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1213 x &= 0xff80ff80ff80ff80ull;
1214 x += 0x0080008000800080ull;
1215 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1216 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1219 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1221 x &= 0xffff8000ffff8000ull;
1222 x += 0x0000800000008000ull;
1223 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1226 uint32_t HELPER(neon_unarrow_sat8)(CPUState *env, uint64_t x)
1228 uint16_t s;
1229 uint8_t d;
1230 uint32_t res = 0;
1231 #define SAT8(n) \
1232 s = x >> n; \
1233 if (s & 0x8000) { \
1234 SET_QC(); \
1235 } else { \
1236 if (s > 0xff) { \
1237 d = 0xff; \
1238 SET_QC(); \
1239 } else { \
1240 d = s; \
1242 res |= (uint32_t)d << (n / 2); \
1245 SAT8(0);
1246 SAT8(16);
1247 SAT8(32);
1248 SAT8(48);
1249 #undef SAT8
1250 return res;
1253 uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
1255 uint16_t s;
1256 uint8_t d;
1257 uint32_t res = 0;
1258 #define SAT8(n) \
1259 s = x >> n; \
1260 if (s > 0xff) { \
1261 d = 0xff; \
1262 SET_QC(); \
1263 } else { \
1264 d = s; \
1266 res |= (uint32_t)d << (n / 2);
1268 SAT8(0);
1269 SAT8(16);
1270 SAT8(32);
1271 SAT8(48);
1272 #undef SAT8
1273 return res;
1276 uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
1278 int16_t s;
1279 uint8_t d;
1280 uint32_t res = 0;
1281 #define SAT8(n) \
1282 s = x >> n; \
1283 if (s != (int8_t)s) { \
1284 d = (s >> 15) ^ 0x7f; \
1285 SET_QC(); \
1286 } else { \
1287 d = s; \
1289 res |= (uint32_t)d << (n / 2);
1291 SAT8(0);
1292 SAT8(16);
1293 SAT8(32);
1294 SAT8(48);
1295 #undef SAT8
1296 return res;
1299 uint32_t HELPER(neon_unarrow_sat16)(CPUState *env, uint64_t x)
1301 uint32_t high;
1302 uint32_t low;
1303 low = x;
1304 if (low & 0x80000000) {
1305 low = 0;
1306 SET_QC();
1307 } else if (low > 0xffff) {
1308 low = 0xffff;
1309 SET_QC();
1311 high = x >> 32;
1312 if (high & 0x80000000) {
1313 high = 0;
1314 SET_QC();
1315 } else if (high > 0xffff) {
1316 high = 0xffff;
1317 SET_QC();
1319 return low | (high << 16);
1322 uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
1324 uint32_t high;
1325 uint32_t low;
1326 low = x;
1327 if (low > 0xffff) {
1328 low = 0xffff;
1329 SET_QC();
1331 high = x >> 32;
1332 if (high > 0xffff) {
1333 high = 0xffff;
1334 SET_QC();
1336 return low | (high << 16);
1339 uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
1341 int32_t low;
1342 int32_t high;
1343 low = x;
1344 if (low != (int16_t)low) {
1345 low = (low >> 31) ^ 0x7fff;
1346 SET_QC();
1348 high = x >> 32;
1349 if (high != (int16_t)high) {
1350 high = (high >> 31) ^ 0x7fff;
1351 SET_QC();
1353 return (uint16_t)low | (high << 16);
1356 uint32_t HELPER(neon_unarrow_sat32)(CPUState *env, uint64_t x)
1358 if (x & 0x8000000000000000ull) {
1359 SET_QC();
1360 return 0;
1362 if (x > 0xffffffffu) {
1363 SET_QC();
1364 return 0xffffffffu;
1366 return x;
1369 uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
1371 if (x > 0xffffffffu) {
1372 SET_QC();
1373 return 0xffffffffu;
1375 return x;
1378 uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)
1380 if ((int64_t)x != (int32_t)x) {
1381 SET_QC();
1382 return ((int64_t)x >> 63) ^ 0x7fffffff;
1384 return x;
1387 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1389 uint64_t tmp;
1390 uint64_t ret;
1391 ret = (uint8_t)x;
1392 tmp = (uint8_t)(x >> 8);
1393 ret |= tmp << 16;
1394 tmp = (uint8_t)(x >> 16);
1395 ret |= tmp << 32;
1396 tmp = (uint8_t)(x >> 24);
1397 ret |= tmp << 48;
1398 return ret;
1401 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1403 uint64_t tmp;
1404 uint64_t ret;
1405 ret = (uint16_t)(int8_t)x;
1406 tmp = (uint16_t)(int8_t)(x >> 8);
1407 ret |= tmp << 16;
1408 tmp = (uint16_t)(int8_t)(x >> 16);
1409 ret |= tmp << 32;
1410 tmp = (uint16_t)(int8_t)(x >> 24);
1411 ret |= tmp << 48;
1412 return ret;
1415 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1417 uint64_t high = (uint16_t)(x >> 16);
1418 return ((uint16_t)x) | (high << 32);
1421 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1423 uint64_t high = (int16_t)(x >> 16);
1424 return ((uint32_t)(int16_t)x) | (high << 32);
1427 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1429 uint64_t mask;
1430 mask = (a ^ b) & 0x8000800080008000ull;
1431 a &= ~0x8000800080008000ull;
1432 b &= ~0x8000800080008000ull;
1433 return (a + b) ^ mask;
1436 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1438 uint64_t mask;
1439 mask = (a ^ b) & 0x8000000080000000ull;
1440 a &= ~0x8000000080000000ull;
1441 b &= ~0x8000000080000000ull;
1442 return (a + b) ^ mask;
1445 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1447 uint64_t tmp;
1448 uint64_t tmp2;
1450 tmp = a & 0x0000ffff0000ffffull;
1451 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1452 tmp2 = b & 0xffff0000ffff0000ull;
1453 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1454 return ( tmp & 0xffff)
1455 | ((tmp >> 16) & 0xffff0000ull)
1456 | ((tmp2 << 16) & 0xffff00000000ull)
1457 | ( tmp2 & 0xffff000000000000ull);
1460 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1462 uint32_t low = a + (a >> 32);
1463 uint32_t high = b + (b >> 32);
1464 return low + ((uint64_t)high << 32);
1467 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1469 uint64_t mask;
1470 mask = (a ^ ~b) & 0x8000800080008000ull;
1471 a |= 0x8000800080008000ull;
1472 b &= ~0x8000800080008000ull;
1473 return (a - b) ^ mask;
1476 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1478 uint64_t mask;
1479 mask = (a ^ ~b) & 0x8000000080000000ull;
1480 a |= 0x8000000080000000ull;
1481 b &= ~0x8000000080000000ull;
1482 return (a - b) ^ mask;
1485 uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
1487 uint32_t x, y;
1488 uint32_t low, high;
1490 x = a;
1491 y = b;
1492 low = x + y;
1493 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1494 SET_QC();
1495 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1497 x = a >> 32;
1498 y = b >> 32;
1499 high = x + y;
1500 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1501 SET_QC();
1502 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1504 return low | ((uint64_t)high << 32);
1507 uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
1509 uint64_t result;
1511 result = a + b;
1512 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1513 SET_QC();
1514 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1516 return result;
1519 /* We have to do the arithmetic in a larger type than
1520 * the input type, because for example with a signed 32 bit
1521 * op the absolute difference can overflow a signed 32 bit value.
1523 #define DO_ABD(dest, x, y, intype, arithtype) do { \
1524 arithtype tmp_x = (intype)(x); \
1525 arithtype tmp_y = (intype)(y); \
1526 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1527 } while(0)
1529 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1531 uint64_t tmp;
1532 uint64_t result;
1533 DO_ABD(result, a, b, uint8_t, uint32_t);
1534 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1535 result |= tmp << 16;
1536 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1537 result |= tmp << 32;
1538 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1539 result |= tmp << 48;
1540 return result;
1543 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1545 uint64_t tmp;
1546 uint64_t result;
1547 DO_ABD(result, a, b, int8_t, int32_t);
1548 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1549 result |= tmp << 16;
1550 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1551 result |= tmp << 32;
1552 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1553 result |= tmp << 48;
1554 return result;
1557 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1559 uint64_t tmp;
1560 uint64_t result;
1561 DO_ABD(result, a, b, uint16_t, uint32_t);
1562 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1563 return result | (tmp << 32);
1566 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1568 uint64_t tmp;
1569 uint64_t result;
1570 DO_ABD(result, a, b, int16_t, int32_t);
1571 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1572 return result | (tmp << 32);
1575 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1577 uint64_t result;
1578 DO_ABD(result, a, b, uint32_t, uint64_t);
1579 return result;
1582 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1584 uint64_t result;
1585 DO_ABD(result, a, b, int32_t, int64_t);
1586 return result;
1588 #undef DO_ABD
1590 /* Widening multiply. Named type is the source type. */
1591 #define DO_MULL(dest, x, y, type1, type2) do { \
1592 type1 tmp_x = x; \
1593 type1 tmp_y = y; \
1594 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1595 } while(0)
1597 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1599 uint64_t tmp;
1600 uint64_t result;
1602 DO_MULL(result, a, b, uint8_t, uint16_t);
1603 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1604 result |= tmp << 16;
1605 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1606 result |= tmp << 32;
1607 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1608 result |= tmp << 48;
1609 return result;
1612 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1614 uint64_t tmp;
1615 uint64_t result;
1617 DO_MULL(result, a, b, int8_t, uint16_t);
1618 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1619 result |= tmp << 16;
1620 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1621 result |= tmp << 32;
1622 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1623 result |= tmp << 48;
1624 return result;
1627 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1629 uint64_t tmp;
1630 uint64_t result;
1632 DO_MULL(result, a, b, uint16_t, uint32_t);
1633 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1634 return result | (tmp << 32);
1637 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1639 uint64_t tmp;
1640 uint64_t result;
1642 DO_MULL(result, a, b, int16_t, uint32_t);
1643 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1644 return result | (tmp << 32);
1647 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1649 uint16_t tmp;
1650 uint64_t result;
1651 result = (uint16_t)-x;
1652 tmp = -(x >> 16);
1653 result |= (uint64_t)tmp << 16;
1654 tmp = -(x >> 32);
1655 result |= (uint64_t)tmp << 32;
1656 tmp = -(x >> 48);
1657 result |= (uint64_t)tmp << 48;
1658 return result;
1661 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1663 uint32_t low = -x;
1664 uint32_t high = -(x >> 32);
1665 return low | ((uint64_t)high << 32);
1668 /* FIXME: There should be a native op for this. */
1669 uint64_t HELPER(neon_negl_u64)(uint64_t x)
1671 return -x;
1674 /* Saturnating sign manuipulation. */
1675 /* ??? Make these use NEON_VOP1 */
1676 #define DO_QABS8(x) do { \
1677 if (x == (int8_t)0x80) { \
1678 x = 0x7f; \
1679 SET_QC(); \
1680 } else if (x < 0) { \
1681 x = -x; \
1682 }} while (0)
1683 uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
1685 neon_s8 vec;
1686 NEON_UNPACK(neon_s8, vec, x);
1687 DO_QABS8(vec.v1);
1688 DO_QABS8(vec.v2);
1689 DO_QABS8(vec.v3);
1690 DO_QABS8(vec.v4);
1691 NEON_PACK(neon_s8, x, vec);
1692 return x;
1694 #undef DO_QABS8
1696 #define DO_QNEG8(x) do { \
1697 if (x == (int8_t)0x80) { \
1698 x = 0x7f; \
1699 SET_QC(); \
1700 } else { \
1701 x = -x; \
1702 }} while (0)
1703 uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
1705 neon_s8 vec;
1706 NEON_UNPACK(neon_s8, vec, x);
1707 DO_QNEG8(vec.v1);
1708 DO_QNEG8(vec.v2);
1709 DO_QNEG8(vec.v3);
1710 DO_QNEG8(vec.v4);
1711 NEON_PACK(neon_s8, x, vec);
1712 return x;
1714 #undef DO_QNEG8
1716 #define DO_QABS16(x) do { \
1717 if (x == (int16_t)0x8000) { \
1718 x = 0x7fff; \
1719 SET_QC(); \
1720 } else if (x < 0) { \
1721 x = -x; \
1722 }} while (0)
1723 uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
1725 neon_s16 vec;
1726 NEON_UNPACK(neon_s16, vec, x);
1727 DO_QABS16(vec.v1);
1728 DO_QABS16(vec.v2);
1729 NEON_PACK(neon_s16, x, vec);
1730 return x;
1732 #undef DO_QABS16
1734 #define DO_QNEG16(x) do { \
1735 if (x == (int16_t)0x8000) { \
1736 x = 0x7fff; \
1737 SET_QC(); \
1738 } else { \
1739 x = -x; \
1740 }} while (0)
1741 uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
1743 neon_s16 vec;
1744 NEON_UNPACK(neon_s16, vec, x);
1745 DO_QNEG16(vec.v1);
1746 DO_QNEG16(vec.v2);
1747 NEON_PACK(neon_s16, x, vec);
1748 return x;
1750 #undef DO_QNEG16
1752 uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
1754 if (x == SIGNBIT) {
1755 SET_QC();
1756 x = ~SIGNBIT;
1757 } else if ((int32_t)x < 0) {
1758 x = -x;
1760 return x;
1763 uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
1765 if (x == SIGNBIT) {
1766 SET_QC();
1767 x = ~SIGNBIT;
1768 } else {
1769 x = -x;
1771 return x;
1774 /* NEON Float helpers. */
1775 uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b, void *fpstp)
1777 float_status *fpst = fpstp;
1778 return float32_val(float32_min(make_float32(a), make_float32(b), fpst));
1781 uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b, void *fpstp)
1783 float_status *fpst = fpstp;
1784 return float32_val(float32_max(make_float32(a), make_float32(b), fpst));
1787 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1789 float_status *fpst = fpstp;
1790 float32 f0 = make_float32(a);
1791 float32 f1 = make_float32(b);
1792 return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1795 /* Floating point comparisons produce an integer result.
1796 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1797 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1799 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1801 float_status *fpst = fpstp;
1802 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1805 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1807 float_status *fpst = fpstp;
1808 return -float32_le(make_float32(b), make_float32(a), fpst);
1811 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1813 float_status *fpst = fpstp;
1814 return -float32_lt(make_float32(b), make_float32(a), fpst);
1817 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1819 float_status *fpst = fpstp;
1820 float32 f0 = float32_abs(make_float32(a));
1821 float32 f1 = float32_abs(make_float32(b));
1822 return -float32_le(f1, f0, fpst);
1825 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1827 float_status *fpst = fpstp;
1828 float32 f0 = float32_abs(make_float32(a));
1829 float32 f1 = float32_abs(make_float32(b));
1830 return -float32_lt(f1, f0, fpst);
1833 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1835 void HELPER(neon_qunzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1837 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1838 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1839 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1840 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1841 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1842 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1843 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1844 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1845 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1846 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1847 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1848 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1849 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1850 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1851 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1852 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1853 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1854 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1855 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1856 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1857 env->vfp.regs[rm] = make_float64(m0);
1858 env->vfp.regs[rm + 1] = make_float64(m1);
1859 env->vfp.regs[rd] = make_float64(d0);
1860 env->vfp.regs[rd + 1] = make_float64(d1);
1863 void HELPER(neon_qunzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1865 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1866 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1867 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1868 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1869 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1870 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1871 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1872 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1873 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1874 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1875 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1876 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1877 env->vfp.regs[rm] = make_float64(m0);
1878 env->vfp.regs[rm + 1] = make_float64(m1);
1879 env->vfp.regs[rd] = make_float64(d0);
1880 env->vfp.regs[rd + 1] = make_float64(d1);
1883 void HELPER(neon_qunzip32)(CPUState *env, uint32_t rd, uint32_t rm)
1885 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1886 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1887 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1888 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1889 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1890 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1891 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1892 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1893 env->vfp.regs[rm] = make_float64(m0);
1894 env->vfp.regs[rm + 1] = make_float64(m1);
1895 env->vfp.regs[rd] = make_float64(d0);
1896 env->vfp.regs[rd + 1] = make_float64(d1);
1899 void HELPER(neon_unzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1901 uint64_t zm = float64_val(env->vfp.regs[rm]);
1902 uint64_t zd = float64_val(env->vfp.regs[rd]);
1903 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1904 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1905 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1906 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1907 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1908 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1909 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1910 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1911 env->vfp.regs[rm] = make_float64(m0);
1912 env->vfp.regs[rd] = make_float64(d0);
1915 void HELPER(neon_unzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1917 uint64_t zm = float64_val(env->vfp.regs[rm]);
1918 uint64_t zd = float64_val(env->vfp.regs[rd]);
1919 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1920 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1921 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1922 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1923 env->vfp.regs[rm] = make_float64(m0);
1924 env->vfp.regs[rd] = make_float64(d0);
1927 void HELPER(neon_qzip8)(CPUState *env, uint32_t rd, uint32_t rm)
1929 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1930 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1931 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1932 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1933 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1934 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1935 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1936 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1937 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1938 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1939 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1940 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1941 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1942 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1943 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1944 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1945 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1946 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1947 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1948 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1949 env->vfp.regs[rm] = make_float64(m0);
1950 env->vfp.regs[rm + 1] = make_float64(m1);
1951 env->vfp.regs[rd] = make_float64(d0);
1952 env->vfp.regs[rd + 1] = make_float64(d1);
1955 void HELPER(neon_qzip16)(CPUState *env, uint32_t rd, uint32_t rm)
1957 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1958 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1959 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1960 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1961 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1962 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1963 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1964 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1965 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1966 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1967 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1968 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1969 env->vfp.regs[rm] = make_float64(m0);
1970 env->vfp.regs[rm + 1] = make_float64(m1);
1971 env->vfp.regs[rd] = make_float64(d0);
1972 env->vfp.regs[rd + 1] = make_float64(d1);
1975 void HELPER(neon_qzip32)(CPUState *env, uint32_t rd, uint32_t rm)
1977 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1978 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1979 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1980 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1981 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1982 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1983 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1984 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1985 env->vfp.regs[rm] = make_float64(m0);
1986 env->vfp.regs[rm + 1] = make_float64(m1);
1987 env->vfp.regs[rd] = make_float64(d0);
1988 env->vfp.regs[rd + 1] = make_float64(d1);
1991 void HELPER(neon_zip8)(CPUState *env, uint32_t rd, uint32_t rm)
1993 uint64_t zm = float64_val(env->vfp.regs[rm]);
1994 uint64_t zd = float64_val(env->vfp.regs[rd]);
1995 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
1996 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
1997 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1998 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
1999 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2000 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2001 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2002 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2003 env->vfp.regs[rm] = make_float64(m0);
2004 env->vfp.regs[rd] = make_float64(d0);
2007 void HELPER(neon_zip16)(CPUState *env, uint32_t rd, uint32_t rm)
2009 uint64_t zm = float64_val(env->vfp.regs[rm]);
2010 uint64_t zd = float64_val(env->vfp.regs[rd]);
2011 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2012 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2013 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2014 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2015 env->vfp.regs[rm] = make_float64(m0);
2016 env->vfp.regs[rd] = make_float64(d0);