target-arm/neon_helper.c

   1 /*
   2  * ARM NEON vector operations.
   3  *
   4  * Copyright (c) 2007, 2008 CodeSourcery.
   5  * Written by Paul Brook
   6  *
   7  * This code is licenced under the GNU GPL v2.
   8  */
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11
  12 #include "cpu.h"
  13 #include "exec.h"
  14 #include "helper.h"
  15
  16 #define SIGNBIT (uint32_t)0x80000000
  17 #define SIGNBIT64 ((uint64_t)1 << 63)
  18
  19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
  20
  21 #define NFS (&env->vfp.standard_fp_status)
  22
  23 #define NEON_TYPE1(name, type) \
  24 typedef struct \
  25 { \
  26     type v1; \
  27 } neon_##name;
  28 #ifdef HOST_WORDS_BIGENDIAN
  29 #define NEON_TYPE2(name, type) \
  30 typedef struct \
  31 { \
  32     type v2; \
  33     type v1; \
  34 } neon_##name;
  35 #define NEON_TYPE4(name, type) \
  36 typedef struct \
  37 { \
  38     type v4; \
  39     type v3; \
  40     type v2; \
  41     type v1; \
  42 } neon_##name;
  43 #else
  44 #define NEON_TYPE2(name, type) \
  45 typedef struct \
  46 { \
  47     type v1; \
  48     type v2; \
  49 } neon_##name;
  50 #define NEON_TYPE4(name, type) \
  51 typedef struct \
  52 { \
  53     type v1; \
  54     type v2; \
  55     type v3; \
  56     type v4; \
  57 } neon_##name;
  58 #endif
  59
  60 NEON_TYPE4(s8, int8_t)
  61 NEON_TYPE4(u8, uint8_t)
  62 NEON_TYPE2(s16, int16_t)
  63 NEON_TYPE2(u16, uint16_t)
  64 NEON_TYPE1(s32, int32_t)
  65 NEON_TYPE1(u32, uint32_t)
  66 #undef NEON_TYPE4
  67 #undef NEON_TYPE2
  68 #undef NEON_TYPE1
  69
  70 /* Copy from a uint32_t to a vector structure type.  */
  71 #define NEON_UNPACK(vtype, dest, val) do { \
  72     union { \
  73         vtype v; \
  74         uint32_t i; \
  75     } conv_u; \
  76     conv_u.i = (val); \
  77     dest = conv_u.v; \
  78     } while(0)
  79
  80 /* Copy from a vector structure type to a uint32_t.  */
  81 #define NEON_PACK(vtype, dest, val) do { \
  82     union { \
  83         vtype v; \
  84         uint32_t i; \
  85     } conv_u; \
  86     conv_u.v = (val); \
  87     dest = conv_u.i; \
  88     } while(0)
  89
  90 #define NEON_DO1 \
  91     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
  92 #define NEON_DO2 \
  93     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  94     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
  95 #define NEON_DO4 \
  96     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
  97     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
  98     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
  99     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
 100
 101 #define NEON_VOP_BODY(vtype, n) \
 102 { \
 103     uint32_t res; \
 104     vtype vsrc1; \
 105     vtype vsrc2; \
 106     vtype vdest; \
 107     NEON_UNPACK(vtype, vsrc1, arg1); \
 108     NEON_UNPACK(vtype, vsrc2, arg2); \
 109     NEON_DO##n; \
 110     NEON_PACK(vtype, res, vdest); \
 111     return res; \
 112 }
 113
 114 #define NEON_VOP(name, vtype, n) \
 115 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 116 NEON_VOP_BODY(vtype, n)
 117
 118 /* Pairwise operations.  */
 119 /* For 32-bit elements each segment only contains a single element, so
 120    the elementwise and pairwise operations are the same.  */
 121 #define NEON_PDO2 \
 122     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 123     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
 124 #define NEON_PDO4 \
 125     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
 126     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
 127     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
 128     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
 129
 130 #define NEON_POP(name, vtype, n) \
 131 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
 132 { \
 133     uint32_t res; \
 134     vtype vsrc1; \
 135     vtype vsrc2; \
 136     vtype vdest; \
 137     NEON_UNPACK(vtype, vsrc1, arg1); \
 138     NEON_UNPACK(vtype, vsrc2, arg2); \
 139     NEON_PDO##n; \
 140     NEON_PACK(vtype, res, vdest); \
 141     return res; \
 142 }
 143
 144 /* Unary operators.  */
 145 #define NEON_VOP1(name, vtype, n) \
 146 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
 147 { \
 148     vtype vsrc1; \
 149     vtype vdest; \
 150     NEON_UNPACK(vtype, vsrc1, arg); \
 151     NEON_DO##n; \
 152     NEON_PACK(vtype, arg, vdest); \
 153     return arg; \
 154 }
 155
 156
 157 #define NEON_USAT(dest, src1, src2, type) do { \
 158     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 159     if (tmp != (type)tmp) { \
 160         SET_QC(); \
 161         dest = ~0; \
 162     } else { \
 163         dest = tmp; \
 164     }} while(0)
 165 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 166 NEON_VOP(qadd_u8, neon_u8, 4)
 167 #undef NEON_FN
 168 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 169 NEON_VOP(qadd_u16, neon_u16, 2)
 170 #undef NEON_FN
 171 #undef NEON_USAT
 172
 173 uint32_t HELPER(neon_qadd_u32)(uint32_t a, uint32_t b)
 174 {
 175     uint32_t res = a + b;
 176     if (res < a) {
 177         SET_QC();
 178         res = ~0;
 179     }
 180     return res;
 181 }
 182
 183 uint64_t HELPER(neon_qadd_u64)(uint64_t src1, uint64_t src2)
 184 {
 185     uint64_t res;
 186
 187     res = src1 + src2;
 188     if (res < src1) {
 189         SET_QC();
 190         res = ~(uint64_t)0;
 191     }
 192     return res;
 193 }
 194
 195 #define NEON_SSAT(dest, src1, src2, type) do { \
 196     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
 197     if (tmp != (type)tmp) { \
 198         SET_QC(); \
 199         if (src2 > 0) { \
 200             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 201         } else { \
 202             tmp = 1 << (sizeof(type) * 8 - 1); \
 203         } \
 204     } \
 205     dest = tmp; \
 206     } while(0)
 207 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 208 NEON_VOP(qadd_s8, neon_s8, 4)
 209 #undef NEON_FN
 210 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 211 NEON_VOP(qadd_s16, neon_s16, 2)
 212 #undef NEON_FN
 213 #undef NEON_SSAT
 214
 215 uint32_t HELPER(neon_qadd_s32)(uint32_t a, uint32_t b)
 216 {
 217     uint32_t res = a + b;
 218     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 219         SET_QC();
 220         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 221     }
 222     return res;
 223 }
 224
 225 uint64_t HELPER(neon_qadd_s64)(uint64_t src1, uint64_t src2)
 226 {
 227     uint64_t res;
 228
 229     res = src1 + src2;
 230     if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
 231         SET_QC();
 232         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 233     }
 234     return res;
 235 }
 236
 237 #define NEON_USAT(dest, src1, src2, type) do { \
 238     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 239     if (tmp != (type)tmp) { \
 240         SET_QC(); \
 241         dest = 0; \
 242     } else { \
 243         dest = tmp; \
 244     }} while(0)
 245 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
 246 NEON_VOP(qsub_u8, neon_u8, 4)
 247 #undef NEON_FN
 248 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
 249 NEON_VOP(qsub_u16, neon_u16, 2)
 250 #undef NEON_FN
 251 #undef NEON_USAT
 252
 253 uint32_t HELPER(neon_qsub_u32)(uint32_t a, uint32_t b)
 254 {
 255     uint32_t res = a - b;
 256     if (res > a) {
 257         SET_QC();
 258         res = 0;
 259     }
 260     return res;
 261 }
 262
 263 uint64_t HELPER(neon_qsub_u64)(uint64_t src1, uint64_t src2)
 264 {
 265     uint64_t res;
 266
 267     if (src1 < src2) {
 268         SET_QC();
 269         res = 0;
 270     } else {
 271         res = src1 - src2;
 272     }
 273     return res;
 274 }
 275
 276 #define NEON_SSAT(dest, src1, src2, type) do { \
 277     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
 278     if (tmp != (type)tmp) { \
 279         SET_QC(); \
 280         if (src2 < 0) { \
 281             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
 282         } else { \
 283             tmp = 1 << (sizeof(type) * 8 - 1); \
 284         } \
 285     } \
 286     dest = tmp; \
 287     } while(0)
 288 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
 289 NEON_VOP(qsub_s8, neon_s8, 4)
 290 #undef NEON_FN
 291 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
 292 NEON_VOP(qsub_s16, neon_s16, 2)
 293 #undef NEON_FN
 294 #undef NEON_SSAT
 295
 296 uint32_t HELPER(neon_qsub_s32)(uint32_t a, uint32_t b)
 297 {
 298     uint32_t res = a - b;
 299     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 300         SET_QC();
 301         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 302     }
 303     return res;
 304 }
 305
 306 uint64_t HELPER(neon_qsub_s64)(uint64_t src1, uint64_t src2)
 307 {
 308     uint64_t res;
 309
 310     res = src1 - src2;
 311     if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
 312         SET_QC();
 313         res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
 314     }
 315     return res;
 316 }
 317
 318 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
 319 NEON_VOP(hadd_s8, neon_s8, 4)
 320 NEON_VOP(hadd_u8, neon_u8, 4)
 321 NEON_VOP(hadd_s16, neon_s16, 2)
 322 NEON_VOP(hadd_u16, neon_u16, 2)
 323 #undef NEON_FN
 324
 325 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
 326 {
 327     int32_t dest;
 328
 329     dest = (src1 >> 1) + (src2 >> 1);
 330     if (src1 & src2 & 1)
 331         dest++;
 332     return dest;
 333 }
 334
 335 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
 336 {
 337     uint32_t dest;
 338
 339     dest = (src1 >> 1) + (src2 >> 1);
 340     if (src1 & src2 & 1)
 341         dest++;
 342     return dest;
 343 }
 344
 345 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
 346 NEON_VOP(rhadd_s8, neon_s8, 4)
 347 NEON_VOP(rhadd_u8, neon_u8, 4)
 348 NEON_VOP(rhadd_s16, neon_s16, 2)
 349 NEON_VOP(rhadd_u16, neon_u16, 2)
 350 #undef NEON_FN
 351
 352 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
 353 {
 354     int32_t dest;
 355
 356     dest = (src1 >> 1) + (src2 >> 1);
 357     if ((src1 | src2) & 1)
 358         dest++;
 359     return dest;
 360 }
 361
 362 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
 363 {
 364     uint32_t dest;
 365
 366     dest = (src1 >> 1) + (src2 >> 1);
 367     if ((src1 | src2) & 1)
 368         dest++;
 369     return dest;
 370 }
 371
 372 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
 373 NEON_VOP(hsub_s8, neon_s8, 4)
 374 NEON_VOP(hsub_u8, neon_u8, 4)
 375 NEON_VOP(hsub_s16, neon_s16, 2)
 376 NEON_VOP(hsub_u16, neon_u16, 2)
 377 #undef NEON_FN
 378
 379 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
 380 {
 381     int32_t dest;
 382
 383     dest = (src1 >> 1) - (src2 >> 1);
 384     if ((~src1) & src2 & 1)
 385         dest--;
 386     return dest;
 387 }
 388
 389 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
 390 {
 391     uint32_t dest;
 392
 393     dest = (src1 >> 1) - (src2 >> 1);
 394     if ((~src1) & src2 & 1)
 395         dest--;
 396     return dest;
 397 }
 398
 399 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
 400 NEON_VOP(cgt_s8, neon_s8, 4)
 401 NEON_VOP(cgt_u8, neon_u8, 4)
 402 NEON_VOP(cgt_s16, neon_s16, 2)
 403 NEON_VOP(cgt_u16, neon_u16, 2)
 404 NEON_VOP(cgt_s32, neon_s32, 1)
 405 NEON_VOP(cgt_u32, neon_u32, 1)
 406 #undef NEON_FN
 407
 408 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
 409 NEON_VOP(cge_s8, neon_s8, 4)
 410 NEON_VOP(cge_u8, neon_u8, 4)
 411 NEON_VOP(cge_s16, neon_s16, 2)
 412 NEON_VOP(cge_u16, neon_u16, 2)
 413 NEON_VOP(cge_s32, neon_s32, 1)
 414 NEON_VOP(cge_u32, neon_u32, 1)
 415 #undef NEON_FN
 416
 417 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
 418 NEON_VOP(min_s8, neon_s8, 4)
 419 NEON_VOP(min_u8, neon_u8, 4)
 420 NEON_VOP(min_s16, neon_s16, 2)
 421 NEON_VOP(min_u16, neon_u16, 2)
 422 NEON_VOP(min_s32, neon_s32, 1)
 423 NEON_VOP(min_u32, neon_u32, 1)
 424 NEON_POP(pmin_s8, neon_s8, 4)
 425 NEON_POP(pmin_u8, neon_u8, 4)
 426 NEON_POP(pmin_s16, neon_s16, 2)
 427 NEON_POP(pmin_u16, neon_u16, 2)
 428 #undef NEON_FN
 429
 430 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
 431 NEON_VOP(max_s8, neon_s8, 4)
 432 NEON_VOP(max_u8, neon_u8, 4)
 433 NEON_VOP(max_s16, neon_s16, 2)
 434 NEON_VOP(max_u16, neon_u16, 2)
 435 NEON_VOP(max_s32, neon_s32, 1)
 436 NEON_VOP(max_u32, neon_u32, 1)
 437 NEON_POP(pmax_s8, neon_s8, 4)
 438 NEON_POP(pmax_u8, neon_u8, 4)
 439 NEON_POP(pmax_s16, neon_s16, 2)
 440 NEON_POP(pmax_u16, neon_u16, 2)
 441 #undef NEON_FN
 442
 443 #define NEON_FN(dest, src1, src2) \
 444     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
 445 NEON_VOP(abd_s8, neon_s8, 4)
 446 NEON_VOP(abd_u8, neon_u8, 4)
 447 NEON_VOP(abd_s16, neon_s16, 2)
 448 NEON_VOP(abd_u16, neon_u16, 2)
 449 NEON_VOP(abd_s32, neon_s32, 1)
 450 NEON_VOP(abd_u32, neon_u32, 1)
 451 #undef NEON_FN
 452
 453 #define NEON_FN(dest, src1, src2) do { \
 454     int8_t tmp; \
 455     tmp = (int8_t)src2; \
 456     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 457         tmp <= -(ssize_t)sizeof(src1) * 8) { \
 458         dest = 0; \
 459     } else if (tmp < 0) { \
 460         dest = src1 >> -tmp; \
 461     } else { \
 462         dest = src1 << tmp; \
 463     }} while (0)
 464 NEON_VOP(shl_u8, neon_u8, 4)
 465 NEON_VOP(shl_u16, neon_u16, 2)
 466 NEON_VOP(shl_u32, neon_u32, 1)
 467 #undef NEON_FN
 468
 469 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
 470 {
 471     int8_t shift = (int8_t)shiftop;
 472     if (shift >= 64 || shift <= -64) {
 473         val = 0;
 474     } else if (shift < 0) {
 475         val >>= -shift;
 476     } else {
 477         val <<= shift;
 478     }
 479     return val;
 480 }
 481
 482 #define NEON_FN(dest, src1, src2) do { \
 483     int8_t tmp; \
 484     tmp = (int8_t)src2; \
 485     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 486         dest = 0; \
 487     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 488         dest = src1 >> (sizeof(src1) * 8 - 1); \
 489     } else if (tmp < 0) { \
 490         dest = src1 >> -tmp; \
 491     } else { \
 492         dest = src1 << tmp; \
 493     }} while (0)
 494 NEON_VOP(shl_s8, neon_s8, 4)
 495 NEON_VOP(shl_s16, neon_s16, 2)
 496 NEON_VOP(shl_s32, neon_s32, 1)
 497 #undef NEON_FN
 498
 499 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
 500 {
 501     int8_t shift = (int8_t)shiftop;
 502     int64_t val = valop;
 503     if (shift >= 64) {
 504         val = 0;
 505     } else if (shift <= -64) {
 506         val >>= 63;
 507     } else if (shift < 0) {
 508         val >>= -shift;
 509     } else {
 510         val <<= shift;
 511     }
 512     return val;
 513 }
 514
 515 #define NEON_FN(dest, src1, src2) do { \
 516     int8_t tmp; \
 517     tmp = (int8_t)src2; \
 518     if ((tmp >= (ssize_t)sizeof(src1) * 8) \
 519         || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
 520         dest = 0; \
 521     } else if (tmp < 0) { \
 522         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 523     } else { \
 524         dest = src1 << tmp; \
 525     }} while (0)
 526 NEON_VOP(rshl_s8, neon_s8, 4)
 527 NEON_VOP(rshl_s16, neon_s16, 2)
 528 #undef NEON_FN
 529
 530 /* The addition of the rounding constant may overflow, so we use an
 531  * intermediate 64 bits accumulator.  */
 532 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
 533 {
 534     int32_t dest;
 535     int32_t val = (int32_t)valop;
 536     int8_t shift = (int8_t)shiftop;
 537     if ((shift >= 32) || (shift <= -32)) {
 538         dest = 0;
 539     } else if (shift < 0) {
 540         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 541         dest = big_dest >> -shift;
 542     } else {
 543         dest = val << shift;
 544     }
 545     return dest;
 546 }
 547
 548 /* Handling addition overflow with 64 bits inputs values is more
 549  * tricky than with 32 bits values.  */
 550 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
 551 {
 552     int8_t shift = (int8_t)shiftop;
 553     int64_t val = valop;
 554     if ((shift >= 64) || (shift <= -64)) {
 555         val = 0;
 556     } else if (shift < 0) {
 557         val >>= (-shift - 1);
 558         if (val == INT64_MAX) {
 559             /* In this case, it means that the rounding constant is 1,
 560              * and the addition would overflow. Return the actual
 561              * result directly.  */
 562             val = 0x4000000000000000LL;
 563         } else {
 564             val++;
 565             val >>= 1;
 566         }
 567     } else {
 568         val <<= shift;
 569     }
 570     return val;
 571 }
 572
 573 #define NEON_FN(dest, src1, src2) do { \
 574     int8_t tmp; \
 575     tmp = (int8_t)src2; \
 576     if (tmp >= (ssize_t)sizeof(src1) * 8 || \
 577         tmp < -(ssize_t)sizeof(src1) * 8) { \
 578         dest = 0; \
 579     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 580         dest = src1 >> (-tmp - 1); \
 581     } else if (tmp < 0) { \
 582         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 583     } else { \
 584         dest = src1 << tmp; \
 585     }} while (0)
 586 NEON_VOP(rshl_u8, neon_u8, 4)
 587 NEON_VOP(rshl_u16, neon_u16, 2)
 588 #undef NEON_FN
 589
 590 /* The addition of the rounding constant may overflow, so we use an
 591  * intermediate 64 bits accumulator.  */
 592 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
 593 {
 594     uint32_t dest;
 595     int8_t shift = (int8_t)shiftop;
 596     if (shift >= 32 || shift < -32) {
 597         dest = 0;
 598     } else if (shift == -32) {
 599         dest = val >> 31;
 600     } else if (shift < 0) {
 601         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 602         dest = big_dest >> -shift;
 603     } else {
 604         dest = val << shift;
 605     }
 606     return dest;
 607 }
 608
 609 /* Handling addition overflow with 64 bits inputs values is more
 610  * tricky than with 32 bits values.  */
 611 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
 612 {
 613     int8_t shift = (uint8_t)shiftop;
 614     if (shift >= 64 || shift < -64) {
 615         val = 0;
 616     } else if (shift == -64) {
 617         /* Rounding a 1-bit result just preserves that bit.  */
 618         val >>= 63;
 619     } else if (shift < 0) {
 620         val >>= (-shift - 1);
 621         if (val == UINT64_MAX) {
 622             /* In this case, it means that the rounding constant is 1,
 623              * and the addition would overflow. Return the actual
 624              * result directly.  */
 625             val = 0x8000000000000000ULL;
 626         } else {
 627             val++;
 628             val >>= 1;
 629         }
 630     } else {
 631         val <<= shift;
 632     }
 633     return val;
 634 }
 635
 636 #define NEON_FN(dest, src1, src2) do { \
 637     int8_t tmp; \
 638     tmp = (int8_t)src2; \
 639     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 640         if (src1) { \
 641             SET_QC(); \
 642             dest = ~0; \
 643         } else { \
 644             dest = 0; \
 645         } \
 646     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 647         dest = 0; \
 648     } else if (tmp < 0) { \
 649         dest = src1 >> -tmp; \
 650     } else { \
 651         dest = src1 << tmp; \
 652         if ((dest >> tmp) != src1) { \
 653             SET_QC(); \
 654             dest = ~0; \
 655         } \
 656     }} while (0)
 657 NEON_VOP(qshl_u8, neon_u8, 4)
 658 NEON_VOP(qshl_u16, neon_u16, 2)
 659 NEON_VOP(qshl_u32, neon_u32, 1)
 660 #undef NEON_FN
 661
 662 uint64_t HELPER(neon_qshl_u64)(uint64_t val, uint64_t shiftop)
 663 {
 664     int8_t shift = (int8_t)shiftop;
 665     if (shift >= 64) {
 666         if (val) {
 667             val = ~(uint64_t)0;
 668             SET_QC();
 669         }
 670     } else if (shift <= -64) {
 671         val = 0;
 672     } else if (shift < 0) {
 673         val >>= -shift;
 674     } else {
 675         uint64_t tmp = val;
 676         val <<= shift;
 677         if ((val >> shift) != tmp) {
 678             SET_QC();
 679             val = ~(uint64_t)0;
 680         }
 681     }
 682     return val;
 683 }
 684
 685 #define NEON_FN(dest, src1, src2) do { \
 686     int8_t tmp; \
 687     tmp = (int8_t)src2; \
 688     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 689         if (src1) { \
 690             SET_QC(); \
 691             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 692             if (src1 > 0) { \
 693                 dest--; \
 694             } \
 695         } else { \
 696             dest = src1; \
 697         } \
 698     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 699         dest = src1 >> 31; \
 700     } else if (tmp < 0) { \
 701         dest = src1 >> -tmp; \
 702     } else { \
 703         dest = src1 << tmp; \
 704         if ((dest >> tmp) != src1) { \
 705             SET_QC(); \
 706             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 707             if (src1 > 0) { \
 708                 dest--; \
 709             } \
 710         } \
 711     }} while (0)
 712 NEON_VOP(qshl_s8, neon_s8, 4)
 713 NEON_VOP(qshl_s16, neon_s16, 2)
 714 NEON_VOP(qshl_s32, neon_s32, 1)
 715 #undef NEON_FN
 716
 717 uint64_t HELPER(neon_qshl_s64)(uint64_t valop, uint64_t shiftop)
 718 {
 719     int8_t shift = (uint8_t)shiftop;
 720     int64_t val = valop;
 721     if (shift >= 64) {
 722         if (val) {
 723             SET_QC();
 724             val = (val >> 63) ^ ~SIGNBIT64;
 725         }
 726     } else if (shift <= -64) {
 727         val >>= 63;
 728     } else if (shift < 0) {
 729         val >>= -shift;
 730     } else {
 731         int64_t tmp = val;
 732         val <<= shift;
 733         if ((val >> shift) != tmp) {
 734             SET_QC();
 735             val = (tmp >> 63) ^ ~SIGNBIT64;
 736         }
 737     }
 738     return val;
 739 }
 740
 741 #define NEON_FN(dest, src1, src2) do { \
 742     if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
 743         SET_QC(); \
 744         dest = 0; \
 745     } else { \
 746         int8_t tmp; \
 747         tmp = (int8_t)src2; \
 748         if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 749             if (src1) { \
 750                 SET_QC(); \
 751                 dest = ~0; \
 752             } else { \
 753                 dest = 0; \
 754             } \
 755         } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 756             dest = 0; \
 757         } else if (tmp < 0) { \
 758             dest = src1 >> -tmp; \
 759         } else { \
 760             dest = src1 << tmp; \
 761             if ((dest >> tmp) != src1) { \
 762                 SET_QC(); \
 763                 dest = ~0; \
 764             } \
 765         } \
 766     }} while (0)
 767 NEON_VOP(qshlu_s8, neon_u8, 4)
 768 NEON_VOP(qshlu_s16, neon_u16, 2)
 769 #undef NEON_FN
 770
 771 uint32_t HELPER(neon_qshlu_s32)(uint32_t valop, uint32_t shiftop)
 772 {
 773     if ((int32_t)valop < 0) {
 774         SET_QC();
 775         return 0;
 776     }
 777     return helper_neon_qshl_u32(valop, shiftop);
 778 }
 779
 780 uint64_t HELPER(neon_qshlu_s64)(uint64_t valop, uint64_t shiftop)
 781 {
 782     if ((int64_t)valop < 0) {
 783         SET_QC();
 784         return 0;
 785     }
 786     return helper_neon_qshl_u64(valop, shiftop);
 787 }
 788
 789 /* FIXME: This is wrong.  */
 790 #define NEON_FN(dest, src1, src2) do { \
 791     int8_t tmp; \
 792     tmp = (int8_t)src2; \
 793     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 794         if (src1) { \
 795             SET_QC(); \
 796             dest = ~0; \
 797         } else { \
 798             dest = 0; \
 799         } \
 800     } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
 801         dest = 0; \
 802     } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
 803         dest = src1 >> (sizeof(src1) * 8 - 1); \
 804     } else if (tmp < 0) { \
 805         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 806     } else { \
 807         dest = src1 << tmp; \
 808         if ((dest >> tmp) != src1) { \
 809             SET_QC(); \
 810             dest = ~0; \
 811         } \
 812     }} while (0)
 813 NEON_VOP(qrshl_u8, neon_u8, 4)
 814 NEON_VOP(qrshl_u16, neon_u16, 2)
 815 #undef NEON_FN
 816
 817 /* The addition of the rounding constant may overflow, so we use an
 818  * intermediate 64 bits accumulator.  */
 819 uint32_t HELPER(neon_qrshl_u32)(uint32_t val, uint32_t shiftop)
 820 {
 821     uint32_t dest;
 822     int8_t shift = (int8_t)shiftop;
 823     if (shift >= 32) {
 824         if (val) {
 825             SET_QC();
 826             dest = ~0;
 827         } else {
 828             dest = 0;
 829         }
 830     } else if (shift < -32) {
 831         dest = 0;
 832     } else if (shift == -32) {
 833         dest = val >> 31;
 834     } else if (shift < 0) {
 835         uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
 836         dest = big_dest >> -shift;
 837     } else {
 838         dest = val << shift;
 839         if ((dest >> shift) != val) {
 840             SET_QC();
 841             dest = ~0;
 842         }
 843     }
 844     return dest;
 845 }
 846
 847 /* Handling addition overflow with 64 bits inputs values is more
 848  * tricky than with 32 bits values.  */
 849 uint64_t HELPER(neon_qrshl_u64)(uint64_t val, uint64_t shiftop)
 850 {
 851     int8_t shift = (int8_t)shiftop;
 852     if (shift >= 64) {
 853         if (val) {
 854             SET_QC();
 855             val = ~0;
 856         }
 857     } else if (shift < -64) {
 858         val = 0;
 859     } else if (shift == -64) {
 860         val >>= 63;
 861     } else if (shift < 0) {
 862         val >>= (-shift - 1);
 863         if (val == UINT64_MAX) {
 864             /* In this case, it means that the rounding constant is 1,
 865              * and the addition would overflow. Return the actual
 866              * result directly.  */
 867             val = 0x8000000000000000ULL;
 868         } else {
 869             val++;
 870             val >>= 1;
 871         }
 872     } else { \
 873         uint64_t tmp = val;
 874         val <<= shift;
 875         if ((val >> shift) != tmp) {
 876             SET_QC();
 877             val = ~0;
 878         }
 879     }
 880     return val;
 881 }
 882
 883 #define NEON_FN(dest, src1, src2) do { \
 884     int8_t tmp; \
 885     tmp = (int8_t)src2; \
 886     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
 887         if (src1) { \
 888             SET_QC(); \
 889             dest = (1 << (sizeof(src1) * 8 - 1)); \
 890             if (src1 > 0) { \
 891                 dest--; \
 892             } \
 893         } else { \
 894             dest = 0; \
 895         } \
 896     } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
 897         dest = 0; \
 898     } else if (tmp < 0) { \
 899         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
 900     } else { \
 901         dest = src1 << tmp; \
 902         if ((dest >> tmp) != src1) { \
 903             SET_QC(); \
 904             dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
 905             if (src1 > 0) { \
 906                 dest--; \
 907             } \
 908         } \
 909     }} while (0)
 910 NEON_VOP(qrshl_s8, neon_s8, 4)
 911 NEON_VOP(qrshl_s16, neon_s16, 2)
 912 #undef NEON_FN
 913
 914 /* The addition of the rounding constant may overflow, so we use an
 915  * intermediate 64 bits accumulator.  */
 916 uint32_t HELPER(neon_qrshl_s32)(uint32_t valop, uint32_t shiftop)
 917 {
 918     int32_t dest;
 919     int32_t val = (int32_t)valop;
 920     int8_t shift = (int8_t)shiftop;
 921     if (shift >= 32) {
 922         if (val) {
 923             SET_QC();
 924             dest = (val >> 31) ^ ~SIGNBIT;
 925         } else {
 926             dest = 0;
 927         }
 928     } else if (shift <= -32) {
 929         dest = 0;
 930     } else if (shift < 0) {
 931         int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
 932         dest = big_dest >> -shift;
 933     } else {
 934         dest = val << shift;
 935         if ((dest >> shift) != val) {
 936             SET_QC();
 937             dest = (val >> 31) ^ ~SIGNBIT;
 938         }
 939     }
 940     return dest;
 941 }
 942
 943 /* Handling addition overflow with 64 bits inputs values is more
 944  * tricky than with 32 bits values.  */
 945 uint64_t HELPER(neon_qrshl_s64)(uint64_t valop, uint64_t shiftop)
 946 {
 947     int8_t shift = (uint8_t)shiftop;
 948     int64_t val = valop;
 949
 950     if (shift >= 64) {
 951         if (val) {
 952             SET_QC();
 953             val = (val >> 63) ^ ~SIGNBIT64;
 954         }
 955     } else if (shift <= -64) {
 956         val = 0;
 957     } else if (shift < 0) {
 958         val >>= (-shift - 1);
 959         if (val == INT64_MAX) {
 960             /* In this case, it means that the rounding constant is 1,
 961              * and the addition would overflow. Return the actual
 962              * result directly.  */
 963             val = 0x4000000000000000ULL;
 964         } else {
 965             val++;
 966             val >>= 1;
 967         }
 968     } else {
 969         int64_t tmp = val;
 970         val <<= shift;
 971         if ((val >> shift) != tmp) {
 972             SET_QC();
 973             val = (tmp >> 63) ^ ~SIGNBIT64;
 974         }
 975     }
 976     return val;
 977 }
 978
 979 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
 980 {
 981     uint32_t mask;
 982     mask = (a ^ b) & 0x80808080u;
 983     a &= ~0x80808080u;
 984     b &= ~0x80808080u;
 985     return (a + b) ^ mask;
 986 }
 987
 988 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
 989 {
 990     uint32_t mask;
 991     mask = (a ^ b) & 0x80008000u;
 992     a &= ~0x80008000u;
 993     b &= ~0x80008000u;
 994     return (a + b) ^ mask;
 995 }
 996
 997 #define NEON_FN(dest, src1, src2) dest = src1 + src2
 998 NEON_POP(padd_u8, neon_u8, 4)
 999 NEON_POP(padd_u16, neon_u16, 2)
1000 #undef NEON_FN
1001
1002 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1003 NEON_VOP(sub_u8, neon_u8, 4)
1004 NEON_VOP(sub_u16, neon_u16, 2)
1005 #undef NEON_FN
1006
1007 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1008 NEON_VOP(mul_u8, neon_u8, 4)
1009 NEON_VOP(mul_u16, neon_u16, 2)
1010 #undef NEON_FN
1011
1012 /* Polynomial multiplication is like integer multiplication except the
1013    partial products are XORed, not added.  */
1014 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1015 {
1016     uint32_t mask;
1017     uint32_t result;
1018     result = 0;
1019     while (op1) {
1020         mask = 0;
1021         if (op1 & 1)
1022             mask |= 0xff;
1023         if (op1 & (1 << 8))
1024             mask |= (0xff << 8);
1025         if (op1 & (1 << 16))
1026             mask |= (0xff << 16);
1027         if (op1 & (1 << 24))
1028             mask |= (0xff << 24);
1029         result ^= op2 & mask;
1030         op1 = (op1 >> 1) & 0x7f7f7f7f;
1031         op2 = (op2 << 1) & 0xfefefefe;
1032     }
1033     return result;
1034 }
1035
1036 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1037 {
1038     uint64_t result = 0;
1039     uint64_t mask;
1040     uint64_t op2ex = op2;
1041     op2ex = (op2ex & 0xff) |
1042         ((op2ex & 0xff00) << 8) |
1043         ((op2ex & 0xff0000) << 16) |
1044         ((op2ex & 0xff000000) << 24);
1045     while (op1) {
1046         mask = 0;
1047         if (op1 & 1) {
1048             mask |= 0xffff;
1049         }
1050         if (op1 & (1 << 8)) {
1051             mask |= (0xffffU << 16);
1052         }
1053         if (op1 & (1 << 16)) {
1054             mask |= (0xffffULL << 32);
1055         }
1056         if (op1 & (1 << 24)) {
1057             mask |= (0xffffULL << 48);
1058         }
1059         result ^= op2ex & mask;
1060         op1 = (op1 >> 1) & 0x7f7f7f7f;
1061         op2ex <<= 1;
1062     }
1063     return result;
1064 }
1065
1066 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1067 NEON_VOP(tst_u8, neon_u8, 4)
1068 NEON_VOP(tst_u16, neon_u16, 2)
1069 NEON_VOP(tst_u32, neon_u32, 1)
1070 #undef NEON_FN
1071
1072 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1073 NEON_VOP(ceq_u8, neon_u8, 4)
1074 NEON_VOP(ceq_u16, neon_u16, 2)
1075 NEON_VOP(ceq_u32, neon_u32, 1)
1076 #undef NEON_FN
1077
1078 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1079 NEON_VOP1(abs_s8, neon_s8, 4)
1080 NEON_VOP1(abs_s16, neon_s16, 2)
1081 #undef NEON_FN
1082
1083 /* Count Leading Sign/Zero Bits.  */
1084 static inline int do_clz8(uint8_t x)
1085 {
1086     int n;
1087     for (n = 8; x; n--)
1088         x >>= 1;
1089     return n;
1090 }
1091
1092 static inline int do_clz16(uint16_t x)
1093 {
1094     int n;
1095     for (n = 16; x; n--)
1096         x >>= 1;
1097     return n;
1098 }
1099
1100 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1101 NEON_VOP1(clz_u8, neon_u8, 4)
1102 #undef NEON_FN
1103
1104 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1105 NEON_VOP1(clz_u16, neon_u16, 2)
1106 #undef NEON_FN
1107
1108 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1109 NEON_VOP1(cls_s8, neon_s8, 4)
1110 #undef NEON_FN
1111
1112 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1113 NEON_VOP1(cls_s16, neon_s16, 2)
1114 #undef NEON_FN
1115
1116 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1117 {
1118     int count;
1119     if ((int32_t)x < 0)
1120         x = ~x;
1121     for (count = 32; x; count--)
1122         x = x >> 1;
1123     return count - 1;
1124 }
1125
1126 /* Bit count.  */
1127 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1128 {
1129     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
1130     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
1131     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);
1132     return x;
1133 }
1134
1135 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1136     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1137     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1138         SET_QC(); \
1139         tmp = (tmp >> 31) ^ ~SIGNBIT; \
1140     } else { \
1141         tmp <<= 1; \
1142     } \
1143     if (round) { \
1144         int32_t old = tmp; \
1145         tmp += 1 << 15; \
1146         if ((int32_t)tmp < old) { \
1147             SET_QC(); \
1148             tmp = SIGNBIT - 1; \
1149         } \
1150     } \
1151     dest = tmp >> 16; \
1152     } while(0)
1153 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1154 NEON_VOP(qdmulh_s16, neon_s16, 2)
1155 #undef NEON_FN
1156 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1157 NEON_VOP(qrdmulh_s16, neon_s16, 2)
1158 #undef NEON_FN
1159 #undef NEON_QDMULH16
1160
1161 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1162     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1163     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1164         SET_QC(); \
1165         tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1166     } else { \
1167         tmp <<= 1; \
1168     } \
1169     if (round) { \
1170         int64_t old = tmp; \
1171         tmp += (int64_t)1 << 31; \
1172         if ((int64_t)tmp < old) { \
1173             SET_QC(); \
1174             tmp = SIGNBIT64 - 1; \
1175         } \
1176     } \
1177     dest = tmp >> 32; \
1178     } while(0)
1179 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1180 NEON_VOP(qdmulh_s32, neon_s32, 1)
1181 #undef NEON_FN
1182 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1183 NEON_VOP(qrdmulh_s32, neon_s32, 1)
1184 #undef NEON_FN
1185 #undef NEON_QDMULH32
1186
1187 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1188 {
1189     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1190            | ((x >> 24) & 0xff000000u);
1191 }
1192
1193 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1194 {
1195     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1196 }
1197
1198 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1199 {
1200     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1201             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1202 }
1203
1204 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1205 {
1206     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1207 }
1208
1209 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1210 {
1211     x &= 0xff80ff80ff80ff80ull;
1212     x += 0x0080008000800080ull;
1213     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1214             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1215 }
1216
1217 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1218 {
1219     x &= 0xffff8000ffff8000ull;
1220     x += 0x0000800000008000ull;
1221     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1222 }
1223
1224 uint32_t HELPER(neon_unarrow_sat8)(uint64_t x)
1225 {
1226     uint16_t s;
1227     uint8_t d;
1228     uint32_t res = 0;
1229 #define SAT8(n) \
1230     s = x >> n; \
1231     if (s & 0x8000) { \
1232         SET_QC(); \
1233     } else { \
1234         if (s > 0xff) { \
1235             d = 0xff; \
1236             SET_QC(); \
1237         } else  { \
1238             d = s; \
1239         } \
1240         res |= (uint32_t)d << (n / 2); \
1241     }
1242
1243     SAT8(0);
1244     SAT8(16);
1245     SAT8(32);
1246     SAT8(48);
1247 #undef SAT8
1248     return res;
1249 }
1250
1251 uint32_t HELPER(neon_narrow_sat_u8)(uint64_t x)
1252 {
1253     uint16_t s;
1254     uint8_t d;
1255     uint32_t res = 0;
1256 #define SAT8(n) \
1257     s = x >> n; \
1258     if (s > 0xff) { \
1259         d = 0xff; \
1260         SET_QC(); \
1261     } else  { \
1262         d = s; \
1263     } \
1264     res |= (uint32_t)d << (n / 2);
1265
1266     SAT8(0);
1267     SAT8(16);
1268     SAT8(32);
1269     SAT8(48);
1270 #undef SAT8
1271     return res;
1272 }
1273
1274 uint32_t HELPER(neon_narrow_sat_s8)(uint64_t x)
1275 {
1276     int16_t s;
1277     uint8_t d;
1278     uint32_t res = 0;
1279 #define SAT8(n) \
1280     s = x >> n; \
1281     if (s != (int8_t)s) { \
1282         d = (s >> 15) ^ 0x7f; \
1283         SET_QC(); \
1284     } else  { \
1285         d = s; \
1286     } \
1287     res |= (uint32_t)d << (n / 2);
1288
1289     SAT8(0);
1290     SAT8(16);
1291     SAT8(32);
1292     SAT8(48);
1293 #undef SAT8
1294     return res;
1295 }
1296
1297 uint32_t HELPER(neon_unarrow_sat16)(uint64_t x)
1298 {
1299     uint32_t high;
1300     uint32_t low;
1301     low = x;
1302     if (low & 0x80000000) {
1303         low = 0;
1304         SET_QC();
1305     } else if (low > 0xffff) {
1306         low = 0xffff;
1307         SET_QC();
1308     }
1309     high = x >> 32;
1310     if (high & 0x80000000) {
1311         high = 0;
1312         SET_QC();
1313     } else if (high > 0xffff) {
1314         high = 0xffff;
1315         SET_QC();
1316     }
1317     return low | (high << 16);
1318 }
1319
1320 uint32_t HELPER(neon_narrow_sat_u16)(uint64_t x)
1321 {
1322     uint32_t high;
1323     uint32_t low;
1324     low = x;
1325     if (low > 0xffff) {
1326         low = 0xffff;
1327         SET_QC();
1328     }
1329     high = x >> 32;
1330     if (high > 0xffff) {
1331         high = 0xffff;
1332         SET_QC();
1333     }
1334     return low | (high << 16);
1335 }
1336
1337 uint32_t HELPER(neon_narrow_sat_s16)(uint64_t x)
1338 {
1339     int32_t low;
1340     int32_t high;
1341     low = x;
1342     if (low != (int16_t)low) {
1343         low = (low >> 31) ^ 0x7fff;
1344         SET_QC();
1345     }
1346     high = x >> 32;
1347     if (high != (int16_t)high) {
1348         high = (high >> 31) ^ 0x7fff;
1349         SET_QC();
1350     }
1351     return (uint16_t)low | (high << 16);
1352 }
1353
1354 uint32_t HELPER(neon_unarrow_sat32)(uint64_t x)
1355 {
1356     if (x & 0x8000000000000000ull) {
1357         SET_QC();
1358         return 0;
1359     }
1360     if (x > 0xffffffffu) {
1361         SET_QC();
1362         return 0xffffffffu;
1363     }
1364     return x;
1365 }
1366
1367 uint32_t HELPER(neon_narrow_sat_u32)(uint64_t x)
1368 {
1369     if (x > 0xffffffffu) {
1370         SET_QC();
1371         return 0xffffffffu;
1372     }
1373     return x;
1374 }
1375
1376 uint32_t HELPER(neon_narrow_sat_s32)(uint64_t x)
1377 {
1378     if ((int64_t)x != (int32_t)x) {
1379         SET_QC();
1380         return ((int64_t)x >> 63) ^ 0x7fffffff;
1381     }
1382     return x;
1383 }
1384
1385 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1386 {
1387     uint64_t tmp;
1388     uint64_t ret;
1389     ret = (uint8_t)x;
1390     tmp = (uint8_t)(x >> 8);
1391     ret |= tmp << 16;
1392     tmp = (uint8_t)(x >> 16);
1393     ret |= tmp << 32;
1394     tmp = (uint8_t)(x >> 24);
1395     ret |= tmp << 48;
1396     return ret;
1397 }
1398
1399 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1400 {
1401     uint64_t tmp;
1402     uint64_t ret;
1403     ret = (uint16_t)(int8_t)x;
1404     tmp = (uint16_t)(int8_t)(x >> 8);
1405     ret |= tmp << 16;
1406     tmp = (uint16_t)(int8_t)(x >> 16);
1407     ret |= tmp << 32;
1408     tmp = (uint16_t)(int8_t)(x >> 24);
1409     ret |= tmp << 48;
1410     return ret;
1411 }
1412
1413 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1414 {
1415     uint64_t high = (uint16_t)(x >> 16);
1416     return ((uint16_t)x) | (high << 32);
1417 }
1418
1419 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1420 {
1421     uint64_t high = (int16_t)(x >> 16);
1422     return ((uint32_t)(int16_t)x) | (high << 32);
1423 }
1424
1425 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1426 {
1427     uint64_t mask;
1428     mask = (a ^ b) & 0x8000800080008000ull;
1429     a &= ~0x8000800080008000ull;
1430     b &= ~0x8000800080008000ull;
1431     return (a + b) ^ mask;
1432 }
1433
1434 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1435 {
1436     uint64_t mask;
1437     mask = (a ^ b) & 0x8000000080000000ull;
1438     a &= ~0x8000000080000000ull;
1439     b &= ~0x8000000080000000ull;
1440     return (a + b) ^ mask;
1441 }
1442
1443 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1444 {
1445     uint64_t tmp;
1446     uint64_t tmp2;
1447
1448     tmp = a & 0x0000ffff0000ffffull;
1449     tmp += (a >> 16) & 0x0000ffff0000ffffull;
1450     tmp2 = b & 0xffff0000ffff0000ull;
1451     tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1452     return    ( tmp         & 0xffff)
1453             | ((tmp  >> 16) & 0xffff0000ull)
1454             | ((tmp2 << 16) & 0xffff00000000ull)
1455             | ( tmp2        & 0xffff000000000000ull);
1456 }
1457
1458 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1459 {
1460     uint32_t low = a + (a >> 32);
1461     uint32_t high = b + (b >> 32);
1462     return low + ((uint64_t)high << 32);
1463 }
1464
1465 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1466 {
1467     uint64_t mask;
1468     mask = (a ^ ~b) & 0x8000800080008000ull;
1469     a |= 0x8000800080008000ull;
1470     b &= ~0x8000800080008000ull;
1471     return (a - b) ^ mask;
1472 }
1473
1474 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1475 {
1476     uint64_t mask;
1477     mask = (a ^ ~b) & 0x8000000080000000ull;
1478     a |= 0x8000000080000000ull;
1479     b &= ~0x8000000080000000ull;
1480     return (a - b) ^ mask;
1481 }
1482
1483 uint64_t HELPER(neon_addl_saturate_s32)(uint64_t a, uint64_t b)
1484 {
1485     uint32_t x, y;
1486     uint32_t low, high;
1487
1488     x = a;
1489     y = b;
1490     low = x + y;
1491     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1492         SET_QC();
1493         low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1494     }
1495     x = a >> 32;
1496     y = b >> 32;
1497     high = x + y;
1498     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1499         SET_QC();
1500         high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1501     }
1502     return low | ((uint64_t)high << 32);
1503 }
1504
1505 uint64_t HELPER(neon_addl_saturate_s64)(uint64_t a, uint64_t b)
1506 {
1507     uint64_t result;
1508
1509     result = a + b;
1510     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1511         SET_QC();
1512         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1513     }
1514     return result;
1515 }
1516
1517 /* We have to do the arithmetic in a larger type than
1518  * the input type, because for example with a signed 32 bit
1519  * op the absolute difference can overflow a signed 32 bit value.
1520  */
1521 #define DO_ABD(dest, x, y, intype, arithtype) do {            \
1522     arithtype tmp_x = (intype)(x);                            \
1523     arithtype tmp_y = (intype)(y);                            \
1524     dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1525     } while(0)
1526
1527 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1528 {
1529     uint64_t tmp;
1530     uint64_t result;
1531     DO_ABD(result, a, b, uint8_t, uint32_t);
1532     DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1533     result |= tmp << 16;
1534     DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1535     result |= tmp << 32;
1536     DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1537     result |= tmp << 48;
1538     return result;
1539 }
1540
1541 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1542 {
1543     uint64_t tmp;
1544     uint64_t result;
1545     DO_ABD(result, a, b, int8_t, int32_t);
1546     DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1547     result |= tmp << 16;
1548     DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1549     result |= tmp << 32;
1550     DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1551     result |= tmp << 48;
1552     return result;
1553 }
1554
1555 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1556 {
1557     uint64_t tmp;
1558     uint64_t result;
1559     DO_ABD(result, a, b, uint16_t, uint32_t);
1560     DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1561     return result | (tmp << 32);
1562 }
1563
1564 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1565 {
1566     uint64_t tmp;
1567     uint64_t result;
1568     DO_ABD(result, a, b, int16_t, int32_t);
1569     DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1570     return result | (tmp << 32);
1571 }
1572
1573 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1574 {
1575     uint64_t result;
1576     DO_ABD(result, a, b, uint32_t, uint64_t);
1577     return result;
1578 }
1579
1580 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1581 {
1582     uint64_t result;
1583     DO_ABD(result, a, b, int32_t, int64_t);
1584     return result;
1585 }
1586 #undef DO_ABD
1587
1588 /* Widening multiply. Named type is the source type.  */
1589 #define DO_MULL(dest, x, y, type1, type2) do { \
1590     type1 tmp_x = x; \
1591     type1 tmp_y = y; \
1592     dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1593     } while(0)
1594
1595 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1596 {
1597     uint64_t tmp;
1598     uint64_t result;
1599
1600     DO_MULL(result, a, b, uint8_t, uint16_t);
1601     DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1602     result |= tmp << 16;
1603     DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1604     result |= tmp << 32;
1605     DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1606     result |= tmp << 48;
1607     return result;
1608 }
1609
1610 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1611 {
1612     uint64_t tmp;
1613     uint64_t result;
1614
1615     DO_MULL(result, a, b, int8_t, uint16_t);
1616     DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1617     result |= tmp << 16;
1618     DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1619     result |= tmp << 32;
1620     DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1621     result |= tmp << 48;
1622     return result;
1623 }
1624
1625 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1626 {
1627     uint64_t tmp;
1628     uint64_t result;
1629
1630     DO_MULL(result, a, b, uint16_t, uint32_t);
1631     DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1632     return result | (tmp << 32);
1633 }
1634
1635 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1636 {
1637     uint64_t tmp;
1638     uint64_t result;
1639
1640     DO_MULL(result, a, b, int16_t, uint32_t);
1641     DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1642     return result | (tmp << 32);
1643 }
1644
1645 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1646 {
1647     uint16_t tmp;
1648     uint64_t result;
1649     result = (uint16_t)-x;
1650     tmp = -(x >> 16);
1651     result |= (uint64_t)tmp << 16;
1652     tmp = -(x >> 32);
1653     result |= (uint64_t)tmp << 32;
1654     tmp = -(x >> 48);
1655     result |= (uint64_t)tmp << 48;
1656     return result;
1657 }
1658
1659 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1660 {
1661     uint32_t low = -x;
1662     uint32_t high = -(x >> 32);
1663     return low | ((uint64_t)high << 32);
1664 }
1665
1666 /* FIXME:  There should be a native op for this.  */
1667 uint64_t HELPER(neon_negl_u64)(uint64_t x)
1668 {
1669     return -x;
1670 }
1671
1672 /* Saturnating sign manuipulation.  */
1673 /* ??? Make these use NEON_VOP1 */
1674 #define DO_QABS8(x) do { \
1675     if (x == (int8_t)0x80) { \
1676         x = 0x7f; \
1677         SET_QC(); \
1678     } else if (x < 0) { \
1679         x = -x; \
1680     }} while (0)
1681 uint32_t HELPER(neon_qabs_s8)(uint32_t x)
1682 {
1683     neon_s8 vec;
1684     NEON_UNPACK(neon_s8, vec, x);
1685     DO_QABS8(vec.v1);
1686     DO_QABS8(vec.v2);
1687     DO_QABS8(vec.v3);
1688     DO_QABS8(vec.v4);
1689     NEON_PACK(neon_s8, x, vec);
1690     return x;
1691 }
1692 #undef DO_QABS8
1693
1694 #define DO_QNEG8(x) do { \
1695     if (x == (int8_t)0x80) { \
1696         x = 0x7f; \
1697         SET_QC(); \
1698     } else { \
1699         x = -x; \
1700     }} while (0)
1701 uint32_t HELPER(neon_qneg_s8)(uint32_t x)
1702 {
1703     neon_s8 vec;
1704     NEON_UNPACK(neon_s8, vec, x);
1705     DO_QNEG8(vec.v1);
1706     DO_QNEG8(vec.v2);
1707     DO_QNEG8(vec.v3);
1708     DO_QNEG8(vec.v4);
1709     NEON_PACK(neon_s8, x, vec);
1710     return x;
1711 }
1712 #undef DO_QNEG8
1713
1714 #define DO_QABS16(x) do { \
1715     if (x == (int16_t)0x8000) { \
1716         x = 0x7fff; \
1717         SET_QC(); \
1718     } else if (x < 0) { \
1719         x = -x; \
1720     }} while (0)
1721 uint32_t HELPER(neon_qabs_s16)(uint32_t x)
1722 {
1723     neon_s16 vec;
1724     NEON_UNPACK(neon_s16, vec, x);
1725     DO_QABS16(vec.v1);
1726     DO_QABS16(vec.v2);
1727     NEON_PACK(neon_s16, x, vec);
1728     return x;
1729 }
1730 #undef DO_QABS16
1731
1732 #define DO_QNEG16(x) do { \
1733     if (x == (int16_t)0x8000) { \
1734         x = 0x7fff; \
1735         SET_QC(); \
1736     } else { \
1737         x = -x; \
1738     }} while (0)
1739 uint32_t HELPER(neon_qneg_s16)(uint32_t x)
1740 {
1741     neon_s16 vec;
1742     NEON_UNPACK(neon_s16, vec, x);
1743     DO_QNEG16(vec.v1);
1744     DO_QNEG16(vec.v2);
1745     NEON_PACK(neon_s16, x, vec);
1746     return x;
1747 }
1748 #undef DO_QNEG16
1749
1750 uint32_t HELPER(neon_qabs_s32)(uint32_t x)
1751 {
1752     if (x == SIGNBIT) {
1753         SET_QC();
1754         x = ~SIGNBIT;
1755     } else if ((int32_t)x < 0) {
1756         x = -x;
1757     }
1758     return x;
1759 }
1760
1761 uint32_t HELPER(neon_qneg_s32)(uint32_t x)
1762 {
1763     if (x == SIGNBIT) {
1764         SET_QC();
1765         x = ~SIGNBIT;
1766     } else {
1767         x = -x;
1768     }
1769     return x;
1770 }
1771
1772 /* NEON Float helpers.  */
1773 uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1774 {
1775     return float32_val(float32_min(make_float32(a), make_float32(b), NFS));
1776 }
1777
1778 uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1779 {
1780     return float32_val(float32_max(make_float32(a), make_float32(b), NFS));
1781 }
1782
1783 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1784 {
1785     float32 f0 = make_float32(a);
1786     float32 f1 = make_float32(b);
1787     return float32_val(float32_abs(float32_sub(f0, f1, NFS)));
1788 }
1789
1790 uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1791 {
1792     return float32_val(float32_add(make_float32(a), make_float32(b), NFS));
1793 }
1794
1795 uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1796 {
1797     return float32_val(float32_sub(make_float32(a), make_float32(b), NFS));
1798 }
1799
1800 uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1801 {
1802     return float32_val(float32_mul(make_float32(a), make_float32(b), NFS));
1803 }
1804
1805 /* Floating point comparisons produce an integer result.
1806  * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1807  * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1808  */
1809 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b)
1810 {
1811     return -float32_eq_quiet(make_float32(a), make_float32(b), NFS);
1812 }
1813
1814 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b)
1815 {
1816     return -float32_le(make_float32(b), make_float32(a), NFS);
1817 }
1818
1819 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b)
1820 {
1821     return -float32_lt(make_float32(b), make_float32(a), NFS);
1822 }
1823
1824 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1825 {
1826     float32 f0 = float32_abs(make_float32(a));
1827     float32 f1 = float32_abs(make_float32(b));
1828     return -float32_le(f1, f0, NFS);
1829 }
1830
1831 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1832 {
1833     float32 f0 = float32_abs(make_float32(a));
1834     float32 f1 = float32_abs(make_float32(b));
1835     return -float32_lt(f1, f0, NFS);
1836 }
1837
1838 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1839
1840 void HELPER(neon_qunzip8)(uint32_t rd, uint32_t rm)
1841 {
1842     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1843     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1844     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1845     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1846     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1847         | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1848         | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1849         | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1850     uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1851         | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1852         | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1853         | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1854     uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1855         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1856         | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1857         | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1858     uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1859         | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1860         | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1861         | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1862     env->vfp.regs[rm] = make_float64(m0);
1863     env->vfp.regs[rm + 1] = make_float64(m1);
1864     env->vfp.regs[rd] = make_float64(d0);
1865     env->vfp.regs[rd + 1] = make_float64(d1);
1866 }
1867
1868 void HELPER(neon_qunzip16)(uint32_t rd, uint32_t rm)
1869 {
1870     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1871     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1872     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1873     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1874     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1875         | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1876     uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1877         | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1878     uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1879         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1880     uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1881         | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1882     env->vfp.regs[rm] = make_float64(m0);
1883     env->vfp.regs[rm + 1] = make_float64(m1);
1884     env->vfp.regs[rd] = make_float64(d0);
1885     env->vfp.regs[rd + 1] = make_float64(d1);
1886 }
1887
1888 void HELPER(neon_qunzip32)(uint32_t rd, uint32_t rm)
1889 {
1890     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1891     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1892     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1893     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1894     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1895     uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1896     uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1897     uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1898     env->vfp.regs[rm] = make_float64(m0);
1899     env->vfp.regs[rm + 1] = make_float64(m1);
1900     env->vfp.regs[rd] = make_float64(d0);
1901     env->vfp.regs[rd + 1] = make_float64(d1);
1902 }
1903
1904 void HELPER(neon_unzip8)(uint32_t rd, uint32_t rm)
1905 {
1906     uint64_t zm = float64_val(env->vfp.regs[rm]);
1907     uint64_t zd = float64_val(env->vfp.regs[rd]);
1908     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1909         | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1910         | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1911         | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1912     uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1913         | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1914         | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1915         | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1916     env->vfp.regs[rm] = make_float64(m0);
1917     env->vfp.regs[rd] = make_float64(d0);
1918 }
1919
1920 void HELPER(neon_unzip16)(uint32_t rd, uint32_t rm)
1921 {
1922     uint64_t zm = float64_val(env->vfp.regs[rm]);
1923     uint64_t zd = float64_val(env->vfp.regs[rd]);
1924     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1925         | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1926     uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1927         | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1928     env->vfp.regs[rm] = make_float64(m0);
1929     env->vfp.regs[rd] = make_float64(d0);
1930 }
1931
1932 void HELPER(neon_qzip8)(uint32_t rd, uint32_t rm)
1933 {
1934     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1935     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1936     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1937     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1938     uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1939         | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1940         | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1941         | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1942     uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1943         | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1944         | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1945         | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1946     uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1947         | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
1948         | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1949         | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
1950     uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
1951         | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
1952         | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
1953         | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1954     env->vfp.regs[rm] = make_float64(m0);
1955     env->vfp.regs[rm + 1] = make_float64(m1);
1956     env->vfp.regs[rd] = make_float64(d0);
1957     env->vfp.regs[rd + 1] = make_float64(d1);
1958 }
1959
1960 void HELPER(neon_qzip16)(uint32_t rd, uint32_t rm)
1961 {
1962     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1963     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1964     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1965     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1966     uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
1967         | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
1968     uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
1969         | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
1970     uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
1971         | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
1972     uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
1973         | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1974     env->vfp.regs[rm] = make_float64(m0);
1975     env->vfp.regs[rm + 1] = make_float64(m1);
1976     env->vfp.regs[rd] = make_float64(d0);
1977     env->vfp.regs[rd + 1] = make_float64(d1);
1978 }
1979
1980 void HELPER(neon_qzip32)(uint32_t rd, uint32_t rm)
1981 {
1982     uint64_t zm0 = float64_val(env->vfp.regs[rm]);
1983     uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
1984     uint64_t zd0 = float64_val(env->vfp.regs[rd]);
1985     uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
1986     uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
1987     uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
1988     uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1989     uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1990     env->vfp.regs[rm] = make_float64(m0);
1991     env->vfp.regs[rm + 1] = make_float64(m1);
1992     env->vfp.regs[rd] = make_float64(d0);
1993     env->vfp.regs[rd + 1] = make_float64(d1);
1994 }
1995
1996 void HELPER(neon_zip8)(uint32_t rd, uint32_t rm)
1997 {
1998     uint64_t zm = float64_val(env->vfp.regs[rm]);
1999     uint64_t zd = float64_val(env->vfp.regs[rd]);
2000     uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2001         | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2002         | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2003         | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2004     uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2005         | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2006         | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2007         | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2008     env->vfp.regs[rm] = make_float64(m0);
2009     env->vfp.regs[rd] = make_float64(d0);
2010 }
2011
2012 void HELPER(neon_zip16)(uint32_t rd, uint32_t rm)
2013 {
2014     uint64_t zm = float64_val(env->vfp.regs[rm]);
2015     uint64_t zd = float64_val(env->vfp.regs[rd]);
2016     uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2017         | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2018     uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2019         | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2020     env->vfp.regs[rm] = make_float64(m0);
2021     env->vfp.regs[rd] = make_float64(d0);
2022 }