Add a test program for the membarrier() system call

[valgrind.git] / VEX / priv / host_arm64_isel.c

/*---------------------------------------------------------------*/
/*--- begin                                 host_arm64_isel.c ---*/
/*---------------------------------------------------------------*/

/*
   This file is part of Valgrind, a dynamic binary instrumentation
   framework.

   Copyright (C) 2013-2017 OpenWorks
      info@open-works.net

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License as
   published by the Free Software Foundation; either version 2 of the
   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
   02110-1301, USA.

   The GNU General Public License is contained in the file COPYING.
*/

#include "libvex_basictypes.h"
#include "libvex_ir.h"
#include "libvex.h"
#include "ir_match.h"

#include "main_util.h"
#include "main_globals.h"
#include "host_generic_regs.h"
#include "host_generic_simd64.h"  // for 32-bit SIMD helpers
#include "host_arm64_defs.h"


/*---------------------------------------------------------*/
/*--- ISelEnv                                           ---*/
/*---------------------------------------------------------*/

/* This carries around:

   - A mapping from IRTemp to IRType, giving the type of any IRTemp we
     might encounter.  This is computed before insn selection starts,
     and does not change.

   - A mapping from IRTemp to HReg.  This tells the insn selector
     which virtual register is associated with each IRTemp temporary.
     This is computed before insn selection starts, and does not
     change.  We expect this mapping to map precisely the same set of
     IRTemps as the type mapping does.

     |vregmap|   holds the primary register for the IRTemp.
     |vregmapHI| is only used for 128-bit integer-typed
                 IRTemps.  It holds the identity of a second
                 64-bit virtual HReg, which holds the high half
                 of the value.

   - The code array, that is, the insns selected so far.

   - A counter, for generating new virtual registers.

   - The host hardware capabilities word.  This is set at the start
     and does not change.

   - A Bool for indicating whether we may generate chain-me
     instructions for control flow transfers, or whether we must use
     XAssisted.

   - The maximum guest address of any guest insn in this block.
     Actually, the address of the highest-addressed byte from any insn
     in this block.  Is set at the start and does not change.  This is
     used for detecting jumps which are definitely forward-edges from
     this block, and therefore can be made (chained) to the fast entry
     point of the destination, thereby avoiding the destination's
     event check.

    - An IRExpr*, which may be NULL, holding the IR expression (an
      IRRoundingMode-encoded value) to which the FPU's rounding mode
      was most recently set.  Setting to NULL is always safe.  Used to
      avoid redundant settings of the FPU's rounding mode, as
      described in set_FPCR_rounding_mode below.

   Note, this is all (well, mostly) host-independent.
*/

typedef
   struct {
      /* Constant -- are set at the start and do not change. */
      IRTypeEnv*   type_env;

      HReg*        vregmap;
      HReg*        vregmapHI;
      Int          n_vregmap;

      UInt         hwcaps;

      Bool         chainingAllowed;
      Addr64       max_ga;

      /* These are modified as we go along. */
      HInstrArray* code;
      Int          vreg_ctr;

      IRExpr*      previous_rm;
   }
   ISelEnv;

static HReg lookupIRTemp ( ISelEnv* env, IRTemp tmp )
{
   vassert(tmp >= 0);
   vassert(tmp < env->n_vregmap);
   return env->vregmap[tmp];
}

static void lookupIRTempPair ( HReg* vrHI, HReg* vrLO, 
                               ISelEnv* env, IRTemp tmp )
{
   vassert(tmp >= 0);
   vassert(tmp < env->n_vregmap);
   vassert(! hregIsInvalid(env->vregmapHI[tmp]));
   *vrLO = env->vregmap[tmp];
   *vrHI = env->vregmapHI[tmp];
}

static void addInstr ( ISelEnv* env, ARM64Instr* instr )
{
   addHInstr(env->code, instr);
   if (vex_traceflags & VEX_TRACE_VCODE) {
      ppARM64Instr(instr);
      vex_printf("\n");
   }
}

static HReg newVRegI ( ISelEnv* env )
{
   HReg reg = mkHReg(True/*virtual reg*/, HRcInt64, 0, env->vreg_ctr);
   env->vreg_ctr++;
   return reg;
}

static HReg newVRegD ( ISelEnv* env )
{
   HReg reg = mkHReg(True/*virtual reg*/, HRcFlt64, 0, env->vreg_ctr);
   env->vreg_ctr++;
   return reg;
}

static HReg newVRegV ( ISelEnv* env )
{
   HReg reg = mkHReg(True/*virtual reg*/, HRcVec128, 0, env->vreg_ctr);
   env->vreg_ctr++;
   return reg;
}


/*---------------------------------------------------------*/
/*--- ISEL: Forward declarations                        ---*/
/*---------------------------------------------------------*/

/* These are organised as iselXXX and iselXXX_wrk pairs.  The
   iselXXX_wrk do the real work, but are not to be called directly.
   For each XXX, iselXXX calls its iselXXX_wrk counterpart, then
   checks that all returned registers are virtual.  You should not
   call the _wrk version directly.

   Because some forms of ARM64 memory amodes are implicitly scaled by
   the access size, iselIntExpr_AMode takes an IRType which tells it
   the type of the access for which the amode is to be used.  This
   type needs to be correct, else you'll get incorrect code.
*/
static ARM64AMode* iselIntExpr_AMode_wrk ( ISelEnv* env,
                                           IRExpr* e, IRType dty );
static ARM64AMode* iselIntExpr_AMode     ( ISelEnv* env,
                                           IRExpr* e, IRType dty );

static ARM64RIA*   iselIntExpr_RIA_wrk   ( ISelEnv* env, IRExpr* e );
static ARM64RIA*   iselIntExpr_RIA       ( ISelEnv* env, IRExpr* e );

static ARM64RIL*   iselIntExpr_RIL_wrk   ( ISelEnv* env, IRExpr* e );
static ARM64RIL*   iselIntExpr_RIL       ( ISelEnv* env, IRExpr* e );

static ARM64RI6*   iselIntExpr_RI6_wrk   ( ISelEnv* env, IRExpr* e );
static ARM64RI6*   iselIntExpr_RI6       ( ISelEnv* env, IRExpr* e );

static ARM64CondCode iselCondCode_wrk    ( ISelEnv* env, IRExpr* e );
static ARM64CondCode iselCondCode        ( ISelEnv* env, IRExpr* e );

static HReg        iselIntExpr_R_wrk     ( ISelEnv* env, IRExpr* e );
static HReg        iselIntExpr_R         ( ISelEnv* env, IRExpr* e );

static void        iselInt128Expr_wrk    ( /*OUT*/HReg* rHi, HReg* rLo, 
                                           ISelEnv* env, IRExpr* e );
static void        iselInt128Expr        ( /*OUT*/HReg* rHi, HReg* rLo, 
                                           ISelEnv* env, IRExpr* e );

static HReg        iselDblExpr_wrk        ( ISelEnv* env, IRExpr* e );
static HReg        iselDblExpr            ( ISelEnv* env, IRExpr* e );

static HReg        iselFltExpr_wrk        ( ISelEnv* env, IRExpr* e );
static HReg        iselFltExpr            ( ISelEnv* env, IRExpr* e );

static HReg        iselF16Expr_wrk        ( ISelEnv* env, IRExpr* e );
static HReg        iselF16Expr            ( ISelEnv* env, IRExpr* e );

static HReg        iselV128Expr_wrk       ( ISelEnv* env, IRExpr* e );
static HReg        iselV128Expr           ( ISelEnv* env, IRExpr* e );

static void        iselV256Expr_wrk       ( /*OUT*/HReg* rHi, HReg* rLo, 
                                            ISelEnv* env, IRExpr* e );
static void        iselV256Expr           ( /*OUT*/HReg* rHi, HReg* rLo, 
                                            ISelEnv* env, IRExpr* e );

static ARM64RIL* mb_mkARM64RIL_I ( ULong imm64 );


/*---------------------------------------------------------*/
/*--- ISEL: Misc helpers                                ---*/
/*---------------------------------------------------------*/

/* Generate an amode suitable for a 64-bit sized access relative to
   the baseblock register (X21).  This generates an RI12 amode, which
   means its scaled by the access size, which is why the access size
   -- 64 bit -- is stated explicitly here.  Consequently |off| needs
   to be divisible by 8. */
static ARM64AMode* mk_baseblock_64bit_access_amode ( UInt off )
{
   vassert(off < (8 << 12)); /* otherwise it's unrepresentable */
   vassert((off & 7) == 0);  /* ditto */
   return ARM64AMode_RI12(hregARM64_X21(), off >> 3, 8/*scale*/);
}

/* Ditto, for 32 bit accesses. */
static ARM64AMode* mk_baseblock_32bit_access_amode ( UInt off )
{
   vassert(off < (4 << 12)); /* otherwise it's unrepresentable */
   vassert((off & 3) == 0);  /* ditto */
   return ARM64AMode_RI12(hregARM64_X21(), off >> 2, 4/*scale*/);
}

/* Ditto, for 16 bit accesses. */
static ARM64AMode* mk_baseblock_16bit_access_amode ( UInt off )
{
   vassert(off < (2 << 12)); /* otherwise it's unrepresentable */
   vassert((off & 1) == 0);  /* ditto */
   return ARM64AMode_RI12(hregARM64_X21(), off >> 1, 2/*scale*/);
}

/* Ditto, for 8 bit accesses. */
static ARM64AMode* mk_baseblock_8bit_access_amode ( UInt off )
{
   vassert(off < (1 << 12)); /* otherwise it's unrepresentable */
   return ARM64AMode_RI12(hregARM64_X21(), off >> 0, 1/*scale*/);
}

static HReg mk_baseblock_128bit_access_addr ( ISelEnv* env, UInt off )
{
   vassert(off < (1<<12));
   HReg r = newVRegI(env);
   addInstr(env, ARM64Instr_Arith(r, hregARM64_X21(),
                                     ARM64RIA_I12(off,0), True/*isAdd*/));
   return r;
}

static HReg get_baseblock_register ( void )
{
   return hregARM64_X21();
}

/* Generate code to zero extend a 32 bit value in 'src' to 64 bits, in
   a new register, and return the new register. */
static HReg widen_z_32_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst  = newVRegI(env);
   ARM64RIL* mask = ARM64RIL_I13(1, 0, 31); /* encodes 0xFFFFFFFF */
   addInstr(env, ARM64Instr_Logic(dst, src, mask, ARM64lo_AND));
   return dst;
}

/* Generate code to sign extend a 16 bit value in 'src' to 64 bits, in
   a new register, and return the new register. */
static HReg widen_s_16_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst = newVRegI(env);
   ARM64RI6* n48 = ARM64RI6_I6(48);
   addInstr(env, ARM64Instr_Shift(dst, src, n48, ARM64sh_SHL));
   addInstr(env, ARM64Instr_Shift(dst, dst, n48, ARM64sh_SAR));
   return dst;
}

/* Generate code to zero extend a 16 bit value in 'src' to 64 bits, in
   a new register, and return the new register. */
static HReg widen_z_16_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst  = newVRegI(env);
   ARM64RIL* mask = ARM64RIL_I13(1, 0, 15); /* encodes 0xFFFF */
   addInstr(env, ARM64Instr_Logic(dst, src, mask, ARM64lo_AND));
   return dst;
}

/* Generate code to sign extend a 32 bit value in 'src' to 64 bits, in
   a new register, and return the new register. */
static HReg widen_s_32_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst = newVRegI(env);
   ARM64RI6* n32 = ARM64RI6_I6(32);
   addInstr(env, ARM64Instr_Shift(dst, src, n32, ARM64sh_SHL));
   addInstr(env, ARM64Instr_Shift(dst, dst, n32, ARM64sh_SAR));
   return dst;
}

/* Generate code to sign extend a 8 bit value in 'src' to 64 bits, in
   a new register, and return the new register. */
static HReg widen_s_8_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst = newVRegI(env);
   ARM64RI6* n56 = ARM64RI6_I6(56);
   addInstr(env, ARM64Instr_Shift(dst, src, n56, ARM64sh_SHL));
   addInstr(env, ARM64Instr_Shift(dst, dst, n56, ARM64sh_SAR));
   return dst;
}

static HReg widen_z_8_to_64 ( ISelEnv* env, HReg src )
{
   HReg      dst  = newVRegI(env);
   ARM64RIL* mask = ARM64RIL_I13(1, 0, 7); /* encodes 0xFF */
   addInstr(env, ARM64Instr_Logic(dst, src, mask, ARM64lo_AND));
   return dst;
}

/* Is this IRExpr_Const(IRConst_U64(0)) ? */
static Bool isZeroU64 ( IRExpr* e ) {
   if (e->tag != Iex_Const) return False;
   IRConst* con = e->Iex.Const.con;
   vassert(con->tag == Ico_U64);
   return con->Ico.U64 == 0;
}


/*---------------------------------------------------------*/
/*--- ISEL: FP rounding mode helpers                    ---*/
/*---------------------------------------------------------*/

/* Set the FP rounding mode: 'mode' is an I32-typed expression
   denoting a value in the range 0 .. 3, indicating a round mode
   encoded as per type IRRoundingMode -- the first four values only
   (Irrm_NEAREST, Irrm_NegINF, Irrm_PosINF, Irrm_ZERO).  Set the ARM64
   FSCR to have the same rounding.

   For speed & simplicity, we're setting the *entire* FPCR here.

   Setting the rounding mode is expensive.  So this function tries to
   avoid repeatedly setting the rounding mode to the same thing by
   first comparing 'mode' to the 'mode' tree supplied in the previous
   call to this function, if any.  (The previous value is stored in
   env->previous_rm.)  If 'mode' is a single IR temporary 't' and
   env->previous_rm is also just 't', then the setting is skipped.

   This is safe because of the SSA property of IR: an IR temporary can
   only be defined once and so will have the same value regardless of
   where it appears in the block.  Cool stuff, SSA.

   A safety condition: all attempts to set the RM must be aware of
   this mechanism - by being routed through the functions here.

   Of course this only helps if blocks where the RM is set more than
   once and it is set to the same value each time, *and* that value is
   held in the same IR temporary each time.  In order to assure the
   latter as much as possible, the IR optimiser takes care to do CSE
   on any block with any sign of floating point activity.
*/
static
void set_FPCR_rounding_mode ( ISelEnv* env, IRExpr* mode )
{
   vassert(typeOfIRExpr(env->type_env,mode) == Ity_I32);
   
   /* Do we need to do anything? */
   if (env->previous_rm
       && env->previous_rm->tag == Iex_RdTmp
       && mode->tag == Iex_RdTmp
       && env->previous_rm->Iex.RdTmp.tmp == mode->Iex.RdTmp.tmp) {
      /* no - setting it to what it was before.  */
      vassert(typeOfIRExpr(env->type_env, env->previous_rm) == Ity_I32);
      return;
   }

   /* No luck - we better set it, and remember what we set it to. */
   env->previous_rm = mode;

   /* Only supporting the rounding-mode bits - the rest of FPCR is set
      to zero - so we can set the whole register at once (faster). */

   /* This isn't simple, because 'mode' carries an IR rounding
      encoding, and we need to translate that to an ARM64 FP one:
      The IR encoding:
         00  to nearest (the default)
         10  to +infinity
         01  to -infinity
         11  to zero
      The ARM64 FP encoding:
         00  to nearest
         01  to +infinity
         10  to -infinity
         11  to zero
      Easy enough to do; just swap the two bits.
   */
   HReg irrm = iselIntExpr_R(env, mode);
   HReg tL   = newVRegI(env);
   HReg tR   = newVRegI(env);
   HReg t3   = newVRegI(env);
   /* tL = irrm << 1;
      tR = irrm >> 1;  if we're lucky, these will issue together
      tL &= 2;
      tR &= 1;         ditto
      t3 = tL | tR;
      t3 <<= 22;
      fmxr fpscr, t3
   */
   ARM64RIL* ril_one = mb_mkARM64RIL_I(1);
   ARM64RIL* ril_two = mb_mkARM64RIL_I(2);
   vassert(ril_one && ril_two);
   addInstr(env, ARM64Instr_Shift(tL, irrm, ARM64RI6_I6(1), ARM64sh_SHL));
   addInstr(env, ARM64Instr_Shift(tR, irrm, ARM64RI6_I6(1), ARM64sh_SHR));
   addInstr(env, ARM64Instr_Logic(tL, tL, ril_two, ARM64lo_AND));
   addInstr(env, ARM64Instr_Logic(tR, tR, ril_one, ARM64lo_AND));
   addInstr(env, ARM64Instr_Logic(t3, tL, ARM64RIL_R(tR), ARM64lo_OR));
   addInstr(env, ARM64Instr_Shift(t3, t3, ARM64RI6_I6(22), ARM64sh_SHL));
   addInstr(env, ARM64Instr_FPCR(True/*toFPCR*/, t3));
}


/*---------------------------------------------------------*/
/*--- ISEL: Function call helpers                       ---*/
/*---------------------------------------------------------*/

/* Used only in doHelperCall.  See big comment in doHelperCall re
   handling of register-parameter args.  This function figures out
   whether evaluation of an expression might require use of a fixed
   register.  If in doubt return True (safe but suboptimal).
*/
static
Bool mightRequireFixedRegs ( IRExpr* e )
{
   if (UNLIKELY(is_IRExpr_VECRET_or_GSPTR(e))) {
      // These are always "safe" -- either a copy of SP in some
      // arbitrary vreg, or a copy of x21, respectively.
      return False;
   }
   /* Else it's a "normal" expression. */
   switch (e->tag) {
      case Iex_RdTmp: case Iex_Const: case Iex_Get:
         return False;
      default:
         return True;
   }
}
 

/* Do a complete function call.  |guard| is a Ity_Bit expression
   indicating whether or not the call happens.  If guard==NULL, the
   call is unconditional.  |retloc| is set to indicate where the
   return value is after the call.  The caller (of this fn) must
   generate code to add |stackAdjustAfterCall| to the stack pointer
   after the call is done.  Returns True iff it managed to handle this
   combination of arg/return types, else returns False. */

static
Bool doHelperCall ( /*OUT*/UInt*   stackAdjustAfterCall,
                    /*OUT*/RetLoc* retloc,
                    ISelEnv* env,
                    IRExpr* guard,
                    IRCallee* cee, IRType retTy, IRExpr** args )
{
   ARM64CondCode cc;
   HReg          argregs[ARM64_N_ARGREGS];
   HReg          tmpregs[ARM64_N_ARGREGS];
   Bool          go_fast;
   Int           n_args, i, nextArgReg;
   Addr64        target;

   vassert(ARM64_N_ARGREGS == 8);

   /* Set default returns.  We'll update them later if needed. */
   *stackAdjustAfterCall = 0;
   *retloc               = mk_RetLoc_INVALID();

   /* These are used for cross-checking that IR-level constraints on
      the use of IRExpr_VECRET() and IRExpr_GSPTR() are observed. */
   UInt nVECRETs = 0;
   UInt nGSPTRs  = 0;

   /* Marshal args for a call and do the call.

      This function only deals with a tiny set of possibilities, which
      cover all helpers in practice.  The restrictions are that only
      arguments in registers are supported, hence only
      ARM64_N_REGPARMS x 64 integer bits in total can be passed.  In
      fact the only supported arg type is I64.

      The return type can be I{64,32} or V128.  In the V128 case, it
      is expected that |args| will contain the special node
      IRExpr_VECRET(), in which case this routine generates code to
      allocate space on the stack for the vector return value.  Since
      we are not passing any scalars on the stack, it is enough to
      preallocate the return space before marshalling any arguments,
      in this case.

      |args| may also contain IRExpr_GSPTR(), in which case the
      value in x21 is passed as the corresponding argument.

      Generating code which is both efficient and correct when
      parameters are to be passed in registers is difficult, for the
      reasons elaborated in detail in comments attached to
      doHelperCall() in priv/host-x86/isel.c.  Here, we use a variant
      of the method described in those comments.

      The problem is split into two cases: the fast scheme and the
      slow scheme.  In the fast scheme, arguments are computed
      directly into the target (real) registers.  This is only safe
      when we can be sure that computation of each argument will not
      trash any real registers set by computation of any other
      argument.

      In the slow scheme, all args are first computed into vregs, and
      once they are all done, they are moved to the relevant real
      regs.  This always gives correct code, but it also gives a bunch
      of vreg-to-rreg moves which are usually redundant but are hard
      for the register allocator to get rid of.

      To decide which scheme to use, all argument expressions are
      first examined.  If they are all so simple that it is clear they
      will be evaluated without use of any fixed registers, use the
      fast scheme, else use the slow scheme.  Note also that only
      unconditional calls may use the fast scheme, since having to
      compute a condition expression could itself trash real
      registers.

      Note this requires being able to examine an expression and
      determine whether or not evaluation of it might use a fixed
      register.  That requires knowledge of how the rest of this insn
      selector works.  Currently just the following 3 are regarded as
      safe -- hopefully they cover the majority of arguments in
      practice: IRExpr_Tmp IRExpr_Const IRExpr_Get.
   */

   /* Note that the cee->regparms field is meaningless on ARM64 hosts
      (since there is only one calling convention) and so we always
      ignore it. */

   n_args = 0;
   for (i = 0; args[i]; i++) {
      IRExpr* arg = args[i];
      if (UNLIKELY(arg->tag == Iex_VECRET)) {
         nVECRETs++;
      } else if (UNLIKELY(arg->tag == Iex_GSPTR)) {
         nGSPTRs++;
      }
      n_args++;
   }

   /* If this fails, the IR is ill-formed */
   vassert(nGSPTRs == 0 || nGSPTRs == 1);

   /* If we have a VECRET, allocate space on the stack for the return
      value, and record the stack pointer after that. */
   HReg r_vecRetAddr = INVALID_HREG;
   if (nVECRETs == 1) {
      vassert(retTy == Ity_V128 || retTy == Ity_V256);
      vassert(retTy != Ity_V256); // we don't handle that yet (if ever)
      r_vecRetAddr = newVRegI(env);
      addInstr(env, ARM64Instr_AddToSP(-16));
      addInstr(env, ARM64Instr_FromSP(r_vecRetAddr));
   } else {
      // If either of these fail, the IR is ill-formed
      vassert(retTy != Ity_V128 && retTy != Ity_V256);
      vassert(nVECRETs == 0);
   }

   argregs[0] = hregARM64_X0();
   argregs[1] = hregARM64_X1();
   argregs[2] = hregARM64_X2();
   argregs[3] = hregARM64_X3();
   argregs[4] = hregARM64_X4();
   argregs[5] = hregARM64_X5();
   argregs[6] = hregARM64_X6();
   argregs[7] = hregARM64_X7();

   tmpregs[0] = tmpregs[1] = tmpregs[2] = tmpregs[3] = INVALID_HREG;
   tmpregs[4] = tmpregs[5] = tmpregs[6] = tmpregs[7] = INVALID_HREG;

   /* First decide which scheme (slow or fast) is to be used.  First
      assume the fast scheme, and select slow if any contraindications
      (wow) appear. */

   go_fast = True;

   if (guard) {
      if (guard->tag == Iex_Const
          && guard->Iex.Const.con->tag == Ico_U1
          && guard->Iex.Const.con->Ico.U1 == True) {
         /* unconditional */
      } else {
         /* Not manifestly unconditional -- be conservative. */
         go_fast = False;
      }
   }

   if (go_fast) {
      for (i = 0; i < n_args; i++) {
         if (mightRequireFixedRegs(args[i])) {
            go_fast = False;
            break;
         }
      }
   }

   if (go_fast) {
      if (retTy == Ity_V128 || retTy == Ity_V256)
         go_fast = False;
   }

   /* At this point the scheme to use has been established.  Generate
      code to get the arg values into the argument rregs.  If we run
      out of arg regs, give up. */

   if (go_fast) {

      /* FAST SCHEME */
      nextArgReg = 0;

      for (i = 0; i < n_args; i++) {
         IRExpr* arg = args[i];

         IRType  aTy = Ity_INVALID;
         if (LIKELY(!is_IRExpr_VECRET_or_GSPTR(arg)))
            aTy = typeOfIRExpr(env->type_env, args[i]);

         if (nextArgReg >= ARM64_N_ARGREGS)
            return False; /* out of argregs */

         if (aTy == Ity_I64) {
            addInstr(env, ARM64Instr_MovI( argregs[nextArgReg],
                                           iselIntExpr_R(env, args[i]) ));
            nextArgReg++;
         }
         else if (arg->tag == Iex_GSPTR) {
            vassert(0); //ATC
            addInstr(env, ARM64Instr_MovI( argregs[nextArgReg],
                                           hregARM64_X21() ));
            nextArgReg++;
         }
         else if (arg->tag == Iex_VECRET) {
            // because of the go_fast logic above, we can't get here,
            // since vector return values makes us use the slow path
            // instead.
            vassert(0);
         }
         else
            return False; /* unhandled arg type */
      }

      /* Fast scheme only applies for unconditional calls.  Hence: */
      cc = ARM64cc_AL;

   } else {

      /* SLOW SCHEME; move via temporaries */
      nextArgReg = 0;

      for (i = 0; i < n_args; i++) {
         IRExpr* arg = args[i];

         IRType  aTy = Ity_INVALID;
         if (LIKELY(!is_IRExpr_VECRET_or_GSPTR(arg)))
            aTy = typeOfIRExpr(env->type_env, args[i]);

         if (nextArgReg >= ARM64_N_ARGREGS)
            return False; /* out of argregs */

         if (aTy == Ity_I64) {
            tmpregs[nextArgReg] = iselIntExpr_R(env, args[i]);
            nextArgReg++;
         }
         else if (arg->tag == Iex_GSPTR) {
            vassert(0); //ATC
            tmpregs[nextArgReg] = hregARM64_X21();
            nextArgReg++;
         }
         else if (arg->tag == Iex_VECRET) {
            vassert(!hregIsInvalid(r_vecRetAddr));
            tmpregs[nextArgReg] = r_vecRetAddr;
            nextArgReg++;
         }
         else
            return False; /* unhandled arg type */
      }

      /* Now we can compute the condition.  We can't do it earlier
         because the argument computations could trash the condition
         codes.  Be a bit clever to handle the common case where the
         guard is 1:Bit. */
      cc = ARM64cc_AL;
      if (guard) {
         if (guard->tag == Iex_Const
             && guard->Iex.Const.con->tag == Ico_U1
             && guard->Iex.Const.con->Ico.U1 == True) {
            /* unconditional -- do nothing */
         } else {
            cc = iselCondCode( env, guard );
         }
      }

      /* Move the args to their final destinations. */
      for (i = 0; i < nextArgReg; i++) {
         vassert(!(hregIsInvalid(tmpregs[i])));
         /* None of these insns, including any spill code that might
            be generated, may alter the condition codes. */
         addInstr( env, ARM64Instr_MovI( argregs[i], tmpregs[i] ) );
      }

   }

   /* Should be assured by checks above */
   vassert(nextArgReg <= ARM64_N_ARGREGS);

   /* Do final checks, set the return values, and generate the call
      instruction proper. */
   vassert(nGSPTRs == 0 || nGSPTRs == 1);
   vassert(nVECRETs == ((retTy == Ity_V128 || retTy == Ity_V256) ? 1 : 0));
   vassert(*stackAdjustAfterCall == 0);
   vassert(is_RetLoc_INVALID(*retloc));
   switch (retTy) {
      case Ity_INVALID:
         /* Function doesn't return a value. */
         *retloc = mk_RetLoc_simple(RLPri_None);
         break;
      case Ity_I64: case Ity_I32: case Ity_I16: case Ity_I8:
         *retloc = mk_RetLoc_simple(RLPri_Int);
         break;
      case Ity_V128:
         *retloc = mk_RetLoc_spRel(RLPri_V128SpRel, 0);
         *stackAdjustAfterCall = 16;
         break;
      case Ity_V256:
         vassert(0); // ATC
         *retloc = mk_RetLoc_spRel(RLPri_V256SpRel, 0);
         *stackAdjustAfterCall = 32;
         break;
      default:
         /* IR can denote other possible return types, but we don't
            handle those here. */
         vassert(0);
   }

   /* Finally, generate the call itself.  This needs the *retloc value
      set in the switch above, which is why it's at the end. */

   /* nextArgReg doles out argument registers.  Since these are
      assigned in the order x0 .. x7, its numeric value at this point,
      which must be between 0 and 8 inclusive, is going to be equal to
      the number of arg regs in use for the call.  Hence bake that
      number into the call (we'll need to know it when doing register
      allocation, to know what regs the call reads.) */

   target = (Addr)cee->addr;
   addInstr(env, ARM64Instr_Call( cc, target, nextArgReg, *retloc ));

   return True; /* success */
}


/*---------------------------------------------------------*/
/*--- ISEL: Integer expressions (64/32 bit)             ---*/
/*---------------------------------------------------------*/

/* Select insns for an integer-typed expression, and add them to the
   code list.  Return a reg holding the result.  This reg will be a
   virtual register.  THE RETURNED REG MUST NOT BE MODIFIED.  If you
   want to modify it, ask for a new vreg, copy it in there, and modify
   the copy.  The register allocator will do its best to map both
   vregs to the same real register, so the copies will often disappear
   later in the game.

   This should handle expressions of 64- and 32-bit type.  All results
   are returned in a 64-bit register.  For 32-bit expressions, the
   upper 32 bits are arbitrary, so you should mask or sign extend
   partial values if necessary.
*/

/* --------------------- AMode --------------------- */

/* Return an AMode which computes the value of the specified
   expression, possibly also adding insns to the code list as a
   result.  The expression may only be a 64-bit one.
*/

static Bool isValidScale ( UChar scale )
{
   switch (scale) {
      case 1: case 2: case 4: case 8: /* case 16: ??*/ return True;
      default: return False;
   }
}

static Bool sane_AMode ( ARM64AMode* am )
{
   switch (am->tag) {
      case ARM64am_RI9:
         return
            toBool( hregClass(am->ARM64am.RI9.reg) == HRcInt64
                    && (hregIsVirtual(am->ARM64am.RI9.reg)
                        /* || sameHReg(am->ARM64am.RI9.reg, 
                                       hregARM64_X21()) */ )
                    && am->ARM64am.RI9.simm9 >= -256
                    && am->ARM64am.RI9.simm9 <= 255 );
      case ARM64am_RI12:
         return
            toBool( hregClass(am->ARM64am.RI12.reg) == HRcInt64
                    && (hregIsVirtual(am->ARM64am.RI12.reg)
                        /* || sameHReg(am->ARM64am.RI12.reg, 
                                       hregARM64_X21()) */ )
                    && am->ARM64am.RI12.uimm12 < 4096
                    && isValidScale(am->ARM64am.RI12.szB) );
      case ARM64am_RR:
         return
            toBool( hregClass(am->ARM64am.RR.base) == HRcInt64
                    && hregIsVirtual(am->ARM64am.RR.base)
                    && hregClass(am->ARM64am.RR.index) == HRcInt64
                    && hregIsVirtual(am->ARM64am.RR.index) );
      default:
         vpanic("sane_AMode: unknown ARM64 AMode1 tag");
   }
}

static
ARM64AMode* iselIntExpr_AMode ( ISelEnv* env, IRExpr* e, IRType dty )
{
   ARM64AMode* am = iselIntExpr_AMode_wrk(env, e, dty);
   vassert(sane_AMode(am));
   return am;
}

static
ARM64AMode* iselIntExpr_AMode_wrk ( ISelEnv* env, IRExpr* e, IRType dty )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_I64);

   ULong szBbits = 0;
   switch (dty) {
      case Ity_I64: szBbits = 3; break;
      case Ity_I32: szBbits = 2; break;
      case Ity_I16: szBbits = 1; break;
      case Ity_I8:  szBbits = 0; break;
      default: vassert(0);
   }

   /* {Add64,Sub64}(expr,simm9).  We don't care about |dty| here since
      we're going to create an amode suitable for LDU* or STU*
      instructions, which use unscaled immediate offsets.  */
   if (e->tag == Iex_Binop
       && (e->Iex.Binop.op == Iop_Add64 || e->Iex.Binop.op == Iop_Sub64)
       && e->Iex.Binop.arg2->tag == Iex_Const
       && e->Iex.Binop.arg2->Iex.Const.con->tag == Ico_U64) {
      Long simm = (Long)e->Iex.Binop.arg2->Iex.Const.con->Ico.U64;
      if (simm >= -255 && simm <= 255) {
         /* Although the gating condition might seem to be 
               simm >= -256 && simm <= 255
            we will need to negate simm in the case where the op is Sub64.
            Hence limit the lower value to -255 in order that its negation
            is representable. */
         HReg reg = iselIntExpr_R(env, e->Iex.Binop.arg1);
         if (e->Iex.Binop.op == Iop_Sub64) simm = -simm;
         return ARM64AMode_RI9(reg, (Int)simm);
      }
   }

   /* Add64(expr, uimm12 * transfer-size) */
   if (e->tag == Iex_Binop
       && e->Iex.Binop.op == Iop_Add64
       && e->Iex.Binop.arg2->tag == Iex_Const
       && e->Iex.Binop.arg2->Iex.Const.con->tag == Ico_U64) {
      ULong uimm = e->Iex.Binop.arg2->Iex.Const.con->Ico.U64;
      ULong szB  = 1 << szBbits;
      if (0 == (uimm & (szB-1)) /* "uimm is szB-aligned" */
          && (uimm >> szBbits) < 4096) {
         HReg reg = iselIntExpr_R(env, e->Iex.Binop.arg1);
         return ARM64AMode_RI12(reg, (UInt)(uimm >> szBbits), (UChar)szB);
      }
   }

   /* Add64(expr1, expr2) */
   if (e->tag == Iex_Binop
       && e->Iex.Binop.op == Iop_Add64) {
      HReg reg1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
      HReg reg2 = iselIntExpr_R(env, e->Iex.Binop.arg2);
      return ARM64AMode_RR(reg1, reg2);
   }

   /* Doesn't match anything in particular.  Generate it into
      a register and use that. */
   HReg reg = iselIntExpr_R(env, e);
   return ARM64AMode_RI9(reg, 0);
}


/* --------------------- RIA --------------------- */

/* Select instructions to generate 'e' into a RIA. */

static ARM64RIA* iselIntExpr_RIA ( ISelEnv* env, IRExpr* e )
{
   ARM64RIA* ri = iselIntExpr_RIA_wrk(env, e);
   /* sanity checks ... */
   switch (ri->tag) {
      case ARM64riA_I12:
         vassert(ri->ARM64riA.I12.imm12 < 4096);
         vassert(ri->ARM64riA.I12.shift == 0 || ri->ARM64riA.I12.shift == 12);
         return ri;
      case ARM64riA_R:
         vassert(hregClass(ri->ARM64riA.R.reg) == HRcInt64);
         vassert(hregIsVirtual(ri->ARM64riA.R.reg));
         return ri;
      default:
         vpanic("iselIntExpr_RIA: unknown arm RIA tag");
   }
}

/* DO NOT CALL THIS DIRECTLY ! */
static ARM64RIA* iselIntExpr_RIA_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_I64 || ty == Ity_I32);

   /* special case: immediate */
   if (e->tag == Iex_Const) {
      ULong u = 0xF000000ULL; /* invalid */
      switch (e->Iex.Const.con->tag) {
         case Ico_U64: u = e->Iex.Const.con->Ico.U64; break;
         case Ico_U32: u = e->Iex.Const.con->Ico.U32; break;
         default: vpanic("iselIntExpr_RIA.Iex_Const(arm64)");
      }
      if (0 == (u & ~(0xFFFULL << 0)))
         return ARM64RIA_I12((UShort)((u >> 0) & 0xFFFULL), 0);
      if (0 == (u & ~(0xFFFULL << 12)))
         return ARM64RIA_I12((UShort)((u >> 12) & 0xFFFULL), 12);
      /* else fail, fall through to default case */
   }

   /* default case: calculate into a register and return that */
   {
      HReg r = iselIntExpr_R ( env, e );
      return ARM64RIA_R(r);
   }
}


/* --------------------- RIL --------------------- */

/* Select instructions to generate 'e' into a RIL.  At this point we
   have to deal with the strange bitfield-immediate encoding for logic
   instructions. */


// The following four functions
//    CountLeadingZeros CountTrailingZeros CountSetBits isImmLogical
// are copied, with modifications, from
// https://github.com/armvixl/vixl/blob/master/src/a64/assembler-a64.cc
// which has the following copyright notice:
/*
   Copyright 2013, ARM Limited
   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:
   
   * Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
   * Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.
   * Neither the name of ARM Limited nor the names of its contributors may be
     used to endorse or promote products derived from this software without
     specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
   ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

static Int CountLeadingZeros(ULong value, Int width)
{
   vassert(width == 32 || width == 64);
   Int count = 0;
   ULong bit_test = 1ULL << (width - 1);
   while ((count < width) && ((bit_test & value) == 0)) {
      count++;
      bit_test >>= 1;
   }
   return count;
}

static Int CountTrailingZeros(ULong value, Int width)
{
   vassert(width == 32 || width == 64);
   Int count = 0;
   while ((count < width) && (((value >> count) & 1) == 0)) {
      count++;
   }
   return count;
}

static Int CountSetBits(ULong value, Int width)
{
   // TODO: Other widths could be added here, as the implementation already
   // supports them.
   vassert(width == 32 || width == 64);

   // Mask out unused bits to ensure that they are not counted.
   value &= (0xffffffffffffffffULL >> (64-width));

   // Add up the set bits.
   // The algorithm works by adding pairs of bit fields together iteratively,
   // where the size of each bit field doubles each time.
   // An example for an 8-bit value:
   // Bits: h g f e d c b a
   // \ | \ | \ | \ |
   // value = h+g f+e d+c b+a
   // \ | \ |
   // value = h+g+f+e d+c+b+a
   // \ |
   // value = h+g+f+e+d+c+b+a
   value = ((value >>  1) & 0x5555555555555555ULL)
                 + (value & 0x5555555555555555ULL);
   value = ((value >>  2) & 0x3333333333333333ULL)
                 + (value & 0x3333333333333333ULL);
   value = ((value >>  4) & 0x0f0f0f0f0f0f0f0fULL)
                 + (value & 0x0f0f0f0f0f0f0f0fULL);
   value = ((value >>  8) & 0x00ff00ff00ff00ffULL)
                 + (value & 0x00ff00ff00ff00ffULL);
   value = ((value >> 16) & 0x0000ffff0000ffffULL)
                 + (value & 0x0000ffff0000ffffULL);
   value = ((value >> 32) & 0x00000000ffffffffULL)
                 + (value & 0x00000000ffffffffULL);

   return value;
}

static Bool isImmLogical ( /*OUT*/UInt* n,
                           /*OUT*/UInt* imm_s, /*OUT*/UInt* imm_r,
                           ULong value, UInt width )
{
  // Test if a given value can be encoded in the immediate field of a
  // logical instruction.

  // If it can be encoded, the function returns true, and values
  // pointed to by n, imm_s and imm_r are updated with immediates
  // encoded in the format required by the corresponding fields in the
  // logical instruction.  If it can not be encoded, the function
  // returns false, and the values pointed to by n, imm_s and imm_r
  // are undefined.
  vassert(n != NULL && imm_s != NULL && imm_r != NULL);
  vassert(width == 32 || width == 64);

  // Logical immediates are encoded using parameters n, imm_s and imm_r using
  // the following table:
  //
  // N imms immr size S R
  // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
  // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
  // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
  // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
  // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
  // 0 11110s xxxxxr 2 UInt(s) UInt(r)
  // (s bits must not be all set)
  //
  // A pattern is constructed of size bits, where the least significant S+1
  // bits are set. The pattern is rotated right by R, and repeated across a
  // 32 or 64-bit value, depending on destination register width.
  //
  // To test if an arbitrary immediate can be encoded using this scheme, an
  // iterative algorithm is used.
  //
  // TODO: This code does not consider using X/W register overlap to support
  // 64-bit immediates where the top 32-bits are zero, and the bottom 32-bits
  // are an encodable logical immediate.

  // 1. If the value has all set or all clear bits, it can't be encoded.
  if ((value == 0) || (value == 0xffffffffffffffffULL) ||
      ((width == 32) && (value == 0xffffffff))) {
    return False;
  }

  UInt lead_zero = CountLeadingZeros(value, width);
  UInt lead_one = CountLeadingZeros(~value, width);
  UInt trail_zero = CountTrailingZeros(value, width);
  UInt trail_one = CountTrailingZeros(~value, width);
  UInt set_bits = CountSetBits(value, width);

  // The fixed bits in the immediate s field.
  // If width == 64 (X reg), start at 0xFFFFFF80.
  // If width == 32 (W reg), start at 0xFFFFFFC0, as the iteration for 64-bit
  // widths won't be executed.
  Int imm_s_fixed = (width == 64) ? -128 : -64;
  Int imm_s_mask = 0x3F;

  for (;;) {
    // 2. If the value is two bits wide, it can be encoded.
    if (width == 2) {
      *n = 0;
      *imm_s = 0x3C;
      *imm_r = (value & 3) - 1;
      return True;
    }

    *n = (width == 64) ? 1 : 0;
    *imm_s = ((imm_s_fixed | (set_bits - 1)) & imm_s_mask);
    if ((lead_zero + set_bits) == width) {
      *imm_r = 0;
    } else {
      *imm_r = (lead_zero > 0) ? (width - trail_zero) : lead_one;
    }

    // 3. If the sum of leading zeros, trailing zeros and set bits is equal to
    // the bit width of the value, it can be encoded.
    if (lead_zero + trail_zero + set_bits == width) {
      return True;
    }

    // 4. If the sum of leading ones, trailing ones and unset bits in the
    // value is equal to the bit width of the value, it can be encoded.
    if (lead_one + trail_one + (width - set_bits) == width) {
      return True;
    }

    // 5. If the most-significant half of the bitwise value is equal to the
    // least-significant half, return to step 2 using the least-significant
    // half of the value.
    ULong mask = (1ULL << (width >> 1)) - 1;
    if ((value & mask) == ((value >> (width >> 1)) & mask)) {
      width >>= 1;
      set_bits >>= 1;
      imm_s_fixed >>= 1;
      continue;
    }

    // 6. Otherwise, the value can't be encoded.
    return False;
  }
}


/* Create a RIL for the given immediate, if it is representable, or
   return NULL if not. */

static ARM64RIL* mb_mkARM64RIL_I ( ULong imm64 )
{
   UInt n = 0, imm_s = 0, imm_r = 0;
   Bool ok = isImmLogical(&n, &imm_s, &imm_r, imm64, 64);
   if (!ok) return NULL;
   vassert(n < 2 && imm_s < 64 && imm_r < 64);
   return ARM64RIL_I13(n, imm_r, imm_s);
}

/* So, finally .. */

static ARM64RIL* iselIntExpr_RIL ( ISelEnv* env, IRExpr* e )
{
   ARM64RIL* ri = iselIntExpr_RIL_wrk(env, e);
   /* sanity checks ... */
   switch (ri->tag) {
      case ARM64riL_I13:
         vassert(ri->ARM64riL.I13.bitN < 2);
         vassert(ri->ARM64riL.I13.immR < 64);
         vassert(ri->ARM64riL.I13.immS < 64);
         return ri;
      case ARM64riL_R:
         vassert(hregClass(ri->ARM64riL.R.reg) == HRcInt64);
         vassert(hregIsVirtual(ri->ARM64riL.R.reg));
         return ri;
      default:
         vpanic("iselIntExpr_RIL: unknown arm RIL tag");
   }
}

/* DO NOT CALL THIS DIRECTLY ! */
static ARM64RIL* iselIntExpr_RIL_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_I64 || ty == Ity_I32);
   
   /* special case: immediate */
   if (e->tag == Iex_Const) {
      ARM64RIL* maybe = NULL;
      if (ty == Ity_I64) {
         vassert(e->Iex.Const.con->tag == Ico_U64);
         maybe = mb_mkARM64RIL_I(e->Iex.Const.con->Ico.U64);
      } else {
         vassert(ty == Ity_I32);
         vassert(e->Iex.Const.con->tag == Ico_U32);
         UInt  u32 = e->Iex.Const.con->Ico.U32;
         ULong u64 = (ULong)u32;
         /* First try with 32 leading zeroes. */
         maybe = mb_mkARM64RIL_I(u64);
         /* If that doesn't work, try with 2 copies, since it doesn't
            matter what winds up in the upper 32 bits. */
         if (!maybe) {
            maybe = mb_mkARM64RIL_I((u64 << 32) | u64);
         }
      }
      if (maybe) return maybe;
      /* else fail, fall through to default case */
   }

   /* default case: calculate into a register and return that */
   {
      HReg r = iselIntExpr_R ( env, e );
      return ARM64RIL_R(r);
   }
}


/* --------------------- RI6 --------------------- */

/* Select instructions to generate 'e' into a RI6. */

static ARM64RI6* iselIntExpr_RI6 ( ISelEnv* env, IRExpr* e )
{
   ARM64RI6* ri = iselIntExpr_RI6_wrk(env, e);
   /* sanity checks ... */
   switch (ri->tag) {
      case ARM64ri6_I6:
         vassert(ri->ARM64ri6.I6.imm6 < 64);
         vassert(ri->ARM64ri6.I6.imm6 > 0);
         return ri;
      case ARM64ri6_R:
         vassert(hregClass(ri->ARM64ri6.R.reg) == HRcInt64);
         vassert(hregIsVirtual(ri->ARM64ri6.R.reg));
         return ri;
      default:
         vpanic("iselIntExpr_RI6: unknown arm RI6 tag");
   }
}

/* DO NOT CALL THIS DIRECTLY ! */
static ARM64RI6* iselIntExpr_RI6_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_I64 || ty == Ity_I8);

   /* special case: immediate */
   if (e->tag == Iex_Const) {
      switch (e->Iex.Const.con->tag) {
         case Ico_U8: {
            UInt u = e->Iex.Const.con->Ico.U8;
            if (u > 0 && u < 64)
              return ARM64RI6_I6(u);
            break;
         default:
            break;
         }
      }
      /* else fail, fall through to default case */
   }

   /* default case: calculate into a register and return that */
   {
      HReg r = iselIntExpr_R ( env, e );
      return ARM64RI6_R(r);
   }
}


/* ------------------- CondCode ------------------- */

/* Generate code to evaluated a bit-typed expression, returning the
   condition code which would correspond when the expression would
   notionally have returned 1. */

static ARM64CondCode iselCondCode ( ISelEnv* env, IRExpr* e )
{
   ARM64CondCode cc = iselCondCode_wrk(env,e);
   vassert(cc != ARM64cc_NV);
   return cc;
}

static ARM64CondCode iselCondCode_wrk ( ISelEnv* env, IRExpr* e )
{
   vassert(e);
   vassert(typeOfIRExpr(env->type_env,e) == Ity_I1);

   /* var */
   if (e->tag == Iex_RdTmp) {
      HReg rTmp = lookupIRTemp(env, e->Iex.RdTmp.tmp);
      /* Cmp doesn't modify rTmp; so this is OK. */
      ARM64RIL* one = mb_mkARM64RIL_I(1);
      vassert(one);
      addInstr(env, ARM64Instr_Test(rTmp, one));
      return ARM64cc_NE;
   }

   /* Not1(e) */
   if (e->tag == Iex_Unop && e->Iex.Unop.op == Iop_Not1) {
      /* Generate code for the arg, and negate the test condition */
      ARM64CondCode cc = iselCondCode(env, e->Iex.Unop.arg);
      if (cc == ARM64cc_AL || cc == ARM64cc_NV) {
        return ARM64cc_AL;
      } else {
        return 1 ^ cc;
      }
   }

   /* --- patterns rooted at: 64to1 --- */

   if (e->tag == Iex_Unop
       && e->Iex.Unop.op == Iop_64to1) {
      HReg      rTmp = iselIntExpr_R(env, e->Iex.Unop.arg);
      ARM64RIL* one  = mb_mkARM64RIL_I(1);
      vassert(one); /* '1' must be representable */
      addInstr(env, ARM64Instr_Test(rTmp, one));
      return ARM64cc_NE;
   }

   /* --- patterns rooted at: CmpNEZ8 --- */

   if (e->tag == Iex_Unop
       && e->Iex.Unop.op == Iop_CmpNEZ8) {
      HReg      r1  = iselIntExpr_R(env, e->Iex.Unop.arg);
      ARM64RIL* xFF = mb_mkARM64RIL_I(0xFF);
      addInstr(env, ARM64Instr_Test(r1, xFF));
      return ARM64cc_NE;
   }

   /* --- patterns rooted at: CmpNEZ16 --- */

   if (e->tag == Iex_Unop
       && e->Iex.Unop.op == Iop_CmpNEZ16) {
      HReg      r1    = iselIntExpr_R(env, e->Iex.Unop.arg);
      ARM64RIL* xFFFF = mb_mkARM64RIL_I(0xFFFF);
      addInstr(env, ARM64Instr_Test(r1, xFFFF));
      return ARM64cc_NE;
   }

   /* --- patterns rooted at: CmpNEZ64 --- */

   if (e->tag == Iex_Unop
       && e->Iex.Unop.op == Iop_CmpNEZ64) {
      HReg      r1   = iselIntExpr_R(env, e->Iex.Unop.arg);
      ARM64RIA* zero = ARM64RIA_I12(0,0);
      addInstr(env, ARM64Instr_Cmp(r1, zero, True/*is64*/));
      return ARM64cc_NE;
   }

   /* --- patterns rooted at: CmpNEZ32 --- */

   if (e->tag == Iex_Unop
       && e->Iex.Unop.op == Iop_CmpNEZ32) {
      HReg      r1   = iselIntExpr_R(env, e->Iex.Unop.arg);
      ARM64RIA* zero = ARM64RIA_I12(0,0);
      addInstr(env, ARM64Instr_Cmp(r1, zero, False/*!is64*/));
      return ARM64cc_NE;
   }

   /* --- Cmp*64*(x,y) --- */
   if (e->tag == Iex_Binop
       && (e->Iex.Binop.op == Iop_CmpEQ64
           || e->Iex.Binop.op == Iop_CmpNE64
           || e->Iex.Binop.op == Iop_CmpLT64S
           || e->Iex.Binop.op == Iop_CmpLT64U
           || e->Iex.Binop.op == Iop_CmpLE64S
           || e->Iex.Binop.op == Iop_CmpLE64U
           || e->Iex.Binop.op == Iop_CasCmpEQ64)) {
      HReg      argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
      ARM64RIA* argR = iselIntExpr_RIA(env, e->Iex.Binop.arg2);
      addInstr(env, ARM64Instr_Cmp(argL, argR, True/*is64*/));
      switch (e->Iex.Binop.op) {
         case Iop_CmpEQ64: case Iop_CasCmpEQ64: return ARM64cc_EQ;
         case Iop_CmpNE64:  return ARM64cc_NE;
         case Iop_CmpLT64S: return ARM64cc_LT;
         case Iop_CmpLT64U: return ARM64cc_CC;
         case Iop_CmpLE64S: return ARM64cc_LE;
         case Iop_CmpLE64U: return ARM64cc_LS;
         default: vpanic("iselCondCode(arm64): CmpXX64");
      }
   }

   /* --- Cmp*32*(x,y) --- */
   if (e->tag == Iex_Binop
       && (e->Iex.Binop.op == Iop_CmpEQ32
           || e->Iex.Binop.op == Iop_CmpNE32
           || e->Iex.Binop.op == Iop_CmpLT32S
           || e->Iex.Binop.op == Iop_CmpLT32U
           || e->Iex.Binop.op == Iop_CmpLE32S
           || e->Iex.Binop.op == Iop_CmpLE32U
           || e->Iex.Binop.op == Iop_CasCmpEQ32)) {
      HReg      argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
      ARM64RIA* argR = iselIntExpr_RIA(env, e->Iex.Binop.arg2);
      addInstr(env, ARM64Instr_Cmp(argL, argR, False/*!is64*/));
      switch (e->Iex.Binop.op) {
         case Iop_CmpEQ32: case Iop_CasCmpEQ32: return ARM64cc_EQ;
         case Iop_CmpNE32:  return ARM64cc_NE;
         case Iop_CmpLT32S: return ARM64cc_LT;
         case Iop_CmpLT32U: return ARM64cc_CC;
         case Iop_CmpLE32S: return ARM64cc_LE;
         case Iop_CmpLE32U: return ARM64cc_LS;
         default: vpanic("iselCondCode(arm64): CmpXX32");
      }
   }

   /* --- Cmp*16*(x,y) --- */
   if (e->tag == Iex_Binop
       && (e->Iex.Binop.op == Iop_CasCmpEQ16)) {
      HReg argL  = iselIntExpr_R(env, e->Iex.Binop.arg1);
      HReg argR  = iselIntExpr_R(env, e->Iex.Binop.arg2);
      HReg argL2 = widen_z_16_to_64(env, argL);
      HReg argR2 = widen_z_16_to_64(env, argR);
      addInstr(env, ARM64Instr_Cmp(argL2, ARM64RIA_R(argR2), True/*is64*/));
      switch (e->Iex.Binop.op) {
         case Iop_CasCmpEQ16: return ARM64cc_EQ;
         default: vpanic("iselCondCode(arm64): CmpXX16");
      }
   }

   /* --- Cmp*8*(x,y) --- */
   if (e->tag == Iex_Binop
       && (e->Iex.Binop.op == Iop_CasCmpEQ8)) {
      HReg argL  = iselIntExpr_R(env, e->Iex.Binop.arg1);
      HReg argR  = iselIntExpr_R(env, e->Iex.Binop.arg2);
      HReg argL2 = widen_z_8_to_64(env, argL);
      HReg argR2 = widen_z_8_to_64(env, argR);
      addInstr(env, ARM64Instr_Cmp(argL2, ARM64RIA_R(argR2), True/*is64*/));
      switch (e->Iex.Binop.op) {
         case Iop_CasCmpEQ8: return ARM64cc_EQ;
         default: vpanic("iselCondCode(arm64): CmpXX8");
      }
   }

   ppIRExpr(e);
   vpanic("iselCondCode");
}


/* --------------------- Reg --------------------- */

static HReg iselIntExpr_R ( ISelEnv* env, IRExpr* e )
{
   HReg r = iselIntExpr_R_wrk(env, e);
   /* sanity checks ... */
#  if 0
   vex_printf("\n"); ppIRExpr(e); vex_printf("\n");
#  endif
   vassert(hregClass(r) == HRcInt64);
   vassert(hregIsVirtual(r));
   return r;
}

/* DO NOT CALL THIS DIRECTLY ! */
static HReg iselIntExpr_R_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_I64 || ty == Ity_I32 || ty == Ity_I16 || ty == Ity_I8);

   switch (e->tag) {

   /* --------- TEMP --------- */
   case Iex_RdTmp: {
      return lookupIRTemp(env, e->Iex.RdTmp.tmp);
   }

   /* --------- LOAD --------- */
   case Iex_Load: {
      HReg dst  = newVRegI(env);

      if (e->Iex.Load.end != Iend_LE)
         goto irreducible;

      if (ty == Ity_I64) {
         ARM64AMode* amode = iselIntExpr_AMode ( env, e->Iex.Load.addr, ty );
         addInstr(env, ARM64Instr_LdSt64(True/*isLoad*/, dst, amode));
         return dst;
      }
      if (ty == Ity_I32) {
         ARM64AMode* amode = iselIntExpr_AMode ( env, e->Iex.Load.addr, ty );
         addInstr(env, ARM64Instr_LdSt32(True/*isLoad*/, dst, amode));
         return dst;
      }
      if (ty == Ity_I16) {
         ARM64AMode* amode = iselIntExpr_AMode ( env, e->Iex.Load.addr, ty );
         addInstr(env, ARM64Instr_LdSt16(True/*isLoad*/, dst, amode));
         return dst;
      }
      if (ty == Ity_I8) {
         ARM64AMode* amode = iselIntExpr_AMode ( env, e->Iex.Load.addr, ty );
         addInstr(env, ARM64Instr_LdSt8(True/*isLoad*/, dst, amode));
         return dst;
      }
      break;
   }

   /* --------- BINARY OP --------- */
   case Iex_Binop: {

      ARM64LogicOp lop = 0; /* invalid */
      ARM64ShiftOp sop = 0; /* invalid */

      /* Special-case 0-x into a Neg instruction.  Not because it's
         particularly useful but more so as to give value flow using
         this instruction, so as to check its assembly correctness for
         implementation of Left32/Left64. */
      switch (e->Iex.Binop.op) {
         case Iop_Sub64:
            if (isZeroU64(e->Iex.Binop.arg1)) {
               HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
               HReg dst  = newVRegI(env);
               addInstr(env, ARM64Instr_Unary(dst, argR, ARM64un_NEG));
               return dst;
            }
            break;
         default:
            break;
      }

      /* ADD/SUB */
      switch (e->Iex.Binop.op) {
         case Iop_Add64: case Iop_Add32:
         case Iop_Sub64: case Iop_Sub32: {
            Bool      isAdd = e->Iex.Binop.op == Iop_Add64
                              || e->Iex.Binop.op == Iop_Add32;
            HReg      dst   = newVRegI(env);
            HReg      argL  = iselIntExpr_R(env, e->Iex.Binop.arg1);
            ARM64RIA* argR  = iselIntExpr_RIA(env, e->Iex.Binop.arg2);
            addInstr(env, ARM64Instr_Arith(dst, argL, argR, isAdd));
            return dst;
         }
         default:
            break;
      }

      /* AND/OR/XOR */
      switch (e->Iex.Binop.op) {
         case Iop_And64: case Iop_And32: lop = ARM64lo_AND; goto log_binop;
         case Iop_Or64:  case Iop_Or32:  lop = ARM64lo_OR;  goto log_binop;
         case Iop_Xor64: case Iop_Xor32: lop = ARM64lo_XOR; goto log_binop;
         log_binop: {
            HReg      dst  = newVRegI(env);
            HReg      argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
            ARM64RIL* argR = iselIntExpr_RIL(env, e->Iex.Binop.arg2);
            addInstr(env, ARM64Instr_Logic(dst, argL, argR, lop));
            return dst;
         }
         default:
            break;
      }

      /* SHL/SHR/SAR */
      switch (e->Iex.Binop.op) {
         case Iop_Shr64:                 sop = ARM64sh_SHR; goto sh_binop;
         case Iop_Sar64:                 sop = ARM64sh_SAR; goto sh_binop;
         case Iop_Shl64: case Iop_Shl32: sop = ARM64sh_SHL; goto sh_binop;
         sh_binop: {
            HReg      dst  = newVRegI(env);
            HReg      argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
            ARM64RI6* argR = iselIntExpr_RI6(env, e->Iex.Binop.arg2);
            addInstr(env, ARM64Instr_Shift(dst, argL, argR, sop));
            return dst;
         }
         case Iop_Shr32:
         case Iop_Sar32: {
            Bool      zx   = e->Iex.Binop.op == Iop_Shr32;
            HReg      argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
            ARM64RI6* argR = iselIntExpr_RI6(env, e->Iex.Binop.arg2);
            HReg      dst  = zx ? widen_z_32_to_64(env, argL)
                                : widen_s_32_to_64(env, argL);
            addInstr(env, ARM64Instr_Shift(dst, dst, argR, ARM64sh_SHR));
            return dst;
         }
         default: break;
      }

      /* MUL */
      if (e->Iex.Binop.op == Iop_Mul64 || e->Iex.Binop.op == Iop_Mul32) {
         HReg argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
         HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
         HReg dst  = newVRegI(env);
         addInstr(env, ARM64Instr_Mul(dst, argL, argR, ARM64mul_PLAIN));
         return dst;
      }

      /* MULL */
      if (e->Iex.Binop.op == Iop_MullU32 || e->Iex.Binop.op == Iop_MullS32) {
         Bool isS  = e->Iex.Binop.op == Iop_MullS32;
         HReg argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
         HReg extL = (isS ? widen_s_32_to_64 : widen_z_32_to_64)(env, argL);
         HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
         HReg extR = (isS ? widen_s_32_to_64 : widen_z_32_to_64)(env, argR);
         HReg dst  = newVRegI(env);
         addInstr(env, ARM64Instr_Mul(dst, extL, extR, ARM64mul_PLAIN));
         return dst;
      }

      /* Handle misc other ops. */

      if (e->Iex.Binop.op == Iop_Max32U) {
         HReg argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
         HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
         HReg dst  = newVRegI(env);
         addInstr(env, ARM64Instr_Cmp(argL, ARM64RIA_R(argR), False/*!is64*/));
         addInstr(env, ARM64Instr_CSel(dst, argL, argR, ARM64cc_CS));
         return dst;
      }

      if (e->Iex.Binop.op == Iop_32HLto64) {
         HReg hi32s = iselIntExpr_R(env, e->Iex.Binop.arg1);
         HReg lo32s = iselIntExpr_R(env, e->Iex.Binop.arg2);
         HReg lo32  = widen_z_32_to_64(env, lo32s);
         HReg hi32  = newVRegI(env);
         addInstr(env, ARM64Instr_Shift(hi32, hi32s, ARM64RI6_I6(32),
                                        ARM64sh_SHL));
         addInstr(env, ARM64Instr_Logic(hi32, hi32, ARM64RIL_R(lo32),
                                        ARM64lo_OR));
         return hi32;
      }

      if (e->Iex.Binop.op == Iop_CmpF64 || e->Iex.Binop.op == Iop_CmpF32) {
         Bool isD = e->Iex.Binop.op == Iop_CmpF64;
         HReg dL  = (isD ? iselDblExpr : iselFltExpr)(env, e->Iex.Binop.arg1);
         HReg dR  = (isD ? iselDblExpr : iselFltExpr)(env, e->Iex.Binop.arg2);
         HReg dst = newVRegI(env);
         HReg imm = newVRegI(env);
         /* Do the compare (FCMP), which sets NZCV in PSTATE.  Then
            create in dst, the IRCmpF64Result encoded result. */
         addInstr(env, (isD ? ARM64Instr_VCmpD : ARM64Instr_VCmpS)(dL, dR));
         addInstr(env, ARM64Instr_Imm64(dst, 0));
         addInstr(env, ARM64Instr_Imm64(imm, 0x40)); // 0x40 = Ircr_EQ
         addInstr(env, ARM64Instr_CSel(dst, imm, dst, ARM64cc_EQ));
         addInstr(env, ARM64Instr_Imm64(imm, 0x01)); // 0x01 = Ircr_LT
         addInstr(env, ARM64Instr_CSel(dst, imm, dst, ARM64cc_MI));
         addInstr(env, ARM64Instr_Imm64(imm, 0x00)); // 0x00 = Ircr_GT
         addInstr(env, ARM64Instr_CSel(dst, imm, dst, ARM64cc_GT));
         addInstr(env, ARM64Instr_Imm64(imm, 0x45)); // 0x45 = Ircr_UN
         addInstr(env, ARM64Instr_CSel(dst, imm, dst, ARM64cc_VS));
         return dst;
      }

      { /* local scope */
        ARM64CvtOp cvt_op = ARM64cvt_INVALID;
        Bool       srcIsD = False;
        switch (e->Iex.Binop.op) {
           case Iop_F64toI64S:
              cvt_op = ARM64cvt_F64_I64S; srcIsD = True; break;
           case Iop_F64toI64U:
              cvt_op = ARM64cvt_F64_I64U; srcIsD = True; break;
           case Iop_F64toI32S:
              cvt_op = ARM64cvt_F64_I32S; srcIsD = True; break;
           case Iop_F64toI32U:
              cvt_op = ARM64cvt_F64_I32U; srcIsD = True; break;
           case Iop_F32toI32S:
              cvt_op = ARM64cvt_F32_I32S; srcIsD = False; break;
           case Iop_F32toI32U:
              cvt_op = ARM64cvt_F32_I32U; srcIsD = False; break;
           case Iop_F32toI64S:
              cvt_op = ARM64cvt_F32_I64S; srcIsD = False; break;
           case Iop_F32toI64U:
              cvt_op = ARM64cvt_F32_I64U; srcIsD = False; break;
           default:
              break;
        }
        if (cvt_op != ARM64cvt_INVALID) {
           /* This is all a bit dodgy, because we can't handle a
              non-constant (not-known-at-JIT-time) rounding mode
              indication.  That's because there's no instruction
              AFAICS that does this conversion but rounds according to
              FPCR.RM, so we have to bake the rounding mode into the
              instruction right now.  But that should be OK because
              (1) the front end attaches a literal Irrm_ value to the
              conversion binop, and (2) iropt will never float that
              off via CSE, into a literal.  Hence we should always
              have an Irrm_ value as the first arg. */
           IRExpr* arg1 = e->Iex.Binop.arg1;
           if (arg1->tag != Iex_Const) goto irreducible;
           IRConst* arg1con = arg1->Iex.Const.con;
           vassert(arg1con->tag == Ico_U32); // else ill-typed IR
           UInt irrm = arg1con->Ico.U32;
           /* Find the ARM-encoded equivalent for |irrm|. */
           UInt armrm = 4; /* impossible */
           switch (irrm) {
              case Irrm_NEAREST: armrm = 0; break;
              case Irrm_NegINF:  armrm = 2; break;
              case Irrm_PosINF:  armrm = 1; break;
              case Irrm_ZERO:    armrm = 3; break;
              default: goto irreducible;
           }
           HReg src = (srcIsD ? iselDblExpr : iselFltExpr)
                         (env, e->Iex.Binop.arg2);
           HReg dst = newVRegI(env);
           addInstr(env, ARM64Instr_VCvtF2I(cvt_op, dst, src, armrm));
           return dst;
        }
      } /* local scope */

      /* All cases involving host-side helper calls. */
      void* fn = NULL;
      switch (e->Iex.Binop.op) {
         case Iop_DivU32:
            fn = &h_calc_udiv32_w_arm_semantics; break;
         case Iop_DivS32:
            fn = &h_calc_sdiv32_w_arm_semantics; break;
         case Iop_DivU64:
            fn = &h_calc_udiv64_w_arm_semantics; break;
         case Iop_DivS64:
            fn = &h_calc_sdiv64_w_arm_semantics; break;
         default:
            break;
      }

      if (fn) {
         HReg regL = iselIntExpr_R(env, e->Iex.Binop.arg1);
         HReg regR = iselIntExpr_R(env, e->Iex.Binop.arg2);
         HReg res  = newVRegI(env);
         addInstr(env, ARM64Instr_MovI(hregARM64_X0(), regL));
         addInstr(env, ARM64Instr_MovI(hregARM64_X1(), regR));
         addInstr(env, ARM64Instr_Call( ARM64cc_AL, (Addr)fn,
                                        2, mk_RetLoc_simple(RLPri_Int) ));
         addInstr(env, ARM64Instr_MovI(res, hregARM64_X0()));
         return res;
      }

      break;
   }

   /* --------- UNARY OP --------- */
   case Iex_Unop: {

      switch (e->Iex.Unop.op) {
         case Iop_16Uto64: {
            /* This probably doesn't occur often enough to be worth
               rolling the extension into the load. */
            IRExpr* arg = e->Iex.Unop.arg;
            HReg    src = iselIntExpr_R(env, arg);
            HReg    dst = widen_z_16_to_64(env, src);
            return dst;
         }
         case Iop_32Uto64: {
            IRExpr* arg = e->Iex.Unop.arg;
            if (arg->tag == Iex_Load) {
               /* This correctly zero extends because _LdSt32 is
                  defined to do a zero extending load. */
               HReg dst = newVRegI(env);
               ARM64AMode* am
                  = iselIntExpr_AMode(env, arg->Iex.Load.addr, Ity_I32);
               addInstr(env, ARM64Instr_LdSt32(True/*isLoad*/, dst, am));
               return dst;
            }
            /* else be lame and mask it  */
            HReg src  = iselIntExpr_R(env, arg);
            HReg dst  = widen_z_32_to_64(env, src);
            return dst;
         }
         case Iop_8Uto32: /* Just freeload on the 8Uto64 case */
         case Iop_8Uto64: {
            IRExpr* arg = e->Iex.Unop.arg;
            if (arg->tag == Iex_Load) {
               /* This correctly zero extends because _LdSt8 is
                  defined to do a zero extending load. */
               HReg dst = newVRegI(env);
               ARM64AMode* am
                  = iselIntExpr_AMode(env, arg->Iex.Load.addr, Ity_I8);
               addInstr(env, ARM64Instr_LdSt8(True/*isLoad*/, dst, am));
               return dst;
            }
            /* else be lame and mask it  */
            HReg src = iselIntExpr_R(env, arg);
            HReg dst = widen_z_8_to_64(env, src);
            return dst;
         }
         case Iop_128HIto64: {
            HReg rHi, rLo;
            iselInt128Expr(&rHi,&rLo, env, e->Iex.Unop.arg);
            return rHi; /* and abandon rLo */
         }
         case Iop_8Sto32: case Iop_8Sto64: {
            IRExpr* arg = e->Iex.Unop.arg;
            HReg    src = iselIntExpr_R(env, arg);
            HReg    dst = widen_s_8_to_64(env, src);
            return dst;
         }
         case Iop_16Sto32: case Iop_16Sto64: {
            IRExpr* arg = e->Iex.Unop.arg;
            HReg    src = iselIntExpr_R(env, arg);
            HReg    dst = widen_s_16_to_64(env, src);
            return dst;
         }
         case Iop_32Sto64: {
            IRExpr* arg = e->Iex.Unop.arg;
            HReg    src = iselIntExpr_R(env, arg);
            HReg    dst = widen_s_32_to_64(env, src);
            return dst;
         }
         case Iop_Not32:
         case Iop_Not64: {
            HReg dst = newVRegI(env);
            HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_Unary(dst, src, ARM64un_NOT));
            return dst;
         }
         case Iop_Clz64: {
            HReg dst = newVRegI(env);
            HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_Unary(dst, src, ARM64un_CLZ));
            return dst;
         }
         case Iop_Left32:
         case Iop_Left64: {
            /* Left64(src) = src | -src.  Left32 can use the same
               implementation since in that case we don't care what
               the upper 32 bits become. */
            HReg dst = newVRegI(env);
            HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_Unary(dst, src, ARM64un_NEG));
            addInstr(env, ARM64Instr_Logic(dst, dst, ARM64RIL_R(src),
                                           ARM64lo_OR));
            return dst;
         }
         case Iop_CmpwNEZ64: {
           /* CmpwNEZ64(src) = (src == 0) ? 0...0 : 1...1
                             = Left64(src) >>s 63 */
            HReg dst = newVRegI(env);
            HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_Unary(dst, src, ARM64un_NEG));
            addInstr(env, ARM64Instr_Logic(dst, dst, ARM64RIL_R(src),
                                           ARM64lo_OR));
            addInstr(env, ARM64Instr_Shift(dst, dst, ARM64RI6_I6(63),
                                           ARM64sh_SAR));
            return dst;
         }
         case Iop_CmpwNEZ32: {
            /* CmpwNEZ32(src) = CmpwNEZ64(src & 0xFFFFFFFF)
                              = Left64(src & 0xFFFFFFFF) >>s 63 */
            HReg dst = newVRegI(env);
            HReg pre = iselIntExpr_R(env, e->Iex.Unop.arg);
            HReg src = widen_z_32_to_64(env, pre);
            addInstr(env, ARM64Instr_Unary(dst, src, ARM64un_NEG));
            addInstr(env, ARM64Instr_Logic(dst, dst, ARM64RIL_R(src),
                                           ARM64lo_OR));
            addInstr(env, ARM64Instr_Shift(dst, dst, ARM64RI6_I6(63),
                                           ARM64sh_SAR));
            return dst;
         }
         case Iop_V128to64: case Iop_V128HIto64: {
            HReg dst    = newVRegI(env);
            HReg src    = iselV128Expr(env, e->Iex.Unop.arg);
            UInt laneNo = (e->Iex.Unop.op == Iop_V128HIto64) ? 1 : 0;
            addInstr(env, ARM64Instr_VXfromQ(dst, src, laneNo));
            return dst;
         }
         case Iop_ReinterpF64asI64: {
            HReg dst = newVRegI(env);
            HReg src = iselDblExpr(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VXfromDorS(dst, src, True/*fromD*/));
            return dst;
         }
         case Iop_ReinterpF32asI32: {
            HReg dst = newVRegI(env);
            HReg src = iselFltExpr(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VXfromDorS(dst, src, False/*!fromD*/));
            return dst;
         }
         case Iop_1Sto16:
         case Iop_1Sto32:
         case Iop_1Sto64: {
            /* As with the iselStmt case for 'tmp:I1 = expr', we could
               do a lot better here if it ever became necessary. */
            HReg zero = newVRegI(env);
            HReg one  = newVRegI(env);
            HReg dst  = newVRegI(env);
            addInstr(env, ARM64Instr_Imm64(zero, 0));
            addInstr(env, ARM64Instr_Imm64(one,  1));
            ARM64CondCode cc = iselCondCode(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_CSel(dst, one, zero, cc));
            addInstr(env, ARM64Instr_Shift(dst, dst, ARM64RI6_I6(63),
                                           ARM64sh_SHL));
            addInstr(env, ARM64Instr_Shift(dst, dst, ARM64RI6_I6(63),
                                           ARM64sh_SAR));
            return dst;
         }
         case Iop_NarrowUn16to8x8:
         case Iop_NarrowUn32to16x4:
         case Iop_NarrowUn64to32x2:
         case Iop_QNarrowUn16Sto8Sx8:
         case Iop_QNarrowUn32Sto16Sx4:
         case Iop_QNarrowUn64Sto32Sx2:
         case Iop_QNarrowUn16Uto8Ux8:
         case Iop_QNarrowUn32Uto16Ux4:
         case Iop_QNarrowUn64Uto32Ux2:
         case Iop_QNarrowUn16Sto8Ux8:
         case Iop_QNarrowUn32Sto16Ux4:
         case Iop_QNarrowUn64Sto32Ux2:
         {
            HReg src = iselV128Expr(env, e->Iex.Unop.arg);
            HReg tmp = newVRegV(env);
            HReg dst = newVRegI(env);
            UInt dszBlg2 = 3; /* illegal */
            ARM64VecNarrowOp op = ARM64vecna_INVALID;
            switch (e->Iex.Unop.op) {
               case Iop_NarrowUn16to8x8:
                  dszBlg2 = 0; op = ARM64vecna_XTN; break;
               case Iop_NarrowUn32to16x4:
                  dszBlg2 = 1; op = ARM64vecna_XTN; break;
               case Iop_NarrowUn64to32x2:
                  dszBlg2 = 2; op = ARM64vecna_XTN; break;
               case Iop_QNarrowUn16Sto8Sx8:
                  dszBlg2 = 0; op = ARM64vecna_SQXTN; break;
               case Iop_QNarrowUn32Sto16Sx4:
                  dszBlg2 = 1; op = ARM64vecna_SQXTN; break;
               case Iop_QNarrowUn64Sto32Sx2:
                  dszBlg2 = 2; op = ARM64vecna_SQXTN; break;
               case Iop_QNarrowUn16Uto8Ux8:
                  dszBlg2 = 0; op = ARM64vecna_UQXTN; break;
               case Iop_QNarrowUn32Uto16Ux4:
                  dszBlg2 = 1; op = ARM64vecna_UQXTN; break;
               case Iop_QNarrowUn64Uto32Ux2:
                  dszBlg2 = 2; op = ARM64vecna_UQXTN; break;
               case Iop_QNarrowUn16Sto8Ux8:
                  dszBlg2 = 0; op = ARM64vecna_SQXTUN; break;
               case Iop_QNarrowUn32Sto16Ux4:
                  dszBlg2 = 1; op = ARM64vecna_SQXTUN; break;
               case Iop_QNarrowUn64Sto32Ux2:
                  dszBlg2 = 2; op = ARM64vecna_SQXTUN; break;
               default:
                  vassert(0);
            }
            addInstr(env, ARM64Instr_VNarrowV(op, dszBlg2, tmp, src));
            addInstr(env, ARM64Instr_VXfromQ(dst, tmp, 0/*laneNo*/));
            return dst;
         }
         case Iop_1Uto64: {
            /* 1Uto64(tmp). */
            HReg dst = newVRegI(env);
            if (e->Iex.Unop.arg->tag == Iex_RdTmp) {
               ARM64RIL* one = mb_mkARM64RIL_I(1);
               HReg src = lookupIRTemp(env, e->Iex.Unop.arg->Iex.RdTmp.tmp);
               vassert(one);
               addInstr(env, ARM64Instr_Logic(dst, src, one, ARM64lo_AND));
            } else {
               /* CLONE-01 */
               HReg zero = newVRegI(env);
               HReg one  = newVRegI(env);
               addInstr(env, ARM64Instr_Imm64(zero, 0));
               addInstr(env, ARM64Instr_Imm64(one,  1));
               ARM64CondCode cc = iselCondCode(env, e->Iex.Unop.arg);
               addInstr(env, ARM64Instr_CSel(dst, one, zero, cc));
            }
            return dst;
         }
         case Iop_64to32:
         case Iop_64to16:
         case Iop_64to8:
            /* These are no-ops. */
            return iselIntExpr_R(env, e->Iex.Unop.arg);

         default:
            break;
      }

      break;
   }

   /* --------- GET --------- */
   case Iex_Get: {
      if (ty == Ity_I64
          && 0 == (e->Iex.Get.offset & 7) && e->Iex.Get.offset < (8<<12)-8) {
         HReg        dst = newVRegI(env);
         ARM64AMode* am
            = mk_baseblock_64bit_access_amode(e->Iex.Get.offset);
         addInstr(env, ARM64Instr_LdSt64(True/*isLoad*/, dst, am));
         return dst;
      }
      if (ty == Ity_I32
          && 0 == (e->Iex.Get.offset & 3) && e->Iex.Get.offset < (4<<12)-4) {
         HReg        dst = newVRegI(env);
         ARM64AMode* am
            = mk_baseblock_32bit_access_amode(e->Iex.Get.offset);
         addInstr(env, ARM64Instr_LdSt32(True/*isLoad*/, dst, am));
         return dst;
      }
      if (ty == Ity_I16
          && 0 == (e->Iex.Get.offset & 1) && e->Iex.Get.offset < (2<<12)-2) {
         HReg        dst = newVRegI(env);
         ARM64AMode* am
            = mk_baseblock_16bit_access_amode(e->Iex.Get.offset);
         addInstr(env, ARM64Instr_LdSt16(True/*isLoad*/, dst, am));
         return dst;
      }
      if (ty == Ity_I8
          /* && no alignment check */ && e->Iex.Get.offset < (1<<12)-1) {
         HReg        dst = newVRegI(env);
         ARM64AMode* am
            = mk_baseblock_8bit_access_amode(e->Iex.Get.offset);
         addInstr(env, ARM64Instr_LdSt8(True/*isLoad*/, dst, am));
         return dst;
      }
      break;
   }

   /* --------- CCALL --------- */
   case Iex_CCall: {
      HReg    dst = newVRegI(env);
      vassert(ty == e->Iex.CCall.retty);

      /* be very restrictive for now.  Only 64-bit ints allowed for
         args, and 64 bits for return type.  Don't forget to change
         the RetLoc if more types are allowed in future. */
      if (e->Iex.CCall.retty != Ity_I64)
         goto irreducible;

      /* Marshal args, do the call, clear stack. */
      UInt   addToSp = 0;
      RetLoc rloc    = mk_RetLoc_INVALID();
      Bool   ok      = doHelperCall( &addToSp, &rloc, env, NULL/*guard*/,
                                     e->Iex.CCall.cee, e->Iex.CCall.retty,
                                     e->Iex.CCall.args );
      /* */
      if (ok) {
         vassert(is_sane_RetLoc(rloc));
         vassert(rloc.pri == RLPri_Int);
         vassert(addToSp == 0);
         addInstr(env, ARM64Instr_MovI(dst, hregARM64_X0()));
         return dst;
      }
      /* else fall through; will hit the irreducible: label */
   }

   /* --------- LITERAL --------- */
   /* 64-bit literals */
   case Iex_Const: {
      ULong u   = 0;
      HReg  dst = newVRegI(env);
      switch (e->Iex.Const.con->tag) {
         case Ico_U64: u = e->Iex.Const.con->Ico.U64; break;
         case Ico_U32: u = e->Iex.Const.con->Ico.U32; break;
         case Ico_U16: u = e->Iex.Const.con->Ico.U16; break;
         case Ico_U8:  u = e->Iex.Const.con->Ico.U8;  break;
         default: ppIRExpr(e); vpanic("iselIntExpr_R.Iex_Const(arm64)");
      }
      addInstr(env, ARM64Instr_Imm64(dst, u));
      return dst;
   }

   /* --------- MULTIPLEX --------- */
   case Iex_ITE: {
      /* ITE(ccexpr, iftrue, iffalse) */
      if (ty == Ity_I64 || ty == Ity_I32) {
         ARM64CondCode cc;
         HReg r1  = iselIntExpr_R(env, e->Iex.ITE.iftrue);
         HReg r0  = iselIntExpr_R(env, e->Iex.ITE.iffalse);
         HReg dst = newVRegI(env);
         cc = iselCondCode(env, e->Iex.ITE.cond);
         addInstr(env, ARM64Instr_CSel(dst, r1, r0, cc));
         return dst;
      }
      break;
   }

   default: 
   break;
   } /* switch (e->tag) */

   /* We get here if no pattern matched. */
  irreducible:
   ppIRExpr(e);
   vpanic("iselIntExpr_R: cannot reduce tree");
}


/*---------------------------------------------------------*/
/*--- ISEL: Integer expressions (128 bit)               ---*/
/*---------------------------------------------------------*/

/* Compute a 128-bit value into a register pair, which is returned as
   the first two parameters.  As with iselIntExpr_R, these may be
   either real or virtual regs; in any case they must not be changed
   by subsequent code emitted by the caller.  */

static void iselInt128Expr ( HReg* rHi, HReg* rLo, 
                             ISelEnv* env, IRExpr* e )
{
   iselInt128Expr_wrk(rHi, rLo, env, e);
#  if 0
   vex_printf("\n"); ppIRExpr(e); vex_printf("\n");
#  endif
   vassert(hregClass(*rHi) == HRcInt64);
   vassert(hregIsVirtual(*rHi));
   vassert(hregClass(*rLo) == HRcInt64);
   vassert(hregIsVirtual(*rLo));
}

/* DO NOT CALL THIS DIRECTLY ! */
static void iselInt128Expr_wrk ( HReg* rHi, HReg* rLo, 
                                 ISelEnv* env, IRExpr* e )
{
   vassert(e);
   vassert(typeOfIRExpr(env->type_env,e) == Ity_I128);

   /* --------- BINARY ops --------- */
   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         /* 64 x 64 -> 128 multiply */
         case Iop_MullU64:
         case Iop_MullS64: {
            Bool syned = toBool(e->Iex.Binop.op == Iop_MullS64);
            HReg argL  = iselIntExpr_R(env, e->Iex.Binop.arg1);
            HReg argR  = iselIntExpr_R(env, e->Iex.Binop.arg2);
            HReg dstLo = newVRegI(env);
            HReg dstHi = newVRegI(env);
            addInstr(env, ARM64Instr_Mul(dstLo, argL, argR,
                                         ARM64mul_PLAIN));
            addInstr(env, ARM64Instr_Mul(dstHi, argL, argR,
                                         syned ? ARM64mul_SX : ARM64mul_ZX));
            *rHi = dstHi;
            *rLo = dstLo;
            return;
         }
         /* 64HLto128(e1,e2) */
         case Iop_64HLto128:
            *rHi = iselIntExpr_R(env, e->Iex.Binop.arg1);
            *rLo = iselIntExpr_R(env, e->Iex.Binop.arg2);
            return;
         default: 
            break;
      }
   } /* if (e->tag == Iex_Binop) */

   ppIRExpr(e);
   vpanic("iselInt128Expr(arm64)");
}


/*---------------------------------------------------------*/
/*--- ISEL: Vector expressions (128 bit)                ---*/
/*---------------------------------------------------------*/

static HReg iselV128Expr ( ISelEnv* env, IRExpr* e )
{
   HReg r = iselV128Expr_wrk( env, e );
   vassert(hregClass(r) == HRcVec128);
   vassert(hregIsVirtual(r));
   return r;
}

/* DO NOT CALL THIS DIRECTLY */
static HReg iselV128Expr_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env, e);
   vassert(e);
   vassert(ty == Ity_V128);

   if (e->tag == Iex_RdTmp) {
      return lookupIRTemp(env, e->Iex.RdTmp.tmp);
   }

   if (e->tag == Iex_Const) {
      /* Only a very limited range of constants is handled. */
      vassert(e->Iex.Const.con->tag == Ico_V128);
      UShort con = e->Iex.Const.con->Ico.V128;
      HReg   res = newVRegV(env);
      switch (con) {
         case 0x0000: case 0x000F: case 0x003F: case 0x00FF: case 0xFFFF:
            addInstr(env, ARM64Instr_VImmQ(res, con));
            return res;
         case 0x00F0:
            addInstr(env, ARM64Instr_VImmQ(res, 0x000F));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 12));
            return res;
         case 0x0F00:
            addInstr(env, ARM64Instr_VImmQ(res, 0x000F));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 8));
            return res;
         case 0x0FF0:
            addInstr(env, ARM64Instr_VImmQ(res, 0x00FF));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 12));
            return res;
         case 0x0FFF:
            addInstr(env, ARM64Instr_VImmQ(res, 0x000F));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 4));
            addInstr(env, ARM64Instr_VUnaryV(ARM64vecu_NOT, res, res));
            return res;
         case 0xF000:
            addInstr(env, ARM64Instr_VImmQ(res, 0x000F));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 4));
            return res;
         case 0xFF00:
            addInstr(env, ARM64Instr_VImmQ(res, 0x00FF));
            addInstr(env, ARM64Instr_VExtV(res, res, res, 8));
            return res;
         default: 
            break;
      }
      /* Unhandled */
      goto v128_expr_bad;
   }

   if (e->tag == Iex_Load) {
      HReg res = newVRegV(env);
      HReg rN  = iselIntExpr_R(env, e->Iex.Load.addr);
      vassert(ty == Ity_V128);
      addInstr(env, ARM64Instr_VLdStQ(True/*isLoad*/, res, rN));
      return res;
   }

   if (e->tag == Iex_Get) {
      UInt offs = (UInt)e->Iex.Get.offset;
      if (offs < (1<<12)) {
         HReg addr = mk_baseblock_128bit_access_addr(env, offs);
         HReg res  = newVRegV(env);
         vassert(ty == Ity_V128);
         addInstr(env, ARM64Instr_VLdStQ(True/*isLoad*/, res, addr));
         return res;
      }
      goto v128_expr_bad;
   }

   if (e->tag == Iex_Unop) {

      /* Iop_ZeroHIXXofV128 cases */
      UShort imm16 = 0;
      switch (e->Iex.Unop.op) {
         case Iop_ZeroHI64ofV128:  imm16 = 0x00FF; break;
         case Iop_ZeroHI96ofV128:  imm16 = 0x000F; break;
         case Iop_ZeroHI112ofV128: imm16 = 0x0003; break;
         case Iop_ZeroHI120ofV128: imm16 = 0x0001; break;
         default: break;
      }
      if (imm16 != 0) {
         HReg src = iselV128Expr(env, e->Iex.Unop.arg);
         HReg imm = newVRegV(env);
         HReg res = newVRegV(env);
         addInstr(env, ARM64Instr_VImmQ(imm, imm16));
         addInstr(env, ARM64Instr_VBinV(ARM64vecb_AND, res, src, imm));
         return res;
      }

      /* Other cases */
      switch (e->Iex.Unop.op) {
         case Iop_NotV128:
         case Iop_Abs64Fx2: case Iop_Abs32Fx4:
         case Iop_Neg64Fx2: case Iop_Neg32Fx4:
         case Iop_Abs64x2:  case Iop_Abs32x4:
         case Iop_Abs16x8:  case Iop_Abs8x16:
         case Iop_Cls32x4:  case Iop_Cls16x8:  case Iop_Cls8x16:
         case Iop_Clz32x4:  case Iop_Clz16x8:  case Iop_Clz8x16:
         case Iop_Cnt8x16:
         case Iop_Reverse1sIn8_x16:
         case Iop_Reverse8sIn16_x8:
         case Iop_Reverse8sIn32_x4: case Iop_Reverse16sIn32_x4:
         case Iop_Reverse8sIn64_x2: case Iop_Reverse16sIn64_x2:
         case Iop_Reverse32sIn64_x2:
         case Iop_RecipEst32Ux4:
         case Iop_RSqrtEst32Ux4:
         case Iop_RecipEst64Fx2: case Iop_RecipEst32Fx4:
         case Iop_RSqrtEst64Fx2: case Iop_RSqrtEst32Fx4:
         {
            HReg res   = newVRegV(env);
            HReg arg   = iselV128Expr(env, e->Iex.Unop.arg);
            Bool setRM = False;
            ARM64VecUnaryOp op = ARM64vecu_INVALID;
            switch (e->Iex.Unop.op) {
               case Iop_NotV128:           op = ARM64vecu_NOT;         break;
               case Iop_Abs64Fx2:          op = ARM64vecu_FABS64x2;    break;
               case Iop_Abs32Fx4:          op = ARM64vecu_FABS32x4;    break;
               case Iop_Neg64Fx2:          op = ARM64vecu_FNEG64x2;    break;
               case Iop_Neg32Fx4:          op = ARM64vecu_FNEG32x4;    break;
               case Iop_Abs64x2:           op = ARM64vecu_ABS64x2;     break;
               case Iop_Abs32x4:           op = ARM64vecu_ABS32x4;     break;
               case Iop_Abs16x8:           op = ARM64vecu_ABS16x8;     break;
               case Iop_Abs8x16:           op = ARM64vecu_ABS8x16;     break;
               case Iop_Cls32x4:           op = ARM64vecu_CLS32x4;     break;
               case Iop_Cls16x8:           op = ARM64vecu_CLS16x8;     break;
               case Iop_Cls8x16:           op = ARM64vecu_CLS8x16;     break;
               case Iop_Clz32x4:           op = ARM64vecu_CLZ32x4;     break;
               case Iop_Clz16x8:           op = ARM64vecu_CLZ16x8;     break;
               case Iop_Clz8x16:           op = ARM64vecu_CLZ8x16;     break;
               case Iop_Cnt8x16:           op = ARM64vecu_CNT8x16;     break;
               case Iop_Reverse1sIn8_x16:  op = ARM64vecu_RBIT;        break;
               case Iop_Reverse8sIn16_x8:  op = ARM64vecu_REV1616B;    break;
               case Iop_Reverse8sIn32_x4:  op = ARM64vecu_REV3216B;    break;
               case Iop_Reverse16sIn32_x4: op = ARM64vecu_REV328H;     break;
               case Iop_Reverse8sIn64_x2:  op = ARM64vecu_REV6416B;    break;
               case Iop_Reverse16sIn64_x2: op = ARM64vecu_REV648H;     break;
               case Iop_Reverse32sIn64_x2: op = ARM64vecu_REV644S;     break;
               case Iop_RecipEst32Ux4:     op = ARM64vecu_URECPE32x4;  break;
               case Iop_RSqrtEst32Ux4:     op = ARM64vecu_URSQRTE32x4; break;
               case Iop_RecipEst64Fx2:     setRM = True;
                                           op = ARM64vecu_FRECPE64x2;  break;
               case Iop_RecipEst32Fx4:     setRM = True;
                                           op = ARM64vecu_FRECPE32x4;  break;
               case Iop_RSqrtEst64Fx2:     setRM = True;
                                           op = ARM64vecu_FRSQRTE64x2; break;
               case Iop_RSqrtEst32Fx4:     setRM = True;
                                           op = ARM64vecu_FRSQRTE32x4; break;
               default: vassert(0);
            }
            if (setRM) {
               // This is a bit of a kludge.  We should do rm properly for
               // these recip-est insns, but that would require changing the
               // primop's type to take an rmode.
               set_FPCR_rounding_mode(env, IRExpr_Const(
                                              IRConst_U32(Irrm_NEAREST)));
            }
            addInstr(env, ARM64Instr_VUnaryV(op, res, arg));
            return res;
         }
         case Iop_CmpNEZ8x16:
         case Iop_CmpNEZ16x8:
         case Iop_CmpNEZ32x4:
         case Iop_CmpNEZ64x2: {
            HReg arg  = iselV128Expr(env, e->Iex.Unop.arg);
            HReg zero = newVRegV(env);
            HReg res  = newVRegV(env);
            ARM64VecBinOp cmp = ARM64vecb_INVALID;
            switch (e->Iex.Unop.op) {
               case Iop_CmpNEZ64x2: cmp = ARM64vecb_CMEQ64x2; break;
               case Iop_CmpNEZ32x4: cmp = ARM64vecb_CMEQ32x4; break;
               case Iop_CmpNEZ16x8: cmp = ARM64vecb_CMEQ16x8; break;
               case Iop_CmpNEZ8x16: cmp = ARM64vecb_CMEQ8x16; break;
               default: vassert(0);
            }
            // This is pretty feeble.  Better: use CMP against zero
            // and avoid the extra instruction and extra register.
            addInstr(env, ARM64Instr_VImmQ(zero, 0x0000));
            addInstr(env, ARM64Instr_VBinV(cmp, res, arg, zero));
            addInstr(env, ARM64Instr_VUnaryV(ARM64vecu_NOT, res, res));
            return res;
         }
         case Iop_V256toV128_0:
         case Iop_V256toV128_1: {
            HReg vHi, vLo;
            iselV256Expr(&vHi, &vLo, env, e->Iex.Unop.arg);
            return (e->Iex.Unop.op == Iop_V256toV128_1) ? vHi : vLo;
         }
         case Iop_64UtoV128: {
            HReg res = newVRegV(env);
            HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VQfromX(res, arg));
            return res;
         }
         case Iop_Widen8Sto16x8: {
            HReg res = newVRegV(env);
            HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VQfromX(res, arg));
            addInstr(env, ARM64Instr_VBinV(ARM64vecb_ZIP18x16, res, res, res));
            addInstr(env, ARM64Instr_VShiftImmV(ARM64vecshi_SSHR16x8,
                                                res, res, 8));
            return res;
         }
         case Iop_Widen16Sto32x4: {
            HReg res = newVRegV(env);
            HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VQfromX(res, arg));
            addInstr(env, ARM64Instr_VBinV(ARM64vecb_ZIP116x8, res, res, res));
            addInstr(env, ARM64Instr_VShiftImmV(ARM64vecshi_SSHR32x4,
                                                res, res, 16));
            return res;
         }
         case Iop_Widen32Sto64x2: {
            HReg res = newVRegV(env);
            HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
            addInstr(env, ARM64Instr_VQfromX(res, arg));
            addInstr(env, ARM64Instr_VBinV(ARM64vecb_ZIP132x4, res, res, res));
            addInstr(env, ARM64Instr_VShiftImmV(ARM64vecshi_SSHR64x2,
                                                res, res, 32));
            return res;
         }
         /* ... */
         default:
            break;
      } /* switch on the unop */
   } /* if (e->tag == Iex_Unop) */

   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         case Iop_Sqrt32Fx4:
         case Iop_Sqrt64Fx2: {
            HReg arg = iselV128Expr(env, e->Iex.Binop.arg2);
            HReg res = newVRegV(env);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            ARM64VecUnaryOp op 
               = e->Iex.Binop.op == Iop_Sqrt32Fx4
                    ? ARM64vecu_FSQRT32x4 : ARM64vecu_FSQRT64x2;
            addInstr(env, ARM64Instr_VUnaryV(op, res, arg));
            return res;
         }
         case Iop_64HLtoV128: {
            HReg res  = newVRegV(env);
            HReg argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
            HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
            addInstr(env, ARM64Instr_VQfromXX(res, argL, argR));
            return res;
         }
         /* -- Cases where we can generate a simple three-reg instruction. -- */
         case Iop_AndV128:
         case Iop_OrV128:
         case Iop_XorV128:
         case Iop_Max32Ux4: case Iop_Max16Ux8: case Iop_Max8Ux16:
         case Iop_Min32Ux4: case Iop_Min16Ux8: case Iop_Min8Ux16:
         case Iop_Max32Sx4: case Iop_Max16Sx8: case Iop_Max8Sx16:
         case Iop_Min32Sx4: case Iop_Min16Sx8: case Iop_Min8Sx16:
         case Iop_Add64x2: case Iop_Add32x4:
         case Iop_Add16x8: case Iop_Add8x16:
         case Iop_Sub64x2: case Iop_Sub32x4:
         case Iop_Sub16x8: case Iop_Sub8x16:
         case Iop_Mul32x4: case Iop_Mul16x8: case Iop_Mul8x16:
         case Iop_CmpEQ64x2: case Iop_CmpEQ32x4:
         case Iop_CmpEQ16x8:  case Iop_CmpEQ8x16:
         case Iop_CmpGT64Ux2: case Iop_CmpGT32Ux4:
         case Iop_CmpGT16Ux8: case Iop_CmpGT8Ux16:
         case Iop_CmpGT64Sx2: case Iop_CmpGT32Sx4:
         case Iop_CmpGT16Sx8: case Iop_CmpGT8Sx16:
         case Iop_CmpEQ64Fx2: case Iop_CmpEQ32Fx4:
         case Iop_CmpLE64Fx2: case Iop_CmpLE32Fx4:
         case Iop_CmpLT64Fx2: case Iop_CmpLT32Fx4:
         case Iop_Perm8x16:
         case Iop_InterleaveLO64x2: case Iop_CatEvenLanes32x4:
         case Iop_CatEvenLanes16x8: case Iop_CatEvenLanes8x16:
         case Iop_InterleaveHI64x2: case Iop_CatOddLanes32x4:
         case Iop_CatOddLanes16x8:  case Iop_CatOddLanes8x16:
         case Iop_InterleaveHI32x4:
         case Iop_InterleaveHI16x8: case Iop_InterleaveHI8x16:
         case Iop_InterleaveLO32x4:
         case Iop_InterleaveLO16x8: case Iop_InterleaveLO8x16:
         case Iop_PolynomialMul8x16:
         case Iop_QAdd64Sx2: case Iop_QAdd32Sx4:
         case Iop_QAdd16Sx8: case Iop_QAdd8Sx16:
         case Iop_QAdd64Ux2: case Iop_QAdd32Ux4:
         case Iop_QAdd16Ux8: case Iop_QAdd8Ux16:
         case Iop_QSub64Sx2: case Iop_QSub32Sx4:
         case Iop_QSub16Sx8: case Iop_QSub8Sx16:
         case Iop_QSub64Ux2: case Iop_QSub32Ux4:
         case Iop_QSub16Ux8: case Iop_QSub8Ux16:
         case Iop_QDMulHi32Sx4:  case Iop_QDMulHi16Sx8:
         case Iop_QRDMulHi32Sx4: case Iop_QRDMulHi16Sx8:
         case Iop_Sh8Sx16:  case Iop_Sh16Sx8:
         case Iop_Sh32Sx4:  case Iop_Sh64Sx2:
         case Iop_Sh8Ux16:  case Iop_Sh16Ux8:
         case Iop_Sh32Ux4:  case Iop_Sh64Ux2:
         case Iop_Rsh8Sx16: case Iop_Rsh16Sx8:
         case Iop_Rsh32Sx4: case Iop_Rsh64Sx2:
         case Iop_Rsh8Ux16: case Iop_Rsh16Ux8:
         case Iop_Rsh32Ux4: case Iop_Rsh64Ux2:
         case Iop_Max64Fx2: case Iop_Max32Fx4:
         case Iop_Min64Fx2: case Iop_Min32Fx4:
         case Iop_RecipStep64Fx2: case Iop_RecipStep32Fx4:
         case Iop_RSqrtStep64Fx2: case Iop_RSqrtStep32Fx4:
         {
            HReg res   = newVRegV(env);
            HReg argL  = iselV128Expr(env, e->Iex.Binop.arg1);
            HReg argR  = iselV128Expr(env, e->Iex.Binop.arg2);
            Bool sw    = False;
            Bool setRM = False;
            ARM64VecBinOp op = ARM64vecb_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_AndV128:    op = ARM64vecb_AND; break;
               case Iop_OrV128:     op = ARM64vecb_ORR; break;
               case Iop_XorV128:    op = ARM64vecb_XOR; break;
               case Iop_Max32Ux4:   op = ARM64vecb_UMAX32x4; break;
               case Iop_Max16Ux8:   op = ARM64vecb_UMAX16x8; break;
               case Iop_Max8Ux16:   op = ARM64vecb_UMAX8x16; break;
               case Iop_Min32Ux4:   op = ARM64vecb_UMIN32x4; break;
               case Iop_Min16Ux8:   op = ARM64vecb_UMIN16x8; break;
               case Iop_Min8Ux16:   op = ARM64vecb_UMIN8x16; break;
               case Iop_Max32Sx4:   op = ARM64vecb_SMAX32x4; break;
               case Iop_Max16Sx8:   op = ARM64vecb_SMAX16x8; break;
               case Iop_Max8Sx16:   op = ARM64vecb_SMAX8x16; break;
               case Iop_Min32Sx4:   op = ARM64vecb_SMIN32x4; break;
               case Iop_Min16Sx8:   op = ARM64vecb_SMIN16x8; break;
               case Iop_Min8Sx16:   op = ARM64vecb_SMIN8x16; break;
               case Iop_Add64x2:    op = ARM64vecb_ADD64x2; break;
               case Iop_Add32x4:    op = ARM64vecb_ADD32x4; break;
               case Iop_Add16x8:    op = ARM64vecb_ADD16x8; break;
               case Iop_Add8x16:    op = ARM64vecb_ADD8x16; break;
               case Iop_Sub64x2:    op = ARM64vecb_SUB64x2; break;
               case Iop_Sub32x4:    op = ARM64vecb_SUB32x4; break;
               case Iop_Sub16x8:    op = ARM64vecb_SUB16x8; break;
               case Iop_Sub8x16:    op = ARM64vecb_SUB8x16; break;
               case Iop_Mul32x4:    op = ARM64vecb_MUL32x4; break;
               case Iop_Mul16x8:    op = ARM64vecb_MUL16x8; break;
               case Iop_Mul8x16:    op = ARM64vecb_MUL8x16; break;
               case Iop_CmpEQ64x2:  op = ARM64vecb_CMEQ64x2; break;
               case Iop_CmpEQ32x4:  op = ARM64vecb_CMEQ32x4; break;
               case Iop_CmpEQ16x8:  op = ARM64vecb_CMEQ16x8; break;
               case Iop_CmpEQ8x16:  op = ARM64vecb_CMEQ8x16; break;
               case Iop_CmpGT64Ux2: op = ARM64vecb_CMHI64x2; break;
               case Iop_CmpGT32Ux4: op = ARM64vecb_CMHI32x4; break;
               case Iop_CmpGT16Ux8: op = ARM64vecb_CMHI16x8; break;
               case Iop_CmpGT8Ux16: op = ARM64vecb_CMHI8x16; break;
               case Iop_CmpGT64Sx2: op = ARM64vecb_CMGT64x2; break;
               case Iop_CmpGT32Sx4: op = ARM64vecb_CMGT32x4; break;
               case Iop_CmpGT16Sx8: op = ARM64vecb_CMGT16x8; break;
               case Iop_CmpGT8Sx16: op = ARM64vecb_CMGT8x16; break;
               case Iop_CmpEQ64Fx2: op = ARM64vecb_FCMEQ64x2; break;
               case Iop_CmpEQ32Fx4: op = ARM64vecb_FCMEQ32x4; break;
               case Iop_CmpLE64Fx2: op = ARM64vecb_FCMGE64x2; sw = True; break;
               case Iop_CmpLE32Fx4: op = ARM64vecb_FCMGE32x4; sw = True; break;
               case Iop_CmpLT64Fx2: op = ARM64vecb_FCMGT64x2; sw = True; break;
               case Iop_CmpLT32Fx4: op = ARM64vecb_FCMGT32x4; sw = True; break;
               case Iop_Perm8x16:   op = ARM64vecb_TBL1; break;
               case Iop_InterleaveLO64x2: op = ARM64vecb_UZP164x2; sw = True;
                                          break;
               case Iop_CatEvenLanes32x4: op = ARM64vecb_UZP132x4; sw = True;
                                          break;
               case Iop_CatEvenLanes16x8: op = ARM64vecb_UZP116x8; sw = True;
                                          break;
               case Iop_CatEvenLanes8x16: op = ARM64vecb_UZP18x16; sw = True;
                                          break;
               case Iop_InterleaveHI64x2: op = ARM64vecb_UZP264x2; sw = True;
                                          break;
               case Iop_CatOddLanes32x4:  op = ARM64vecb_UZP232x4; sw = True;
                                          break;
               case Iop_CatOddLanes16x8:  op = ARM64vecb_UZP216x8; sw = True;
                                          break;
               case Iop_CatOddLanes8x16:  op = ARM64vecb_UZP28x16; sw = True;
                                          break;
               case Iop_InterleaveHI32x4: op = ARM64vecb_ZIP232x4; sw = True;
                                          break;
               case Iop_InterleaveHI16x8: op = ARM64vecb_ZIP216x8; sw = True;
                                          break;
               case Iop_InterleaveHI8x16: op = ARM64vecb_ZIP28x16; sw = True;
                                          break;
               case Iop_InterleaveLO32x4: op = ARM64vecb_ZIP132x4; sw = True;
                                          break;
               case Iop_InterleaveLO16x8: op = ARM64vecb_ZIP116x8; sw = True;
                                          break;
               case Iop_InterleaveLO8x16: op = ARM64vecb_ZIP18x16; sw = True;
                                          break;
               case Iop_PolynomialMul8x16: op = ARM64vecb_PMUL8x16; break;
               case Iop_QAdd64Sx2:      op = ARM64vecb_SQADD64x2; break;
               case Iop_QAdd32Sx4:      op = ARM64vecb_SQADD32x4; break;
               case Iop_QAdd16Sx8:      op = ARM64vecb_SQADD16x8; break;
               case Iop_QAdd8Sx16:      op = ARM64vecb_SQADD8x16; break;
               case Iop_QAdd64Ux2:      op = ARM64vecb_UQADD64x2; break;
               case Iop_QAdd32Ux4:      op = ARM64vecb_UQADD32x4; break;
               case Iop_QAdd16Ux8:      op = ARM64vecb_UQADD16x8; break;
               case Iop_QAdd8Ux16:      op = ARM64vecb_UQADD8x16; break;
               case Iop_QSub64Sx2:      op = ARM64vecb_SQSUB64x2; break;
               case Iop_QSub32Sx4:      op = ARM64vecb_SQSUB32x4; break;
               case Iop_QSub16Sx8:      op = ARM64vecb_SQSUB16x8; break;
               case Iop_QSub8Sx16:      op = ARM64vecb_SQSUB8x16; break;
               case Iop_QSub64Ux2:      op = ARM64vecb_UQSUB64x2; break;
               case Iop_QSub32Ux4:      op = ARM64vecb_UQSUB32x4; break;
               case Iop_QSub16Ux8:      op = ARM64vecb_UQSUB16x8; break;
               case Iop_QSub8Ux16:      op = ARM64vecb_UQSUB8x16; break;
               case Iop_QDMulHi32Sx4:   op = ARM64vecb_SQDMULH32x4; break;
               case Iop_QDMulHi16Sx8:   op = ARM64vecb_SQDMULH16x8; break;
               case Iop_QRDMulHi32Sx4:  op = ARM64vecb_SQRDMULH32x4; break;
               case Iop_QRDMulHi16Sx8:  op = ARM64vecb_SQRDMULH16x8; break;
               case Iop_Sh8Sx16:        op = ARM64vecb_SSHL8x16; break;
               case Iop_Sh16Sx8:        op = ARM64vecb_SSHL16x8; break;
               case Iop_Sh32Sx4:        op = ARM64vecb_SSHL32x4; break;
               case Iop_Sh64Sx2:        op = ARM64vecb_SSHL64x2; break;
               case Iop_Sh8Ux16:        op = ARM64vecb_USHL8x16; break;
               case Iop_Sh16Ux8:        op = ARM64vecb_USHL16x8; break;
               case Iop_Sh32Ux4:        op = ARM64vecb_USHL32x4; break;
               case Iop_Sh64Ux2:        op = ARM64vecb_USHL64x2; break;
               case Iop_Rsh8Sx16:       op = ARM64vecb_SRSHL8x16; break;
               case Iop_Rsh16Sx8:       op = ARM64vecb_SRSHL16x8; break;
               case Iop_Rsh32Sx4:       op = ARM64vecb_SRSHL32x4; break;
               case Iop_Rsh64Sx2:       op = ARM64vecb_SRSHL64x2; break;
               case Iop_Rsh8Ux16:       op = ARM64vecb_URSHL8x16; break;
               case Iop_Rsh16Ux8:       op = ARM64vecb_URSHL16x8; break;
               case Iop_Rsh32Ux4:       op = ARM64vecb_URSHL32x4; break;
               case Iop_Rsh64Ux2:       op = ARM64vecb_URSHL64x2; break;
               case Iop_Max64Fx2:       op = ARM64vecb_FMAX64x2; break;
               case Iop_Max32Fx4:       op = ARM64vecb_FMAX32x4; break;
               case Iop_Min64Fx2:       op = ARM64vecb_FMIN64x2; break;
               case Iop_Min32Fx4:       op = ARM64vecb_FMIN32x4; break;
               case Iop_RecipStep64Fx2: setRM = True;
                                        op = ARM64vecb_FRECPS64x2; break;
               case Iop_RecipStep32Fx4: setRM = True;
                                        op = ARM64vecb_FRECPS32x4; break;
               case Iop_RSqrtStep64Fx2: setRM = True;
                                        op = ARM64vecb_FRSQRTS64x2; break;
               case Iop_RSqrtStep32Fx4: setRM = True;
                                        op = ARM64vecb_FRSQRTS32x4; break;
               default: vassert(0);
            }
            if (setRM) {
               // This is a bit of a kludge.  We should do rm properly for
               // these recip-step insns, but that would require changing the
               // primop's type to take an rmode.
               set_FPCR_rounding_mode(env, IRExpr_Const(
                                              IRConst_U32(Irrm_NEAREST)));
            }
            if (sw) {
               addInstr(env, ARM64Instr_VBinV(op, res, argR, argL));
            } else {
               addInstr(env, ARM64Instr_VBinV(op, res, argL, argR));
            }
            return res;
         }
         /* -- These only have 2 operand instructions, so we have to first move
            the first argument into a new register, for modification. -- */
         case Iop_QAddExtUSsatSS8x16: case Iop_QAddExtUSsatSS16x8:
         case Iop_QAddExtUSsatSS32x4: case Iop_QAddExtUSsatSS64x2:
         case Iop_QAddExtSUsatUU8x16: case Iop_QAddExtSUsatUU16x8:
         case Iop_QAddExtSUsatUU32x4: case Iop_QAddExtSUsatUU64x2:
         {
            HReg res  = newVRegV(env);
            HReg argL = iselV128Expr(env, e->Iex.Binop.arg1);
            HReg argR = iselV128Expr(env, e->Iex.Binop.arg2);
            ARM64VecModifyOp op = ARM64vecmo_INVALID;
            switch (e->Iex.Binop.op) {
               /* In the following 8 cases, the US - SU switching is intended.
                  See comments on the libvex_ir.h for details.  Also in the 
                  ARM64 front end, where used these primops are generated. */
               case Iop_QAddExtUSsatSS8x16: op = ARM64vecmo_SUQADD8x16; break;
               case Iop_QAddExtUSsatSS16x8: op = ARM64vecmo_SUQADD16x8; break;
               case Iop_QAddExtUSsatSS32x4: op = ARM64vecmo_SUQADD32x4; break;
               case Iop_QAddExtUSsatSS64x2: op = ARM64vecmo_SUQADD64x2; break;
               case Iop_QAddExtSUsatUU8x16: op = ARM64vecmo_USQADD8x16; break;
               case Iop_QAddExtSUsatUU16x8: op = ARM64vecmo_USQADD16x8; break;
               case Iop_QAddExtSUsatUU32x4: op = ARM64vecmo_USQADD32x4; break;
               case Iop_QAddExtSUsatUU64x2: op = ARM64vecmo_USQADD64x2; break;
               default: vassert(0);
            }
            /* The order of the operands is important.  Although this is
               basically addition, the two operands are extended differently,
               making it important to get them into the correct registers in
               the instruction. */
            addInstr(env, ARM64Instr_VMov(16, res, argR));
            addInstr(env, ARM64Instr_VModifyV(op, res, argL));
            return res;
         }
         /* -- Shifts by an immediate. -- */
         case Iop_ShrN64x2: case Iop_ShrN32x4:
         case Iop_ShrN16x8: case Iop_ShrN8x16:
         case Iop_SarN64x2: case Iop_SarN32x4:
         case Iop_SarN16x8: case Iop_SarN8x16:
         case Iop_ShlN64x2: case Iop_ShlN32x4:
         case Iop_ShlN16x8: case Iop_ShlN8x16:
         case Iop_QShlNsatUU64x2: case Iop_QShlNsatUU32x4:
         case Iop_QShlNsatUU16x8: case Iop_QShlNsatUU8x16:
         case Iop_QShlNsatSS64x2: case Iop_QShlNsatSS32x4:
         case Iop_QShlNsatSS16x8: case Iop_QShlNsatSS8x16:
         case Iop_QShlNsatSU64x2: case Iop_QShlNsatSU32x4:
         case Iop_QShlNsatSU16x8: case Iop_QShlNsatSU8x16:
         {
            IRExpr* argL = e->Iex.Binop.arg1;
            IRExpr* argR = e->Iex.Binop.arg2;
            if (argR->tag == Iex_Const && argR->Iex.Const.con->tag == Ico_U8) {
               UInt amt   = argR->Iex.Const.con->Ico.U8;
               UInt limLo = 0;
               UInt limHi = 0;
               ARM64VecShiftImmOp op = ARM64vecshi_INVALID;
               /* Establish the instruction to use. */
               switch (e->Iex.Binop.op) {
                  case Iop_ShrN64x2:       op = ARM64vecshi_USHR64x2;   break;
                  case Iop_ShrN32x4:       op = ARM64vecshi_USHR32x4;   break;
                  case Iop_ShrN16x8:       op = ARM64vecshi_USHR16x8;   break;
                  case Iop_ShrN8x16:       op = ARM64vecshi_USHR8x16;   break;
                  case Iop_SarN64x2:       op = ARM64vecshi_SSHR64x2;   break;
                  case Iop_SarN32x4:       op = ARM64vecshi_SSHR32x4;   break;
                  case Iop_SarN16x8:       op = ARM64vecshi_SSHR16x8;   break;
                  case Iop_SarN8x16:       op = ARM64vecshi_SSHR8x16;   break;
                  case Iop_ShlN64x2:       op = ARM64vecshi_SHL64x2;    break;
                  case Iop_ShlN32x4:       op = ARM64vecshi_SHL32x4;    break;
                  case Iop_ShlN16x8:       op = ARM64vecshi_SHL16x8;    break;
                  case Iop_ShlN8x16:       op = ARM64vecshi_SHL8x16;    break;
                  case Iop_QShlNsatUU64x2: op = ARM64vecshi_UQSHL64x2;  break;
                  case Iop_QShlNsatUU32x4: op = ARM64vecshi_UQSHL32x4;  break;
                  case Iop_QShlNsatUU16x8: op = ARM64vecshi_UQSHL16x8;  break;
                  case Iop_QShlNsatUU8x16: op = ARM64vecshi_UQSHL8x16;  break;
                  case Iop_QShlNsatSS64x2: op = ARM64vecshi_SQSHL64x2;  break;
                  case Iop_QShlNsatSS32x4: op = ARM64vecshi_SQSHL32x4;  break;
                  case Iop_QShlNsatSS16x8: op = ARM64vecshi_SQSHL16x8;  break;
                  case Iop_QShlNsatSS8x16: op = ARM64vecshi_SQSHL8x16;  break;
                  case Iop_QShlNsatSU64x2: op = ARM64vecshi_SQSHLU64x2; break;
                  case Iop_QShlNsatSU32x4: op = ARM64vecshi_SQSHLU32x4; break;
                  case Iop_QShlNsatSU16x8: op = ARM64vecshi_SQSHLU16x8; break;
                  case Iop_QShlNsatSU8x16: op = ARM64vecshi_SQSHLU8x16; break;
                  default: vassert(0);
               }
               /* Establish the shift limits, for sanity check purposes only. */
               switch (e->Iex.Binop.op) {
                  case Iop_ShrN64x2:       limLo = 1; limHi = 64; break;
                  case Iop_ShrN32x4:       limLo = 1; limHi = 32; break;
                  case Iop_ShrN16x8:       limLo = 1; limHi = 16; break;
                  case Iop_ShrN8x16:       limLo = 1; limHi = 8;  break;
                  case Iop_SarN64x2:       limLo = 1; limHi = 64; break;
                  case Iop_SarN32x4:       limLo = 1; limHi = 32; break;
                  case Iop_SarN16x8:       limLo = 1; limHi = 16; break;
                  case Iop_SarN8x16:       limLo = 1; limHi = 8;  break;
                  case Iop_ShlN64x2:       limLo = 0; limHi = 63; break;
                  case Iop_ShlN32x4:       limLo = 0; limHi = 31; break;
                  case Iop_ShlN16x8:       limLo = 0; limHi = 15; break;
                  case Iop_ShlN8x16:       limLo = 0; limHi = 7;  break;
                  case Iop_QShlNsatUU64x2: limLo = 0; limHi = 63; break;
                  case Iop_QShlNsatUU32x4: limLo = 0; limHi = 31; break;
                  case Iop_QShlNsatUU16x8: limLo = 0; limHi = 15; break;
                  case Iop_QShlNsatUU8x16: limLo = 0; limHi = 7;  break;
                  case Iop_QShlNsatSS64x2: limLo = 0; limHi = 63; break;
                  case Iop_QShlNsatSS32x4: limLo = 0; limHi = 31; break;
                  case Iop_QShlNsatSS16x8: limLo = 0; limHi = 15; break;
                  case Iop_QShlNsatSS8x16: limLo = 0; limHi = 7;  break;
                  case Iop_QShlNsatSU64x2: limLo = 0; limHi = 63; break;
                  case Iop_QShlNsatSU32x4: limLo = 0; limHi = 31; break;
                  case Iop_QShlNsatSU16x8: limLo = 0; limHi = 15; break;
                  case Iop_QShlNsatSU8x16: limLo = 0; limHi = 7;  break;
                  default: vassert(0);
               }
               /* For left shifts, the allowable amt values are
                  0 .. lane_bits-1.  For right shifts the allowable
                  values are 1 .. lane_bits. */
               if (op != ARM64vecshi_INVALID && amt >= limLo && amt <= limHi) {
                  HReg src = iselV128Expr(env, argL);
                  HReg dst = newVRegV(env);
                  addInstr(env, ARM64Instr_VShiftImmV(op, dst, src, amt));
                  return dst;
               }
               /* Special case some no-op shifts that the arm64 front end
                  throws at us.  We can't generate any instructions for these,
                  but we don't need to either. */
               switch (e->Iex.Binop.op) {
                  case Iop_ShrN64x2: case Iop_ShrN32x4:
                  case Iop_ShrN16x8: case Iop_ShrN8x16:
                     if (amt == 0) {
                        return iselV128Expr(env, argL);
                     }
                     break;
                  default:
                     break;
               }
               /* otherwise unhandled */
            }
            /* else fall out; this is unhandled */
            break;
         }
         /* -- Saturating narrowing by an immediate -- */
         /* uu */
         case Iop_QandQShrNnarrow16Uto8Ux8:
         case Iop_QandQShrNnarrow32Uto16Ux4:
         case Iop_QandQShrNnarrow64Uto32Ux2:
         /* ss */
         case Iop_QandQSarNnarrow16Sto8Sx8:
         case Iop_QandQSarNnarrow32Sto16Sx4:
         case Iop_QandQSarNnarrow64Sto32Sx2:
         /* su */
         case Iop_QandQSarNnarrow16Sto8Ux8:
         case Iop_QandQSarNnarrow32Sto16Ux4:
         case Iop_QandQSarNnarrow64Sto32Ux2:
         /* ruu */
         case Iop_QandQRShrNnarrow16Uto8Ux8:
         case Iop_QandQRShrNnarrow32Uto16Ux4:
         case Iop_QandQRShrNnarrow64Uto32Ux2:
         /* rss */
         case Iop_QandQRSarNnarrow16Sto8Sx8:
         case Iop_QandQRSarNnarrow32Sto16Sx4:
         case Iop_QandQRSarNnarrow64Sto32Sx2:
         /* rsu */
         case Iop_QandQRSarNnarrow16Sto8Ux8:
         case Iop_QandQRSarNnarrow32Sto16Ux4:
         case Iop_QandQRSarNnarrow64Sto32Ux2:
         {
            IRExpr* argL = e->Iex.Binop.arg1;
            IRExpr* argR = e->Iex.Binop.arg2;
            if (argR->tag == Iex_Const && argR->Iex.Const.con->tag == Ico_U8) {
               UInt amt   = argR->Iex.Const.con->Ico.U8;
               UInt limit = 0;
               ARM64VecShiftImmOp op = ARM64vecshi_INVALID;
               switch (e->Iex.Binop.op) {
                  /* uu */
                  case Iop_QandQShrNnarrow64Uto32Ux2:
                     op = ARM64vecshi_UQSHRN2SD; limit = 64; break;
                  case Iop_QandQShrNnarrow32Uto16Ux4:
                     op = ARM64vecshi_UQSHRN4HS; limit = 32; break;
                  case Iop_QandQShrNnarrow16Uto8Ux8:
                     op = ARM64vecshi_UQSHRN8BH; limit = 16; break;
                  /* ss */
                  case Iop_QandQSarNnarrow64Sto32Sx2:
                     op = ARM64vecshi_SQSHRN2SD; limit = 64; break;
                  case Iop_QandQSarNnarrow32Sto16Sx4:
                     op = ARM64vecshi_SQSHRN4HS; limit = 32; break;
                  case Iop_QandQSarNnarrow16Sto8Sx8:
                     op = ARM64vecshi_SQSHRN8BH; limit = 16; break;
                  /* su */
                  case Iop_QandQSarNnarrow64Sto32Ux2:
                     op = ARM64vecshi_SQSHRUN2SD; limit = 64; break;
                  case Iop_QandQSarNnarrow32Sto16Ux4:
                     op = ARM64vecshi_SQSHRUN4HS; limit = 32; break;
                  case Iop_QandQSarNnarrow16Sto8Ux8:
                     op = ARM64vecshi_SQSHRUN8BH; limit = 16; break;
                  /* ruu */
                  case Iop_QandQRShrNnarrow64Uto32Ux2:
                     op = ARM64vecshi_UQRSHRN2SD; limit = 64; break;
                  case Iop_QandQRShrNnarrow32Uto16Ux4:
                     op = ARM64vecshi_UQRSHRN4HS; limit = 32; break;
                  case Iop_QandQRShrNnarrow16Uto8Ux8:
                     op = ARM64vecshi_UQRSHRN8BH; limit = 16; break;
                  /* rss */
                  case Iop_QandQRSarNnarrow64Sto32Sx2:
                     op = ARM64vecshi_SQRSHRN2SD; limit = 64; break;
                  case Iop_QandQRSarNnarrow32Sto16Sx4:
                     op = ARM64vecshi_SQRSHRN4HS; limit = 32; break;
                  case Iop_QandQRSarNnarrow16Sto8Sx8:
                     op = ARM64vecshi_SQRSHRN8BH; limit = 16; break;
                  /* rsu */
                  case Iop_QandQRSarNnarrow64Sto32Ux2:
                     op = ARM64vecshi_SQRSHRUN2SD; limit = 64; break;
                  case Iop_QandQRSarNnarrow32Sto16Ux4:
                     op = ARM64vecshi_SQRSHRUN4HS; limit = 32; break;
                  case Iop_QandQRSarNnarrow16Sto8Ux8:
                     op = ARM64vecshi_SQRSHRUN8BH; limit = 16; break;
                  /**/
                  default:
                     vassert(0);
               }
               if (op != ARM64vecshi_INVALID && amt >= 1 && amt <= limit) {
                  HReg src  = iselV128Expr(env, argL);
                  HReg dst  = newVRegV(env);
                  HReg fpsr = newVRegI(env);
                  /* Clear FPSR.Q, do the operation, and return both its
                     result and the new value of FPSR.Q.  We can simply
                     zero out FPSR since all the other bits have no relevance
                     in VEX generated code. */
                  addInstr(env, ARM64Instr_Imm64(fpsr, 0));
                  addInstr(env, ARM64Instr_FPSR(True/*toFPSR*/, fpsr));
                  addInstr(env, ARM64Instr_VShiftImmV(op, dst, src, amt));
                  addInstr(env, ARM64Instr_FPSR(False/*!toFPSR*/, fpsr));
                  addInstr(env, ARM64Instr_Shift(fpsr, fpsr, ARM64RI6_I6(27),
                                                             ARM64sh_SHR));
                  ARM64RIL* ril_one = mb_mkARM64RIL_I(1);
                  vassert(ril_one);
                  addInstr(env, ARM64Instr_Logic(fpsr,
                                                 fpsr, ril_one, ARM64lo_AND));
                  /* Now we have: the main (shift) result in the bottom half
                     of |dst|, and the Q bit at the bottom of |fpsr|.  
                     Combining them with a "InterleaveLO64x2" style operation 
                     produces a 128 bit value, dst[63:0]:fpsr[63:0], 
                     which is what we want. */
                  HReg scratch = newVRegV(env);
                  addInstr(env, ARM64Instr_VQfromX(scratch, fpsr));
                  addInstr(env, ARM64Instr_VBinV(ARM64vecb_UZP164x2,
                                                 dst, dst, scratch));
                  return dst;
               }
            }
            /* else fall out; this is unhandled */
            break;
         }

         // Use Iop_SliceV128 in preference to Iop_ShlV128 and Iop_ShrV128,
         // as it is in some ways more general and often leads to better
         // code overall. 
         case Iop_ShlV128:
         case Iop_ShrV128: {
            Bool isSHR = e->Iex.Binop.op == Iop_ShrV128;
            /* This is tricky.  Generate an EXT instruction with zeroes in
               the high operand (shift right) or low operand (shift left).
               Note that we can only slice in the EXT instruction at a byte
               level of granularity, so the shift amount needs careful
               checking. */
            IRExpr* argL = e->Iex.Binop.arg1;
            IRExpr* argR = e->Iex.Binop.arg2;
            if (argR->tag == Iex_Const && argR->Iex.Const.con->tag == Ico_U8) {
               UInt amt   = argR->Iex.Const.con->Ico.U8;
               Bool amtOK = False;
               switch (amt) {
                  case 0x08: case 0x10: case 0x18: case 0x20: case 0x28:
                  case 0x30: case 0x38: case 0x40: case 0x48: case 0x50:
                  case 0x58: case 0x60: case 0x68: case 0x70: case 0x78:
                     amtOK = True; break;
               }
               /* We could also deal with amt==0 by copying the source to
                  the destination, but there's no need for that so far. */
               if (amtOK) {
                  HReg src  = iselV128Expr(env, argL);
                  HReg srcZ = newVRegV(env);
                  addInstr(env, ARM64Instr_VImmQ(srcZ, 0x0000));
                  UInt immB = amt / 8;
                  vassert(immB >= 1 && immB <= 15);
                  HReg dst = newVRegV(env);
                  if (isSHR) {
                    addInstr(env, ARM64Instr_VExtV(dst, src/*lo*/, srcZ/*hi*/,
                                                         immB));
                  } else {
                    addInstr(env, ARM64Instr_VExtV(dst, srcZ/*lo*/, src/*hi*/,
                                                         16 - immB));
                  }
                  return dst;
               }
            }
            /* else fall out; this is unhandled */
            break;
         }

         case Iop_PolynomialMull8x8:
         case Iop_Mull32Ux2:
         case Iop_Mull16Ux4:
         case Iop_Mull8Ux8:
         case Iop_Mull32Sx2:
         case Iop_Mull16Sx4:
         case Iop_Mull8Sx8:
         case Iop_QDMull32Sx2:
         case Iop_QDMull16Sx4:
         {
            HReg iSrcL = iselIntExpr_R(env, e->Iex.Binop.arg1);
            HReg iSrcR = iselIntExpr_R(env, e->Iex.Binop.arg2);
            HReg vSrcL = newVRegV(env);
            HReg vSrcR = newVRegV(env);
            HReg dst   = newVRegV(env);
            ARM64VecBinOp op = ARM64vecb_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_PolynomialMull8x8: op = ARM64vecb_PMULL8x8;    break;
               case Iop_Mull32Ux2:         op = ARM64vecb_UMULL2DSS;   break;
               case Iop_Mull16Ux4:         op = ARM64vecb_UMULL4SHH;   break;
               case Iop_Mull8Ux8:          op = ARM64vecb_UMULL8HBB;   break;
               case Iop_Mull32Sx2:         op = ARM64vecb_SMULL2DSS;   break;
               case Iop_Mull16Sx4:         op = ARM64vecb_SMULL4SHH;   break;
               case Iop_Mull8Sx8:          op = ARM64vecb_SMULL8HBB;   break;
               case Iop_QDMull32Sx2:       op = ARM64vecb_SQDMULL2DSS; break;
               case Iop_QDMull16Sx4:       op = ARM64vecb_SQDMULL4SHH; break;
               default: vassert(0);
            }
            addInstr(env, ARM64Instr_VQfromXX(vSrcL, iSrcL, iSrcL));
            addInstr(env, ARM64Instr_VQfromXX(vSrcR, iSrcR, iSrcR));
            addInstr(env, ARM64Instr_VBinV(op, dst, vSrcL, vSrcR));
            return dst;
         }

         /* ... */
         default:
            break;
      } /* switch on the binop */
   } /* if (e->tag == Iex_Binop) */

   if (e->tag == Iex_Triop) {
      IRTriop*      triop  = e->Iex.Triop.details;
      ARM64VecBinOp vecbop = ARM64vecb_INVALID;
      switch (triop->op) {
         case Iop_Add64Fx2: vecbop = ARM64vecb_FADD64x2; break;
         case Iop_Sub64Fx2: vecbop = ARM64vecb_FSUB64x2; break;
         case Iop_Mul64Fx2: vecbop = ARM64vecb_FMUL64x2; break;
         case Iop_Div64Fx2: vecbop = ARM64vecb_FDIV64x2; break;
         case Iop_Add32Fx4: vecbop = ARM64vecb_FADD32x4; break;
         case Iop_Sub32Fx4: vecbop = ARM64vecb_FSUB32x4; break;
         case Iop_Mul32Fx4: vecbop = ARM64vecb_FMUL32x4; break;
         case Iop_Div32Fx4: vecbop = ARM64vecb_FDIV32x4; break;
         default: break;
      }
      if (vecbop != ARM64vecb_INVALID) {
         HReg argL = iselV128Expr(env, triop->arg2);
         HReg argR = iselV128Expr(env, triop->arg3);
         HReg dst  = newVRegV(env);
         set_FPCR_rounding_mode(env, triop->arg1);
         addInstr(env, ARM64Instr_VBinV(vecbop, dst, argL, argR));
         return dst;
      }

      if (triop->op == Iop_SliceV128) {
         /* Note that, compared to ShlV128/ShrV128 just above, the shift
            amount here is in bytes, not bits. */
         IRExpr* argHi  = triop->arg1;
         IRExpr* argLo  = triop->arg2;
         IRExpr* argAmt = triop->arg3;
         if (argAmt->tag == Iex_Const && argAmt->Iex.Const.con->tag == Ico_U8) {
            UInt amt   = argAmt->Iex.Const.con->Ico.U8;
            Bool amtOK = amt >= 1 && amt <= 15;
            /* We could also deal with amt==0 by copying argLO to
               the destination, but there's no need for that so far. */
            if (amtOK) {
               HReg srcHi = iselV128Expr(env, argHi);
               HReg srcLo = iselV128Expr(env, argLo);
               HReg dst = newVRegV(env);
              addInstr(env, ARM64Instr_VExtV(dst, srcLo, srcHi, amt));
               return dst;
            }
         }
         /* else fall out; this is unhandled */
      }

   } /* if (e->tag == Iex_Triop) */

  v128_expr_bad:
   ppIRExpr(e);
   vpanic("iselV128Expr_wrk");
}


/*---------------------------------------------------------*/
/*--- ISEL: Floating point expressions (64 bit)         ---*/
/*---------------------------------------------------------*/

/* Compute a 64-bit floating point value into a register, the identity
   of which is returned.  As with iselIntExpr_R, the reg may be either
   real or virtual; in any case it must not be changed by subsequent
   code emitted by the caller.  */

static HReg iselDblExpr ( ISelEnv* env, IRExpr* e )
{
   HReg r = iselDblExpr_wrk( env, e );
#  if 0
   vex_printf("\n"); ppIRExpr(e); vex_printf("\n");
#  endif
   vassert(hregClass(r) == HRcFlt64);
   vassert(hregIsVirtual(r));
   return r;
}

/* DO NOT CALL THIS DIRECTLY */
static HReg iselDblExpr_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(e);
   vassert(ty == Ity_F64);

   if (e->tag == Iex_RdTmp) {
      return lookupIRTemp(env, e->Iex.RdTmp.tmp);
   }

   if (e->tag == Iex_Const) {
      IRConst* con = e->Iex.Const.con;
      if (con->tag == Ico_F64i) {
         HReg src = newVRegI(env);
         HReg dst = newVRegD(env);
         addInstr(env, ARM64Instr_Imm64(src, con->Ico.F64i));
         addInstr(env, ARM64Instr_VDfromX(dst, src));
         return dst;
      }
      if (con->tag == Ico_F64) {
         HReg src = newVRegI(env);
         HReg dst = newVRegD(env);
         union { Double d64; ULong u64; } u;
         vassert(sizeof(u) == 8);
         u.d64 = con->Ico.F64;
         addInstr(env, ARM64Instr_Imm64(src, u.u64));
         addInstr(env, ARM64Instr_VDfromX(dst, src));
         return dst;
      }
   }

   if (e->tag == Iex_Load && e->Iex.Load.end == Iend_LE) {
      vassert(e->Iex.Load.ty == Ity_F64);
      HReg addr = iselIntExpr_R(env, e->Iex.Load.addr);
      HReg res  = newVRegD(env);
      addInstr(env, ARM64Instr_VLdStD(True/*isLoad*/, res, addr, 0));
      return res;
   }

   if (e->tag == Iex_Get) {
      Int offs = e->Iex.Get.offset;
      if (offs >= 0 && offs < 32768 && 0 == (offs & 7)) {
         HReg rD = newVRegD(env);
         HReg rN = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStD(True/*isLoad*/, rD, rN, offs));
         return rD;
      }
   }

   if (e->tag == Iex_Unop) {
      switch (e->Iex.Unop.op) {
         case Iop_NegF64: {
            HReg src = iselDblExpr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VUnaryD(ARM64fpu_NEG, dst, src));
            return dst;
         }
         case Iop_AbsF64: {
            HReg src = iselDblExpr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VUnaryD(ARM64fpu_ABS, dst, src));
            return dst;
         }
         case Iop_F32toF64: {
            HReg src = iselFltExpr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtSD(True/*sToD*/, dst, src));
            return dst;
         }
         case Iop_F16toF64: {
            HReg src = iselF16Expr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtHD(True/*hToD*/, dst, src));
            return dst;
         }
         case Iop_I32UtoF64:
         case Iop_I32StoF64: {
            /* Rounding mode is not involved here, since the
               conversion can always be done without loss of
               precision. */
            HReg src   = iselIntExpr_R(env, e->Iex.Unop.arg);
            HReg dst   = newVRegD(env);
            Bool syned = e->Iex.Unop.op == Iop_I32StoF64;
            ARM64CvtOp cvt_op = syned ? ARM64cvt_F64_I32S : ARM64cvt_F64_I32U;
            addInstr(env, ARM64Instr_VCvtI2F(cvt_op, dst, src));
            return dst;
         }
         default:
            break;
      }
   }

   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         case Iop_RoundF64toInt:
         case Iop_SqrtF64:
         case Iop_RecpExpF64: {
            HReg src = iselDblExpr(env, e->Iex.Binop.arg2);
            HReg dst = newVRegD(env);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            ARM64FpUnaryOp op = ARM64fpu_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_RoundF64toInt: op = ARM64fpu_RINT;  break;
               case Iop_SqrtF64:       op = ARM64fpu_SQRT;  break;
               case Iop_RecpExpF64:    op = ARM64fpu_RECPX; break;
               default: vassert(0);
            }
            addInstr(env, ARM64Instr_VUnaryD(op, dst, src));
            return dst;
         }
         case Iop_I64StoF64:
         case Iop_I64UtoF64: {
            ARM64CvtOp cvt_op = e->Iex.Binop.op == Iop_I64StoF64
                                   ? ARM64cvt_F64_I64S : ARM64cvt_F64_I64U;
            HReg srcI = iselIntExpr_R(env, e->Iex.Binop.arg2);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            HReg dstS = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtI2F(cvt_op, dstS, srcI));
            return dstS;
         }
         default:
            break;
      }
   }

   if (e->tag == Iex_Triop) {
      IRTriop*     triop = e->Iex.Triop.details;
      ARM64FpBinOp dblop = ARM64fpb_INVALID;
      switch (triop->op) {
         case Iop_DivF64: dblop = ARM64fpb_DIV; break;
         case Iop_MulF64: dblop = ARM64fpb_MUL; break;
         case Iop_SubF64: dblop = ARM64fpb_SUB; break;
         case Iop_AddF64: dblop = ARM64fpb_ADD; break;
         default: break;
      }
      if (dblop != ARM64fpb_INVALID) {
         HReg argL = iselDblExpr(env, triop->arg2);
         HReg argR = iselDblExpr(env, triop->arg3);
         HReg dst  = newVRegD(env);
         set_FPCR_rounding_mode(env, triop->arg1);
         addInstr(env, ARM64Instr_VBinD(dblop, dst, argL, argR));
         return dst;
      }
   }

   if (e->tag == Iex_ITE) {
      /* ITE(ccexpr, iftrue, iffalse) */
      ARM64CondCode cc;
      HReg r1  = iselDblExpr(env, e->Iex.ITE.iftrue);
      HReg r0  = iselDblExpr(env, e->Iex.ITE.iffalse);
      HReg dst = newVRegD(env);
      cc = iselCondCode(env, e->Iex.ITE.cond);
      addInstr(env, ARM64Instr_VFCSel(dst, r1, r0, cc, True/*64-bit*/));
      return dst;
   }

   ppIRExpr(e);
   vpanic("iselDblExpr_wrk");
}


/*---------------------------------------------------------*/
/*--- ISEL: Floating point expressions (32 bit)         ---*/
/*---------------------------------------------------------*/

/* Compute a 32-bit floating point value into a register, the identity
   of which is returned.  As with iselIntExpr_R, the reg may be either
   real or virtual; in any case it must not be changed by subsequent
   code emitted by the caller.  Values are generated into HRcFlt64
   registers despite the values themselves being Ity_F32s. */

static HReg iselFltExpr ( ISelEnv* env, IRExpr* e )
{
   HReg r = iselFltExpr_wrk( env, e );
#  if 0
   vex_printf("\n"); ppIRExpr(e); vex_printf("\n");
#  endif
   vassert(hregClass(r) == HRcFlt64);
   vassert(hregIsVirtual(r));
   return r;
}

/* DO NOT CALL THIS DIRECTLY */
static HReg iselFltExpr_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(e);
   vassert(ty == Ity_F32);

   if (e->tag == Iex_RdTmp) {
      return lookupIRTemp(env, e->Iex.RdTmp.tmp);
   }

   if (e->tag == Iex_Const) {
      /* This is something of a kludge.  Since a 32 bit floating point
         zero is just .. all zeroes, just create a 64 bit zero word
         and transfer it.  This avoids having to create a SfromW
         instruction for this specific case. */
      IRConst* con = e->Iex.Const.con;
      if (con->tag == Ico_F32i && con->Ico.F32i == 0) {
         HReg src = newVRegI(env);
         HReg dst = newVRegD(env);
         addInstr(env, ARM64Instr_Imm64(src, 0));
         addInstr(env, ARM64Instr_VDfromX(dst, src));
         return dst;
      }
      if (con->tag == Ico_F32) {
         HReg src = newVRegI(env);
         HReg dst = newVRegD(env);
         union { Float f32; UInt u32; } u;
         vassert(sizeof(u) == 4);
         u.f32 = con->Ico.F32;
         addInstr(env, ARM64Instr_Imm64(src, (ULong)u.u32));
         addInstr(env, ARM64Instr_VDfromX(dst, src));
         return dst;
      }
   }

   if (e->tag == Iex_Load && e->Iex.Load.end == Iend_LE) {
      vassert(e->Iex.Load.ty == Ity_F32);
      HReg addr = iselIntExpr_R(env, e->Iex.Load.addr);
      HReg res  = newVRegD(env);
      addInstr(env, ARM64Instr_VLdStS(True/*isLoad*/, res, addr, 0));
      return res;
   }

   if (e->tag == Iex_Get) {
      Int offs = e->Iex.Get.offset;
      if (offs >= 0 && offs < 16384 && 0 == (offs & 3)) {
         HReg rD = newVRegD(env);
         HReg rN = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStS(True/*isLoad*/, rD, rN, offs));
         return rD;
      }
   }

   if (e->tag == Iex_Unop) {
      switch (e->Iex.Unop.op) {
         case Iop_NegF32: {
            HReg src = iselFltExpr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VUnaryS(ARM64fpu_NEG, dst, src));
            return dst;
         }
         case Iop_AbsF32: {
            HReg src = iselFltExpr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VUnaryS(ARM64fpu_ABS, dst, src));
            return dst;
         }
         case Iop_F16toF32: {
            HReg src = iselF16Expr(env, e->Iex.Unop.arg);
            HReg dst = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtHS(True/*hToS*/, dst, src));
            return dst;
         }
         default:
            break;
      }
   }

   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         case Iop_RoundF32toInt:
         case Iop_SqrtF32:
         case Iop_RecpExpF32: {
            HReg src = iselFltExpr(env, e->Iex.Binop.arg2);
            HReg dst = newVRegD(env);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            ARM64FpUnaryOp op = ARM64fpu_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_RoundF32toInt: op = ARM64fpu_RINT;  break;
               case Iop_SqrtF32:       op = ARM64fpu_SQRT;  break;
               case Iop_RecpExpF32:    op = ARM64fpu_RECPX; break;
               default: vassert(0);
            }
            addInstr(env, ARM64Instr_VUnaryS(op, dst, src));
            return dst;
         }
         case Iop_F64toF32: {
            HReg srcD = iselDblExpr(env, e->Iex.Binop.arg2);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            HReg dstS = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtSD(False/*!sToD*/, dstS, srcD));
            return dstS;
         }
         case Iop_I32UtoF32:
         case Iop_I32StoF32:
         case Iop_I64UtoF32:
         case Iop_I64StoF32: {
            ARM64CvtOp cvt_op = ARM64cvt_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_I32UtoF32: cvt_op = ARM64cvt_F32_I32U; break;
               case Iop_I32StoF32: cvt_op = ARM64cvt_F32_I32S; break;
               case Iop_I64UtoF32: cvt_op = ARM64cvt_F32_I64U; break;
               case Iop_I64StoF32: cvt_op = ARM64cvt_F32_I64S; break;
               default: vassert(0);
            }
            HReg srcI = iselIntExpr_R(env, e->Iex.Binop.arg2);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            HReg dstS = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtI2F(cvt_op, dstS, srcI));
            return dstS;
         }
         default:
            break;
      }
   }

   if (e->tag == Iex_Triop) {
      IRTriop*     triop = e->Iex.Triop.details;
      ARM64FpBinOp sglop = ARM64fpb_INVALID;
      switch (triop->op) {
         case Iop_DivF32: sglop = ARM64fpb_DIV; break;
         case Iop_MulF32: sglop = ARM64fpb_MUL; break;
         case Iop_SubF32: sglop = ARM64fpb_SUB; break;
         case Iop_AddF32: sglop = ARM64fpb_ADD; break;
         default: break;
      }
      if (sglop != ARM64fpb_INVALID) {
         HReg argL = iselFltExpr(env, triop->arg2);
         HReg argR = iselFltExpr(env, triop->arg3);
         HReg dst  = newVRegD(env);
         set_FPCR_rounding_mode(env, triop->arg1);
         addInstr(env, ARM64Instr_VBinS(sglop, dst, argL, argR));
         return dst;
      }
   }

   if (e->tag == Iex_ITE) {
      /* ITE(ccexpr, iftrue, iffalse) */
      ARM64CondCode cc;
      HReg r1  = iselFltExpr(env, e->Iex.ITE.iftrue);
      HReg r0  = iselFltExpr(env, e->Iex.ITE.iffalse);
      HReg dst = newVRegD(env);
      cc = iselCondCode(env, e->Iex.ITE.cond);
      addInstr(env, ARM64Instr_VFCSel(dst, r1, r0, cc, False/*!64-bit*/));
      return dst;
   }

   ppIRExpr(e);
   vpanic("iselFltExpr_wrk");
}


/*---------------------------------------------------------*/
/*--- ISEL: Floating point expressions (16 bit)         ---*/
/*---------------------------------------------------------*/

/* Compute a 16-bit floating point value into a register, the identity
   of which is returned.  As with iselIntExpr_R, the reg may be either
   real or virtual; in any case it must not be changed by subsequent
   code emitted by the caller.  Values are generated into HRcFlt64
   registers despite the values themselves being Ity_F16s. */

static HReg iselF16Expr ( ISelEnv* env, IRExpr* e )
{
   HReg r = iselF16Expr_wrk( env, e );
#  if 0
   vex_printf("\n"); ppIRExpr(e); vex_printf("\n");
#  endif
   vassert(hregClass(r) == HRcFlt64);
   vassert(hregIsVirtual(r));
   return r;
}

/* DO NOT CALL THIS DIRECTLY */
static HReg iselF16Expr_wrk ( ISelEnv* env, IRExpr* e )
{
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(e);
   vassert(ty == Ity_F16);

   if (e->tag == Iex_Get) {
      Int offs = e->Iex.Get.offset;
      if (offs >= 0 && offs < 8192 && 0 == (offs & 1)) {
         HReg rD = newVRegD(env);
         HReg rN = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStH(True/*isLoad*/, rD, rN, offs));
         return rD;
      }
   }

   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         case Iop_F32toF16: {
            HReg srcS = iselFltExpr(env, e->Iex.Binop.arg2);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            HReg dstH = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtHS(False/*!hToS*/, dstH, srcS));
            return dstH;
         }
         case Iop_F64toF16: {
            HReg srcD = iselDblExpr(env, e->Iex.Binop.arg2);
            set_FPCR_rounding_mode(env, e->Iex.Binop.arg1);
            HReg dstH = newVRegD(env);
            addInstr(env, ARM64Instr_VCvtHD(False/*!hToD*/, dstH, srcD));
            return dstH;
         }
         default:
            break;
      }
   }

   ppIRExpr(e);
   vpanic("iselF16Expr_wrk");
}


/*---------------------------------------------------------*/
/*--- ISEL: Vector expressions (256 bit)                ---*/
/*---------------------------------------------------------*/

static void iselV256Expr ( /*OUT*/HReg* rHi, HReg* rLo, 
                           ISelEnv* env, IRExpr* e )
{
   iselV256Expr_wrk( rHi, rLo, env, e );
   vassert(hregClass(*rHi) == HRcVec128);
   vassert(hregClass(*rLo) == HRcVec128);
   vassert(hregIsVirtual(*rHi));
   vassert(hregIsVirtual(*rLo));
}

/* DO NOT CALL THIS DIRECTLY */
static void iselV256Expr_wrk ( /*OUT*/HReg* rHi, /*OUT*/HReg* rLo, 
                               ISelEnv* env, IRExpr* e )
{
   vassert(e);
   IRType ty = typeOfIRExpr(env->type_env,e);
   vassert(ty == Ity_V256);

   /* read 256-bit IRTemp */
   if (e->tag == Iex_RdTmp) {
      lookupIRTempPair( rHi, rLo, env, e->Iex.RdTmp.tmp);
      return;
   }
 
   if (e->tag == Iex_Binop) {
      switch (e->Iex.Binop.op) {
         case Iop_V128HLtoV256: {
            *rHi = iselV128Expr(env, e->Iex.Binop.arg1);
            *rLo = iselV128Expr(env, e->Iex.Binop.arg2);
            return;
         }
         case Iop_QandSQsh64x2:
         case Iop_QandSQsh32x4:
         case Iop_QandSQsh16x8:
         case Iop_QandSQsh8x16:
         case Iop_QandUQsh64x2:
         case Iop_QandUQsh32x4:
         case Iop_QandUQsh16x8:
         case Iop_QandUQsh8x16:
         case Iop_QandSQRsh64x2:
         case Iop_QandSQRsh32x4:
         case Iop_QandSQRsh16x8:
         case Iop_QandSQRsh8x16:
         case Iop_QandUQRsh64x2:
         case Iop_QandUQRsh32x4:
         case Iop_QandUQRsh16x8:
         case Iop_QandUQRsh8x16:
         {
            HReg argL  = iselV128Expr(env, e->Iex.Binop.arg1);
            HReg argR  = iselV128Expr(env, e->Iex.Binop.arg2);
            HReg fpsr  = newVRegI(env);
            HReg resHi = newVRegV(env);
            HReg resLo = newVRegV(env);
            ARM64VecBinOp op = ARM64vecb_INVALID;
            switch (e->Iex.Binop.op) {
               case Iop_QandSQsh64x2:  op = ARM64vecb_SQSHL64x2;  break;
               case Iop_QandSQsh32x4:  op = ARM64vecb_SQSHL32x4;  break;
               case Iop_QandSQsh16x8:  op = ARM64vecb_SQSHL16x8;  break;
               case Iop_QandSQsh8x16:  op = ARM64vecb_SQSHL8x16;  break;
               case Iop_QandUQsh64x2:  op = ARM64vecb_UQSHL64x2;  break;
               case Iop_QandUQsh32x4:  op = ARM64vecb_UQSHL32x4;  break;
               case Iop_QandUQsh16x8:  op = ARM64vecb_UQSHL16x8;  break;
               case Iop_QandUQsh8x16:  op = ARM64vecb_UQSHL8x16;  break;
               case Iop_QandSQRsh64x2: op = ARM64vecb_SQRSHL64x2; break;
               case Iop_QandSQRsh32x4: op = ARM64vecb_SQRSHL32x4; break;
               case Iop_QandSQRsh16x8: op = ARM64vecb_SQRSHL16x8; break;
               case Iop_QandSQRsh8x16: op = ARM64vecb_SQRSHL8x16; break;
               case Iop_QandUQRsh64x2: op = ARM64vecb_UQRSHL64x2; break;
               case Iop_QandUQRsh32x4: op = ARM64vecb_UQRSHL32x4; break;
               case Iop_QandUQRsh16x8: op = ARM64vecb_UQRSHL16x8; break;
               case Iop_QandUQRsh8x16: op = ARM64vecb_UQRSHL8x16; break;
               default: vassert(0);
            }
            /* Clear FPSR.Q, do the operation, and return both its result
               and the new value of FPSR.Q.  We can simply zero out FPSR
               since all the other bits have no relevance in VEX generated
               code. */
            addInstr(env, ARM64Instr_Imm64(fpsr, 0));
            addInstr(env, ARM64Instr_FPSR(True/*toFPSR*/, fpsr));
            addInstr(env, ARM64Instr_VBinV(op, resLo, argL, argR));
            addInstr(env, ARM64Instr_FPSR(False/*!toFPSR*/, fpsr));
            addInstr(env, ARM64Instr_Shift(fpsr, fpsr, ARM64RI6_I6(27),
                                                       ARM64sh_SHR));
            ARM64RIL* ril_one = mb_mkARM64RIL_I(1);
            vassert(ril_one);
            addInstr(env, ARM64Instr_Logic(fpsr, fpsr, ril_one, ARM64lo_AND));
            /* Now we have: the main (shift) result in |resLo|, and the
               Q bit at the bottom of |fpsr|. */
            addInstr(env, ARM64Instr_VQfromX(resHi, fpsr));
            *rHi = resHi;
            *rLo = resLo;
            return;
         }

         /* ... */
         default:
            break;
      } /* switch on the binop */
   } /* if (e->tag == Iex_Binop) */

   ppIRExpr(e);
   vpanic("iselV256Expr_wrk");
}


/*---------------------------------------------------------*/
/*--- ISEL: Statements                                  ---*/
/*---------------------------------------------------------*/

static void iselStmt ( ISelEnv* env, IRStmt* stmt )
{
   if (vex_traceflags & VEX_TRACE_VCODE) {
      vex_printf("\n-- ");
      ppIRStmt(stmt);
      vex_printf("\n");
   }
   switch (stmt->tag) {

   /* --------- STORE --------- */
   /* little-endian write to memory */
   case Ist_Store: {
      IRType    tya  = typeOfIRExpr(env->type_env, stmt->Ist.Store.addr);
      IRType    tyd  = typeOfIRExpr(env->type_env, stmt->Ist.Store.data);
      IREndness end  = stmt->Ist.Store.end;

      if (tya != Ity_I64 || end != Iend_LE) 
         goto stmt_fail;

      if (tyd == Ity_I64) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Store.data);
         ARM64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr, tyd);
         addInstr(env, ARM64Instr_LdSt64(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I32) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Store.data);
         ARM64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr, tyd);
         addInstr(env, ARM64Instr_LdSt32(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I16) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Store.data);
         ARM64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr, tyd);
         addInstr(env, ARM64Instr_LdSt16(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I8) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Store.data);
         ARM64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr, tyd);
         addInstr(env, ARM64Instr_LdSt8(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_V128) {
         HReg qD   = iselV128Expr(env, stmt->Ist.Store.data);
         HReg addr = iselIntExpr_R(env, stmt->Ist.Store.addr);
         addInstr(env, ARM64Instr_VLdStQ(False/*!isLoad*/, qD, addr));
         return;
      }
      if (tyd == Ity_F64) {
         HReg dD   = iselDblExpr(env, stmt->Ist.Store.data);
         HReg addr = iselIntExpr_R(env, stmt->Ist.Store.addr);
         addInstr(env, ARM64Instr_VLdStD(False/*!isLoad*/, dD, addr, 0));
         return;
      }
      if (tyd == Ity_F32) {
         HReg sD   = iselFltExpr(env, stmt->Ist.Store.data);
         HReg addr = iselIntExpr_R(env, stmt->Ist.Store.addr);
         addInstr(env, ARM64Instr_VLdStS(False/*!isLoad*/, sD, addr, 0));
         return;
      }
      break;
   }

   /* --------- PUT --------- */
   /* write guest state, fixed offset */
   case Ist_Put: {
      IRType tyd  = typeOfIRExpr(env->type_env, stmt->Ist.Put.data);
      UInt   offs = (UInt)stmt->Ist.Put.offset;
      if (tyd == Ity_I64 && 0 == (offs & 7) && offs < (8<<12)) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Put.data);
         ARM64AMode* am = mk_baseblock_64bit_access_amode(offs);
         addInstr(env, ARM64Instr_LdSt64(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I32 && 0 == (offs & 3) && offs < (4<<12)) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Put.data);
         ARM64AMode* am = mk_baseblock_32bit_access_amode(offs);
         addInstr(env, ARM64Instr_LdSt32(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I16 && 0 == (offs & 1) && offs < (2<<12)) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Put.data);
         ARM64AMode* am = mk_baseblock_16bit_access_amode(offs);
         addInstr(env, ARM64Instr_LdSt16(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_I8 && offs < (1<<12)) {
         HReg        rD = iselIntExpr_R(env, stmt->Ist.Put.data);
         ARM64AMode* am = mk_baseblock_8bit_access_amode(offs);
         addInstr(env, ARM64Instr_LdSt8(False/*!isLoad*/, rD, am));
         return;
      }
      if (tyd == Ity_V128 && offs < (1<<12)) {
         HReg qD   = iselV128Expr(env, stmt->Ist.Put.data);
         HReg addr = mk_baseblock_128bit_access_addr(env, offs);
         addInstr(env, ARM64Instr_VLdStQ(False/*!isLoad*/, qD, addr));
         return;
      }
      if (tyd == Ity_F64 && 0 == (offs & 7) && offs < (8<<12)) {
         HReg dD   = iselDblExpr(env, stmt->Ist.Put.data);
         HReg bbp  = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStD(False/*!isLoad*/, dD, bbp, offs));
         return;
      }
      if (tyd == Ity_F32 && 0 == (offs & 3) && offs < (4<<12)) {
         HReg sD   = iselFltExpr(env, stmt->Ist.Put.data);
         HReg bbp  = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStS(False/*!isLoad*/, sD, bbp, offs));
         return;
      }
      if (tyd == Ity_F16 && 0 == (offs & 1) && offs < (2<<12)) {
         HReg hD   = iselF16Expr(env, stmt->Ist.Put.data);
         HReg bbp  = get_baseblock_register();
         addInstr(env, ARM64Instr_VLdStH(False/*!isLoad*/, hD, bbp, offs));
         return;
      }

      break;
   }

   /* --------- TMP --------- */
   /* assign value to temporary */
   case Ist_WrTmp: {
      IRTemp tmp = stmt->Ist.WrTmp.tmp;
      IRType ty  = typeOfIRTemp(env->type_env, tmp);

      if (ty == Ity_I64 || ty == Ity_I32 || ty == Ity_I16 || ty == Ity_I8) {
         /* We could do a lot better here.  But for the time being: */
         HReg dst = lookupIRTemp(env, tmp);
         HReg rD  = iselIntExpr_R(env, stmt->Ist.WrTmp.data);
         addInstr(env, ARM64Instr_MovI(dst, rD));
         return;
      }
      if (ty == Ity_I1) {
         /* Here, we are generating a I1 value into a 64 bit register.
            Make sure the value in the register is only zero or one,
            but no other.  This allows optimisation of the
            1Uto64(tmp:I1) case, by making it simply a copy of the
            register holding 'tmp'.  The point being that the value in
            the register holding 'tmp' can only have been created
            here.  LATER: that seems dangerous; safer to do 'tmp & 1'
            in that case.  Also, could do this just with a single CINC
            insn. */
         /* CLONE-01 */
         HReg zero = newVRegI(env);
         HReg one  = newVRegI(env);
         HReg dst  = lookupIRTemp(env, tmp);
         addInstr(env, ARM64Instr_Imm64(zero, 0));
         addInstr(env, ARM64Instr_Imm64(one,  1));
         ARM64CondCode cc = iselCondCode(env, stmt->Ist.WrTmp.data);
         addInstr(env, ARM64Instr_CSel(dst, one, zero, cc));
         return;
      }
      if (ty == Ity_F64) {
         HReg src = iselDblExpr(env, stmt->Ist.WrTmp.data);
         HReg dst = lookupIRTemp(env, tmp);
         addInstr(env, ARM64Instr_VMov(8, dst, src));
         return;
      }
      if (ty == Ity_F32) {
         HReg src = iselFltExpr(env, stmt->Ist.WrTmp.data);
         HReg dst = lookupIRTemp(env, tmp);
         addInstr(env, ARM64Instr_VMov(8/*yes, really*/, dst, src));
         return;
      }
      if (ty == Ity_V128) {
         HReg src = iselV128Expr(env, stmt->Ist.WrTmp.data);
         HReg dst = lookupIRTemp(env, tmp);
         addInstr(env, ARM64Instr_VMov(16, dst, src));
         return;
      }
      if (ty == Ity_V256) {
         HReg srcHi, srcLo, dstHi, dstLo;
         iselV256Expr(&srcHi,&srcLo, env, stmt->Ist.WrTmp.data);
         lookupIRTempPair( &dstHi, &dstLo, env, tmp);
         addInstr(env, ARM64Instr_VMov(16, dstHi, srcHi));
         addInstr(env, ARM64Instr_VMov(16, dstLo, srcLo));
         return;
      }
      break;
   }

   /* --------- Call to DIRTY helper --------- */
   /* call complex ("dirty") helper function */
   case Ist_Dirty: {
      IRDirty* d = stmt->Ist.Dirty.details;

      /* Figure out the return type, if any. */
      IRType retty = Ity_INVALID;
      if (d->tmp != IRTemp_INVALID)
         retty = typeOfIRTemp(env->type_env, d->tmp);

      Bool retty_ok = False;
      switch (retty) {
         case Ity_INVALID: /* function doesn't return anything */
         case Ity_I64: case Ity_I32: case Ity_I16: case Ity_I8:
         case Ity_V128:
            retty_ok = True; break;
         default:
            break;
      }
      if (!retty_ok)
         break; /* will go to stmt_fail: */

      /* Marshal args, do the call, and set the return value to 0x555..555
         if this is a conditional call that returns a value and the
         call is skipped. */
      UInt   addToSp = 0;
      RetLoc rloc    = mk_RetLoc_INVALID();
      doHelperCall( &addToSp, &rloc, env, d->guard, d->cee, retty, d->args );
      vassert(is_sane_RetLoc(rloc));

      /* Now figure out what to do with the returned value, if any. */
      switch (retty) {
         case Ity_INVALID: {
            /* No return value.  Nothing to do. */
            vassert(d->tmp == IRTemp_INVALID);
            vassert(rloc.pri == RLPri_None);
            vassert(addToSp == 0);
            return;
         }
         case Ity_I64: case Ity_I32: case Ity_I16: case Ity_I8: {
            vassert(rloc.pri == RLPri_Int);
            vassert(addToSp == 0);
            /* The returned value is in x0.  Park it in the register
               associated with tmp. */
            HReg dst = lookupIRTemp(env, d->tmp);
            addInstr(env, ARM64Instr_MovI(dst, hregARM64_X0()) );
            return;
         }
         case Ity_V128: {
            /* The returned value is on the stack, and *retloc tells
               us where.  Fish it off the stack and then move the
               stack pointer upwards to clear it, as directed by
               doHelperCall. */
            vassert(rloc.pri == RLPri_V128SpRel);
            vassert(rloc.spOff < 256); // stay sane
            vassert(addToSp >= 16); // ditto
            vassert(addToSp < 256); // ditto
            HReg dst = lookupIRTemp(env, d->tmp);
            HReg tmp = newVRegI(env); // the address of the returned value
            addInstr(env, ARM64Instr_FromSP(tmp)); // tmp = SP
            addInstr(env, ARM64Instr_Arith(tmp, tmp,
                                           ARM64RIA_I12((UShort)rloc.spOff, 0),
                                           True/*isAdd*/ ));
            addInstr(env, ARM64Instr_VLdStQ(True/*isLoad*/, dst, tmp));
            addInstr(env, ARM64Instr_AddToSP(addToSp));
            return;
         }
         default:
            /*NOTREACHED*/
            vassert(0);
      }
      break;
   }

   /* --------- Load Linked and Store Conditional --------- */
   case Ist_LLSC: {
      if (stmt->Ist.LLSC.storedata == NULL) {
         /* LL */
         IRTemp res = stmt->Ist.LLSC.result;
         IRType ty  = typeOfIRTemp(env->type_env, res);
         if (ty == Ity_I64 || ty == Ity_I32 
             || ty == Ity_I16 || ty == Ity_I8) {
            Int  szB   = 0;
            HReg r_dst = lookupIRTemp(env, res);
            HReg raddr = iselIntExpr_R(env, stmt->Ist.LLSC.addr);
            switch (ty) {
               case Ity_I8:  szB = 1; break;
               case Ity_I16: szB = 2; break;
               case Ity_I32: szB = 4; break;
               case Ity_I64: szB = 8; break;
               default:      vassert(0);
            }
            addInstr(env, ARM64Instr_MovI(hregARM64_X4(), raddr));
            addInstr(env, ARM64Instr_LdrEX(szB));
            addInstr(env, ARM64Instr_MovI(r_dst, hregARM64_X2()));
            return;
         }
         goto stmt_fail;
      } else {
         /* SC */
         IRType tyd = typeOfIRExpr(env->type_env, stmt->Ist.LLSC.storedata);
         if (tyd == Ity_I64 || tyd == Ity_I32
             || tyd == Ity_I16 || tyd == Ity_I8) {
            Int  szB = 0;
            HReg rD  = iselIntExpr_R(env, stmt->Ist.LLSC.storedata);
            HReg rA  = iselIntExpr_R(env, stmt->Ist.LLSC.addr);
            switch (tyd) {
               case Ity_I8:  szB = 1; break;
               case Ity_I16: szB = 2; break;
               case Ity_I32: szB = 4; break;
               case Ity_I64: szB = 8; break;
               default:      vassert(0);
            }
            addInstr(env, ARM64Instr_MovI(hregARM64_X2(), rD));
            addInstr(env, ARM64Instr_MovI(hregARM64_X4(), rA));
            addInstr(env, ARM64Instr_StrEX(szB));
         } else {
            goto stmt_fail;
         }
         /* now r0 is 1 if failed, 0 if success.  Change to IR
            conventions (0 is fail, 1 is success).  Also transfer
            result to r_res. */
         IRTemp    res   = stmt->Ist.LLSC.result;
         IRType    ty    = typeOfIRTemp(env->type_env, res);
         HReg      r_res = lookupIRTemp(env, res);
         ARM64RIL* one   = mb_mkARM64RIL_I(1);
         vassert(ty == Ity_I1);
         vassert(one);
         addInstr(env, ARM64Instr_Logic(r_res, hregARM64_X0(), one,
                                        ARM64lo_XOR));
         /* And be conservative -- mask off all but the lowest bit. */
         addInstr(env, ARM64Instr_Logic(r_res, r_res, one,
                                        ARM64lo_AND));
         return;
      }
      break;
   }

   /* --------- ACAS --------- */
   case Ist_CAS: {
      if (stmt->Ist.CAS.details->oldHi == IRTemp_INVALID) {
         /* "normal" singleton CAS */
         UChar  sz;
         IRCAS* cas = stmt->Ist.CAS.details;
         IRType ty  = typeOfIRExpr(env->type_env, cas->dataLo);
         switch (ty) { 
            case Ity_I64: sz = 8; break;
            case Ity_I32: sz = 4; break;
            case Ity_I16: sz = 2; break;
            case Ity_I8:  sz = 1; break; 
            default: goto unhandled_cas;
         }
         HReg rAddr = iselIntExpr_R(env, cas->addr);
         HReg rExpd = iselIntExpr_R(env, cas->expdLo);
         HReg rData = iselIntExpr_R(env, cas->dataLo);
         vassert(cas->expdHi == NULL);
         vassert(cas->dataHi == NULL);
         addInstr(env, ARM64Instr_MovI(hregARM64_X3(), rAddr));
         addInstr(env, ARM64Instr_MovI(hregARM64_X5(), rExpd));
         addInstr(env, ARM64Instr_MovI(hregARM64_X7(), rData));
         addInstr(env, ARM64Instr_CAS(sz));
         /* Now we have the lowest szB bytes of x1 are either equal to
            the lowest szB bytes of x5, indicating success, or they
            aren't, indicating failure.  The IR semantics actually
            require us to return the old value at the location,
            regardless of success or failure, but in the case of
            failure it's not clear how to do this, since
            ARM64Instr_CAS can't provide that.  Instead we'll just
            return the relevant bit of x1, since that's at least
            guaranteed to be different from the lowest bits of x5 on
            failure. */
         HReg rResult = hregARM64_X1();
         switch (sz) {
            case 8:  break;
            case 4:  rResult = widen_z_32_to_64(env, rResult); break;
            case 2:  rResult = widen_z_16_to_64(env, rResult); break;
            case 1:  rResult = widen_z_8_to_64(env, rResult); break;
            default: vassert(0);
         }
         // "old" in this case is interpreted somewhat liberally, per
         // the previous comment.
         HReg rOld = lookupIRTemp(env, cas->oldLo);
         addInstr(env, ARM64Instr_MovI(rOld, rResult));
         return;
      }
      unhandled_cas:
      break;
   }

   /* --------- MEM FENCE --------- */
   case Ist_MBE:
      switch (stmt->Ist.MBE.event) {
         case Imbe_Fence:
            addInstr(env, ARM64Instr_MFence());
            return;
         case Imbe_CancelReservation:
            addInstr(env, ARM64Instr_ClrEX());
            return;
         default:
            break;
      }
      break;

   /* --------- INSTR MARK --------- */
   /* Doesn't generate any executable code ... */
   case Ist_IMark:
       return;

   /* --------- ABI HINT --------- */
   /* These have no meaning (denotation in the IR) and so we ignore
      them ... if any actually made it this far. */
   case Ist_AbiHint:
       return;

   /* --------- NO-OP --------- */
   case Ist_NoOp:
       return;

   /* --------- EXIT --------- */
   case Ist_Exit: {
      if (stmt->Ist.Exit.dst->tag != Ico_U64)
         vpanic("isel_arm: Ist_Exit: dst is not a 64-bit value");

      ARM64CondCode cc 
         = iselCondCode(env, stmt->Ist.Exit.guard);
      ARM64AMode* amPC
         = mk_baseblock_64bit_access_amode(stmt->Ist.Exit.offsIP);

      /* Case: boring transfer to known address */
      if (stmt->Ist.Exit.jk == Ijk_Boring) {
         if (env->chainingAllowed) {
            /* .. almost always true .. */
            /* Skip the event check at the dst if this is a forwards
               edge. */
            Bool toFastEP
               = ((Addr64)stmt->Ist.Exit.dst->Ico.U64) > env->max_ga;
            if (0) vex_printf("%s", toFastEP ? "Y" : ",");
            addInstr(env, ARM64Instr_XDirect(stmt->Ist.Exit.dst->Ico.U64,
                                             amPC, cc, toFastEP));
         } else {
            /* .. very occasionally .. */
            /* We can't use chaining, so ask for an assisted transfer,
               as that's the only alternative that is allowable. */
            HReg r = iselIntExpr_R(env, IRExpr_Const(stmt->Ist.Exit.dst));
            addInstr(env, ARM64Instr_XAssisted(r, amPC, cc, Ijk_Boring));
         }
         return;
      }

      /* Case: assisted transfer to arbitrary address */
      switch (stmt->Ist.Exit.jk) {
         /* Keep this list in sync with that for iselNext below */
         case Ijk_ClientReq:
         case Ijk_NoDecode:
         case Ijk_NoRedir:
         case Ijk_Sys_syscall:
         case Ijk_InvalICache:
         case Ijk_FlushDCache:
         case Ijk_SigTRAP:
         case Ijk_Yield: {
            HReg r = iselIntExpr_R(env, IRExpr_Const(stmt->Ist.Exit.dst));
            addInstr(env, ARM64Instr_XAssisted(r, amPC, cc,
                                               stmt->Ist.Exit.jk));
            return;
         }
         default:
            break;
      }

      /* Do we ever expect to see any other kind? */
      goto stmt_fail;
   }

   default: break;
   }
  stmt_fail:
   ppIRStmt(stmt);
   vpanic("iselStmt");
}


/*---------------------------------------------------------*/
/*--- ISEL: Basic block terminators (Nexts)             ---*/
/*---------------------------------------------------------*/

static void iselNext ( ISelEnv* env,
                       IRExpr* next, IRJumpKind jk, Int offsIP )
{
   if (vex_traceflags & VEX_TRACE_VCODE) {
      vex_printf( "\n-- PUT(%d) = ", offsIP);
      ppIRExpr( next );
      vex_printf( "; exit-");
      ppIRJumpKind(jk);
      vex_printf( "\n");
   }

   /* Case: boring transfer to known address */
   if (next->tag == Iex_Const) {
      IRConst* cdst = next->Iex.Const.con;
      vassert(cdst->tag == Ico_U64);
      if (jk == Ijk_Boring || jk == Ijk_Call) {
         /* Boring transfer to known address */
         ARM64AMode* amPC = mk_baseblock_64bit_access_amode(offsIP);
         if (env->chainingAllowed) {
            /* .. almost always true .. */
            /* Skip the event check at the dst if this is a forwards
               edge. */
            Bool toFastEP
               = ((Addr64)cdst->Ico.U64) > env->max_ga;
            if (0) vex_printf("%s", toFastEP ? "X" : ".");
            addInstr(env, ARM64Instr_XDirect(cdst->Ico.U64,
                                             amPC, ARM64cc_AL, 
                                             toFastEP));
         } else {
            /* .. very occasionally .. */
            /* We can't use chaining, so ask for an assisted transfer,
               as that's the only alternative that is allowable. */
            HReg r = iselIntExpr_R(env, next);
            addInstr(env, ARM64Instr_XAssisted(r, amPC, ARM64cc_AL,
                                               Ijk_Boring));
         }
         return;
      }
   }

   /* Case: call/return (==boring) transfer to any address */
   switch (jk) {
      case Ijk_Boring: case Ijk_Ret: case Ijk_Call: {
         HReg        r    = iselIntExpr_R(env, next);
         ARM64AMode* amPC = mk_baseblock_64bit_access_amode(offsIP);
         if (env->chainingAllowed) {
            addInstr(env, ARM64Instr_XIndir(r, amPC, ARM64cc_AL));
         } else {
            addInstr(env, ARM64Instr_XAssisted(r, amPC, ARM64cc_AL,
                                               Ijk_Boring));
         }
         return;
      }
      default:
         break;
   }

   /* Case: assisted transfer to arbitrary address */
   switch (jk) {
      /* Keep this list in sync with that for Ist_Exit above */
      case Ijk_ClientReq:
      case Ijk_NoDecode:
      case Ijk_NoRedir:
      case Ijk_Sys_syscall:
      case Ijk_InvalICache:
      case Ijk_FlushDCache:
      case Ijk_SigTRAP:
      case Ijk_Yield:
      {
         HReg        r    = iselIntExpr_R(env, next);
         ARM64AMode* amPC = mk_baseblock_64bit_access_amode(offsIP);
         addInstr(env, ARM64Instr_XAssisted(r, amPC, ARM64cc_AL, jk));
         return;
      }
      default:
         break;
   }

   vex_printf( "\n-- PUT(%d) = ", offsIP);
   ppIRExpr( next );
   vex_printf( "; exit-");
   ppIRJumpKind(jk);
   vex_printf( "\n");
   vassert(0); // are we expecting any other kind?
}


/*---------------------------------------------------------*/
/*--- Insn selector top-level                           ---*/
/*---------------------------------------------------------*/

/* Translate an entire SB to arm64 code. */

HInstrArray* iselSB_ARM64 ( const IRSB* bb,
                            VexArch      arch_host,
                            const VexArchInfo* archinfo_host,
                            const VexAbiInfo*  vbi/*UNUSED*/,
                            Int offs_Host_EvC_Counter,
                            Int offs_Host_EvC_FailAddr,
                            Bool chainingAllowed,
                            Bool addProfInc,
                            Addr max_ga )
{
   Int        i, j;
   HReg       hreg, hregHI;
   ISelEnv*   env;
   UInt       hwcaps_host = archinfo_host->hwcaps;
   ARM64AMode *amCounter, *amFailAddr;

   /* sanity ... */
   vassert(arch_host == VexArchARM64);

   /* Check that the host's endianness is as expected. */
   vassert(archinfo_host->endness == VexEndnessLE);

   /* guard against unexpected space regressions */
   vassert(sizeof(ARM64Instr) <= 32);

   /* Make up an initial environment to use. */
   env = LibVEX_Alloc_inline(sizeof(ISelEnv));
   env->vreg_ctr = 0;

   /* Set up output code array. */
   env->code = newHInstrArray();
    
   /* Copy BB's type env. */
   env->type_env = bb->tyenv;

   /* Make up an IRTemp -> virtual HReg mapping.  This doesn't
      change as we go along. */
   env->n_vregmap = bb->tyenv->types_used;
   env->vregmap   = LibVEX_Alloc_inline(env->n_vregmap * sizeof(HReg));
   env->vregmapHI = LibVEX_Alloc_inline(env->n_vregmap * sizeof(HReg));

   /* and finally ... */
   env->chainingAllowed = chainingAllowed;
   env->hwcaps          = hwcaps_host;
   env->previous_rm     = NULL;
   env->max_ga          = max_ga;

   /* For each IR temporary, allocate a suitably-kinded virtual
      register. */
   j = 0;
   for (i = 0; i < env->n_vregmap; i++) {
      hregHI = hreg = INVALID_HREG;
      switch (bb->tyenv->types[i]) {
         case Ity_I1:
         case Ity_I8: case Ity_I16: case Ity_I32: case Ity_I64:
            hreg = mkHReg(True, HRcInt64, 0, j++);
            break;
         case Ity_I128:
            hreg   = mkHReg(True, HRcInt64, 0, j++);
            hregHI = mkHReg(True, HRcInt64, 0, j++);
            break;
         case Ity_F16: // we'll use HRcFlt64 regs for F16 too
         case Ity_F32: // we'll use HRcFlt64 regs for F32 too
         case Ity_F64:
            hreg = mkHReg(True, HRcFlt64, 0, j++);
            break;
         case Ity_V128:
            hreg = mkHReg(True, HRcVec128, 0, j++);
            break;
         case Ity_V256:
            hreg   = mkHReg(True, HRcVec128, 0, j++);
            hregHI = mkHReg(True, HRcVec128, 0, j++);
            break;
         default:
            ppIRType(bb->tyenv->types[i]);
            vpanic("iselBB(arm64): IRTemp type");
      }
      env->vregmap[i]   = hreg;
      env->vregmapHI[i] = hregHI;
   }
   env->vreg_ctr = j;

   /* The very first instruction must be an event check. */
   amCounter  = ARM64AMode_RI9(hregARM64_X21(), offs_Host_EvC_Counter);
   amFailAddr = ARM64AMode_RI9(hregARM64_X21(), offs_Host_EvC_FailAddr);
   addInstr(env, ARM64Instr_EvCheck(amCounter, amFailAddr));

   /* Possibly a block counter increment (for profiling).  At this
      point we don't know the address of the counter, so just pretend
      it is zero.  It will have to be patched later, but before this
      translation is used, by a call to LibVEX_patchProfCtr. */
   if (addProfInc) {
      addInstr(env, ARM64Instr_ProfInc());
   }

   /* Ok, finally we can iterate over the statements. */
   for (i = 0; i < bb->stmts_used; i++)
      iselStmt(env, bb->stmts[i]);

   iselNext(env, bb->next, bb->jumpkind, bb->offsIP);

   /* record the number of vregs we used. */
   env->code->n_vregs = env->vreg_ctr;
   return env->code;
}


/*---------------------------------------------------------------*/
/*--- end                                   host_arm64_isel.c ---*/
/*---------------------------------------------------------------*/