js/src/methodjit/FastArithmetic.cpp

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
   2  * vim: set ts=4 sw=4 et tw=99:
   3  *
   4  * ***** BEGIN LICENSE BLOCK *****
   5  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   6  *
   7  * The contents of this file are subject to the Mozilla Public License Version
   8  * 1.1 (the "License"); you may not use this file except in compliance with
   9  * the License. You may obtain a copy of the License at
  10  * http://www.mozilla.org/MPL/
  11  *
  12  * Software distributed under the License is distributed on an "AS IS" basis,
  13  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  14  * for the specific language governing rights and limitations under the
  15  * License.
  16  *
  17  * The Original Code is Mozilla SpiderMonkey JavaScript 1.9 code, released
  18  * May 28, 2008.
  19  *
  20  * The Initial Developer of the Original Code is
  21  *   Brendan Eich <brendan@mozilla.org>
  22  *
  23  * Contributor(s):
  24  *   David Anderson <danderson@mozilla.com>
  25  *   David Mandelin <dmandelin@mozilla.com>
  26  *   Sean Stangl    <sstangl@mozilla.com>
  27  *
  28  * Alternatively, the contents of this file may be used under the terms of
  29  * either of the GNU General Public License Version 2 or later (the "GPL"),
  30  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  31  * in which case the provisions of the GPL or the LGPL are applicable instead
  32  * of those above. If you wish to allow use of your version of this file only
  33  * under the terms of either the GPL or the LGPL, and not to allow others to
  34  * use your version of this file under the terms of the MPL, indicate your
  35  * decision by deleting the provisions above and replace them with the notice
  36  * and other provisions required by the GPL or the LGPL. If you do not delete
  37  * the provisions above, a recipient may use your version of this file under
  38  * the terms of any one of the MPL, the GPL or the LGPL.
  39  *
  40  * ***** END LICENSE BLOCK ***** */
  41 #include "jsbool.h"
  42 #include "jslibmath.h"
  43 #include "jsnum.h"
  44 #include "methodjit/MethodJIT.h"
  45 #include "methodjit/Compiler.h"
  46 #include "methodjit/StubCalls.h"
  47 #include "methodjit/FrameState-inl.h"
  48
  49 using namespace js;
  50 using namespace js::mjit;
  51 using namespace JSC;
  52
  53 typedef JSC::MacroAssembler::FPRegisterID FPRegisterID;
  54
  55 bool
  56 mjit::Compiler::tryBinaryConstantFold(JSContext *cx, FrameState &frame, JSOp op,
  57                                       FrameEntry *lhs, FrameEntry *rhs)
  58 {
  59     if (!lhs->isConstant() || !rhs->isConstant())
  60         return false;
  61
  62     const Value &L = lhs->getValue();
  63     const Value &R = rhs->getValue();
  64
  65     if (!L.isPrimitive() || !R.isPrimitive() ||
  66         (op == JSOP_ADD && (L.isString() || R.isString()))) {
  67         return false;
  68     }
  69
  70     bool needInt;
  71     switch (op) {
  72       case JSOP_ADD:
  73       case JSOP_SUB:
  74       case JSOP_MUL:
  75       case JSOP_DIV:
  76         needInt = false;
  77         break;
  78
  79       case JSOP_MOD:
  80         needInt = (L.isInt32() && R.isInt32() &&
  81                    L.toInt32() >= 0 && R.toInt32() > 0);
  82         break;
  83
  84       case JSOP_RSH:
  85         needInt = true;
  86         break;
  87
  88       default:
  89         JS_NOT_REACHED("NYI");
  90         needInt = false; /* Silence compiler warning. */
  91         break;
  92     }
  93
  94     double dL = 0, dR = 0;
  95     int32_t nL = 0, nR = 0;
  96     /*
  97      * We don't need to check for conversion failure, since primitive conversion
  98      * is infallible.
  99      */
 100     if (needInt) {
 101         ValueToECMAInt32(cx, L, &nL);
 102         ValueToECMAInt32(cx, R, &nR);
 103     } else {
 104         ValueToNumber(cx, L, &dL);
 105         ValueToNumber(cx, R, &dR);
 106     }
 107
 108     switch (op) {
 109       case JSOP_ADD:
 110         dL += dR;
 111         break;
 112       case JSOP_SUB:
 113         dL -= dR;
 114         break;
 115       case JSOP_MUL:
 116         dL *= dR;
 117         break;
 118       case JSOP_DIV:
 119         if (dR == 0) {
 120 #ifdef XP_WIN
 121             if (JSDOUBLE_IS_NaN(dR))
 122                 dL = js_NaN;
 123             else
 124 #endif
 125             if (dL == 0 || JSDOUBLE_IS_NaN(dL))
 126                 dL = js_NaN;
 127             else if (JSDOUBLE_IS_NEG(dL) != JSDOUBLE_IS_NEG(dR))
 128                 dL = cx->runtime->negativeInfinityValue.toDouble();
 129             else
 130                 dL = cx->runtime->positiveInfinityValue.toDouble();
 131         } else {
 132             dL /= dR;
 133         }
 134         break;
 135       case JSOP_MOD:
 136         if (needInt)
 137             nL %= nR;
 138         else if (dR == 0)
 139             dL = js_NaN;
 140         else
 141             dL = js_fmod(dL, dR);
 142         break;
 143
 144       case JSOP_RSH:
 145         nL >>= (nR & 31);
 146         break;
 147
 148       default:
 149         JS_NOT_REACHED("NYI");
 150         break;
 151     }
 152
 153     Value v;
 154     if (needInt)
 155         v.setInt32(nL);
 156     else
 157         v.setNumber(dL);
 158     frame.popn(2);
 159     frame.push(v);
 160
 161     return true;
 162 }
 163
 164 void
 165 mjit::Compiler::slowLoadConstantDouble(Assembler &masm,
 166                                        FrameEntry *fe, FPRegisterID fpreg)
 167 {
 168     DoublePatch patch;
 169     if (fe->getKnownType() == JSVAL_TYPE_INT32)
 170         patch.d = (double)fe->getValue().toInt32();
 171     else
 172         patch.d = fe->getValue().toDouble();
 173     patch.label = masm.loadDouble(NULL, fpreg);
 174     patch.ool = &masm != &this->masm;
 175     JS_ASSERT_IF(patch.ool, &masm == &stubcc.masm);
 176     doubleList.append(patch);
 177 }
 178
 179 void
 180 mjit::Compiler::maybeJumpIfNotInt32(Assembler &masm, MaybeJump &mj, FrameEntry *fe,
 181                                     MaybeRegisterID &mreg)
 182 {
 183     if (!fe->isTypeKnown()) {
 184         if (mreg.isSet())
 185             mj.setJump(masm.testInt32(Assembler::NotEqual, mreg.reg()));
 186         else
 187             mj.setJump(masm.testInt32(Assembler::NotEqual, frame.addressOf(fe)));
 188     } else if (fe->getKnownType() != JSVAL_TYPE_INT32) {
 189         mj.setJump(masm.jump());
 190     }
 191 }
 192
 193 void
 194 mjit::Compiler::maybeJumpIfNotDouble(Assembler &masm, MaybeJump &mj, FrameEntry *fe,
 195                                     MaybeRegisterID &mreg)
 196 {
 197     if (!fe->isTypeKnown()) {
 198         if (mreg.isSet())
 199             mj.setJump(masm.testDouble(Assembler::NotEqual, mreg.reg()));
 200         else
 201             mj.setJump(masm.testDouble(Assembler::NotEqual, frame.addressOf(fe)));
 202     } else if (fe->getKnownType() != JSVAL_TYPE_DOUBLE) {
 203         mj.setJump(masm.jump());
 204     }
 205 }
 206
 207 void
 208 mjit::Compiler::jsop_binary(JSOp op, VoidStub stub)
 209 {
 210     FrameEntry *rhs = frame.peek(-1);
 211     FrameEntry *lhs = frame.peek(-2);
 212
 213     if (tryBinaryConstantFold(cx, frame, op, lhs, rhs))
 214         return;
 215
 216     /*
 217      * Bail out if there are unhandled types or ops.
 218      * This is temporary while ops are still being implemented.
 219      */
 220     if ((op == JSOP_MOD) ||
 221         (lhs->isTypeKnown() && (lhs->getKnownType() > JSVAL_UPPER_INCL_TYPE_OF_NUMBER_SET)) ||
 222         (rhs->isTypeKnown() && (rhs->getKnownType() > JSVAL_UPPER_INCL_TYPE_OF_NUMBER_SET))
 223 #if defined(JS_CPU_ARM)
 224         /* ARM cannot detect integer overflow with multiplication. */
 225         || op == JSOP_MUL
 226 #endif /* JS_CPU_ARM */
 227     ) {
 228         bool isStringResult = (op == JSOP_ADD) &&
 229                               (lhs->isType(JSVAL_TYPE_STRING) ||
 230                                rhs->isType(JSVAL_TYPE_STRING));
 231
 232         prepareStubCall(Uses(2));
 233         INLINE_STUBCALL(stub);
 234         frame.popn(2);
 235         if (isStringResult)
 236             frame.pushSyncedType(JSVAL_TYPE_STRING);
 237         else
 238             frame.pushSynced();
 239         return;
 240     }
 241
 242     /* Can do int math iff there is no double constant and the op is not division. */
 243     bool canDoIntMath = op != JSOP_DIV &&
 244                         !((rhs->isTypeKnown() && rhs->getKnownType() == JSVAL_TYPE_DOUBLE) ||
 245                           (lhs->isTypeKnown() && lhs->getKnownType() == JSVAL_TYPE_DOUBLE));
 246
 247     if (canDoIntMath)
 248         jsop_binary_full(lhs, rhs, op, stub);
 249     else
 250         jsop_binary_double(lhs, rhs, op, stub);
 251 }
 252
 253 static void
 254 EmitDoubleOp(JSOp op, FPRegisterID fpRight, FPRegisterID fpLeft, Assembler &masm)
 255 {
 256     switch (op) {
 257       case JSOP_ADD:
 258         masm.addDouble(fpRight, fpLeft);
 259         break;
 260
 261       case JSOP_SUB:
 262         masm.subDouble(fpRight, fpLeft);
 263         break;
 264
 265       case JSOP_MUL:
 266         masm.mulDouble(fpRight, fpLeft);
 267         break;
 268
 269       case JSOP_DIV:
 270         masm.divDouble(fpRight, fpLeft);
 271         break;
 272
 273       default:
 274         JS_NOT_REACHED("unrecognized binary op");
 275     }
 276 }
 277
 278 mjit::MaybeJump
 279 mjit::Compiler::loadDouble(FrameEntry *fe, FPRegisterID fpReg)
 280 {
 281     MaybeJump notNumber;
 282
 283     if (fe->isConstant()) {
 284         slowLoadConstantDouble(masm, fe, fpReg);
 285     } else if (!fe->isTypeKnown()) {
 286         frame.tempRegForType(fe);
 287         Jump j = frame.testDouble(Assembler::Equal, fe);
 288         notNumber = frame.testInt32(Assembler::NotEqual, fe);
 289         frame.convertInt32ToDouble(masm, fe, fpReg);
 290         Jump converted = masm.jump();
 291         j.linkTo(masm.label(), &masm);
 292         // CANDIDATE
 293         frame.loadDouble(fe, fpReg, masm);
 294         converted.linkTo(masm.label(), &masm);
 295     } else if (fe->getKnownType() == JSVAL_TYPE_INT32) {
 296         frame.tempRegForData(fe);
 297         frame.convertInt32ToDouble(masm, fe, fpReg);
 298     } else {
 299         JS_ASSERT(fe->getKnownType() == JSVAL_TYPE_DOUBLE);
 300         frame.loadDouble(fe, fpReg, masm);
 301     }
 302
 303     return notNumber;
 304 }
 305
 306 /*
 307  * This function emits a single fast-path for handling numerical arithmetic.
 308  * Unlike jsop_binary_full(), all integers are converted to doubles.
 309  */
 310 void
 311 mjit::Compiler::jsop_binary_double(FrameEntry *lhs, FrameEntry *rhs, JSOp op, VoidStub stub)
 312 {
 313     FPRegisterID fpLeft = FPRegisters::First;
 314     FPRegisterID fpRight = FPRegisters::Second;
 315
 316     MaybeJump lhsNotNumber = loadDouble(lhs, fpLeft);
 317     if (lhsNotNumber.isSet())
 318         stubcc.linkExit(lhsNotNumber.get(), Uses(2));
 319
 320     MaybeJump rhsNotNumber;
 321     if (frame.haveSameBacking(lhs, rhs)) {
 322         masm.moveDouble(fpLeft, fpRight);
 323     } else {
 324         rhsNotNumber = loadDouble(rhs, fpRight);
 325         if (rhsNotNumber.isSet())
 326             stubcc.linkExit(rhsNotNumber.get(), Uses(2));
 327     }
 328
 329     EmitDoubleOp(op, fpRight, fpLeft, masm);
 330
 331     MaybeJump done;
 332
 333     /*
 334      * Try to convert result to integer. Skip this for 1/x or -1/x, as the
 335      * result is unlikely to fit in an int.
 336      */
 337     if (op == JSOP_DIV && !(lhs->isConstant() && lhs->isType(JSVAL_TYPE_INT32) &&
 338         abs(lhs->getValue().toInt32()) == 1)) {
 339         RegisterID reg = frame.allocReg();
 340         JumpList isDouble;
 341         masm.branchConvertDoubleToInt32(fpLeft, reg, isDouble, fpRight);
 342
 343         masm.storeValueFromComponents(ImmType(JSVAL_TYPE_INT32), reg,
 344                                       frame.addressOf(lhs));
 345
 346         frame.freeReg(reg);
 347         done.setJump(masm.jump());
 348         isDouble.linkTo(masm.label(), &masm);
 349     }
 350
 351     masm.storeDouble(fpLeft, frame.addressOf(lhs));
 352
 353     if (done.isSet())
 354         done.getJump().linkTo(masm.label(), &masm);
 355
 356     if (lhsNotNumber.isSet() || rhsNotNumber.isSet()) {
 357         stubcc.leave();
 358         OOL_STUBCALL(stub);
 359     }
 360
 361     frame.popn(2);
 362     frame.pushNumber(MaybeRegisterID());
 363
 364     if (lhsNotNumber.isSet() || rhsNotNumber.isSet())
 365         stubcc.rejoin(Changes(1));
 366 }
 367
 368 /*
 369  * Simpler version of jsop_binary_full() for when lhs == rhs.
 370  */
 371 void
 372 mjit::Compiler::jsop_binary_full_simple(FrameEntry *fe, JSOp op, VoidStub stub)
 373 {
 374     FrameEntry *lhs = frame.peek(-2);
 375
 376     /* Easiest case: known double. Don't bother conversion back yet? */
 377     if (fe->isTypeKnown() && fe->getKnownType() == JSVAL_TYPE_DOUBLE) {
 378         loadDouble(fe, FPRegisters::First);
 379         EmitDoubleOp(op, FPRegisters::First, FPRegisters::First, masm);
 380         frame.popn(2);
 381         frame.pushNumber(MaybeRegisterID());
 382         return;
 383     }
 384
 385     /* Allocate all registers up-front. */
 386     FrameState::BinaryAlloc regs;
 387     frame.allocForSameBinary(fe, op, regs);
 388
 389     MaybeJump notNumber;
 390     MaybeJump doublePathDone;
 391     if (!fe->isTypeKnown()) {
 392         Jump notInt = masm.testInt32(Assembler::NotEqual, regs.lhsType.reg());
 393         stubcc.linkExitDirect(notInt, stubcc.masm.label());
 394
 395         notNumber = stubcc.masm.testDouble(Assembler::NotEqual, regs.lhsType.reg());
 396         frame.loadDouble(fe, FPRegisters::First, stubcc.masm);
 397         EmitDoubleOp(op, FPRegisters::First, FPRegisters::First, stubcc.masm);
 398
 399         /* Force the double back to memory. */
 400         Address result = frame.addressOf(lhs);
 401         stubcc.masm.storeDouble(FPRegisters::First, result);
 402
 403         /* Load the payload into the result reg so the rejoin is safe. */
 404         stubcc.masm.loadPayload(result, regs.result);
 405
 406         doublePathDone = stubcc.masm.jump();
 407     }
 408
 409     /* Okay - good to emit the integer fast-path. */
 410     MaybeJump overflow;
 411     switch (op) {
 412       case JSOP_ADD:
 413         overflow = masm.branchAdd32(Assembler::Overflow, regs.result, regs.result);
 414         break;
 415
 416       case JSOP_SUB:
 417         overflow = masm.branchSub32(Assembler::Overflow, regs.result, regs.result);
 418         break;
 419
 420 #if !defined(JS_CPU_ARM)
 421       case JSOP_MUL:
 422         overflow = masm.branchMul32(Assembler::Overflow, regs.result, regs.result);
 423         break;
 424 #endif
 425
 426       default:
 427         JS_NOT_REACHED("unrecognized op");
 428     }
 429
 430     JS_ASSERT(overflow.isSet());
 431
 432     /*
 433      * Integer overflow path. Separate from the first double path, since we
 434      * know never to try and convert back to integer.
 435      */
 436     MaybeJump overflowDone;
 437     stubcc.linkExitDirect(overflow.get(), stubcc.masm.label());
 438     {
 439         if (regs.lhsNeedsRemat) {
 440             Address address = masm.payloadOf(frame.addressForDataRemat(lhs));
 441             stubcc.masm.convertInt32ToDouble(address, FPRegisters::First);
 442         } else if (!lhs->isConstant()) {
 443             stubcc.masm.convertInt32ToDouble(regs.lhsData.reg(), FPRegisters::First);
 444         } else {
 445             slowLoadConstantDouble(stubcc.masm, lhs, FPRegisters::First);
 446         }
 447
 448         EmitDoubleOp(op, FPRegisters::First, FPRegisters::First, stubcc.masm);
 449
 450         Address address = frame.addressOf(lhs);
 451         stubcc.masm.storeDouble(FPRegisters::First, address);
 452         stubcc.masm.loadPayload(address, regs.result);
 453
 454         overflowDone = stubcc.masm.jump();
 455     }
 456
 457     /* Slow paths funnel here. */
 458     if (notNumber.isSet())
 459         notNumber.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 460     overflowDone.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 461
 462     /* Slow call - use frame.sync to avoid erroneous jump repatching in stubcc. */
 463     frame.sync(stubcc.masm, Uses(2));
 464     stubcc.leave();
 465     OOL_STUBCALL(stub);
 466
 467     /* Finish up stack operations. */
 468     frame.popn(2);
 469     frame.pushNumber(regs.result, true);
 470
 471     /* Merge back OOL double paths. */
 472     if (doublePathDone.isSet())
 473         stubcc.linkRejoin(doublePathDone.get());
 474     stubcc.linkRejoin(overflowDone.get());
 475
 476     stubcc.rejoin(Changes(1));
 477 }
 478
 479 /*
 480  * This function emits multiple fast-paths for handling numerical arithmetic.
 481  * Currently, it handles only ADD, SUB, and MUL, where both LHS and RHS are
 482  * known not to be doubles.
 483  *
 484  * The control flow of the emitted code depends on which types are known.
 485  * Given both types are unknown, the full spread looks like:
 486  *
 487  * Inline                              OOL
 488  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 489  * Is LHS Int32?  ------ No -------->  Is LHS Double?  ----- No -------,
 490  *                                     Sync LHS                        |
 491  *                                     Load LHS into XMM1              |
 492  *                                     Is RHS Double? ---- Yes --,     |
 493  *                                       Is RHS Int32? ---- No --|-----|
 494  *                                       Convert RHS into XMM0   |     |
 495  *                                     Else  <-------------------'     |
 496  *                                       Sync RHS                      |
 497  *                                       Load RHS into XMM0            |
 498  *                                     [Add,Sub,Mul] XMM0,XMM1         |
 499  *                                     Jump ---------------------,     |
 500  *                                                               |     |
 501  * Is RHS Int32?  ------ No ------->   Is RHS Double? ----- No --|-----|
 502  *                                     Sync RHS                  |     |
 503  *                                     Load RHS into XMM0        |     |
 504  *                                     Convert LHS into XMM1     |     |
 505  *                                     [Add,Sub,Mul] XMM0,XMM1   |     |
 506  *                                     Jump ---------------------|   Slow Call
 507  *                                                               |
 508  * [Add,Sub,Mul] RHS, LHS                                        |
 509  * Overflow      ------ Yes ------->   Convert RHS into XMM0     |
 510  *                                     Convert LHS into XMM1     |
 511  *                                     [Add,Sub,Mul] XMM0,XMM1   |
 512  *                                     Sync XMM1 to stack    <---'
 513  *  <--------------------------------- Rejoin
 514  */
 515 void
 516 mjit::Compiler::jsop_binary_full(FrameEntry *lhs, FrameEntry *rhs, JSOp op, VoidStub stub)
 517 {
 518     if (frame.haveSameBacking(lhs, rhs)) {
 519         jsop_binary_full_simple(lhs, op, stub);
 520         return;
 521     }
 522
 523     /* Allocate all registers up-front. */
 524     FrameState::BinaryAlloc regs;
 525     frame.allocForBinary(lhs, rhs, op, regs);
 526
 527     /* Quick-test some invariants. */
 528     JS_ASSERT_IF(lhs->isTypeKnown(), lhs->getKnownType() == JSVAL_TYPE_INT32);
 529     JS_ASSERT_IF(rhs->isTypeKnown(), rhs->getKnownType() == JSVAL_TYPE_INT32);
 530
 531     FPRegisterID fpLeft = FPRegisters::First;
 532     FPRegisterID fpRight = FPRegisters::Second;
 533
 534     MaybeJump lhsNotDouble, rhsNotNumber, lhsUnknownDone;
 535     if (!lhs->isTypeKnown())
 536         emitLeftDoublePath(lhs, rhs, regs, lhsNotDouble, rhsNotNumber, lhsUnknownDone);
 537
 538     MaybeJump rhsNotNumber2;
 539     if (!rhs->isTypeKnown())
 540         emitRightDoublePath(lhs, rhs, regs, rhsNotNumber2);
 541
 542     /* Perform the double addition. */
 543     MaybeJump doublePathDone;
 544     if (!rhs->isTypeKnown() || lhsUnknownDone.isSet()) {
 545         /* If the LHS type was not known, link its path here. */
 546         if (lhsUnknownDone.isSet())
 547             lhsUnknownDone.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 548
 549         /* Perform the double operation. */
 550         EmitDoubleOp(op, fpRight, fpLeft, stubcc.masm);
 551
 552         /* Force the double back to memory. */
 553         Address result = frame.addressOf(lhs);
 554         stubcc.masm.storeDouble(fpLeft, result);
 555
 556         /* Load the payload into the result reg so the rejoin is safe. */
 557         stubcc.masm.loadPayload(result, regs.result);
 558
 559         /* We'll link this back up later, at the bottom of the op. */
 560         doublePathDone = stubcc.masm.jump();
 561     }
 562
 563     /* Time to do the integer path. Figure out the immutable side. */
 564     int32 value = 0;
 565     JSOp origOp = op;
 566     MaybeRegisterID reg;
 567     MaybeJump preOverflow;
 568     if (!regs.resultHasRhs) {
 569         if (!regs.rhsData.isSet())
 570             value = rhs->getValue().toInt32();
 571         else
 572             reg = regs.rhsData.reg();
 573     } else {
 574         if (!regs.lhsData.isSet())
 575             value = lhs->getValue().toInt32();
 576         else
 577             reg = regs.lhsData.reg();
 578         if (op == JSOP_SUB) {
 579             // If the RHS is 0x80000000, the smallest negative value, neg does
 580             // not work. Guard against this and treat it as an overflow.
 581             preOverflow = masm.branch32(Assembler::Equal, regs.result, Imm32(0x80000000));
 582             masm.neg32(regs.result);
 583             op = JSOP_ADD;
 584         }
 585     }
 586
 587     /* Okay - good to emit the integer fast-path. */
 588     MaybeJump overflow, negZeroDone;
 589     switch (op) {
 590       case JSOP_ADD:
 591         if (reg.isSet())
 592             overflow = masm.branchAdd32(Assembler::Overflow, reg.reg(), regs.result);
 593         else
 594             overflow = masm.branchAdd32(Assembler::Overflow, Imm32(value), regs.result);
 595         break;
 596
 597       case JSOP_SUB:
 598         if (reg.isSet())
 599             overflow = masm.branchSub32(Assembler::Overflow, reg.reg(), regs.result);
 600         else
 601             overflow = masm.branchSub32(Assembler::Overflow, Imm32(value), regs.result);
 602         break;
 603
 604 #if !defined(JS_CPU_ARM)
 605       case JSOP_MUL:
 606       {
 607         JS_ASSERT(reg.isSet());
 608
 609         MaybeJump storeNegZero;
 610         bool maybeNegZero = true;
 611         bool hasConstant = (lhs->isConstant() || rhs->isConstant());
 612
 613         if (hasConstant) {
 614             value = (lhs->isConstant() ? lhs : rhs)->getValue().toInt32();
 615             RegisterID nonConstReg = lhs->isConstant() ? regs.rhsData.reg() : regs.lhsData.reg();
 616
 617             if (value > 0)
 618                 maybeNegZero = false;
 619             else if (value < 0)
 620                 storeNegZero = masm.branchTest32(Assembler::Zero, nonConstReg);
 621             else
 622                 storeNegZero = masm.branch32(Assembler::LessThan, nonConstReg, Imm32(0));
 623         }
 624         overflow = masm.branchMul32(Assembler::Overflow, reg.reg(), regs.result);
 625
 626         if (maybeNegZero) {
 627             if (!hasConstant) {
 628                 Jump isZero = masm.branchTest32(Assembler::Zero, regs.result);
 629                 stubcc.linkExitDirect(isZero, stubcc.masm.label());
 630
 631                 /* Restore original value. */
 632                 if (regs.resultHasRhs) {
 633                     if (regs.rhsNeedsRemat)
 634                         stubcc.masm.loadPayload(frame.addressForDataRemat(rhs), regs.result);
 635                     else
 636                         stubcc.masm.move(regs.rhsData.reg(), regs.result);
 637                 } else {
 638                     if (regs.lhsNeedsRemat)
 639                         stubcc.masm.loadPayload(frame.addressForDataRemat(lhs), regs.result);
 640                     else
 641                         stubcc.masm.move(regs.lhsData.reg(), regs.result);
 642                 }
 643                 storeNegZero = stubcc.masm.branchOr32(Assembler::Signed, reg.reg(), regs.result);
 644                 stubcc.masm.xor32(regs.result, regs.result);
 645                 stubcc.crossJump(stubcc.masm.jump(), masm.label());
 646                 storeNegZero.getJump().linkTo(stubcc.masm.label(), &stubcc.masm);
 647             } else {
 648                 JS_ASSERT(storeNegZero.isSet());
 649                 stubcc.linkExitDirect(storeNegZero.get(), stubcc.masm.label());
 650             }
 651             stubcc.masm.storeValue(DoubleValue(-0.0), frame.addressOf(lhs));
 652             stubcc.masm.loadPayload(frame.addressOf(lhs), regs.result);
 653             negZeroDone = stubcc.masm.jump();
 654         }
 655         break;
 656       }
 657 #endif
 658
 659       default:
 660         JS_NOT_REACHED("unrecognized op");
 661     }
 662     op = origOp;
 663
 664     JS_ASSERT(overflow.isSet());
 665
 666     /*
 667      * Integer overflow path. Separate from the first double path, since we
 668      * know never to try and convert back to integer.
 669      */
 670     MaybeJump overflowDone;
 671     if (preOverflow.isSet())
 672         stubcc.linkExitDirect(preOverflow.get(), stubcc.masm.label());
 673     stubcc.linkExitDirect(overflow.get(), stubcc.masm.label());
 674     {
 675         if (regs.lhsNeedsRemat) {
 676             Address address = masm.payloadOf(frame.addressForDataRemat(lhs));
 677             stubcc.masm.convertInt32ToDouble(address, fpLeft);
 678         } else if (!lhs->isConstant()) {
 679             stubcc.masm.convertInt32ToDouble(regs.lhsData.reg(), fpLeft);
 680         } else {
 681             slowLoadConstantDouble(stubcc.masm, lhs, fpLeft);
 682         }
 683
 684         if (regs.rhsNeedsRemat) {
 685             Address address = masm.payloadOf(frame.addressForDataRemat(rhs));
 686             stubcc.masm.convertInt32ToDouble(address, fpRight);
 687         } else if (!rhs->isConstant()) {
 688             stubcc.masm.convertInt32ToDouble(regs.rhsData.reg(), fpRight);
 689         } else {
 690             slowLoadConstantDouble(stubcc.masm, rhs, fpRight);
 691         }
 692
 693         EmitDoubleOp(op, fpRight, fpLeft, stubcc.masm);
 694
 695         Address address = frame.addressOf(lhs);
 696         stubcc.masm.storeDouble(fpLeft, address);
 697         stubcc.masm.loadPayload(address, regs.result);
 698
 699         overflowDone = stubcc.masm.jump();
 700     }
 701
 702     /* The register allocator creates at most one temporary. */
 703     if (regs.extraFree.isSet())
 704         frame.freeReg(regs.extraFree.reg());
 705
 706     /* Slow paths funnel here. */
 707     if (lhsNotDouble.isSet()) {
 708         lhsNotDouble.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 709         if (rhsNotNumber.isSet())
 710             rhsNotNumber.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 711     }
 712     if (rhsNotNumber2.isSet())
 713         rhsNotNumber2.get().linkTo(stubcc.masm.label(), &stubcc.masm);
 714
 715     /* Slow call - use frame.sync to avoid erroneous jump repatching in stubcc. */
 716     frame.sync(stubcc.masm, Uses(2));
 717     stubcc.leave();
 718     OOL_STUBCALL(stub);
 719
 720     /* Finish up stack operations. */
 721     frame.popn(2);
 722     frame.pushNumber(regs.result, true);
 723
 724     /* Merge back OOL double paths. */
 725     if (doublePathDone.isSet())
 726         stubcc.linkRejoin(doublePathDone.get());
 727     if (negZeroDone.isSet())
 728         stubcc.linkRejoin(negZeroDone.get());
 729     stubcc.linkRejoin(overflowDone.get());
 730
 731     stubcc.rejoin(Changes(1));
 732 }
 733
 734 static const uint64 DoubleNegMask = 0x8000000000000000ULL;
 735
 736 void
 737 mjit::Compiler::jsop_neg()
 738 {
 739     FrameEntry *fe = frame.peek(-1);
 740
 741     if (fe->isTypeKnown() && fe->getKnownType() > JSVAL_UPPER_INCL_TYPE_OF_NUMBER_SET) {
 742         prepareStubCall(Uses(1));
 743         INLINE_STUBCALL(stubs::Neg);
 744         frame.pop();
 745         frame.pushSynced();
 746         return;
 747     }
 748
 749     JS_ASSERT(!fe->isConstant());
 750
 751     /* Load type information into register. */
 752     MaybeRegisterID feTypeReg;
 753     if (!fe->isTypeKnown() && !frame.shouldAvoidTypeRemat(fe)) {
 754         /* Safe because only one type is loaded. */
 755         feTypeReg.setReg(frame.tempRegForType(fe));
 756
 757         /* Don't get clobbered by copyDataIntoReg(). */
 758         frame.pinReg(feTypeReg.reg());
 759     }
 760
 761     RegisterID reg = frame.copyDataIntoReg(masm, fe);
 762     Label feSyncTarget = stubcc.syncExitAndJump(Uses(1));
 763
 764     /* Try a double path (inline). */
 765     MaybeJump jmpNotDbl;
 766     {
 767         maybeJumpIfNotDouble(masm, jmpNotDbl, fe, feTypeReg);
 768
 769         FPRegisterID fpreg = frame.copyEntryIntoFPReg(fe, FPRegisters::First);
 770
 771 #if defined JS_CPU_X86 || defined JS_CPU_X64
 772         masm.loadDouble(&DoubleNegMask, FPRegisters::Second);
 773         masm.xorDouble(FPRegisters::Second, fpreg);
 774 #elif defined JS_CPU_ARM
 775         masm.negDouble(fpreg, fpreg);
 776 #endif
 777
 778         /* Overwrite pushed frame's memory (before push). */
 779         masm.storeDouble(fpreg, frame.addressOf(fe));
 780     }
 781
 782     /* Try an integer path (out-of-line). */
 783     MaybeJump jmpNotInt;
 784     MaybeJump jmpIntZero;
 785     MaybeJump jmpMinInt;
 786     MaybeJump jmpIntRejoin;
 787     Label lblIntPath = stubcc.masm.label();
 788     {
 789         maybeJumpIfNotInt32(stubcc.masm, jmpNotInt, fe, feTypeReg);
 790
 791         /* 0 (int) -> -0 (double). */
 792         jmpIntZero.setJump(stubcc.masm.branch32(Assembler::Equal, reg, Imm32(0)));
 793         /* int32 negation on (-2147483648) yields (-2147483648). */
 794         jmpMinInt.setJump(stubcc.masm.branch32(Assembler::Equal, reg, Imm32(1 << 31)));
 795
 796         stubcc.masm.neg32(reg);
 797
 798         /* Sync back with double path. */
 799         stubcc.masm.storeValueFromComponents(ImmType(JSVAL_TYPE_INT32), reg,
 800                                              frame.addressOf(fe));
 801
 802         jmpIntRejoin.setJump(stubcc.masm.jump());
 803     }
 804
 805     frame.freeReg(reg);
 806     if (feTypeReg.isSet())
 807         frame.unpinReg(feTypeReg.reg());
 808
 809     stubcc.leave();
 810     OOL_STUBCALL(stubs::Neg);
 811
 812     frame.pop();
 813     frame.pushSynced();
 814
 815     /* Link jumps. */
 816     if (jmpNotDbl.isSet())
 817         stubcc.linkExitDirect(jmpNotDbl.getJump(), lblIntPath);
 818
 819     if (jmpNotInt.isSet())
 820         jmpNotInt.getJump().linkTo(feSyncTarget, &stubcc.masm);
 821     if (jmpIntZero.isSet())
 822         jmpIntZero.getJump().linkTo(feSyncTarget, &stubcc.masm);
 823     if (jmpMinInt.isSet())
 824         jmpMinInt.getJump().linkTo(feSyncTarget, &stubcc.masm);
 825     if (jmpIntRejoin.isSet())
 826         stubcc.crossJump(jmpIntRejoin.getJump(), masm.label());
 827
 828     stubcc.rejoin(Changes(1));
 829 }
 830
 831 void
 832 mjit::Compiler::jsop_mod()
 833 {
 834 #if defined(JS_CPU_X86)
 835     FrameEntry *lhs = frame.peek(-2);
 836     FrameEntry *rhs = frame.peek(-1);
 837
 838     if (tryBinaryConstantFold(cx, frame, JSOP_MOD, lhs, rhs))
 839         return;
 840
 841     if ((lhs->isTypeKnown() && lhs->getKnownType() != JSVAL_TYPE_INT32) ||
 842         (rhs->isTypeKnown() && rhs->getKnownType() != JSVAL_TYPE_INT32))
 843 #endif
 844     {
 845         prepareStubCall(Uses(2));
 846         INLINE_STUBCALL(stubs::Mod);
 847         frame.popn(2);
 848         frame.pushSynced();
 849         return;
 850     }
 851
 852 #if defined(JS_CPU_X86)
 853     if (!lhs->isTypeKnown()) {
 854         Jump j = frame.testInt32(Assembler::NotEqual, lhs);
 855         stubcc.linkExit(j, Uses(2));
 856     }
 857     if (!rhs->isTypeKnown()) {
 858         Jump j = frame.testInt32(Assembler::NotEqual, rhs);
 859         stubcc.linkExit(j, Uses(2));
 860     }
 861
 862     /* LHS must be in EAX:EDX */
 863     if (!lhs->isConstant()) {
 864         frame.copyDataIntoReg(lhs, X86Registers::eax);
 865     } else {
 866         frame.takeReg(X86Registers::eax);
 867         masm.move(Imm32(lhs->getValue().toInt32()), X86Registers::eax);
 868     }
 869
 870     /* Get RHS into anything but EDX - could avoid more spilling? */
 871     MaybeRegisterID temp;
 872     RegisterID rhsReg;
 873     if (!rhs->isConstant()) {
 874         uint32 mask = Registers::AvailRegs & ~Registers::maskReg(X86Registers::edx);
 875         rhsReg = frame.tempRegInMaskForData(rhs, mask);
 876         JS_ASSERT(rhsReg != X86Registers::edx);
 877     } else {
 878         rhsReg = frame.allocReg(Registers::AvailRegs & ~Registers::maskReg(X86Registers::edx));
 879         JS_ASSERT(rhsReg != X86Registers::edx);
 880         masm.move(Imm32(rhs->getValue().toInt32()), rhsReg);
 881         temp = rhsReg;
 882     }
 883     frame.takeReg(X86Registers::edx);
 884     frame.freeReg(X86Registers::eax);
 885
 886     if (temp.isSet())
 887         frame.freeReg(temp.reg());
 888
 889     bool slowPath = !(lhs->isTypeKnown() && rhs->isTypeKnown());
 890     if (rhs->isConstant() && rhs->getValue().toInt32() != 0) {
 891         if (rhs->getValue().toInt32() == -1) {
 892             /* Guard against -1 / INT_MIN which throws a hardware exception. */
 893             Jump checkDivExc = masm.branch32(Assembler::Equal, X86Registers::eax,
 894                                              Imm32(0x80000000));
 895             stubcc.linkExit(checkDivExc, Uses(2));
 896             slowPath = true;
 897         }
 898     } else {
 899         Jump checkDivExc = masm.branch32(Assembler::Equal, X86Registers::eax, Imm32(0x80000000));
 900         stubcc.linkExit(checkDivExc, Uses(2));
 901         Jump checkZero = masm.branchTest32(Assembler::Zero, rhsReg, rhsReg);
 902         stubcc.linkExit(checkZero, Uses(2));
 903         slowPath = true;
 904     }
 905
 906     /* Perform division. */
 907     masm.idiv(rhsReg);
 908
 909     /* ECMA-262 11.5.3 requires the result to have the same sign as the lhs.
 910      * Thus, if the remainder of the div instruction is zero and the lhs is
 911      * negative, we must return negative 0. */
 912
 913     bool lhsMaybeNeg = true;
 914     bool lhsIsNeg = false;
 915     if (lhs->isConstant()) {
 916         /* This condition is established at the top of this function. */
 917         JS_ASSERT(lhs->getValue().isInt32());
 918         lhsMaybeNeg = lhsIsNeg = (lhs->getValue().toInt32() < 0);
 919     }
 920
 921     MaybeJump done;
 922     if (lhsMaybeNeg) {
 923         MaybeRegisterID lhsData;
 924         if (!lhsIsNeg)
 925             lhsData = frame.tempRegForData(lhs);
 926         Jump negZero1 = masm.branchTest32(Assembler::NonZero, X86Registers::edx);
 927         MaybeJump negZero2;
 928         if (!lhsIsNeg)
 929             negZero2 = masm.branchTest32(Assembler::Zero, lhsData.reg(), Imm32(0x80000000));
 930         /* Darn, negative 0. */
 931         masm.storeValue(DoubleValue(-0.0), frame.addressOf(lhs));
 932
 933         /* :TODO: This is wrong, must load into EDX as well. */
 934
 935         done = masm.jump();
 936         negZero1.linkTo(masm.label(), &masm);
 937         if (negZero2.isSet())
 938             negZero2.getJump().linkTo(masm.label(), &masm);
 939     }
 940
 941     /* Better - integer. */
 942     masm.storeTypeTag(ImmType(JSVAL_TYPE_INT32), frame.addressOf(lhs));
 943
 944     if (done.isSet())
 945         done.getJump().linkTo(masm.label(), &masm);
 946
 947     if (slowPath) {
 948         stubcc.leave();
 949         OOL_STUBCALL(stubs::Mod);
 950     }
 951
 952     frame.popn(2);
 953     frame.pushNumber(X86Registers::edx);
 954
 955     if (slowPath)
 956         stubcc.rejoin(Changes(1));
 957 #endif
 958 }
 959
 960 bool
 961 mjit::Compiler::jsop_equality_int_string(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused)
 962 {
 963     FrameEntry *rhs = frame.peek(-1);
 964     FrameEntry *lhs = frame.peek(-2);
 965
 966     /* Swap the LHS and RHS if it makes register allocation better... or possible. */
 967     if (lhs->isConstant() ||
 968         (frame.shouldAvoidDataRemat(lhs) && !rhs->isConstant())) {
 969         FrameEntry *temp = rhs;
 970         rhs = lhs;
 971         lhs = temp;
 972     }
 973
 974     bool lhsInt = lhs->isType(JSVAL_TYPE_INT32);
 975     bool rhsInt = rhs->isType(JSVAL_TYPE_INT32);
 976
 977     /* Invert the condition if fusing with an IFEQ branch. */
 978     bool flipCondition = (target && fused == JSOP_IFEQ);
 979
 980     /* Get the condition being tested. */
 981     Assembler::Condition cond;
 982     switch (op) {
 983       case JSOP_EQ:
 984         cond = flipCondition ? Assembler::NotEqual : Assembler::Equal;
 985         break;
 986       case JSOP_NE:
 987         cond = flipCondition ? Assembler::Equal : Assembler::NotEqual;
 988         break;
 989       default:
 990         JS_NOT_REACHED("wat");
 991         return false;
 992     }
 993
 994     if (target) {
 995         Value rval = UndefinedValue();  /* quiet gcc warning */
 996         bool rhsConst = false;
 997         if (rhs->isConstant()) {
 998             rhsConst = true;
 999             rval = rhs->getValue();
1000         }
1001
1002         ValueRemat lvr, rvr;
1003         frame.pinEntry(lhs, lvr);
1004         frame.pinEntry(rhs, rvr);
1005
1006         /*
1007          * Sync everything except the top two entries.
1008          * We will handle the lhs/rhs in the stub call path.
1009          */
1010         frame.syncAndKill(Registers(Registers::AvailRegs), Uses(frame.frameSlots()), Uses(2));
1011
1012         RegisterID tempReg = frame.allocReg();
1013
1014         frame.pop();
1015         frame.pop();
1016         frame.discardFrame();
1017
1018         JaegerSpew(JSpew_Insns, " ---- BEGIN STUB CALL CODE ---- \n");
1019
1020         RESERVE_OOL_SPACE(stubcc.masm);
1021
1022         /* Start of the slow path for equality stub call. */
1023         Label stubEntry = stubcc.masm.label();
1024
1025         /* The lhs/rhs need to be synced in the stub call path. */
1026         frame.ensureValueSynced(stubcc.masm, lhs, lvr);
1027         frame.ensureValueSynced(stubcc.masm, rhs, rvr);
1028
1029         bool needStub = true;
1030
1031 #ifdef JS_MONOIC
1032         EqualityGenInfo ic;
1033
1034         ic.cond = cond;
1035         ic.tempReg = tempReg;
1036         ic.lvr = lvr;
1037         ic.rvr = rvr;
1038         ic.stubEntry = stubEntry;
1039         ic.stub = stub;
1040
1041         bool useIC = !addTraceHints || target >= PC;
1042
1043         /* Call the IC stub, which may generate a fast path. */
1044         if (useIC) {
1045             /* Adjust for the two values just pushed. */
1046             ic.addrLabel = stubcc.masm.moveWithPatch(ImmPtr(NULL), Registers::ArgReg1);
1047             ic.stubCall = OOL_STUBCALL_LOCAL_SLOTS(ic::Equality,
1048                                                    frame.stackDepth() + script->nfixed + 2);
1049             needStub = false;
1050         }
1051 #endif
1052
1053         if (needStub)
1054             OOL_STUBCALL_LOCAL_SLOTS(stub, frame.stackDepth() + script->nfixed + 2);
1055
1056         /*
1057          * The stub call has no need to rejoin, since state is synced.
1058          * Instead, we can just test the return value.
1059          */
1060         Assembler::Condition ncond = (fused == JSOP_IFEQ)
1061                                    ? Assembler::Zero
1062                                    : Assembler::NonZero;
1063         Jump stubBranch =
1064             stubcc.masm.branchTest32(ncond, Registers::ReturnReg, Registers::ReturnReg);
1065         Jump stubFallthrough = stubcc.masm.jump();
1066
1067         JaegerSpew(JSpew_Insns, " ---- END STUB CALL CODE ---- \n");
1068         CHECK_OOL_SPACE();
1069
1070         Jump fast;
1071         MaybeJump firstStubJump;
1072
1073         if ((!lhs->isTypeKnown() || lhsInt) && (!rhs->isTypeKnown() || rhsInt)) {
1074             if (!lhsInt) {
1075                 Jump lhsFail = masm.testInt32(Assembler::NotEqual, lvr.typeReg());
1076                 stubcc.linkExitDirect(lhsFail, stubEntry);
1077                 firstStubJump = lhsFail;
1078             }
1079             if (!rhsInt) {
1080                 Jump rhsFail = masm.testInt32(Assembler::NotEqual, rvr.typeReg());
1081                 stubcc.linkExitDirect(rhsFail, stubEntry);
1082                 if (!firstStubJump.isSet())
1083                     firstStubJump = rhsFail;
1084             }
1085
1086             if (rhsConst)
1087                 fast = masm.branch32(cond, lvr.dataReg(), Imm32(rval.toInt32()));
1088             else
1089                 fast = masm.branch32(cond, lvr.dataReg(), rvr.dataReg());
1090
1091             if (!jumpInScript(fast, target))
1092                 return false;
1093         } else {
1094             Jump j = masm.jump();
1095             stubcc.linkExitDirect(j, stubEntry);
1096             firstStubJump = j;
1097
1098             /* This is just a dummy jump. */
1099             fast = masm.jump();
1100         }
1101
1102 #ifdef JS_MONOIC
1103         ic.jumpToStub = firstStubJump;
1104         if (useIC) {
1105             ic.fallThrough = masm.label();
1106             ic.jumpTarget = target;
1107             equalityICs.append(ic);
1108         }
1109 #endif
1110
1111         /* Jump from the stub call fallthrough to here. */
1112         stubcc.crossJump(stubFallthrough, masm.label());
1113
1114         /*
1115          * NB: jumpAndTrace emits to the OOL path, so make sure not to use it
1116          * in the middle of an in-progress slow path.
1117          */
1118         if (!jumpAndTrace(fast, target, &stubBranch))
1119             return false;
1120     } else {
1121         /* No fusing. Compare, set, and push a boolean. */
1122
1123         /* Should have filtered these out in the caller. */
1124         JS_ASSERT(!lhs->isType(JSVAL_TYPE_STRING) && !rhs->isType(JSVAL_TYPE_STRING));
1125
1126         /* Test the types. */
1127         if ((lhs->isTypeKnown() && !lhsInt) || (rhs->isTypeKnown() && !rhsInt)) {
1128             stubcc.linkExit(masm.jump(), Uses(2));
1129         } else {
1130             if (!lhsInt) {
1131                 Jump lhsFail = frame.testInt32(Assembler::NotEqual, lhs);
1132                 stubcc.linkExit(lhsFail, Uses(2));
1133             }
1134             if (!rhsInt) {
1135                 Jump rhsFail = frame.testInt32(Assembler::NotEqual, rhs);
1136                 stubcc.linkExit(rhsFail, Uses(2));
1137             }
1138         }
1139
1140         stubcc.leave();
1141         OOL_STUBCALL(stub);
1142
1143         RegisterID reg = frame.ownRegForData(lhs);
1144
1145         /* x86/64's SET instruction can only take single-byte regs.*/
1146         RegisterID resultReg = reg;
1147         if (!(Registers::maskReg(reg) & Registers::SingleByteRegs))
1148             resultReg = frame.allocReg(Registers::SingleByteRegs);
1149
1150         /* Emit the compare & set. */
1151         if (rhs->isConstant()) {
1152             masm.set32(cond, reg, Imm32(rhs->getValue().toInt32()), resultReg);
1153         } else if (frame.shouldAvoidDataRemat(rhs)) {
1154             masm.set32(cond, reg,
1155                        masm.payloadOf(frame.addressOf(rhs)),
1156                        resultReg);
1157         } else {
1158             masm.set32(cond, reg, frame.tempRegForData(rhs), resultReg);
1159         }
1160
1161         /* Clean up and push a boolean. */
1162         frame.pop();
1163         frame.pop();
1164         if (reg != resultReg)
1165             frame.freeReg(reg);
1166         frame.pushTypedPayload(JSVAL_TYPE_BOOLEAN, resultReg);
1167         stubcc.rejoin(Changes(1));
1168     }
1169     return true;
1170 }
1171
1172 /*
1173  * Emit an OOL path for a possibly double LHS, and possibly int32 or number RHS.
1174  */
1175 void
1176 mjit::Compiler::emitLeftDoublePath(FrameEntry *lhs, FrameEntry *rhs, FrameState::BinaryAlloc &regs,
1177                                    MaybeJump &lhsNotDouble, MaybeJump &rhsNotNumber,
1178                                    MaybeJump &lhsUnknownDone)
1179 {
1180     FPRegisterID fpLeft = FPRegisters::First;
1181     FPRegisterID fpRight = FPRegisters::Second;
1182
1183     /* If the LHS is not a 32-bit integer, take OOL path. */
1184     Jump lhsNotInt32 = masm.testInt32(Assembler::NotEqual, regs.lhsType.reg());
1185     stubcc.linkExitDirect(lhsNotInt32, stubcc.masm.label());
1186
1187     /* OOL path for LHS as a double - first test LHS is double. */
1188     lhsNotDouble = stubcc.masm.testDouble(Assembler::NotEqual, regs.lhsType.reg());
1189
1190     /* Ensure the RHS is a number. */
1191     MaybeJump rhsIsDouble;
1192     if (!rhs->isTypeKnown()) {
1193         rhsIsDouble = stubcc.masm.testDouble(Assembler::Equal, regs.rhsType.reg());
1194         rhsNotNumber = stubcc.masm.testInt32(Assembler::NotEqual, regs.rhsType.reg());
1195     }
1196
1197     /* If RHS is constant, convert now. */
1198     if (rhs->isConstant())
1199         slowLoadConstantDouble(stubcc.masm, rhs, fpRight);
1200     else
1201         stubcc.masm.convertInt32ToDouble(regs.rhsData.reg(), fpRight);
1202
1203     if (!rhs->isTypeKnown()) {
1204         /* Jump past double load, bind double type check. */
1205         Jump converted = stubcc.masm.jump();
1206         rhsIsDouble.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1207
1208         /* Load the double. */
1209         frame.loadDouble(regs.rhsType.reg(), regs.rhsData.reg(),
1210                          rhs, fpRight, stubcc.masm);
1211
1212         converted.linkTo(stubcc.masm.label(), &stubcc.masm);
1213     }
1214
1215     /* Load the LHS. */
1216     frame.loadDouble(regs.lhsType.reg(), regs.lhsData.reg(),
1217                      lhs, fpLeft, stubcc.masm);
1218     lhsUnknownDone = stubcc.masm.jump();
1219 }
1220
1221 /*
1222  * Emit an OOL path for an integer LHS, possibly double RHS.
1223  */
1224 void
1225 mjit::Compiler::emitRightDoublePath(FrameEntry *lhs, FrameEntry *rhs, FrameState::BinaryAlloc &regs,
1226                                     MaybeJump &rhsNotNumber2)
1227 {
1228     FPRegisterID fpLeft = FPRegisters::First;
1229     FPRegisterID fpRight = FPRegisters::Second;
1230
1231     /* If the RHS is not a double, take OOL path. */
1232     Jump notInt32 = masm.testInt32(Assembler::NotEqual, regs.rhsType.reg());
1233     stubcc.linkExitDirect(notInt32, stubcc.masm.label());
1234
1235     /* Now test if RHS is a double. */
1236     rhsNotNumber2 = stubcc.masm.testDouble(Assembler::NotEqual, regs.rhsType.reg());
1237
1238     /* We know LHS is an integer. */
1239     if (lhs->isConstant())
1240         slowLoadConstantDouble(stubcc.masm, lhs, fpLeft);
1241     else
1242         stubcc.masm.convertInt32ToDouble(regs.lhsData.reg(), fpLeft);
1243
1244     /* Load the RHS. */
1245     frame.loadDouble(regs.rhsType.reg(), regs.rhsData.reg(),
1246                      rhs, fpRight, stubcc.masm);
1247 }
1248
1249 static inline Assembler::DoubleCondition
1250 DoubleCondForOp(JSOp op, JSOp fused)
1251 {
1252     bool ifeq = fused == JSOP_IFEQ;
1253     switch (op) {
1254       case JSOP_GT:
1255         return ifeq
1256                ? Assembler::DoubleLessThanOrEqualOrUnordered
1257                : Assembler::DoubleGreaterThan;
1258       case JSOP_GE:
1259         return ifeq
1260                ? Assembler::DoubleLessThanOrUnordered
1261                : Assembler::DoubleGreaterThanOrEqual;
1262       case JSOP_LT:
1263         return ifeq
1264                ? Assembler::DoubleGreaterThanOrEqualOrUnordered
1265                : Assembler::DoubleLessThan;
1266       case JSOP_LE:
1267         return ifeq
1268                ? Assembler::DoubleGreaterThanOrUnordered
1269                : Assembler::DoubleLessThanOrEqual;
1270       default:
1271         JS_NOT_REACHED("unrecognized op");
1272         return Assembler::DoubleLessThan;
1273     }
1274 }
1275
1276 bool
1277 mjit::Compiler::jsop_relational_double(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused)
1278 {
1279     FrameEntry *rhs = frame.peek(-1);
1280     FrameEntry *lhs = frame.peek(-2);
1281
1282     FPRegisterID fpLeft = FPRegisters::First;
1283     FPRegisterID fpRight = FPRegisters::Second;
1284
1285     JS_ASSERT_IF(!target, fused != JSOP_IFEQ);
1286
1287     MaybeJump lhsNotNumber = loadDouble(lhs, fpLeft);
1288     MaybeJump rhsNotNumber = loadDouble(rhs, fpRight);
1289
1290     Assembler::DoubleCondition dblCond = DoubleCondForOp(op, fused);
1291
1292     if (target) {
1293         if (lhsNotNumber.isSet())
1294             stubcc.linkExitForBranch(lhsNotNumber.get());
1295         if (rhsNotNumber.isSet())
1296             stubcc.linkExitForBranch(rhsNotNumber.get());
1297         stubcc.leave();
1298         OOL_STUBCALL(stub);
1299
1300         frame.popn(2);
1301         frame.syncAndForgetEverything();
1302
1303         Jump j = masm.branchDouble(dblCond, fpLeft, fpRight);
1304
1305         /*
1306          * The stub call has no need to rejoin since the state is synced.
1307          * Instead, we can just test the return value.
1308          */
1309         Assembler::Condition cond = (fused == JSOP_IFEQ)
1310                                     ? Assembler::Zero
1311                                     : Assembler::NonZero;
1312         Jump sj = stubcc.masm.branchTest32(cond, Registers::ReturnReg, Registers::ReturnReg);
1313
1314         /* Rejoin from the slow path. */
1315         Jump j2 = stubcc.masm.jump();
1316         stubcc.crossJump(j2, masm.label());
1317
1318         /*
1319          * NB: jumpAndTrace emits to the OOL path, so make sure not to use it
1320          * in the middle of an in-progress slow path.
1321          */
1322         if (!jumpAndTrace(j, target, &sj))
1323             return false;
1324     } else {
1325         if (lhsNotNumber.isSet())
1326             stubcc.linkExit(lhsNotNumber.get(), Uses(2));
1327         if (rhsNotNumber.isSet())
1328             stubcc.linkExit(rhsNotNumber.get(), Uses(2));
1329         stubcc.leave();
1330         OOL_STUBCALL(stub);
1331
1332         frame.popn(2);
1333
1334         RegisterID reg = frame.allocReg();
1335         Jump j = masm.branchDouble(dblCond, fpLeft, fpRight);
1336         masm.move(Imm32(0), reg);
1337         Jump skip = masm.jump();
1338         j.linkTo(masm.label(), &masm);
1339         masm.move(Imm32(1), reg);
1340         skip.linkTo(masm.label(), &masm);
1341
1342         frame.pushTypedPayload(JSVAL_TYPE_BOOLEAN, reg);
1343
1344         stubcc.rejoin(Changes(1));
1345     }
1346     return true;
1347 }
1348
1349 bool
1350 mjit::Compiler::jsop_relational_self(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused)
1351 {
1352 #ifdef DEBUG
1353     FrameEntry *rhs = frame.peek(-1);
1354     FrameEntry *lhs = frame.peek(-2);
1355
1356     JS_ASSERT(frame.haveSameBacking(lhs, rhs));
1357 #endif
1358
1359     /* :TODO: optimize this?  */
1360     return emitStubCmpOp(stub, target, fused);
1361 }
1362
1363 /* See jsop_binary_full() for more information on how this works. */
1364 bool
1365 mjit::Compiler::jsop_relational_full(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused)
1366 {
1367     FrameEntry *rhs = frame.peek(-1);
1368     FrameEntry *lhs = frame.peek(-2);
1369
1370     /* Allocate all registers up-front. */
1371     FrameState::BinaryAlloc regs;
1372     frame.allocForBinary(lhs, rhs, op, regs, !target);
1373
1374     FPRegisterID fpLeft = FPRegisters::First;
1375     FPRegisterID fpRight = FPRegisters::Second;
1376
1377     MaybeJump lhsNotDouble, rhsNotNumber, lhsUnknownDone;
1378     if (!lhs->isTypeKnown())
1379         emitLeftDoublePath(lhs, rhs, regs, lhsNotDouble, rhsNotNumber, lhsUnknownDone);
1380
1381     MaybeJump rhsNotNumber2;
1382     if (!rhs->isTypeKnown())
1383         emitRightDoublePath(lhs, rhs, regs, rhsNotNumber2);
1384
1385     /* Both double paths will join here. */
1386     bool hasDoublePath = false;
1387     if (!rhs->isTypeKnown() || lhsUnknownDone.isSet())
1388         hasDoublePath = true;
1389
1390     /* Integer path - figure out the immutable side. */
1391     JSOp cmpOp = op;
1392     int32 value = 0;
1393     RegisterID cmpReg;
1394     MaybeRegisterID reg;
1395     if (regs.lhsData.isSet()) {
1396         cmpReg = regs.lhsData.reg();
1397         if (!regs.rhsData.isSet())
1398             value = rhs->getValue().toInt32();
1399         else
1400             reg = regs.rhsData.reg();
1401     } else {
1402         cmpReg = regs.rhsData.reg();
1403         value = lhs->getValue().toInt32();
1404         switch (op) {
1405           case JSOP_GT:
1406             cmpOp = JSOP_LT;
1407             break;
1408           case JSOP_GE:
1409             cmpOp = JSOP_LE;
1410             break;
1411           case JSOP_LT:
1412             cmpOp = JSOP_GT;
1413             break;
1414           case JSOP_LE:
1415             cmpOp = JSOP_GE;
1416             break;
1417           default:
1418             JS_NOT_REACHED("unrecognized op");
1419             break;
1420         }
1421     }
1422
1423     /*
1424      * Emit the actual comparisons. When a fusion is in play, it's faster to
1425      * combine the comparison with the jump, so these two cases are implemented
1426      * separately.
1427      */
1428
1429     if (target) {
1430         /*
1431          * Emit the double path now, necessary to complete the OOL fast-path
1432          * before emitting the slow path.
1433          *
1434          * Note: doubles have not been swapped yet. Use original op.
1435          */
1436         MaybeJump doubleTest, doubleFall;
1437         Assembler::DoubleCondition dblCond = DoubleCondForOp(op, fused);
1438         if (hasDoublePath) {
1439             if (lhsUnknownDone.isSet())
1440                 lhsUnknownDone.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1441             frame.sync(stubcc.masm, Uses(frame.frameSlots()));
1442             doubleTest = stubcc.masm.branchDouble(dblCond, fpLeft, fpRight);
1443             doubleFall = stubcc.masm.jump();
1444
1445             /* Link all incoming slow paths to here. */
1446             if (lhsNotDouble.isSet()) {
1447                 lhsNotDouble.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1448                 if (rhsNotNumber.isSet())
1449                     rhsNotNumber.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1450             }
1451             if (rhsNotNumber2.isSet())
1452                 rhsNotNumber2.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1453
1454             /*
1455              * For fusions, spill the tracker state. xmm* remain intact. Note
1456              * that frame.sync() must be used directly, to avoid syncExit()'s
1457              * jumping logic.
1458              */
1459             frame.sync(stubcc.masm, Uses(frame.frameSlots()));
1460             stubcc.leave();
1461             OOL_STUBCALL(stub);
1462         }
1463
1464         /* Forget the world, preserving data. */
1465         frame.pinReg(cmpReg);
1466         if (reg.isSet())
1467             frame.pinReg(reg.reg());
1468
1469         frame.popn(2);
1470
1471         frame.syncAndKillEverything();
1472         frame.unpinKilledReg(cmpReg);
1473         if (reg.isSet())
1474             frame.unpinKilledReg(reg.reg());
1475         frame.syncAndForgetEverything();
1476
1477         /* Operands could have been reordered, so use cmpOp. */
1478         Assembler::Condition i32Cond;
1479         bool ifeq = fused == JSOP_IFEQ;
1480         switch (cmpOp) {
1481           case JSOP_GT:
1482             i32Cond = ifeq ? Assembler::LessThanOrEqual : Assembler::GreaterThan;
1483             break;
1484           case JSOP_GE:
1485             i32Cond = ifeq ? Assembler::LessThan : Assembler::GreaterThanOrEqual;
1486             break;
1487           case JSOP_LT:
1488             i32Cond = ifeq ? Assembler::GreaterThanOrEqual : Assembler::LessThan;
1489             break;
1490           case JSOP_LE:
1491             i32Cond = ifeq ? Assembler::GreaterThan : Assembler::LessThanOrEqual;
1492             break;
1493           default:
1494             JS_NOT_REACHED("unrecognized op");
1495             return false;
1496         }
1497
1498         /* Emit the i32 path. */
1499         Jump fast;
1500         if (reg.isSet())
1501             fast = masm.branch32(i32Cond, cmpReg, reg.reg());
1502         else
1503             fast = masm.branch32(i32Cond, cmpReg, Imm32(value));
1504
1505         /*
1506          * The stub call has no need to rejoin since state is synced. Instead,
1507          * we can just test the return value.
1508          */
1509         Assembler::Condition cond = (fused == JSOP_IFEQ)
1510                                     ? Assembler::Zero
1511                                     : Assembler::NonZero;
1512         Jump j = stubcc.masm.branchTest32(cond, Registers::ReturnReg, Registers::ReturnReg);
1513
1514         /* Rejoin from the slow path. */
1515         Jump j2 = stubcc.masm.jump();
1516         stubcc.crossJump(j2, masm.label());
1517
1518         /* :TODO: make double path invoke tracer. */
1519         if (hasDoublePath) {
1520             j.linkTo(stubcc.masm.label(), &stubcc.masm);
1521             doubleTest.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1522             j = stubcc.masm.jump();
1523         }
1524
1525         /*
1526          * NB: jumpAndTrace emits to the OOL path, so make sure not to use it
1527          * in the middle of an in-progress slow path.
1528          */
1529         if (!jumpAndTrace(fast, target, &j))
1530             return false;
1531
1532         /* Rejoin from the double path. */
1533         if (hasDoublePath)
1534             stubcc.crossJump(doubleFall.get(), masm.label());
1535     } else {
1536         /*
1537          * Emit the double path now, necessary to complete the OOL fast-path
1538          * before emitting the slow path.
1539          */
1540         MaybeJump doubleDone;
1541         Assembler::DoubleCondition dblCond = DoubleCondForOp(op, JSOP_NOP);
1542         if (hasDoublePath) {
1543             if (lhsUnknownDone.isSet())
1544                 lhsUnknownDone.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1545             /* :FIXME: Use SET if we can? */
1546             Jump test = stubcc.masm.branchDouble(dblCond, fpLeft, fpRight);
1547             stubcc.masm.move(Imm32(0), regs.result);
1548             Jump skip = stubcc.masm.jump();
1549             test.linkTo(stubcc.masm.label(), &stubcc.masm);
1550             stubcc.masm.move(Imm32(1), regs.result);
1551             skip.linkTo(stubcc.masm.label(), &stubcc.masm);
1552             doubleDone = stubcc.masm.jump();
1553
1554             /* Link all incoming slow paths to here. */
1555             if (lhsNotDouble.isSet()) {
1556                 lhsNotDouble.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1557                 if (rhsNotNumber.isSet())
1558                     rhsNotNumber.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1559             }
1560             if (rhsNotNumber2.isSet())
1561                 rhsNotNumber2.get().linkTo(stubcc.masm.label(), &stubcc.masm);
1562
1563             /* Emit the slow path - note full frame syncage. */
1564             frame.sync(stubcc.masm, Uses(2));
1565             stubcc.leave();
1566             OOL_STUBCALL(stub);
1567         }
1568
1569         /* Get an integer comparison condition. */
1570         Assembler::Condition i32Cond;
1571         switch (cmpOp) {
1572           case JSOP_GT:
1573             i32Cond = Assembler::GreaterThan;
1574             break;
1575           case JSOP_GE:
1576             i32Cond = Assembler::GreaterThanOrEqual;
1577             break;
1578           case JSOP_LT:
1579             i32Cond = Assembler::LessThan;
1580             break;
1581           case JSOP_LE:
1582             i32Cond = Assembler::LessThanOrEqual;
1583             break;
1584           default:
1585             JS_NOT_REACHED("unrecognized op");
1586             return false;
1587         }
1588
1589         /* Emit the compare & set. */
1590         if (Registers::maskReg(regs.result) & Registers::SingleByteRegs) {
1591             if (reg.isSet())
1592                 masm.set32(i32Cond, cmpReg, reg.reg(), regs.result);
1593             else
1594                 masm.set32(i32Cond, cmpReg, Imm32(value), regs.result);
1595         } else {
1596             Jump j;
1597             if (reg.isSet())
1598                 j = masm.branch32(i32Cond, cmpReg, reg.reg());
1599             else
1600                 j = masm.branch32(i32Cond, cmpReg, Imm32(value));
1601             masm.move(Imm32(0), regs.result);
1602             Jump skip = masm.jump();
1603             j.linkTo(masm.label(),  &masm);
1604             masm.move(Imm32(1), regs.result);
1605             skip.linkTo(masm.label(), &masm);
1606         }
1607
1608         frame.popn(2);
1609         frame.pushTypedPayload(JSVAL_TYPE_BOOLEAN, regs.result);
1610
1611         if (hasDoublePath)
1612             stubcc.crossJump(doubleDone.get(), masm.label());
1613         stubcc.rejoin(Changes(1));
1614     }
1615     return true;
1616 }
1617