llvm/lib/Target/AMDGPU/R600ISelLowering.cpp

   1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// Custom DAG lowering for R600
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "R600ISelLowering.h"
  15 #include "AMDGPU.h"
  16 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  17 #include "R600Defines.h"
  18 #include "R600InstrInfo.h"
  19 #include "R600MachineFunctionInfo.h"
  20 #include "R600Subtarget.h"
  21 #include "llvm/CodeGen/MachineFunction.h"
  22 #include "llvm/IR/IntrinsicsAMDGPU.h"
  23 #include "llvm/IR/IntrinsicsR600.h"
  24
  25 using namespace llvm;
  26
  27 #include "R600GenCallingConv.inc"
  28
  29 R600TargetLowering::R600TargetLowering(const TargetMachine &TM,
  30                                        const R600Subtarget &STI)
  31     : AMDGPUTargetLowering(TM, STI), Subtarget(&STI), Gen(STI.getGeneration()) {
  32   addRegisterClass(MVT::f32, &R600::R600_Reg32RegClass);
  33   addRegisterClass(MVT::i32, &R600::R600_Reg32RegClass);
  34   addRegisterClass(MVT::v2f32, &R600::R600_Reg64RegClass);
  35   addRegisterClass(MVT::v2i32, &R600::R600_Reg64RegClass);
  36   addRegisterClass(MVT::v4f32, &R600::R600_Reg128RegClass);
  37   addRegisterClass(MVT::v4i32, &R600::R600_Reg128RegClass);
  38
  39   setBooleanContents(ZeroOrNegativeOneBooleanContent);
  40   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
  41
  42   computeRegisterProperties(Subtarget->getRegisterInfo());
  43
  44   // Legalize loads and stores to the private address space.
  45   setOperationAction(ISD::LOAD, MVT::i32, Custom);
  46   setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
  47   setOperationAction(ISD::LOAD, MVT::v4i32, Custom);
  48
  49   // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address
  50   // spaces, so it is custom lowered to handle those where it isn't.
  51   for (MVT VT : MVT::integer_valuetypes()) {
  52     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
  53     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Custom);
  54     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Custom);
  55
  56     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
  57     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Custom);
  58     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Custom);
  59
  60     setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
  61     setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Custom);
  62     setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Custom);
  63   }
  64
  65   // Workaround for LegalizeDAG asserting on expansion of i1 vector loads.
  66   setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i1, Expand);
  67   setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i1, Expand);
  68   setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i1, Expand);
  69
  70   setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i1, Expand);
  71   setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i1, Expand);
  72   setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i1, Expand);
  73
  74   setOperationAction(ISD::STORE, MVT::i8, Custom);
  75   setOperationAction(ISD::STORE, MVT::i32, Custom);
  76   setOperationAction(ISD::STORE, MVT::v2i32, Custom);
  77   setOperationAction(ISD::STORE, MVT::v4i32, Custom);
  78
  79   setTruncStoreAction(MVT::i32, MVT::i8, Custom);
  80   setTruncStoreAction(MVT::i32, MVT::i16, Custom);
  81   // We need to include these since trunc STORES to PRIVATE need
  82   // special handling to accommodate RMW
  83   setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
  84   setTruncStoreAction(MVT::v4i32, MVT::v4i16, Custom);
  85   setTruncStoreAction(MVT::v8i32, MVT::v8i16, Custom);
  86   setTruncStoreAction(MVT::v16i32, MVT::v16i16, Custom);
  87   setTruncStoreAction(MVT::v32i32, MVT::v32i16, Custom);
  88   setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
  89   setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom);
  90   setTruncStoreAction(MVT::v8i32, MVT::v8i8, Custom);
  91   setTruncStoreAction(MVT::v16i32, MVT::v16i8, Custom);
  92   setTruncStoreAction(MVT::v32i32, MVT::v32i8, Custom);
  93
  94   // Workaround for LegalizeDAG asserting on expansion of i1 vector stores.
  95   setTruncStoreAction(MVT::v2i32, MVT::v2i1, Expand);
  96   setTruncStoreAction(MVT::v4i32, MVT::v4i1, Expand);
  97
  98   // Set condition code actions
  99   setCondCodeAction(ISD::SETO,   MVT::f32, Expand);
 100   setCondCodeAction(ISD::SETUO,  MVT::f32, Expand);
 101   setCondCodeAction(ISD::SETLT,  MVT::f32, Expand);
 102   setCondCodeAction(ISD::SETLE,  MVT::f32, Expand);
 103   setCondCodeAction(ISD::SETOLT, MVT::f32, Expand);
 104   setCondCodeAction(ISD::SETOLE, MVT::f32, Expand);
 105   setCondCodeAction(ISD::SETONE, MVT::f32, Expand);
 106   setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand);
 107   setCondCodeAction(ISD::SETUGE, MVT::f32, Expand);
 108   setCondCodeAction(ISD::SETUGT, MVT::f32, Expand);
 109   setCondCodeAction(ISD::SETULT, MVT::f32, Expand);
 110   setCondCodeAction(ISD::SETULE, MVT::f32, Expand);
 111
 112   setCondCodeAction(ISD::SETLE, MVT::i32, Expand);
 113   setCondCodeAction(ISD::SETLT, MVT::i32, Expand);
 114   setCondCodeAction(ISD::SETULE, MVT::i32, Expand);
 115   setCondCodeAction(ISD::SETULT, MVT::i32, Expand);
 116
 117   setOperationAction(ISD::FCOS, MVT::f32, Custom);
 118   setOperationAction(ISD::FSIN, MVT::f32, Custom);
 119
 120   setOperationAction(ISD::SETCC, MVT::v4i32, Expand);
 121   setOperationAction(ISD::SETCC, MVT::v2i32, Expand);
 122
 123   setOperationAction(ISD::BR_CC, MVT::i32, Expand);
 124   setOperationAction(ISD::BR_CC, MVT::f32, Expand);
 125   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
 126
 127   setOperationAction(ISD::FSUB, MVT::f32, Expand);
 128
 129   setOperationAction(ISD::FCEIL, MVT::f64, Custom);
 130   setOperationAction(ISD::FTRUNC, MVT::f64, Custom);
 131   setOperationAction(ISD::FRINT, MVT::f64, Custom);
 132   setOperationAction(ISD::FFLOOR, MVT::f64, Custom);
 133
 134   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
 135   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
 136
 137   setOperationAction(ISD::SETCC, MVT::i32, Expand);
 138   setOperationAction(ISD::SETCC, MVT::f32, Expand);
 139   setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom);
 140   setOperationAction(ISD::FP_TO_SINT, MVT::i1, Custom);
 141   setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
 142   setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
 143
 144   setOperationAction(ISD::SELECT, MVT::i32, Expand);
 145   setOperationAction(ISD::SELECT, MVT::f32, Expand);
 146   setOperationAction(ISD::SELECT, MVT::v2i32, Expand);
 147   setOperationAction(ISD::SELECT, MVT::v4i32, Expand);
 148
 149   // ADD, SUB overflow.
 150   // TODO: turn these into Legal?
 151   if (Subtarget->hasCARRY())
 152     setOperationAction(ISD::UADDO, MVT::i32, Custom);
 153
 154   if (Subtarget->hasBORROW())
 155     setOperationAction(ISD::USUBO, MVT::i32, Custom);
 156
 157   // Expand sign extension of vectors
 158   if (!Subtarget->hasBFE())
 159     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 160
 161   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Expand);
 162   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Expand);
 163
 164   if (!Subtarget->hasBFE())
 165     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
 166   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Expand);
 167   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Expand);
 168
 169   if (!Subtarget->hasBFE())
 170     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
 171   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand);
 172   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Expand);
 173
 174   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
 175   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Expand);
 176   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Expand);
 177
 178   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Expand);
 179
 180   setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
 181
 182   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Custom);
 183   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom);
 184   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
 185   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
 186
 187   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i32, Custom);
 188   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Custom);
 189   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
 190   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
 191
 192   // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32
 193   //  to be Legal/Custom in order to avoid library calls.
 194   setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
 195   setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
 196   setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
 197
 198   if (!Subtarget->hasFMA()) {
 199     setOperationAction(ISD::FMA, MVT::f32, Expand);
 200     setOperationAction(ISD::FMA, MVT::f64, Expand);
 201   }
 202
 203   // FIXME: May need no denormals check
 204   setOperationAction(ISD::FMAD, MVT::f32, Legal);
 205
 206   if (!Subtarget->hasBFI()) {
 207     // fcopysign can be done in a single instruction with BFI.
 208     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
 209     setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
 210   }
 211
 212   if (!Subtarget->hasBCNT(32))
 213     setOperationAction(ISD::CTPOP, MVT::i32, Expand);
 214
 215   if (!Subtarget->hasBCNT(64))
 216     setOperationAction(ISD::CTPOP, MVT::i64, Expand);
 217
 218   if (Subtarget->hasFFBH())
 219     setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
 220
 221   if (Subtarget->hasFFBL())
 222     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
 223
 224   // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we
 225   // need it for R600.
 226   if (Subtarget->hasBFE())
 227     setHasExtractBitsInsn(true);
 228
 229   setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
 230
 231   const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
 232   for (MVT VT : ScalarIntVTs) {
 233     setOperationAction(ISD::ADDC, VT, Expand);
 234     setOperationAction(ISD::SUBC, VT, Expand);
 235     setOperationAction(ISD::ADDE, VT, Expand);
 236     setOperationAction(ISD::SUBE, VT, Expand);
 237   }
 238
 239   // LLVM will expand these to atomic_cmp_swap(0)
 240   // and atomic_swap, respectively.
 241   setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Expand);
 242   setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Expand);
 243
 244   // We need to custom lower some of the intrinsics
 245   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 246   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 247
 248   setSchedulingPreference(Sched::Source);
 249
 250   setTargetDAGCombine(ISD::FP_ROUND);
 251   setTargetDAGCombine(ISD::FP_TO_SINT);
 252   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
 253   setTargetDAGCombine(ISD::SELECT_CC);
 254   setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
 255   setTargetDAGCombine(ISD::LOAD);
 256 }
 257
 258 static inline bool isEOP(MachineBasicBlock::iterator I) {
 259   if (std::next(I) == I->getParent()->end())
 260     return false;
 261   return std::next(I)->getOpcode() == R600::RETURN;
 262 }
 263
 264 MachineBasicBlock *
 265 R600TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
 266                                                 MachineBasicBlock *BB) const {
 267   MachineFunction *MF = BB->getParent();
 268   MachineRegisterInfo &MRI = MF->getRegInfo();
 269   MachineBasicBlock::iterator I = MI;
 270   const R600InstrInfo *TII = Subtarget->getInstrInfo();
 271
 272   switch (MI.getOpcode()) {
 273   default:
 274     // Replace LDS_*_RET instruction that don't have any uses with the
 275     // equivalent LDS_*_NORET instruction.
 276     if (TII->isLDSRetInstr(MI.getOpcode())) {
 277       int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst);
 278       assert(DstIdx != -1);
 279       MachineInstrBuilder NewMI;
 280       // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add
 281       //        LDS_1A2D support and remove this special case.
 282       if (!MRI.use_empty(MI.getOperand(DstIdx).getReg()) ||
 283           MI.getOpcode() == R600::LDS_CMPST_RET)
 284         return BB;
 285
 286       NewMI = BuildMI(*BB, I, BB->findDebugLoc(I),
 287                       TII->get(R600::getLDSNoRetOp(MI.getOpcode())));
 288       for (unsigned i = 1, e = MI.getNumOperands(); i < e; ++i) {
 289         NewMI.add(MI.getOperand(i));
 290       }
 291     } else {
 292       return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
 293     }
 294     break;
 295
 296   case R600::FABS_R600: {
 297     MachineInstr *NewMI = TII->buildDefaultInstruction(
 298         *BB, I, R600::MOV, MI.getOperand(0).getReg(),
 299         MI.getOperand(1).getReg());
 300     TII->addFlag(*NewMI, 0, MO_FLAG_ABS);
 301     break;
 302   }
 303
 304   case R600::FNEG_R600: {
 305     MachineInstr *NewMI = TII->buildDefaultInstruction(
 306         *BB, I, R600::MOV, MI.getOperand(0).getReg(),
 307         MI.getOperand(1).getReg());
 308     TII->addFlag(*NewMI, 0, MO_FLAG_NEG);
 309     break;
 310   }
 311
 312   case R600::MASK_WRITE: {
 313     Register maskedRegister = MI.getOperand(0).getReg();
 314     assert(maskedRegister.isVirtual());
 315     MachineInstr * defInstr = MRI.getVRegDef(maskedRegister);
 316     TII->addFlag(*defInstr, 0, MO_FLAG_MASK);
 317     break;
 318   }
 319
 320   case R600::MOV_IMM_F32:
 321     TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), MI.getOperand(1)
 322                                                             .getFPImm()
 323                                                             ->getValueAPF()
 324                                                             .bitcastToAPInt()
 325                                                             .getZExtValue());
 326     break;
 327
 328   case R600::MOV_IMM_I32:
 329     TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(),
 330                      MI.getOperand(1).getImm());
 331     break;
 332
 333   case R600::MOV_IMM_GLOBAL_ADDR: {
 334     //TODO: Perhaps combine this instruction with the next if possible
 335     auto MIB = TII->buildDefaultInstruction(
 336         *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_LITERAL_X);
 337     int Idx = TII->getOperandIdx(*MIB, R600::OpName::literal);
 338     //TODO: Ugh this is rather ugly
 339     MIB->getOperand(Idx) = MI.getOperand(1);
 340     break;
 341   }
 342
 343   case R600::CONST_COPY: {
 344     MachineInstr *NewMI = TII->buildDefaultInstruction(
 345         *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_CONST);
 346     TII->setImmOperand(*NewMI, R600::OpName::src0_sel,
 347                        MI.getOperand(1).getImm());
 348     break;
 349   }
 350
 351   case R600::RAT_WRITE_CACHELESS_32_eg:
 352   case R600::RAT_WRITE_CACHELESS_64_eg:
 353   case R600::RAT_WRITE_CACHELESS_128_eg:
 354     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
 355         .add(MI.getOperand(0))
 356         .add(MI.getOperand(1))
 357         .addImm(isEOP(I)); // Set End of program bit
 358     break;
 359
 360   case R600::RAT_STORE_TYPED_eg:
 361     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
 362         .add(MI.getOperand(0))
 363         .add(MI.getOperand(1))
 364         .add(MI.getOperand(2))
 365         .addImm(isEOP(I)); // Set End of program bit
 366     break;
 367
 368   case R600::BRANCH:
 369     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP))
 370         .add(MI.getOperand(0));
 371     break;
 372
 373   case R600::BRANCH_COND_f32: {
 374     MachineInstr *NewMI =
 375         BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X),
 376                 R600::PREDICATE_BIT)
 377             .add(MI.getOperand(1))
 378             .addImm(R600::PRED_SETNE)
 379             .addImm(0); // Flags
 380     TII->addFlag(*NewMI, 0, MO_FLAG_PUSH);
 381     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND))
 382         .add(MI.getOperand(0))
 383         .addReg(R600::PREDICATE_BIT, RegState::Kill);
 384     break;
 385   }
 386
 387   case R600::BRANCH_COND_i32: {
 388     MachineInstr *NewMI =
 389         BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X),
 390                 R600::PREDICATE_BIT)
 391             .add(MI.getOperand(1))
 392             .addImm(R600::PRED_SETNE_INT)
 393             .addImm(0); // Flags
 394     TII->addFlag(*NewMI, 0, MO_FLAG_PUSH);
 395     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND))
 396         .add(MI.getOperand(0))
 397         .addReg(R600::PREDICATE_BIT, RegState::Kill);
 398     break;
 399   }
 400
 401   case R600::EG_ExportSwz:
 402   case R600::R600_ExportSwz: {
 403     // Instruction is left unmodified if its not the last one of its type
 404     bool isLastInstructionOfItsType = true;
 405     unsigned InstExportType = MI.getOperand(1).getImm();
 406     for (MachineBasicBlock::iterator NextExportInst = std::next(I),
 407          EndBlock = BB->end(); NextExportInst != EndBlock;
 408          NextExportInst = std::next(NextExportInst)) {
 409       if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
 410           NextExportInst->getOpcode() == R600::R600_ExportSwz) {
 411         unsigned CurrentInstExportType = NextExportInst->getOperand(1)
 412             .getImm();
 413         if (CurrentInstExportType == InstExportType) {
 414           isLastInstructionOfItsType = false;
 415           break;
 416         }
 417       }
 418     }
 419     bool EOP = isEOP(I);
 420     if (!EOP && !isLastInstructionOfItsType)
 421       return BB;
 422     unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
 423     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
 424         .add(MI.getOperand(0))
 425         .add(MI.getOperand(1))
 426         .add(MI.getOperand(2))
 427         .add(MI.getOperand(3))
 428         .add(MI.getOperand(4))
 429         .add(MI.getOperand(5))
 430         .add(MI.getOperand(6))
 431         .addImm(CfInst)
 432         .addImm(EOP);
 433     break;
 434   }
 435   case R600::RETURN: {
 436     return BB;
 437   }
 438   }
 439
 440   MI.eraseFromParent();
 441   return BB;
 442 }
 443
 444 //===----------------------------------------------------------------------===//
 445 // Custom DAG Lowering Operations
 446 //===----------------------------------------------------------------------===//
 447
 448 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
 449   MachineFunction &MF = DAG.getMachineFunction();
 450   R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
 451   switch (Op.getOpcode()) {
 452   default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
 453   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
 454   case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
 455   case ISD::SHL_PARTS:
 456   case ISD::SRA_PARTS:
 457   case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
 458   case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY);
 459   case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW);
 460   case ISD::FCOS:
 461   case ISD::FSIN: return LowerTrig(Op, DAG);
 462   case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
 463   case ISD::STORE: return LowerSTORE(Op, DAG);
 464   case ISD::LOAD: {
 465     SDValue Result = LowerLOAD(Op, DAG);
 466     assert((!Result.getNode() ||
 467             Result.getNode()->getNumValues() == 2) &&
 468            "Load should return a value and a chain");
 469     return Result;
 470   }
 471
 472   case ISD::BRCOND: return LowerBRCOND(Op, DAG);
 473   case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG);
 474   case ISD::FrameIndex: return lowerFrameIndex(Op, DAG);
 475   case ISD::INTRINSIC_VOID: {
 476     SDValue Chain = Op.getOperand(0);
 477     unsigned IntrinsicID =
 478                          cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
 479     switch (IntrinsicID) {
 480     case Intrinsic::r600_store_swizzle: {
 481       SDLoc DL(Op);
 482       const SDValue Args[8] = {
 483         Chain,
 484         Op.getOperand(2), // Export Value
 485         Op.getOperand(3), // ArrayBase
 486         Op.getOperand(4), // Type
 487         DAG.getConstant(0, DL, MVT::i32), // SWZ_X
 488         DAG.getConstant(1, DL, MVT::i32), // SWZ_Y
 489         DAG.getConstant(2, DL, MVT::i32), // SWZ_Z
 490         DAG.getConstant(3, DL, MVT::i32) // SWZ_W
 491       };
 492       return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, Op.getValueType(), Args);
 493     }
 494
 495     // default for switch(IntrinsicID)
 496     default: break;
 497     }
 498     // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode())
 499     break;
 500   }
 501   case ISD::INTRINSIC_WO_CHAIN: {
 502     unsigned IntrinsicID =
 503                          cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
 504     EVT VT = Op.getValueType();
 505     SDLoc DL(Op);
 506     switch (IntrinsicID) {
 507     case Intrinsic::r600_tex:
 508     case Intrinsic::r600_texc: {
 509       unsigned TextureOp;
 510       switch (IntrinsicID) {
 511       case Intrinsic::r600_tex:
 512         TextureOp = 0;
 513         break;
 514       case Intrinsic::r600_texc:
 515         TextureOp = 1;
 516         break;
 517       default:
 518         llvm_unreachable("unhandled texture operation");
 519       }
 520
 521       SDValue TexArgs[19] = {
 522         DAG.getConstant(TextureOp, DL, MVT::i32),
 523         Op.getOperand(1),
 524         DAG.getConstant(0, DL, MVT::i32),
 525         DAG.getConstant(1, DL, MVT::i32),
 526         DAG.getConstant(2, DL, MVT::i32),
 527         DAG.getConstant(3, DL, MVT::i32),
 528         Op.getOperand(2),
 529         Op.getOperand(3),
 530         Op.getOperand(4),
 531         DAG.getConstant(0, DL, MVT::i32),
 532         DAG.getConstant(1, DL, MVT::i32),
 533         DAG.getConstant(2, DL, MVT::i32),
 534         DAG.getConstant(3, DL, MVT::i32),
 535         Op.getOperand(5),
 536         Op.getOperand(6),
 537         Op.getOperand(7),
 538         Op.getOperand(8),
 539         Op.getOperand(9),
 540         Op.getOperand(10)
 541       };
 542       return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs);
 543     }
 544     case Intrinsic::r600_dot4: {
 545       SDValue Args[8] = {
 546       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
 547           DAG.getConstant(0, DL, MVT::i32)),
 548       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
 549           DAG.getConstant(0, DL, MVT::i32)),
 550       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
 551           DAG.getConstant(1, DL, MVT::i32)),
 552       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
 553           DAG.getConstant(1, DL, MVT::i32)),
 554       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
 555           DAG.getConstant(2, DL, MVT::i32)),
 556       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
 557           DAG.getConstant(2, DL, MVT::i32)),
 558       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
 559           DAG.getConstant(3, DL, MVT::i32)),
 560       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
 561           DAG.getConstant(3, DL, MVT::i32))
 562       };
 563       return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args);
 564     }
 565
 566     case Intrinsic::r600_implicitarg_ptr: {
 567       MVT PtrVT = getPointerTy(DAG.getDataLayout(), AMDGPUAS::PARAM_I_ADDRESS);
 568       uint32_t ByteOffset = getImplicitParameterOffset(MF, FIRST_IMPLICIT);
 569       return DAG.getConstant(ByteOffset, DL, PtrVT);
 570     }
 571     case Intrinsic::r600_read_ngroups_x:
 572       return LowerImplicitParameter(DAG, VT, DL, 0);
 573     case Intrinsic::r600_read_ngroups_y:
 574       return LowerImplicitParameter(DAG, VT, DL, 1);
 575     case Intrinsic::r600_read_ngroups_z:
 576       return LowerImplicitParameter(DAG, VT, DL, 2);
 577     case Intrinsic::r600_read_global_size_x:
 578       return LowerImplicitParameter(DAG, VT, DL, 3);
 579     case Intrinsic::r600_read_global_size_y:
 580       return LowerImplicitParameter(DAG, VT, DL, 4);
 581     case Intrinsic::r600_read_global_size_z:
 582       return LowerImplicitParameter(DAG, VT, DL, 5);
 583     case Intrinsic::r600_read_local_size_x:
 584       return LowerImplicitParameter(DAG, VT, DL, 6);
 585     case Intrinsic::r600_read_local_size_y:
 586       return LowerImplicitParameter(DAG, VT, DL, 7);
 587     case Intrinsic::r600_read_local_size_z:
 588       return LowerImplicitParameter(DAG, VT, DL, 8);
 589
 590     case Intrinsic::r600_read_tgid_x:
 591     case Intrinsic::amdgcn_workgroup_id_x:
 592       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 593                                      R600::T1_X, VT);
 594     case Intrinsic::r600_read_tgid_y:
 595     case Intrinsic::amdgcn_workgroup_id_y:
 596       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 597                                      R600::T1_Y, VT);
 598     case Intrinsic::r600_read_tgid_z:
 599     case Intrinsic::amdgcn_workgroup_id_z:
 600       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 601                                      R600::T1_Z, VT);
 602     case Intrinsic::r600_read_tidig_x:
 603     case Intrinsic::amdgcn_workitem_id_x:
 604       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 605                                      R600::T0_X, VT);
 606     case Intrinsic::r600_read_tidig_y:
 607     case Intrinsic::amdgcn_workitem_id_y:
 608       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 609                                      R600::T0_Y, VT);
 610     case Intrinsic::r600_read_tidig_z:
 611     case Intrinsic::amdgcn_workitem_id_z:
 612       return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
 613                                      R600::T0_Z, VT);
 614
 615     case Intrinsic::r600_recipsqrt_ieee:
 616       return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
 617
 618     case Intrinsic::r600_recipsqrt_clamped:
 619       return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1));
 620     default:
 621       return Op;
 622     }
 623
 624     // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode())
 625     break;
 626   }
 627   } // end switch(Op.getOpcode())
 628   return SDValue();
 629 }
 630
 631 void R600TargetLowering::ReplaceNodeResults(SDNode *N,
 632                                             SmallVectorImpl<SDValue> &Results,
 633                                             SelectionDAG &DAG) const {
 634   switch (N->getOpcode()) {
 635   default:
 636     AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
 637     return;
 638   case ISD::FP_TO_UINT:
 639     if (N->getValueType(0) == MVT::i1) {
 640       Results.push_back(lowerFP_TO_UINT(N->getOperand(0), DAG));
 641       return;
 642     }
 643     // Since we don't care about out of bounds values we can use FP_TO_SINT for
 644     // uints too. The DAGLegalizer code for uint considers some extra cases
 645     // which are not necessary here.
 646     LLVM_FALLTHROUGH;
 647   case ISD::FP_TO_SINT: {
 648     if (N->getValueType(0) == MVT::i1) {
 649       Results.push_back(lowerFP_TO_SINT(N->getOperand(0), DAG));
 650       return;
 651     }
 652
 653     SDValue Result;
 654     if (expandFP_TO_SINT(N, Result, DAG))
 655       Results.push_back(Result);
 656     return;
 657   }
 658   case ISD::SDIVREM: {
 659     SDValue Op = SDValue(N, 1);
 660     SDValue RES = LowerSDIVREM(Op, DAG);
 661     Results.push_back(RES);
 662     Results.push_back(RES.getValue(1));
 663     break;
 664   }
 665   case ISD::UDIVREM: {
 666     SDValue Op = SDValue(N, 0);
 667     LowerUDIVREM64(Op, DAG, Results);
 668     break;
 669   }
 670   }
 671 }
 672
 673 SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
 674                                                    SDValue Vector) const {
 675   SDLoc DL(Vector);
 676   EVT VecVT = Vector.getValueType();
 677   EVT EltVT = VecVT.getVectorElementType();
 678   SmallVector<SDValue, 8> Args;
 679
 680   for (unsigned i = 0, e = VecVT.getVectorNumElements(); i != e; ++i) {
 681     Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector,
 682                                DAG.getVectorIdxConstant(i, DL)));
 683   }
 684
 685   return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args);
 686 }
 687
 688 SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 689                                                     SelectionDAG &DAG) const {
 690   SDLoc DL(Op);
 691   SDValue Vector = Op.getOperand(0);
 692   SDValue Index = Op.getOperand(1);
 693
 694   if (isa<ConstantSDNode>(Index) ||
 695       Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
 696     return Op;
 697
 698   Vector = vectorToVerticalVector(DAG, Vector);
 699   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(),
 700                      Vector, Index);
 701 }
 702
 703 SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
 704                                                    SelectionDAG &DAG) const {
 705   SDLoc DL(Op);
 706   SDValue Vector = Op.getOperand(0);
 707   SDValue Value = Op.getOperand(1);
 708   SDValue Index = Op.getOperand(2);
 709
 710   if (isa<ConstantSDNode>(Index) ||
 711       Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
 712     return Op;
 713
 714   Vector = vectorToVerticalVector(DAG, Vector);
 715   SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(),
 716                                Vector, Value, Index);
 717   return vectorToVerticalVector(DAG, Insert);
 718 }
 719
 720 SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
 721                                                SDValue Op,
 722                                                SelectionDAG &DAG) const {
 723   GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
 724   if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
 725     return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
 726
 727   const DataLayout &DL = DAG.getDataLayout();
 728   const GlobalValue *GV = GSD->getGlobal();
 729   MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 730
 731   SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT);
 732   return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA);
 733 }
 734
 735 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
 736   // On hw >= R700, COS/SIN input must be between -1. and 1.
 737   // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
 738   EVT VT = Op.getValueType();
 739   SDValue Arg = Op.getOperand(0);
 740   SDLoc DL(Op);
 741
 742   // TODO: Should this propagate fast-math-flags?
 743   SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT,
 744       DAG.getNode(ISD::FADD, DL, VT,
 745         DAG.getNode(ISD::FMUL, DL, VT, Arg,
 746           DAG.getConstantFP(0.15915494309, DL, MVT::f32)),
 747         DAG.getConstantFP(0.5, DL, MVT::f32)));
 748   unsigned TrigNode;
 749   switch (Op.getOpcode()) {
 750   case ISD::FCOS:
 751     TrigNode = AMDGPUISD::COS_HW;
 752     break;
 753   case ISD::FSIN:
 754     TrigNode = AMDGPUISD::SIN_HW;
 755     break;
 756   default:
 757     llvm_unreachable("Wrong trig opcode");
 758   }
 759   SDValue TrigVal = DAG.getNode(TrigNode, DL, VT,
 760       DAG.getNode(ISD::FADD, DL, VT, FractPart,
 761         DAG.getConstantFP(-0.5, DL, MVT::f32)));
 762   if (Gen >= AMDGPUSubtarget::R700)
 763     return TrigVal;
 764   // On R600 hw, COS/SIN input must be between -Pi and Pi.
 765   return DAG.getNode(ISD::FMUL, DL, VT, TrigVal,
 766       DAG.getConstantFP(numbers::pif, DL, MVT::f32));
 767 }
 768
 769 SDValue R600TargetLowering::LowerShiftParts(SDValue Op,
 770                                             SelectionDAG &DAG) const {
 771   SDValue Lo, Hi;
 772   expandShiftParts(Op.getNode(), Lo, Hi, DAG);
 773   return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
 774 }
 775
 776 SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
 777                                           unsigned mainop, unsigned ovf) const {
 778   SDLoc DL(Op);
 779   EVT VT = Op.getValueType();
 780
 781   SDValue Lo = Op.getOperand(0);
 782   SDValue Hi = Op.getOperand(1);
 783
 784   SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi);
 785   // Extend sign.
 786   OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF,
 787                     DAG.getValueType(MVT::i1));
 788
 789   SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi);
 790
 791   return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF);
 792 }
 793
 794 SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
 795   SDLoc DL(Op);
 796   return DAG.getNode(
 797       ISD::SETCC,
 798       DL,
 799       MVT::i1,
 800       Op, DAG.getConstantFP(1.0f, DL, MVT::f32),
 801       DAG.getCondCode(ISD::SETEQ));
 802 }
 803
 804 SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
 805   SDLoc DL(Op);
 806   return DAG.getNode(
 807       ISD::SETCC,
 808       DL,
 809       MVT::i1,
 810       Op, DAG.getConstantFP(-1.0f, DL, MVT::f32),
 811       DAG.getCondCode(ISD::SETEQ));
 812 }
 813
 814 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
 815                                                    const SDLoc &DL,
 816                                                    unsigned DwordOffset) const {
 817   unsigned ByteOffset = DwordOffset * 4;
 818   PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()),
 819                                       AMDGPUAS::PARAM_I_ADDRESS);
 820
 821   // We shouldn't be using an offset wider than 16-bits for implicit parameters.
 822   assert(isInt<16>(ByteOffset));
 823
 824   return DAG.getLoad(VT, DL, DAG.getEntryNode(),
 825                      DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR
 826                      MachinePointerInfo(ConstantPointerNull::get(PtrType)));
 827 }
 828
 829 bool R600TargetLowering::isZero(SDValue Op) const {
 830   if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
 831     return Cst->isNullValue();
 832   } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){
 833     return CstFP->isZero();
 834   } else {
 835     return false;
 836   }
 837 }
 838
 839 bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
 840   if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
 841     return CFP->isExactlyValue(1.0);
 842   }
 843   return isAllOnesConstant(Op);
 844 }
 845
 846 bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
 847   if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
 848     return CFP->getValueAPF().isZero();
 849   }
 850   return isNullConstant(Op);
 851 }
 852
 853 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
 854   SDLoc DL(Op);
 855   EVT VT = Op.getValueType();
 856
 857   SDValue LHS = Op.getOperand(0);
 858   SDValue RHS = Op.getOperand(1);
 859   SDValue True = Op.getOperand(2);
 860   SDValue False = Op.getOperand(3);
 861   SDValue CC = Op.getOperand(4);
 862   SDValue Temp;
 863
 864   if (VT == MVT::f32) {
 865     DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
 866     SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI);
 867     if (MinMax)
 868       return MinMax;
 869   }
 870
 871   // LHS and RHS are guaranteed to be the same value type
 872   EVT CompareVT = LHS.getValueType();
 873
 874   // Check if we can lower this to a native operation.
 875
 876   // Try to lower to a SET* instruction:
 877   //
 878   // SET* can match the following patterns:
 879   //
 880   // select_cc f32, f32, -1,  0, cc_supported
 881   // select_cc f32, f32, 1.0f, 0.0f, cc_supported
 882   // select_cc i32, i32, -1,  0, cc_supported
 883   //
 884
 885   // Move hardware True/False values to the correct operand.
 886   if (isHWTrueValue(False) && isHWFalseValue(True)) {
 887     ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
 888     ISD::CondCode InverseCC = ISD::getSetCCInverse(CCOpcode, CompareVT);
 889     if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) {
 890       std::swap(False, True);
 891       CC = DAG.getCondCode(InverseCC);
 892     } else {
 893       ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC);
 894       if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) {
 895         std::swap(False, True);
 896         std::swap(LHS, RHS);
 897         CC = DAG.getCondCode(SwapInvCC);
 898       }
 899     }
 900   }
 901
 902   if (isHWTrueValue(True) && isHWFalseValue(False) &&
 903       (CompareVT == VT || VT == MVT::i32)) {
 904     // This can be matched by a SET* instruction.
 905     return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC);
 906   }
 907
 908   // Try to lower to a CND* instruction:
 909   //
 910   // CND* can match the following patterns:
 911   //
 912   // select_cc f32, 0.0, f32, f32, cc_supported
 913   // select_cc f32, 0.0, i32, i32, cc_supported
 914   // select_cc i32, 0,   f32, f32, cc_supported
 915   // select_cc i32, 0,   i32, i32, cc_supported
 916   //
 917
 918   // Try to move the zero value to the RHS
 919   if (isZero(LHS)) {
 920     ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
 921     // Try swapping the operands
 922     ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode);
 923     if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) {
 924       std::swap(LHS, RHS);
 925       CC = DAG.getCondCode(CCSwapped);
 926     } else {
 927       // Try inverting the conditon and then swapping the operands
 928       ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT);
 929       CCSwapped = ISD::getSetCCSwappedOperands(CCInv);
 930       if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) {
 931         std::swap(True, False);
 932         std::swap(LHS, RHS);
 933         CC = DAG.getCondCode(CCSwapped);
 934       }
 935     }
 936   }
 937   if (isZero(RHS)) {
 938     SDValue Cond = LHS;
 939     SDValue Zero = RHS;
 940     ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
 941     if (CompareVT != VT) {
 942       // Bitcast True / False to the correct types.  This will end up being
 943       // a nop, but it allows us to define only a single pattern in the
 944       // .TD files for each CND* instruction rather than having to have
 945       // one pattern for integer True/False and one for fp True/False
 946       True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True);
 947       False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False);
 948     }
 949
 950     switch (CCOpcode) {
 951     case ISD::SETONE:
 952     case ISD::SETUNE:
 953     case ISD::SETNE:
 954       CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT);
 955       Temp = True;
 956       True = False;
 957       False = Temp;
 958       break;
 959     default:
 960       break;
 961     }
 962     SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
 963         Cond, Zero,
 964         True, False,
 965         DAG.getCondCode(CCOpcode));
 966     return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode);
 967   }
 968
 969   // If we make it this for it means we have no native instructions to handle
 970   // this SELECT_CC, so we must lower it.
 971   SDValue HWTrue, HWFalse;
 972
 973   if (CompareVT == MVT::f32) {
 974     HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT);
 975     HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT);
 976   } else if (CompareVT == MVT::i32) {
 977     HWTrue = DAG.getConstant(-1, DL, CompareVT);
 978     HWFalse = DAG.getConstant(0, DL, CompareVT);
 979   }
 980   else {
 981     llvm_unreachable("Unhandled value type in LowerSELECT_CC");
 982   }
 983
 984   // Lower this unsupported SELECT_CC into a combination of two supported
 985   // SELECT_CC operations.
 986   SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC);
 987
 988   return DAG.getNode(ISD::SELECT_CC, DL, VT,
 989       Cond, HWFalse,
 990       True, False,
 991       DAG.getCondCode(ISD::SETNE));
 992 }
 993
 994 /// LLVM generates byte-addressed pointers.  For indirect addressing, we need to
 995 /// convert these pointers to a register index.  Each register holds
 996 /// 16 bytes, (4 x 32bit sub-register), but we need to take into account the
 997 /// \p StackWidth, which tells us how many of the 4 sub-registrers will be used
 998 /// for indirect addressing.
 999 SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
1000                                                unsigned StackWidth,
1001                                                SelectionDAG &DAG) const {
1002   unsigned SRLPad;
1003   switch(StackWidth) {
1004   case 1:
1005     SRLPad = 2;
1006     break;
1007   case 2:
1008     SRLPad = 3;
1009     break;
1010   case 4:
1011     SRLPad = 4;
1012     break;
1013   default: llvm_unreachable("Invalid stack width");
1014   }
1015
1016   SDLoc DL(Ptr);
1017   return DAG.getNode(ISD::SRL, DL, Ptr.getValueType(), Ptr,
1018                      DAG.getConstant(SRLPad, DL, MVT::i32));
1019 }
1020
1021 void R600TargetLowering::getStackAddress(unsigned StackWidth,
1022                                          unsigned ElemIdx,
1023                                          unsigned &Channel,
1024                                          unsigned &PtrIncr) const {
1025   switch (StackWidth) {
1026   default:
1027   case 1:
1028     Channel = 0;
1029     if (ElemIdx > 0) {
1030       PtrIncr = 1;
1031     } else {
1032       PtrIncr = 0;
1033     }
1034     break;
1035   case 2:
1036     Channel = ElemIdx % 2;
1037     if (ElemIdx == 2) {
1038       PtrIncr = 1;
1039     } else {
1040       PtrIncr = 0;
1041     }
1042     break;
1043   case 4:
1044     Channel = ElemIdx;
1045     PtrIncr = 0;
1046     break;
1047   }
1048 }
1049
1050 SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
1051                                                    SelectionDAG &DAG) const {
1052   SDLoc DL(Store);
1053   //TODO: Who creates the i8 stores?
1054   assert(Store->isTruncatingStore()
1055          || Store->getValue().getValueType() == MVT::i8);
1056   assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS);
1057
1058   SDValue Mask;
1059   if (Store->getMemoryVT() == MVT::i8) {
1060     assert(Store->getAlignment() >= 1);
1061     Mask = DAG.getConstant(0xff, DL, MVT::i32);
1062   } else if (Store->getMemoryVT() == MVT::i16) {
1063     assert(Store->getAlignment() >= 2);
1064     Mask = DAG.getConstant(0xffff, DL, MVT::i32);
1065   } else {
1066     llvm_unreachable("Unsupported private trunc store");
1067   }
1068
1069   SDValue OldChain = Store->getChain();
1070   bool VectorTrunc = (OldChain.getOpcode() == AMDGPUISD::DUMMY_CHAIN);
1071   // Skip dummy
1072   SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain;
1073   SDValue BasePtr = Store->getBasePtr();
1074   SDValue Offset = Store->getOffset();
1075   EVT MemVT = Store->getMemoryVT();
1076
1077   SDValue LoadPtr = BasePtr;
1078   if (!Offset.isUndef()) {
1079     LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset);
1080   }
1081
1082   // Get dword location
1083   // TODO: this should be eliminated by the future SHR ptr, 2
1084   SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1085                             DAG.getConstant(0xfffffffc, DL, MVT::i32));
1086
1087   // Load dword
1088   // TODO: can we be smarter about machine pointer info?
1089   MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1090   SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1091
1092   Chain = Dst.getValue(1);
1093
1094   // Get offset in dword
1095   SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1096                                 DAG.getConstant(0x3, DL, MVT::i32));
1097
1098   // Convert byte offset to bit shift
1099   SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1100                                  DAG.getConstant(3, DL, MVT::i32));
1101
1102   // TODO: Contrary to the name of the functiom,
1103   // it also handles sub i32 non-truncating stores (like i1)
1104   SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1105                                   Store->getValue());
1106
1107   // Mask the value to the right type
1108   SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1109
1110   // Shift the value in place
1111   SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1112                                      MaskedValue, ShiftAmt);
1113
1114   // Shift the mask in place
1115   SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, Mask, ShiftAmt);
1116
1117   // Invert the mask. NOTE: if we had native ROL instructions we could
1118   // use inverted mask
1119   DstMask = DAG.getNOT(DL, DstMask, MVT::i32);
1120
1121   // Cleanup the target bits
1122   Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
1123
1124   // Add the new bits
1125   SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1126
1127   // Store dword
1128   // TODO: Can we be smarter about MachinePointerInfo?
1129   SDValue NewStore = DAG.getStore(Chain, DL, Value, Ptr, PtrInfo);
1130
1131   // If we are part of expanded vector, make our neighbors depend on this store
1132   if (VectorTrunc) {
1133     // Make all other vector elements depend on this store
1134     Chain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, NewStore);
1135     DAG.ReplaceAllUsesOfValueWith(OldChain, Chain);
1136   }
1137   return NewStore;
1138 }
1139
1140 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1141   StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
1142   unsigned AS = StoreNode->getAddressSpace();
1143
1144   SDValue Chain = StoreNode->getChain();
1145   SDValue Ptr = StoreNode->getBasePtr();
1146   SDValue Value = StoreNode->getValue();
1147
1148   EVT VT = Value.getValueType();
1149   EVT MemVT = StoreNode->getMemoryVT();
1150   EVT PtrVT = Ptr.getValueType();
1151
1152   SDLoc DL(Op);
1153
1154   const bool TruncatingStore = StoreNode->isTruncatingStore();
1155
1156   // Neither LOCAL nor PRIVATE can do vectors at the moment
1157   if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS ||
1158        TruncatingStore) &&
1159       VT.isVector()) {
1160     if ((AS == AMDGPUAS::PRIVATE_ADDRESS) && TruncatingStore) {
1161       // Add an extra level of chain to isolate this vector
1162       SDValue NewChain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, Chain);
1163       // TODO: can the chain be replaced without creating a new store?
1164       SDValue NewStore = DAG.getTruncStore(
1165           NewChain, DL, Value, Ptr, StoreNode->getPointerInfo(),
1166           MemVT, StoreNode->getAlignment(),
1167           StoreNode->getMemOperand()->getFlags(), StoreNode->getAAInfo());
1168       StoreNode = cast<StoreSDNode>(NewStore);
1169     }
1170
1171     return scalarizeVectorStore(StoreNode, DAG);
1172   }
1173
1174   Align Alignment = StoreNode->getAlign();
1175   if (Alignment < MemVT.getStoreSize() &&
1176       !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment,
1177                                       StoreNode->getMemOperand()->getFlags(),
1178                                       nullptr)) {
1179     return expandUnalignedStore(StoreNode, DAG);
1180   }
1181
1182   SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, PtrVT, Ptr,
1183                                   DAG.getConstant(2, DL, PtrVT));
1184
1185   if (AS == AMDGPUAS::GLOBAL_ADDRESS) {
1186     // It is beneficial to create MSKOR here instead of combiner to avoid
1187     // artificial dependencies introduced by RMW
1188     if (TruncatingStore) {
1189       assert(VT.bitsLE(MVT::i32));
1190       SDValue MaskConstant;
1191       if (MemVT == MVT::i8) {
1192         MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32);
1193       } else {
1194         assert(MemVT == MVT::i16);
1195         assert(StoreNode->getAlignment() >= 2);
1196         MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32);
1197       }
1198
1199       SDValue ByteIndex = DAG.getNode(ISD::AND, DL, PtrVT, Ptr,
1200                                       DAG.getConstant(0x00000003, DL, PtrVT));
1201       SDValue BitShift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex,
1202                                      DAG.getConstant(3, DL, VT));
1203
1204       // Put the mask in correct place
1205       SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, BitShift);
1206
1207       // Put the value bits in correct place
1208       SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant);
1209       SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, BitShift);
1210
1211       // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32
1212       // vector instead.
1213       SDValue Src[4] = {
1214         ShiftedValue,
1215         DAG.getConstant(0, DL, MVT::i32),
1216         DAG.getConstant(0, DL, MVT::i32),
1217         Mask
1218       };
1219       SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src);
1220       SDValue Args[3] = { Chain, Input, DWordAddr };
1221       return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL,
1222                                      Op->getVTList(), Args, MemVT,
1223                                      StoreNode->getMemOperand());
1224     } else if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(MVT::i32)) {
1225       // Convert pointer from byte address to dword address.
1226       Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1227
1228       if (StoreNode->isIndexed()) {
1229         llvm_unreachable("Indexed stores not supported yet");
1230       } else {
1231         Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand());
1232       }
1233       return Chain;
1234     }
1235   }
1236
1237   // GLOBAL_ADDRESS has been handled above, LOCAL_ADDRESS allows all sizes
1238   if (AS != AMDGPUAS::PRIVATE_ADDRESS)
1239     return SDValue();
1240
1241   if (MemVT.bitsLT(MVT::i32))
1242     return lowerPrivateTruncStore(StoreNode, DAG);
1243
1244   // Standard i32+ store, tag it with DWORDADDR to note that the address
1245   // has been shifted
1246   if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1247     Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1248     return DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand());
1249   }
1250
1251   // Tagged i32+ stores will be matched by patterns
1252   return SDValue();
1253 }
1254
1255 // return (512 + (kc_bank << 12)
1256 static int
1257 ConstantAddressBlock(unsigned AddressSpace) {
1258   switch (AddressSpace) {
1259   case AMDGPUAS::CONSTANT_BUFFER_0:
1260     return 512;
1261   case AMDGPUAS::CONSTANT_BUFFER_1:
1262     return 512 + 4096;
1263   case AMDGPUAS::CONSTANT_BUFFER_2:
1264     return 512 + 4096 * 2;
1265   case AMDGPUAS::CONSTANT_BUFFER_3:
1266     return 512 + 4096 * 3;
1267   case AMDGPUAS::CONSTANT_BUFFER_4:
1268     return 512 + 4096 * 4;
1269   case AMDGPUAS::CONSTANT_BUFFER_5:
1270     return 512 + 4096 * 5;
1271   case AMDGPUAS::CONSTANT_BUFFER_6:
1272     return 512 + 4096 * 6;
1273   case AMDGPUAS::CONSTANT_BUFFER_7:
1274     return 512 + 4096 * 7;
1275   case AMDGPUAS::CONSTANT_BUFFER_8:
1276     return 512 + 4096 * 8;
1277   case AMDGPUAS::CONSTANT_BUFFER_9:
1278     return 512 + 4096 * 9;
1279   case AMDGPUAS::CONSTANT_BUFFER_10:
1280     return 512 + 4096 * 10;
1281   case AMDGPUAS::CONSTANT_BUFFER_11:
1282     return 512 + 4096 * 11;
1283   case AMDGPUAS::CONSTANT_BUFFER_12:
1284     return 512 + 4096 * 12;
1285   case AMDGPUAS::CONSTANT_BUFFER_13:
1286     return 512 + 4096 * 13;
1287   case AMDGPUAS::CONSTANT_BUFFER_14:
1288     return 512 + 4096 * 14;
1289   case AMDGPUAS::CONSTANT_BUFFER_15:
1290     return 512 + 4096 * 15;
1291   default:
1292     return -1;
1293   }
1294 }
1295
1296 SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1297                                                 SelectionDAG &DAG) const {
1298   SDLoc DL(Op);
1299   LoadSDNode *Load = cast<LoadSDNode>(Op);
1300   ISD::LoadExtType ExtType = Load->getExtensionType();
1301   EVT MemVT = Load->getMemoryVT();
1302   assert(Load->getAlignment() >= MemVT.getStoreSize());
1303
1304   SDValue BasePtr = Load->getBasePtr();
1305   SDValue Chain = Load->getChain();
1306   SDValue Offset = Load->getOffset();
1307
1308   SDValue LoadPtr = BasePtr;
1309   if (!Offset.isUndef()) {
1310     LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset);
1311   }
1312
1313   // Get dword location
1314   // NOTE: this should be eliminated by the future SHR ptr, 2
1315   SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1316                             DAG.getConstant(0xfffffffc, DL, MVT::i32));
1317
1318   // Load dword
1319   // TODO: can we be smarter about machine pointer info?
1320   MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1321   SDValue Read = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1322
1323   // Get offset within the register.
1324   SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
1325                                 LoadPtr, DAG.getConstant(0x3, DL, MVT::i32));
1326
1327   // Bit offset of target byte (byteIdx * 8).
1328   SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1329                                  DAG.getConstant(3, DL, MVT::i32));
1330
1331   // Shift to the right.
1332   SDValue Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Read, ShiftAmt);
1333
1334   // Eliminate the upper bits by setting them to ...
1335   EVT MemEltVT = MemVT.getScalarType();
1336
1337   if (ExtType == ISD::SEXTLOAD) { // ... ones.
1338     SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1339     Ret = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode);
1340   } else { // ... or zeros.
1341     Ret = DAG.getZeroExtendInReg(Ret, DL, MemEltVT);
1342   }
1343
1344   SDValue Ops[] = {
1345     Ret,
1346     Read.getValue(1) // This should be our output chain
1347   };
1348
1349   return DAG.getMergeValues(Ops, DL);
1350 }
1351
1352 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1353   LoadSDNode *LoadNode = cast<LoadSDNode>(Op);
1354   unsigned AS = LoadNode->getAddressSpace();
1355   EVT MemVT = LoadNode->getMemoryVT();
1356   ISD::LoadExtType ExtType = LoadNode->getExtensionType();
1357
1358   if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
1359       ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(MVT::i32)) {
1360     return lowerPrivateExtLoad(Op, DAG);
1361   }
1362
1363   SDLoc DL(Op);
1364   EVT VT = Op.getValueType();
1365   SDValue Chain = LoadNode->getChain();
1366   SDValue Ptr = LoadNode->getBasePtr();
1367
1368   if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1369       LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) &&
1370       VT.isVector()) {
1371     SDValue Ops[2];
1372     std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LoadNode, DAG);
1373     return DAG.getMergeValues(Ops, DL);
1374   }
1375
1376   // This is still used for explicit load from addrspace(8)
1377   int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace());
1378   if (ConstantBlock > -1 &&
1379       ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) ||
1380        (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) {
1381     SDValue Result;
1382     if (isa<Constant>(LoadNode->getMemOperand()->getValue()) ||
1383         isa<ConstantSDNode>(Ptr)) {
1384       return constBufferLoad(LoadNode, LoadNode->getAddressSpace(), DAG);
1385     } else {
1386       //TODO: Does this even work?
1387       // non-constant ptr can't be folded, keeps it as a v4f32 load
1388       Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32,
1389           DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
1390                       DAG.getConstant(4, DL, MVT::i32)),
1391                       DAG.getConstant(LoadNode->getAddressSpace() -
1392                                       AMDGPUAS::CONSTANT_BUFFER_0, DL, MVT::i32)
1393           );
1394     }
1395
1396     if (!VT.isVector()) {
1397       Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
1398                            DAG.getConstant(0, DL, MVT::i32));
1399     }
1400
1401     SDValue MergedValues[2] = {
1402       Result,
1403       Chain
1404     };
1405     return DAG.getMergeValues(MergedValues, DL);
1406   }
1407
1408   // For most operations returning SDValue() will result in the node being
1409   // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we
1410   // need to manually expand loads that may be legal in some address spaces and
1411   // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for
1412   // compute shaders, since the data is sign extended when it is uploaded to the
1413   // buffer. However SEXT loads from other address spaces are not supported, so
1414   // we need to expand them here.
1415   if (LoadNode->getExtensionType() == ISD::SEXTLOAD) {
1416     assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1417     SDValue NewLoad = DAG.getExtLoad(
1418         ISD::EXTLOAD, DL, VT, Chain, Ptr, LoadNode->getPointerInfo(), MemVT,
1419         LoadNode->getAlignment(), LoadNode->getMemOperand()->getFlags());
1420     SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad,
1421                               DAG.getValueType(MemVT));
1422
1423     SDValue MergedValues[2] = { Res, Chain };
1424     return DAG.getMergeValues(MergedValues, DL);
1425   }
1426
1427   if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
1428     return SDValue();
1429   }
1430
1431   // DWORDADDR ISD marks already shifted address
1432   if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1433     assert(VT == MVT::i32);
1434     Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(2, DL, MVT::i32));
1435     Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, MVT::i32, Ptr);
1436     return DAG.getLoad(MVT::i32, DL, Chain, Ptr, LoadNode->getMemOperand());
1437   }
1438   return SDValue();
1439 }
1440
1441 SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1442   SDValue Chain = Op.getOperand(0);
1443   SDValue Cond  = Op.getOperand(1);
1444   SDValue Jump  = Op.getOperand(2);
1445
1446   return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(),
1447                      Chain, Jump, Cond);
1448 }
1449
1450 SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1451                                             SelectionDAG &DAG) const {
1452   MachineFunction &MF = DAG.getMachineFunction();
1453   const R600FrameLowering *TFL = Subtarget->getFrameLowering();
1454
1455   FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op);
1456
1457   unsigned FrameIndex = FIN->getIndex();
1458   Register IgnoredFrameReg;
1459   StackOffset Offset =
1460       TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg);
1461   return DAG.getConstant(Offset.getFixed() * 4 * TFL->getStackWidth(MF),
1462                          SDLoc(Op), Op.getValueType());
1463 }
1464
1465 CCAssignFn *R600TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1466                                                   bool IsVarArg) const {
1467   switch (CC) {
1468   case CallingConv::AMDGPU_KERNEL:
1469   case CallingConv::SPIR_KERNEL:
1470   case CallingConv::C:
1471   case CallingConv::Fast:
1472   case CallingConv::Cold:
1473     llvm_unreachable("kernels should not be handled here");
1474   case CallingConv::AMDGPU_VS:
1475   case CallingConv::AMDGPU_GS:
1476   case CallingConv::AMDGPU_PS:
1477   case CallingConv::AMDGPU_CS:
1478   case CallingConv::AMDGPU_HS:
1479   case CallingConv::AMDGPU_ES:
1480   case CallingConv::AMDGPU_LS:
1481     return CC_R600;
1482   default:
1483     report_fatal_error("Unsupported calling convention.");
1484   }
1485 }
1486
1487 /// XXX Only kernel functions are supported, so we can assume for now that
1488 /// every function is a kernel function, but in the future we should use
1489 /// separate calling conventions for kernel and non-kernel functions.
1490 SDValue R600TargetLowering::LowerFormalArguments(
1491     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1492     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1493     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1494   SmallVector<CCValAssign, 16> ArgLocs;
1495   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1496                  *DAG.getContext());
1497   MachineFunction &MF = DAG.getMachineFunction();
1498   SmallVector<ISD::InputArg, 8> LocalIns;
1499
1500   if (AMDGPU::isShader(CallConv)) {
1501     CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
1502   } else {
1503     analyzeFormalArgumentsCompute(CCInfo, Ins);
1504   }
1505
1506   for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1507     CCValAssign &VA = ArgLocs[i];
1508     const ISD::InputArg &In = Ins[i];
1509     EVT VT = In.VT;
1510     EVT MemVT = VA.getLocVT();
1511     if (!VT.isVector() && MemVT.isVector()) {
1512       // Get load source type if scalarized.
1513       MemVT = MemVT.getVectorElementType();
1514     }
1515
1516     if (AMDGPU::isShader(CallConv)) {
1517       Register Reg = MF.addLiveIn(VA.getLocReg(), &R600::R600_Reg128RegClass);
1518       SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT);
1519       InVals.push_back(Register);
1520       continue;
1521     }
1522
1523     // i64 isn't a legal type, so the register type used ends up as i32, which
1524     // isn't expected here. It attempts to create this sextload, but it ends up
1525     // being invalid. Somehow this seems to work with i64 arguments, but breaks
1526     // for <1 x i64>.
1527
1528     // The first 36 bytes of the input buffer contains information about
1529     // thread group and global sizes.
1530     ISD::LoadExtType Ext = ISD::NON_EXTLOAD;
1531     if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
1532       // FIXME: This should really check the extload type, but the handling of
1533       // extload vector parameters seems to be broken.
1534
1535       // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
1536       Ext = ISD::SEXTLOAD;
1537     }
1538
1539     // Compute the offset from the value.
1540     // XXX - I think PartOffset should give you this, but it seems to give the
1541     // size of the register which isn't useful.
1542
1543     unsigned PartOffset = VA.getLocMemOffset();
1544     unsigned Alignment = MinAlign(VT.getStoreSize(), PartOffset);
1545
1546     MachinePointerInfo PtrInfo(AMDGPUAS::PARAM_I_ADDRESS);
1547     SDValue Arg = DAG.getLoad(
1548         ISD::UNINDEXED, Ext, VT, DL, Chain,
1549         DAG.getConstant(PartOffset, DL, MVT::i32), DAG.getUNDEF(MVT::i32),
1550         PtrInfo,
1551         MemVT, Alignment, MachineMemOperand::MONonTemporal |
1552                                         MachineMemOperand::MODereferenceable |
1553                                         MachineMemOperand::MOInvariant);
1554
1555     InVals.push_back(Arg);
1556   }
1557   return Chain;
1558 }
1559
1560 EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1561                                            EVT VT) const {
1562    if (!VT.isVector())
1563      return MVT::i32;
1564    return VT.changeVectorElementTypeToInteger();
1565 }
1566
1567 bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1568                                           const MachineFunction &MF) const {
1569   // Local and Private addresses do not handle vectors. Limit to i32
1570   if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS)) {
1571     return (MemVT.getSizeInBits() <= 32);
1572   }
1573   return true;
1574 }
1575
1576 bool R600TargetLowering::allowsMisalignedMemoryAccesses(
1577     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1578     bool *IsFast) const {
1579   if (IsFast)
1580     *IsFast = false;
1581
1582   if (!VT.isSimple() || VT == MVT::Other)
1583     return false;
1584
1585   if (VT.bitsLT(MVT::i32))
1586     return false;
1587
1588   // TODO: This is a rough estimate.
1589   if (IsFast)
1590     *IsFast = true;
1591
1592   return VT.bitsGT(MVT::i32) && Alignment >= Align(4);
1593 }
1594
1595 static SDValue CompactSwizzlableVector(
1596   SelectionDAG &DAG, SDValue VectorEntry,
1597   DenseMap<unsigned, unsigned> &RemapSwizzle) {
1598   assert(RemapSwizzle.empty());
1599
1600   SDLoc DL(VectorEntry);
1601   EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1602
1603   SDValue NewBldVec[4];
1604   for (unsigned i = 0; i < 4; i++)
1605     NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry,
1606                                DAG.getIntPtrConstant(i, DL));
1607
1608   for (unsigned i = 0; i < 4; i++) {
1609     if (NewBldVec[i].isUndef())
1610       // We mask write here to teach later passes that the ith element of this
1611       // vector is undef. Thus we can use it to reduce 128 bits reg usage,
1612       // break false dependencies and additionnaly make assembly easier to read.
1613       RemapSwizzle[i] = 7; // SEL_MASK_WRITE
1614     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) {
1615       if (C->isZero()) {
1616         RemapSwizzle[i] = 4; // SEL_0
1617         NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1618       } else if (C->isExactlyValue(1.0)) {
1619         RemapSwizzle[i] = 5; // SEL_1
1620         NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1621       }
1622     }
1623
1624     if (NewBldVec[i].isUndef())
1625       continue;
1626
1627     for (unsigned j = 0; j < i; j++) {
1628       if (NewBldVec[i] == NewBldVec[j]) {
1629         NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType());
1630         RemapSwizzle[i] = j;
1631         break;
1632       }
1633     }
1634   }
1635
1636   return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry),
1637                             NewBldVec);
1638 }
1639
1640 static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
1641                                 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1642   assert(RemapSwizzle.empty());
1643
1644   SDLoc DL(VectorEntry);
1645   EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1646
1647   SDValue NewBldVec[4];
1648   bool isUnmovable[4] = {false, false, false, false};
1649   for (unsigned i = 0; i < 4; i++)
1650     NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry,
1651                                DAG.getIntPtrConstant(i, DL));
1652
1653   for (unsigned i = 0; i < 4; i++) {
1654     RemapSwizzle[i] = i;
1655     if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1656       unsigned Idx = cast<ConstantSDNode>(NewBldVec[i].getOperand(1))
1657           ->getZExtValue();
1658       if (i == Idx)
1659         isUnmovable[Idx] = true;
1660     }
1661   }
1662
1663   for (unsigned i = 0; i < 4; i++) {
1664     if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1665       unsigned Idx = cast<ConstantSDNode>(NewBldVec[i].getOperand(1))
1666           ->getZExtValue();
1667       if (isUnmovable[Idx])
1668         continue;
1669       // Swap i and Idx
1670       std::swap(NewBldVec[Idx], NewBldVec[i]);
1671       std::swap(RemapSwizzle[i], RemapSwizzle[Idx]);
1672       break;
1673     }
1674   }
1675
1676   return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry),
1677                             NewBldVec);
1678 }
1679
1680 SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[4],
1681                                             SelectionDAG &DAG,
1682                                             const SDLoc &DL) const {
1683   // Old -> New swizzle values
1684   DenseMap<unsigned, unsigned> SwizzleRemap;
1685
1686   BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap);
1687   for (unsigned i = 0; i < 4; i++) {
1688     unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue();
1689     if (SwizzleRemap.find(Idx) != SwizzleRemap.end())
1690       Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32);
1691   }
1692
1693   SwizzleRemap.clear();
1694   BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap);
1695   for (unsigned i = 0; i < 4; i++) {
1696     unsigned Idx = cast<ConstantSDNode>(Swz[i])->getZExtValue();
1697     if (SwizzleRemap.find(Idx) != SwizzleRemap.end())
1698       Swz[i] = DAG.getConstant(SwizzleRemap[Idx], DL, MVT::i32);
1699   }
1700
1701   return BuildVector;
1702 }
1703
1704 SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block,
1705                                             SelectionDAG &DAG) const {
1706   SDLoc DL(LoadNode);
1707   EVT VT = LoadNode->getValueType(0);
1708   SDValue Chain = LoadNode->getChain();
1709   SDValue Ptr = LoadNode->getBasePtr();
1710   assert (isa<ConstantSDNode>(Ptr));
1711
1712   //TODO: Support smaller loads
1713   if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(LoadNode))
1714     return SDValue();
1715
1716   if (LoadNode->getAlignment() < 4)
1717     return SDValue();
1718
1719   int ConstantBlock = ConstantAddressBlock(Block);
1720
1721   SDValue Slots[4];
1722   for (unsigned i = 0; i < 4; i++) {
1723     // We want Const position encoded with the following formula :
1724     // (((512 + (kc_bank << 12) + const_index) << 2) + chan)
1725     // const_index is Ptr computed by llvm using an alignment of 16.
1726     // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
1727     // then div by 4 at the ISel step
1728     SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1729         DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32));
1730     Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr);
1731   }
1732   EVT NewVT = MVT::v4i32;
1733   unsigned NumElements = 4;
1734   if (VT.isVector()) {
1735     NewVT = VT;
1736     NumElements = VT.getVectorNumElements();
1737   }
1738   SDValue Result = DAG.getBuildVector(NewVT, DL, makeArrayRef(Slots, NumElements));
1739   if (!VT.isVector()) {
1740     Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
1741                          DAG.getConstant(0, DL, MVT::i32));
1742   }
1743   SDValue MergedValues[2] = {
1744     Result,
1745     Chain
1746   };
1747   return DAG.getMergeValues(MergedValues, DL);
1748 }
1749
1750 //===----------------------------------------------------------------------===//
1751 // Custom DAG Optimizations
1752 //===----------------------------------------------------------------------===//
1753
1754 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
1755                                               DAGCombinerInfo &DCI) const {
1756   SelectionDAG &DAG = DCI.DAG;
1757   SDLoc DL(N);
1758
1759   switch (N->getOpcode()) {
1760   // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a)
1761   case ISD::FP_ROUND: {
1762       SDValue Arg = N->getOperand(0);
1763       if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) {
1764         return DAG.getNode(ISD::UINT_TO_FP, DL, N->getValueType(0),
1765                            Arg.getOperand(0));
1766       }
1767       break;
1768     }
1769
1770   // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) ->
1771   // (i32 select_cc f32, f32, -1, 0 cc)
1772   //
1773   // Mesa's GLSL frontend generates the above pattern a lot and we can lower
1774   // this to one of the SET*_DX10 instructions.
1775   case ISD::FP_TO_SINT: {
1776     SDValue FNeg = N->getOperand(0);
1777     if (FNeg.getOpcode() != ISD::FNEG) {
1778       return SDValue();
1779     }
1780     SDValue SelectCC = FNeg.getOperand(0);
1781     if (SelectCC.getOpcode() != ISD::SELECT_CC ||
1782         SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS
1783         SelectCC.getOperand(2).getValueType() != MVT::f32 || // True
1784         !isHWTrueValue(SelectCC.getOperand(2)) ||
1785         !isHWFalseValue(SelectCC.getOperand(3))) {
1786       return SDValue();
1787     }
1788
1789     return DAG.getNode(ISD::SELECT_CC, DL, N->getValueType(0),
1790                            SelectCC.getOperand(0), // LHS
1791                            SelectCC.getOperand(1), // RHS
1792                            DAG.getConstant(-1, DL, MVT::i32), // True
1793                            DAG.getConstant(0, DL, MVT::i32),  // False
1794                            SelectCC.getOperand(4)); // CC
1795   }
1796
1797   // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx
1798   // => build_vector elt0, ... , NewEltIdx, ... , eltN
1799   case ISD::INSERT_VECTOR_ELT: {
1800     SDValue InVec = N->getOperand(0);
1801     SDValue InVal = N->getOperand(1);
1802     SDValue EltNo = N->getOperand(2);
1803
1804     // If the inserted element is an UNDEF, just use the input vector.
1805     if (InVal.isUndef())
1806       return InVec;
1807
1808     EVT VT = InVec.getValueType();
1809
1810     // If we can't generate a legal BUILD_VECTOR, exit
1811     if (!isOperationLegal(ISD::BUILD_VECTOR, VT))
1812       return SDValue();
1813
1814     // Check that we know which element is being inserted
1815     if (!isa<ConstantSDNode>(EltNo))
1816       return SDValue();
1817     unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
1818
1819     // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
1820     // be converted to a BUILD_VECTOR).  Fill in the Ops vector with the
1821     // vector elements.
1822     SmallVector<SDValue, 8> Ops;
1823     if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
1824       Ops.append(InVec.getNode()->op_begin(),
1825                  InVec.getNode()->op_end());
1826     } else if (InVec.isUndef()) {
1827       unsigned NElts = VT.getVectorNumElements();
1828       Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
1829     } else {
1830       return SDValue();
1831     }
1832
1833     // Insert the element
1834     if (Elt < Ops.size()) {
1835       // All the operands of BUILD_VECTOR must have the same type;
1836       // we enforce that here.
1837       EVT OpVT = Ops[0].getValueType();
1838       if (InVal.getValueType() != OpVT)
1839         InVal = OpVT.bitsGT(InVal.getValueType()) ?
1840           DAG.getNode(ISD::ANY_EXTEND, DL, OpVT, InVal) :
1841           DAG.getNode(ISD::TRUNCATE, DL, OpVT, InVal);
1842       Ops[Elt] = InVal;
1843     }
1844
1845     // Return the new vector
1846     return DAG.getBuildVector(VT, DL, Ops);
1847   }
1848
1849   // Extract_vec (Build_vector) generated by custom lowering
1850   // also needs to be customly combined
1851   case ISD::EXTRACT_VECTOR_ELT: {
1852     SDValue Arg = N->getOperand(0);
1853     if (Arg.getOpcode() == ISD::BUILD_VECTOR) {
1854       if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1855         unsigned Element = Const->getZExtValue();
1856         return Arg->getOperand(Element);
1857       }
1858     }
1859     if (Arg.getOpcode() == ISD::BITCAST &&
1860         Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
1861         (Arg.getOperand(0).getValueType().getVectorNumElements() ==
1862          Arg.getValueType().getVectorNumElements())) {
1863       if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1864         unsigned Element = Const->getZExtValue();
1865         return DAG.getNode(ISD::BITCAST, DL, N->getVTList(),
1866                            Arg->getOperand(0).getOperand(Element));
1867       }
1868     }
1869     break;
1870   }
1871
1872   case ISD::SELECT_CC: {
1873     // Try common optimizations
1874     if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI))
1875       return Ret;
1876
1877     // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq ->
1878     //      selectcc x, y, a, b, inv(cc)
1879     //
1880     // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne ->
1881     //      selectcc x, y, a, b, cc
1882     SDValue LHS = N->getOperand(0);
1883     if (LHS.getOpcode() != ISD::SELECT_CC) {
1884       return SDValue();
1885     }
1886
1887     SDValue RHS = N->getOperand(1);
1888     SDValue True = N->getOperand(2);
1889     SDValue False = N->getOperand(3);
1890     ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get();
1891
1892     if (LHS.getOperand(2).getNode() != True.getNode() ||
1893         LHS.getOperand(3).getNode() != False.getNode() ||
1894         RHS.getNode() != False.getNode()) {
1895       return SDValue();
1896     }
1897
1898     switch (NCC) {
1899     default: return SDValue();
1900     case ISD::SETNE: return LHS;
1901     case ISD::SETEQ: {
1902       ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get();
1903       LHSCC = ISD::getSetCCInverse(LHSCC, LHS.getOperand(0).getValueType());
1904       if (DCI.isBeforeLegalizeOps() ||
1905           isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType()))
1906         return DAG.getSelectCC(DL,
1907                                LHS.getOperand(0),
1908                                LHS.getOperand(1),
1909                                LHS.getOperand(2),
1910                                LHS.getOperand(3),
1911                                LHSCC);
1912       break;
1913     }
1914     }
1915     return SDValue();
1916   }
1917
1918   case AMDGPUISD::R600_EXPORT: {
1919     SDValue Arg = N->getOperand(1);
1920     if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1921       break;
1922
1923     SDValue NewArgs[8] = {
1924       N->getOperand(0), // Chain
1925       SDValue(),
1926       N->getOperand(2), // ArrayBase
1927       N->getOperand(3), // Type
1928       N->getOperand(4), // SWZ_X
1929       N->getOperand(5), // SWZ_Y
1930       N->getOperand(6), // SWZ_Z
1931       N->getOperand(7) // SWZ_W
1932     };
1933     NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL);
1934     return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, N->getVTList(), NewArgs);
1935   }
1936   case AMDGPUISD::TEXTURE_FETCH: {
1937     SDValue Arg = N->getOperand(1);
1938     if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1939       break;
1940
1941     SDValue NewArgs[19] = {
1942       N->getOperand(0),
1943       N->getOperand(1),
1944       N->getOperand(2),
1945       N->getOperand(3),
1946       N->getOperand(4),
1947       N->getOperand(5),
1948       N->getOperand(6),
1949       N->getOperand(7),
1950       N->getOperand(8),
1951       N->getOperand(9),
1952       N->getOperand(10),
1953       N->getOperand(11),
1954       N->getOperand(12),
1955       N->getOperand(13),
1956       N->getOperand(14),
1957       N->getOperand(15),
1958       N->getOperand(16),
1959       N->getOperand(17),
1960       N->getOperand(18),
1961     };
1962     NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL);
1963     return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs);
1964   }
1965
1966   case ISD::LOAD: {
1967     LoadSDNode *LoadNode = cast<LoadSDNode>(N);
1968     SDValue Ptr = LoadNode->getBasePtr();
1969     if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS &&
1970          isa<ConstantSDNode>(Ptr))
1971       return constBufferLoad(LoadNode, AMDGPUAS::CONSTANT_BUFFER_0, DAG);
1972     break;
1973   }
1974
1975   default: break;
1976   }
1977
1978   return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
1979 }
1980
1981 bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx,
1982                                      SDValue &Src, SDValue &Neg, SDValue &Abs,
1983                                      SDValue &Sel, SDValue &Imm,
1984                                      SelectionDAG &DAG) const {
1985   const R600InstrInfo *TII = Subtarget->getInstrInfo();
1986   if (!Src.isMachineOpcode())
1987     return false;
1988
1989   switch (Src.getMachineOpcode()) {
1990   case R600::FNEG_R600:
1991     if (!Neg.getNode())
1992       return false;
1993     Src = Src.getOperand(0);
1994     Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
1995     return true;
1996   case R600::FABS_R600:
1997     if (!Abs.getNode())
1998       return false;
1999     Src = Src.getOperand(0);
2000     Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
2001     return true;
2002   case R600::CONST_COPY: {
2003     unsigned Opcode = ParentNode->getMachineOpcode();
2004     bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2005
2006     if (!Sel.getNode())
2007       return false;
2008
2009     SDValue CstOffset = Src.getOperand(0);
2010     if (ParentNode->getValueType(0).isVector())
2011       return false;
2012
2013     // Gather constants values
2014     int SrcIndices[] = {
2015       TII->getOperandIdx(Opcode, R600::OpName::src0),
2016       TII->getOperandIdx(Opcode, R600::OpName::src1),
2017       TII->getOperandIdx(Opcode, R600::OpName::src2),
2018       TII->getOperandIdx(Opcode, R600::OpName::src0_X),
2019       TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
2020       TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
2021       TII->getOperandIdx(Opcode, R600::OpName::src0_W),
2022       TII->getOperandIdx(Opcode, R600::OpName::src1_X),
2023       TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
2024       TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
2025       TII->getOperandIdx(Opcode, R600::OpName::src1_W)
2026     };
2027     std::vector<unsigned> Consts;
2028     for (int OtherSrcIdx : SrcIndices) {
2029       int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx);
2030       if (OtherSrcIdx < 0 || OtherSelIdx < 0)
2031         continue;
2032       if (HasDst) {
2033         OtherSrcIdx--;
2034         OtherSelIdx--;
2035       }
2036       if (RegisterSDNode *Reg =
2037           dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) {
2038         if (Reg->getReg() == R600::ALU_CONST) {
2039           ConstantSDNode *Cst
2040             = cast<ConstantSDNode>(ParentNode->getOperand(OtherSelIdx));
2041           Consts.push_back(Cst->getZExtValue());
2042         }
2043       }
2044     }
2045
2046     ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset);
2047     Consts.push_back(Cst->getZExtValue());
2048     if (!TII->fitsConstReadLimitations(Consts)) {
2049       return false;
2050     }
2051
2052     Sel = CstOffset;
2053     Src = DAG.getRegister(R600::ALU_CONST, MVT::f32);
2054     return true;
2055   }
2056   case R600::MOV_IMM_GLOBAL_ADDR:
2057     // Check if the Imm slot is used. Taken from below.
2058     if (cast<ConstantSDNode>(Imm)->getZExtValue())
2059       return false;
2060     Imm = Src.getOperand(0);
2061     Src = DAG.getRegister(R600::ALU_LITERAL_X, MVT::i32);
2062     return true;
2063   case R600::MOV_IMM_I32:
2064   case R600::MOV_IMM_F32: {
2065     unsigned ImmReg = R600::ALU_LITERAL_X;
2066     uint64_t ImmValue = 0;
2067
2068     if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
2069       ConstantFPSDNode *FPC = cast<ConstantFPSDNode>(Src.getOperand(0));
2070       float FloatValue = FPC->getValueAPF().convertToFloat();
2071       if (FloatValue == 0.0) {
2072         ImmReg = R600::ZERO;
2073       } else if (FloatValue == 0.5) {
2074         ImmReg = R600::HALF;
2075       } else if (FloatValue == 1.0) {
2076         ImmReg = R600::ONE;
2077       } else {
2078         ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue();
2079       }
2080     } else {
2081       ConstantSDNode *C = cast<ConstantSDNode>(Src.getOperand(0));
2082       uint64_t Value = C->getZExtValue();
2083       if (Value == 0) {
2084         ImmReg = R600::ZERO;
2085       } else if (Value == 1) {
2086         ImmReg = R600::ONE_INT;
2087       } else {
2088         ImmValue = Value;
2089       }
2090     }
2091
2092     // Check that we aren't already using an immediate.
2093     // XXX: It's possible for an instruction to have more than one
2094     // immediate operand, but this is not supported yet.
2095     if (ImmReg == R600::ALU_LITERAL_X) {
2096       if (!Imm.getNode())
2097         return false;
2098       ConstantSDNode *C = cast<ConstantSDNode>(Imm);
2099       if (C->getZExtValue())
2100         return false;
2101       Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32);
2102     }
2103     Src = DAG.getRegister(ImmReg, MVT::i32);
2104     return true;
2105   }
2106   default:
2107     return false;
2108   }
2109 }
2110
2111 /// Fold the instructions after selecting them
2112 SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
2113                                             SelectionDAG &DAG) const {
2114   const R600InstrInfo *TII = Subtarget->getInstrInfo();
2115   if (!Node->isMachineOpcode())
2116     return Node;
2117
2118   unsigned Opcode = Node->getMachineOpcode();
2119   SDValue FakeOp;
2120
2121   std::vector<SDValue> Ops(Node->op_begin(), Node->op_end());
2122
2123   if (Opcode == R600::DOT_4) {
2124     int OperandIdx[] = {
2125       TII->getOperandIdx(Opcode, R600::OpName::src0_X),
2126       TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
2127       TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
2128       TII->getOperandIdx(Opcode, R600::OpName::src0_W),
2129       TII->getOperandIdx(Opcode, R600::OpName::src1_X),
2130       TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
2131       TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
2132       TII->getOperandIdx(Opcode, R600::OpName::src1_W)
2133         };
2134     int NegIdx[] = {
2135       TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X),
2136       TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y),
2137       TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z),
2138       TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W),
2139       TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X),
2140       TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y),
2141       TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z),
2142       TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W)
2143     };
2144     int AbsIdx[] = {
2145       TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X),
2146       TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y),
2147       TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z),
2148       TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W),
2149       TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X),
2150       TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y),
2151       TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z),
2152       TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W)
2153     };
2154     for (unsigned i = 0; i < 8; i++) {
2155       if (OperandIdx[i] < 0)
2156         return Node;
2157       SDValue &Src = Ops[OperandIdx[i] - 1];
2158       SDValue &Neg = Ops[NegIdx[i] - 1];
2159       SDValue &Abs = Ops[AbsIdx[i] - 1];
2160       bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2161       int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2162       if (HasDst)
2163         SelIdx--;
2164       SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2165       if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG))
2166         return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2167     }
2168   } else if (Opcode == R600::REG_SEQUENCE) {
2169     for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) {
2170       SDValue &Src = Ops[i];
2171       if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG))
2172         return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2173     }
2174   } else {
2175     if (!TII->hasInstrModifiers(Opcode))
2176       return Node;
2177     int OperandIdx[] = {
2178       TII->getOperandIdx(Opcode, R600::OpName::src0),
2179       TII->getOperandIdx(Opcode, R600::OpName::src1),
2180       TII->getOperandIdx(Opcode, R600::OpName::src2)
2181     };
2182     int NegIdx[] = {
2183       TII->getOperandIdx(Opcode, R600::OpName::src0_neg),
2184       TII->getOperandIdx(Opcode, R600::OpName::src1_neg),
2185       TII->getOperandIdx(Opcode, R600::OpName::src2_neg)
2186     };
2187     int AbsIdx[] = {
2188       TII->getOperandIdx(Opcode, R600::OpName::src0_abs),
2189       TII->getOperandIdx(Opcode, R600::OpName::src1_abs),
2190       -1
2191     };
2192     for (unsigned i = 0; i < 3; i++) {
2193       if (OperandIdx[i] < 0)
2194         return Node;
2195       SDValue &Src = Ops[OperandIdx[i] - 1];
2196       SDValue &Neg = Ops[NegIdx[i] - 1];
2197       SDValue FakeAbs;
2198       SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2199       bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2200       int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2201       int ImmIdx = TII->getOperandIdx(Opcode, R600::OpName::literal);
2202       if (HasDst) {
2203         SelIdx--;
2204         ImmIdx--;
2205       }
2206       SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2207       SDValue &Imm = Ops[ImmIdx];
2208       if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG))
2209         return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2210     }
2211   }
2212
2213   return Node;
2214 }