llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp

   1 //===- LegalizerHelperTest.cpp
   2 //-----------------------------------------------===//
   3 //
   4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 // See https://llvm.org/LICENSE.txt for license information.
   6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "GISelMITest.h"
  11 #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
  12
  13 using namespace LegalizeActions;
  14 using namespace LegalizeMutations;
  15 using namespace LegalityPredicates;
  16
  17 namespace {
  18
  19 class DummyGISelObserver : public GISelChangeObserver {
  20 public:
  21   void changingInstr(MachineInstr &MI) override {}
  22   void changedInstr(MachineInstr &MI) override {}
  23   void createdInstr(MachineInstr &MI) override {}
  24   void erasingInstr(MachineInstr &MI) override {}
  25 };
  26
  27 // Test G_ROTL/G_ROTR lowering.
  28 TEST_F(AArch64GISelMITest, LowerRotates) {
  29   setUp();
  30   if (!TM)
  31     GTEST_SKIP();
  32
  33   // Declare your legalization info
  34   DefineLegalizerInfo(A, {
  35     getActionDefinitionsBuilder({G_ROTR, G_ROTL}).lower(); });
  36
  37   LLT S32 = LLT::scalar(32);
  38   auto Src = B.buildTrunc(S32, Copies[0]);
  39   auto Amt = B.buildTrunc(S32, Copies[1]);
  40   auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {S32}, {Src, Amt});
  41   auto ROTL = B.buildInstr(TargetOpcode::G_ROTL, {S32}, {Src, Amt});
  42
  43   AInfo Info(MF->getSubtarget());
  44   DummyGISelObserver Observer;
  45   LegalizerHelper Helper(*MF, Info, Observer, B);
  46   // Perform Legalization
  47   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
  48             Helper.lower(*ROTR, 0, S32));
  49   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
  50             Helper.lower(*ROTL, 0, S32));
  51
  52   auto CheckStr = R"(
  53   ; Check G_ROTR
  54   CHECK: [[SRC:%[0-9]+]]:_(s32) = G_TRUNC
  55   CHECK: [[AMT:%[0-9]+]]:_(s32) = G_TRUNC
  56   CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
  57   CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
  58   CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]]:_, [[AMT]]:_
  59   CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMT]]:_, [[C1]]:_
  60   CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[SRC]]:_, [[AND]]:_(s32)
  61   CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]]:_, [[C1]]:_
  62   CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SRC]]:_, [[AND1]]:_(s32)
  63   CHECK: G_OR [[LSHR]]:_, [[SHL]]:_
  64
  65   ; Check G_ROTL
  66   CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
  67   CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
  68   CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]]:_, [[AMT]]:_
  69   CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMT]]:_, [[C1]]:_
  70   CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SRC]]:_, [[AND]]:_(s32)
  71   CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB]]:_, [[C1]]:_
  72   CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[SRC]]:_, [[AND1]]:_(s32)
  73   CHECK: G_OR [[SHL]]:_, [[LSHR]]:_
  74   )";
  75
  76   // Check
  77   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
  78 }
  79
  80 // Test G_ROTL/G_ROTR non-pow2 lowering.
  81 TEST_F(AArch64GISelMITest, LowerRotatesNonPow2) {
  82   setUp();
  83   if (!TM)
  84     GTEST_SKIP();
  85
  86   // Declare your legalization info
  87   DefineLegalizerInfo(A, {
  88     getActionDefinitionsBuilder({G_ROTR, G_ROTL}).lower(); });
  89
  90   LLT S24 = LLT::scalar(24);
  91   auto Src = B.buildTrunc(S24, Copies[0]);
  92   auto Amt = B.buildTrunc(S24, Copies[1]);
  93   auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {S24}, {Src, Amt});
  94   auto ROTL = B.buildInstr(TargetOpcode::G_ROTL, {S24}, {Src, Amt});
  95
  96   AInfo Info(MF->getSubtarget());
  97   DummyGISelObserver Observer;
  98   LegalizerHelper Helper(*MF, Info, Observer, B);
  99   // Perform Legalization
 100   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 101             Helper.lower(*ROTR, 0, S24));
 102   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 103             Helper.lower(*ROTL, 0, S24));
 104
 105   auto CheckStr = R"(
 106   ; Check G_ROTR
 107   CHECK: [[SRC:%[0-9]+]]:_(s24) = G_TRUNC
 108   CHECK: [[AMT:%[0-9]+]]:_(s24) = G_TRUNC
 109   CHECK: [[C:%[0-9]+]]:_(s24) = G_CONSTANT i24 0
 110   CHECK: [[C1:%[0-9]+]]:_(s24) = G_CONSTANT i24 23
 111   CHECK: [[C2:%[0-9]+]]:_(s24) = G_CONSTANT i24 24
 112   CHECK: [[UREM:%[0-9]+]]:_(s24) = G_UREM [[AMT]]:_, [[C2]]:_
 113   CHECK: [[LSHR:%[0-9]+]]:_(s24) = G_LSHR [[SRC]]:_, [[UREM]]:_(s24)
 114   CHECK: [[SUB:%[0-9]+]]:_(s24) = G_SUB [[C1]]:_, [[UREM]]:_
 115   CHECK: [[C4:%[0-9]+]]:_(s24) = G_CONSTANT i24 1
 116   CHECK: [[SHL:%[0-9]+]]:_(s24) = G_SHL [[SRC]]:_, [[C4]]:_(s24)
 117   CHECK: [[SHL2:%[0-9]+]]:_(s24) = G_SHL [[SHL]]:_, [[SUB]]:_(s24)
 118   CHECK: G_OR [[LSHR]]:_, [[SHL2]]:_
 119
 120   ; Check G_ROTL
 121   CHECK: [[C:%[0-9]+]]:_(s24) = G_CONSTANT i24 0
 122   CHECK: [[C1:%[0-9]+]]:_(s24) = G_CONSTANT i24 23
 123   CHECK: [[C2:%[0-9]+]]:_(s24) = G_CONSTANT i24 24
 124   CHECK: [[UREM:%[0-9]+]]:_(s24) = G_UREM [[AMT]]:_, [[C2]]:_
 125   CHECK: [[SHL:%[0-9]+]]:_(s24) = G_SHL [[SRC]]:_, [[UREM]]:_(s24)
 126   CHECK: [[SUB:%[0-9]+]]:_(s24) = G_SUB [[C1]]:_, [[UREM]]:_
 127   CHECK: [[C4:%[0-9]+]]:_(s24) = G_CONSTANT i24 1
 128   CHECK: [[LSHR:%[0-9]+]]:_(s24) = G_LSHR [[SRC]]:_, [[C4]]:_(s24)
 129   CHECK: [[LSHR2:%[0-9]+]]:_(s24) = G_LSHR [[LSHR]]:_, [[SUB]]:_(s24)
 130   CHECK: G_OR [[SHL]]:_, [[LSHR2]]:_
 131   )";
 132
 133   // Check
 134   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 135 }
 136
 137 // Test vector G_ROTR lowering.
 138 TEST_F(AArch64GISelMITest, LowerRotatesVector) {
 139   setUp();
 140   if (!TM)
 141     GTEST_SKIP();
 142
 143   // Declare your legalization info
 144   DefineLegalizerInfo(A, {
 145     getActionDefinitionsBuilder({G_ROTR, G_ROTL}).lower(); });
 146
 147   LLT S32 = LLT::scalar(32);
 148   LLT V4S32 = LLT::fixed_vector(4, S32);
 149   auto SrcTrunc = B.buildTrunc(S32, Copies[0]);
 150   auto Src = B.buildSplatBuildVector(V4S32, SrcTrunc);
 151   auto AmtTrunc = B.buildTrunc(S32, Copies[1]);
 152   auto Amt = B.buildSplatBuildVector(V4S32, AmtTrunc);
 153   auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {V4S32}, {Src, Amt});
 154
 155   AInfo Info(MF->getSubtarget());
 156   DummyGISelObserver Observer;
 157   LegalizerHelper Helper(*MF, Info, Observer, B);
 158   // Perform Legalization
 159   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 160             Helper.lower(*ROTR, 0, V4S32));
 161
 162   auto CheckStr = R"(
 163   CHECK: [[SRCTRUNC:%[0-9]+]]:_(s32) = G_TRUNC
 164   CHECK: [[SRC:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[SRCTRUNC]]
 165   CHECK: [[AMTTRUNC:%[0-9]+]]:_(s32) = G_TRUNC
 166   CHECK: [[AMT:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AMTTRUNC]]
 167   CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 168   CHECK: [[ZERO:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]]
 169   CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
 170   CHECK: [[VEC31:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C1]]
 171   CHECK: [[SUB:%[0-9]+]]:_(<4 x s32>) = G_SUB [[ZERO]]:_, [[AMT]]:_
 172   CHECK: [[AND:%[0-9]+]]:_(<4 x s32>) = G_AND [[AMT]]:_, [[VEC31]]:_
 173   CHECK: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[SRC]]:_, [[AND]]:_(<4 x s32>)
 174   CHECK: [[AND1:%[0-9]+]]:_(<4 x s32>) = G_AND [[SUB]]:_, [[VEC31]]:_
 175   CHECK: [[SHL:%[0-9]+]]:_(<4 x s32>) = G_SHL [[SRC]]:_, [[AND1]]:_(<4 x s32>)
 176   CHECK: G_OR [[LSHR]]:_, [[SHL]]:_
 177   )";
 178
 179   // Check
 180   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 181 }
 182
 183 // Test CTTZ expansion when CTTZ_ZERO_UNDEF is legal or custom,
 184 // in which case it becomes CTTZ_ZERO_UNDEF with select.
 185 TEST_F(AArch64GISelMITest, LowerBitCountingCTTZ0) {
 186   setUp();
 187   if (!TM)
 188     GTEST_SKIP();
 189
 190   // Declare your legalization info
 191   DefineLegalizerInfo(A, {
 192     getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({{s32, s64}});
 193   });
 194   // Build Instr
 195   auto MIBCTTZ =
 196       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(32)}, {Copies[0]});
 197   AInfo Info(MF->getSubtarget());
 198   DummyGISelObserver Observer;
 199   LegalizerHelper Helper(*MF, Info, Observer, B);
 200   // Perform Legalization
 201   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 202             Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)));
 203
 204   auto CheckStr = R"(
 205   CHECK: [[CZU:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF %0
 206   CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 207   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), %0:_(s64), [[ZERO]]
 208   CHECK: [[SIXTY4:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
 209   CHECK: [[SEL:%[0-9]+]]:_(s32) = G_SELECT [[CMP]]:_(s1), [[SIXTY4]]:_, [[CZU]]
 210   )";
 211
 212   // Check
 213   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 214 }
 215
 216 // CTTZ expansion in terms of CTLZ
 217 TEST_F(AArch64GISelMITest, LowerBitCountingCTTZ1) {
 218   setUp();
 219   if (!TM)
 220     GTEST_SKIP();
 221
 222   // Declare your legalization info
 223   DefineLegalizerInfo(A, {
 224     getActionDefinitionsBuilder(G_CTLZ).legalFor({{s64, s64}});
 225   });
 226   // Build Instr
 227   auto MIBCTTZ =
 228       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]});
 229   AInfo Info(MF->getSubtarget());
 230   DummyGISelObserver Observer;
 231   LegalizerHelper Helper(*MF, Info, Observer, B);
 232   // Perform Legalization
 233   EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
 234               LegalizerHelper::LegalizeResult::Legalized);
 235
 236   auto CheckStr = R"(
 237   CHECK: [[NEG1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
 238   CHECK: [[NOT:%[0-9]+]]:_(s64) = G_XOR %0:_, [[NEG1]]
 239   CHECK: [[SUB1:%[0-9]+]]:_(s64) = G_ADD %0:_, [[NEG1]]
 240   CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[NOT]]:_, [[SUB1]]:_
 241   CHECK: [[CST64:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
 242   CHECK: [[CTLZ:%[0-9]+]]:_(s64) = G_CTLZ [[AND1]]:_
 243   CHECK: G_SUB [[CST64]]:_, [[CTLZ]]:_
 244   )";
 245
 246   // Check
 247   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 248 }
 249
 250 // CTLZ scalar narrowing
 251 TEST_F(AArch64GISelMITest, NarrowScalarCTLZ) {
 252   setUp();
 253   if (!TM)
 254     GTEST_SKIP();
 255
 256   // Declare your legalization info
 257   DefineLegalizerInfo(A, {
 258     getActionDefinitionsBuilder(G_CTLZ).legalFor({{s32, s32}});
 259   });
 260   // Build Instr
 261   auto CTLZ =
 262       B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(32)}, {Copies[0]});
 263   AInfo Info(MF->getSubtarget());
 264   DummyGISelObserver Observer;
 265   LegalizerHelper Helper(*MF, Info, Observer, B);
 266   // Perform Legalization
 267   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 268             Helper.narrowScalar(*CTLZ, 1, LLT::scalar(32)));
 269
 270   auto CheckStr = R"(
 271   CHECK: [[UNMERGE_LO:%[0-9]+]]:_(s32), [[UNMERGE_HI:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %0:_(s64)
 272   CHECK: [[ZERO:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 273   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UNMERGE_HI]]:_(s32), [[ZERO]]:_
 274   CHECK: [[CTLZ_LO:%[0-9]+]]:_(s32) = G_CTLZ [[UNMERGE_LO]]:_(s32)
 275   CHECK: [[THIRTYTWO:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
 276   CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTLZ_LO]]:_, [[THIRTYTWO]]:_
 277   CHECK: [[CTLZ_HI:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[UNMERGE_HI]]:_(s32)
 278   CHECK: %{{[0-9]+}}:_(s32) = G_SELECT [[CMP]]:_(s1), [[ADD]]:_, [[CTLZ_HI]]:_
 279   )";
 280
 281   // Check
 282   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 283 }
 284
 285 // CTTZ scalar narrowing
 286 TEST_F(AArch64GISelMITest, NarrowScalarCTTZ) {
 287   setUp();
 288   if (!TM)
 289     GTEST_SKIP();
 290
 291   // Declare your legalization info
 292   DefineLegalizerInfo(A, {
 293     getActionDefinitionsBuilder(G_CTTZ).legalFor({{s32, s64}});
 294   });
 295   // Build Instr
 296   auto CTTZ =
 297       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(32)}, {Copies[0]});
 298   AInfo Info(MF->getSubtarget());
 299   DummyGISelObserver Observer;
 300   LegalizerHelper Helper(*MF, Info, Observer, B);
 301   // Perform Legalization
 302   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 303             Helper.narrowScalar(*CTTZ, 1, LLT::scalar(32)));
 304
 305   auto CheckStr = R"(
 306   CHECK: [[UNMERGE_LO:%[0-9]+]]:_(s32), [[UNMERGE_HI:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %0:_(s64)
 307   CHECK: [[ZERO:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 308   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UNMERGE_LO]]:_(s32), [[ZERO]]:_
 309   CHECK: [[CTTZ_HI:%[0-9]+]]:_(s32) = G_CTTZ [[UNMERGE_HI]]:_(s32)
 310   CHECK: [[THIRTYTWO:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
 311   CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[CTTZ_HI]]:_, [[THIRTYTWO]]:_
 312   CHECK: [[CTTZ_LO:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[UNMERGE_LO]]:_(s32)
 313   CHECK: %{{[0-9]+}}:_(s32) = G_SELECT [[CMP]]:_(s1), [[ADD]]:_, [[CTTZ_LO]]:_
 314   )";
 315
 316   // Check
 317   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 318 }
 319
 320 // CTTZ expansion in terms of CTPOP
 321 TEST_F(AArch64GISelMITest, LowerBitCountingCTTZ2) {
 322   setUp();
 323   if (!TM)
 324     GTEST_SKIP();
 325
 326   // Declare your legalization info
 327   DefineLegalizerInfo(A, {
 328     getActionDefinitionsBuilder(G_CTPOP).legalFor({{s64, s64}});
 329   });
 330   // Build
 331   auto MIBCTTZ =
 332       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]});
 333   AInfo Info(MF->getSubtarget());
 334   DummyGISelObserver Observer;
 335   LegalizerHelper Helper(*MF, Info, Observer, B);
 336
 337   B.setInsertPt(*EntryMBB, MIBCTTZ->getIterator());
 338   EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
 339               LegalizerHelper::LegalizeResult::Legalized);
 340
 341   auto CheckStr = R"(
 342   CHECK: [[NEG1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
 343   CHECK: [[NOT:%[0-9]+]]:_(s64) = G_XOR %0:_, [[NEG1]]
 344   CHECK: [[SUB1:%[0-9]+]]:_(s64) = G_ADD %0:_, [[NEG1]]
 345   CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[NOT]]:_, [[SUB1]]:_
 346   CHECK: [[POP:%[0-9]+]]:_(s64) = G_CTPOP [[AND1]]
 347   )";
 348
 349   // Check
 350   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 351 }
 352
 353 // CTPOP widening.
 354 TEST_F(AArch64GISelMITest, WidenBitCountingCTPOP1) {
 355   setUp();
 356   if (!TM)
 357     GTEST_SKIP();
 358
 359   // Declare your legalization info
 360   DefineLegalizerInfo(A, {
 361       getActionDefinitionsBuilder(G_CTPOP).legalFor({{s16, s16}});
 362     });
 363
 364   // Build
 365   // Trunc it to s8.
 366   LLT s8{LLT::scalar(8)};
 367   LLT s16{LLT::scalar(16)};
 368   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 369   auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s16}, {MIBTrunc});
 370   AInfo Info(MF->getSubtarget());
 371   DummyGISelObserver Observer;
 372   LegalizerHelper Helper(*MF, Info, Observer, B);
 373   B.setInstr(*MIBCTPOP);
 374   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 375             Helper.widenScalar(*MIBCTPOP, 1, s16));
 376
 377   auto CheckStr = R"(
 378   CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC %0:_(s64)
 379   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC]]:_(s8)
 380   CHECK: [[CTPOP:%[0-9]+]]:_(s16) = G_CTPOP [[ZEXT]]
 381   CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY [[CTPOP]]:_(s16)
 382   )";
 383
 384   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 385 }
 386
 387 // Test a strange case where the result is wider than the source
 388 TEST_F(AArch64GISelMITest, WidenBitCountingCTPOP2) {
 389   setUp();
 390   if (!TM)
 391     GTEST_SKIP();
 392
 393   // Declare your legalization info
 394   DefineLegalizerInfo(A, {
 395       getActionDefinitionsBuilder(G_CTPOP).legalFor({{s32, s16}});
 396     });
 397
 398   // Build
 399   // Trunc it to s8.
 400   LLT s8{LLT::scalar(8)};
 401   LLT s16{LLT::scalar(16)};
 402   LLT s32{LLT::scalar(32)};
 403   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 404   auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s32}, {MIBTrunc});
 405   AInfo Info(MF->getSubtarget());
 406   DummyGISelObserver Observer;
 407   LegalizerHelper Helper(*MF, Info, Observer, B);
 408   B.setInstr(*MIBCTPOP);
 409   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 410             Helper.widenScalar(*MIBCTPOP, 1, s16));
 411
 412   auto CheckStr = R"(
 413   CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC %0:_(s64)
 414   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC]]:_(s8)
 415   CHECK: [[CTPOP:%[0-9]+]]:_(s16) = G_CTPOP [[ZEXT]]
 416   CHECK: [[COPY:%[0-9]+]]:_(s32) = G_ZEXT [[CTPOP]]:_(s16)
 417   )";
 418
 419   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 420 }
 421
 422 // CTTZ_ZERO_UNDEF expansion in terms of CTTZ
 423 TEST_F(AArch64GISelMITest, LowerBitCountingCTTZ3) {
 424   setUp();
 425   if (!TM)
 426     GTEST_SKIP();
 427
 428   // Declare your legalization info
 429   DefineLegalizerInfo(A, {
 430     getActionDefinitionsBuilder(G_CTTZ).legalFor({{s64, s64}});
 431   });
 432   // Build
 433   auto MIBCTTZ = B.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
 434                               {LLT::scalar(64)}, {Copies[0]});
 435   AInfo Info(MF->getSubtarget());
 436   DummyGISelObserver Observer;
 437   LegalizerHelper Helper(*MF, Info, Observer, B);
 438   EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
 439               LegalizerHelper::LegalizeResult::Legalized);
 440
 441   auto CheckStr = R"(
 442   CHECK: CTTZ
 443   )";
 444
 445   // Check
 446   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 447 }
 448
 449 // CTLZ expansion in terms of CTLZ_ZERO_UNDEF
 450 TEST_F(AArch64GISelMITest, LowerBitCountingCTLZ0) {
 451   setUp();
 452   if (!TM)
 453     GTEST_SKIP();
 454
 455   // Declare your legalization info
 456   DefineLegalizerInfo(A, {
 457     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({{s64, s64}});
 458   });
 459   // Build
 460   auto MIBCTLZ =
 461       B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(64)}, {Copies[0]});
 462   AInfo Info(MF->getSubtarget());
 463   DummyGISelObserver Observer;
 464   LegalizerHelper Helper(*MF, Info, Observer, B);
 465   EXPECT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) ==
 466               LegalizerHelper::LegalizeResult::Legalized);
 467
 468   auto CheckStr = R"(
 469   CHECK: [[CZU:%[0-9]+]]:_(s64) = G_CTLZ_ZERO_UNDEF %0
 470   CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 471   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), %0:_(s64), [[ZERO]]
 472   CHECK: [[SIXTY4:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
 473   CHECK: [[SEL:%[0-9]+]]:_(s64) = G_SELECT [[CMP]]:_(s1), [[SIXTY4]]:_, [[CZU]]
 474   )";
 475
 476   // Check
 477   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 478 }
 479
 480 // CTLZ expansion in terms of CTLZ_ZERO_UNDEF if the latter is a libcall
 481 TEST_F(AArch64GISelMITest, LowerBitCountingCTLZLibcall) {
 482   setUp();
 483   if (!TM)
 484     GTEST_SKIP();
 485
 486   // Declare your legalization info
 487   DefineLegalizerInfo(A, {
 488     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).libcallFor({{s32, s64}});
 489   });
 490   // Build
 491   auto MIBCTLZ =
 492       B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(32)}, {Copies[0]});
 493   AInfo Info(MF->getSubtarget());
 494   DummyGISelObserver Observer;
 495   LegalizerHelper Helper(*MF, Info, Observer, B);
 496   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
 497             Helper.lower(*MIBCTLZ, 0, LLT::scalar(32)));
 498
 499   auto CheckStr = R"(
 500   CHECK: [[CZU:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF %0
 501   CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 502   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), %0:_(s64), [[ZERO]]
 503   CHECK: [[THIRTY2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
 504   CHECK: [[SEL:%[0-9]+]]:_(s32) = G_SELECT [[CMP]]:_(s1), [[THIRTY2]]:_, [[CZU]]
 505   )";
 506
 507   // Check
 508   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 509 }
 510
 511 // CTLZ expansion
 512 TEST_F(AArch64GISelMITest, LowerBitCountingCTLZ1) {
 513   setUp();
 514   if (!TM)
 515     GTEST_SKIP();
 516
 517   // Declare your legalization info
 518   DefineLegalizerInfo(A, {
 519     getActionDefinitionsBuilder(G_CTPOP).legalFor({{s8, s8}});
 520   });
 521   // Build
 522   // Trunc it to s8.
 523   LLT s8{LLT::scalar(8)};
 524   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 525   auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc});
 526   AInfo Info(MF->getSubtarget());
 527   DummyGISelObserver Observer;
 528   LegalizerHelper Helper(*MF, Info, Observer, B);
 529   EXPECT_TRUE(Helper.lower(*MIBCTLZ, 0, s8) ==
 530               LegalizerHelper::LegalizeResult::Legalized);
 531
 532   auto CheckStr = R"(
 533   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 534   CHECK: [[Cst1:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
 535   CHECK: [[Sh1:%[0-9]+]]:_(s8) = G_LSHR [[Trunc]]:_, [[Cst1]]:_
 536   CHECK: [[Or1:%[0-9]+]]:_(s8) = G_OR [[Trunc]]:_, [[Sh1]]:_
 537   CHECK: [[Cst2:%[0-9]+]]:_(s8) = G_CONSTANT i8 2
 538   CHECK: [[Sh2:%[0-9]+]]:_(s8) = G_LSHR [[Or1]]:_, [[Cst2]]:_
 539   CHECK: [[Or2:%[0-9]+]]:_(s8) = G_OR [[Or1]]:_, [[Sh2]]:_
 540   CHECK: [[Cst4:%[0-9]+]]:_(s8) = G_CONSTANT i8 4
 541   CHECK: [[Sh4:%[0-9]+]]:_(s8) = G_LSHR [[Or2]]:_, [[Cst4]]:_
 542   CHECK: [[Or4:%[0-9]+]]:_(s8) = G_OR [[Or2]]:_, [[Sh4]]:_
 543   CHECK: [[CTPOP:%[0-9]+]]:_(s8) = G_CTPOP [[Or4]]:_
 544   CHECK: [[Len:%[0-9]+]]:_(s8) = G_CONSTANT i8 8
 545   CHECK: [[Sub:%[0-9]+]]:_(s8) = G_SUB [[Len]]:_, [[CTPOP]]:_
 546   )";
 547
 548   // Check
 549   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 550 }
 551
 552 // CTLZ widening.
 553 TEST_F(AArch64GISelMITest, WidenBitCountingCTLZ) {
 554   setUp();
 555   if (!TM)
 556     GTEST_SKIP();
 557
 558   // Declare your legalization info
 559   DefineLegalizerInfo(A, {
 560     getActionDefinitionsBuilder(G_CTLZ).legalFor({{s16, s16}});
 561   });
 562   // Build
 563   // Trunc it to s8.
 564   LLT s8{LLT::scalar(8)};
 565   LLT s16{LLT::scalar(16)};
 566   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 567   auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc});
 568   AInfo Info(MF->getSubtarget());
 569   DummyGISelObserver Observer;
 570   LegalizerHelper Helper(*MF, Info, Observer, B);
 571   EXPECT_TRUE(Helper.widenScalar(*MIBCTLZ, 1, s16) ==
 572               LegalizerHelper::LegalizeResult::Legalized);
 573
 574   auto CheckStr = R"(
 575   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 576   CHECK: [[Zext:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 577   CHECK: [[Ctlz:%[0-9]+]]:_(s16) = G_CTLZ [[Zext]]
 578   CHECK: [[Cst8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
 579   CHECK: [[Sub:%[0-9]+]]:_(s16) = G_SUB [[Ctlz]]:_, [[Cst8]]:_
 580   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC [[Sub]]
 581   )";
 582
 583   // Check
 584   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 585 }
 586
 587 // CTLZ_ZERO_UNDEF widening.
 588 TEST_F(AArch64GISelMITest, WidenBitCountingCTLZZeroUndef) {
 589   setUp();
 590   if (!TM)
 591     GTEST_SKIP();
 592
 593   // Declare your legalization info
 594   DefineLegalizerInfo(A, {
 595     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({{s16, s16}});
 596   });
 597   // Build
 598   // Trunc it to s8.
 599   LLT s8{LLT::scalar(8)};
 600   LLT s16{LLT::scalar(16)};
 601   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 602   auto MIBCTLZ_ZU =
 603       B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {s8}, {MIBTrunc});
 604   AInfo Info(MF->getSubtarget());
 605   DummyGISelObserver Observer;
 606   LegalizerHelper Helper(*MF, Info, Observer, B);
 607   EXPECT_TRUE(Helper.widenScalar(*MIBCTLZ_ZU, 1, s16) ==
 608               LegalizerHelper::LegalizeResult::Legalized);
 609
 610   auto CheckStr = R"(
 611   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 612   CHECK: [[Zext:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 613   CHECK: [[Cst8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
 614   CHECK: [[Shl:%[0-9]+]]:_(s16) = G_SHL [[Zext]]:_, [[Cst8]]:_
 615   CHECK: [[CtlzZu:%[0-9]+]]:_(s16) = G_CTLZ_ZERO_UNDEF [[Shl]]
 616   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC [[CtlzZu]]
 617   )";
 618
 619   // Check
 620   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 621 }
 622
 623 // CTPOP widening.
 624 TEST_F(AArch64GISelMITest, WidenBitCountingCTPOP) {
 625   setUp();
 626   if (!TM)
 627     GTEST_SKIP();
 628
 629   // Declare your legalization info
 630   DefineLegalizerInfo(A, {
 631     getActionDefinitionsBuilder(G_CTPOP).legalFor({{s16, s16}});
 632   });
 633   // Build
 634   // Trunc it to s8.
 635   LLT s8{LLT::scalar(8)};
 636   LLT s16{LLT::scalar(16)};
 637   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 638   auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s8}, {MIBTrunc});
 639   AInfo Info(MF->getSubtarget());
 640   DummyGISelObserver Observer;
 641   LegalizerHelper Helper(*MF, Info, Observer, B);
 642   EXPECT_TRUE(Helper.widenScalar(*MIBCTPOP, 1, s16) ==
 643               LegalizerHelper::LegalizeResult::Legalized);
 644
 645   auto CheckStr = R"(
 646   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 647   CHECK: [[Zext:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 648   CHECK: [[Ctpop:%[0-9]+]]:_(s16) = G_CTPOP [[Zext]]
 649   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC [[Ctpop]]
 650   )";
 651
 652   // Check
 653   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 654 }
 655
 656 // CTTZ_ZERO_UNDEF widening.
 657 TEST_F(AArch64GISelMITest, WidenBitCountingCTTZ_ZERO_UNDEF) {
 658   setUp();
 659   if (!TM)
 660     GTEST_SKIP();
 661
 662   // Declare your legalization info
 663   DefineLegalizerInfo(A, {
 664     getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({{s16, s16}});
 665   });
 666   // Build
 667   // Trunc it to s8.
 668   LLT s8{LLT::scalar(8)};
 669   LLT s16{LLT::scalar(16)};
 670   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 671   auto MIBCTTZ_ZERO_UNDEF =
 672       B.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {s8}, {MIBTrunc});
 673   AInfo Info(MF->getSubtarget());
 674   DummyGISelObserver Observer;
 675   LegalizerHelper Helper(*MF, Info, Observer, B);
 676   EXPECT_TRUE(Helper.widenScalar(*MIBCTTZ_ZERO_UNDEF, 1, s16) ==
 677               LegalizerHelper::LegalizeResult::Legalized);
 678
 679   auto CheckStr = R"(
 680   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 681   CHECK: [[AnyExt:%[0-9]+]]:_(s16) = G_ANYEXT [[Trunc]]
 682   CHECK: [[CttzZu:%[0-9]+]]:_(s16) = G_CTTZ_ZERO_UNDEF [[AnyExt]]
 683   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC [[CttzZu]]
 684   )";
 685
 686   // Check
 687   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 688 }
 689
 690 // CTTZ widening.
 691 TEST_F(AArch64GISelMITest, WidenBitCountingCTTZ) {
 692   setUp();
 693   if (!TM)
 694     GTEST_SKIP();
 695
 696   // Declare your legalization info
 697   DefineLegalizerInfo(A, {
 698     getActionDefinitionsBuilder(G_CTTZ).legalFor({{s16, s16}});
 699   });
 700   // Build
 701   // Trunc it to s8.
 702   LLT s8{LLT::scalar(8)};
 703   LLT s16{LLT::scalar(16)};
 704   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 705   auto MIBCTTZ = B.buildInstr(TargetOpcode::G_CTTZ, {s8}, {MIBTrunc});
 706   AInfo Info(MF->getSubtarget());
 707   DummyGISelObserver Observer;
 708   LegalizerHelper Helper(*MF, Info, Observer, B);
 709   EXPECT_TRUE(Helper.widenScalar(*MIBCTTZ, 1, s16) ==
 710               LegalizerHelper::LegalizeResult::Legalized);
 711
 712   auto CheckStr = R"(
 713   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 714   CHECK: [[AnyExt:%[0-9]+]]:_(s16) = G_ANYEXT [[Trunc]]
 715   CHECK: [[Cst:%[0-9]+]]:_(s16) = G_CONSTANT i16 256
 716   CHECK: [[Or:%[0-9]+]]:_(s16) = G_OR [[AnyExt]]:_, [[Cst]]
 717   CHECK: [[Cttz:%[0-9]+]]:_(s16) = G_CTTZ_ZERO_UNDEF [[Or]]
 718   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC [[Cttz]]
 719   )";
 720
 721   // Check
 722   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 723 }
 724 // UADDO widening.
 725 TEST_F(AArch64GISelMITest, WidenUADDO) {
 726   setUp();
 727   if (!TM)
 728     GTEST_SKIP();
 729
 730   // Declare your legalization info
 731   DefineLegalizerInfo(A, {
 732     getActionDefinitionsBuilder(G_ADD).legalFor({{s16, s16}});
 733   });
 734   // Build
 735   // Trunc it to s8.
 736   LLT s8{LLT::scalar(8)};
 737   LLT s16{LLT::scalar(16)};
 738   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 739   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 740   auto MIBUAddO =
 741       B.buildInstr(TargetOpcode::G_UADDO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
 742   AInfo Info(MF->getSubtarget());
 743   DummyGISelObserver Observer;
 744   LegalizerHelper Helper(*MF, Info, Observer, B);
 745   EXPECT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) ==
 746               LegalizerHelper::LegalizeResult::Legalized);
 747
 748   auto CheckStr = R"(
 749   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 750   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 751   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 752   CHECK: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[LHS]]:_, [[RHS]]:_
 753   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[ADD]]
 754   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]]
 755   CHECK: G_ICMP intpred(ne), [[ADD]]:_(s16), [[ZEXT]]:_
 756   CHECK: G_TRUNC [[ADD]]
 757   )";
 758
 759   // Check
 760   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 761 }
 762
 763 // USUBO widening.
 764 TEST_F(AArch64GISelMITest, WidenUSUBO) {
 765   setUp();
 766   if (!TM)
 767     GTEST_SKIP();
 768
 769   // Declare your legalization info
 770   DefineLegalizerInfo(A, {
 771     getActionDefinitionsBuilder(G_SUB).legalFor({{s16, s16}});
 772   });
 773   // Build
 774   // Trunc it to s8.
 775   LLT s8{LLT::scalar(8)};
 776   LLT s16{LLT::scalar(16)};
 777   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 778   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 779   auto MIBUSUBO =
 780       B.buildInstr(TargetOpcode::G_USUBO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
 781   AInfo Info(MF->getSubtarget());
 782   DummyGISelObserver Observer;
 783   LegalizerHelper Helper(*MF, Info, Observer, B);
 784   EXPECT_TRUE(Helper.widenScalar(*MIBUSUBO, 0, s16) ==
 785               LegalizerHelper::LegalizeResult::Legalized);
 786
 787   auto CheckStr = R"(
 788   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 789   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 790   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 791   CHECK: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[LHS]]:_, [[RHS]]:_
 792   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SUB]]
 793   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]]
 794   CHECK: G_ICMP intpred(ne), [[SUB]]:_(s16), [[ZEXT]]:_
 795   CHECK: G_TRUNC [[SUB]]
 796   )";
 797
 798   // Check
 799   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 800 }
 801
 802 // SADDO widening.
 803 TEST_F(AArch64GISelMITest, WidenSADDO) {
 804   setUp();
 805   if (!TM)
 806     GTEST_SKIP();
 807
 808   // Declare your legalization info
 809   DefineLegalizerInfo(A, {
 810     getActionDefinitionsBuilder(G_ADD).legalFor({{s16, s16}});
 811   });
 812   // Build
 813   // Trunc it to s8.
 814   LLT s8{LLT::scalar(8)};
 815   LLT s16{LLT::scalar(16)};
 816   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 817   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 818   auto MIBSAddO =
 819       B.buildInstr(TargetOpcode::G_SADDO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
 820   AInfo Info(MF->getSubtarget());
 821   DummyGISelObserver Observer;
 822   LegalizerHelper Helper(*MF, Info, Observer, B);
 823   EXPECT_TRUE(Helper.widenScalar(*MIBSAddO, 0, s16) ==
 824               LegalizerHelper::LegalizeResult::Legalized);
 825
 826   auto CheckStr = R"(
 827   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 828   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 829   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 830   CHECK: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[LHS]]:_, [[RHS]]:_
 831   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[ADD]]
 832   CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]]
 833   CHECK: G_ICMP intpred(ne), [[ADD]]:_(s16), [[SEXT]]:_
 834   CHECK: G_TRUNC [[ADD]]
 835   )";
 836
 837   // Check
 838   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 839 }
 840
 841 // SSUBO widening.
 842 TEST_F(AArch64GISelMITest, WidenSSUBO) {
 843   setUp();
 844   if (!TM)
 845     GTEST_SKIP();
 846
 847   // Declare your legalization info
 848   DefineLegalizerInfo(A, {
 849     getActionDefinitionsBuilder(G_SUB).legalFor({{s16, s16}});
 850   });
 851   // Build
 852   // Trunc it to s8.
 853   LLT s8{LLT::scalar(8)};
 854   LLT s16{LLT::scalar(16)};
 855   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 856   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 857   auto MIBSSUBO =
 858       B.buildInstr(TargetOpcode::G_SSUBO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
 859   AInfo Info(MF->getSubtarget());
 860   DummyGISelObserver Observer;
 861   LegalizerHelper Helper(*MF, Info, Observer, B);
 862   EXPECT_TRUE(Helper.widenScalar(*MIBSSUBO, 0, s16) ==
 863               LegalizerHelper::LegalizeResult::Legalized);
 864
 865   auto CheckStr = R"(
 866   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 867   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 868   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 869   CHECK: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[LHS]]:_, [[RHS]]:_
 870   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SUB]]
 871   CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]]
 872   CHECK: G_ICMP intpred(ne), [[SUB]]:_(s16), [[SEXT]]:_
 873   CHECK: G_TRUNC [[SUB]]
 874   )";
 875
 876   // Check
 877   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 878 }
 879
 880 TEST_F(AArch64GISelMITest, WidenUADDE) {
 881   setUp();
 882   if (!TM)
 883     GTEST_SKIP();
 884
 885   // Declare your legalization info
 886   DefineLegalizerInfo(A, {
 887     getActionDefinitionsBuilder(G_UADDE).legalFor({{s16, s16}});
 888   });
 889   // Build
 890   // Trunc it to s8.
 891   LLT s8{LLT::scalar(8)};
 892   LLT s16{LLT::scalar(16)};
 893   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 894   auto CarryIn = B.buildUndef(LLT::scalar(1));
 895   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 896   auto MIBUAddO = B.buildInstr(TargetOpcode::G_UADDE, {s8, CarryReg},
 897                                {MIBTrunc, MIBTrunc, CarryIn});
 898   AInfo Info(MF->getSubtarget());
 899   DummyGISelObserver Observer;
 900   LegalizerHelper Helper(*MF, Info, Observer, B);
 901   EXPECT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) ==
 902               LegalizerHelper::LegalizeResult::Legalized);
 903
 904   const char *CheckStr = R"(
 905   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 906   CHECK: [[Implicit:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
 907   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 908   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 909   CHECK: [[UADDE:%[0-9]+]]:_(s16), [[CARRY:%[0-9]+]]:_(s1) = G_UADDE [[LHS]]:_, [[RHS]]:_, [[Implicit]]:_
 910   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UADDE]]
 911   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]]
 912   CHECK: G_ICMP intpred(ne), [[UADDE]]:_(s16), [[ZEXT]]:_
 913   CHECK: G_TRUNC [[UADDE]]
 914   )";
 915
 916   // Check
 917   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 918 }
 919
 920 TEST_F(AArch64GISelMITest, WidenUSUBE) {
 921   setUp();
 922   if (!TM)
 923     GTEST_SKIP();
 924
 925   // Declare your legalization info
 926   DefineLegalizerInfo(A, {
 927     getActionDefinitionsBuilder(G_USUBE).legalFor({{s16, s16}});
 928   });
 929   // Build
 930   // Trunc it to s8.
 931   LLT s8{LLT::scalar(8)};
 932   LLT s16{LLT::scalar(16)};
 933   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 934   auto CarryIn = B.buildUndef(LLT::scalar(1));
 935   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 936   auto MIBUSUBE = B.buildInstr(TargetOpcode::G_USUBE, {s8, CarryReg},
 937                                {MIBTrunc, MIBTrunc, CarryIn});
 938   AInfo Info(MF->getSubtarget());
 939   DummyGISelObserver Observer;
 940   LegalizerHelper Helper(*MF, Info, Observer, B);
 941   EXPECT_TRUE(Helper.widenScalar(*MIBUSUBE, 0, s16) ==
 942               LegalizerHelper::LegalizeResult::Legalized);
 943
 944   const char *CheckStr = R"(
 945   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 946   CHECK: [[Implicit:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
 947   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 948   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_ZEXT [[Trunc]]
 949   CHECK: [[USUBE:%[0-9]+]]:_(s16), [[CARRY:%[0-9]+]]:_(s1) = G_USUBE [[LHS]]:_, [[RHS]]:_, [[Implicit]]:_
 950   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[USUBE]]
 951   CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC1]]
 952   CHECK: G_ICMP intpred(ne), [[USUBE]]:_(s16), [[ZEXT]]:_
 953   CHECK: G_TRUNC [[USUBE]]
 954   )";
 955
 956   // Check
 957   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 958 }
 959
 960 TEST_F(AArch64GISelMITest, WidenSADDE) {
 961   setUp();
 962   if (!TM)
 963     GTEST_SKIP();
 964
 965   // Declare your legalization info
 966   DefineLegalizerInfo(A, {
 967     getActionDefinitionsBuilder({G_SADDE, G_UADDE}).legalFor({{s16, s16}});
 968   });
 969   // Build
 970   // Trunc it to s8.
 971   LLT s8{LLT::scalar(8)};
 972   LLT s16{LLT::scalar(16)};
 973   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
 974   auto CarryIn = B.buildUndef(LLT::scalar(1));
 975   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
 976   auto MIBUAddO = B.buildInstr(TargetOpcode::G_SADDE, {s8, CarryReg},
 977                                {MIBTrunc, MIBTrunc, CarryIn});
 978   AInfo Info(MF->getSubtarget());
 979   DummyGISelObserver Observer;
 980   LegalizerHelper Helper(*MF, Info, Observer, B);
 981   EXPECT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) ==
 982               LegalizerHelper::LegalizeResult::Legalized);
 983
 984   const char *CheckStr = R"(
 985   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
 986   CHECK: [[Implicit:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
 987   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 988   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
 989   CHECK: [[SADDE:%[0-9]+]]:_(s16), [[CARRY:%[0-9]+]]:_(s1) = G_UADDE [[LHS]]:_, [[RHS]]:_, [[Implicit]]:_
 990   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SADDE]]
 991   CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]]
 992   CHECK: G_ICMP intpred(ne), [[SADDE]]:_(s16), [[SEXT]]:_
 993   CHECK: G_TRUNC [[SADDE]]
 994   )";
 995
 996   // Check
 997   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 998 }
 999
1000 TEST_F(AArch64GISelMITest, WidenSSUBE) {
1001   setUp();
1002   if (!TM)
1003     GTEST_SKIP();
1004
1005   // Declare your legalization info
1006   DefineLegalizerInfo(A, {
1007     getActionDefinitionsBuilder({G_SSUBE, G_USUBE}).legalFor({{s16, s16}});
1008   });
1009   // Build
1010   // Trunc it to s8.
1011   LLT s8{LLT::scalar(8)};
1012   LLT s16{LLT::scalar(16)};
1013   auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
1014   auto CarryIn = B.buildUndef(LLT::scalar(1));
1015   Register CarryReg = MRI->createGenericVirtualRegister(LLT::scalar(1));
1016   auto MIBSSUBE = B.buildInstr(TargetOpcode::G_SSUBE, {s8, CarryReg},
1017                                {MIBTrunc, MIBTrunc, CarryIn});
1018   AInfo Info(MF->getSubtarget());
1019   DummyGISelObserver Observer;
1020   LegalizerHelper Helper(*MF, Info, Observer, B);
1021   EXPECT_TRUE(Helper.widenScalar(*MIBSSUBE, 0, s16) ==
1022               LegalizerHelper::LegalizeResult::Legalized);
1023
1024   const char *CheckStr = R"(
1025   CHECK: [[Trunc:%[0-9]+]]:_(s8) = G_TRUNC
1026   CHECK: [[Implicit:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
1027   CHECK: [[LHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
1028   CHECK: [[RHS:%[0-9]+]]:_(s16) = G_SEXT [[Trunc]]
1029   CHECK: [[SSUBE:%[0-9]+]]:_(s16), [[CARRY:%[0-9]+]]:_(s1) = G_USUBE [[LHS]]:_, [[RHS]]:_, [[Implicit]]:_
1030   CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[SSUBE]]
1031   CHECK: [[SEXT:%[0-9]+]]:_(s16) = G_SEXT [[TRUNC1]]
1032   CHECK: G_ICMP intpred(ne), [[SSUBE]]:_(s16), [[SEXT]]:_
1033   CHECK: G_TRUNC [[SSUBE]]
1034   )";
1035
1036   // Check
1037   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1038 }
1039
1040 TEST_F(AArch64GISelMITest, WidenUMULOCondition) {
1041   setUp();
1042   if (!TM)
1043     GTEST_SKIP();
1044
1045   // Declare your legalization info
1046   DefineLegalizerInfo(A, {
1047     getActionDefinitionsBuilder(G_ADD).legalFor({{s16, s16}});
1048   });
1049
1050   LLT s32 = LLT::scalar(32);
1051   LLT s64 = LLT::scalar(64);
1052
1053   auto UMulo =
1054     B.buildInstr(TargetOpcode::G_UMULO, {s64, LLT::scalar(1)},
1055                  {Copies[0], Copies[1]});
1056   AInfo Info(MF->getSubtarget());
1057   DummyGISelObserver Observer;
1058   LegalizerHelper Helper(*MF, Info, Observer, B);
1059
1060   B.setInstrAndDebugLoc(*UMulo);
1061   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1062             Helper.widenScalar(*UMulo, 1, s32));
1063
1064   auto CheckStr = R"(
1065   CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY
1066   CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY
1067   CHECK: [[ADD:%[0-9]+]]:_(s64), [[OVERFLOW:%[0-9]+]]:_(s32) = G_UMULO [[COPY0]]:_, [[COPY1]]:_
1068   CHECK: {{[0-9]+}}:_(s1) = G_TRUNC [[OVERFLOW]]
1069   )";
1070
1071   // Check
1072   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1073 }
1074
1075 TEST_F(AArch64GISelMITest, NarrowUADDO) {
1076   setUp();
1077   if (!TM)
1078     GTEST_SKIP();
1079
1080   LLT S1 = LLT::scalar(1);
1081   LLT S32 = LLT::scalar(32);
1082   LLT S96 = LLT::scalar(96);
1083   DefineLegalizerInfo(A, {
1084     getActionDefinitionsBuilder({G_UADDO, G_UADDE})
1085         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1086   });
1087
1088   auto Op0 = B.buildUndef(S96);
1089   auto Op1 = B.buildUndef(S96);
1090   auto UADDO = B.buildUAddo(S96, S1, Op0, Op1);
1091
1092   AInfo Info(MF->getSubtarget());
1093   DummyGISelObserver Observer;
1094   LegalizerHelper Helper(*MF, Info, Observer, B);
1095   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1096             Helper.narrowScalar(*UADDO, 0, S32));
1097
1098   const char *CheckStr = R"(
1099   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1100   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1101   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1102   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1103   CHECK: [[UADDO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDO [[OP0_0]]:_, [[OP1_0]]:_
1104   CHECK: [[UADDO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1105   CHECK: [[UADDO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_UADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1106   CHECK: [[UADDO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[UADDO0]]:_(s32), [[UADDO1]]:_(s32), [[UADDO2]]:_(s32)
1107   )";
1108
1109   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1110 }
1111
1112 TEST_F(AArch64GISelMITest, NarrowUSUBO) {
1113   setUp();
1114   if (!TM)
1115     GTEST_SKIP();
1116
1117   LLT S1 = LLT::scalar(1);
1118   LLT S32 = LLT::scalar(32);
1119   LLT S96 = LLT::scalar(96);
1120   DefineLegalizerInfo(A, {
1121     getActionDefinitionsBuilder({G_USUBO, G_USUBE})
1122         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1123   });
1124
1125   auto Op0 = B.buildUndef(S96);
1126   auto Op1 = B.buildUndef(S96);
1127   auto USUBO = B.buildUSubo(S96, S1, Op0, Op1);
1128
1129   AInfo Info(MF->getSubtarget());
1130   DummyGISelObserver Observer;
1131   LegalizerHelper Helper(*MF, Info, Observer, B);
1132   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1133             Helper.narrowScalar(*USUBO, 0, S32));
1134
1135   const char *CheckStr = R"(
1136   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1137   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1138   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1139   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1140   CHECK: [[USUBO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBO [[OP0_0]]:_, [[OP1_0]]:_
1141   CHECK: [[USUBO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1142   CHECK: [[USUBO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_USUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1143   CHECK: [[USUBO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[USUBO0]]:_(s32), [[USUBO1]]:_(s32), [[USUBO2]]:_(s32)
1144   )";
1145
1146   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1147 }
1148
1149 TEST_F(AArch64GISelMITest, NarrowSADDO) {
1150   setUp();
1151   if (!TM)
1152     GTEST_SKIP();
1153
1154   LLT S1 = LLT::scalar(1);
1155   LLT S32 = LLT::scalar(32);
1156   LLT S96 = LLT::scalar(96);
1157   DefineLegalizerInfo(A, {
1158     getActionDefinitionsBuilder({G_UADDO, G_UADDE, G_SADDE})
1159         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1160   });
1161
1162   auto Op0 = B.buildUndef(S96);
1163   auto Op1 = B.buildUndef(S96);
1164   auto SADDO = B.buildSAddo(S96, S1, Op0, Op1);
1165
1166   AInfo Info(MF->getSubtarget());
1167   DummyGISelObserver Observer;
1168   LegalizerHelper Helper(*MF, Info, Observer, B);
1169   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1170             Helper.narrowScalar(*SADDO, 0, S32));
1171
1172   const char *CheckStr = R"(
1173   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1174   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1175   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1176   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1177   CHECK: [[SADDO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDO [[OP0_0]]:_, [[OP1_0]]:_
1178   CHECK: [[SADDO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1179   CHECK: [[SADDO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1180   CHECK: [[SADDO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SADDO0]]:_(s32), [[SADDO1]]:_(s32), [[SADDO2]]:_(s32)
1181   )";
1182
1183   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1184 }
1185
1186 TEST_F(AArch64GISelMITest, NarrowSSUBO) {
1187   setUp();
1188   if (!TM)
1189     GTEST_SKIP();
1190
1191   LLT S1 = LLT::scalar(1);
1192   LLT S32 = LLT::scalar(32);
1193   LLT S96 = LLT::scalar(96);
1194   DefineLegalizerInfo(A, {
1195     getActionDefinitionsBuilder({G_USUBO, G_USUBE, G_SSUBE})
1196         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1197   });
1198
1199   auto Op0 = B.buildUndef(S96);
1200   auto Op1 = B.buildUndef(S96);
1201   auto SSUBO = B.buildSSubo(S96, S1, Op0, Op1);
1202
1203   AInfo Info(MF->getSubtarget());
1204   DummyGISelObserver Observer;
1205   LegalizerHelper Helper(*MF, Info, Observer, B);
1206   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1207             Helper.narrowScalar(*SSUBO, 0, S32));
1208
1209   const char *CheckStr = R"(
1210   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1211   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1212   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1213   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1214   CHECK: [[SSUBO0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBO [[OP0_0]]:_, [[OP1_0]]:_
1215   CHECK: [[SSUBO1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1216   CHECK: [[SSUBO2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SSUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1217   CHECK: [[SSUBO:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SSUBO0]]:_(s32), [[SSUBO1]]:_(s32), [[SSUBO2]]:_(s32)
1218   )";
1219
1220   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1221 }
1222
1223 TEST_F(AArch64GISelMITest, NarrowUADDE) {
1224   setUp();
1225   if (!TM)
1226     GTEST_SKIP();
1227
1228   LLT S1 = LLT::scalar(1);
1229   LLT S32 = LLT::scalar(32);
1230   LLT S96 = LLT::scalar(96);
1231   DefineLegalizerInfo(A, {
1232     getActionDefinitionsBuilder(G_UADDE).legalFor(
1233         {{LLT::scalar(32), LLT::scalar(1)}});
1234   });
1235
1236   auto Op0 = B.buildUndef(S96);
1237   auto Op1 = B.buildUndef(S96);
1238   auto Op2 = B.buildUndef(S1);
1239   auto UADDE = B.buildUAdde(S96, S1, Op0, Op1, Op2);
1240
1241   AInfo Info(MF->getSubtarget());
1242   DummyGISelObserver Observer;
1243   LegalizerHelper Helper(*MF, Info, Observer, B);
1244   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1245             Helper.narrowScalar(*UADDE, 0, S32));
1246
1247   const char *CheckStr = R"(
1248   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1249   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1250   CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
1251   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1252   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1253   CHECK: [[UADDE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_
1254   CHECK: [[UADDE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1255   CHECK: [[UADDE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_UADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1256   CHECK: [[UADDE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[UADDE0]]:_(s32), [[UADDE1]]:_(s32), [[UADDE2]]:_(s32)
1257   )";
1258
1259   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1260 }
1261
1262 TEST_F(AArch64GISelMITest, NarrowUSUBE) {
1263   setUp();
1264   if (!TM)
1265     GTEST_SKIP();
1266
1267   LLT S1 = LLT::scalar(1);
1268   LLT S32 = LLT::scalar(32);
1269   LLT S96 = LLT::scalar(96);
1270   DefineLegalizerInfo(A, {
1271     getActionDefinitionsBuilder(G_USUBE).legalFor(
1272         {{LLT::scalar(32), LLT::scalar(1)}});
1273   });
1274
1275   auto Op0 = B.buildUndef(S96);
1276   auto Op1 = B.buildUndef(S96);
1277   auto Op2 = B.buildUndef(S1);
1278   auto USUBE = B.buildUSube(S96, S1, Op0, Op1, Op2);
1279
1280   AInfo Info(MF->getSubtarget());
1281   DummyGISelObserver Observer;
1282   LegalizerHelper Helper(*MF, Info, Observer, B);
1283   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1284             Helper.narrowScalar(*USUBE, 0, S32));
1285
1286   const char *CheckStr = R"(
1287   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1288   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1289   CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
1290   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1291   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1292   CHECK: [[USUBE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_
1293   CHECK: [[USUBE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1294   CHECK: [[USUBE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_USUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1295   CHECK: [[USUBE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[USUBE0]]:_(s32), [[USUBE1]]:_(s32), [[USUBE2]]:_(s32)
1296   )";
1297
1298   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1299 }
1300
1301 TEST_F(AArch64GISelMITest, NarrowSADDE) {
1302   setUp();
1303   if (!TM)
1304     GTEST_SKIP();
1305
1306   LLT S1 = LLT::scalar(1);
1307   LLT S32 = LLT::scalar(32);
1308   LLT S96 = LLT::scalar(96);
1309   DefineLegalizerInfo(A, {
1310     getActionDefinitionsBuilder({G_SADDE, G_UADDE})
1311         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1312   });
1313
1314   auto Op0 = B.buildUndef(S96);
1315   auto Op1 = B.buildUndef(S96);
1316   auto Op2 = B.buildUndef(S1);
1317   auto SADDE = B.buildSAdde(S96, S1, Op0, Op1, Op2);
1318
1319   AInfo Info(MF->getSubtarget());
1320   DummyGISelObserver Observer;
1321   LegalizerHelper Helper(*MF, Info, Observer, B);
1322   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1323             Helper.narrowScalar(*SADDE, 0, S32));
1324
1325   const char *CheckStr = R"(
1326   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1327   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1328   CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
1329   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1330   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1331   CHECK: [[SADDE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_UADDE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_
1332   CHECK: [[SADDE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_UADDE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1333   CHECK: [[SADDE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SADDE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1334   CHECK: [[SADDE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SADDE0]]:_(s32), [[SADDE1]]:_(s32), [[SADDE2]]:_(s32)
1335   )";
1336
1337   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1338 }
1339
1340 TEST_F(AArch64GISelMITest, NarrowSSUBE) {
1341   setUp();
1342   if (!TM)
1343     GTEST_SKIP();
1344
1345   LLT S1 = LLT::scalar(1);
1346   LLT S32 = LLT::scalar(32);
1347   LLT S96 = LLT::scalar(96);
1348   DefineLegalizerInfo(A, {
1349     getActionDefinitionsBuilder({G_SSUBE, G_USUBE})
1350         .legalFor({{LLT::scalar(32), LLT::scalar(1)}});
1351   });
1352
1353   auto Op0 = B.buildUndef(S96);
1354   auto Op1 = B.buildUndef(S96);
1355   auto Op2 = B.buildUndef(S1);
1356   auto SSUBE = B.buildSSube(S96, S1, Op0, Op1, Op2);
1357
1358   AInfo Info(MF->getSubtarget());
1359   DummyGISelObserver Observer;
1360   LegalizerHelper Helper(*MF, Info, Observer, B);
1361   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1362             Helper.narrowScalar(*SSUBE, 0, S32));
1363
1364   const char *CheckStr = R"(
1365   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1366   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
1367   CHECK: [[IMP_DEF2:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
1368   CHECK: [[OP0_0:%[0-9]+]]:_(s32), [[OP0_1:%[0-9]+]]:_(s32), [[OP0_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]
1369   CHECK: [[OP1_0:%[0-9]+]]:_(s32), [[OP1_1:%[0-9]+]]:_(s32), [[OP1_2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]
1370   CHECK: [[SSUBE0:%[0-9]+]]:_(s32), [[CARRY0:%[0-9]+]]:_(s1) = G_USUBE [[OP0_0]]:_, [[OP1_0]]:_, [[IMP_DEF2]]:_
1371   CHECK: [[SSUBE1:%[0-9]+]]:_(s32), [[CARRY1:%[0-9]+]]:_(s1) = G_USUBE [[OP0_1]]:_, [[OP1_1]]:_, [[CARRY0]]:_
1372   CHECK: [[SSUBE2:%[0-9]+]]:_(s32), [[CARRY2:%[0-9]+]]:_(s1) = G_SSUBE [[OP0_2]]:_, [[OP1_2]]:_, [[CARRY1]]:_
1373   CHECK: [[SSUBE:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[SSUBE0]]:_(s32), [[SSUBE1]]:_(s32), [[SSUBE2]]:_(s32)
1374   )";
1375
1376   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1377 }
1378
1379 TEST_F(AArch64GISelMITest, FewerElementsAnd) {
1380   setUp();
1381   if (!TM)
1382     GTEST_SKIP();
1383
1384   const LLT V2S32 = LLT::fixed_vector(2, 32);
1385   const LLT V5S32 = LLT::fixed_vector(5, 32);
1386
1387   // Declare your legalization info
1388   DefineLegalizerInfo(A, {
1389     getActionDefinitionsBuilder(G_AND)
1390       .legalFor({s32});
1391   });
1392
1393   auto Op0 = B.buildUndef(V5S32);
1394   auto Op1 = B.buildUndef(V5S32);
1395   auto And = B.buildAnd(V5S32, Op0, Op1);
1396
1397   AInfo Info(MF->getSubtarget());
1398   DummyGISelObserver Observer;
1399   LegalizerHelper Helper(*MF, Info, Observer, B);
1400   B.setInstr(*And);
1401   EXPECT_TRUE(Helper.fewerElementsVector(*And, 0, V2S32) ==
1402               LegalizerHelper::LegalizeResult::Legalized);
1403
1404   auto CheckStr = R"(
1405   CHECK: [[IMP_DEF0:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
1406   CHECK: [[IMP_DEF1:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
1407   CHECK: [[VALUE0:%[0-9]+]]:_(s32), [[VALUE1:%[0-9]+]]:_(s32), [[VALUE2:%[0-9]+]]:_(s32), [[VALUE3:%[0-9]+]]:_(s32), [[VALUE4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF0]]:_(<5 x s32>)
1408   CHECK: [[VECTOR0:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[VALUE0]]:_(s32), [[VALUE1]]:_(s32)
1409   CHECK: [[VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[VALUE2]]:_(s32), [[VALUE3]]:_(s32)
1410   CHECK: [[VALUE5:%[0-9]+]]:_(s32), [[VALUE6:%[0-9]+]]:_(s32), [[VALUE7:%[0-9]+]]:_(s32), [[VALUE8:%[0-9]+]]:_(s32), [[VALUE9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IMP_DEF1]]:_(<5 x s32>)
1411   CHECK: [[VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[VALUE5]]:_(s32), [[VALUE6]]:_(s32)
1412   CHECK: [[VECTOR3:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[VALUE7]]:_(s32), [[VALUE8]]:_(s32)
1413   CHECK: [[AND0:%[0-9]+]]:_(<2 x s32>) = G_AND [[VECTOR0]]:_, [[VECTOR2]]:_
1414   CHECK: [[AND1:%[0-9]+]]:_(<2 x s32>) = G_AND [[VECTOR1]]:_, [[VECTOR3]]:_
1415   CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[VALUE4]]:_, [[VALUE9]]:_
1416   CHECK: [[AND0_E0:%[0-9]+]]:_(s32), [[AND0_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND0]]:_(<2 x s32>)
1417   CHECK: [[AND1_E0:%[0-9]+]]:_(s32), [[AND1_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]]:_(<2 x s32>)
1418   CHECK: [[RESULT:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[AND0_E0]]:_(s32), [[AND0_E1]]:_(s32), [[AND1_E0]]:_(s32), [[AND1_E1]]:_(s32), [[AND2]]:_(s32)
1419   )";
1420
1421   // Check
1422   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1423 }
1424
1425 TEST_F(AArch64GISelMITest, MoreElementsAnd) {
1426   setUp();
1427   if (!TM)
1428     GTEST_SKIP();
1429
1430   LLT s32 = LLT::scalar(32);
1431   LLT v2s32 = LLT::fixed_vector(2, 32);
1432   LLT v6s32 = LLT::fixed_vector(6, 32);
1433
1434   LegalizerInfo LI;
1435   LI.getActionDefinitionsBuilder(TargetOpcode::G_AND)
1436     .legalFor({v6s32})
1437     .clampMinNumElements(0, s32, 6);
1438   LI.getLegacyLegalizerInfo().computeTables();
1439
1440   DummyGISelObserver Observer;
1441   LegalizerHelper Helper(*MF, LI, Observer, B);
1442
1443   B.setInsertPt(*EntryMBB, EntryMBB->end());
1444
1445   auto Val0 = B.buildBitcast(v2s32, Copies[0]);
1446   auto Val1 = B.buildBitcast(v2s32, Copies[1]);
1447
1448   auto And = B.buildAnd(v2s32, Val0, Val1);
1449
1450   B.setInstr(*And);
1451   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1452             Helper.moreElementsVector(*And, 0, v6s32));
1453
1454   auto CheckStr = R"(
1455   CHECK: [[BITCAST0:%[0-9]+]]:_(<2 x s32>) = G_BITCAST
1456   CHECK: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST
1457
1458   CHECK: [[BITCAST0_E0:%[0-9]+]]:_(s32), [[BITCAST0_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST0]]:_(<2 x s32>)
1459   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
1460   CHECK: [[BITCAST0_LARGE:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[BITCAST0_E0]]:_(s32), [[BITCAST0_E1]]:_(s32), [[IMP_DEF0]]:_(s32), [[IMP_DEF0]]:_(s32), [[IMP_DEF0]]:_(s32), [[IMP_DEF0]]:_(s32)
1461
1462   CHECK: [[BITCAST1_E0:%[0-9]+]]:_(s32), [[BITCAST1_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]]:_(<2 x s32>)
1463   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
1464   CHECK: [[BITCAST1_LARGE:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[BITCAST1_E0]]:_(s32), [[BITCAST1_E1]]:_(s32), [[IMP_DEF1]]:_(s32), [[IMP_DEF1]]:_(s32), [[IMP_DEF1]]:_(s32), [[IMP_DEF1]]:_(s32)
1465
1466   CHECK: [[AND:%[0-9]+]]:_(<6 x s32>) = G_AND [[BITCAST0_LARGE]]:_, [[BITCAST1_LARGE]]:_
1467
1468   CHECK: [[AND_E0:%[0-9]+]]:_(s32), [[AND_E1:%[0-9]+]]:_(s32), [[AND_E2:%[0-9]+]]:_(s32), [[AND_E3:%[0-9]+]]:_(s32), [[AND_E4:%[0-9]+]]:_(s32), [[AND_E5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]]:_(<6 x s32>)
1469   CHECK: (<2 x s32>) = G_BUILD_VECTOR [[AND_E0]]:_(s32), [[AND_E1]]:_(s32)
1470   )";
1471
1472   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1473 }
1474
1475 TEST_F(AArch64GISelMITest, FewerElementsPhi) {
1476   setUp();
1477   if (!TM)
1478     GTEST_SKIP();
1479
1480   LLT s1 = LLT::scalar(1);
1481   LLT s32 = LLT::scalar(32);
1482   LLT s64 = LLT::scalar(64);
1483   LLT v2s32 = LLT::fixed_vector(2, 32);
1484   LLT v5s32 = LLT::fixed_vector(5, 32);
1485
1486   LegalizerInfo LI;
1487   LI.getActionDefinitionsBuilder(TargetOpcode::G_PHI)
1488     .legalFor({v2s32})
1489     .clampMinNumElements(0, s32, 2);
1490   LI.getLegacyLegalizerInfo().computeTables();
1491
1492   LLT PhiTy = v5s32;
1493   DummyGISelObserver Observer;
1494   LegalizerHelper Helper(*MF, LI, Observer, B);
1495   B.setMBB(*EntryMBB);
1496
1497   MachineBasicBlock *MidMBB = MF->CreateMachineBasicBlock();
1498   MachineBasicBlock *EndMBB = MF->CreateMachineBasicBlock();
1499   MF->insert(MF->end(), MidMBB);
1500   MF->insert(MF->end(), EndMBB);
1501
1502   EntryMBB->addSuccessor(MidMBB);
1503   EntryMBB->addSuccessor(EndMBB);
1504   MidMBB->addSuccessor(EndMBB);
1505
1506   auto InitVal = B.buildUndef(PhiTy);
1507   auto InitOtherVal = B.buildConstant(s64, 999);
1508
1509   auto ICmp = B.buildICmp(CmpInst::ICMP_EQ, s1, Copies[0], Copies[1]);
1510   B.buildBrCond(ICmp.getReg(0), *MidMBB);
1511   B.buildBr(*EndMBB);
1512
1513
1514   B.setMBB(*MidMBB);
1515   auto MidVal = B.buildUndef(PhiTy);
1516   auto MidOtherVal = B.buildConstant(s64, 345);
1517   B.buildBr(*EndMBB);
1518
1519   B.setMBB(*EndMBB);
1520   auto Phi = B.buildInstr(TargetOpcode::G_PHI)
1521     .addDef(MRI->createGenericVirtualRegister(PhiTy))
1522     .addUse(InitVal.getReg(0))
1523     .addMBB(EntryMBB)
1524     .addUse(MidVal.getReg(0))
1525     .addMBB(MidMBB);
1526
1527   // Insert another irrelevant phi to make sure the rebuild is inserted after
1528   // it.
1529   B.buildInstr(TargetOpcode::G_PHI)
1530     .addDef(MRI->createGenericVirtualRegister(s64))
1531     .addUse(InitOtherVal.getReg(0))
1532     .addMBB(EntryMBB)
1533     .addUse(MidOtherVal.getReg(0))
1534     .addMBB(MidMBB);
1535
1536   // Add some use instruction after the phis.
1537   B.buildAnd(PhiTy, Phi.getReg(0), Phi.getReg(0));
1538
1539   B.setInstr(*Phi);
1540   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1541             Helper.fewerElementsVector(*Phi, 0, v2s32));
1542
1543   auto CheckStr = R"(
1544   CHECK: [[INITVAL:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
1545   CHECK: [[INITVAL_E0:%[0-9]+]]:_(s32), [[INITVAL_E1:%[0-9]+]]:_(s32), [[INITVAL_E2:%[0-9]+]]:_(s32), [[INITVAL_E3:%[0-9]+]]:_(s32), [[INITVAL_E4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INITVAL]]:_(<5 x s32>)
1546   CHECK: [[INITVAL_E01:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[INITVAL_E0]]:_(s32), [[INITVAL_E1]]:_(s32)
1547   CHECK: [[INITVAL_E23:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[INITVAL_E2]]:_(s32), [[INITVAL_E3]]:_(s32)
1548   CHECK: G_BRCOND
1549
1550   CHECK: [[MIDVAL:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
1551   CHECK: [[MIDVAL_E0:%[0-9]+]]:_(s32), [[MIDVAL_E1:%[0-9]+]]:_(s32), [[MIDVAL_E2:%[0-9]+]]:_(s32), [[MIDVAL_E3:%[0-9]+]]:_(s32), [[MIDVAL_E4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MIDVAL]]:_(<5 x s32>)
1552   CHECK: [[MIDVAL_E01:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MIDVAL_E0]]:_(s32), [[MIDVAL_E1]]:_(s32)
1553   CHECK: [[MIDVAL_E23:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MIDVAL_E2]]:_(s32), [[MIDVAL_E3]]:_(s32)
1554   CHECK: G_BR
1555
1556   CHECK: [[PHI0:%[0-9]+]]:_(<2 x s32>) = G_PHI [[INITVAL_E01]]:_(<2 x s32>), %bb.0, [[MIDVAL_E01]]:_(<2 x s32>), %bb.1
1557   CHECK: [[PHI1:%[0-9]+]]:_(<2 x s32>) = G_PHI [[INITVAL_E23]]:_(<2 x s32>), %bb.0, [[MIDVAL_E23]]:_(<2 x s32>), %bb.1
1558   CHECK: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[INITVAL_E4]]:_(s32), %bb.0, [[MIDVAL_E4]]:_(s32), %bb.1
1559
1560   CHECK: [[OTHER_PHI:%[0-9]+]]:_(s64) = G_PHI
1561
1562   CHECK: [[UNMERGE0:%[0-9]+]]:_(s32), [[UNMERGE1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI0]]:_(<2 x s32>)
1563   CHECK: [[UNMERGE2:%[0-9]+]]:_(s32), [[UNMERGE3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI1]]:_(<2 x s32>)
1564   CHECK: [[BV:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[UNMERGE0]]:_(s32), [[UNMERGE1]]:_(s32), [[UNMERGE2]]:_(s32), [[UNMERGE3]]:_(s32), [[PHI2]]:_(s32)
1565   CHECK: [[USE_OP:%[0-9]+]]:_(<5 x s32>) = G_AND [[BV]]:_, [[BV]]:_
1566   )";
1567
1568   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1569 }
1570
1571 // FNEG expansion in terms of XOR
1572 TEST_F(AArch64GISelMITest, LowerFNEG) {
1573   setUp();
1574   if (!TM)
1575     GTEST_SKIP();
1576
1577   // Declare your legalization info
1578   DefineLegalizerInfo(A, {
1579     getActionDefinitionsBuilder(G_FSUB).legalFor({s64});
1580   });
1581
1582   // Build Instr. Make sure FMF are preserved.
1583   auto FAdd =
1584     B.buildInstr(TargetOpcode::G_FADD, {LLT::scalar(64)}, {Copies[0], Copies[1]},
1585                  MachineInstr::MIFlag::FmNsz);
1586
1587   // Should not propagate the flags of src instruction.
1588   auto FNeg0 =
1589     B.buildInstr(TargetOpcode::G_FNEG, {LLT::scalar(64)}, {FAdd.getReg(0)},
1590                  {MachineInstr::MIFlag::FmArcp});
1591
1592   // Preserve the one flag.
1593   auto FNeg1 =
1594     B.buildInstr(TargetOpcode::G_FNEG, {LLT::scalar(64)}, {Copies[0]},
1595                  MachineInstr::MIFlag::FmNoInfs);
1596
1597   AInfo Info(MF->getSubtarget());
1598   DummyGISelObserver Observer;
1599   LegalizerHelper Helper(*MF, Info, Observer, B);
1600   // Perform Legalization
1601   B.setInstr(*FNeg0);
1602   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1603             Helper.lower(*FNeg0, 0, LLT::scalar(64)));
1604   B.setInstr(*FNeg1);
1605   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1606             Helper.lower(*FNeg1, 0, LLT::scalar(64)));
1607
1608   auto CheckStr = R"(
1609   CHECK: [[FADD:%[0-9]+]]:_(s64) = nsz G_FADD %0:_, %1:_
1610   CHECK: [[CONST0:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
1611   CHECK: [[FSUB0:%[0-9]+]]:_(s64) = G_XOR [[FADD]]:_, [[CONST0]]:_
1612   CHECK: [[CONST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
1613   CHECK: [[FSUB1:%[0-9]+]]:_(s64) = G_XOR %0:_, [[CONST1]]:_
1614   )";
1615
1616   // Check
1617   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1618 }
1619
1620 TEST_F(AArch64GISelMITest, LowerMinMax) {
1621   setUp();
1622   if (!TM)
1623     GTEST_SKIP();
1624
1625   LLT s64 = LLT::scalar(64);
1626   LLT v2s32 = LLT::fixed_vector(2, 32);
1627
1628   DefineLegalizerInfo(A, {
1629     getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1630         .lowerFor({s64, LLT::fixed_vector(2, s32)});
1631   });
1632
1633   auto SMin = B.buildSMin(s64, Copies[0], Copies[1]);
1634   auto SMax = B.buildSMax(s64, Copies[0], Copies[1]);
1635   auto UMin = B.buildUMin(s64, Copies[0], Copies[1]);
1636   auto UMax = B.buildUMax(s64, Copies[0], Copies[1]);
1637
1638   auto VecVal0 = B.buildBitcast(v2s32, Copies[0]);
1639   auto VecVal1 = B.buildBitcast(v2s32, Copies[1]);
1640
1641   auto SMinV = B.buildSMin(v2s32, VecVal0, VecVal1);
1642   auto SMaxV = B.buildSMax(v2s32, VecVal0, VecVal1);
1643   auto UMinV = B.buildUMin(v2s32, VecVal0, VecVal1);
1644   auto UMaxV = B.buildUMax(v2s32, VecVal0, VecVal1);
1645
1646   AInfo Info(MF->getSubtarget());
1647   DummyGISelObserver Observer;
1648   LegalizerHelper Helper(*MF, Info, Observer, B);
1649   B.setInstr(*SMin);
1650   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1651             Helper.lower(*SMin, 0, s64));
1652   B.setInstr(*SMax);
1653   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1654             Helper.lower(*SMax, 0, s64));
1655   B.setInstr(*UMin);
1656   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1657             Helper.lower(*UMin, 0, s64));
1658   B.setInstr(*UMax);
1659   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1660             Helper.lower(*UMax, 0, s64));
1661
1662   B.setInstr(*SMinV);
1663   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1664             Helper.lower(*SMinV, 0, v2s32));
1665   B.setInstr(*SMaxV);
1666   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1667             Helper.lower(*SMaxV, 0, v2s32));
1668   B.setInstr(*UMinV);
1669   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1670             Helper.lower(*UMinV, 0, v2s32));
1671   B.setInstr(*UMaxV);
1672   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1673             Helper.lower(*UMaxV, 0, v2s32));
1674
1675   auto CheckStr = R"(
1676   CHECK: [[CMP0:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), %0:_(s64), %1:_
1677   CHECK: [[SMIN:%[0-9]+]]:_(s64) = G_SELECT [[CMP0]]:_(s1), %0:_, %1:_
1678
1679   CHECK: [[CMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), %0:_(s64), %1:_
1680   CHECK: [[SMAX:%[0-9]+]]:_(s64) = G_SELECT [[CMP1]]:_(s1), %0:_, %1:_
1681
1682   CHECK: [[CMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), %0:_(s64), %1:_
1683   CHECK: [[UMIN:%[0-9]+]]:_(s64) = G_SELECT [[CMP2]]:_(s1), %0:_, %1:_
1684
1685   CHECK: [[CMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ugt), %0:_(s64), %1:_
1686   CHECK: [[UMAX:%[0-9]+]]:_(s64) = G_SELECT [[CMP3]]:_(s1), %0:_, %1:_
1687
1688   CHECK: [[VEC0:%[0-9]+]]:_(<2 x s32>) = G_BITCAST %0:_(s64)
1689   CHECK: [[VEC1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST %1:_(s64)
1690
1691   CHECK: [[VCMP0:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(slt), [[VEC0]]:_(<2 x s32>), [[VEC1]]:_
1692   CHECK: [[SMINV:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[VCMP0]]:_(<2 x s1>), [[VEC0]]:_, [[VEC1]]:_
1693
1694   CHECK: [[VCMP1:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(sgt), [[VEC0]]:_(<2 x s32>), [[VEC1]]:_
1695   CHECK: [[SMAXV:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[VCMP1]]:_(<2 x s1>), [[VEC0]]:_, [[VEC1]]:_
1696
1697   CHECK: [[VCMP2:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(ult), [[VEC0]]:_(<2 x s32>), [[VEC1]]:_
1698   CHECK: [[UMINV:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[VCMP2]]:_(<2 x s1>), [[VEC0]]:_, [[VEC1]]:_
1699
1700   CHECK: [[VCMP3:%[0-9]+]]:_(<2 x s1>) = G_ICMP intpred(ugt), [[VEC0]]:_(<2 x s32>), [[VEC1]]:_
1701   CHECK: [[UMAXV:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[VCMP3]]:_(<2 x s1>), [[VEC0]]:_, [[VEC1]]:_
1702   )";
1703
1704   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1705 }
1706
1707 TEST_F(AArch64GISelMITest, WidenScalarBuildVector) {
1708   setUp();
1709   if (!TM)
1710     GTEST_SKIP();
1711
1712   LLT S32 = LLT::scalar(32);
1713   LLT S16 = LLT::scalar(16);
1714   LLT V2S16 = LLT::fixed_vector(2, S16);
1715   LLT V2S32 = LLT::fixed_vector(2, S32);
1716
1717   DefineLegalizerInfo(A, {
1718     getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1719         .lowerFor({s64, LLT::fixed_vector(2, s32)});
1720   });
1721
1722   AInfo Info(MF->getSubtarget());
1723   DummyGISelObserver Observer;
1724   LegalizerHelper Helper(*MF, Info, Observer, B);
1725   B.setInsertPt(*EntryMBB, EntryMBB->end());
1726
1727   Register Constant0 = B.buildConstant(S16, 1).getReg(0);
1728   Register Constant1 = B.buildConstant(S16, 2).getReg(0);
1729   auto BV0 = B.buildBuildVector(V2S16, {Constant0, Constant1});
1730   auto BV1 = B.buildBuildVector(V2S16, {Constant0, Constant1});
1731
1732   B.setInstr(*BV0);
1733   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1734             Helper.widenScalar(*BV0, 0, V2S32));
1735   B.setInstr(*BV1);
1736   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1737             Helper.widenScalar(*BV1, 1, S32));
1738
1739   auto CheckStr = R"(
1740   CHECK: [[K0:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
1741   CHECK-NEXT: [[K1:%[0-9]+]]:_(s16) = G_CONSTANT i16 2
1742   CHECK-NEXT: [[EXT_K0_0:%[0-9]+]]:_(s32) = G_ANYEXT [[K0]]
1743   CHECK-NEXT: [[EXT_K1_0:%[0-9]+]]:_(s32) = G_ANYEXT [[K1]]
1744   CHECK-NEXT: [[BV0:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[EXT_K0_0]]:_(s32), [[EXT_K1_0]]:_(s32)
1745   CHECK-NEXT: [[BV0_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[BV0]]
1746
1747   CHECK: [[EXT_K0_1:%[0-9]+]]:_(s32) = G_ANYEXT [[K0]]
1748   CHECK-NEXT: [[EXT_K1_1:%[0-9]+]]:_(s32) = G_ANYEXT [[K1]]
1749
1750   CHECK-NEXT: [[BV1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[EXT_K0_1]]:_(s32), [[EXT_K1_1]]:_(s32)
1751   )";
1752
1753   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1754 }
1755
1756 TEST_F(AArch64GISelMITest, LowerMergeValues) {
1757   setUp();
1758   if (!TM)
1759     GTEST_SKIP();
1760
1761   const LLT S32 = LLT::scalar(32);
1762   const LLT S24 = LLT::scalar(24);
1763   const LLT S21 = LLT::scalar(21);
1764   const LLT S16 = LLT::scalar(16);
1765   const LLT S9 = LLT::scalar(9);
1766   const LLT S8 = LLT::scalar(8);
1767   const LLT S3 = LLT::scalar(3);
1768
1769   DefineLegalizerInfo(A, {
1770     getActionDefinitionsBuilder(G_UNMERGE_VALUES)
1771       .widenScalarIf(typeIs(1, LLT::scalar(3)), changeTo(1, LLT::scalar(9)));
1772   });
1773
1774   AInfo Info(MF->getSubtarget());
1775   DummyGISelObserver Observer;
1776   LegalizerHelper Helper(*MF, Info, Observer, B);
1777   B.setInsertPt(*EntryMBB, EntryMBB->end());
1778
1779   // 24 = 3 3 3   3 3 3   3 3
1780   //     => 9
1781   //
1782   // This can do 3 merges, but need an extra implicit_def.
1783   SmallVector<Register, 8> Merge0Ops;
1784   for (int I = 0; I != 8; ++I)
1785     Merge0Ops.push_back(B.buildConstant(S3, I).getReg(0));
1786
1787   auto Merge0 = B.buildMergeLikeInstr(S24, Merge0Ops);
1788
1789   // 21 = 3 3 3   3 3 3   3
1790   //     => 9, 2 extra implicit_def needed
1791   //
1792   SmallVector<Register, 8> Merge1Ops;
1793   for (int I = 0; I != 7; ++I)
1794     Merge1Ops.push_back(B.buildConstant(S3, I).getReg(0));
1795
1796   auto Merge1 = B.buildMergeLikeInstr(S21, Merge1Ops);
1797
1798   SmallVector<Register, 8> Merge2Ops;
1799   for (int I = 0; I != 2; ++I)
1800     Merge2Ops.push_back(B.buildConstant(S8, I).getReg(0));
1801
1802   auto Merge2 = B.buildMergeLikeInstr(S16, Merge2Ops);
1803
1804   B.setInstr(*Merge0);
1805   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1806             Helper.widenScalar(*Merge0, 1, S9));
1807   B.setInstr(*Merge1);
1808   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1809             Helper.widenScalar(*Merge1, 1, S9));
1810
1811   // Request a source size greater than the original destination size.
1812   B.setInstr(*Merge2);
1813   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1814             Helper.widenScalar(*Merge2, 1, S32));
1815
1816   auto CheckStr = R"(
1817   CHECK: [[K0:%[0-9]+]]:_(s3) = G_CONSTANT i3 0
1818   CHECK-NEXT: [[K1:%[0-9]+]]:_(s3) = G_CONSTANT i3 1
1819   CHECK-NEXT: [[K2:%[0-9]+]]:_(s3) = G_CONSTANT i3 2
1820   CHECK-NEXT: [[K3:%[0-9]+]]:_(s3) = G_CONSTANT i3 3
1821   CHECK-NEXT: [[K4:%[0-9]+]]:_(s3) = G_CONSTANT i3 -4
1822   CHECK-NEXT: [[K5:%[0-9]+]]:_(s3) = G_CONSTANT i3 -3
1823   CHECK-NEXT: [[K6:%[0-9]+]]:_(s3) = G_CONSTANT i3 -2
1824   CHECK-NEXT: [[K7:%[0-9]+]]:_(s3) = G_CONSTANT i3 -1
1825   CHECK-NEXT: [[IMPDEF0:%[0-9]+]]:_(s3) = G_IMPLICIT_DEF
1826   CHECK-NEXT: [[MERGE0:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K0]]:_(s3), [[K1]]:_(s3), [[K2]]:_(s3)
1827   CHECK-NEXT: [[MERGE1:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K3]]:_(s3), [[K4]]:_(s3), [[K5]]:_(s3)
1828   CHECK-NEXT: [[MERGE2:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K6]]:_(s3), [[K7]]:_(s3), [[IMPDEF0]]:_(s3)
1829   CHECK-NEXT: [[MERGE3:%[0-9]+]]:_(s27) = G_MERGE_VALUES [[MERGE0]]:_(s9), [[MERGE1]]:_(s9), [[MERGE2]]:_(s9)
1830   CHECK-NEXT: (s24) = G_TRUNC [[MERGE3]]:_(s27)
1831
1832
1833   CHECK: [[K8:%[0-9]+]]:_(s3) = G_CONSTANT i3 0
1834   CHECK-NEXT: [[K9:%[0-9]+]]:_(s3) = G_CONSTANT i3 1
1835   CHECK-NEXT: [[K10:%[0-9]+]]:_(s3) = G_CONSTANT i3 2
1836   CHECK-NEXT: [[K11:%[0-9]+]]:_(s3) = G_CONSTANT i3 3
1837   CHECK-NEXT: [[K12:%[0-9]+]]:_(s3) = G_CONSTANT i3 -4
1838   CHECK-NEXT: [[K13:%[0-9]+]]:_(s3) = G_CONSTANT i3 -3
1839   CHECK-NEXT: [[K14:%[0-9]+]]:_(s3) = G_CONSTANT i3 -2
1840   CHECK-NEXT: [[IMPDEF1:%[0-9]+]]:_(s3) = G_IMPLICIT_DEF
1841   CHECK-NEXT: [[MERGE4:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K8]]:_(s3), [[K9]]:_(s3), [[K10]]:_(s3)
1842   CHECK-NEXT: [[MERGE5:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K11]]:_(s3), [[K12]]:_(s3), [[K13]]:_(s3)
1843   CHECK-NEXT: [[MERGE6:%[0-9]+]]:_(s9) = G_MERGE_VALUES [[K14]]:_(s3), [[IMPDEF1]]:_(s3), [[IMPDEF1]]:_(s3)
1844   CHECK-NEXT: [[MERGE7:%[0-9]+]]:_(s27) = G_MERGE_VALUES [[MERGE4]]:_(s9), [[MERGE5]]:_(s9), [[MERGE6]]:_(s9)
1845   CHECK-NEXT: (s21) = G_TRUNC [[MERGE7]]:_(s27)
1846
1847
1848   CHECK: [[K15:%[0-9]+]]:_(s8) = G_CONSTANT i8 0
1849   CHECK-NEXT: [[K16:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
1850   CHECK-NEXT: [[ZEXT_K15:[0-9]+]]:_(s32) = G_ZEXT [[K15]]:_(s8)
1851   CHECK-NEXT: [[ZEXT_K16:[0-9]+]]:_(s32) = G_ZEXT [[K16]]:_(s8)
1852   [[K16:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
1853   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT_K16]]:_, [[K16]]:_(s32)
1854   [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT_K16]]:_, [[SHL]]:_
1855   (s16) = G_TRUNC [[OR]]:_(s32)
1856   )";
1857
1858   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1859 }
1860
1861 TEST_F(AArch64GISelMITest, WidenScalarMergeValuesPointer) {
1862   setUp();
1863   if (!TM)
1864     GTEST_SKIP();
1865
1866   DefineLegalizerInfo(A, {});
1867
1868   AInfo Info(MF->getSubtarget());
1869   DummyGISelObserver Observer;
1870   LegalizerHelper Helper(*MF, Info, Observer, B);
1871   B.setInsertPt(*EntryMBB, EntryMBB->end());
1872
1873   const LLT S32 = LLT::scalar(32);
1874   const LLT S64 = LLT::scalar(64);
1875   const LLT P0 = LLT::pointer(0, 64);
1876
1877   auto Lo = B.buildTrunc(S32, Copies[0]);
1878   auto Hi = B.buildTrunc(S32, Copies[1]);
1879
1880   auto Merge = B.buildMergeLikeInstr(P0, {Lo, Hi});
1881
1882   B.setInstr(*Merge);
1883   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
1884             Helper.widenScalar(*Merge, 1, S64));
1885
1886   auto CheckStr = R"(
1887    CHECK: [[TRUNC0:%[0-9]+]]:_(s32) = G_TRUNC
1888    CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC
1889    CHECK: [[ZEXT_TRUNC0:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC0]]
1890    CHECK: [[ZEXT_TRUNC1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]]
1891    CHECK: [[SHIFT_AMT:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
1892    CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ZEXT_TRUNC1]]:_, [[SHIFT_AMT]]
1893    CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[ZEXT_TRUNC0]]:_, [[SHL]]
1894    CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[OR]]:_(s64)
1895   )";
1896
1897   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1898 }
1899
1900 TEST_F(AArch64GISelMITest, WidenSEXTINREG) {
1901   setUp();
1902   if (!TM)
1903     GTEST_SKIP();
1904
1905   // Declare your legalization info
1906   DefineLegalizerInfo(A, {
1907     getActionDefinitionsBuilder(G_SEXT_INREG).legalForTypeWithAnyImm({s64});
1908   });
1909   // Build Instr
1910   auto MIB = B.buildInstr(
1911       TargetOpcode::G_SEXT_INREG, {LLT::scalar(32)},
1912       {B.buildInstr(TargetOpcode::G_TRUNC, {LLT::scalar(32)}, {Copies[0]}),
1913        uint64_t(8)});
1914   AInfo Info(MF->getSubtarget());
1915   DummyGISelObserver Observer;
1916   LegalizerHelper Helper(*MF, Info, Observer, B);
1917   // Perform Legalization
1918   B.setInstr(*MIB);
1919   ASSERT_TRUE(Helper.widenScalar(*MIB, 0, LLT::scalar(64)) ==
1920               LegalizerHelper::LegalizeResult::Legalized);
1921
1922   auto CheckStr = R"(
1923   CHECK: [[T0:%[0-9]+]]:_(s32) = G_TRUNC
1924   CHECK: [[T1:%[0-9]+]]:_(s64) = G_ANYEXT [[T0]]:_(s32)
1925   CHECK: [[T2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[T1]]:_, 8
1926   CHECK: [[T3:%[0-9]+]]:_(s32) = G_TRUNC [[T2]]:_(s64)
1927   )";
1928
1929   // Check
1930   ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1931 }
1932
1933 TEST_F(AArch64GISelMITest, NarrowSEXTINREG) {
1934   setUp();
1935   if (!TM)
1936     GTEST_SKIP();
1937
1938   // Declare your legalization info, these aren't actually relevant to the test.
1939   DefineLegalizerInfo(A, {
1940     getActionDefinitionsBuilder(G_SEXT_INREG).legalForTypeWithAnyImm({s64});
1941   });
1942   // Build Instr
1943   auto MIB = B.buildInstr(
1944       TargetOpcode::G_SEXT_INREG, {LLT::scalar(16)},
1945       {B.buildInstr(TargetOpcode::G_TRUNC, {LLT::scalar(16)}, {Copies[0]}),
1946        uint64_t(8)});
1947   AInfo Info(MF->getSubtarget());
1948   DummyGISelObserver Observer;
1949   LegalizerHelper Helper(*MF, Info, Observer, B);
1950   // Perform Legalization
1951   B.setInstr(*MIB);
1952   ASSERT_TRUE(Helper.narrowScalar(*MIB, 0, LLT::scalar(10)) ==
1953               LegalizerHelper::LegalizeResult::Legalized);
1954
1955   auto CheckStr = R"(
1956   CHECK: [[T0:%[0-9]+]]:_(s16) = G_TRUNC
1957   CHECK: [[T1:%[0-9]+]]:_(s10) = G_TRUNC [[T0]]:_(s16)
1958   CHECK: [[T2:%[0-9]+]]:_(s10) = G_SEXT_INREG [[T1]]:_, 8
1959   CHECK: [[T3:%[0-9]+]]:_(s16) = G_SEXT [[T2]]:_(s10)
1960   )";
1961
1962   // Check
1963   ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
1964 }
1965
1966 TEST_F(AArch64GISelMITest, NarrowSEXTINREG2) {
1967   setUp();
1968   if (!TM)
1969     GTEST_SKIP();
1970
1971   // Declare your legalization info, these aren't actually relevant to the test.
1972   DefineLegalizerInfo(
1973       A, { getActionDefinitionsBuilder(G_SEXT_INREG).legalForTypeWithAnyImm({s64}); });
1974   // Build Instr
1975   auto MIB = B.buildInstr(
1976       TargetOpcode::G_SEXT_INREG, {LLT::scalar(32)},
1977       {B.buildInstr(TargetOpcode::G_TRUNC, {LLT::scalar(32)}, {Copies[0]}),
1978        uint64_t(9)});
1979   AInfo Info(MF->getSubtarget());
1980   DummyGISelObserver Observer;
1981   LegalizerHelper Helper(*MF, Info, Observer, B);
1982   // Perform Legalization
1983   B.setInstr(*MIB);
1984   ASSERT_TRUE(Helper.narrowScalar(*MIB, 0, LLT::scalar(8)) ==
1985               LegalizerHelper::LegalizeResult::Legalized);
1986
1987   auto CheckStr = R"(
1988   CHECK: [[T0:%[0-9]+]]:_(s32) = G_TRUNC
1989   CHECK: [[T1:%[0-9]+]]:_(s8), [[T2:%[0-9]+]]:_(s8), [[T3:%[0-9]+]]:_(s8), [[T4:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[T0]]:_(s32)
1990   CHECK: [[CST2:%[0-9]+]]:_(s8) = G_CONSTANT i8 7
1991   CHECK: [[T5:%[0-9]+]]:_(s8) = G_SEXT_INREG [[T2]]:_, 1
1992   CHECK: [[T6:%[0-9]+]]:_(s8) = G_ASHR [[T5]]:_, [[CST2]]:_
1993   CHECK: [[T7:%[0-9]+]]:_(s32) = G_MERGE_VALUES [[T1]]:_(s8), [[T5]]:_(s8), [[T6]]:_(s8), [[T6]]:_(s8)
1994   )";
1995
1996   // Check
1997   ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
1998 }
1999
2000 TEST_F(AArch64GISelMITest, LowerSEXTINREG) {
2001   setUp();
2002   if (!TM)
2003     GTEST_SKIP();
2004
2005   // Declare your legalization info, these aren't actually relevant to the test.
2006   DefineLegalizerInfo(
2007       A, { getActionDefinitionsBuilder(G_SEXT_INREG).legalForTypeWithAnyImm({s64}); });
2008   // Build Instr
2009   auto MIB = B.buildInstr(
2010       TargetOpcode::G_SEXT_INREG, {LLT::scalar(32)},
2011       {B.buildInstr(TargetOpcode::G_TRUNC, {LLT::scalar(32)}, {Copies[0]}),
2012        uint64_t(8)});
2013   AInfo Info(MF->getSubtarget());
2014   DummyGISelObserver Observer;
2015   LegalizerHelper Helper(*MF, Info, Observer, B);
2016   // Perform Legalization
2017   B.setInstr(*MIB);
2018   ASSERT_TRUE(Helper.lower(*MIB, 0, LLT()) ==
2019               LegalizerHelper::LegalizeResult::Legalized);
2020
2021   auto CheckStr = R"(
2022   CHECK: [[T1:%[0-9]+]]:_(s32) = G_TRUNC
2023   CHECK: [[CST:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
2024   CHECK: [[T2:%[0-9]+]]:_(s32) = G_SHL [[T1]]:_, [[CST]]:_
2025   CHECK: [[T3:%[0-9]+]]:_(s32) = G_ASHR [[T2]]:_, [[CST]]:_
2026   )";
2027
2028   // Check
2029   ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
2030 }
2031
2032 TEST_F(AArch64GISelMITest, LibcallFPExt) {
2033   setUp();
2034   if (!TM)
2035     GTEST_SKIP();
2036
2037   // Declare your legalization info
2038   DefineLegalizerInfo(A, {
2039     getActionDefinitionsBuilder(G_FPEXT).libcallFor({{s32, s16}, {s128, s64}});
2040   });
2041
2042   LLT S16{LLT::scalar(16)};
2043   LLT S32{LLT::scalar(32)};
2044   LLT S128{LLT::scalar(128)};
2045   auto MIBTrunc = B.buildTrunc(S16, Copies[0]);
2046   auto MIBFPExt1 =
2047       B.buildInstr(TargetOpcode::G_FPEXT, {S32}, {MIBTrunc});
2048
2049   auto MIBFPExt2 =
2050       B.buildInstr(TargetOpcode::G_FPEXT, {S128}, {Copies[1]});
2051   AInfo Info(MF->getSubtarget());
2052   DummyGISelObserver Observer;
2053   LegalizerHelper Helper(*MF, Info, Observer, B);
2054   LostDebugLocObserver DummyLocObserver("");
2055   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2056               Helper.libcall(*MIBFPExt1, DummyLocObserver));
2057
2058   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2059               Helper.libcall(*MIBFPExt2, DummyLocObserver));
2060   auto CheckStr = R"(
2061   CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC
2062   CHECK: $h0 = COPY [[TRUNC]]
2063   CHECK: BL &__gnu_h2f_ieee
2064   CHECK: $d0 = COPY
2065   CHECK: BL &__extenddftf2
2066   )";
2067
2068   // Check
2069   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2070 }
2071
2072 TEST_F(AArch64GISelMITest, LibcallFPTrunc) {
2073   setUp();
2074   if (!TM)
2075     GTEST_SKIP();
2076
2077   // Declare your legalization info
2078   DefineLegalizerInfo(A, {
2079     getActionDefinitionsBuilder(G_FPTRUNC).libcallFor({{s16, s32}, {s64, s128}});
2080   });
2081
2082   LLT S16{LLT::scalar(16)};
2083   LLT S32{LLT::scalar(32)};
2084   LLT S64{LLT::scalar(64)};
2085   LLT S128{LLT::scalar(128)};
2086   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2087   auto MIBFPTrunc1 =
2088       B.buildInstr(TargetOpcode::G_FPTRUNC, {S16}, {MIBTrunc});
2089
2090   auto MIBMerge = B.buildMergeLikeInstr(S128, {Copies[1], Copies[2]});
2091
2092   auto MIBFPTrunc2 =
2093       B.buildInstr(TargetOpcode::G_FPTRUNC, {S64}, {MIBMerge});
2094   AInfo Info(MF->getSubtarget());
2095   DummyGISelObserver Observer;
2096   LostDebugLocObserver DummyLocObserver("");
2097   LegalizerHelper Helper(*MF, Info, Observer, B);
2098   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2099             Helper.libcall(*MIBFPTrunc1, DummyLocObserver));
2100
2101   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2102             Helper.libcall(*MIBFPTrunc2, DummyLocObserver));
2103   auto CheckStr = R"(
2104   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2105   CHECK: $s0 = COPY [[TRUNC]]
2106   CHECK: BL &__gnu_f2h_ieee
2107   CHECK: $q0 = COPY
2108   CHECK: BL &__trunctfdf2
2109   )";
2110
2111   // Check
2112   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2113 }
2114
2115 TEST_F(AArch64GISelMITest, LibcallSimple) {
2116   setUp();
2117   if (!TM)
2118     GTEST_SKIP();
2119
2120   // Declare your legalization info
2121   DefineLegalizerInfo(A, {
2122     getActionDefinitionsBuilder(G_FADD).libcallFor({s16});
2123   });
2124
2125   LLT S16{LLT::scalar(16)};
2126   auto MIBTrunc = B.buildTrunc(S16, Copies[0]);
2127   auto MIBFADD =
2128       B.buildInstr(TargetOpcode::G_FADD, {S16}, {MIBTrunc, MIBTrunc});
2129
2130   AInfo Info(MF->getSubtarget());
2131   DummyGISelObserver Observer;
2132   LostDebugLocObserver DummyLocObserver("");
2133   LegalizerHelper Helper(*MF, Info, Observer, B);
2134   // Make sure we do not crash anymore
2135   EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize,
2136             Helper.libcall(*MIBFADD, DummyLocObserver));
2137 }
2138
2139 TEST_F(AArch64GISelMITest, LibcallMul) {
2140   setUp();
2141   if (!TM)
2142     GTEST_SKIP();
2143
2144   // Declare your legalization info
2145   DefineLegalizerInfo(A, {
2146     getActionDefinitionsBuilder(G_MUL).libcallFor({s32, s64, s128});
2147   });
2148
2149   LLT S32{LLT::scalar(32)};
2150   LLT S64{LLT::scalar(64)};
2151   LLT S128{LLT::scalar(128)};
2152   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2153   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2154
2155   auto MIBMul32 =
2156       B.buildInstr(TargetOpcode::G_MUL, {S32}, {MIBTrunc, MIBTrunc});
2157   auto MIBMul64 =
2158       B.buildInstr(TargetOpcode::G_MUL, {S64}, {Copies[0], Copies[0]});
2159   auto MIBMul128 =
2160       B.buildInstr(TargetOpcode::G_MUL, {S128}, {MIBExt, MIBExt});
2161
2162   AInfo Info(MF->getSubtarget());
2163   DummyGISelObserver Observer;
2164   LostDebugLocObserver DummyLocObserver("");
2165   LegalizerHelper Helper(*MF, Info, Observer, B);
2166
2167   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2168             Helper.libcall(*MIBMul32, DummyLocObserver));
2169   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2170             Helper.libcall(*MIBMul64, DummyLocObserver));
2171   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2172             Helper.libcall(*MIBMul128, DummyLocObserver));
2173
2174   auto CheckStr = R"(
2175   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2176   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2177   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2178   CHECK: $w0 = COPY [[TRUNC]]
2179   CHECK: $w1 = COPY [[TRUNC]]
2180   CHECK: BL &__mulsi3
2181   CHECK: $x0 = COPY [[COPY]]
2182   CHECK: $x1 = COPY [[COPY]]
2183   CHECK: BL &__muldi3
2184   CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2185   CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2186   CHECK: $x0 = COPY [[UV]]
2187   CHECK: $x1 = COPY [[UV1]]
2188   CHECK: $x2 = COPY [[UV2]]
2189   CHECK: $x3 = COPY [[UV3]]
2190   CHECK: BL &__multi3
2191   )";
2192
2193   // Check
2194   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2195 }
2196
2197 TEST_F(AArch64GISelMITest, LibcallSRem) {
2198   setUp();
2199   if (!TM)
2200     GTEST_SKIP();
2201
2202   // Declare your legalization info
2203   DefineLegalizerInfo(A, {
2204     getActionDefinitionsBuilder(G_SREM).libcallFor({s32, s64, s128});
2205   });
2206
2207   LLT S32{LLT::scalar(32)};
2208   LLT S64{LLT::scalar(64)};
2209   LLT S128{LLT::scalar(128)};
2210   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2211   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2212
2213   auto MIBSRem32 =
2214       B.buildInstr(TargetOpcode::G_SREM, {S32}, {MIBTrunc, MIBTrunc});
2215   auto MIBSRem64 =
2216       B.buildInstr(TargetOpcode::G_SREM, {S64}, {Copies[0], Copies[0]});
2217   auto MIBSRem128 =
2218       B.buildInstr(TargetOpcode::G_SREM, {S128}, {MIBExt, MIBExt});
2219
2220   AInfo Info(MF->getSubtarget());
2221   DummyGISelObserver Observer;
2222   LostDebugLocObserver DummyLocObserver("");
2223   LegalizerHelper Helper(*MF, Info, Observer, B);
2224
2225   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2226             Helper.libcall(*MIBSRem32, DummyLocObserver));
2227   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2228             Helper.libcall(*MIBSRem64, DummyLocObserver));
2229   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2230             Helper.libcall(*MIBSRem128, DummyLocObserver));
2231
2232   auto CheckStr = R"(
2233   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2234   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2235   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2236   CHECK: $w0 = COPY [[TRUNC]]
2237   CHECK: $w1 = COPY [[TRUNC]]
2238   CHECK: BL &__modsi3
2239   CHECK: $x0 = COPY [[COPY]]
2240   CHECK: $x1 = COPY [[COPY]]
2241   CHECK: BL &__moddi3
2242   CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2243   CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2244   CHECK: $x0 = COPY [[UV]]
2245   CHECK: $x1 = COPY [[UV1]]
2246   CHECK: $x2 = COPY [[UV2]]
2247   CHECK: $x3 = COPY [[UV3]]
2248   CHECK: BL &__modti3
2249   )";
2250
2251   // Check
2252   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2253 }
2254
2255 TEST_F(AArch64GISelMITest, LibcallURem) {
2256   setUp();
2257   if (!TM)
2258     GTEST_SKIP();
2259
2260   // Declare your legalization info
2261   DefineLegalizerInfo(A, {
2262     getActionDefinitionsBuilder(G_UREM).libcallFor({s32, s64, s128});
2263   });
2264
2265   LLT S32{LLT::scalar(32)};
2266   LLT S64{LLT::scalar(64)};
2267   LLT S128{LLT::scalar(128)};
2268   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2269   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2270
2271   auto MIBURem32 =
2272       B.buildInstr(TargetOpcode::G_UREM, {S32}, {MIBTrunc, MIBTrunc});
2273   auto MIBURem64 =
2274       B.buildInstr(TargetOpcode::G_UREM, {S64}, {Copies[0], Copies[0]});
2275   auto MIBURem128 =
2276       B.buildInstr(TargetOpcode::G_UREM, {S128}, {MIBExt, MIBExt});
2277
2278   AInfo Info(MF->getSubtarget());
2279   DummyGISelObserver Observer;
2280   LostDebugLocObserver DummyLocObserver("");
2281   LegalizerHelper Helper(*MF, Info, Observer, B);
2282
2283   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2284             Helper.libcall(*MIBURem32, DummyLocObserver));
2285   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2286             Helper.libcall(*MIBURem64, DummyLocObserver));
2287   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2288             Helper.libcall(*MIBURem128, DummyLocObserver));
2289
2290   const auto *CheckStr = R"(
2291   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2292   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2293   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2294   CHECK: $w0 = COPY [[TRUNC]]
2295   CHECK: $w1 = COPY [[TRUNC]]
2296   CHECK: BL &__umodsi3
2297   CHECK: $x0 = COPY [[COPY]]
2298   CHECK: $x1 = COPY [[COPY]]
2299   CHECK: BL &__umoddi3
2300   CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2301   CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2302   CHECK: $x0 = COPY [[UV]]
2303   CHECK: $x1 = COPY [[UV1]]
2304   CHECK: $x2 = COPY [[UV2]]
2305   CHECK: $x3 = COPY [[UV3]]
2306   CHECK: BL &__umodti3
2307   )";
2308
2309   // Check
2310   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2311 }
2312
2313 TEST_F(AArch64GISelMITest, LibcallCtlzZeroUndef) {
2314   setUp();
2315   if (!TM)
2316     GTEST_SKIP();
2317
2318   // Declare your legalization info
2319   DefineLegalizerInfo(A, {
2320     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF)
2321         .libcallFor({{s32, s32}, {s64, s64}, {s128, s128}});
2322   });
2323
2324   LLT S32{LLT::scalar(32)};
2325   LLT S64{LLT::scalar(64)};
2326   LLT S128{LLT::scalar(128)};
2327   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2328   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2329
2330   auto MIBCtlz32 =
2331       B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {S32}, {MIBTrunc});
2332   auto MIBCtlz64 =
2333       B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {S64}, {Copies[0]});
2334   auto MIBCtlz128 =
2335       B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {S128}, {MIBExt});
2336
2337   AInfo Info(MF->getSubtarget());
2338   DummyGISelObserver Observer;
2339   LostDebugLocObserver DummyLocObserver("");
2340   LegalizerHelper Helper(*MF, Info, Observer, B);
2341
2342   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2343             Helper.libcall(*MIBCtlz32, DummyLocObserver));
2344   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2345             Helper.libcall(*MIBCtlz64, DummyLocObserver));
2346   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2347             Helper.libcall(*MIBCtlz128, DummyLocObserver));
2348
2349   const auto *CheckStr = R"(
2350   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2351   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2352   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2353   CHECK: $w0 = COPY [[TRUNC]]
2354   CHECK: BL &__clzsi2
2355   CHECK: $x0 = COPY [[COPY]]
2356   CHECK: BL &__clzdi2
2357   CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]]
2358   CHECK: $x0 = COPY [[UV]]
2359   CHECK: $x1 = COPY [[UV1]]
2360   CHECK: BL &__clzti2
2361   )";
2362
2363   // Check
2364   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2365 }
2366
2367 TEST_F(AArch64GISelMITest, LibcallFAdd) {
2368   setUp();
2369   if (!TM)
2370     GTEST_SKIP();
2371
2372   // Declare your legalization info
2373   DefineLegalizerInfo(A, {
2374     getActionDefinitionsBuilder(G_FADD).libcallFor({s32, s64, s128});
2375   });
2376
2377   LLT S32{LLT::scalar(32)};
2378   LLT S64{LLT::scalar(64)};
2379   LLT S128{LLT::scalar(128)};
2380   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2381   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2382
2383   auto MIBAdd32 =
2384       B.buildInstr(TargetOpcode::G_FADD, {S32}, {MIBTrunc, MIBTrunc});
2385   auto MIBAdd64 =
2386       B.buildInstr(TargetOpcode::G_FADD, {S64}, {Copies[0], Copies[0]});
2387   auto MIBAdd128 = B.buildInstr(TargetOpcode::G_FADD, {S128}, {MIBExt, MIBExt});
2388
2389   AInfo Info(MF->getSubtarget());
2390   DummyGISelObserver Observer;
2391   LostDebugLocObserver DummyLocObserver("");
2392   LegalizerHelper Helper(*MF, Info, Observer, B);
2393
2394   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2395             Helper.libcall(*MIBAdd32, DummyLocObserver));
2396   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2397             Helper.libcall(*MIBAdd64, DummyLocObserver));
2398   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2399             Helper.libcall(*MIBAdd128, DummyLocObserver));
2400
2401   const auto *CheckStr = R"(
2402   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2403   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2404   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2405   CHECK: $s0 = COPY [[TRUNC]]
2406   CHECK: $s1 = COPY [[TRUNC]]
2407   CHECK: BL &__addsf3
2408   CHECK: $d0 = COPY [[COPY]]
2409   CHECK: $d1 = COPY [[COPY]]
2410   CHECK: BL &__adddf3
2411   CHECK: $q0 = COPY [[ANYEXT]]
2412   CHECK: $q1 = COPY [[ANYEXT]]
2413   CHECK: BL &__addtf3
2414   )";
2415
2416   // Check
2417   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2418 }
2419
2420 TEST_F(AArch64GISelMITest, LibcallFSub) {
2421   setUp();
2422   if (!TM)
2423     GTEST_SKIP();
2424
2425   // Declare your legalization info
2426   DefineLegalizerInfo(A, {
2427     getActionDefinitionsBuilder(G_FSUB).libcallFor({s32, s64, s128});
2428   });
2429
2430   LLT S32{LLT::scalar(32)};
2431   LLT S64{LLT::scalar(64)};
2432   LLT S128{LLT::scalar(128)};
2433   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2434   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2435
2436   auto MIBSub32 =
2437       B.buildInstr(TargetOpcode::G_FSUB, {S32}, {MIBTrunc, MIBTrunc});
2438   auto MIBSub64 =
2439       B.buildInstr(TargetOpcode::G_FSUB, {S64}, {Copies[0], Copies[0]});
2440   auto MIBSub128 = B.buildInstr(TargetOpcode::G_FSUB, {S128}, {MIBExt, MIBExt});
2441
2442   AInfo Info(MF->getSubtarget());
2443   DummyGISelObserver Observer;
2444   LostDebugLocObserver DummyLocObserver("");
2445   LegalizerHelper Helper(*MF, Info, Observer, B);
2446
2447   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2448             Helper.libcall(*MIBSub32, DummyLocObserver));
2449   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2450             Helper.libcall(*MIBSub64, DummyLocObserver));
2451   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2452             Helper.libcall(*MIBSub128, DummyLocObserver));
2453
2454   const auto *CheckStr = R"(
2455   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2456   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2457   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2458   CHECK: $s0 = COPY [[TRUNC]]
2459   CHECK: $s1 = COPY [[TRUNC]]
2460   CHECK: BL &__subsf3
2461   CHECK: $d0 = COPY [[COPY]]
2462   CHECK: $d1 = COPY [[COPY]]
2463   CHECK: BL &__subdf3
2464   CHECK: $q0 = COPY [[ANYEXT]]
2465   CHECK: $q1 = COPY [[ANYEXT]]
2466   CHECK: BL &__subtf3
2467   )";
2468
2469   // Check
2470   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2471 }
2472
2473 TEST_F(AArch64GISelMITest, LibcallFMul) {
2474   setUp();
2475   if (!TM)
2476     GTEST_SKIP();
2477
2478   // Declare your legalization info
2479   DefineLegalizerInfo(A, {
2480     getActionDefinitionsBuilder(G_FMUL).libcallFor({s32, s64, s128});
2481   });
2482
2483   LLT S32{LLT::scalar(32)};
2484   LLT S64{LLT::scalar(64)};
2485   LLT S128{LLT::scalar(128)};
2486   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2487   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2488
2489   auto MIBMul32 =
2490       B.buildInstr(TargetOpcode::G_FMUL, {S32}, {MIBTrunc, MIBTrunc});
2491   auto MIBMul64 =
2492       B.buildInstr(TargetOpcode::G_FMUL, {S64}, {Copies[0], Copies[0]});
2493   auto MIBMul128 = B.buildInstr(TargetOpcode::G_FMUL, {S128}, {MIBExt, MIBExt});
2494
2495   AInfo Info(MF->getSubtarget());
2496   DummyGISelObserver Observer;
2497   LegalizerHelper Helper(*MF, Info, Observer, B);
2498   LostDebugLocObserver DummyLocObserver("");
2499
2500   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2501             Helper.libcall(*MIBMul32, DummyLocObserver));
2502   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2503             Helper.libcall(*MIBMul64, DummyLocObserver));
2504   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2505             Helper.libcall(*MIBMul128, DummyLocObserver));
2506
2507   const auto *CheckStr = R"(
2508   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2509   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2510   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2511   CHECK: $s0 = COPY [[TRUNC]]
2512   CHECK: $s1 = COPY [[TRUNC]]
2513   CHECK: BL &__mulsf3
2514   CHECK: $d0 = COPY [[COPY]]
2515   CHECK: $d1 = COPY [[COPY]]
2516   CHECK: BL &__muldf3
2517   CHECK: $q0 = COPY [[ANYEXT]]
2518   CHECK: $q1 = COPY [[ANYEXT]]
2519   CHECK: BL &__multf3
2520   )";
2521
2522   // Check
2523   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2524 }
2525
2526 TEST_F(AArch64GISelMITest, LibcallFDiv) {
2527   setUp();
2528   if (!TM)
2529     GTEST_SKIP();
2530
2531   // Declare your legalization info
2532   DefineLegalizerInfo(A, {
2533     getActionDefinitionsBuilder(G_FDIV).libcallFor({s32, s64, s128});
2534   });
2535
2536   LLT S32{LLT::scalar(32)};
2537   LLT S64{LLT::scalar(64)};
2538   LLT S128{LLT::scalar(128)};
2539   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2540   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2541
2542   auto MIBDiv32 =
2543       B.buildInstr(TargetOpcode::G_FDIV, {S32}, {MIBTrunc, MIBTrunc});
2544   auto MIBDiv64 =
2545       B.buildInstr(TargetOpcode::G_FDIV, {S64}, {Copies[0], Copies[0]});
2546   auto MIBDiv128 = B.buildInstr(TargetOpcode::G_FDIV, {S128}, {MIBExt, MIBExt});
2547
2548   AInfo Info(MF->getSubtarget());
2549   DummyGISelObserver Observer;
2550   LostDebugLocObserver DummyLocObserver("");
2551   LegalizerHelper Helper(*MF, Info, Observer, B);
2552
2553   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2554             Helper.libcall(*MIBDiv32, DummyLocObserver));
2555   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2556             Helper.libcall(*MIBDiv64, DummyLocObserver));
2557   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2558             Helper.libcall(*MIBDiv128, DummyLocObserver));
2559
2560   const auto *CheckStr = R"(
2561   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2562   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2563   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2564   CHECK: $s0 = COPY [[TRUNC]]
2565   CHECK: $s1 = COPY [[TRUNC]]
2566   CHECK: BL &__divsf3
2567   CHECK: $d0 = COPY [[COPY]]
2568   CHECK: $d1 = COPY [[COPY]]
2569   CHECK: BL &__divdf3
2570   CHECK: $q0 = COPY [[ANYEXT]]
2571   CHECK: $q1 = COPY [[ANYEXT]]
2572   CHECK: BL &__divtf3
2573   )";
2574
2575   // Check
2576   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2577 }
2578
2579 TEST_F(AArch64GISelMITest, LibcallFExp) {
2580   setUp();
2581   if (!TM)
2582     GTEST_SKIP();
2583
2584   // Declare your legalization info
2585   DefineLegalizerInfo(A, {
2586     getActionDefinitionsBuilder(G_FEXP).libcallFor({s32, s64, s128});
2587   });
2588
2589   LLT S32{LLT::scalar(32)};
2590   LLT S64{LLT::scalar(64)};
2591   LLT S128{LLT::scalar(128)};
2592   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2593   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2594
2595   auto MIBExp32 = B.buildInstr(TargetOpcode::G_FEXP, {S32}, {MIBTrunc});
2596   auto MIBExp64 = B.buildInstr(TargetOpcode::G_FEXP, {S64}, {Copies[0]});
2597   auto MIBExp128 = B.buildInstr(TargetOpcode::G_FEXP, {S128}, {MIBExt});
2598
2599   AInfo Info(MF->getSubtarget());
2600   DummyGISelObserver Observer;
2601   LostDebugLocObserver DummyLocObserver("");
2602   LegalizerHelper Helper(*MF, Info, Observer, B);
2603
2604   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2605             Helper.libcall(*MIBExp32, DummyLocObserver));
2606   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2607             Helper.libcall(*MIBExp64, DummyLocObserver));
2608   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2609             Helper.libcall(*MIBExp128, DummyLocObserver));
2610
2611   const auto *CheckStr = R"(
2612   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2613   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2614   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2615   CHECK: $s0 = COPY [[TRUNC]]
2616   CHECK: BL &expf
2617   CHECK: $d0 = COPY [[COPY]]
2618   CHECK: BL &exp
2619   CHECK: $q0 = COPY [[ANYEXT]]
2620   CHECK: BL &expl
2621   )";
2622
2623   // Check
2624   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2625 }
2626
2627 TEST_F(AArch64GISelMITest, LibcallFExp2) {
2628   setUp();
2629   if (!TM)
2630     GTEST_SKIP();
2631
2632   // Declare your legalization info
2633   DefineLegalizerInfo(A, {
2634     getActionDefinitionsBuilder(G_FEXP2).libcallFor({s32, s64, s128});
2635   });
2636
2637   LLT S32{LLT::scalar(32)};
2638   LLT S64{LLT::scalar(64)};
2639   LLT S128{LLT::scalar(128)};
2640   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2641   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2642
2643   auto MIBExp232 = B.buildInstr(TargetOpcode::G_FEXP2, {S32}, {MIBTrunc});
2644   auto MIBExp264 = B.buildInstr(TargetOpcode::G_FEXP2, {S64}, {Copies[0]});
2645   auto MIBExp2128 = B.buildInstr(TargetOpcode::G_FEXP2, {S128}, {MIBExt});
2646
2647   AInfo Info(MF->getSubtarget());
2648   DummyGISelObserver Observer;
2649   LostDebugLocObserver DummyLocObserver("");
2650   LegalizerHelper Helper(*MF, Info, Observer, B);
2651
2652   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2653             Helper.libcall(*MIBExp232, DummyLocObserver));
2654   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2655             Helper.libcall(*MIBExp264, DummyLocObserver));
2656   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2657             Helper.libcall(*MIBExp2128, DummyLocObserver));
2658
2659   const auto *CheckStr = R"(
2660   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2661   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2662   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2663   CHECK: $s0 = COPY [[TRUNC]]
2664   CHECK: BL &exp2f
2665   CHECK: $d0 = COPY [[COPY]]
2666   CHECK: BL &exp2
2667   CHECK: $q0 = COPY [[ANYEXT]]
2668   CHECK: BL &exp2l
2669   )";
2670
2671   // Check
2672   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2673 }
2674
2675 TEST_F(AArch64GISelMITest, LibcallFRem) {
2676   setUp();
2677   if (!TM)
2678     GTEST_SKIP();
2679
2680   // Declare your legalization info
2681   DefineLegalizerInfo(A, {
2682     getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64, s128});
2683   });
2684
2685   LLT S32{LLT::scalar(32)};
2686   LLT S64{LLT::scalar(64)};
2687   LLT S128{LLT::scalar(128)};
2688   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2689   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2690
2691   auto MIBFRem32 = B.buildInstr(TargetOpcode::G_FREM, {S32}, {MIBTrunc});
2692   auto MIBFRem64 = B.buildInstr(TargetOpcode::G_FREM, {S64}, {Copies[0]});
2693   auto MIBFRem128 = B.buildInstr(TargetOpcode::G_FREM, {S128}, {MIBExt});
2694
2695   AInfo Info(MF->getSubtarget());
2696   DummyGISelObserver Observer;
2697   LostDebugLocObserver DummyLocObserver("");
2698   LegalizerHelper Helper(*MF, Info, Observer, B);
2699
2700   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2701             Helper.libcall(*MIBFRem32, DummyLocObserver));
2702   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2703             Helper.libcall(*MIBFRem64, DummyLocObserver));
2704   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2705             Helper.libcall(*MIBFRem128, DummyLocObserver));
2706
2707   const auto *CheckStr = R"(
2708   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2709   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2710   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2711   CHECK: $s0 = COPY [[TRUNC]]
2712   CHECK: BL &fmodf
2713   CHECK: $d0 = COPY [[COPY]]
2714   CHECK: BL &fmod
2715   CHECK: $q0 = COPY [[ANYEXT]]
2716   CHECK: BL &fmodl
2717   )";
2718
2719   // Check
2720   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2721 }
2722
2723 TEST_F(AArch64GISelMITest, LibcallFPow) {
2724   setUp();
2725   if (!TM)
2726     GTEST_SKIP();
2727
2728   // Declare your legalization info
2729   DefineLegalizerInfo(A, {
2730     getActionDefinitionsBuilder(G_FPOW).libcallFor({s32, s64, s128});
2731   });
2732
2733   LLT S32{LLT::scalar(32)};
2734   LLT S64{LLT::scalar(64)};
2735   LLT S128{LLT::scalar(128)};
2736   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2737   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2738
2739   auto MIBPow32 = B.buildInstr(TargetOpcode::G_FPOW, {S32}, {MIBTrunc});
2740   auto MIBPow64 = B.buildInstr(TargetOpcode::G_FPOW, {S64}, {Copies[0]});
2741   auto MIBPow128 = B.buildInstr(TargetOpcode::G_FPOW, {S128}, {MIBExt});
2742
2743   AInfo Info(MF->getSubtarget());
2744   DummyGISelObserver Observer;
2745   LostDebugLocObserver DummyLocObserver("");
2746   LegalizerHelper Helper(*MF, Info, Observer, B);
2747
2748   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2749             Helper.libcall(*MIBPow32, DummyLocObserver));
2750   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2751             Helper.libcall(*MIBPow64, DummyLocObserver));
2752   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2753             Helper.libcall(*MIBPow128, DummyLocObserver));
2754
2755   const auto *CheckStr = R"(
2756   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2757   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2758   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2759   CHECK: $s0 = COPY [[TRUNC]]
2760   CHECK: BL &powf
2761   CHECK: $d0 = COPY [[COPY]]
2762   CHECK: BL &pow
2763   CHECK: $q0 = COPY [[ANYEXT]]
2764   CHECK: BL &powl
2765   )";
2766
2767   // Check
2768   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2769 }
2770
2771 TEST_F(AArch64GISelMITest, LibcallFMa) {
2772   setUp();
2773   if (!TM)
2774     GTEST_SKIP();
2775
2776   // Declare your legalization info
2777   DefineLegalizerInfo(A, {
2778     getActionDefinitionsBuilder(G_FMA).libcallFor({s32, s64, s128});
2779   });
2780
2781   LLT S32{LLT::scalar(32)};
2782   LLT S64{LLT::scalar(64)};
2783   LLT S128{LLT::scalar(128)};
2784   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2785   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2786
2787   auto MIBMa32 = B.buildInstr(TargetOpcode::G_FMA, {S32}, {MIBTrunc, MIBTrunc});
2788   auto MIBMa64 =
2789       B.buildInstr(TargetOpcode::G_FMA, {S64}, {Copies[0], Copies[0]});
2790   auto MIBMa128 = B.buildInstr(TargetOpcode::G_FMA, {S128}, {MIBExt, MIBExt});
2791
2792   AInfo Info(MF->getSubtarget());
2793   DummyGISelObserver Observer;
2794   LostDebugLocObserver DummyLocObserver("");
2795   LegalizerHelper Helper(*MF, Info, Observer, B);
2796
2797   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2798             Helper.libcall(*MIBMa32, DummyLocObserver));
2799   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2800             Helper.libcall(*MIBMa64, DummyLocObserver));
2801   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2802             Helper.libcall(*MIBMa128, DummyLocObserver));
2803
2804   const auto *CheckStr = R"(
2805   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2806   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2807   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2808   CHECK: $s0 = COPY [[TRUNC]]
2809   CHECK: BL &fmaf
2810   CHECK: $d0 = COPY [[COPY]]
2811   CHECK: BL &fma
2812   CHECK: $q0 = COPY [[ANYEXT]]
2813   CHECK: BL &fmal
2814   )";
2815
2816   // Check
2817   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2818 }
2819
2820 TEST_F(AArch64GISelMITest, LibcallFCeil) {
2821   setUp();
2822   if (!TM)
2823     GTEST_SKIP();
2824
2825   // Declare your legalization info
2826   DefineLegalizerInfo(A, {
2827     getActionDefinitionsBuilder(G_FCEIL).libcallFor({s32, s64, s128});
2828   });
2829
2830   LLT S32{LLT::scalar(32)};
2831   LLT S64{LLT::scalar(64)};
2832   LLT S128{LLT::scalar(128)};
2833   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2834   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2835
2836   auto MIBCeil32 = B.buildInstr(TargetOpcode::G_FCEIL, {S32}, {MIBTrunc});
2837   auto MIBCeil64 = B.buildInstr(TargetOpcode::G_FCEIL, {S64}, {Copies[0]});
2838   auto MIBCeil128 = B.buildInstr(TargetOpcode::G_FCEIL, {S128}, {MIBExt});
2839
2840   AInfo Info(MF->getSubtarget());
2841   DummyGISelObserver Observer;
2842   LegalizerHelper Helper(*MF, Info, Observer, B);
2843   LostDebugLocObserver DummyLocObserver("");
2844
2845   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2846             Helper.libcall(*MIBCeil32, DummyLocObserver));
2847   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2848             Helper.libcall(*MIBCeil64, DummyLocObserver));
2849   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2850             Helper.libcall(*MIBCeil128, DummyLocObserver));
2851
2852   const auto *CheckStr = R"(
2853   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2854   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2855   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2856   CHECK: $s0 = COPY [[TRUNC]]
2857   CHECK: BL &ceilf
2858   CHECK: $d0 = COPY [[COPY]]
2859   CHECK: BL &ceil
2860   CHECK: $q0 = COPY [[ANYEXT]]
2861   CHECK: BL &ceill
2862   )";
2863
2864   // Check
2865   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2866 }
2867
2868 TEST_F(AArch64GISelMITest, LibcallFFloor) {
2869   setUp();
2870   if (!TM)
2871     GTEST_SKIP();
2872
2873   // Declare your legalization info
2874   DefineLegalizerInfo(A, {
2875     getActionDefinitionsBuilder(G_FFLOOR).libcallFor({s32, s64, s128});
2876   });
2877
2878   LLT S32{LLT::scalar(32)};
2879   LLT S64{LLT::scalar(64)};
2880   LLT S128{LLT::scalar(128)};
2881   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2882   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2883
2884   auto MIBFloor32 = B.buildInstr(TargetOpcode::G_FFLOOR, {S32}, {MIBTrunc});
2885   auto MIBFloor64 = B.buildInstr(TargetOpcode::G_FFLOOR, {S64}, {Copies[0]});
2886   auto MIBFloor128 = B.buildInstr(TargetOpcode::G_FFLOOR, {S128}, {MIBExt});
2887
2888   AInfo Info(MF->getSubtarget());
2889   DummyGISelObserver Observer;
2890   LegalizerHelper Helper(*MF, Info, Observer, B);
2891   LostDebugLocObserver DummyLocObserver("");
2892
2893   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2894             Helper.libcall(*MIBFloor32, DummyLocObserver));
2895   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2896             Helper.libcall(*MIBFloor64, DummyLocObserver));
2897   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2898             Helper.libcall(*MIBFloor128, DummyLocObserver));
2899
2900   const auto *CheckStr = R"(
2901   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2902   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2903   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2904   CHECK: $s0 = COPY [[TRUNC]]
2905   CHECK: BL &floorf
2906   CHECK: $d0 = COPY [[COPY]]
2907   CHECK: BL &floor
2908   CHECK: $q0 = COPY [[ANYEXT]]
2909   CHECK: BL &floorl
2910   )";
2911
2912   // Check
2913   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2914 }
2915
2916 TEST_F(AArch64GISelMITest, LibcallFMinNum) {
2917   setUp();
2918   if (!TM)
2919     GTEST_SKIP();
2920
2921   // Declare your legalization info
2922   DefineLegalizerInfo(A, {
2923     getActionDefinitionsBuilder(G_FMINNUM).libcallFor({s32, s64, s128});
2924   });
2925
2926   LLT S32{LLT::scalar(32)};
2927   LLT S64{LLT::scalar(64)};
2928   LLT S128{LLT::scalar(128)};
2929   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2930   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2931
2932   auto MIBMin32 = B.buildFMinNum(S32, MIBTrunc, MIBTrunc);
2933   auto MIBMin64 = B.buildFMinNum(S64, Copies[0], Copies[0]);
2934   auto MIBMin128 = B.buildFMinNum(S128, MIBExt, MIBExt);
2935
2936   AInfo Info(MF->getSubtarget());
2937   DummyGISelObserver Observer;
2938   LegalizerHelper Helper(*MF, Info, Observer, B);
2939   LostDebugLocObserver DummyLocObserver("");
2940
2941   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2942             Helper.libcall(*MIBMin32, DummyLocObserver));
2943   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2944             Helper.libcall(*MIBMin64, DummyLocObserver));
2945   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2946             Helper.libcall(*MIBMin128, DummyLocObserver));
2947
2948   const auto *CheckStr = R"(
2949   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
2950   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
2951   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
2952   CHECK: $s0 = COPY [[TRUNC]]
2953   CHECK: $s1 = COPY [[TRUNC]]
2954   CHECK: BL &fminf
2955   CHECK: $d0 = COPY [[COPY]]
2956   CHECK: $d1 = COPY [[COPY]]
2957   CHECK: BL &fmin
2958   CHECK: $q0 = COPY [[ANYEXT]]
2959   CHECK: $q1 = COPY [[ANYEXT]]
2960   CHECK: BL &fminl
2961   )";
2962
2963   // Check
2964   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
2965 }
2966
2967 TEST_F(AArch64GISelMITest, LibcallFMaxNum) {
2968   setUp();
2969   if (!TM)
2970     GTEST_SKIP();
2971
2972   // Declare your legalization info
2973   DefineLegalizerInfo(A, {
2974     getActionDefinitionsBuilder(G_FMAXNUM).libcallFor({s32, s64, s128});
2975   });
2976
2977   LLT S32{LLT::scalar(32)};
2978   LLT S64{LLT::scalar(64)};
2979   LLT S128{LLT::scalar(128)};
2980   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
2981   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
2982
2983   auto MIBMax32 = B.buildFMaxNum(S32, MIBTrunc, MIBTrunc);
2984   auto MIBMax64 = B.buildFMaxNum(S64, Copies[0], Copies[0]);
2985   auto MIBMax128 = B.buildFMaxNum(S128, MIBExt, MIBExt);
2986
2987   AInfo Info(MF->getSubtarget());
2988   DummyGISelObserver Observer;
2989   LegalizerHelper Helper(*MF, Info, Observer, B);
2990   LostDebugLocObserver DummyLocObserver("");
2991
2992   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2993             Helper.libcall(*MIBMax32, DummyLocObserver));
2994   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2995             Helper.libcall(*MIBMax64, DummyLocObserver));
2996   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
2997             Helper.libcall(*MIBMax128, DummyLocObserver));
2998
2999   const auto *CheckStr = R"(
3000   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3001   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
3002   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
3003   CHECK: $s0 = COPY [[TRUNC]]
3004   CHECK: $s1 = COPY [[TRUNC]]
3005   CHECK: BL &fmaxf
3006   CHECK: $d0 = COPY [[COPY]]
3007   CHECK: $d1 = COPY [[COPY]]
3008   CHECK: BL &fmax
3009   CHECK: $q0 = COPY [[ANYEXT]]
3010   CHECK: $q1 = COPY [[ANYEXT]]
3011   CHECK: BL &fmaxl
3012   )";
3013
3014   // Check
3015   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3016 }
3017
3018 TEST_F(AArch64GISelMITest, LibcallFSqrt) {
3019   setUp();
3020   if (!TM)
3021     GTEST_SKIP();
3022
3023   // Declare your legalization info
3024   DefineLegalizerInfo(A, {
3025     getActionDefinitionsBuilder(G_FSQRT).libcallFor({s32, s64, s128});
3026   });
3027
3028   LLT S32{LLT::scalar(32)};
3029   LLT S64{LLT::scalar(64)};
3030   LLT S128{LLT::scalar(128)};
3031   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
3032   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
3033
3034   auto MIBSqrt32 = B.buildInstr(TargetOpcode::G_FSQRT, {S32}, {MIBTrunc});
3035   auto MIBSqrt64 = B.buildInstr(TargetOpcode::G_FSQRT, {S64}, {Copies[0]});
3036   auto MIBSqrt128 = B.buildInstr(TargetOpcode::G_FSQRT, {S128}, {MIBExt});
3037
3038   AInfo Info(MF->getSubtarget());
3039   DummyGISelObserver Observer;
3040   LegalizerHelper Helper(*MF, Info, Observer, B);
3041   LostDebugLocObserver DummyLocObserver("");
3042
3043   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3044             Helper.libcall(*MIBSqrt32, DummyLocObserver));
3045   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3046             Helper.libcall(*MIBSqrt64, DummyLocObserver));
3047   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3048             Helper.libcall(*MIBSqrt128, DummyLocObserver));
3049
3050   const auto *CheckStr = R"(
3051   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3052   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
3053   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
3054   CHECK: $s0 = COPY [[TRUNC]]
3055   CHECK: BL &sqrtf
3056   CHECK: $d0 = COPY [[COPY]]
3057   CHECK: BL &sqrt
3058   CHECK: $q0 = COPY [[ANYEXT]]
3059   CHECK: BL &sqrtl
3060   )";
3061
3062   // Check
3063   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3064 }
3065
3066 TEST_F(AArch64GISelMITest, LibcallFRint) {
3067   setUp();
3068   if (!TM)
3069     GTEST_SKIP();
3070
3071   // Declare your legalization info
3072   DefineLegalizerInfo(A, {
3073     getActionDefinitionsBuilder(G_FRINT).libcallFor({s32, s64, s128});
3074   });
3075
3076   LLT S32{LLT::scalar(32)};
3077   LLT S64{LLT::scalar(64)};
3078   LLT S128{LLT::scalar(128)};
3079   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
3080   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
3081
3082   auto MIBRint32 = B.buildInstr(TargetOpcode::G_FRINT, {S32}, {MIBTrunc});
3083   auto MIBRint64 = B.buildInstr(TargetOpcode::G_FRINT, {S64}, {Copies[0]});
3084   auto MIBRint128 = B.buildInstr(TargetOpcode::G_FRINT, {S128}, {MIBExt});
3085
3086   AInfo Info(MF->getSubtarget());
3087   DummyGISelObserver Observer;
3088   LegalizerHelper Helper(*MF, Info, Observer, B);
3089   LostDebugLocObserver DummyLocObserver("");
3090
3091   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3092             Helper.libcall(*MIBRint32, DummyLocObserver));
3093   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3094             Helper.libcall(*MIBRint64, DummyLocObserver));
3095   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3096             Helper.libcall(*MIBRint128, DummyLocObserver));
3097
3098   const auto *CheckStr = R"(
3099   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3100   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
3101   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
3102   CHECK: $s0 = COPY [[TRUNC]]
3103   CHECK: BL &rintf
3104   CHECK: $d0 = COPY [[COPY]]
3105   CHECK: BL &rint
3106   CHECK: $q0 = COPY [[ANYEXT]]
3107   CHECK: BL &rintl
3108   )";
3109
3110   // Check
3111   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3112 }
3113
3114 TEST_F(AArch64GISelMITest, LibcallFNearbyInt) {
3115   setUp();
3116   if (!TM)
3117     GTEST_SKIP();
3118
3119   // Declare your legalization info
3120   DefineLegalizerInfo(A, {
3121     getActionDefinitionsBuilder(G_FNEARBYINT).libcallFor({s32, s64, s128});
3122   });
3123
3124   LLT S32{LLT::scalar(32)};
3125   LLT S64{LLT::scalar(64)};
3126   LLT S128{LLT::scalar(128)};
3127   auto MIBTrunc = B.buildTrunc(S32, Copies[0]);
3128   auto MIBExt = B.buildAnyExt(S128, Copies[0]);
3129
3130   auto MIBNearbyInt32 =
3131       B.buildInstr(TargetOpcode::G_FNEARBYINT, {S32}, {MIBTrunc});
3132   auto MIBNearbyInt64 =
3133       B.buildInstr(TargetOpcode::G_FNEARBYINT, {S64}, {Copies[0]});
3134   auto MIBNearbyInt128 =
3135       B.buildInstr(TargetOpcode::G_FNEARBYINT, {S128}, {MIBExt});
3136
3137   AInfo Info(MF->getSubtarget());
3138   DummyGISelObserver Observer;
3139   LegalizerHelper Helper(*MF, Info, Observer, B);
3140   LostDebugLocObserver DummyLocObserver("");
3141
3142   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3143             Helper.libcall(*MIBNearbyInt32, DummyLocObserver));
3144   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3145             Helper.libcall(*MIBNearbyInt64, DummyLocObserver));
3146   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3147             Helper.libcall(*MIBNearbyInt128, DummyLocObserver));
3148
3149   const auto *CheckStr = R"(
3150   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3151   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC
3152   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT
3153   CHECK: $s0 = COPY [[TRUNC]]
3154   CHECK: BL &nearbyintf
3155   CHECK: $d0 = COPY [[COPY]]
3156   CHECK: BL &nearbyint
3157   CHECK: $q0 = COPY [[ANYEXT]]
3158   CHECK: BL &nearbyintl
3159   )";
3160
3161   // Check
3162   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3163 }
3164
3165 TEST_F(AArch64GISelMITest, NarrowScalarExtract) {
3166   setUp();
3167   if (!TM)
3168     GTEST_SKIP();
3169
3170   // Declare your legalization info
3171   DefineLegalizerInfo(A, {
3172     getActionDefinitionsBuilder(G_UNMERGE_VALUES).legalFor({{s32, s64}});
3173     getActionDefinitionsBuilder(G_EXTRACT).legalForTypeWithAnyImm({{s16, s32}});
3174   });
3175
3176   LLT S16{LLT::scalar(16)};
3177   LLT S32{LLT::scalar(32)};
3178
3179   auto MIBExtractS32 = B.buildExtract(S32, Copies[1], 32);
3180   auto MIBExtractS16 = B.buildExtract(S16, Copies[1], 0);
3181
3182   AInfo Info(MF->getSubtarget());
3183   DummyGISelObserver Observer;
3184   LegalizerHelper Helper(*MF, Info, Observer, B);
3185
3186   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3187             Helper.narrowScalar(*MIBExtractS32, 1, S32));
3188
3189   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3190             Helper.narrowScalar(*MIBExtractS16, 1, S32));
3191
3192   const auto *CheckStr = R"(
3193   CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES
3194   CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[UV1]]
3195   CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES
3196   CHECK: [[EXTR:%[0-9]+]]:_(s16) = G_EXTRACT [[UV3]]:_(s32), 0
3197   CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY [[EXTR]]
3198   )";
3199
3200   // Check
3201   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3202 }
3203
3204 TEST_F(AArch64GISelMITest, LowerInsert) {
3205   setUp();
3206   if (!TM)
3207     GTEST_SKIP();
3208
3209   // Declare your legalization info
3210   DefineLegalizerInfo(A, { getActionDefinitionsBuilder(G_INSERT).lower(); });
3211
3212   LLT S32{LLT::scalar(32)};
3213   LLT S64{LLT::scalar(64)};
3214   LLT P0{LLT::pointer(0, 64)};
3215   LLT P1{LLT::pointer(1, 32)};
3216   LLT V2S32{LLT::fixed_vector(2, 32)};
3217
3218   auto TruncS32 = B.buildTrunc(S32, Copies[0]);
3219   auto IntToPtrP0 = B.buildIntToPtr(P0, Copies[0]);
3220   auto IntToPtrP1 = B.buildIntToPtr(P1, TruncS32);
3221   auto BitcastV2S32 = B.buildBitcast(V2S32, Copies[0]);
3222
3223   auto InsertS64S32 = B.buildInsert(S64, Copies[0], TruncS32, 0);
3224   auto InsertS64P1 = B.buildInsert(S64, Copies[0], IntToPtrP1, 8);
3225   auto InsertP0S32 = B.buildInsert(P0, IntToPtrP0, TruncS32, 16);
3226   auto InsertP0P1 = B.buildInsert(P0, IntToPtrP0, IntToPtrP1, 4);
3227   auto InsertV2S32S32 = B.buildInsert(V2S32, BitcastV2S32, TruncS32, 32);
3228   auto InsertV2S32P1 = B.buildInsert(V2S32, BitcastV2S32, IntToPtrP1, 0);
3229
3230   AInfo Info(MF->getSubtarget());
3231   DummyGISelObserver Observer;
3232   LegalizerHelper Helper(*MF, Info, Observer, B);
3233
3234   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3235             Helper.lower(*InsertS64S32, 0, LLT{}));
3236
3237   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3238             Helper.lower(*InsertS64P1, 0, LLT{}));
3239
3240   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3241             Helper.lower(*InsertP0S32, 0, LLT{}));
3242
3243   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3244             Helper.lower(*InsertP0P1, 0, LLT{}));
3245
3246   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3247             Helper.lower(*InsertV2S32S32, 0, LLT{}));
3248
3249   EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize,
3250             Helper.lower(*InsertV2S32P1, 0, LLT{}));
3251
3252   const auto *CheckStr = R"(
3253   CHECK: [[S64:%[0-9]+]]:_(s64) = COPY
3254   CHECK: [[S32:%[0-9]+]]:_(s32) = G_TRUNC [[S64]]
3255   CHECK: [[P0:%[0-9]+]]:_(p0) = G_INTTOPTR [[S64]]
3256   CHECK: [[P1:%[0-9]+]]:_(p1) = G_INTTOPTR [[S32]]
3257   CHECK: [[V2S32:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[S64]]
3258   CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[S32]]
3259   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3260   CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[S64]]:_, [[C]]:_
3261   CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]]:_, [[ZEXT]]:_
3262
3263   CHECK: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[P1]]
3264   CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[PTRTOINT]]
3265   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3266   CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ZEXT]]:_, [[C]]:_(s64)
3267   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3268   CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[S64]]:_, [[C]]:_
3269   CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]]:_, [[SHL]]:_
3270
3271   CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[P0]]
3272   CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[S32]]
3273   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3274   CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ZEXT]]:_, [[C]]:_(s64)
3275   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3276   CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[PTRTOINT]]:_, [[C]]:_
3277   CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]]:_, [[SHL]]:_
3278   CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[OR]]
3279
3280   CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[P0]]
3281   CHECK: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[P1]]
3282   CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[PTRTOINT1]]
3283   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3284   CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ZEXT]]:_, [[C]]:_(s64)
3285   CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT
3286   CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[PTRTOINT]]:_, [[C]]:_
3287   CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]]:_, [[SHL]]:_
3288   CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[OR]]
3289
3290   CHECK: [[V2S32_E0:%[0-9]+]]:_(s32), [[V2S32_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[V2S32]]
3291   CHECK: [[BV:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[V2S32_E0]]:_(s32), [[S32]]:_(s32)
3292   )";
3293
3294   // Check
3295   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3296 }
3297
3298 // Test lowering of G_FFLOOR
3299 TEST_F(AArch64GISelMITest, LowerFFloor) {
3300   setUp();
3301   if (!TM)
3302     GTEST_SKIP();
3303
3304   // Declare your legalization info
3305   DefineLegalizerInfo(A, {});
3306   // Build Instr
3307   auto Floor = B.buildFFloor(LLT::scalar(64), Copies[0], MachineInstr::MIFlag::FmNoInfs);
3308   AInfo Info(MF->getSubtarget());
3309   DummyGISelObserver Observer;
3310   LegalizerHelper Helper(*MF, Info, Observer, B);
3311   // Perform Legalization
3312   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3313             Helper.lower(*Floor, 0, LLT()));
3314
3315   auto CheckStr = R"(
3316   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3317   CHECK: [[TRUNC:%[0-9]+]]:_(s64) = ninf G_INTRINSIC_TRUNC [[COPY]]
3318   CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
3319   CHECK: [[CMP0:%[0-9]+]]:_(s1) = ninf G_FCMP floatpred(olt), [[COPY]]:_(s64), [[ZERO]]:_
3320   CHECK: [[CMP1:%[0-9]+]]:_(s1) = ninf G_FCMP floatpred(one), [[COPY]]:_(s64), [[TRUNC]]:_
3321   CHECK: [[AND:%[0-9]+]]:_(s1) = G_AND [[CMP0]]:_, [[CMP1]]:_
3322   CHECK: [[ITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[AND]]
3323   = ninf G_FADD [[TRUNC]]:_, [[ITOFP]]:_
3324   )";
3325
3326   // Check
3327   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3328 }
3329
3330 // Test lowering of G_BSWAP
3331 TEST_F(AArch64GISelMITest, LowerBSWAP) {
3332   setUp();
3333   if (!TM)
3334     GTEST_SKIP();
3335
3336   DefineLegalizerInfo(A, {});
3337
3338   // Make sure vector lowering doesn't assert.
3339   auto Cast = B.buildBitcast(LLT::fixed_vector(2, 32), Copies[0]);
3340   auto BSwap = B.buildBSwap(LLT::fixed_vector(2, 32), Cast);
3341   AInfo Info(MF->getSubtarget());
3342   DummyGISelObserver Observer;
3343   LegalizerHelper Helper(*MF, Info, Observer, B);
3344   // Perform Legalization
3345   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3346             Helper.lower(*BSwap, 0, LLT()));
3347
3348   auto CheckStr = R"(
3349   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3350   CHECK: [[VEC:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]]
3351   CHECK: [[K24:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
3352   CHECK: [[SPLAT24:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[K24]]:_(s32), [[K24]]:_(s32)
3353   CHECK: [[SHL0:%[0-9]+]]:_(<2 x s32>) = G_SHL [[VEC]]:_, [[SPLAT24]]
3354   CHECK: [[SHR0:%[0-9]+]]:_(<2 x s32>) = G_LSHR [[VEC]]:_, [[SPLAT24]]
3355   CHECK: [[OR0:%[0-9]+]]:_(<2 x s32>) = G_OR [[SHR0]]:_, [[SHL0]]:_
3356   CHECK: [[KMASK:%[0-9]+]]:_(s32) = G_CONSTANT i32 65280
3357   CHECK: [[SPLATMASK:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[KMASK]]:_(s32), [[KMASK]]:_(s32)
3358   CHECK: [[K8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
3359   CHECK: [[SPLAT8:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[K8]]:_(s32), [[K8]]:_(s32)
3360   CHECK: [[AND0:%[0-9]+]]:_(<2 x s32>) = G_AND [[VEC]]:_, [[SPLATMASK]]:_
3361   CHECK: [[SHL1:%[0-9]+]]:_(<2 x s32>) = G_SHL [[AND0]]:_, [[SPLAT8]]
3362   CHECK: [[OR1:%[0-9]+]]:_(<2 x s32>) = G_OR [[OR0]]:_, [[SHL1]]:_
3363   CHECK: [[SHR1:%[0-9]+]]:_(<2 x s32>) = G_LSHR [[VEC]]:_, [[SPLAT8]]
3364   CHECK: [[AND1:%[0-9]+]]:_(<2 x s32>) = G_AND [[SHR1]]:_, [[SPLATMASK]]:_
3365   CHECK: [[BSWAP:%[0-9]+]]:_(<2 x s32>) = G_OR [[OR1]]:_, [[AND1]]:_
3366   )";
3367
3368   // Check
3369   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3370 }
3371
3372 // Test lowering of G_SDIVREM into G_SDIV and G_SREM
3373 TEST_F(AArch64GISelMITest, LowerSDIVREM) {
3374   setUp();
3375   if (!TM)
3376     GTEST_SKIP();
3377
3378   // Declare your legalization info
3379   DefineLegalizerInfo(
3380       A, { getActionDefinitionsBuilder(G_SDIVREM).lowerFor({s64}); });
3381
3382   LLT S64{LLT::scalar(64)};
3383
3384   // Build Instr
3385   auto SDivrem =
3386       B.buildInstr(TargetOpcode::G_SDIVREM, {S64, S64}, {Copies[0], Copies[1]});
3387   AInfo Info(MF->getSubtarget());
3388   DummyGISelObserver Observer;
3389   LegalizerHelper Helper(*MF, Info, Observer, B);
3390   // Perform Legalization
3391   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3392             Helper.lower(*SDivrem, 0, S64));
3393
3394   const auto *CheckStr = R"(
3395   CHECK: [[DIV:%[0-9]+]]:_(s64) = G_SDIV %0:_, %1:_
3396   CHECK: [[REM:%[0-9]+]]:_(s64) = G_SREM %0:_, %1:_
3397   )";
3398
3399   // Check
3400   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3401 }
3402
3403 // Test lowering of G_UDIVREM into G_UDIV and G_UREM
3404 TEST_F(AArch64GISelMITest, LowerUDIVREM) {
3405   setUp();
3406   if (!TM)
3407     GTEST_SKIP();
3408
3409   // Declare your legalization info
3410   DefineLegalizerInfo(
3411       A, { getActionDefinitionsBuilder(G_UDIVREM).lowerFor({s64}); });
3412
3413   LLT S64{LLT::scalar(64)};
3414
3415   // Build Instr
3416   auto UDivrem =
3417       B.buildInstr(TargetOpcode::G_UDIVREM, {S64, S64}, {Copies[0], Copies[1]});
3418   AInfo Info(MF->getSubtarget());
3419   DummyGISelObserver Observer;
3420   LegalizerHelper Helper(*MF, Info, Observer, B);
3421   // Perform Legalization
3422   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3423             Helper.lower(*UDivrem, 0, S64));
3424
3425   const auto *CheckStr = R"(
3426   CHECK: [[DIV:%[0-9]+]]:_(s64) = G_UDIV %0:_, %1:_
3427   CHECK: [[REM:%[0-9]+]]:_(s64) = G_UREM %0:_, %1:_
3428   )";
3429
3430   // Check
3431   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3432 }
3433
3434 // Test G_SELECT lowering.
3435 // Note: This is for testing the legalizer, aarch64 does not lower scalar
3436 // selects like this.
3437 TEST_F(AArch64GISelMITest, LowerSelect) {
3438   setUp();
3439   if (!TM)
3440     GTEST_SKIP();
3441
3442   // Declare your legalization info
3443   DefineLegalizerInfo(A, { getActionDefinitionsBuilder(G_SELECT).lower(); });
3444
3445   LLT S1 = LLT::scalar(1);
3446   LLT S32 = LLT::scalar(32);
3447   auto Tst = B.buildTrunc(S1, Copies[0]);
3448   auto SrcA = B.buildTrunc(S32, Copies[1]);
3449   auto SrcB = B.buildTrunc(S32, Copies[2]);
3450   auto SELECT = B.buildInstr(TargetOpcode::G_SELECT, {S32}, {Tst, SrcA, SrcB});
3451
3452   AInfo Info(MF->getSubtarget());
3453   DummyGISelObserver Observer;
3454   LegalizerHelper Helper(*MF, Info, Observer, B);
3455   // Perform Legalization
3456   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3457             Helper.lower(*SELECT, 0, S32));
3458
3459   auto CheckStr = R"(
3460   CHECK: [[TST:%[0-9]+]]:_(s1) = G_TRUNC
3461   CHECK: [[TRUE:%[0-9]+]]:_(s32) = G_TRUNC
3462   CHECK: [[FALSE:%[0-9]+]]:_(s32) = G_TRUNC
3463   CHECK: [[MSK:%[0-9]+]]:_(s32) = G_SEXT [[TST]]
3464   CHECK: [[M:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
3465   CHECK: [[NEGMSK:%[0-9]+]]:_(s32) = G_XOR [[MSK]]:_, [[M]]:_
3466   CHECK: [[TVAL:%[0-9]+]]:_(s32) = G_AND [[TRUE]]:_, [[MSK]]:_
3467   CHECK: [[FVAL:%[0-9]+]]:_(s32) = G_AND [[FALSE]]:_, [[NEGMSK]]:_
3468   CHECK: [[RES:%[0-9]+]]:_(s32) = G_OR [[TVAL]]:_, [[FVAL]]:_
3469   )";
3470
3471   // Check
3472   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3473 }
3474
3475 // Test widening of G_UNMERGE_VALUES
3476 TEST_F(AArch64GISelMITest, WidenUnmerge) {
3477   setUp();
3478   if (!TM)
3479     GTEST_SKIP();
3480
3481   DefineLegalizerInfo(A, {});
3482
3483   // Check that widening G_UNMERGE_VALUES to a larger type than the source type
3484   // works as expected
3485   LLT P0{LLT::pointer(0, 64)};
3486   LLT S32{LLT::scalar(32)};
3487   LLT S96{LLT::scalar(96)};
3488
3489   auto IntToPtr = B.buildIntToPtr(P0, Copies[0]);
3490   auto UnmergePtr = B.buildUnmerge(S32, IntToPtr);
3491   auto UnmergeScalar = B.buildUnmerge(S32, Copies[0]);
3492
3493   AInfo Info(MF->getSubtarget());
3494   DummyGISelObserver Observer;
3495   LegalizerHelper Helper(*MF, Info, Observer, B);
3496
3497   // Perform Legalization
3498   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3499             Helper.widenScalar(*UnmergePtr, 0, S96));
3500
3501   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3502             Helper.widenScalar(*UnmergeScalar, 0, S96));
3503
3504   const auto *CheckStr = R"(
3505   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3506   CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]]
3507   CHECK: [[INT:%[0-9]+]]:_(s64) = G_PTRTOINT [[PTR]]
3508   CHECK: [[ANYEXT:%[0-9]+]]:_(s96) = G_ANYEXT [[INT]]
3509   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ANYEXT]]
3510   CHECK: [[C:%[0-9]+]]:_(s96) = G_CONSTANT i96 32
3511   CHECK: [[LSHR:%[0-9]+]]:_(s96) = G_LSHR [[ANYEXT]]:_, [[C]]
3512   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]]
3513   CHECK: [[ANYEXT:%[0-9]+]]:_(s96) = G_ANYEXT [[COPY]]
3514   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ANYEXT]]
3515   CHECK: [[C:%[0-9]+]]:_(s96) = G_CONSTANT i96 32
3516   CHECK: [[LSHR:%[0-9]+]]:_(s96) = G_LSHR [[ANYEXT]]:_, [[C]]
3517   CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]]
3518   )";
3519
3520   // Check
3521   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3522 }
3523
3524 TEST_F(AArch64GISelMITest, BitcastLoad) {
3525   setUp();
3526   if (!TM)
3527     GTEST_SKIP();
3528
3529   LLT P0 = LLT::pointer(0, 64);
3530   LLT S32 = LLT::scalar(32);
3531   LLT V4S8 = LLT::fixed_vector(4, 8);
3532   auto Ptr = B.buildUndef(P0);
3533
3534   DefineLegalizerInfo(A, {});
3535
3536   MachineMemOperand *MMO = B.getMF().getMachineMemOperand(
3537       MachinePointerInfo(), MachineMemOperand::MOLoad, 4, Align(4));
3538   auto Load = B.buildLoad(V4S8, Ptr, *MMO);
3539
3540   AInfo Info(MF->getSubtarget());
3541   DummyGISelObserver Observer;
3542   B.setInsertPt(*EntryMBB, Load->getIterator());
3543   LegalizerHelper Helper(*MF, Info, Observer, B);
3544   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3545             Helper.bitcast(*Load, 0, S32));
3546
3547   auto CheckStr = R"(
3548   CHECK: [[PTR:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
3549   CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD
3550   CHECK: [[CAST:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[LOAD]]
3551
3552   )";
3553
3554   // Check
3555   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3556 }
3557
3558 TEST_F(AArch64GISelMITest, BitcastStore) {
3559   setUp();
3560   if (!TM)
3561     GTEST_SKIP();
3562
3563   LLT P0 = LLT::pointer(0, 64);
3564   LLT S32 = LLT::scalar(32);
3565   LLT V4S8 = LLT::fixed_vector(4, 8);
3566   auto Ptr = B.buildUndef(P0);
3567
3568   DefineLegalizerInfo(A, {});
3569
3570   MachineMemOperand *MMO = B.getMF().getMachineMemOperand(
3571       MachinePointerInfo(), MachineMemOperand::MOStore, 4, Align(4));
3572   auto Val = B.buildUndef(V4S8);
3573   auto Store = B.buildStore(Val, Ptr, *MMO);
3574
3575   AInfo Info(MF->getSubtarget());
3576   DummyGISelObserver Observer;
3577   LegalizerHelper Helper(*MF, Info, Observer, B);
3578   B.setInsertPt(*EntryMBB, Store->getIterator());
3579   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3580             Helper.bitcast(*Store, 0, S32));
3581
3582   auto CheckStr = R"(
3583   CHECK: [[VAL:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
3584   CHECK: [[CAST:%[0-9]+]]:_(s32) = G_BITCAST [[VAL]]
3585   CHECK: G_STORE [[CAST]]
3586   )";
3587
3588   // Check
3589   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3590 }
3591
3592 TEST_F(AArch64GISelMITest, BitcastSelect) {
3593   setUp();
3594   if (!TM)
3595     GTEST_SKIP();
3596
3597   LLT S1 = LLT::scalar(1);
3598   LLT S32 = LLT::scalar(32);
3599   LLT V4S8 = LLT::fixed_vector(4, 8);
3600
3601   DefineLegalizerInfo(A, {});
3602
3603   auto Cond = B.buildUndef(S1);
3604   auto Val0 = B.buildConstant(V4S8, 123);
3605   auto Val1 = B.buildConstant(V4S8, 99);
3606
3607   auto Select = B.buildSelect(V4S8, Cond, Val0, Val1);
3608
3609   AInfo Info(MF->getSubtarget());
3610   DummyGISelObserver Observer;
3611   LegalizerHelper Helper(*MF, Info, Observer, B);
3612   B.setInsertPt(*EntryMBB, Select->getIterator());
3613   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3614             Helper.bitcast(*Select, 0, S32));
3615
3616   auto CheckStr = R"(
3617   CHECK: [[VAL0:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR
3618   CHECK: [[VAL1:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR
3619   CHECK: [[CAST0:%[0-9]+]]:_(s32) = G_BITCAST [[VAL0]]
3620   CHECK: [[CAST1:%[0-9]+]]:_(s32) = G_BITCAST [[VAL1]]
3621   CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT %{{[0-9]+}}:_(s1), [[CAST0]]:_, [[CAST1]]:_
3622   CHECK: [[CAST2:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[SELECT]]
3623   )";
3624
3625   // Check
3626   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3627
3628   // Doesn't make sense
3629   auto VCond = B.buildUndef(LLT::fixed_vector(4, 1));
3630   auto VSelect = B.buildSelect(V4S8, VCond, Val0, Val1);
3631
3632   B.setInsertPt(*EntryMBB, VSelect->getIterator());
3633   EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize,
3634             Helper.bitcast(*VSelect, 0, S32));
3635   EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize,
3636             Helper.bitcast(*VSelect, 1, LLT::scalar(4)));
3637 }
3638
3639 TEST_F(AArch64GISelMITest, BitcastBitOps) {
3640   setUp();
3641   if (!TM)
3642     GTEST_SKIP();
3643
3644   LLT S32 = LLT::scalar(32);
3645   LLT V4S8 = LLT::fixed_vector(4, 8);
3646
3647   DefineLegalizerInfo(A, {});
3648
3649   auto Val0 = B.buildConstant(V4S8, 123);
3650   auto Val1 = B.buildConstant(V4S8, 99);
3651   auto And = B.buildAnd(V4S8, Val0, Val1);
3652   auto Or = B.buildOr(V4S8, Val0, Val1);
3653   auto Xor = B.buildXor(V4S8, Val0, Val1);
3654
3655   AInfo Info(MF->getSubtarget());
3656   DummyGISelObserver Observer;
3657   LegalizerHelper Helper(*MF, Info, Observer, B);
3658   B.setInsertPt(*EntryMBB, And->getIterator());
3659   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3660             Helper.bitcast(*And, 0, S32));
3661
3662   B.setInsertPt(*EntryMBB, Or->getIterator());
3663   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3664             Helper.bitcast(*Or, 0, S32));
3665
3666   B.setInsertPt(*EntryMBB, Xor->getIterator());
3667   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3668             Helper.bitcast(*Xor, 0, S32));
3669
3670   auto CheckStr = R"(
3671   CHECK: [[VAL0:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR
3672   CHECK: [[VAL1:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR
3673   CHECK: [[CAST0:%[0-9]+]]:_(s32) = G_BITCAST [[VAL0]]
3674   CHECK: [[CAST1:%[0-9]+]]:_(s32) = G_BITCAST [[VAL1]]
3675   CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[CAST0]]:_, [[CAST1]]:_
3676   CHECK: [[CAST_AND:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[AND]]
3677   CHECK: [[CAST2:%[0-9]+]]:_(s32) = G_BITCAST [[VAL0]]
3678   CHECK: [[CAST3:%[0-9]+]]:_(s32) = G_BITCAST [[VAL1]]
3679   CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[CAST2]]:_, [[CAST3]]:_
3680   CHECK: [[CAST_OR:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[OR]]
3681   CHECK: [[CAST4:%[0-9]+]]:_(s32) = G_BITCAST [[VAL0]]
3682   CHECK: [[CAST5:%[0-9]+]]:_(s32) = G_BITCAST [[VAL1]]
3683   CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[CAST4]]:_, [[CAST5]]:_
3684   CHECK: [[CAST_XOR:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[XOR]]
3685   )";
3686
3687   // Check
3688   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3689 }
3690
3691 TEST_F(AArch64GISelMITest, CreateLibcall) {
3692   setUp();
3693   if (!TM)
3694     GTEST_SKIP();
3695
3696   DefineLegalizerInfo(A, {});
3697
3698   AInfo Info(MF->getSubtarget());
3699   DummyGISelObserver Observer;
3700   LostDebugLocObserver DummyLocObserver("");
3701
3702   LLVMContext &Ctx = MF->getFunction().getContext();
3703   auto *RetTy = Type::getVoidTy(Ctx);
3704
3705   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3706             createLibcall(B, "abort", {{}, RetTy, 0}, {}, CallingConv::C,
3707                           DummyLocObserver, nullptr));
3708
3709   auto CheckStr = R"(
3710   CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
3711   CHECK: BL &abort
3712   CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
3713   )";
3714
3715   // Check
3716   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3717 }
3718
3719 // Test narrowing of G_IMPLICIT_DEF
3720 TEST_F(AArch64GISelMITest, NarrowImplicitDef) {
3721   setUp();
3722   if (!TM)
3723     GTEST_SKIP();
3724
3725   DefineLegalizerInfo(A, {});
3726
3727   // Make sure that G_IMPLICIT_DEF can be narrowed if the original size is not a
3728   // multiple of narrow size
3729   LLT S32{LLT::scalar(32)};
3730   LLT S48{LLT::scalar(48)};
3731   LLT S64{LLT::scalar(64)};
3732   LLT V2S64{{LLT::fixed_vector(2, 64)}};
3733
3734   auto Implicit1 = B.buildUndef(S64);
3735   auto Implicit2 = B.buildUndef(S64);
3736   auto Implicit3 = B.buildUndef(V2S64);
3737   auto Implicit4 = B.buildUndef(V2S64);
3738
3739   AInfo Info(MF->getSubtarget());
3740   DummyGISelObserver Observer;
3741   LegalizerHelper Helper(*MF, Info, Observer, B);
3742
3743   // Perform Legalization
3744
3745   B.setInsertPt(*EntryMBB, Implicit1->getIterator());
3746   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3747             Helper.narrowScalar(*Implicit1, 0, S48));
3748
3749   B.setInsertPt(*EntryMBB, Implicit2->getIterator());
3750   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3751             Helper.narrowScalar(*Implicit2, 0, S32));
3752
3753   B.setInsertPt(*EntryMBB, Implicit3->getIterator());
3754   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3755             Helper.narrowScalar(*Implicit3, 0, S48));
3756
3757   B.setInsertPt(*EntryMBB, Implicit4->getIterator());
3758   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3759             Helper.narrowScalar(*Implicit4, 0, S32));
3760
3761   const auto *CheckStr = R"(
3762   CHECK: [[DEF:%[0-9]+]]:_(s48) = G_IMPLICIT_DEF
3763   CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[DEF]]
3764
3765   CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3766   CHECK: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3767   CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[DEF]]:_(s32), [[DEF1]]
3768
3769   CHECK: [[DEF:%[0-9]+]]:_(<2 x s48>) = G_IMPLICIT_DEF
3770   CHECK: [[ANYEXT:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[DEF]]
3771
3772   CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3773   CHECK: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3774   CHECK: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3775   CHECK: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3776   CHECK: [[BV:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[DEF]]:_(s32), [[DEF1]]:_(s32), [[DEF2]]:_(s32), [[DEF3]]:_(s32)
3777   )";
3778
3779   // Check
3780   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3781 }
3782
3783 // Test widening of G_FREEZE
3784 TEST_F(AArch64GISelMITest, WidenFreeze) {
3785   setUp();
3786   if (!TM)
3787     GTEST_SKIP();
3788
3789   DefineLegalizerInfo(A, {});
3790
3791   // Make sure that G_FREEZE is widened with anyext
3792   LLT S64{LLT::scalar(64)};
3793   LLT S128{LLT::scalar(128)};
3794   LLT V2S32{LLT::fixed_vector(2, 32)};
3795   LLT V2S64{LLT::fixed_vector(2, 64)};
3796
3797   auto Vector = B.buildBitcast(V2S32, Copies[0]);
3798
3799   auto FreezeScalar = B.buildInstr(TargetOpcode::G_FREEZE, {S64}, {Copies[0]});
3800   auto FreezeVector = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector});
3801
3802   AInfo Info(MF->getSubtarget());
3803   DummyGISelObserver Observer;
3804   LegalizerHelper Helper(*MF, Info, Observer, B);
3805
3806   // Perform Legalization
3807
3808   B.setInsertPt(*EntryMBB, FreezeScalar->getIterator());
3809   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3810             Helper.widenScalar(*FreezeScalar, 0, S128));
3811
3812   B.setInsertPt(*EntryMBB, FreezeVector->getIterator());
3813   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3814             Helper.widenScalar(*FreezeVector, 0, V2S64));
3815
3816   const auto *CheckStr = R"(
3817   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3818   CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]]
3819
3820   CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[COPY]]
3821   CHECK: [[FREEZE:%[0-9]+]]:_(s128) = G_FREEZE [[ANYEXT]]
3822   CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[FREEZE]]
3823
3824   CHECK: [[ANYEXT1:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[BITCAST]]
3825   CHECK: [[FREEZE1:%[0-9]+]]:_(<2 x s64>) = G_FREEZE [[ANYEXT1]]
3826   CHECK: [[TRUNC1:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[FREEZE1]]
3827   )";
3828
3829   // Check
3830   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3831 }
3832
3833 // Test narrowing of G_FREEZE
3834 TEST_F(AArch64GISelMITest, NarrowFreeze) {
3835   setUp();
3836   if (!TM)
3837     GTEST_SKIP();
3838
3839   DefineLegalizerInfo(A, {});
3840
3841   // Make sure that G_FREEZE is narrowed using unmerge/extract
3842   LLT S32{LLT::scalar(32)};
3843   LLT S33{LLT::scalar(33)};
3844   LLT S48{LLT::scalar(48)};
3845   LLT S64{LLT::scalar(64)};
3846   LLT V2S16{LLT::fixed_vector(2, 16)};
3847   LLT V3S16{LLT::fixed_vector(3, 16)};
3848   LLT V4S16{LLT::fixed_vector(4, 16)};
3849
3850   auto Trunc = B.buildTrunc(S33, {Copies[0]});
3851   auto Trunc1 = B.buildTrunc(S48, {Copies[0]});
3852   auto Vector = B.buildBitcast(V3S16, Trunc1);
3853
3854   auto FreezeScalar = B.buildInstr(TargetOpcode::G_FREEZE, {S64}, {Copies[0]});
3855   auto FreezeOdd = B.buildInstr(TargetOpcode::G_FREEZE, {S33}, {Trunc});
3856   auto FreezeVector = B.buildInstr(TargetOpcode::G_FREEZE, {V3S16}, {Vector});
3857   auto FreezeVector1 = B.buildInstr(TargetOpcode::G_FREEZE, {V3S16}, {Vector});
3858
3859   AInfo Info(MF->getSubtarget());
3860   DummyGISelObserver Observer;
3861   LegalizerHelper Helper(*MF, Info, Observer, B);
3862
3863   // Perform Legalization
3864
3865   B.setInsertPt(*EntryMBB, FreezeScalar->getIterator());
3866   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3867             Helper.narrowScalar(*FreezeScalar, 0, S32));
3868
3869   // This should be followed by narrowScalar to S32.
3870   B.setInsertPt(*EntryMBB, FreezeOdd->getIterator());
3871   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3872             Helper.widenScalar(*FreezeOdd, 0, S64));
3873
3874   B.setInsertPt(*EntryMBB, FreezeVector->getIterator());
3875   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3876             Helper.fewerElementsVector(*FreezeVector, 0, V2S16));
3877
3878   // This should be followed by fewerElements to V2S16.
3879   B.setInsertPt(*EntryMBB, FreezeVector1->getIterator());
3880   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3881             Helper.moreElementsVector(*FreezeVector1, 0, V4S16));
3882
3883   const auto *CheckStr = R"(
3884   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3885   CHECK: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]]
3886   CHECK: [[TRUNC1:%[0-9]+]]:_(s48) = G_TRUNC [[COPY]]
3887   CHECK: [[BITCAST:%[0-9]+]]:_(<3 x s16>) = G_BITCAST [[TRUNC1]]
3888
3889   CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]]
3890   CHECK: [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[UV]]
3891   CHECK: [[FREEZE1:%[0-9]+]]:_(s32) = G_FREEZE [[UV1]]
3892   CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FREEZE]]:_(s32), [[FREEZE1]]
3893
3894   CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]]
3895   CHECK: [[FREEZE2:%[0-9]+]]:_(s64) = G_FREEZE [[ANYEXT]]
3896   CHECK: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[FREEZE2]]
3897
3898   CHECK: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[BITCAST]]
3899   CHECK: [[BV:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[UV2]]:_(s16), [[UV3]]:_(s16)
3900   CHECK: [[FREEZE3:%[0-9]+]]:_(<2 x s16>) = G_FREEZE [[BV]]
3901   CHECK: [[FREEZE4:%[0-9]+]]:_(s16) = G_FREEZE [[UV4]]
3902   CHECK: [[FREEZE3_E0:%[0-9]+]]:_(s16), [[FREEZE3_E1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[FREEZE3]]
3903   CHECK: [[BV1:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FREEZE3_E0]]:_(s16), [[FREEZE3_E1]]:_(s16), [[FREEZE4]]:_(s16)
3904
3905   CHECK: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[BITCAST]]
3906   CHECK: [[IMP_DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
3907   CHECK: [[BV1:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UV5]]:_(s16), [[UV6]]:_(s16), [[UV7]]:_(s16), [[IMP_DEF]]:_(s16)
3908   CHECK: [[FREEZE5:%[0-9]+]]:_(<4 x s16>) = G_FREEZE [[BV1]]
3909   CHECK: [[FREEZE5_E0:%[0-9]+]]:_(s16), [[FREEZE5_E1:%[0-9]+]]:_(s16), [[FREEZE5_E2:%[0-9]+]]:_(s16), [[FREEZE5_E3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[FREEZE5]]
3910   CHECK: [[BV2:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FREEZE5_E0]]:_(s16), [[FREEZE5_E1]]:_(s16), [[FREEZE5_E2]]:_(s16)
3911   )";
3912
3913   // Check
3914   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3915 }
3916
3917 // Test fewer elements of G_FREEZE
3918 TEST_F(AArch64GISelMITest, FewerElementsFreeze) {
3919   setUp();
3920   if (!TM)
3921     GTEST_SKIP();
3922
3923   DefineLegalizerInfo(A, {});
3924
3925   LLT S32{LLT::scalar(32)};
3926   LLT V2S16{LLT::fixed_vector(2, 16)};
3927   LLT V2S32{LLT::fixed_vector(2, 32)};
3928   LLT V4S16{LLT::fixed_vector(4, 16)};
3929
3930   auto Vector1 = B.buildBitcast(V2S32, Copies[0]);
3931   auto Vector2 = B.buildBitcast(V4S16, Copies[0]);
3932
3933   auto FreezeVector1 = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector1});
3934   auto FreezeVector2 = B.buildInstr(TargetOpcode::G_FREEZE, {V4S16}, {Vector2});
3935
3936   AInfo Info(MF->getSubtarget());
3937   DummyGISelObserver Observer;
3938   LegalizerHelper Helper(*MF, Info, Observer, B);
3939
3940   // Perform Legalization
3941
3942   B.setInsertPt(*EntryMBB, FreezeVector1->getIterator());
3943   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3944             Helper.fewerElementsVector(*FreezeVector1, 0, S32));
3945
3946   B.setInsertPt(*EntryMBB, FreezeVector2->getIterator());
3947   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3948             Helper.fewerElementsVector(*FreezeVector2, 0, V2S16));
3949
3950   const auto *CheckStr = R"(
3951   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3952   CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]]
3953   CHECK: [[BITCAST1:%[0-9]+]]:_(<4 x s16>) = G_BITCAST [[COPY]]
3954
3955   CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]]
3956   CHECK: [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[UV]]
3957   CHECK: [[FREEZE1:%[0-9]+]]:_(s32) = G_FREEZE [[UV1]]
3958   CHECK: [[MV:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FREEZE]]:_(s32), [[FREEZE1]]
3959
3960   CHECK: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BITCAST1]]
3961   CHECK: [[FREEZE2:%[0-9]+]]:_(<2 x s16>) = G_FREEZE [[UV]]
3962   CHECK: [[FREEZE3:%[0-9]+]]:_(<2 x s16>) = G_FREEZE [[UV1]]
3963   CHECK: [[MV:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FREEZE2]]:_(<2 x s16>), [[FREEZE3]]
3964   )";
3965
3966   // Check
3967   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
3968 }
3969
3970 // Test more elements of G_FREEZE
3971 TEST_F(AArch64GISelMITest, MoreElementsFreeze) {
3972   setUp();
3973   if (!TM)
3974     GTEST_SKIP();
3975
3976   DefineLegalizerInfo(A, {});
3977
3978   LLT V2S32{LLT::fixed_vector(2, 32)};
3979   LLT V4S32{LLT::fixed_vector(4, 32)};
3980
3981   auto Vector1 = B.buildBitcast(V2S32, Copies[0]);
3982   auto FreezeVector1 = B.buildInstr(TargetOpcode::G_FREEZE, {V2S32}, {Vector1});
3983
3984   AInfo Info(MF->getSubtarget());
3985   DummyGISelObserver Observer;
3986   LegalizerHelper Helper(*MF, Info, Observer, B);
3987
3988   // Perform Legalization
3989   B.setInsertPt(*EntryMBB, FreezeVector1->getIterator());
3990   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
3991             Helper.moreElementsVector(*FreezeVector1, 0, V4S32));
3992
3993   const auto *CheckStr = R"(
3994   CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY
3995   CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]]
3996   CHECK: [[BITCAST_E0:%[0-9]+]]:_(s32), [[BITCAST_E1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]]:_(<2 x s32>)
3997   CHECK: [[IMP_DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
3998   CHECK: [[BITCAST_LARGE:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[BITCAST_E0]]:_(s32), [[BITCAST_E1]]:_(s32), [[IMP_DEF]]:_(s32), [[IMP_DEF]]:_(s32)
3999   CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[BITCAST_LARGE]]
4000   CHECK: [[FREEZE_E0:%[0-9]+]]:_(s32), [[FREEZE_E1:%[0-9]+]]:_(s32), [[FREEZE_E2:%[0-9]+]]:_(s32), [[FREEZE_E3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[FREEZE]]:_(<4 x s32>)
4001   CHECK: (<2 x s32>) = G_BUILD_VECTOR [[FREEZE_E0]]:_(s32), [[FREEZE_E1]]:_(s32)
4002   )";
4003
4004   // Check
4005   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4006 }
4007
4008 // Test fewer elements of G_INSERT_VECTOR_ELEMENT
4009 TEST_F(AArch64GISelMITest, FewerElementsInsertVectorElt) {
4010   setUp();
4011   if (!TM)
4012     GTEST_SKIP();
4013
4014   DefineLegalizerInfo(A, {});
4015
4016   LLT P0{LLT::pointer(0, 64)};
4017   LLT S64{LLT::scalar(64)};
4018   LLT S16{LLT::scalar(16)};
4019   LLT V2S16{LLT::fixed_vector(2, 16)};
4020   LLT V3S16{LLT::fixed_vector(3, 16)};
4021   LLT V8S16{LLT::fixed_vector(8, 16)};
4022
4023   auto Ptr0 = B.buildIntToPtr(P0, Copies[0]);
4024   auto VectorV8 = B.buildLoad(V8S16, Ptr0, MachinePointerInfo(), Align(8));
4025   auto Value = B.buildTrunc(S16, Copies[1]);
4026
4027   auto Seven = B.buildConstant(S64, 7);
4028   auto InsertV8Constant7_0 =
4029       B.buildInsertVectorElement(V8S16, VectorV8, Value, Seven);
4030   auto InsertV8Constant7_1 =
4031       B.buildInsertVectorElement(V8S16, VectorV8, Value, Seven);
4032
4033   B.buildStore(InsertV8Constant7_0, Ptr0, MachinePointerInfo(), Align(8),
4034                MachineMemOperand::MOVolatile);
4035   B.buildStore(InsertV8Constant7_1, Ptr0, MachinePointerInfo(), Align(8),
4036                MachineMemOperand::MOVolatile);
4037
4038   AInfo Info(MF->getSubtarget());
4039   DummyGISelObserver Observer;
4040   LegalizerHelper Helper(*MF, Info, Observer, B);
4041
4042   // Perform Legalization
4043   B.setInsertPt(*EntryMBB, InsertV8Constant7_0->getIterator());
4044
4045   // This should index the high element of the 4th piece of an unmerge.
4046   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4047             Helper.fewerElementsVector(*InsertV8Constant7_0, 0, V2S16));
4048
4049   // This case requires extracting an intermediate vector type into the target
4050   // v4s16.
4051   B.setInsertPt(*EntryMBB, InsertV8Constant7_1->getIterator());
4052   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4053             Helper.fewerElementsVector(*InsertV8Constant7_1, 0, V3S16));
4054
4055   const auto *CheckStr = R"(
4056   CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY
4057   CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY
4058   CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY
4059   CHECK: [[PTR0:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY0]]
4060   CHECK: [[VEC8:%[0-9]+]]:_(<8 x s16>) = G_LOAD [[PTR0]]:_(p0) :: (load (<8 x s16>), align 8)
4061   CHECK: [[INSERT_VAL:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]]
4062
4063
4064   CHECK: [[UNMERGE0:%[0-9]+]]:_(<2 x s16>), [[UNMERGE1:%[0-9]+]]:_(<2 x s16>), [[UNMERGE2:%[0-9]+]]:_(<2 x s16>), [[UNMERGE3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[VEC8]]
4065   CHECK: [[ONE:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
4066   CHECK: [[SUB_INSERT_7:%[0-9]+]]:_(<2 x s16>) = G_INSERT_VECTOR_ELT [[UNMERGE3]]:_, [[INSERT_VAL]]:_(s16), [[ONE]]
4067   CHECK: [[INSERT_V8_7_0:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[UNMERGE0]]:_(<2 x s16>), [[UNMERGE1]]:_(<2 x s16>), [[UNMERGE2]]:_(<2 x s16>), [[SUB_INSERT_7]]:_(<2 x s16>)
4068
4069
4070   CHECK: [[UNMERGE1_0:%[0-9]+]]:_(s16), [[UNMERGE1_1:%[0-9]+]]:_(s16), [[UNMERGE1_2:%[0-9]+]]:_(s16), [[UNMERGE1_3:%[0-9]+]]:_(s16), [[UNMERGE1_4:%[0-9]+]]:_(s16), [[UNMERGE1_5:%[0-9]+]]:_(s16), [[UNMERGE1_6:%[0-9]+]]:_(s16), [[UNMERGE1_7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[VEC8]]:_(<8 x s16>)
4071   CHECK: [[IMPDEF_S16:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
4072   CHECK: [[BUILD0:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UNMERGE1_0]]:_(s16), [[UNMERGE1_1]]:_(s16), [[UNMERGE1_2]]:_(s16)
4073   CHECK: [[BUILD1:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UNMERGE1_3]]:_(s16), [[UNMERGE1_4]]:_(s16), [[UNMERGE1_5]]:_(s16)
4074   CHECK: [[BUILD2:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UNMERGE1_6]]:_(s16), [[UNMERGE1_7]]:_(s16), [[IMPDEF_S16]]:_(s16)
4075   CHECK: [[IMPDEF_V3S16:%[0-9]+]]:_(<3 x s16>) = G_IMPLICIT_DEF
4076   CHECK: [[ONE_1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
4077   CHECK: [[SUB_INSERT_7_V3S16:%[0-9]+]]:_(<3 x s16>) = G_INSERT_VECTOR_ELT [[BUILD2]]:_, [[INSERT_VAL]]:_(s16), [[ONE_1]]
4078
4079   CHECK: [[WIDE_CONCAT_DEAD:%[0-9]+]]:_(<24 x s16>) = G_CONCAT_VECTORS [[BUILD0]]:_(<3 x s16>), [[BUILD1]]:_(<3 x s16>), [[SUB_INSERT_7_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>)
4080   CHECK: [[WIDE_CONCAT:%[0-9]+]]:_(<24 x s16>) = G_CONCAT_VECTORS [[BUILD0]]:_(<3 x s16>), [[BUILD1]]:_(<3 x s16>), [[SUB_INSERT_7_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>), [[IMPDEF_V3S16]]:_(<3 x s16>)
4081   CHECK: [[INSERT_V8_7_1:%[0-9]+]]:_(<8 x s16>), %{{[0-9]+}}:_(<8 x s16>), %{{[0-9]+}}:_(<8 x s16>) = G_UNMERGE_VALUES [[WIDE_CONCAT]]:_(<24 x s16>)
4082
4083
4084   CHECK: G_STORE [[INSERT_V8_7_0]]
4085   CHECK: G_STORE [[INSERT_V8_7_1]]
4086   )";
4087
4088   // Check
4089   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4090 }
4091
4092 // Test widen scalar of G_UNMERGE_VALUES
4093 TEST_F(AArch64GISelMITest, widenScalarUnmerge) {
4094   setUp();
4095   if (!TM)
4096     GTEST_SKIP();
4097
4098   DefineLegalizerInfo(A, {});
4099
4100   LLT S96{LLT::scalar(96)};
4101   LLT S64{LLT::scalar(64)};
4102   LLT S48{LLT::scalar(48)};
4103
4104   auto Src = B.buildAnyExt(S96, Copies[0]);
4105   auto Unmerge = B.buildUnmerge(S48, Src);
4106
4107   AInfo Info(MF->getSubtarget());
4108   DummyGISelObserver Observer;
4109   LegalizerHelper Helper(*MF, Info, Observer, B);
4110
4111   // Perform Legalization
4112   B.setInsertPt(*EntryMBB, Unmerge->getIterator());
4113
4114   // This should create unmerges to a GCD type (S16), then remerge to S48
4115   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4116             Helper.widenScalar(*Unmerge, 0, S64));
4117
4118   const auto *CheckStr = R"(
4119   CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY
4120   CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY
4121   CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY
4122   CHECK: [[ANYEXT:%[0-9]+]]:_(s96) = G_ANYEXT [[COPY0]]
4123   CHECK: [[ANYEXT1:%[0-9]+]]:_(s192) = G_ANYEXT [[ANYEXT]]
4124   CHECK: [[UNMERGE:%[0-9]+]]:_(s64), [[UNMERGE1:%[0-9]+]]:_(s64), [[UNMERGE2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT1]]
4125   CHECK: [[UNMERGE3:%[0-9]+]]:_(s16), [[UNMERGE4:%[0-9]+]]:_(s16), [[UNMERGE5:%[0-9]+]]:_(s16), [[UNMERGE6:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UNMERGE]]
4126   CHECK: [[UNMERGE7:%[0-9]+]]:_(s16), [[UNMERGE8:%[0-9]+]]:_(s16), [[UNMERGE9:%[0-9]+]]:_(s16), [[UNMERGE10:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UNMERGE1]]
4127   CHECK: [[UNMERGE11:%[0-9]+]]:_(s16), [[UNMERGE12:%[0-9]+]]:_(s16), [[UNMERGE13:%[0-9]+]]:_(s16), [[UNMERGE14:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UNMERGE2]]
4128   CHECK: [[MERGE:%[0-9]+]]:_(s48) = G_MERGE_VALUES [[UNMERGE3]]:_(s16), [[UNMERGE4]]:_(s16), [[UNMERGE5]]:_(s16)
4129   CHECK: [[MERGE1:%[0-9]+]]:_(s48) = G_MERGE_VALUES [[UNMERGE6]]:_(s16), [[UNMERGE7]]:_(s16), [[UNMERGE8]]:_(s16)
4130   )";
4131
4132   // Check
4133   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4134 }
4135
4136 // Test moreElements of G_SHUFFLE_VECTOR.
4137 TEST_F(AArch64GISelMITest, moreElementsShuffle) {
4138   setUp();
4139   if (!TM)
4140     GTEST_SKIP();
4141
4142   DefineLegalizerInfo(A, {});
4143
4144   LLT S64{LLT::scalar(64)};
4145   LLT V6S64 = LLT::fixed_vector(6, S64);
4146
4147   auto V1 = B.buildBuildVector(V6S64, {Copies[0], Copies[1], Copies[0],
4148                                        Copies[1], Copies[0], Copies[1]});
4149   auto V2 = B.buildBuildVector(V6S64, {Copies[0], Copies[1], Copies[0],
4150                                        Copies[1], Copies[0], Copies[1]});
4151   auto Shuffle = B.buildShuffleVector(V6S64, V1, V2, {3, 4, 7, 0, 1, 11});
4152
4153   AInfo Info(MF->getSubtarget());
4154   DummyGISelObserver Observer;
4155   LegalizerHelper Helper(*MF, Info, Observer, B);
4156
4157   // Perform Legalization
4158   B.setInsertPt(*EntryMBB, Shuffle->getIterator());
4159
4160   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4161             Helper.moreElementsVector(*Shuffle, 0, LLT::fixed_vector(8, S64)));
4162
4163   const auto *CheckStr = R"(
4164   CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY
4165   CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY
4166   CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY
4167   CHECK: [[BV1:%[0-9]+]]:_(<6 x s64>) = G_BUILD_VECTOR
4168   CHECK: [[BV2:%[0-9]+]]:_(<6 x s64>) = G_BUILD_VECTOR
4169
4170   CHECK: [[BV1_E0:%[0-9]+]]:_(s64), [[BV1_E1:%[0-9]+]]:_(s64), [[BV1_E2:%[0-9]+]]:_(s64), [[BV1_E3:%[0-9]+]]:_(s64), [[BV1_E4:%[0-9]+]]:_(s64), [[BV1_E5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[BV1]]:_(<6 x s64>)
4171   CHECK: [[IMP_DEF0:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
4172   CHECK: [[BV1_LARGE:%[0-9]+]]:_(<8 x s64>) = G_BUILD_VECTOR [[BV1_E0]]:_(s64), [[BV1_E1]]:_(s64), [[BV1_E2]]:_(s64), [[BV1_E3]]:_(s64), [[BV1_E4]]:_(s64), [[BV1_E5]]:_(s64), [[IMP_DEF0]]:_(s64), [[IMP_DEF0]]:_(s64)
4173
4174   CHECK: [[BV2_E0:%[0-9]+]]:_(s64), [[BV2_E1:%[0-9]+]]:_(s64), [[BV2_E2:%[0-9]+]]:_(s64), [[BV2_E3:%[0-9]+]]:_(s64), [[BV2_E4:%[0-9]+]]:_(s64), [[BV2_E5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[BV2]]:_(<6 x s64>)
4175   CHECK: [[IMP_DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
4176   CHECK: [[BV2_LARGE:%[0-9]+]]:_(<8 x s64>) = G_BUILD_VECTOR [[BV2_E0]]:_(s64), [[BV2_E1]]:_(s64), [[BV2_E2]]:_(s64), [[BV2_E3]]:_(s64), [[BV2_E4]]:_(s64), [[BV2_E5]]:_(s64), [[IMP_DEF1]]:_(s64), [[IMP_DEF1]]:_(s64)
4177
4178   CHECK: [[SHUF:%[0-9]+]]:_(<8 x s64>) = G_SHUFFLE_VECTOR [[BV1_LARGE]]:_(<8 x s64>), [[BV2_LARGE]]:_, shufflemask(3, 4, 9, 0, 1, 13, undef, undef)
4179
4180   CHECK: [[SHUF_E0:%[0-9]+]]:_(s64), [[SHUF_E1:%[0-9]+]]:_(s64), [[SHUF_E2:%[0-9]+]]:_(s64), [[SHUF_E3:%[0-9]+]]:_(s64), [[SHUF_E4:%[0-9]+]]:_(s64), [[SHUF_E5:%[0-9]+]]:_(s64), [[SHUF_E6:%[0-9]+]]:_(s64), [[SHUF_E7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[SHUF]]:_(<8 x s64>)
4181   CHECK: (<6 x s64>) = G_BUILD_VECTOR [[SHUF_E0]]:_(s64), [[SHUF_E1]]:_(s64), [[SHUF_E2]]:_(s64), [[SHUF_E3]]:_(s64), [[SHUF_E4]]:_(s64), [[SHUF_E5]]:_(s64)
4182   )";
4183
4184   // Check
4185   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4186 }
4187
4188 // Test narror scalar of G_SHL with constant shift amount
4189 TEST_F(AArch64GISelMITest, narrowScalarShiftByConstant) {
4190   setUp();
4191   if (!TM)
4192     GTEST_SKIP();
4193
4194   DefineLegalizerInfo(A, {});
4195
4196   LLT S64{LLT::scalar(64)};
4197   LLT S32{LLT::scalar(32)};
4198
4199   auto Constant = B.buildConstant(S64, 33);
4200   auto Trunc = B.buildTrunc(S32, Constant);
4201   auto Shift = B.buildShl(S64, Copies[0], Trunc);
4202
4203   AInfo Info(MF->getSubtarget());
4204   DummyGISelObserver Observer;
4205   LegalizerHelper Helper(*MF, Info, Observer, B);
4206
4207   // Perform Legalization
4208   B.setInsertPt(*EntryMBB, Shift->getIterator());
4209
4210   // This should detect the G_CONSTANT feeding the G_SHL through a G_TRUNC
4211   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4212             Helper.narrowScalarShift(*Shift, 0, S32));
4213
4214   const auto *CheckStr = R"(
4215   CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY
4216   CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY
4217   CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY
4218   CHECK: [[THIRTY3:%[0-9]+]]:_(s64) = G_CONSTANT i64 33
4219   CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC %4:_(s64)
4220   CHECK: [[UNMERGE:%[0-9]+]]:_(s32), [[UNMERGE2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY0]]
4221   CHECK: [[ZERO:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
4222   CHECK: [[ONE:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
4223   CHECK: [[SHIFT:%[0-9]+]]:_(s32) = G_SHL [[UNMERGE]]:_, [[ONE]]:_(s32)
4224   CHECK: [[MERGE:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[ZERO]]:_(s32), [[SHIFT]]:_(s32)
4225   )";
4226
4227   // Check
4228   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4229 }
4230
4231 TEST_F(AArch64GISelMITest, MoreElementsSelect) {
4232   setUp();
4233   if (!TM)
4234     GTEST_SKIP();
4235
4236   LLT s1 = LLT::scalar(1);
4237   LLT s64 = LLT::scalar(64);
4238   LLT v2s1 = LLT::fixed_vector(2, 1);
4239   LLT v2s32 = LLT::fixed_vector(2, 32);
4240
4241   LegalizerInfo LI;
4242   DummyGISelObserver Observer;
4243   LegalizerHelper Helper(*MF, LI, Observer, B);
4244
4245   B.setInsertPt(*EntryMBB, EntryMBB->end());
4246
4247   auto Val0 = B.buildBitcast(v2s32, Copies[0]);
4248   auto Val1 = B.buildBitcast(v2s32, Copies[1]);
4249
4250   // Build select of vectors with scalar condition.
4251   auto Zero = B.buildConstant(s64, 0);
4252   auto Cond = B.buildICmp(CmpInst::ICMP_EQ, s1, Copies[2], Zero);
4253   auto Select = B.buildSelect(v2s32, Cond, Val0, Val1);
4254
4255   // Splat the condition into a vector select
4256   B.setInstr(*Select);
4257
4258   EXPECT_EQ(LegalizerHelper::LegalizeResult::UnableToLegalize,
4259             Helper.moreElementsVector(*Select, 1, LLT::fixed_vector(3, 1)));
4260   EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
4261             Helper.moreElementsVector(*Select, 1, v2s1));
4262
4263   auto CheckStr = R"(
4264   CHECK: [[BITCAST0:%[0-9]+]]:_(<2 x s32>) = G_BITCAST
4265   CHECK: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST
4266   CHECK: [[ZERO0:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
4267   CHECK: [[CMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), %{{[0-9]+}}:_(s64), [[ZERO0]]
4268   CHECK: [[IMPDEF:%[0-9]+]]:_(<2 x s1>) = G_IMPLICIT_DEF
4269   CHECK: [[ZERO1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
4270   CHECK: [[INSERT:%[0-9]+]]:_(<2 x s1>) = G_INSERT_VECTOR_ELT [[IMPDEF]]:_, [[CMP]]:_(s1), [[ZERO1]]
4271   CHECK: [[SHUFFLE:%[0-9]+]]:_(<2 x s1>) = G_SHUFFLE_VECTOR [[INSERT]]:_(<2 x s1>), [[IMPDEF]]:_, shufflemask(0, 0)
4272   CHECK: [[SELECT:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[SHUFFLE]]:_(<2 x s1>), [[BITCAST0]]:_, [[BITCAST1]]:_
4273   )";
4274
4275   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
4276 }
4277
4278 } // namespace