llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td

   1 //===-- RISCVInstrInfoZvk.td - RISC-V 'Zvk' instructions ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file describes the RISC-V instructions from the standard 'Zvk',
  10 // Vector Cryptography Instructions extension, version Release 1.0.0.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------------------------------------------===//
  15 // Operand and SDNode transformation definitions.
  16 //===----------------------------------------------------------------------===//
  17
  18 def tuimm5 : RISCVOp, TImmLeaf<XLenVT, [{return isUInt<5>(Imm);}]>;
  19
  20 //===----------------------------------------------------------------------===//
  21 // Instruction class templates
  22 //===----------------------------------------------------------------------===//
  23
  24 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
  25 multiclass VCLMUL_MV_V_X<string opcodestr, bits<6> funct6> {
  26   def V  : VALUVV<funct6, OPMVV, opcodestr # "." # "vv">,
  27            SchedBinaryMC<"WriteVCLMULV", "ReadVCLMULV", "ReadVCLMULV">;
  28   def X  : VALUVX<funct6, OPMVX, opcodestr # "." # "vx">,
  29            SchedBinaryMC<"WriteVCLMULX", "ReadVCLMULV", "ReadVCLMULX">;
  30 }
  31
  32 class RVInstIVI_VROR<bits<6> funct6, dag outs, dag ins, string opcodestr,
  33                      string argstr>
  34     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
  35   bits<5> vs2;
  36   bits<6> imm;
  37   bits<5> vd;
  38   bit vm;
  39
  40   let Inst{31-27} = funct6{5-1};
  41   let Inst{26} = imm{5};
  42   let Inst{25} = vm;
  43   let Inst{24-20} = vs2;
  44   let Inst{19-15} = imm{4-0};
  45   let Inst{14-12} = OPIVI.Value;
  46   let Inst{11-7} = vd;
  47   let Inst{6-0} = OPC_OP_V.Value;
  48
  49   let Uses = [VTYPE, VL];
  50   let RVVConstraint = VMConstraint;
  51 }
  52
  53 multiclass VROR_IV_V_X_I<string opcodestr, bits<6> funct6>
  54     : VALU_IV_V_X<opcodestr, funct6> {
  55   def I : RVInstIVI_VROR<funct6, (outs VR:$vd),
  56               (ins VR:$vs2, uimm6:$imm, VMaskOp:$vm),
  57               opcodestr # ".vi", "$vd, $vs2, $imm$vm">,
  58           SchedUnaryMC<"WriteVRotI", "ReadVRotV">;
  59 }
  60
  61 // op vd, vs2, vs1
  62 class PALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
  63     : VALUVVNoVm<funct6, opv, opcodestr> {
  64   let Inst{6-0} = OPC_OP_VE.Value;
  65 }
  66
  67 // op vd, vs2, vs1
  68 class PALUVVNoVmTernary<bits<6> funct6, RISCVVFormat opv, string opcodestr>
  69     : RVInstVV<funct6, opv, (outs VR:$vd_wb),
  70                (ins VR:$vd, VR:$vs2, VR:$vs1),
  71                opcodestr, "$vd, $vs2, $vs1"> {
  72   let Constraints = "$vd = $vd_wb";
  73   let vm = 1;
  74   let Inst{6-0} = OPC_OP_VE.Value;
  75 }
  76
  77 // op vd, vs2, imm
  78 class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype>
  79     : VALUVINoVm<funct6, opcodestr, optype> {
  80   let Inst{6-0} = OPC_OP_VE.Value;
  81   let Inst{14-12} = OPMVV.Value;
  82 }
  83
  84 // op vd, vs2, imm where vd is also a source regardless of tail policy
  85 class PALUVINoVmBinary<bits<6> funct6, string opcodestr, Operand optype>
  86     : RVInstIVI<funct6, (outs VR:$vd_wb),
  87                 (ins VR:$vd, VR:$vs2, optype:$imm),
  88                 opcodestr, "$vd, $vs2, $imm"> {
  89   let Constraints = "$vd = $vd_wb";
  90   let vm = 1;
  91   let Inst{6-0} = OPC_OP_VE.Value;
  92   let Inst{14-12} = OPMVV.Value;
  93 }
  94
  95 // op vd, vs2 (use vs1 as instruction encoding) where vd is also a source
  96 // regardless of tail policy
  97 class PALUVs2NoVmBinary<bits<6> funct6, bits<5> vs1, RISCVVFormat opv,
  98                         string opcodestr>
  99     : RVInstV<funct6, vs1, opv, (outs VR:$vd_wb), (ins VR:$vd, VR:$vs2),
 100               opcodestr, "$vd, $vs2"> {
 101   let Constraints = "$vd = $vd_wb";
 102   let vm = 1;
 103   let Inst{6-0} = OPC_OP_VE.Value;
 104 }
 105
 106 multiclass VAES_MV_V_S<bits<6> funct6_vv, bits<6> funct6_vs, bits<5> vs1,
 107                          RISCVVFormat opv, string opcodestr> {
 108   let RVVConstraint = NoConstraint in
 109   def NAME # _VV : PALUVs2NoVmBinary<funct6_vv, vs1, opv, opcodestr # ".vv">,
 110                    SchedBinaryMC<"WriteVAESMVV", "ReadVAESMVV", "ReadVAESMVV">;
 111   let RVVConstraint = VS2Constraint in
 112   def NAME # _VS : PALUVs2NoVmBinary<funct6_vs, vs1, opv, opcodestr # ".vs">,
 113                    SchedBinaryMC<"WriteVAESMVV", "ReadVAESMVV", "ReadVAESMVV">;
 114 }
 115 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 116
 117 //===----------------------------------------------------------------------===//
 118 // Instructions
 119 //===----------------------------------------------------------------------===//
 120
 121 let Predicates = [HasStdExtZvbb] in {
 122   def  VBREV_V  : VALUVs2<0b010010, 0b01010, OPMVV, "vbrev.v">;
 123   def  VCLZ_V   : VALUVs2<0b010010, 0b01100, OPMVV, "vclz.v">;
 124   def  VCPOP_V  : VALUVs2<0b010010, 0b01110, OPMVV, "vcpop.v">;
 125   def  VCTZ_V   : VALUVs2<0b010010, 0b01101, OPMVV, "vctz.v">;
 126   let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in
 127   defm VWSLL_V  : VSHT_IV_V_X_I<"vwsll", 0b110101>;
 128 } // Predicates = [HasStdExtZvbb]
 129
 130 let Predicates = [HasStdExtZvbc] in {
 131   defm VCLMUL_V  : VCLMUL_MV_V_X<"vclmul", 0b001100>;
 132   defm VCLMULH_V : VCLMUL_MV_V_X<"vclmulh", 0b001101>;
 133 } // Predicates = [HasStdExtZvbc]
 134
 135 let Predicates = [HasStdExtZvkb] in {
 136   defm VANDN_V  : VALU_IV_V_X<"vandn", 0b000001>;
 137   def  VBREV8_V : VALUVs2<0b010010, 0b01000, OPMVV, "vbrev8.v">;
 138   def  VREV8_V  : VALUVs2<0b010010, 0b01001, OPMVV, "vrev8.v">;
 139   defm VROL_V   : VALU_IV_V_X<"vrol", 0b010101>;
 140   defm VROR_V   : VROR_IV_V_X_I<"vror", 0b010100>;
 141 } // Predicates = [HasStdExtZvkb]
 142
 143 let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in {
 144   def VGHSH_VV : PALUVVNoVmTernary<0b101100, OPMVV, "vghsh.vv">,
 145                  SchedTernaryMC<"WriteVGHSHV", "ReadVGHSHV", "ReadVGHSHV",
 146                                 "ReadVGHSHV">;
 147   def VGMUL_VV : PALUVs2NoVmBinary<0b101000, 0b10001, OPMVV, "vgmul.vv">,
 148                  SchedBinaryMC<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV">;
 149 } // Predicates = [HasStdExtZvkg]
 150
 151 let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in {
 152   def VSHA2CH_VV : PALUVVNoVmTernary<0b101110, OPMVV, "vsha2ch.vv">,
 153                    SchedTernaryMC<"WriteVSHA2CHV", "ReadVSHA2CHV", "ReadVSHA2CHV",
 154                                   "ReadVSHA2CHV">;
 155   def VSHA2CL_VV : PALUVVNoVmTernary<0b101111, OPMVV, "vsha2cl.vv">,
 156                    SchedTernaryMC<"WriteVSHA2CLV", "ReadVSHA2CLV", "ReadVSHA2CLV",
 157                                   "ReadVSHA2CLV">;
 158   def VSHA2MS_VV : PALUVVNoVmTernary<0b101101, OPMVV, "vsha2ms.vv">,
 159                    SchedTernaryMC<"WriteVSHA2MSV", "ReadVSHA2MSV", "ReadVSHA2MSV",
 160                                   "ReadVSHA2MSV">;
 161 } // Predicates = [HasStdExtZvknhaOrZvknhb]
 162
 163 let Predicates = [HasStdExtZvkned] in {
 164   defm VAESDF     : VAES_MV_V_S<0b101000, 0b101001, 0b00001, OPMVV, "vaesdf">;
 165   defm VAESDM     : VAES_MV_V_S<0b101000, 0b101001, 0b00000, OPMVV, "vaesdm">;
 166   defm VAESEF     : VAES_MV_V_S<0b101000, 0b101001, 0b00011, OPMVV, "vaesef">;
 167   defm VAESEM     : VAES_MV_V_S<0b101000, 0b101001, 0b00010, OPMVV, "vaesem">;
 168   def  VAESKF1_VI : PALUVINoVm<0b100010, "vaeskf1.vi", uimm5>,
 169                     SchedUnaryMC<"WriteVAESKF1V", "ReadVAESKF1V">;
 170   def  VAESKF2_VI : PALUVINoVmBinary<0b101010, "vaeskf2.vi", uimm5>,
 171                     SchedBinaryMC<"WriteVAESKF2V", "ReadVAESKF2V", "ReadVAESKF2V">;
 172   let RVVConstraint = VS2Constraint in
 173   def  VAESZ_VS   : PALUVs2NoVmBinary<0b101001, 0b00111, OPMVV, "vaesz.vs">,
 174                     SchedBinaryMC<"WriteVAESZV", "ReadVAESZV", "ReadVAESZV">;
 175 } // Predicates = [HasStdExtZvkned]
 176
 177 let Predicates = [HasStdExtZvksed] in {
 178   let RVVConstraint = NoConstraint in
 179   def  VSM4K_VI : PALUVINoVm<0b100001, "vsm4k.vi", uimm5>,
 180                   SchedUnaryMC<"WriteVSM4KV", "ReadVSM4KV">;
 181   defm VSM4R    : VAES_MV_V_S<0b101000, 0b101001, 0b10000, OPMVV, "vsm4r">;
 182 } // Predicates = [HasStdExtZvksed]
 183
 184 let Predicates = [HasStdExtZvksh], RVVConstraint = VS2Constraint in {
 185   def VSM3C_VI  : PALUVINoVmBinary<0b101011, "vsm3c.vi", uimm5>,
 186                   SchedBinaryMC<"WriteVSM3CV", "ReadVSM3CV", "ReadVSM3CV">;
 187   def VSM3ME_VV : PALUVVNoVm<0b100000, OPMVV, "vsm3me.vv">,
 188                   SchedUnaryMC<"WriteVSM3MEV", "ReadVSM3MEV">;
 189 } // Predicates = [HasStdExtZvksh]
 190
 191 //===----------------------------------------------------------------------===//
 192 // Pseudo instructions
 193 //===----------------------------------------------------------------------===//
 194
 195 defvar I32IntegerVectors = !filter(vti, AllIntegerVectors, !eq(vti.SEW, 32));
 196 defvar I32I64IntegerVectors = !filter(vti, AllIntegerVectors,
 197                                       !or(!eq(vti.SEW, 32), !eq(vti.SEW, 64)));
 198
 199 class ZvkI32IntegerVectors<string vd_lmul> {
 200   list<VTypeInfo> vs2_types = !cond(!eq(vd_lmul, "M8")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 32)),
 201                                     !eq(vd_lmul, "M4")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 32)),
 202                                     !eq(vd_lmul, "M2")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 16)),
 203                                     !eq(vd_lmul, "M1")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 8)),
 204                                     !eq(vd_lmul, "MF2")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 4)),
 205                                     !eq(vd_lmul, "MF4")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 2)),
 206                                     !eq(vd_lmul, "MF8")  : !filter(vti, I32IntegerVectors, !le(vti.LMul.octuple, 1)));
 207 }
 208
 209 class ZvkMxSet<string vd_lmul> {
 210   list<LMULInfo> vs2_lmuls = !cond(!eq(vd_lmul, "M8")  : [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4],
 211                                    !eq(vd_lmul, "M4")  : [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4],
 212                                    !eq(vd_lmul, "M2")  : [V_MF8, V_MF4, V_MF2, V_M1, V_M2],
 213                                    !eq(vd_lmul, "M1")  : [V_MF8, V_MF4, V_MF2, V_M1],
 214                                    !eq(vd_lmul, "MF2") : [V_MF8, V_MF4, V_MF2],
 215                                    !eq(vd_lmul, "MF4") : [V_MF8, V_MF4],
 216                                    !eq(vd_lmul, "MF8") : [V_MF8]);
 217 }
 218
 219 class VPseudoBinaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
 220       Pseudo<(outs RetClass:$rd_wb),
 221         (ins RetClass:$rd, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
 222         RISCVVPseudo {
 223   let mayLoad = 0;
 224   let mayStore = 0;
 225   let hasSideEffects = 0;
 226   let Constraints = "$rd_wb = $rd";
 227   let HasVLOp = 1;
 228   let HasSEWOp = 1;
 229   let HasVecPolicyOp = 1;
 230   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 231 }
 232
 233 class VPseudoTernaryNoMask_Zvk<VReg RetClass,
 234                                VReg Op1Class,
 235                                DAGOperand Op2Class> :
 236         Pseudo<(outs RetClass:$rd_wb),
 237                (ins RetClass:$rd, Op1Class:$rs2, Op2Class:$rs1,
 238                     AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
 239         RISCVVPseudo {
 240   let mayLoad = 0;
 241   let mayStore = 0;
 242   let hasSideEffects = 0;
 243   let Constraints = "$rd_wb = $rd";
 244   let HasVLOp = 1;
 245   let HasSEWOp = 1;
 246   let HasVecPolicyOp = 1;
 247   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 248 }
 249
 250 multiclass VPseudoBinaryNoMaskTU_Zvk<VReg RetClass,
 251                                      VReg Op1Class,
 252                                      DAGOperand Op2Class,
 253                                      LMULInfo MInfo,
 254                                      string Constraint = "",
 255                                      int sew = 0> {
 256   let VLMul = MInfo.value, SEW=sew in {
 257     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
 258     def suffix : VPseudoBinaryNoMaskTU<RetClass, Op1Class, Op2Class,
 259                                        Constraint>;
 260   }
 261 }
 262
 263 multiclass VPseudoTernaryNoMask_Zvk<VReg RetClass,
 264                                    VReg Op1Class,
 265                                    DAGOperand Op2Class,
 266                                    LMULInfo MInfo> {
 267   let VLMul = MInfo.value in
 268     def "_" # MInfo.MX : VPseudoTernaryNoMask_Zvk<RetClass, Op1Class, Op2Class>;
 269 }
 270
 271 multiclass VPseudoBinaryV_V_NoMask_Zvk<LMULInfo m> {
 272   let VLMul = m.value in {
 273     def "_VV_" # m.MX : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass>;
 274   }
 275 }
 276
 277 multiclass VPseudoBinaryV_S_NoMask_Zvk<LMULInfo m> {
 278   let VLMul = m.value in
 279     foreach vs2_lmul = ZvkMxSet<m.MX>.vs2_lmuls in
 280       def "_VS_" # m.MX # "_" # vs2_lmul.MX : VPseudoBinaryNoMask_Zvk<m.vrclass, vs2_lmul.vrclass>;
 281 }
 282
 283 multiclass VPseudoVGMUL {
 284   foreach m = MxListVF4 in {
 285     defvar mx = m.MX;
 286     defm "" : VPseudoBinaryV_V_NoMask_Zvk<m>,
 287               SchedBinary<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV", mx>;
 288   }
 289 }
 290
 291 multiclass VPseudoVAESMV {
 292   foreach m = MxListVF4 in {
 293     defvar mx = m.MX;
 294     defm "" : VPseudoBinaryV_V_NoMask_Zvk<m>,
 295               SchedBinary<"WriteVAESMVV", "ReadVAESMVV", "ReadVAESMVV", mx>;
 296     defm "" : VPseudoBinaryV_S_NoMask_Zvk<m>,
 297               SchedBinary<"WriteVAESMVV", "ReadVAESMVV", "ReadVAESMVV", mx>;
 298
 299   }
 300 }
 301
 302 multiclass VPseudoVSM4R {
 303   foreach m = MxListVF4 in {
 304     defvar mx = m.MX;
 305     defm "" : VPseudoBinaryV_V_NoMask_Zvk<m>,
 306               SchedBinary<"WriteVSM4RV", "ReadVSM4RV", "ReadVSM4RV", mx>;
 307     defm "" : VPseudoBinaryV_S_NoMask_Zvk<m>,
 308               SchedBinary<"WriteVSM4RV", "ReadVSM4RV", "ReadVSM4RV", mx>;
 309
 310   }
 311 }
 312
 313 multiclass VPseudoVGHSH {
 314   foreach m = MxListVF4 in {
 315     defvar mx = m.MX;
 316     defm _VV : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
 317                SchedTernary<"WriteVGHSHV", "ReadVGHSHV", "ReadVGHSHV",
 318                             "ReadVGHSHV", mx>;
 319   }
 320 }
 321
 322 multiclass VPseudoVSHA2CH {
 323   foreach m = MxListVF4 in {
 324     defvar mx = m.MX;
 325     defm _VV : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
 326                SchedTernary<"WriteVSHA2CHV", "ReadVSHA2CHV", "ReadVSHA2CHV",
 327                             "ReadVSHA2CHV", mx>;
 328   }
 329 }
 330
 331 multiclass VPseudoVSHA2CL {
 332   foreach m = MxListVF4 in {
 333     defvar mx = m.MX;
 334     defm _VV : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
 335                SchedTernary<"WriteVSHA2CLV", "ReadVSHA2CLV", "ReadVSHA2CLV",
 336                             "ReadVSHA2CLV", mx>;
 337   }
 338 }
 339
 340 multiclass VPseudoVSHA2MS {
 341   foreach m = MxListVF4 in {
 342     defvar mx = m.MX;
 343     defm _VV : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
 344                SchedTernary<"WriteVSHA2MSV", "ReadVSHA2MSV", "ReadVSHA2MSV",
 345                             "ReadVSHA2MSV", mx>;
 346   }
 347 }
 348
 349 multiclass VPseudoVAESKF1 {
 350   foreach m = MxListVF4 in {
 351     defvar mx = m.MX;
 352     defm _VI : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, uimm5, m>,
 353                SchedBinary<"WriteVAESKF1V", "ReadVAESKF1V", "ReadVAESKF1V", mx,
 354                            forceMergeOpRead=true>;
 355   }
 356 }
 357
 358 multiclass VPseudoVAESKF2 {
 359   foreach m = MxListVF4 in {
 360     defvar mx = m.MX;
 361     defm _VI : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, uimm5, m>,
 362                SchedTernary<"WriteVAESKF2V", "ReadVAESKF2V", "ReadVAESKF2V",
 363                             "ReadVAESKF2V", mx>;
 364   }
 365 }
 366
 367 multiclass VPseudoVAESZ {
 368   foreach m = MxListVF4 in {
 369     defvar mx = m.MX;
 370     defm "" : VPseudoBinaryV_S_NoMask_Zvk<m>,
 371               SchedBinary<"WriteVAESZV", "ReadVAESZV", "ReadVAESZV", mx>;
 372   }
 373 }
 374
 375 multiclass VPseudoVSM3C {
 376   foreach m = MxListVF4 in {
 377     defvar mx = m.MX;
 378     defm _VI : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, uimm5, m>,
 379                SchedTernary<"WriteVSM3CV", "ReadVSM3CV", "ReadVSM3CV",
 380                             "ReadVSM3CV", mx>;
 381   }
 382 }
 383
 384 multiclass VPseudoVSM4K {
 385   foreach m = MxListVF4 in {
 386     defvar mx = m.MX;
 387     defm _VI : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, uimm5, m>,
 388                SchedBinary<"WriteVSM4KV", "ReadVSM4KV", "ReadVSM4KV", mx,
 389                            forceMergeOpRead=true>;
 390   }
 391 }
 392
 393 multiclass VPseudoVSM3ME {
 394   foreach m = MxListVF4 in {
 395     defvar mx = m.MX;
 396     defm _VV : VPseudoBinaryNoMaskTU_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
 397                SchedBinary<"WriteVSM3MEV", "ReadVSM3MEV", "ReadVSM3MEV", mx,
 398                            forceMergeOpRead=true>;
 399   }
 400 }
 401
 402 multiclass VPseudoVCLMUL_VV_VX {
 403   foreach m = MxList in {
 404     defvar mx = m.MX;
 405     defm "" : VPseudoBinaryV_VV<m>,
 406               SchedBinary<"WriteVCLMULV", "ReadVCLMULV", "ReadVCLMULV", mx,
 407                           forceMergeOpRead=true>;
 408     defm "" : VPseudoBinaryV_VX<m>,
 409               SchedBinary<"WriteVCLMULX", "ReadVCLMULV", "ReadVCLMULX", mx,
 410                           forceMergeOpRead=true>;
 411   }
 412 }
 413
 414 multiclass VPseudoUnaryV_V<LMULInfo m> {
 415   let VLMul = m.value in {
 416     defvar suffix = "_V_" # m.MX;
 417     def suffix : VPseudoUnaryNoMask<m.vrclass, m.vrclass>;
 418     def suffix # "_MASK" : VPseudoUnaryMask<m.vrclass, m.vrclass>,
 419                                             RISCVMaskedPseudo<MaskIdx=2>;
 420   }
 421 }
 422
 423 multiclass VPseudoVBREV {
 424   foreach m = MxList in {
 425     defvar mx = m.MX;
 426     defm "" : VPseudoUnaryV_V<m>,
 427               SchedUnary<"WriteVBREVV", "ReadVBREVV", mx, forceMergeOpRead=true>;
 428   }
 429 }
 430
 431 multiclass VPseudoVCLZ {
 432   foreach m = MxList in {
 433     defvar mx = m.MX;
 434     defm "" : VPseudoUnaryV_V<m>,
 435               SchedUnary<"WriteVCLZV", "ReadVCLZV", mx, forceMergeOpRead=true>;
 436   }
 437 }
 438
 439 multiclass VPseudoVCTZ {
 440   foreach m = MxList in {
 441     defvar mx = m.MX;
 442     defm "" : VPseudoUnaryV_V<m>,
 443               SchedUnary<"WriteVCTZV", "ReadVCTZV", mx, forceMergeOpRead=true>;
 444   }
 445 }
 446
 447 multiclass VPseudoVCPOP {
 448   foreach m = MxList in {
 449     defvar mx = m.MX;
 450     defm "" : VPseudoUnaryV_V<m>,
 451               SchedUnary<"WriteVCPOPV", "ReadVCPOPV", mx, forceMergeOpRead=true>;
 452   }
 453 }
 454
 455 multiclass VPseudoVWALU_VV_VX_VI<Operand ImmType> {
 456   foreach m = MxListW in {
 457     defvar mx = m.MX;
 458     defm "" : VPseudoBinaryW_VV<m>,
 459               SchedBinary<"WriteVWSLLV", "ReadVWSLLV", "ReadVWSLLV", mx,
 460                           forceMergeOpRead=true>;
 461     defm "" : VPseudoBinaryW_VX<m>,
 462               SchedBinary<"WriteVWSLLX", "ReadVWSLLV", "ReadVWSLLX", mx,
 463                           forceMergeOpRead=true>;
 464     defm "" : VPseudoBinaryW_VI<ImmType, m>,
 465               SchedUnary<"WriteVWSLLI", "ReadVWSLLV", mx,
 466                          forceMergeOpRead=true>;
 467   }
 468 }
 469
 470 multiclass VPseudoVANDN {
 471  foreach m = MxList in {
 472     defm "" : VPseudoBinaryV_VV<m>,
 473               SchedBinary<"WriteVIALUV", "ReadVIALUV", "ReadVIALUV", m.MX,
 474                           forceMergeOpRead=true>;
 475     defm "" : VPseudoBinaryV_VX<m>,
 476               SchedBinary<"WriteVIALUX", "ReadVIALUV", "ReadVIALUX", m.MX,
 477                           forceMergeOpRead=true>;
 478   }
 479 }
 480
 481 multiclass VPseudoVBREV8 {
 482   foreach m = MxList in {
 483     defvar mx = m.MX;
 484     defm "" : VPseudoUnaryV_V<m>,
 485               SchedUnary<"WriteVBREV8V", "ReadVBREV8V", mx, forceMergeOpRead=true>;
 486   }
 487 }
 488
 489 multiclass VPseudoVREV8 {
 490   foreach m = MxList in {
 491     defvar mx = m.MX;
 492     defm "" : VPseudoUnaryV_V<m>,
 493               SchedUnary<"WriteVREV8V", "ReadVREV8V", mx, forceMergeOpRead=true>;
 494   }
 495 }
 496
 497 multiclass VPseudoVROL {
 498  foreach m = MxList in {
 499     defm "" : VPseudoBinaryV_VV<m>,
 500               SchedBinary<"WriteVRotV", "ReadVRotV", "ReadVRotV", m.MX,
 501                           forceMergeOpRead=true>;
 502     defm "" : VPseudoBinaryV_VX<m>,
 503               SchedBinary<"WriteVRotX", "ReadVRotV", "ReadVRotX", m.MX,
 504                           forceMergeOpRead=true>;
 505   }
 506 }
 507
 508 multiclass VPseudoVROR<Operand ImmType> {
 509   defvar Constraint = "";
 510   foreach m = MxList in {
 511     defvar mx = m.MX;
 512     defm "" : VPseudoBinaryV_VV<m>,
 513               SchedBinary<"WriteVRotV", "ReadVRotV", "ReadVRotV", mx,
 514                           forceMergeOpRead=true>;
 515     defm "" : VPseudoBinaryV_VX<m>,
 516               SchedBinary<"WriteVRotX", "ReadVRotV", "ReadVRotX", mx,
 517                           forceMergeOpRead=true>;
 518     defm "" : VPseudoBinaryV_VI<ImmType, m>,
 519               SchedUnary<"WriteVRotI", "ReadVRotV", mx, forceMergeOpRead=true>;
 520   }
 521 }
 522
 523 let Predicates = [HasStdExtZvbb] in {
 524   defm PseudoVBREV  : VPseudoVBREV;
 525   defm PseudoVCLZ   : VPseudoVCLZ;
 526   defm PseudoVCTZ   : VPseudoVCTZ;
 527   defm PseudoVCPOP  : VPseudoVCPOP;
 528   defm PseudoVWSLL : VPseudoVWALU_VV_VX_VI<uimm5>;
 529 } // Predicates = [HasStdExtZvbb]
 530
 531 let Predicates = [HasStdExtZvbc] in {
 532   defm PseudoVCLMUL  : VPseudoVCLMUL_VV_VX;
 533   defm PseudoVCLMULH : VPseudoVCLMUL_VV_VX;
 534 } // Predicates = [HasStdExtZvbc]
 535
 536 let Predicates = [HasStdExtZvkb] in {
 537   defm PseudoVANDN  : VPseudoVANDN;
 538   defm PseudoVBREV8 : VPseudoVBREV8;
 539   defm PseudoVREV8  : VPseudoVREV8;
 540   defm PseudoVROL   : VPseudoVROL;
 541   defm PseudoVROR   : VPseudoVROR<uimm6>;
 542 } // Predicates = [HasStdExtZvkb]
 543
 544 let Predicates = [HasStdExtZvkg] in {
 545   defm PseudoVGHSH : VPseudoVGHSH;
 546   defm PseudoVGMUL : VPseudoVGMUL;
 547 } // Predicates = [HasStdExtZvkg]
 548
 549 let Predicates = [HasStdExtZvkned] in {
 550   defm PseudoVAESDF  : VPseudoVAESMV;
 551   defm PseudoVAESDM  : VPseudoVAESMV;
 552   defm PseudoVAESEF  : VPseudoVAESMV;
 553   defm PseudoVAESEM  : VPseudoVAESMV;
 554   defm PseudoVAESKF1 : VPseudoVAESKF1;
 555   defm PseudoVAESKF2 : VPseudoVAESKF2;
 556   defm PseudoVAESZ   : VPseudoVAESZ;
 557 } // Predicates = [HasStdExtZvkned]
 558
 559 let Predicates = [HasStdExtZvknhaOrZvknhb] in {
 560   defm PseudoVSHA2CH : VPseudoVSHA2CH;
 561   defm PseudoVSHA2CL : VPseudoVSHA2CL;
 562   defm PseudoVSHA2MS : VPseudoVSHA2MS;
 563 } // Predicates = [HasStdExtZvknhaOrZvknhb]
 564
 565 let Predicates = [HasStdExtZvksed] in {
 566   defm PseudoVSM4K : VPseudoVSM4K;
 567   defm PseudoVSM4R : VPseudoVSM4R;
 568 } // Predicates = [HasStdExtZvksed]
 569
 570 let Predicates = [HasStdExtZvksh] in {
 571   defm PseudoVSM3C  : VPseudoVSM3C;
 572   defm PseudoVSM3ME : VPseudoVSM3ME;
 573 } // Predicates = [HasStdExtZvksh]
 574
 575 //===----------------------------------------------------------------------===//
 576 // SDNode patterns
 577 //===----------------------------------------------------------------------===//
 578
 579 multiclass VPatUnarySDNode_V<SDPatternOperator op, string instruction_name,
 580                              Predicate predicate = HasStdExtZvbb> {
 581   foreach vti = AllIntegerVectors in {
 582     let Predicates = !listconcat([predicate],
 583                                  GetVTypePredicates<vti>.Predicates) in {
 584       def : Pat<(vti.Vector (op (vti.Vector vti.RegClass:$rs1))),
 585                 (!cast<Instruction>(instruction_name#"_V_"#vti.LMul.MX)
 586                    (vti.Vector (IMPLICIT_DEF)),
 587                    vti.RegClass:$rs1,
 588                    vti.AVL, vti.Log2SEW, TA_MA)>;
 589     }
 590   }
 591 }
 592
 593 // Helpers for detecting splats since we preprocess splat_vector to vmv.v.x
 594 // This should match the logic in RISCVDAGToDAGISel::selectVSplat
 595 def riscv_splat_vector : PatFrag<(ops node:$rs1),
 596                                  (riscv_vmv_v_x_vl undef, node:$rs1, srcvalue)>;
 597 def riscv_vnot : PatFrag<(ops node:$rs1), (xor node:$rs1,
 598                                                (riscv_splat_vector -1))>;
 599
 600 foreach vti = AllIntegerVectors in {
 601   let Predicates = !listconcat([HasStdExtZvkb],
 602                                GetVTypePredicates<vti>.Predicates) in {
 603     def : Pat<(vti.Vector (and (riscv_vnot vti.RegClass:$rs1),
 604                                vti.RegClass:$rs2)),
 605               (!cast<Instruction>("PseudoVANDN_VV_"#vti.LMul.MX)
 606                  (vti.Vector (IMPLICIT_DEF)),
 607                  vti.RegClass:$rs2,
 608                  vti.RegClass:$rs1,
 609                  vti.AVL, vti.Log2SEW, TA_MA)>;
 610     def : Pat<(vti.Vector (and (riscv_splat_vector
 611                                  (not vti.ScalarRegClass:$rs1)),
 612                                vti.RegClass:$rs2)),
 613               (!cast<Instruction>("PseudoVANDN_VX_"#vti.LMul.MX)
 614                  (vti.Vector (IMPLICIT_DEF)),
 615                  vti.RegClass:$rs2,
 616                  vti.ScalarRegClass:$rs1,
 617                  vti.AVL, vti.Log2SEW, TA_MA)>;
 618   }
 619 }
 620
 621 defm : VPatUnarySDNode_V<bitreverse, "PseudoVBREV">;
 622 defm : VPatUnarySDNode_V<bswap, "PseudoVREV8", HasStdExtZvkb>;
 623 defm : VPatUnarySDNode_V<ctlz, "PseudoVCLZ">;
 624 defm : VPatUnarySDNode_V<cttz, "PseudoVCTZ">;
 625 defm : VPatUnarySDNode_V<ctpop, "PseudoVCPOP">;
 626
 627 defm : VPatBinarySDNode_VV_VX<rotl, "PseudoVROL">;
 628
 629 // Invert the immediate and mask it to SEW for readability.
 630 def InvRot8Imm : SDNodeXForm<imm, [{
 631   return CurDAG->getTargetConstant(0x7 & (64 - N->getZExtValue()), SDLoc(N),
 632                                    N->getValueType(0));
 633 }]>;
 634 def InvRot16Imm : SDNodeXForm<imm, [{
 635   return CurDAG->getTargetConstant(0xf & (64 - N->getZExtValue()), SDLoc(N),
 636                                    N->getValueType(0));
 637 }]>;
 638 def InvRot32Imm : SDNodeXForm<imm, [{
 639   return CurDAG->getTargetConstant(0x1f & (64 - N->getZExtValue()), SDLoc(N),
 640                                    N->getValueType(0));
 641 }]>;
 642 def InvRot64Imm : SDNodeXForm<imm, [{
 643   return CurDAG->getTargetConstant(0x3f & (64 - N->getZExtValue()), SDLoc(N),
 644                                    N->getValueType(0));
 645 }]>;
 646
 647 // Although there is no vrol.vi, an immediate rotate left can be achieved by
 648 // negating the immediate in vror.vi
 649 foreach vti = AllIntegerVectors in {
 650   let Predicates = !listconcat([HasStdExtZvkb],
 651                                GetVTypePredicates<vti>.Predicates) in {
 652     def : Pat<(vti.Vector (rotl vti.RegClass:$rs2,
 653                                 (vti.Vector (SplatPat_uimm6 uimm6:$rs1)))),
 654               (!cast<Instruction>("PseudoVROR_VI_"#vti.LMul.MX)
 655                  (vti.Vector (IMPLICIT_DEF)),
 656                  vti.RegClass:$rs2,
 657                  (!cast<SDNodeXForm>("InvRot" # vti.SEW # "Imm") uimm6:$rs1),
 658                  vti.AVL, vti.Log2SEW, TA_MA)>;
 659   }
 660 }
 661 defm : VPatBinarySDNode_VV_VX_VI<rotr, "PseudoVROR", uimm6>;
 662
 663 foreach vtiToWti = AllWidenableIntVectors in {
 664   defvar vti = vtiToWti.Vti;
 665   defvar wti = vtiToWti.Wti;
 666   let Predicates = !listconcat([HasStdExtZvbb],
 667                                GetVTypePredicates<vti>.Predicates,
 668                                GetVTypePredicates<wti>.Predicates) in {
 669     def : Pat<(shl (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 670                    (wti.Vector (ext_oneuse (vti.Vector vti.RegClass:$rs1)))),
 671               (!cast<Instruction>("PseudoVWSLL_VV_"#vti.LMul.MX)
 672                  (wti.Vector (IMPLICIT_DEF)),
 673                  vti.RegClass:$rs2, vti.RegClass:$rs1,
 674                  vti.AVL, vti.Log2SEW, TA_MA)>;
 675
 676     def : Pat<(shl (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 677                    (wti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1)))),
 678               (!cast<Instruction>("PseudoVWSLL_VX_"#vti.LMul.MX)
 679                  (wti.Vector (IMPLICIT_DEF)),
 680                  vti.RegClass:$rs2, GPR:$rs1,
 681                  vti.AVL, vti.Log2SEW, TA_MA)>;
 682
 683     def : Pat<(shl (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 684                    (wti.Vector (SplatPat_uimm5 uimm5:$rs1))),
 685               (!cast<Instruction>("PseudoVWSLL_VI_"#vti.LMul.MX)
 686                  (wti.Vector (IMPLICIT_DEF)),
 687                  vti.RegClass:$rs2, uimm5:$rs1,
 688                  vti.AVL, vti.Log2SEW, TA_MA)>;
 689   }
 690 }
 691
 692 //===----------------------------------------------------------------------===//
 693 // VL patterns
 694 //===----------------------------------------------------------------------===//
 695
 696 multiclass VPatUnaryVL_V<SDPatternOperator op, string instruction_name,
 697                          Predicate predicate = HasStdExtZvbb> {
 698   foreach vti = AllIntegerVectors in {
 699     let Predicates = !listconcat([predicate],
 700                                  GetVTypePredicates<vti>.Predicates) in {
 701       def : Pat<(vti.Vector (op (vti.Vector vti.RegClass:$rs1),
 702                                 (vti.Vector vti.RegClass:$merge),
 703                                 (vti.Mask V0),
 704                                 VLOpFrag)),
 705                 (!cast<Instruction>(instruction_name#"_V_"#vti.LMul.MX#"_MASK")
 706                    vti.RegClass:$merge,
 707                    vti.RegClass:$rs1,
 708                    (vti.Mask V0),
 709                    GPR:$vl,
 710                    vti.Log2SEW,
 711                    TAIL_AGNOSTIC)>;
 712     }
 713   }
 714 }
 715
 716 foreach vti = AllIntegerVectors in {
 717   let Predicates = !listconcat([HasStdExtZvkb],
 718                                GetVTypePredicates<vti>.Predicates) in {
 719     def : Pat<(vti.Vector (riscv_and_vl (riscv_xor_vl
 720                                            (vti.Vector vti.RegClass:$rs1),
 721                                            (riscv_splat_vector -1),
 722                                            (vti.Vector vti.RegClass:$merge),
 723                                            (vti.Mask V0),
 724                                            VLOpFrag),
 725                                         (vti.Vector vti.RegClass:$rs2),
 726                                         (vti.Vector vti.RegClass:$merge),
 727                                         (vti.Mask V0),
 728                                         VLOpFrag)),
 729               (!cast<Instruction>("PseudoVANDN_VV_"#vti.LMul.MX#"_MASK")
 730                  vti.RegClass:$merge,
 731                  vti.RegClass:$rs2,
 732                  vti.RegClass:$rs1,
 733                  (vti.Mask V0),
 734                  GPR:$vl,
 735                  vti.Log2SEW,
 736                  TAIL_AGNOSTIC)>;
 737
 738     def : Pat<(vti.Vector (riscv_and_vl (riscv_splat_vector
 739                                            (not vti.ScalarRegClass:$rs1)),
 740                                         (vti.Vector vti.RegClass:$rs2),
 741                                         (vti.Vector vti.RegClass:$merge),
 742                                         (vti.Mask V0),
 743                                         VLOpFrag)),
 744               (!cast<Instruction>("PseudoVANDN_VX_"#vti.LMul.MX#"_MASK")
 745                  vti.RegClass:$merge,
 746                  vti.RegClass:$rs2,
 747                  vti.ScalarRegClass:$rs1,
 748                  (vti.Mask V0),
 749                  GPR:$vl,
 750                  vti.Log2SEW,
 751                  TAIL_AGNOSTIC)>;
 752   }
 753 }
 754
 755 defm : VPatUnaryVL_V<riscv_bitreverse_vl, "PseudoVBREV">;
 756 defm : VPatUnaryVL_V<riscv_bswap_vl, "PseudoVREV8", HasStdExtZvkb>;
 757 defm : VPatUnaryVL_V<riscv_ctlz_vl, "PseudoVCLZ">;
 758 defm : VPatUnaryVL_V<riscv_cttz_vl, "PseudoVCTZ">;
 759 defm : VPatUnaryVL_V<riscv_ctpop_vl, "PseudoVCPOP">;
 760
 761 defm : VPatBinaryVL_VV_VX<riscv_rotl_vl, "PseudoVROL">;
 762 // Although there is no vrol.vi, an immediate rotate left can be achieved by
 763 // negating the immediate in vror.vi
 764 foreach vti = AllIntegerVectors in {
 765   let Predicates = !listconcat([HasStdExtZvkb],
 766                                GetVTypePredicates<vti>.Predicates) in {
 767     def : Pat<(riscv_rotl_vl vti.RegClass:$rs2,
 768                              (vti.Vector (SplatPat_uimm6 uimm6:$rs1)),
 769                              (vti.Vector vti.RegClass:$merge),
 770                              (vti.Mask V0), VLOpFrag),
 771               (!cast<Instruction>("PseudoVROR_VI_"#vti.LMul.MX#"_MASK")
 772                  vti.RegClass:$merge,
 773                  vti.RegClass:$rs2,
 774                  (!cast<SDNodeXForm>("InvRot" # vti.SEW # "Imm") uimm6:$rs1),
 775                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 776   }
 777 }
 778 defm : VPatBinaryVL_VV_VX_VI<riscv_rotr_vl, "PseudoVROR", uimm6>;
 779
 780 foreach vtiToWti = AllWidenableIntVectors in {
 781   defvar vti = vtiToWti.Vti;
 782   defvar wti = vtiToWti.Wti;
 783   let Predicates = !listconcat([HasStdExtZvbb],
 784                                GetVTypePredicates<vti>.Predicates,
 785                                GetVTypePredicates<wti>.Predicates) in {
 786     def : Pat<(riscv_shl_vl
 787                  (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 788                  (wti.Vector (ext_oneuse (vti.Vector vti.RegClass:$rs1))),
 789                  (wti.Vector wti.RegClass:$merge),
 790                  (vti.Mask V0), VLOpFrag),
 791               (!cast<Instruction>("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK")
 792                  wti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1,
 793                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 794
 795     def : Pat<(riscv_shl_vl
 796                  (wti.Vector (riscv_zext_vl_oneuse
 797                                 (vti.Vector vti.RegClass:$rs2),
 798                                 (vti.Mask V0), VLOpFrag)),
 799                  (wti.Vector (riscv_ext_vl_oneuse
 800                                 (vti.Vector vti.RegClass:$rs1),
 801                                 (vti.Mask V0), VLOpFrag)),
 802                  (wti.Vector wti.RegClass:$merge),
 803                  (vti.Mask V0), VLOpFrag),
 804               (!cast<Instruction>("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK")
 805                  wti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1,
 806                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 807
 808     def : Pat<(riscv_shl_vl
 809                  (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 810                  (wti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))),
 811                  (wti.Vector wti.RegClass:$merge),
 812                  (vti.Mask V0), VLOpFrag),
 813               (!cast<Instruction>("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK")
 814                  wti.RegClass:$merge, vti.RegClass:$rs2, GPR:$rs1,
 815                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 816
 817     def : Pat<(riscv_shl_vl
 818                  (wti.Vector (riscv_zext_vl_oneuse
 819                                 (vti.Vector vti.RegClass:$rs2),
 820                                 (vti.Mask V0), VLOpFrag)),
 821                  (wti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))),
 822                  (wti.Vector wti.RegClass:$merge),
 823                  (vti.Mask V0), VLOpFrag),
 824               (!cast<Instruction>("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK")
 825                  wti.RegClass:$merge, vti.RegClass:$rs2, GPR:$rs1,
 826                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 827
 828     def : Pat<(riscv_shl_vl
 829                  (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))),
 830                  (wti.Vector (SplatPat_uimm5 uimm5:$rs1)),
 831                  (wti.Vector wti.RegClass:$merge),
 832                  (vti.Mask V0), VLOpFrag),
 833               (!cast<Instruction>("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK")
 834                  wti.RegClass:$merge, vti.RegClass:$rs2, uimm5:$rs1,
 835                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 836
 837     def : Pat<(riscv_shl_vl
 838                  (wti.Vector (riscv_zext_vl_oneuse
 839                                 (vti.Vector vti.RegClass:$rs2),
 840                                 (vti.Mask V0), VLOpFrag)),
 841                  (wti.Vector (SplatPat_uimm5 uimm5:$rs1)),
 842                  (wti.Vector wti.RegClass:$merge),
 843                  (vti.Mask V0), VLOpFrag),
 844               (!cast<Instruction>("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK")
 845                  wti.RegClass:$merge, vti.RegClass:$rs2, uimm5:$rs1,
 846                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 847
 848     def : Pat<(riscv_vwsll_vl
 849                  (vti.Vector vti.RegClass:$rs2),
 850                  (vti.Vector vti.RegClass:$rs1),
 851                  (wti.Vector wti.RegClass:$merge),
 852                  (vti.Mask V0), VLOpFrag),
 853               (!cast<Instruction>("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK")
 854                  wti.RegClass:$merge, vti.RegClass:$rs2, vti.RegClass:$rs1,
 855                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 856
 857     def : Pat<(riscv_vwsll_vl
 858                  (vti.Vector vti.RegClass:$rs2),
 859                  (vti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))),
 860                  (wti.Vector wti.RegClass:$merge),
 861                  (vti.Mask V0), VLOpFrag),
 862               (!cast<Instruction>("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK")
 863                  wti.RegClass:$merge, vti.RegClass:$rs2, GPR:$rs1,
 864                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 865
 866     def : Pat<(riscv_vwsll_vl
 867                  (vti.Vector vti.RegClass:$rs2),
 868                  (vti.Vector (SplatPat_uimm5 uimm5:$rs1)),
 869                  (wti.Vector wti.RegClass:$merge),
 870                  (vti.Mask V0), VLOpFrag),
 871               (!cast<Instruction>("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK")
 872                  wti.RegClass:$merge, vti.RegClass:$rs2, uimm5:$rs1,
 873                  (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
 874   }
 875 }
 876
 877 //===----------------------------------------------------------------------===//
 878 // Codegen patterns
 879 //===----------------------------------------------------------------------===//
 880
 881 class VPatUnaryNoMask_Zvk<string intrinsic_name,
 882                           string inst,
 883                           string kind,
 884                           ValueType result_type,
 885                           ValueType op2_type,
 886                           int sew,
 887                           LMULInfo vlmul,
 888                           VReg result_reg_class,
 889                           VReg op2_reg_class> :
 890   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
 891                    (result_type result_reg_class:$rd),
 892                    (op2_type op2_reg_class:$rs2),
 893                    VLOpFrag, (XLenVT timm:$policy))),
 894                    (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX)
 895                    (result_type result_reg_class:$rd),
 896                    (op2_type op2_reg_class:$rs2),
 897                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 898
 899 class VPatUnaryNoMask_VS_Zvk<string intrinsic_name,
 900                              string inst,
 901                              string kind,
 902                              ValueType result_type,
 903                              ValueType op2_type,
 904                              int sew,
 905                              LMULInfo vlmul,
 906                              LMULInfo vs2_lmul,
 907                              VReg result_reg_class,
 908                              VReg op2_reg_class> :
 909   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
 910                    (result_type result_reg_class:$rd),
 911                    (op2_type op2_reg_class:$rs2),
 912                    VLOpFrag, (XLenVT timm:$policy))),
 913                    (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX#"_"#vs2_lmul.MX)
 914                    (result_type result_reg_class:$rd),
 915                    (op2_type op2_reg_class:$rs2),
 916                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 917
 918 multiclass VPatUnaryV_V_NoMask_Zvk<string intrinsic, string instruction,
 919                                    list<VTypeInfo> vtilist> {
 920   foreach vti = vtilist in
 921     def : VPatUnaryNoMask_Zvk<intrinsic # "_vv", instruction, "VV",
 922                           vti.Vector, vti.Vector, vti.Log2SEW,
 923                           vti.LMul, vti.RegClass, vti.RegClass>;
 924 }
 925
 926 multiclass VPatUnaryV_S_NoMaskVectorCrypto<string intrinsic, string instruction,
 927                                            list<VTypeInfo> vtilist> {
 928   foreach vti = vtilist in
 929     foreach vti_vs2 = ZvkI32IntegerVectors<vti.LMul.MX>.vs2_types in
 930       def : VPatUnaryNoMask_VS_Zvk<intrinsic # "_vs", instruction, "VS",
 931                             vti.Vector, vti_vs2.Vector, vti.Log2SEW,
 932                             vti.LMul, vti_vs2.LMul, vti.RegClass, vti_vs2.RegClass>;
 933 }
 934
 935 multiclass VPatUnaryV_V_S_NoMask_Zvk<string intrinsic, string instruction,
 936                                      list<VTypeInfo> vtilist> {
 937   defm : VPatUnaryV_V_NoMask_Zvk<intrinsic, instruction, vtilist>;
 938   defm : VPatUnaryV_S_NoMaskVectorCrypto<intrinsic, instruction, vtilist>;
 939 }
 940
 941 multiclass VPatBinaryV_VV_NoMask<string intrinsic, string instruction,
 942                                  list<VTypeInfo> vtilist> {
 943   foreach vti = vtilist in
 944     def : VPatTernaryNoMaskWithPolicy<intrinsic, instruction, "VV",
 945                                       vti.Vector, vti.Vector, vti.Vector,
 946                                       vti.Log2SEW, vti.LMul, vti.RegClass,
 947                                       vti.RegClass, vti.RegClass>;
 948 }
 949
 950 multiclass VPatBinaryV_VI_NoMask<string intrinsic, string instruction,
 951                                  list<VTypeInfo> vtilist,
 952                                  Operand imm_type = tuimm5> {
 953   foreach vti = vtilist in
 954     def : VPatTernaryNoMaskWithPolicy<intrinsic, instruction, "VI",
 955                                       vti.Vector, vti.Vector, XLenVT,
 956                                       vti.Log2SEW, vti.LMul, vti.RegClass,
 957                                       vti.RegClass, imm_type>;
 958 }
 959
 960 multiclass VPatBinaryV_VI_NoMaskTU<string intrinsic, string instruction,
 961                                    list<VTypeInfo> vtilist,
 962                                    Operand imm_type = tuimm5> {
 963   foreach vti = vtilist in
 964     def : VPatBinaryNoMaskTU<intrinsic, instruction # "_VI_" # vti.LMul.MX,
 965                              vti.Vector, vti.Vector, XLenVT, vti.Log2SEW,
 966                              vti.RegClass, vti.RegClass, imm_type>;
 967 }
 968
 969 multiclass VPatBinaryV_VV_NoMaskTU<string intrinsic, string instruction,
 970                                    list<VTypeInfo> vtilist> {
 971   foreach vti = vtilist in
 972     def : VPatBinaryNoMaskTU<intrinsic, instruction # "_VV_" # vti.LMul.MX,
 973                              vti.Vector, vti.Vector, vti.Vector, vti.Log2SEW,
 974                              vti.RegClass, vti.RegClass, vti.RegClass>;
 975 }
 976
 977 multiclass VPatBinaryV_VX_VROTATE<string intrinsic, string instruction,
 978                                   list<VTypeInfo> vtilist, bit isSEWAware = 0> {
 979   foreach vti = vtilist in {
 980     defvar kind = "V"#vti.ScalarSuffix;
 981     let Predicates = GetVTypePredicates<vti>.Predicates in
 982     defm : VPatBinary<intrinsic,
 983                       !if(isSEWAware,
 984                           instruction#"_"#kind#"_"#vti.LMul.MX#"_E"#vti.SEW,
 985                           instruction#"_"#kind#"_"#vti.LMul.MX),
 986                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
 987                       vti.Log2SEW, vti.RegClass,
 988                       vti.RegClass, vti.ScalarRegClass>;
 989   }
 990 }
 991
 992 multiclass VPatBinaryV_VI_VROL<string intrinsic, string instruction,
 993                                list<VTypeInfo> vtilist, bit isSEWAware = 0> {
 994   foreach vti = vtilist in {
 995     defvar Intr = !cast<Intrinsic>(intrinsic);
 996     defvar Pseudo = !cast<Instruction>(
 997         !if(isSEWAware, instruction#"_VI_"#vti.LMul.MX#"_E"#vti.SEW,
 998                         instruction#"_VI_"#vti.LMul.MX));
 999     let Predicates = GetVTypePredicates<vti>.Predicates in
1000     def : Pat<(vti.Vector (Intr (vti.Vector vti.RegClass:$merge),
1001                           (vti.Vector vti.RegClass:$rs2),
1002                           (XLenVT uimm6:$rs1),
1003                           VLOpFrag)),
1004                           (Pseudo (vti.Vector vti.RegClass:$merge),
1005                           (vti.Vector vti.RegClass:$rs2),
1006                           (InvRot64Imm uimm6:$rs1),
1007                           GPR:$vl, vti.Log2SEW, TU_MU)>;
1008
1009     defvar IntrMask = !cast<Intrinsic>(intrinsic#"_mask");
1010     defvar PseudoMask = !cast<Instruction>(
1011         !if(isSEWAware, instruction#"_VI_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK",
1012                         instruction#"_VI_"#vti.LMul.MX#"_MASK"));
1013     let Predicates = GetVTypePredicates<vti>.Predicates in
1014     def : Pat<(vti.Vector (IntrMask (vti.Vector vti.RegClass:$merge),
1015                           (vti.Vector vti.RegClass:$rs2),
1016                           (XLenVT uimm6:$rs1),
1017                           (vti.Mask V0),
1018                           VLOpFrag, (XLenVT timm:$policy))),
1019                           (PseudoMask (vti.Vector vti.RegClass:$merge),
1020                           (vti.Vector vti.RegClass:$rs2),
1021                           (InvRot64Imm uimm6:$rs1),
1022                           (vti.Mask V0),
1023                           GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>;
1024     }
1025 }
1026
1027 multiclass VPatBinaryV_VV_VX_VROL<string intrinsic, string instruction,
1028                                   string instruction2, list<VTypeInfo> vtilist>
1029     : VPatBinaryV_VV<intrinsic, instruction, vtilist>,
1030       VPatBinaryV_VX_VROTATE<intrinsic, instruction, vtilist>,
1031       VPatBinaryV_VI_VROL<intrinsic, instruction2, vtilist>;
1032
1033 multiclass VPatBinaryV_VV_VX_VI_VROR<string intrinsic, string instruction,
1034                                      list<VTypeInfo> vtilist,
1035                                      Operand ImmType = uimm6>
1036     : VPatBinaryV_VV<intrinsic, instruction, vtilist>,
1037       VPatBinaryV_VX_VROTATE<intrinsic, instruction, vtilist>,
1038       VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>;
1039
1040 multiclass VPatBinaryW_VI_VWSLL<string intrinsic, string instruction,
1041                                 list<VTypeInfoToWide> vtilist> {
1042   foreach VtiToWti = vtilist in {
1043     defvar Vti = VtiToWti.Vti;
1044     defvar Wti = VtiToWti.Wti;
1045     defm : VPatBinary<intrinsic, instruction # "_VI_" # Vti.LMul.MX,
1046                       Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
1047                       Vti.Log2SEW, Wti.RegClass,
1048                       Vti.RegClass, uimm5>;
1049   }
1050 }
1051
1052 multiclass VPatBinaryW_VX_VWSLL<string intrinsic, string instruction,
1053                                 list<VTypeInfoToWide> vtilist> {
1054   foreach VtiToWti = vtilist in {
1055     defvar Vti = VtiToWti.Vti;
1056     defvar Wti = VtiToWti.Wti;
1057     defvar kind = "V"#Vti.ScalarSuffix;
1058     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
1059                                  GetVTypePredicates<Wti>.Predicates) in
1060     defm : VPatBinary<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX,
1061                       Wti.Vector, Vti.Vector, XLenVT, Vti.Mask,
1062                       Vti.Log2SEW, Wti.RegClass,
1063                       Vti.RegClass, Vti.ScalarRegClass>;
1064   }
1065 }
1066
1067 multiclass VPatBinaryW_VV_VX_VI_VWSLL<string intrinsic, string instruction,
1068                                       list<VTypeInfoToWide> vtilist>
1069     : VPatBinaryW_VV<intrinsic, instruction, vtilist>,
1070       VPatBinaryW_VX_VWSLL<intrinsic, instruction, vtilist>,
1071       VPatBinaryW_VI_VWSLL<intrinsic, instruction, vtilist>;
1072
1073 let Predicates = [HasStdExtZvbb] in {
1074   defm : VPatUnaryV_V<"int_riscv_vbrev", "PseudoVBREV", AllIntegerVectors>;
1075   defm : VPatUnaryV_V<"int_riscv_vclz", "PseudoVCLZ", AllIntegerVectors>;
1076   defm : VPatUnaryV_V<"int_riscv_vctz", "PseudoVCTZ", AllIntegerVectors>;
1077   defm : VPatUnaryV_V<"int_riscv_vcpopv", "PseudoVCPOP", AllIntegerVectors>;
1078   defm : VPatBinaryW_VV_VX_VI_VWSLL<"int_riscv_vwsll", "PseudoVWSLL", AllWidenableIntVectors>;
1079 } // Predicates = [HasStdExtZvbb]
1080
1081 let Predicates = [HasStdExtZvbc] in {
1082   defm : VPatBinaryV_VV_VX<"int_riscv_vclmul", "PseudoVCLMUL", I64IntegerVectors>;
1083   defm : VPatBinaryV_VV_VX<"int_riscv_vclmulh", "PseudoVCLMULH", I64IntegerVectors>;
1084 } // Predicates = [HasStdExtZvbc]
1085
1086 let Predicates = [HasStdExtZvkb] in {
1087   defm : VPatBinaryV_VV_VX<"int_riscv_vandn", "PseudoVANDN", AllIntegerVectors>;
1088   defm : VPatUnaryV_V<"int_riscv_vbrev8", "PseudoVBREV8", AllIntegerVectors>;
1089   defm : VPatUnaryV_V<"int_riscv_vrev8", "PseudoVREV8", AllIntegerVectors>;
1090   defm : VPatBinaryV_VV_VX_VROL<"int_riscv_vrol", "PseudoVROL", "PseudoVROR", AllIntegerVectors>;
1091   defm : VPatBinaryV_VV_VX_VI_VROR<"int_riscv_vror", "PseudoVROR", AllIntegerVectors>;
1092 } // Predicates = [HasStdExtZvkb]
1093
1094 let Predicates = [HasStdExtZvkg] in {
1095   defm : VPatBinaryV_VV_NoMask<"int_riscv_vghsh", "PseudoVGHSH", I32IntegerVectors>;
1096   defm : VPatUnaryV_V_NoMask_Zvk<"int_riscv_vgmul", "PseudoVGMUL", I32IntegerVectors>;
1097 } // Predicates = [HasStdExtZvkg]
1098
1099 let Predicates = [HasStdExtZvkned] in {
1100   defm : VPatUnaryV_V_S_NoMask_Zvk<"int_riscv_vaesdf", "PseudoVAESDF", I32IntegerVectors>;
1101   defm : VPatUnaryV_V_S_NoMask_Zvk<"int_riscv_vaesdm", "PseudoVAESDM", I32IntegerVectors>;
1102   defm : VPatUnaryV_V_S_NoMask_Zvk<"int_riscv_vaesef", "PseudoVAESEF", I32IntegerVectors>;
1103   defm : VPatUnaryV_V_S_NoMask_Zvk<"int_riscv_vaesem", "PseudoVAESEM", I32IntegerVectors>;
1104   defm : VPatBinaryV_VI_NoMaskTU<"int_riscv_vaeskf1", "PseudoVAESKF1", I32IntegerVectors>;
1105   defm : VPatBinaryV_VI_NoMask<"int_riscv_vaeskf2", "PseudoVAESKF2", I32IntegerVectors>;
1106   defm : VPatUnaryV_S_NoMaskVectorCrypto<"int_riscv_vaesz", "PseudoVAESZ", I32IntegerVectors>;
1107 } // Predicates = [HasStdExtZvkned]
1108
1109 let Predicates = [HasStdExtZvknha] in {
1110   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ch", "PseudoVSHA2CH", I32IntegerVectors>;
1111   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CH", I32IntegerVectors>;
1112   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ms", "PseudoVSHA2MS", I32IntegerVectors>;
1113 } // Predicates = [HasStdExtZvknha]
1114
1115 let Predicates = [HasStdExtZvknhb] in {
1116   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ch", "PseudoVSHA2CH", I32I64IntegerVectors>;
1117   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2cl", "PseudoVSHA2CH", I32I64IntegerVectors>;
1118   defm : VPatBinaryV_VV_NoMask<"int_riscv_vsha2ms", "PseudoVSHA2MS", I32I64IntegerVectors>;
1119 } // Predicates = [HasStdExtZvknhb]
1120
1121 let Predicates = [HasStdExtZvksed] in {
1122   defm : VPatBinaryV_VI_NoMaskTU<"int_riscv_vsm4k", "PseudoVSM4K", I32IntegerVectors>;
1123   defm : VPatUnaryV_V_S_NoMask_Zvk<"int_riscv_vsm4r", "PseudoVSM4R", I32IntegerVectors>;
1124 } // Predicates = [HasStdExtZvksed]
1125
1126 let Predicates = [HasStdExtZvksh] in {
1127   defm : VPatBinaryV_VI_NoMask<"int_riscv_vsm3c", "PseudoVSM3C", I32IntegerVectors>;
1128   defm : VPatBinaryV_VV_NoMaskTU<"int_riscv_vsm3me", "PseudoVSM3ME", I32IntegerVectors>;
1129 } // Predicates = [HasStdExtZvksh]