llvm/lib/Target/AArch64/AArch64SchedCyclone.td

   1 //=- AArch64SchedCyclone.td - Cyclone Scheduling Definitions -*- tablegen -*-=//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines the machine model for AArch64 Cyclone to support
  10 // instruction scheduling and other instruction cost heuristics.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 def CycloneModel : SchedMachineModel {
  15   let IssueWidth = 6; // 6 micro-ops are dispatched per cycle.
  16   let MicroOpBufferSize = 192; // Based on the reorder buffer.
  17   let LoadLatency = 4; // Optimistic load latency.
  18   let MispredictPenalty = 16; // 14-19 cycles are typical.
  19   let CompleteModel = 1;
  20
  21   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
  22                                                     PAUnsupported.F,
  23                                                     SMEUnsupported.F,
  24                                                     [HasMTE, HasCSSC]);
  25 }
  26
  27 //===----------------------------------------------------------------------===//
  28 // Define each kind of processor resource and number available on Cyclone.
  29
  30 // 4 integer pipes
  31 def CyUnitI : ProcResource<4> {
  32   let BufferSize = 48;
  33 }
  34
  35 // 2 branch units: I[0..1]
  36 def CyUnitB : ProcResource<2> {
  37   let Super  = CyUnitI;
  38   let BufferSize = 24;
  39 }
  40
  41 // 1 indirect-branch unit: I[0]
  42 def CyUnitBR : ProcResource<1> {
  43   let Super  = CyUnitB;
  44 }
  45
  46 // 2 shifter pipes: I[2..3]
  47 // When an instruction consumes a CyUnitIS, it also consumes a CyUnitI
  48 def CyUnitIS : ProcResource<2> {
  49   let Super = CyUnitI;
  50   let BufferSize = 24;
  51 }
  52
  53 // 1 mul pipe: I[0]
  54 def CyUnitIM : ProcResource<1> {
  55   let Super = CyUnitBR;
  56   let BufferSize = 32;
  57 }
  58
  59 // 1 div pipe: I[1]
  60 def CyUnitID : ProcResource<1> {
  61   let Super = CyUnitB;
  62   let BufferSize = 16;
  63 }
  64
  65 // 1 integer division unit. This is driven by the ID pipe, but only
  66 // consumes the pipe for one cycle at issue and another cycle at writeback.
  67 def CyUnitIntDiv : ProcResource<1>;
  68
  69 // 2 ld/st pipes.
  70 def CyUnitLS : ProcResource<2> {
  71   let BufferSize = 28;
  72 }
  73
  74 // 3 fp/vector pipes.
  75 def CyUnitV : ProcResource<3> {
  76   let BufferSize = 48;
  77 }
  78 // 2 fp/vector arithmetic and multiply pipes: V[0-1]
  79 def CyUnitVM : ProcResource<2> {
  80   let Super = CyUnitV;
  81   let BufferSize = 32;
  82 }
  83 // 1 fp/vector division/sqrt pipe: V[2]
  84 def CyUnitVD : ProcResource<1> {
  85   let Super = CyUnitV;
  86   let BufferSize = 16;
  87 }
  88 // 1 fp compare pipe: V[0]
  89 def CyUnitVC : ProcResource<1> {
  90   let Super = CyUnitVM;
  91   let BufferSize = 16;
  92 }
  93
  94 // 2 fp division/square-root units.  These are driven by the VD pipe,
  95 // but only consume the pipe for one cycle at issue and a cycle at writeback.
  96 def CyUnitFloatDiv : ProcResource<2>;
  97
  98 //===----------------------------------------------------------------------===//
  99 // Define scheduler read/write resources and latency on Cyclone.
 100 // This mirrors sections 7.7-7.9 of the Tuning Guide v1.0.1.
 101
 102 let SchedModel = CycloneModel in {
 103
 104 //---
 105 // 7.8.1. Moves
 106 //---
 107
 108 // A single nop micro-op (uX).
 109 def WriteX : SchedWriteRes<[]> { let Latency = 0; }
 110
 111 // Move zero is a register rename (to machine register zero).
 112 // The move is replaced by a single nop micro-op.
 113 // MOVZ Rd, #0
 114 // AND Rd, Rzr, #imm
 115 def WriteZPred : SchedPredicate<[{TII->isGPRZero(*MI)}]>;
 116 def WriteImmZ  : SchedWriteVariant<[
 117                    SchedVar<WriteZPred, [WriteX]>,
 118                    SchedVar<NoSchedPred, [WriteImm]>]>;
 119 def : InstRW<[WriteImmZ], (instrs MOVZWi,MOVZXi,ANDWri,ANDXri)>;
 120
 121 // Move GPR is a register rename and single nop micro-op.
 122 // ORR Xd, XZR, Xm
 123 // ADD Xd, Xn, #0
 124 def WriteIMovPred : SchedPredicate<[{TII->isGPRCopy(*MI)}]>;
 125 def WriteVMovPred : SchedPredicate<[{TII->isFPRCopy(*MI)}]>;
 126 def WriteMov      : SchedWriteVariant<[
 127                       SchedVar<WriteIMovPred, [WriteX]>,
 128                       SchedVar<WriteVMovPred, [WriteX]>,
 129                       SchedVar<NoSchedPred,   [WriteI]>]>;
 130 def : InstRW<[WriteMov], (instrs COPY,ORRXrr,ADDXrr)>;
 131
 132 // Move non-zero immediate is an integer ALU op.
 133 // MOVN,MOVZ,MOVK
 134 def : WriteRes<WriteImm, [CyUnitI]>;
 135
 136 //---
 137 // 7.8.2-7.8.5. Arithmetic and Logical, Comparison, Conditional,
 138 //              Shifts and Bitfield Operations
 139 //---
 140
 141 // ADR,ADRP
 142 // ADD(S)ri,SUB(S)ri,AND(S)ri,EORri,ORRri
 143 // ADD(S)rr,SUB(S)rr,AND(S)rr,BIC(S)rr,EONrr,EORrr,ORNrr,ORRrr
 144 // ADC(S),SBC(S)
 145 // Aliases: CMN, CMP, TST
 146 //
 147 // Conditional operations.
 148 // CCMNi,CCMPi,CCMNr,CCMPr,
 149 // CSEL,CSINC,CSINV,CSNEG
 150 //
 151 // Bit counting and reversal operations.
 152 // CLS,CLZ,RBIT,REV,REV16,REV32
 153 def : WriteRes<WriteI, [CyUnitI]>;
 154
 155 // ADD with shifted register operand is a single micro-op that
 156 // consumes a shift pipeline for two cycles.
 157 // ADD(S)rs,SUB(S)rs,AND(S)rs,BIC(S)rs,EONrs,EORrs,ORNrs,ORRrs
 158 // EXAMPLE: ADDrs Xn, Xm LSL #imm
 159 def : WriteRes<WriteISReg, [CyUnitIS]> {
 160   let Latency = 2;
 161   let ReleaseAtCycles = [2];
 162 }
 163
 164 // ADD with extended register operand is the same as shifted reg operand.
 165 // ADD(S)re,SUB(S)re
 166 // EXAMPLE: ADDXre Xn, Xm, UXTB #1
 167 def : WriteRes<WriteIEReg, [CyUnitIS]> {
 168   let Latency = 2;
 169   let ReleaseAtCycles = [2];
 170 }
 171
 172 // Variable shift and bitfield operations.
 173 // ASRV,LSLV,LSRV,RORV,BFM,SBFM,UBFM
 174 def : WriteRes<WriteIS, [CyUnitIS]>;
 175
 176 // EXTR Shifts a pair of registers and requires two micro-ops.
 177 // The second micro-op is delayed, as modeled by ReadExtrHi.
 178 // EXTR Xn, Xm, #imm
 179 def : WriteRes<WriteExtr, [CyUnitIS, CyUnitIS]> {
 180   let Latency = 2;
 181   let NumMicroOps = 2;
 182 }
 183
 184 // EXTR's first register read is delayed by one cycle, effectively
 185 // shortening its writer's latency.
 186 // EXTR Xn, Xm, #imm
 187 def : ReadAdvance<ReadExtrHi, 1>;
 188
 189 //---
 190 // 7.8.6. Multiplies
 191 //---
 192
 193 // MUL/MNEG are aliases for MADD/MSUB.
 194 // MADDW,MSUBW,SMADDL,SMSUBL,UMADDL,UMSUBL
 195 def : WriteRes<WriteIM32, [CyUnitIM]> {
 196   let Latency = 4;
 197 }
 198 // MADDX,MSUBX,SMULH,UMULH
 199 def : WriteRes<WriteIM64, [CyUnitIM]> {
 200   let Latency = 5;
 201 }
 202
 203 //---
 204 // 7.8.7. Divide
 205 //---
 206
 207 // 32-bit divide takes 7-13 cycles. 10 cycles covers a 20-bit quotient.
 208 // The ID pipe is consumed for 2 cycles: issue and writeback.
 209 // SDIVW,UDIVW
 210 def : WriteRes<WriteID32, [CyUnitID, CyUnitIntDiv]> {
 211   let Latency = 10;
 212   let ReleaseAtCycles = [2, 10];
 213 }
 214 // 64-bit divide takes 7-21 cycles. 13 cycles covers a 32-bit quotient.
 215 // The ID pipe is consumed for 2 cycles: issue and writeback.
 216 // SDIVX,UDIVX
 217 def : WriteRes<WriteID64, [CyUnitID, CyUnitIntDiv]> {
 218   let Latency = 13;
 219   let ReleaseAtCycles = [2, 13];
 220 }
 221
 222 //---
 223 // 7.8.8,7.8.10. Load/Store, single element
 224 //---
 225
 226 // Integer loads take 4 cycles and use one LS unit for one cycle.
 227 def : WriteRes<WriteLD, [CyUnitLS]> {
 228   let Latency = 4;
 229 }
 230
 231 // Store-load forwarding is 4 cycles.
 232 //
 233 // Note: The store-exclusive sequence incorporates this
 234 // latency. However, general heuristics should not model the
 235 // dependence between a store and subsequent may-alias load because
 236 // hardware speculation works.
 237 def : WriteRes<WriteST, [CyUnitLS]> {
 238   let Latency = 4;
 239 }
 240
 241 // Load from base address plus an optionally scaled register offset.
 242 // Rt latency is latency WriteIS + WriteLD.
 243 // EXAMPLE: LDR Xn, Xm [, lsl 3]
 244 def CyWriteLDIdx : SchedWriteVariant<[
 245   SchedVar<ScaledIdxPred, [WriteIS, WriteLD]>, // Load from scaled register.
 246   SchedVar<NoSchedPred,   [WriteLD]>]>;        // Load from register offset.
 247 def : SchedAlias<WriteLDIdx, CyWriteLDIdx>;    // Map AArch64->Cyclone type.
 248
 249 // EXAMPLE: STR Xn, Xm [, lsl 3]
 250 def CyWriteSTIdx : SchedWriteVariant<[
 251   SchedVar<ScaledIdxPred, [WriteIS, WriteST]>, // Store to scaled register.
 252   SchedVar<NoSchedPred,   [WriteST]>]>;        // Store to register offset.
 253 def : SchedAlias<WriteSTIdx, CyWriteSTIdx>;    // Map AArch64->Cyclone type.
 254
 255 // Read the (unshifted) base register Xn in the second micro-op one cycle later.
 256 // EXAMPLE: LDR Xn, Xm [, lsl 3]
 257 def ReadBaseRS : SchedReadAdvance<1>;
 258 def CyReadAdrBase : SchedReadVariant<[
 259   SchedVar<ScaledIdxPred, [ReadBaseRS]>, // Read base reg after shifting offset.
 260   SchedVar<NoSchedPred,   [ReadDefault]>]>;   // Read base reg with no shift.
 261 def : SchedAlias<ReadAdrBase, CyReadAdrBase>; // Map AArch64->Cyclone type.
 262 def : ReadAdvance<ReadST, 0>;
 263
 264 //---
 265 // 7.8.9,7.8.11. Load/Store, paired
 266 //---
 267
 268 // Address pre/post increment is a simple ALU op with one cycle latency.
 269 def : WriteRes<WriteAdr, [CyUnitI]>;
 270
 271 // LDP high register write is fused with the load, but a nop micro-op remains.
 272 def : WriteRes<WriteLDHi, []> {
 273   let Latency = 4;
 274 }
 275
 276 // STP is a vector op and store, except for QQ, which is just two stores.
 277 def : SchedAlias<WriteSTP, WriteVSTShuffle>;
 278 def : InstRW<[WriteST, WriteST], (instrs STPQi)>;
 279
 280 //---
 281 // 7.8.13. Branches
 282 //---
 283
 284 // Branches take a single micro-op.
 285 // The misprediction penalty is defined as a SchedMachineModel property.
 286 def : WriteRes<WriteBr,    [CyUnitB]>  {let Latency = 0;}
 287 def : WriteRes<WriteBrReg, [CyUnitBR]> {let Latency = 0;}
 288
 289 //---
 290 // 7.8.14. Never-issued Instructions, Barrier and Hint Operations
 291 //---
 292
 293 // NOP,SEV,SEVL,WFE,WFI,YIELD
 294 def : WriteRes<WriteHint, []> {let Latency = 0;}
 295 // ISB
 296 def : InstRW<[WriteI], (instrs ISB)>;
 297 // SLREX,DMB,DSB
 298 def : WriteRes<WriteBarrier, [CyUnitLS]>;
 299
 300 // System instructions get an invalid latency because the latency of
 301 // other operations across them is meaningless.
 302 def : WriteRes<WriteSys, []> {let Latency = -1;}
 303
 304 //===----------------------------------------------------------------------===//
 305 // 7.9 Vector Unit Instructions
 306
 307 // Simple vector operations take 2 cycles.
 308 def : WriteRes<WriteVd, [CyUnitV]> {let Latency = 2;}
 309 def : WriteRes<WriteVq, [CyUnitV]> {let Latency = 2;}
 310
 311 // Define some longer latency vector op types for Cyclone.
 312 def CyWriteV3 : SchedWriteRes<[CyUnitV]> {let Latency = 3;}
 313 def CyWriteV4 : SchedWriteRes<[CyUnitV]> {let Latency = 4;}
 314 def CyWriteV5 : SchedWriteRes<[CyUnitV]> {let Latency = 5;}
 315 def CyWriteV6 : SchedWriteRes<[CyUnitV]> {let Latency = 6;}
 316
 317 // Simple floating-point operations take 2 cycles.
 318 def : WriteRes<WriteF, [CyUnitV]> {let Latency = 2;}
 319
 320 //---
 321 // 7.9.1 Vector Moves
 322 //---
 323
 324 // TODO: Add Cyclone-specific zero-cycle zeros. LLVM currently
 325 // generates expensive int-float conversion instead:
 326 // FMOVDi Dd, #0.0
 327 // FMOVv2f64ns Vd.2d, #0.0
 328
 329 // FMOVSi,FMOVDi
 330 def : WriteRes<WriteFImm, [CyUnitV]> {let Latency = 2;}
 331
 332 // MOVI,MVNI are WriteV
 333 // FMOVv2f32ns,FMOVv2f64ns,FMOVv4f32ns are WriteV
 334
 335 // Move FPR is a register rename and single nop micro-op.
 336 // ORR.16b Vd,Vn,Vn
 337 // COPY is handled above in the WriteMov Variant.
 338 def WriteVMov    : SchedWriteVariant<[
 339                      SchedVar<WriteVMovPred, [WriteX]>,
 340                      SchedVar<NoSchedPred,   [WriteVq]>]>;
 341 def : InstRW<[WriteVMov], (instrs ORRv16i8)>;
 342
 343 // FMOVSr,FMOVDr are WriteF.
 344
 345 // MOV V,V is a WriteV.
 346
 347 // CPY D,V[x] is a WriteV
 348
 349 // INS V[x],V[y] is a WriteV.
 350
 351 // FMOVWSr,FMOVXDr,FMOVXDHighr
 352 def : WriteRes<WriteFCopy, [CyUnitLS]> {
 353   let Latency = 5;
 354 }
 355
 356 // FMOVSWr,FMOVDXr
 357 def : InstRW<[WriteLD], (instrs FMOVSWr,FMOVDXr,FMOVDXHighr)>;
 358
 359 // INS V[x],R
 360 def CyWriteCopyToFPR : WriteSequence<[WriteVLD, WriteVq]>;
 361 def : InstRW<[CyWriteCopyToFPR], (instregex "INSv")>;
 362
 363 // SMOV,UMOV R,V[x]
 364 def CyWriteCopyToGPR : WriteSequence<[WriteLD, WriteI]>;
 365 def : InstRW<[CyWriteCopyToGPR], (instregex "SMOVv","UMOVv")>;
 366
 367 // DUP V,R
 368 def : InstRW<[CyWriteCopyToFPR], (instregex "DUPv")>;
 369
 370 // DUP V,V[x] is a WriteV.
 371
 372 //---
 373 // 7.9.2 Integer Arithmetic, Logical, and Comparisons
 374 //---
 375
 376 // BIC,ORR V,#imm are WriteV
 377
 378 def : InstRW<[CyWriteV3], (instregex "ABSv")>;
 379
 380 // MVN,NEG,NOT are WriteV
 381
 382 def : InstRW<[CyWriteV3], (instregex "SQABSv","SQNEGv")>;
 383
 384 // ADDP is a WriteV.
 385 def CyWriteVADDLP : SchedWriteRes<[CyUnitV]> {let Latency = 2;}
 386 def : InstRW<[CyWriteVADDLP], (instregex "SADDLPv","UADDLPv")>;
 387
 388 def : InstRW<[CyWriteV3],
 389              (instregex "ADDVv","SMAXVv","UMAXVv","SMINVv","UMINVv")>;
 390
 391 def : InstRW<[CyWriteV3], (instregex "SADDLV","UADDLV")>;
 392
 393 // ADD,SUB are WriteV
 394
 395 // Forward declare.
 396 def CyWriteVABD : SchedWriteRes<[CyUnitV]> {let Latency = 3;}
 397
 398 // Add/Diff and accumulate uses the vector multiply unit.
 399 def CyWriteVAccum : SchedWriteRes<[CyUnitVM]> {let Latency = 3;}
 400 def CyReadVAccum  : SchedReadAdvance<1,
 401                     [CyWriteVAccum, CyWriteVADDLP, CyWriteVABD]>;
 402
 403 def : InstRW<[CyWriteVAccum, CyReadVAccum],
 404              (instregex "SADALP","UADALP")>;
 405
 406 def : InstRW<[CyWriteVAccum, CyReadVAccum],
 407              (instregex "SABAv","UABAv","SABALv","UABALv")>;
 408
 409 def : InstRW<[CyWriteV3], (instregex "SQADDv","SQSUBv","UQADDv","UQSUBv")>;
 410
 411 def : InstRW<[CyWriteV3], (instregex "SUQADDv","USQADDv")>;
 412
 413 def : InstRW<[CyWriteV4], (instregex "ADDHNv","RADDHNv", "RSUBHNv", "SUBHNv")>;
 414
 415 // WriteV includes:
 416 // AND,BIC,CMTST,EOR,ORN,ORR
 417 // ADDP
 418 // SHADD,SHSUB,SRHADD,UHADD,UHSUB,URHADD
 419 // SADDL,SSUBL,UADDL,USUBL
 420 // SADDW,SSUBW,UADDW,USUBW
 421
 422 def : InstRW<[CyWriteV3], (instregex "CMEQv","CMGEv","CMGTv",
 423                                      "CMLEv","CMLTv",
 424                                      "CMHIv","CMHSv")>;
 425
 426 def : InstRW<[CyWriteV3], (instregex "SMAXv","SMINv","UMAXv","UMINv",
 427                                      "SMAXPv","SMINPv","UMAXPv","UMINPv")>;
 428
 429 def : InstRW<[CyWriteVABD], (instregex "SABDv","UABDv",
 430                                        "SABDLv","UABDLv")>;
 431
 432 //---
 433 // 7.9.3 Floating Point Arithmetic and Comparisons
 434 //---
 435
 436 // FABS,FNEG are WriteF
 437
 438 def : InstRW<[CyWriteV4], (instrs FADDPv2i32p)>;
 439 def : InstRW<[CyWriteV5], (instrs FADDPv2i64p)>;
 440
 441 def : InstRW<[CyWriteV3], (instregex "FMAXPv2i","FMAXNMPv2i",
 442                                      "FMINPv2i","FMINNMPv2i")>;
 443
 444 def : InstRW<[CyWriteV4], (instregex "FMAXVv","FMAXNMVv","FMINVv","FMINNMVv")>;
 445
 446 def : InstRW<[CyWriteV4], (instrs FADDSrr,FADDv2f32,FADDv4f32,
 447                                   FSUBSrr,FSUBv2f32,FSUBv4f32,
 448                                   FADDPv2f32,FADDPv4f32,
 449                                   FABD32,FABDv2f32,FABDv4f32)>;
 450 def : InstRW<[CyWriteV5], (instrs FADDDrr,FADDv2f64,
 451                                   FSUBDrr,FSUBv2f64,
 452                                   FADDPv2f64,
 453                                   FABD64,FABDv2f64)>;
 454
 455 def : InstRW<[CyWriteV3], (instregex "FCMEQ","FCMGT","FCMLE","FCMLT")>;
 456
 457 def : InstRW<[CyWriteV3], (instregex "FACGE","FACGT",
 458                                      "FMAXS","FMAXD","FMAXv",
 459                                      "FMINS","FMIND","FMINv",
 460                                      "FMAXNMS","FMAXNMD","FMAXNMv",
 461                                      "FMINNMS","FMINNMD","FMINNMv",
 462                                      "FMAXPv2f","FMAXPv4f",
 463                                      "FMINPv2f","FMINPv4f",
 464                                      "FMAXNMPv2f","FMAXNMPv4f",
 465                                      "FMINNMPv2f","FMINNMPv4f")>;
 466
 467 // FCMP,FCMPE,FCCMP,FCCMPE
 468 def : WriteRes<WriteFCmp, [CyUnitVC]> {let Latency = 4;}
 469
 470 // FCSEL is a WriteF.
 471
 472 //---
 473 // 7.9.4 Shifts and Bitfield Operations
 474 //---
 475
 476 // SHL is a WriteV
 477
 478 def CyWriteVSHR : SchedWriteRes<[CyUnitV]> {let Latency = 2;}
 479 def : InstRW<[CyWriteVSHR], (instregex "SSHRv","USHRv")>;
 480
 481 def CyWriteVSRSHR : SchedWriteRes<[CyUnitV]> {let Latency = 3;}
 482 def : InstRW<[CyWriteVSRSHR], (instregex "SRSHRv","URSHRv")>;
 483
 484 // Shift and accumulate uses the vector multiply unit.
 485 def CyWriteVShiftAcc : SchedWriteRes<[CyUnitVM]> {let Latency = 3;}
 486 def CyReadVShiftAcc  : SchedReadAdvance<1,
 487                         [CyWriteVShiftAcc, CyWriteVSHR, CyWriteVSRSHR]>;
 488 def : InstRW<[CyWriteVShiftAcc, CyReadVShiftAcc],
 489              (instregex "SRSRAv","SSRAv","URSRAv","USRAv")>;
 490
 491 // SSHL,USHL are WriteV.
 492
 493 def : InstRW<[CyWriteV3], (instregex "SRSHLv","URSHLv")>;
 494
 495 // SQSHL,SQSHLU,UQSHL are WriteV.
 496
 497 def : InstRW<[CyWriteV3], (instregex "SQRSHLv","UQRSHLv")>;
 498
 499 // WriteV includes:
 500 // SHLL,SSHLL,USHLL
 501 // SLI,SRI
 502 // BIF,BIT,BSL,BSP
 503 // EXT
 504 // CLS,CLZ,CNT,RBIT,REV16,REV32,REV64,XTN
 505 // XTN2
 506
 507 def : InstRW<[CyWriteV4],
 508              (instregex "RSHRNv","SHRNv",
 509                         "SQRSHRNv","SQRSHRUNv","SQSHRNv","SQSHRUNv",
 510                         "UQRSHRNv","UQSHRNv","SQXTNv","SQXTUNv","UQXTNv")>;
 511
 512 //---
 513 // 7.9.5 Multiplication
 514 //---
 515
 516 def CyWriteVMul : SchedWriteRes<[CyUnitVM]> { let Latency = 4;}
 517 def : InstRW<[CyWriteVMul], (instregex "MULv","SMULLv","UMULLv",
 518                              "SQDMULLv","SQDMULHv","SQRDMULHv")>;
 519
 520 // FMUL,FMULX,FNMUL default to WriteFMul.
 521 def : WriteRes<WriteFMul, [CyUnitVM]> { let Latency = 4;}
 522
 523 def CyWriteV64Mul : SchedWriteRes<[CyUnitVM]> { let Latency = 5;}
 524 def : InstRW<[CyWriteV64Mul], (instrs FMULDrr,FMULv2f64,FMULv2i64_indexed,
 525                                FNMULDrr,FMULX64,FMULXv2f64,FMULXv2i64_indexed)>;
 526
 527 def CyReadVMulAcc : SchedReadAdvance<1, [CyWriteVMul, CyWriteV64Mul]>;
 528 def : InstRW<[CyWriteVMul, CyReadVMulAcc],
 529              (instregex "MLA","MLS","SMLAL","SMLSL","UMLAL","UMLSL",
 530               "SQDMLAL","SQDMLSL")>;
 531
 532 def CyWriteSMul : SchedWriteRes<[CyUnitVM]> { let Latency = 8;}
 533 def CyWriteDMul : SchedWriteRes<[CyUnitVM]> { let Latency = 10;}
 534 def CyReadSMul : SchedReadAdvance<4, [CyWriteSMul]>;
 535 def CyReadDMul : SchedReadAdvance<5, [CyWriteDMul]>;
 536
 537 def : InstRW<[CyWriteSMul, CyReadSMul],
 538              (instrs FMADDSrrr,FMSUBSrrr,FNMADDSrrr,FNMSUBSrrr,
 539               FMLAv2f32,FMLAv4f32,
 540               FMLAv1i32_indexed,FMLAv1i64_indexed,FMLAv2i32_indexed)>;
 541 def : InstRW<[CyWriteDMul, CyReadDMul],
 542              (instrs FMADDDrrr,FMSUBDrrr,FNMADDDrrr,FNMSUBDrrr,
 543               FMLAv2f64,FMLAv2i64_indexed,
 544               FMLSv2f64,FMLSv2i64_indexed)>;
 545
 546 def CyWritePMUL : SchedWriteRes<[CyUnitVD]> { let Latency = 3; }
 547 def : InstRW<[CyWritePMUL], (instregex "PMULv", "PMULLv")>;
 548
 549 //---
 550 // 7.9.6 Divide and Square Root
 551 //---
 552
 553 // FDIV,FSQRT
 554 // TODO: Add 64-bit variant with 19 cycle latency.
 555 // TODO: Specialize FSQRT for longer latency.
 556 def : WriteRes<WriteFDiv, [CyUnitVD, CyUnitFloatDiv]> {
 557   let Latency = 17;
 558   let ReleaseAtCycles = [2, 17];
 559 }
 560
 561 def : InstRW<[CyWriteV4], (instregex "FRECPEv","FRECPXv","URECPEv","URSQRTEv")>;
 562
 563 def WriteFRSQRTE : SchedWriteRes<[CyUnitVM]> { let Latency = 4; }
 564 def : InstRW<[WriteFRSQRTE], (instregex "FRSQRTEv")>;
 565
 566 def WriteFRECPS : SchedWriteRes<[CyUnitVM]> { let Latency = 8; }
 567 def WriteFRSQRTS : SchedWriteRes<[CyUnitVM]> { let Latency = 10; }
 568 def : InstRW<[WriteFRECPS],  (instregex "FRECPSv")>;
 569 def : InstRW<[WriteFRSQRTS], (instregex "FRSQRTSv")>;
 570
 571 //---
 572 // 7.9.7 Integer-FP Conversions
 573 //---
 574
 575 // FCVT lengthen f16/s32
 576 def : InstRW<[WriteVq], (instrs FCVTSHr,FCVTDHr,FCVTDSr)>;
 577
 578 // FCVT,FCVTN,FCVTXN
 579 // SCVTF,UCVTF V,V
 580 // FRINT(AIMNPXZ) V,V
 581 def : WriteRes<WriteFCvt, [CyUnitV]> {let Latency = 4;}
 582
 583 // SCVT/UCVT S/D, Rd = VLD5+V4: 9 cycles.
 584 def CyWriteCvtToFPR : WriteSequence<[WriteVLD, CyWriteV4]>;
 585 def : InstRW<[CyWriteCopyToFPR], (instregex "FCVT[AMNPZ][SU][SU][WX][SD]r")>;
 586
 587 // FCVT Rd, S/D = V6+LD4: 10 cycles
 588 def CyWriteCvtToGPR : WriteSequence<[CyWriteV6, WriteLD]>;
 589 def : InstRW<[CyWriteCvtToGPR], (instregex "[SU]CVTF[SU][WX][SD]r")>;
 590
 591 // FCVTL is a WriteV
 592
 593 //---
 594 // 7.9.8-7.9.10 Cryptography, Data Transposition, Table Lookup
 595 //---
 596
 597 def CyWriteCrypto2 : SchedWriteRes<[CyUnitVD]> {let Latency = 2;}
 598 def : InstRW<[CyWriteCrypto2], (instrs AESIMCrr, AESMCrr, SHA1Hrr,
 599                                        AESDrr, AESErr, SHA1SU1rr, SHA256SU0rr,
 600                                        SHA1SU0rrr)>;
 601
 602 def CyWriteCrypto3 : SchedWriteRes<[CyUnitVD]> {let Latency = 3;}
 603 def : InstRW<[CyWriteCrypto3], (instrs SHA256SU1rrr)>;
 604
 605 def CyWriteCrypto6 : SchedWriteRes<[CyUnitVD]> {let Latency = 6;}
 606 def : InstRW<[CyWriteCrypto6], (instrs SHA1Crrr, SHA1Mrrr, SHA1Prrr,
 607                                        SHA256Hrrr,SHA256H2rrr)>;
 608
 609 // TRN,UZP,ZUP are WriteV.
 610
 611 // TBL,TBX are WriteV.
 612
 613 //---
 614 // 7.9.11-7.9.14 Load/Store, single element and paired
 615 //---
 616
 617 // Loading into the vector unit takes 5 cycles vs 4 for integer loads.
 618 def : WriteRes<WriteVLD, [CyUnitLS]> {
 619   let Latency = 5;
 620 }
 621
 622 // Store-load forwarding is 4 cycles.
 623 def : WriteRes<WriteVST, [CyUnitLS]> {
 624   let Latency = 4;
 625 }
 626
 627 // WriteVLDPair/VSTPair sequences are expanded by the target description.
 628
 629 //---
 630 // 7.9.15 Load, element operations
 631 //---
 632
 633 // Only the first WriteVLD and WriteAdr for writeback matches def operands.
 634 // Subsequent WriteVLDs consume resources. Since all loaded values have the
 635 // same latency, this is acceptable.
 636
 637 // Vd is read 5 cycles after issuing the vector load.
 638 def : ReadAdvance<ReadVLD, 5>;
 639
 640 def : InstRW<[WriteVLD],
 641              (instregex "LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
 642 def : InstRW<[WriteVLD, WriteAdr],
 643              (instregex "LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST")>;
 644
 645 // Register writes from the load's high half are fused micro-ops.
 646 def : InstRW<[WriteVLD],
 647              (instregex "LD1Twov(8b|4h|2s|1d)$")>;
 648 def : InstRW<[WriteVLD, WriteAdr],
 649              (instregex "LD1Twov(8b|4h|2s|1d)_POST")>;
 650 def : InstRW<[WriteVLD, WriteVLD],
 651              (instregex "LD1Twov(16b|8h|4s|2d)$")>;
 652 def : InstRW<[WriteVLD, WriteAdr, WriteVLD],
 653              (instregex "LD1Twov(16b|8h|4s|2d)_POST")>;
 654
 655 def : InstRW<[WriteVLD, WriteVLD],
 656              (instregex "LD1Threev(8b|4h|2s|1d)$")>;
 657 def : InstRW<[WriteVLD, WriteAdr, WriteVLD],
 658              (instregex "LD1Threev(8b|4h|2s|1d)_POST")>;
 659 def : InstRW<[WriteVLD, WriteVLD, WriteVLD],
 660              (instregex "LD1Threev(16b|8h|4s|2d)$")>;
 661 def : InstRW<[WriteVLD, WriteAdr, WriteVLD, WriteVLD],
 662              (instregex "LD1Threev(16b|8h|4s|2d)_POST")>;
 663
 664 def : InstRW<[WriteVLD, WriteVLD],
 665              (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
 666 def : InstRW<[WriteVLD, WriteAdr, WriteVLD],
 667              (instregex "LD1Fourv(8b|4h|2s|1d)_POST")>;
 668 def : InstRW<[WriteVLD, WriteVLD, WriteVLD, WriteVLD],
 669              (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
 670 def : InstRW<[WriteVLD, WriteAdr, WriteVLD, WriteVLD, WriteVLD],
 671              (instregex "LD1Fourv(16b|8h|4s|2d)_POST")>;
 672
 673 def : InstRW<[WriteVLDShuffle, ReadVLD],
 674              (instregex "LD1i(8|16|32)$")>;
 675 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr],
 676              (instregex "LD1i(8|16|32)_POST")>;
 677
 678 def : InstRW<[WriteVLDShuffle, ReadVLD],          (instrs LD1i64)>;
 679 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr],(instrs LD1i64_POST)>;
 680
 681 def : InstRW<[WriteVLDShuffle],
 682              (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
 683 def : InstRW<[WriteVLDShuffle, WriteAdr],
 684              (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 685
 686 def : InstRW<[WriteVLDShuffle, WriteVq],
 687              (instregex "LD2Twov(8b|4h|2s)$")>;
 688 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVq],
 689              (instregex "LD2Twov(8b|4h|2s)_POST$")>;
 690 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle],
 691              (instregex "LD2Twov(16b|8h|4s|2d)$")>;
 692 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle],
 693              (instregex "LD2Twov(16b|8h|4s|2d)_POST")>;
 694
 695 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVq],
 696              (instregex "LD2i(8|16|32)$")>;
 697 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVq],
 698              (instregex "LD2i(8|16|32)_POST")>;
 699 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVq],
 700              (instregex "LD2i64$")>;
 701 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVq],
 702              (instregex "LD2i64_POST")>;
 703
 704 def : InstRW<[WriteVLDShuffle, WriteVq],
 705              (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
 706 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVq],
 707              (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST")>;
 708
 709 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle, WriteVq],
 710              (instregex "LD3Threev(8b|4h|2s)$")>;
 711 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle, WriteVq],
 712              (instregex "LD3Threev(8b|4h|2s)_POST")>;
 713 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle, WriteVLDShuffle],
 714              (instregex "LD3Threev(16b|8h|4s|2d)$")>;
 715 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle, WriteVLDShuffle],
 716              (instregex "LD3Threev(16b|8h|4s|2d)_POST")>;
 717
 718 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVq, WriteVq],
 719              (instregex "LD3i(8|16|32)$")>;
 720 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVq, WriteVq],
 721              (instregex "LD3i(8|16|32)_POST")>;
 722
 723 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVLDShuffle, WriteVq],
 724              (instregex "LD3i64$")>;
 725 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVLDShuffle, WriteVq],
 726              (instregex "LD3i64_POST")>;
 727
 728 def : InstRW<[WriteVLDShuffle, WriteVq, WriteVq],
 729              (instregex "LD3Rv(8b|4h|2s|16b|8h|4s)$")>;
 730 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVq, WriteVq],
 731              (instregex "LD3Rv(8b|4h|2s|16b|8h|4s)_POST")>;
 732
 733 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle, WriteVq],
 734              (instrs LD3Rv1d,LD3Rv2d)>;
 735 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle, WriteVq],
 736              (instrs LD3Rv1d_POST,LD3Rv2d_POST)>;
 737
 738 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle, WriteVq, WriteVq],
 739              (instregex "LD4Fourv(8b|4h|2s)$")>;
 740 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle, WriteVq, WriteVq],
 741              (instregex "LD4Fourv(8b|4h|2s)_POST")>;
 742 def : InstRW<[WriteVLDPairShuffle, WriteVLDPairShuffle,
 743               WriteVLDPairShuffle, WriteVLDPairShuffle],
 744              (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
 745 def : InstRW<[WriteVLDPairShuffle, WriteAdr, WriteVLDPairShuffle,
 746               WriteVLDPairShuffle, WriteVLDPairShuffle],
 747              (instregex "LD4Fourv(16b|8h|4s|2d)_POST")>;
 748
 749 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVq, WriteVq, WriteVq],
 750              (instregex "LD4i(8|16|32)$")>;
 751 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVq, WriteVq, WriteVq],
 752              (instregex "LD4i(8|16|32)_POST")>;
 753
 754
 755 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteVLDShuffle, WriteVq, WriteVq],
 756              (instrs LD4i64)>;
 757 def : InstRW<[WriteVLDShuffle, ReadVLD, WriteAdr, WriteVLDShuffle, WriteVq],
 758              (instrs LD4i64_POST)>;
 759
 760 def : InstRW<[WriteVLDShuffle, WriteVq, WriteVq, WriteVq],
 761              (instregex "LD4Rv(8b|4h|2s|16b|8h|4s)$")>;
 762 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVq, WriteVq, WriteVq],
 763              (instregex "LD4Rv(8b|4h|2s|16b|8h|4s)_POST")>;
 764
 765 def : InstRW<[WriteVLDShuffle, WriteVLDShuffle, WriteVq, WriteVq],
 766              (instrs LD4Rv1d,LD4Rv2d)>;
 767 def : InstRW<[WriteVLDShuffle, WriteAdr, WriteVLDShuffle, WriteVq, WriteVq],
 768              (instrs LD4Rv1d_POST,LD4Rv2d_POST)>;
 769
 770 //---
 771 // 7.9.16 Store, element operations
 772 //---
 773
 774 // Only the WriteAdr for writeback matches a def operands.
 775 // Subsequent WriteVLDs only consume resources.
 776
 777 def : InstRW<[WriteVST],
 778              (instregex "ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
 779 def : InstRW<[WriteAdr, WriteVST],
 780              (instregex "ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST")>;
 781
 782 def : InstRW<[WriteVSTShuffle],
 783              (instregex "ST1Twov(8b|4h|2s|1d)$")>;
 784 def : InstRW<[WriteAdr, WriteVSTShuffle],
 785              (instregex "ST1Twov(8b|4h|2s|1d)_POST")>;
 786 def : InstRW<[WriteVST, WriteVST],
 787              (instregex "ST1Twov(16b|8h|4s|2d)$")>;
 788 def : InstRW<[WriteAdr, WriteVST, WriteVST],
 789              (instregex "ST1Twov(16b|8h|4s|2d)_POST")>;
 790
 791 def : InstRW<[WriteVSTShuffle, WriteVST],
 792              (instregex "ST1Threev(8b|4h|2s|1d)$")>;
 793 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVST],
 794              (instregex "ST1Threev(8b|4h|2s|1d)_POST")>;
 795 def : InstRW<[WriteVST, WriteVST, WriteVST],
 796              (instregex "ST1Threev(16b|8h|4s|2d)$")>;
 797 def : InstRW<[WriteAdr, WriteVST, WriteVST, WriteVST],
 798              (instregex "ST1Threev(16b|8h|4s|2d)_POST")>;
 799
 800 def : InstRW<[WriteVSTShuffle, WriteVSTShuffle],
 801              (instregex "ST1Fourv(8b|4h|2s|1d)$")>;
 802 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle],
 803              (instregex "ST1Fourv(8b|4h|2s|1d)_POST")>;
 804 def : InstRW<[WriteVST, WriteVST, WriteVST, WriteVST],
 805              (instregex "ST1Fourv(16b|8h|4s|2d)$")>;
 806 def : InstRW<[WriteAdr, WriteVST, WriteVST, WriteVST, WriteVST],
 807              (instregex "ST1Fourv(16b|8h|4s|2d)_POST")>;
 808
 809 def : InstRW<[WriteVSTShuffle],           (instregex "ST1i(8|16|32)$")>;
 810 def : InstRW<[WriteAdr, WriteVSTShuffle], (instregex "ST1i(8|16|32)_POST")>;
 811
 812 def : InstRW<[WriteVSTShuffle],           (instrs ST1i64)>;
 813 def : InstRW<[WriteAdr, WriteVSTShuffle], (instrs ST1i64_POST)>;
 814
 815 def : InstRW<[WriteVSTShuffle],
 816              (instregex "ST2Twov(8b|4h|2s)$")>;
 817 def : InstRW<[WriteAdr, WriteVSTShuffle],
 818              (instregex "ST2Twov(8b|4h|2s)_POST")>;
 819 def : InstRW<[WriteVSTShuffle, WriteVSTShuffle],
 820              (instregex "ST2Twov(16b|8h|4s|2d)$")>;
 821 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle],
 822              (instregex "ST2Twov(16b|8h|4s|2d)_POST")>;
 823
 824 def : InstRW<[WriteVSTShuffle],           (instregex "ST2i(8|16|32)$")>;
 825 def : InstRW<[WriteAdr, WriteVSTShuffle], (instregex "ST2i(8|16|32)_POST")>;
 826 def : InstRW<[WriteVSTShuffle],           (instrs ST2i64)>;
 827 def : InstRW<[WriteAdr, WriteVSTShuffle], (instrs ST2i64_POST)>;
 828
 829 def : InstRW<[WriteVSTShuffle, WriteVSTShuffle],
 830              (instregex "ST3Threev(8b|4h|2s)$")>;
 831 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle],
 832              (instregex "ST3Threev(8b|4h|2s)_POST")>;
 833 def : InstRW<[WriteVSTShuffle, WriteVSTShuffle, WriteVSTShuffle],
 834              (instregex "ST3Threev(16b|8h|4s|2d)$")>;
 835 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle, WriteVSTShuffle],
 836              (instregex "ST3Threev(16b|8h|4s|2d)_POST")>;
 837
 838 def : InstRW<[WriteVSTShuffle],           (instregex "ST3i(8|16|32)$")>;
 839 def : InstRW<[WriteAdr, WriteVSTShuffle], (instregex "ST3i(8|16|32)_POST")>;
 840
 841 def :InstRW<[WriteVSTShuffle, WriteVSTShuffle],           (instrs ST3i64)>;
 842 def :InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle], (instrs ST3i64_POST)>;
 843
 844 def : InstRW<[WriteVSTPairShuffle, WriteVSTPairShuffle],
 845             (instregex "ST4Fourv(8b|4h|2s|1d)$")>;
 846 def : InstRW<[WriteAdr, WriteVSTPairShuffle, WriteVSTPairShuffle],
 847             (instregex "ST4Fourv(8b|4h|2s|1d)_POST")>;
 848 def : InstRW<[WriteVSTPairShuffle, WriteVSTPairShuffle,
 849               WriteVSTPairShuffle, WriteVSTPairShuffle],
 850              (instregex "ST4Fourv(16b|8h|4s|2d)$")>;
 851 def : InstRW<[WriteAdr, WriteVSTPairShuffle, WriteVSTPairShuffle,
 852               WriteVSTPairShuffle, WriteVSTPairShuffle],
 853              (instregex "ST4Fourv(16b|8h|4s|2d)_POST")>;
 854
 855 def : InstRW<[WriteVSTPairShuffle],           (instregex "ST4i(8|16|32)$")>;
 856 def : InstRW<[WriteAdr, WriteVSTPairShuffle], (instregex "ST4i(8|16|32)_POST")>;
 857
 858 def : InstRW<[WriteVSTShuffle, WriteVSTShuffle],          (instrs ST4i64)>;
 859 def : InstRW<[WriteAdr, WriteVSTShuffle, WriteVSTShuffle],(instrs ST4i64_POST)>;
 860
 861 // Atomic operations are not supported.
 862 def : WriteRes<WriteAtomic, []> { let Unsupported = 1; }
 863
 864 //---
 865 // Unused SchedRead types
 866 //---
 867
 868 def : ReadAdvance<ReadI, 0>;
 869 def : ReadAdvance<ReadISReg, 0>;
 870 def : ReadAdvance<ReadIEReg, 0>;
 871 def : ReadAdvance<ReadIM, 0>;
 872 def : ReadAdvance<ReadIMA, 0>;
 873 def : ReadAdvance<ReadID, 0>;
 874
 875 } // SchedModel = CycloneModel