utils/TableGen/CodeGenDAGPatterns.h

   1 //===- CodeGenDAGPatterns.h - Read DAG patterns from .td file ---*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file declares the CodeGenDAGPatterns class, which is used to read and
  10 // represent the patterns present in a .td file for instructions.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
  15 #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
  16
  17 #include "CodeGenHwModes.h"
  18 #include "CodeGenIntrinsics.h"
  19 #include "CodeGenTarget.h"
  20 #include "SDNodeProperties.h"
  21 #include "llvm/ADT/MapVector.h"
  22 #include "llvm/ADT/SmallVector.h"
  23 #include "llvm/ADT/StringMap.h"
  24 #include "llvm/ADT/StringSet.h"
  25 #include "llvm/Support/ErrorHandling.h"
  26 #include "llvm/Support/MathExtras.h"
  27 #include <algorithm>
  28 #include <array>
  29 #include <functional>
  30 #include <map>
  31 #include <numeric>
  32 #include <set>
  33 #include <vector>
  34
  35 namespace llvm {
  36
  37 class Record;
  38 class Init;
  39 class ListInit;
  40 class DagInit;
  41 class SDNodeInfo;
  42 class TreePattern;
  43 class TreePatternNode;
  44 class CodeGenDAGPatterns;
  45 class ComplexPattern;
  46
  47 /// Shared pointer for TreePatternNode.
  48 using TreePatternNodePtr = std::shared_ptr<TreePatternNode>;
  49
  50 /// This represents a set of MVTs. Since the underlying type for the MVT
  51 /// is uint8_t, there are at most 256 values. To reduce the number of memory
  52 /// allocations and deallocations, represent the set as a sequence of bits.
  53 /// To reduce the allocations even further, make MachineValueTypeSet own
  54 /// the storage and use std::array as the bit container.
  55 struct MachineValueTypeSet {
  56   static_assert(std::is_same<std::underlying_type<MVT::SimpleValueType>::type,
  57                              uint8_t>::value,
  58                 "Change uint8_t here to the SimpleValueType's type");
  59   static unsigned constexpr Capacity = std::numeric_limits<uint8_t>::max()+1;
  60   using WordType = uint64_t;
  61   static unsigned constexpr WordWidth = CHAR_BIT*sizeof(WordType);
  62   static unsigned constexpr NumWords = Capacity/WordWidth;
  63   static_assert(NumWords*WordWidth == Capacity,
  64                 "Capacity should be a multiple of WordWidth");
  65
  66   LLVM_ATTRIBUTE_ALWAYS_INLINE
  67   MachineValueTypeSet() {
  68     clear();
  69   }
  70
  71   LLVM_ATTRIBUTE_ALWAYS_INLINE
  72   unsigned size() const {
  73     unsigned Count = 0;
  74     for (WordType W : Words)
  75       Count += countPopulation(W);
  76     return Count;
  77   }
  78   LLVM_ATTRIBUTE_ALWAYS_INLINE
  79   void clear() {
  80     std::memset(Words.data(), 0, NumWords*sizeof(WordType));
  81   }
  82   LLVM_ATTRIBUTE_ALWAYS_INLINE
  83   bool empty() const {
  84     for (WordType W : Words)
  85       if (W != 0)
  86         return false;
  87     return true;
  88   }
  89   LLVM_ATTRIBUTE_ALWAYS_INLINE
  90   unsigned count(MVT T) const {
  91     return (Words[T.SimpleTy / WordWidth] >> (T.SimpleTy % WordWidth)) & 1;
  92   }
  93   std::pair<MachineValueTypeSet&,bool> insert(MVT T) {
  94     bool V = count(T.SimpleTy);
  95     Words[T.SimpleTy / WordWidth] |= WordType(1) << (T.SimpleTy % WordWidth);
  96     return {*this, V};
  97   }
  98   MachineValueTypeSet &insert(const MachineValueTypeSet &S) {
  99     for (unsigned i = 0; i != NumWords; ++i)
 100       Words[i] |= S.Words[i];
 101     return *this;
 102   }
 103   LLVM_ATTRIBUTE_ALWAYS_INLINE
 104   void erase(MVT T) {
 105     Words[T.SimpleTy / WordWidth] &= ~(WordType(1) << (T.SimpleTy % WordWidth));
 106   }
 107
 108   struct const_iterator {
 109     // Some implementations of the C++ library require these traits to be
 110     // defined.
 111     using iterator_category = std::forward_iterator_tag;
 112     using value_type = MVT;
 113     using difference_type = ptrdiff_t;
 114     using pointer = const MVT*;
 115     using reference = const MVT&;
 116
 117     LLVM_ATTRIBUTE_ALWAYS_INLINE
 118     MVT operator*() const {
 119       assert(Pos != Capacity);
 120       return MVT::SimpleValueType(Pos);
 121     }
 122     LLVM_ATTRIBUTE_ALWAYS_INLINE
 123     const_iterator(const MachineValueTypeSet *S, bool End) : Set(S) {
 124       Pos = End ? Capacity : find_from_pos(0);
 125     }
 126     LLVM_ATTRIBUTE_ALWAYS_INLINE
 127     const_iterator &operator++() {
 128       assert(Pos != Capacity);
 129       Pos = find_from_pos(Pos+1);
 130       return *this;
 131     }
 132
 133     LLVM_ATTRIBUTE_ALWAYS_INLINE
 134     bool operator==(const const_iterator &It) const {
 135       return Set == It.Set && Pos == It.Pos;
 136     }
 137     LLVM_ATTRIBUTE_ALWAYS_INLINE
 138     bool operator!=(const const_iterator &It) const {
 139       return !operator==(It);
 140     }
 141
 142   private:
 143     unsigned find_from_pos(unsigned P) const {
 144       unsigned SkipWords = P / WordWidth;
 145       unsigned SkipBits = P % WordWidth;
 146       unsigned Count = SkipWords * WordWidth;
 147
 148       // If P is in the middle of a word, process it manually here, because
 149       // the trailing bits need to be masked off to use findFirstSet.
 150       if (SkipBits != 0) {
 151         WordType W = Set->Words[SkipWords];
 152         W &= maskLeadingOnes<WordType>(WordWidth-SkipBits);
 153         if (W != 0)
 154           return Count + findFirstSet(W);
 155         Count += WordWidth;
 156         SkipWords++;
 157       }
 158
 159       for (unsigned i = SkipWords; i != NumWords; ++i) {
 160         WordType W = Set->Words[i];
 161         if (W != 0)
 162           return Count + findFirstSet(W);
 163         Count += WordWidth;
 164       }
 165       return Capacity;
 166     }
 167
 168     const MachineValueTypeSet *Set;
 169     unsigned Pos;
 170   };
 171
 172   LLVM_ATTRIBUTE_ALWAYS_INLINE
 173   const_iterator begin() const { return const_iterator(this, false); }
 174   LLVM_ATTRIBUTE_ALWAYS_INLINE
 175   const_iterator end()   const { return const_iterator(this, true); }
 176
 177   LLVM_ATTRIBUTE_ALWAYS_INLINE
 178   bool operator==(const MachineValueTypeSet &S) const {
 179     return Words == S.Words;
 180   }
 181   LLVM_ATTRIBUTE_ALWAYS_INLINE
 182   bool operator!=(const MachineValueTypeSet &S) const {
 183     return !operator==(S);
 184   }
 185
 186 private:
 187   friend struct const_iterator;
 188   std::array<WordType,NumWords> Words;
 189 };
 190
 191 struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
 192   using SetType = MachineValueTypeSet;
 193   std::vector<unsigned> AddrSpaces;
 194
 195   TypeSetByHwMode() = default;
 196   TypeSetByHwMode(const TypeSetByHwMode &VTS) = default;
 197   TypeSetByHwMode(MVT::SimpleValueType VT)
 198     : TypeSetByHwMode(ValueTypeByHwMode(VT)) {}
 199   TypeSetByHwMode(ValueTypeByHwMode VT)
 200     : TypeSetByHwMode(ArrayRef<ValueTypeByHwMode>(&VT, 1)) {}
 201   TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList);
 202
 203   SetType &getOrCreate(unsigned Mode) {
 204     if (hasMode(Mode))
 205       return get(Mode);
 206     return Map.insert({Mode,SetType()}).first->second;
 207   }
 208
 209   bool isValueTypeByHwMode(bool AllowEmpty) const;
 210   ValueTypeByHwMode getValueTypeByHwMode() const;
 211
 212   LLVM_ATTRIBUTE_ALWAYS_INLINE
 213   bool isMachineValueType() const {
 214     return isDefaultOnly() && Map.begin()->second.size() == 1;
 215   }
 216
 217   LLVM_ATTRIBUTE_ALWAYS_INLINE
 218   MVT getMachineValueType() const {
 219     assert(isMachineValueType());
 220     return *Map.begin()->second.begin();
 221   }
 222
 223   bool isPossible() const;
 224
 225   LLVM_ATTRIBUTE_ALWAYS_INLINE
 226   bool isDefaultOnly() const {
 227     return Map.size() == 1 && Map.begin()->first == DefaultMode;
 228   }
 229
 230   bool isPointer() const {
 231     return getValueTypeByHwMode().isPointer();
 232   }
 233
 234   unsigned getPtrAddrSpace() const {
 235     assert(isPointer());
 236     return getValueTypeByHwMode().PtrAddrSpace;
 237   }
 238
 239   bool insert(const ValueTypeByHwMode &VVT);
 240   bool constrain(const TypeSetByHwMode &VTS);
 241   template <typename Predicate> bool constrain(Predicate P);
 242   template <typename Predicate>
 243   bool assign_if(const TypeSetByHwMode &VTS, Predicate P);
 244
 245   void writeToStream(raw_ostream &OS) const;
 246   static void writeToStream(const SetType &S, raw_ostream &OS);
 247
 248   bool operator==(const TypeSetByHwMode &VTS) const;
 249   bool operator!=(const TypeSetByHwMode &VTS) const { return !(*this == VTS); }
 250
 251   void dump() const;
 252   bool validate() const;
 253
 254 private:
 255   unsigned PtrAddrSpace = std::numeric_limits<unsigned>::max();
 256   /// Intersect two sets. Return true if anything has changed.
 257   bool intersect(SetType &Out, const SetType &In);
 258 };
 259
 260 raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T);
 261
 262 struct TypeInfer {
 263   TypeInfer(TreePattern &T) : TP(T), ForceMode(0) {}
 264
 265   bool isConcrete(const TypeSetByHwMode &VTS, bool AllowEmpty) const {
 266     return VTS.isValueTypeByHwMode(AllowEmpty);
 267   }
 268   ValueTypeByHwMode getConcrete(const TypeSetByHwMode &VTS,
 269                                 bool AllowEmpty) const {
 270     assert(VTS.isValueTypeByHwMode(AllowEmpty));
 271     return VTS.getValueTypeByHwMode();
 272   }
 273
 274   /// The protocol in the following functions (Merge*, force*, Enforce*,
 275   /// expand*) is to return "true" if a change has been made, "false"
 276   /// otherwise.
 277
 278   bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In);
 279   bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) {
 280     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
 281   }
 282   bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) {
 283     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
 284   }
 285
 286   /// Reduce the set \p Out to have at most one element for each mode.
 287   bool forceArbitrary(TypeSetByHwMode &Out);
 288
 289   /// The following four functions ensure that upon return the set \p Out
 290   /// will only contain types of the specified kind: integer, floating-point,
 291   /// scalar, or vector.
 292   /// If \p Out is empty, all legal types of the specified kind will be added
 293   /// to it. Otherwise, all types that are not of the specified kind will be
 294   /// removed from \p Out.
 295   bool EnforceInteger(TypeSetByHwMode &Out);
 296   bool EnforceFloatingPoint(TypeSetByHwMode &Out);
 297   bool EnforceScalar(TypeSetByHwMode &Out);
 298   bool EnforceVector(TypeSetByHwMode &Out);
 299
 300   /// If \p Out is empty, fill it with all legal types. Otherwise, leave it
 301   /// unchanged.
 302   bool EnforceAny(TypeSetByHwMode &Out);
 303   /// Make sure that for each type in \p Small, there exists a larger type
 304   /// in \p Big.
 305   bool EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big);
 306   /// 1. Ensure that for each type T in \p Vec, T is a vector type, and that
 307   ///    for each type U in \p Elem, U is a scalar type.
 308   /// 2. Ensure that for each (scalar) type U in \p Elem, there exists a
 309   ///    (vector) type T in \p Vec, such that U is the element type of T.
 310   bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, TypeSetByHwMode &Elem);
 311   bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
 312                               const ValueTypeByHwMode &VVT);
 313   /// Ensure that for each type T in \p Sub, T is a vector type, and there
 314   /// exists a type U in \p Vec such that U is a vector type with the same
 315   /// element type as T and at least as many elements as T.
 316   bool EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
 317                                     TypeSetByHwMode &Sub);
 318   /// 1. Ensure that \p V has a scalar type iff \p W has a scalar type.
 319   /// 2. Ensure that for each vector type T in \p V, there exists a vector
 320   ///    type U in \p W, such that T and U have the same number of elements.
 321   /// 3. Ensure that for each vector type U in \p W, there exists a vector
 322   ///    type T in \p V, such that T and U have the same number of elements
 323   ///    (reverse of 2).
 324   bool EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W);
 325   /// 1. Ensure that for each type T in \p A, there exists a type U in \p B,
 326   ///    such that T and U have equal size in bits.
 327   /// 2. Ensure that for each type U in \p B, there exists a type T in \p A
 328   ///    such that T and U have equal size in bits (reverse of 1).
 329   bool EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B);
 330
 331   /// For each overloaded type (i.e. of form *Any), replace it with the
 332   /// corresponding subset of legal, specific types.
 333   void expandOverloads(TypeSetByHwMode &VTS);
 334   void expandOverloads(TypeSetByHwMode::SetType &Out,
 335                        const TypeSetByHwMode::SetType &Legal);
 336
 337   struct ValidateOnExit {
 338     ValidateOnExit(TypeSetByHwMode &T, TypeInfer &TI) : Infer(TI), VTS(T) {}
 339   #ifndef NDEBUG
 340     ~ValidateOnExit();
 341   #else
 342     ~ValidateOnExit() {}  // Empty destructor with NDEBUG.
 343   #endif
 344     TypeInfer &Infer;
 345     TypeSetByHwMode &VTS;
 346   };
 347
 348   struct SuppressValidation {
 349     SuppressValidation(TypeInfer &TI) : Infer(TI), SavedValidate(TI.Validate) {
 350       Infer.Validate = false;
 351     }
 352     ~SuppressValidation() {
 353       Infer.Validate = SavedValidate;
 354     }
 355     TypeInfer &Infer;
 356     bool SavedValidate;
 357   };
 358
 359   TreePattern &TP;
 360   unsigned ForceMode;     // Mode to use when set.
 361   bool CodeGen = false;   // Set during generation of matcher code.
 362   bool Validate = true;   // Indicate whether to validate types.
 363
 364 private:
 365   const TypeSetByHwMode &getLegalTypes();
 366
 367   /// Cached legal types (in default mode).
 368   bool LegalTypesCached = false;
 369   TypeSetByHwMode LegalCache;
 370 };
 371
 372 /// Set type used to track multiply used variables in patterns
 373 typedef StringSet<> MultipleUseVarSet;
 374
 375 /// SDTypeConstraint - This is a discriminated union of constraints,
 376 /// corresponding to the SDTypeConstraint tablegen class in Target.td.
 377 struct SDTypeConstraint {
 378   SDTypeConstraint(Record *R, const CodeGenHwModes &CGH);
 379
 380   unsigned OperandNo;   // The operand # this constraint applies to.
 381   enum {
 382     SDTCisVT, SDTCisPtrTy, SDTCisInt, SDTCisFP, SDTCisVec, SDTCisSameAs,
 383     SDTCisVTSmallerThanOp, SDTCisOpSmallerThanOp, SDTCisEltOfVec,
 384     SDTCisSubVecOfVec, SDTCVecEltisVT, SDTCisSameNumEltsAs, SDTCisSameSizeAs
 385   } ConstraintType;
 386
 387   union {   // The discriminated union.
 388     struct {
 389       unsigned OtherOperandNum;
 390     } SDTCisSameAs_Info;
 391     struct {
 392       unsigned OtherOperandNum;
 393     } SDTCisVTSmallerThanOp_Info;
 394     struct {
 395       unsigned BigOperandNum;
 396     } SDTCisOpSmallerThanOp_Info;
 397     struct {
 398       unsigned OtherOperandNum;
 399     } SDTCisEltOfVec_Info;
 400     struct {
 401       unsigned OtherOperandNum;
 402     } SDTCisSubVecOfVec_Info;
 403     struct {
 404       unsigned OtherOperandNum;
 405     } SDTCisSameNumEltsAs_Info;
 406     struct {
 407       unsigned OtherOperandNum;
 408     } SDTCisSameSizeAs_Info;
 409   } x;
 410
 411   // The VT for SDTCisVT and SDTCVecEltisVT.
 412   // Must not be in the union because it has a non-trivial destructor.
 413   ValueTypeByHwMode VVT;
 414
 415   /// ApplyTypeConstraint - Given a node in a pattern, apply this type
 416   /// constraint to the nodes operands.  This returns true if it makes a
 417   /// change, false otherwise.  If a type contradiction is found, an error
 418   /// is flagged.
 419   bool ApplyTypeConstraint(TreePatternNode *N, const SDNodeInfo &NodeInfo,
 420                            TreePattern &TP) const;
 421 };
 422
 423 /// ScopedName - A name of a node associated with a "scope" that indicates
 424 /// the context (e.g. instance of Pattern or PatFrag) in which the name was
 425 /// used. This enables substitution of pattern fragments while keeping track
 426 /// of what name(s) were originally given to various nodes in the tree.
 427 class ScopedName {
 428   unsigned Scope;
 429   std::string Identifier;
 430 public:
 431   ScopedName(unsigned Scope, StringRef Identifier)
 432     : Scope(Scope), Identifier(Identifier) {
 433     assert(Scope != 0 &&
 434            "Scope == 0 is used to indicate predicates without arguments");
 435   }
 436
 437   unsigned getScope() const { return Scope; }
 438   const std::string &getIdentifier() const { return Identifier; }
 439
 440   std::string getFullName() const;
 441
 442   bool operator==(const ScopedName &o) const;
 443   bool operator!=(const ScopedName &o) const;
 444 };
 445
 446 /// SDNodeInfo - One of these records is created for each SDNode instance in
 447 /// the target .td file.  This represents the various dag nodes we will be
 448 /// processing.
 449 class SDNodeInfo {
 450   Record *Def;
 451   StringRef EnumName;
 452   StringRef SDClassName;
 453   unsigned Properties;
 454   unsigned NumResults;
 455   int NumOperands;
 456   std::vector<SDTypeConstraint> TypeConstraints;
 457 public:
 458   // Parse the specified record.
 459   SDNodeInfo(Record *R, const CodeGenHwModes &CGH);
 460
 461   unsigned getNumResults() const { return NumResults; }
 462
 463   /// getNumOperands - This is the number of operands required or -1 if
 464   /// variadic.
 465   int getNumOperands() const { return NumOperands; }
 466   Record *getRecord() const { return Def; }
 467   StringRef getEnumName() const { return EnumName; }
 468   StringRef getSDClassName() const { return SDClassName; }
 469
 470   const std::vector<SDTypeConstraint> &getTypeConstraints() const {
 471     return TypeConstraints;
 472   }
 473
 474   /// getKnownType - If the type constraints on this node imply a fixed type
 475   /// (e.g. all stores return void, etc), then return it as an
 476   /// MVT::SimpleValueType.  Otherwise, return MVT::Other.
 477   MVT::SimpleValueType getKnownType(unsigned ResNo) const;
 478
 479   /// hasProperty - Return true if this node has the specified property.
 480   ///
 481   bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }
 482
 483   /// ApplyTypeConstraints - Given a node in a pattern, apply the type
 484   /// constraints for this node to the operands of the node.  This returns
 485   /// true if it makes a change, false otherwise.  If a type contradiction is
 486   /// found, an error is flagged.
 487   bool ApplyTypeConstraints(TreePatternNode *N, TreePattern &TP) const;
 488 };
 489
 490 /// TreePredicateFn - This is an abstraction that represents the predicates on
 491 /// a PatFrag node.  This is a simple one-word wrapper around a pointer to
 492 /// provide nice accessors.
 493 class TreePredicateFn {
 494   /// PatFragRec - This is the TreePattern for the PatFrag that we
 495   /// originally came from.
 496   TreePattern *PatFragRec;
 497 public:
 498   /// TreePredicateFn constructor.  Here 'N' is a subclass of PatFrag.
 499   TreePredicateFn(TreePattern *N);
 500
 501
 502   TreePattern *getOrigPatFragRecord() const { return PatFragRec; }
 503
 504   /// isAlwaysTrue - Return true if this is a noop predicate.
 505   bool isAlwaysTrue() const;
 506
 507   bool isImmediatePattern() const { return hasImmCode(); }
 508
 509   /// getImmediatePredicateCode - Return the code that evaluates this pattern if
 510   /// this is an immediate predicate.  It is an error to call this on a
 511   /// non-immediate pattern.
 512   std::string getImmediatePredicateCode() const {
 513     std::string Result = getImmCode();
 514     assert(!Result.empty() && "Isn't an immediate pattern!");
 515     return Result;
 516   }
 517
 518   bool operator==(const TreePredicateFn &RHS) const {
 519     return PatFragRec == RHS.PatFragRec;
 520   }
 521
 522   bool operator!=(const TreePredicateFn &RHS) const { return !(*this == RHS); }
 523
 524   /// Return the name to use in the generated code to reference this, this is
 525   /// "Predicate_foo" if from a pattern fragment "foo".
 526   std::string getFnName() const;
 527
 528   /// getCodeToRunOnSDNode - Return the code for the function body that
 529   /// evaluates this predicate.  The argument is expected to be in "Node",
 530   /// not N.  This handles casting and conversion to a concrete node type as
 531   /// appropriate.
 532   std::string getCodeToRunOnSDNode() const;
 533
 534   /// Get the data type of the argument to getImmediatePredicateCode().
 535   StringRef getImmType() const;
 536
 537   /// Get a string that describes the type returned by getImmType() but is
 538   /// usable as part of an identifier.
 539   StringRef getImmTypeIdentifier() const;
 540
 541   // Predicate code uses the PatFrag's captured operands.
 542   bool usesOperands() const;
 543
 544   // Is the desired predefined predicate for a load?
 545   bool isLoad() const;
 546   // Is the desired predefined predicate for a store?
 547   bool isStore() const;
 548   // Is the desired predefined predicate for an atomic?
 549   bool isAtomic() const;
 550
 551   /// Is this predicate the predefined unindexed load predicate?
 552   /// Is this predicate the predefined unindexed store predicate?
 553   bool isUnindexed() const;
 554   /// Is this predicate the predefined non-extending load predicate?
 555   bool isNonExtLoad() const;
 556   /// Is this predicate the predefined any-extend load predicate?
 557   bool isAnyExtLoad() const;
 558   /// Is this predicate the predefined sign-extend load predicate?
 559   bool isSignExtLoad() const;
 560   /// Is this predicate the predefined zero-extend load predicate?
 561   bool isZeroExtLoad() const;
 562   /// Is this predicate the predefined non-truncating store predicate?
 563   bool isNonTruncStore() const;
 564   /// Is this predicate the predefined truncating store predicate?
 565   bool isTruncStore() const;
 566
 567   /// Is this predicate the predefined monotonic atomic predicate?
 568   bool isAtomicOrderingMonotonic() const;
 569   /// Is this predicate the predefined acquire atomic predicate?
 570   bool isAtomicOrderingAcquire() const;
 571   /// Is this predicate the predefined release atomic predicate?
 572   bool isAtomicOrderingRelease() const;
 573   /// Is this predicate the predefined acquire-release atomic predicate?
 574   bool isAtomicOrderingAcquireRelease() const;
 575   /// Is this predicate the predefined sequentially consistent atomic predicate?
 576   bool isAtomicOrderingSequentiallyConsistent() const;
 577
 578   /// Is this predicate the predefined acquire-or-stronger atomic predicate?
 579   bool isAtomicOrderingAcquireOrStronger() const;
 580   /// Is this predicate the predefined weaker-than-acquire atomic predicate?
 581   bool isAtomicOrderingWeakerThanAcquire() const;
 582
 583   /// Is this predicate the predefined release-or-stronger atomic predicate?
 584   bool isAtomicOrderingReleaseOrStronger() const;
 585   /// Is this predicate the predefined weaker-than-release atomic predicate?
 586   bool isAtomicOrderingWeakerThanRelease() const;
 587
 588   /// If non-null, indicates that this predicate is a predefined memory VT
 589   /// predicate for a load/store and returns the ValueType record for the memory VT.
 590   Record *getMemoryVT() const;
 591   /// If non-null, indicates that this predicate is a predefined memory VT
 592   /// predicate (checking only the scalar type) for load/store and returns the
 593   /// ValueType record for the memory VT.
 594   Record *getScalarMemoryVT() const;
 595
 596   ListInit *getAddressSpaces() const;
 597   int64_t getMinAlignment() const;
 598
 599   // If true, indicates that GlobalISel-based C++ code was supplied.
 600   bool hasGISelPredicateCode() const;
 601   std::string getGISelPredicateCode() const;
 602
 603 private:
 604   bool hasPredCode() const;
 605   bool hasImmCode() const;
 606   std::string getPredCode() const;
 607   std::string getImmCode() const;
 608   bool immCodeUsesAPInt() const;
 609   bool immCodeUsesAPFloat() const;
 610
 611   bool isPredefinedPredicateEqualTo(StringRef Field, bool Value) const;
 612 };
 613
 614 struct TreePredicateCall {
 615   TreePredicateFn Fn;
 616
 617   // Scope -- unique identifier for retrieving named arguments. 0 is used when
 618   // the predicate does not use named arguments.
 619   unsigned Scope;
 620
 621   TreePredicateCall(const TreePredicateFn &Fn, unsigned Scope)
 622     : Fn(Fn), Scope(Scope) {}
 623
 624   bool operator==(const TreePredicateCall &o) const {
 625     return Fn == o.Fn && Scope == o.Scope;
 626   }
 627   bool operator!=(const TreePredicateCall &o) const {
 628     return !(*this == o);
 629   }
 630 };
 631
 632 class TreePatternNode {
 633   /// The type of each node result.  Before and during type inference, each
 634   /// result may be a set of possible types.  After (successful) type inference,
 635   /// each is a single concrete type.
 636   std::vector<TypeSetByHwMode> Types;
 637
 638   /// The index of each result in results of the pattern.
 639   std::vector<unsigned> ResultPerm;
 640
 641   /// Operator - The Record for the operator if this is an interior node (not
 642   /// a leaf).
 643   Record *Operator;
 644
 645   /// Val - The init value (e.g. the "GPRC" record, or "7") for a leaf.
 646   ///
 647   Init *Val;
 648
 649   /// Name - The name given to this node with the :$foo notation.
 650   ///
 651   std::string Name;
 652
 653   std::vector<ScopedName> NamesAsPredicateArg;
 654
 655   /// PredicateCalls - The predicate functions to execute on this node to check
 656   /// for a match.  If this list is empty, no predicate is involved.
 657   std::vector<TreePredicateCall> PredicateCalls;
 658
 659   /// TransformFn - The transformation function to execute on this node before
 660   /// it can be substituted into the resulting instruction on a pattern match.
 661   Record *TransformFn;
 662
 663   std::vector<TreePatternNodePtr> Children;
 664
 665 public:
 666   TreePatternNode(Record *Op, std::vector<TreePatternNodePtr> Ch,
 667                   unsigned NumResults)
 668       : Operator(Op), Val(nullptr), TransformFn(nullptr),
 669         Children(std::move(Ch)) {
 670     Types.resize(NumResults);
 671     ResultPerm.resize(NumResults);
 672     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
 673   }
 674   TreePatternNode(Init *val, unsigned NumResults)    // leaf ctor
 675     : Operator(nullptr), Val(val), TransformFn(nullptr) {
 676     Types.resize(NumResults);
 677     ResultPerm.resize(NumResults);
 678     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
 679   }
 680
 681   bool hasName() const { return !Name.empty(); }
 682   const std::string &getName() const { return Name; }
 683   void setName(StringRef N) { Name.assign(N.begin(), N.end()); }
 684
 685   const std::vector<ScopedName> &getNamesAsPredicateArg() const {
 686     return NamesAsPredicateArg;
 687   }
 688   void setNamesAsPredicateArg(const std::vector<ScopedName>& Names) {
 689     NamesAsPredicateArg = Names;
 690   }
 691   void addNameAsPredicateArg(const ScopedName &N) {
 692     NamesAsPredicateArg.push_back(N);
 693   }
 694
 695   bool isLeaf() const { return Val != nullptr; }
 696
 697   // Type accessors.
 698   unsigned getNumTypes() const { return Types.size(); }
 699   ValueTypeByHwMode getType(unsigned ResNo) const {
 700     return Types[ResNo].getValueTypeByHwMode();
 701   }
 702   const std::vector<TypeSetByHwMode> &getExtTypes() const { return Types; }
 703   const TypeSetByHwMode &getExtType(unsigned ResNo) const {
 704     return Types[ResNo];
 705   }
 706   TypeSetByHwMode &getExtType(unsigned ResNo) { return Types[ResNo]; }
 707   void setType(unsigned ResNo, const TypeSetByHwMode &T) { Types[ResNo] = T; }
 708   MVT::SimpleValueType getSimpleType(unsigned ResNo) const {
 709     return Types[ResNo].getMachineValueType().SimpleTy;
 710   }
 711
 712   bool hasConcreteType(unsigned ResNo) const {
 713     return Types[ResNo].isValueTypeByHwMode(false);
 714   }
 715   bool isTypeCompletelyUnknown(unsigned ResNo, TreePattern &TP) const {
 716     return Types[ResNo].empty();
 717   }
 718
 719   unsigned getNumResults() const { return ResultPerm.size(); }
 720   unsigned getResultIndex(unsigned ResNo) const { return ResultPerm[ResNo]; }
 721   void setResultIndex(unsigned ResNo, unsigned RI) { ResultPerm[ResNo] = RI; }
 722
 723   Init *getLeafValue() const { assert(isLeaf()); return Val; }
 724   Record *getOperator() const { assert(!isLeaf()); return Operator; }
 725
 726   unsigned getNumChildren() const { return Children.size(); }
 727   TreePatternNode *getChild(unsigned N) const { return Children[N].get(); }
 728   const TreePatternNodePtr &getChildShared(unsigned N) const {
 729     return Children[N];
 730   }
 731   void setChild(unsigned i, TreePatternNodePtr N) { Children[i] = N; }
 732
 733   /// hasChild - Return true if N is any of our children.
 734   bool hasChild(const TreePatternNode *N) const {
 735     for (unsigned i = 0, e = Children.size(); i != e; ++i)
 736       if (Children[i].get() == N)
 737         return true;
 738     return false;
 739   }
 740
 741   bool hasProperTypeByHwMode() const;
 742   bool hasPossibleType() const;
 743   bool setDefaultMode(unsigned Mode);
 744
 745   bool hasAnyPredicate() const { return !PredicateCalls.empty(); }
 746
 747   const std::vector<TreePredicateCall> &getPredicateCalls() const {
 748     return PredicateCalls;
 749   }
 750   void clearPredicateCalls() { PredicateCalls.clear(); }
 751   void setPredicateCalls(const std::vector<TreePredicateCall> &Calls) {
 752     assert(PredicateCalls.empty() && "Overwriting non-empty predicate list!");
 753     PredicateCalls = Calls;
 754   }
 755   void addPredicateCall(const TreePredicateCall &Call) {
 756     assert(!Call.Fn.isAlwaysTrue() && "Empty predicate string!");
 757     assert(!is_contained(PredicateCalls, Call) && "predicate applied recursively");
 758     PredicateCalls.push_back(Call);
 759   }
 760   void addPredicateCall(const TreePredicateFn &Fn, unsigned Scope) {
 761     assert((Scope != 0) == Fn.usesOperands());
 762     addPredicateCall(TreePredicateCall(Fn, Scope));
 763   }
 764
 765   Record *getTransformFn() const { return TransformFn; }
 766   void setTransformFn(Record *Fn) { TransformFn = Fn; }
 767
 768   /// getIntrinsicInfo - If this node corresponds to an intrinsic, return the
 769   /// CodeGenIntrinsic information for it, otherwise return a null pointer.
 770   const CodeGenIntrinsic *getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const;
 771
 772   /// getComplexPatternInfo - If this node corresponds to a ComplexPattern,
 773   /// return the ComplexPattern information, otherwise return null.
 774   const ComplexPattern *
 775   getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const;
 776
 777   /// Returns the number of MachineInstr operands that would be produced by this
 778   /// node if it mapped directly to an output Instruction's
 779   /// operand. ComplexPattern specifies this explicitly; MIOperandInfo gives it
 780   /// for Operands; otherwise 1.
 781   unsigned getNumMIResults(const CodeGenDAGPatterns &CGP) const;
 782
 783   /// NodeHasProperty - Return true if this node has the specified property.
 784   bool NodeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const;
 785
 786   /// TreeHasProperty - Return true if any node in this tree has the specified
 787   /// property.
 788   bool TreeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const;
 789
 790   /// isCommutativeIntrinsic - Return true if the node is an intrinsic which is
 791   /// marked isCommutative.
 792   bool isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const;
 793
 794   void print(raw_ostream &OS) const;
 795   void dump() const;
 796
 797 public:   // Higher level manipulation routines.
 798
 799   /// clone - Return a new copy of this tree.
 800   ///
 801   TreePatternNodePtr clone() const;
 802
 803   /// RemoveAllTypes - Recursively strip all the types of this tree.
 804   void RemoveAllTypes();
 805
 806   /// isIsomorphicTo - Return true if this node is recursively isomorphic to
 807   /// the specified node.  For this comparison, all of the state of the node
 808   /// is considered, except for the assigned name.  Nodes with differing names
 809   /// that are otherwise identical are considered isomorphic.
 810   bool isIsomorphicTo(const TreePatternNode *N,
 811                       const MultipleUseVarSet &DepVars) const;
 812
 813   /// SubstituteFormalArguments - Replace the formal arguments in this tree
 814   /// with actual values specified by ArgMap.
 815   void
 816   SubstituteFormalArguments(std::map<std::string, TreePatternNodePtr> &ArgMap);
 817
 818   /// InlinePatternFragments - If this pattern refers to any pattern
 819   /// fragments, return the set of inlined versions (this can be more than
 820   /// one if a PatFrags record has multiple alternatives).
 821   void InlinePatternFragments(TreePatternNodePtr T,
 822                               TreePattern &TP,
 823                               std::vector<TreePatternNodePtr> &OutAlternatives);
 824
 825   /// ApplyTypeConstraints - Apply all of the type constraints relevant to
 826   /// this node and its children in the tree.  This returns true if it makes a
 827   /// change, false otherwise.  If a type contradiction is found, flag an error.
 828   bool ApplyTypeConstraints(TreePattern &TP, bool NotRegisters);
 829
 830   /// UpdateNodeType - Set the node type of N to VT if VT contains
 831   /// information.  If N already contains a conflicting type, then flag an
 832   /// error.  This returns true if any information was updated.
 833   ///
 834   bool UpdateNodeType(unsigned ResNo, const TypeSetByHwMode &InTy,
 835                       TreePattern &TP);
 836   bool UpdateNodeType(unsigned ResNo, MVT::SimpleValueType InTy,
 837                       TreePattern &TP);
 838   bool UpdateNodeType(unsigned ResNo, ValueTypeByHwMode InTy,
 839                       TreePattern &TP);
 840
 841   // Update node type with types inferred from an instruction operand or result
 842   // def from the ins/outs lists.
 843   // Return true if the type changed.
 844   bool UpdateNodeTypeFromInst(unsigned ResNo, Record *Operand, TreePattern &TP);
 845
 846   /// ContainsUnresolvedType - Return true if this tree contains any
 847   /// unresolved types.
 848   bool ContainsUnresolvedType(TreePattern &TP) const;
 849
 850   /// canPatternMatch - If it is impossible for this pattern to match on this
 851   /// target, fill in Reason and return false.  Otherwise, return true.
 852   bool canPatternMatch(std::string &Reason, const CodeGenDAGPatterns &CDP);
 853 };
 854
 855 inline raw_ostream &operator<<(raw_ostream &OS, const TreePatternNode &TPN) {
 856   TPN.print(OS);
 857   return OS;
 858 }
 859
 860
 861 /// TreePattern - Represent a pattern, used for instructions, pattern
 862 /// fragments, etc.
 863 ///
 864 class TreePattern {
 865   /// Trees - The list of pattern trees which corresponds to this pattern.
 866   /// Note that PatFrag's only have a single tree.
 867   ///
 868   std::vector<TreePatternNodePtr> Trees;
 869
 870   /// NamedNodes - This is all of the nodes that have names in the trees in this
 871   /// pattern.
 872   StringMap<SmallVector<TreePatternNode *, 1>> NamedNodes;
 873
 874   /// TheRecord - The actual TableGen record corresponding to this pattern.
 875   ///
 876   Record *TheRecord;
 877
 878   /// Args - This is a list of all of the arguments to this pattern (for
 879   /// PatFrag patterns), which are the 'node' markers in this pattern.
 880   std::vector<std::string> Args;
 881
 882   /// CDP - the top-level object coordinating this madness.
 883   ///
 884   CodeGenDAGPatterns &CDP;
 885
 886   /// isInputPattern - True if this is an input pattern, something to match.
 887   /// False if this is an output pattern, something to emit.
 888   bool isInputPattern;
 889
 890   /// hasError - True if the currently processed nodes have unresolvable types
 891   /// or other non-fatal errors
 892   bool HasError;
 893
 894   /// It's important that the usage of operands in ComplexPatterns is
 895   /// consistent: each named operand can be defined by at most one
 896   /// ComplexPattern. This records the ComplexPattern instance and the operand
 897   /// number for each operand encountered in a ComplexPattern to aid in that
 898   /// check.
 899   StringMap<std::pair<Record *, unsigned>> ComplexPatternOperands;
 900
 901   TypeInfer Infer;
 902
 903 public:
 904
 905   /// TreePattern constructor - Parse the specified DagInits into the
 906   /// current record.
 907   TreePattern(Record *TheRec, ListInit *RawPat, bool isInput,
 908               CodeGenDAGPatterns &ise);
 909   TreePattern(Record *TheRec, DagInit *Pat, bool isInput,
 910               CodeGenDAGPatterns &ise);
 911   TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput,
 912               CodeGenDAGPatterns &ise);
 913
 914   /// getTrees - Return the tree patterns which corresponds to this pattern.
 915   ///
 916   const std::vector<TreePatternNodePtr> &getTrees() const { return Trees; }
 917   unsigned getNumTrees() const { return Trees.size(); }
 918   const TreePatternNodePtr &getTree(unsigned i) const { return Trees[i]; }
 919   void setTree(unsigned i, TreePatternNodePtr Tree) { Trees[i] = Tree; }
 920   const TreePatternNodePtr &getOnlyTree() const {
 921     assert(Trees.size() == 1 && "Doesn't have exactly one pattern!");
 922     return Trees[0];
 923   }
 924
 925   const StringMap<SmallVector<TreePatternNode *, 1>> &getNamedNodesMap() {
 926     if (NamedNodes.empty())
 927       ComputeNamedNodes();
 928     return NamedNodes;
 929   }
 930
 931   /// getRecord - Return the actual TableGen record corresponding to this
 932   /// pattern.
 933   ///
 934   Record *getRecord() const { return TheRecord; }
 935
 936   unsigned getNumArgs() const { return Args.size(); }
 937   const std::string &getArgName(unsigned i) const {
 938     assert(i < Args.size() && "Argument reference out of range!");
 939     return Args[i];
 940   }
 941   std::vector<std::string> &getArgList() { return Args; }
 942
 943   CodeGenDAGPatterns &getDAGPatterns() const { return CDP; }
 944
 945   /// InlinePatternFragments - If this pattern refers to any pattern
 946   /// fragments, inline them into place, giving us a pattern without any
 947   /// PatFrags references.  This may increase the number of trees in the
 948   /// pattern if a PatFrags has multiple alternatives.
 949   void InlinePatternFragments() {
 950     std::vector<TreePatternNodePtr> Copy = Trees;
 951     Trees.clear();
 952     for (unsigned i = 0, e = Copy.size(); i != e; ++i)
 953       Copy[i]->InlinePatternFragments(Copy[i], *this, Trees);
 954   }
 955
 956   /// InferAllTypes - Infer/propagate as many types throughout the expression
 957   /// patterns as possible.  Return true if all types are inferred, false
 958   /// otherwise.  Bail out if a type contradiction is found.
 959   bool InferAllTypes(
 960       const StringMap<SmallVector<TreePatternNode *, 1>> *NamedTypes = nullptr);
 961
 962   /// error - If this is the first error in the current resolution step,
 963   /// print it and set the error flag.  Otherwise, continue silently.
 964   void error(const Twine &Msg);
 965   bool hasError() const {
 966     return HasError;
 967   }
 968   void resetError() {
 969     HasError = false;
 970   }
 971
 972   TypeInfer &getInfer() { return Infer; }
 973
 974   void print(raw_ostream &OS) const;
 975   void dump() const;
 976
 977 private:
 978   TreePatternNodePtr ParseTreePattern(Init *DI, StringRef OpName);
 979   void ComputeNamedNodes();
 980   void ComputeNamedNodes(TreePatternNode *N);
 981 };
 982
 983
 984 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
 985                                             const TypeSetByHwMode &InTy,
 986                                             TreePattern &TP) {
 987   TypeSetByHwMode VTS(InTy);
 988   TP.getInfer().expandOverloads(VTS);
 989   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
 990 }
 991
 992 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
 993                                             MVT::SimpleValueType InTy,
 994                                             TreePattern &TP) {
 995   TypeSetByHwMode VTS(InTy);
 996   TP.getInfer().expandOverloads(VTS);
 997   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
 998 }
 999
1000 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
1001                                             ValueTypeByHwMode InTy,
1002                                             TreePattern &TP) {
1003   TypeSetByHwMode VTS(InTy);
1004   TP.getInfer().expandOverloads(VTS);
1005   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
1006 }
1007
1008
1009 /// DAGDefaultOperand - One of these is created for each OperandWithDefaultOps
1010 /// that has a set ExecuteAlways / DefaultOps field.
1011 struct DAGDefaultOperand {
1012   std::vector<TreePatternNodePtr> DefaultOps;
1013 };
1014
1015 class DAGInstruction {
1016   std::vector<Record*> Results;
1017   std::vector<Record*> Operands;
1018   std::vector<Record*> ImpResults;
1019   TreePatternNodePtr SrcPattern;
1020   TreePatternNodePtr ResultPattern;
1021
1022 public:
1023   DAGInstruction(const std::vector<Record*> &results,
1024                  const std::vector<Record*> &operands,
1025                  const std::vector<Record*> &impresults,
1026                  TreePatternNodePtr srcpattern = nullptr,
1027                  TreePatternNodePtr resultpattern = nullptr)
1028     : Results(results), Operands(operands), ImpResults(impresults),
1029       SrcPattern(srcpattern), ResultPattern(resultpattern) {}
1030
1031   unsigned getNumResults() const { return Results.size(); }
1032   unsigned getNumOperands() const { return Operands.size(); }
1033   unsigned getNumImpResults() const { return ImpResults.size(); }
1034   const std::vector<Record*>& getImpResults() const { return ImpResults; }
1035
1036   Record *getResult(unsigned RN) const {
1037     assert(RN < Results.size());
1038     return Results[RN];
1039   }
1040
1041   Record *getOperand(unsigned ON) const {
1042     assert(ON < Operands.size());
1043     return Operands[ON];
1044   }
1045
1046   Record *getImpResult(unsigned RN) const {
1047     assert(RN < ImpResults.size());
1048     return ImpResults[RN];
1049   }
1050
1051   TreePatternNodePtr getSrcPattern() const { return SrcPattern; }
1052   TreePatternNodePtr getResultPattern() const { return ResultPattern; }
1053 };
1054
1055 /// This class represents a condition that has to be satisfied for a pattern
1056 /// to be tried. It is a generalization of a class "Pattern" from Target.td:
1057 /// in addition to the Target.td's predicates, this class can also represent
1058 /// conditions associated with HW modes. Both types will eventually become
1059 /// strings containing C++ code to be executed, the difference is in how
1060 /// these strings are generated.
1061 class Predicate {
1062 public:
1063   Predicate(Record *R, bool C = true) : Def(R), IfCond(C), IsHwMode(false) {
1064     assert(R->isSubClassOf("Predicate") &&
1065            "Predicate objects should only be created for records derived"
1066            "from Predicate class");
1067   }
1068   Predicate(StringRef FS, bool C = true) : Def(nullptr), Features(FS.str()),
1069     IfCond(C), IsHwMode(true) {}
1070
1071   /// Return a string which contains the C++ condition code that will serve
1072   /// as a predicate during instruction selection.
1073   std::string getCondString() const {
1074     // The string will excute in a subclass of SelectionDAGISel.
1075     // Cast to std::string explicitly to avoid ambiguity with StringRef.
1076     std::string C = IsHwMode
1077         ? std::string("MF->getSubtarget().checkFeatures(\"" + Features + "\")")
1078         : std::string(Def->getValueAsString("CondString"));
1079     if (C.empty())
1080       return "";
1081     return IfCond ? C : "!("+C+')';
1082   }
1083
1084   bool operator==(const Predicate &P) const {
1085     return IfCond == P.IfCond && IsHwMode == P.IsHwMode && Def == P.Def;
1086   }
1087   bool operator<(const Predicate &P) const {
1088     if (IsHwMode != P.IsHwMode)
1089       return IsHwMode < P.IsHwMode;
1090     assert(!Def == !P.Def && "Inconsistency between Def and IsHwMode");
1091     if (IfCond != P.IfCond)
1092       return IfCond < P.IfCond;
1093     if (Def)
1094       return LessRecord()(Def, P.Def);
1095     return Features < P.Features;
1096   }
1097   Record *Def;            ///< Predicate definition from .td file, null for
1098                           ///< HW modes.
1099   std::string Features;   ///< Feature string for HW mode.
1100   bool IfCond;            ///< The boolean value that the condition has to
1101                           ///< evaluate to for this predicate to be true.
1102   bool IsHwMode;          ///< Does this predicate correspond to a HW mode?
1103 };
1104
1105 /// PatternToMatch - Used by CodeGenDAGPatterns to keep tab of patterns
1106 /// processed to produce isel.
1107 class PatternToMatch {
1108 public:
1109   PatternToMatch(Record *srcrecord, std::vector<Predicate> preds,
1110                  TreePatternNodePtr src, TreePatternNodePtr dst,
1111                  std::vector<Record *> dstregs, int complexity,
1112                  unsigned uid, unsigned setmode = 0)
1113       : SrcRecord(srcrecord), SrcPattern(src), DstPattern(dst),
1114         Predicates(std::move(preds)), Dstregs(std::move(dstregs)),
1115         AddedComplexity(complexity), ID(uid), ForceMode(setmode) {}
1116
1117   Record          *SrcRecord;   // Originating Record for the pattern.
1118   TreePatternNodePtr SrcPattern;      // Source pattern to match.
1119   TreePatternNodePtr DstPattern;      // Resulting pattern.
1120   std::vector<Predicate> Predicates;  // Top level predicate conditions
1121                                       // to match.
1122   std::vector<Record*> Dstregs; // Physical register defs being matched.
1123   int              AddedComplexity; // Add to matching pattern complexity.
1124   unsigned         ID;          // Unique ID for the record.
1125   unsigned         ForceMode;   // Force this mode in type inference when set.
1126
1127   Record          *getSrcRecord()  const { return SrcRecord; }
1128   TreePatternNode *getSrcPattern() const { return SrcPattern.get(); }
1129   TreePatternNodePtr getSrcPatternShared() const { return SrcPattern; }
1130   TreePatternNode *getDstPattern() const { return DstPattern.get(); }
1131   TreePatternNodePtr getDstPatternShared() const { return DstPattern; }
1132   const std::vector<Record*> &getDstRegs() const { return Dstregs; }
1133   int         getAddedComplexity() const { return AddedComplexity; }
1134   const std::vector<Predicate> &getPredicates() const { return Predicates; }
1135
1136   std::string getPredicateCheck() const;
1137
1138   /// Compute the complexity metric for the input pattern.  This roughly
1139   /// corresponds to the number of nodes that are covered.
1140   int getPatternComplexity(const CodeGenDAGPatterns &CGP) const;
1141 };
1142
1143 class CodeGenDAGPatterns {
1144   RecordKeeper &Records;
1145   CodeGenTarget Target;
1146   CodeGenIntrinsicTable Intrinsics;
1147   CodeGenIntrinsicTable TgtIntrinsics;
1148
1149   std::map<Record*, SDNodeInfo, LessRecordByID> SDNodes;
1150   std::map<Record*, std::pair<Record*, std::string>, LessRecordByID>
1151       SDNodeXForms;
1152   std::map<Record*, ComplexPattern, LessRecordByID> ComplexPatterns;
1153   std::map<Record *, std::unique_ptr<TreePattern>, LessRecordByID>
1154       PatternFragments;
1155   std::map<Record*, DAGDefaultOperand, LessRecordByID> DefaultOperands;
1156   std::map<Record*, DAGInstruction, LessRecordByID> Instructions;
1157
1158   // Specific SDNode definitions:
1159   Record *intrinsic_void_sdnode;
1160   Record *intrinsic_w_chain_sdnode, *intrinsic_wo_chain_sdnode;
1161
1162   /// PatternsToMatch - All of the things we are matching on the DAG.  The first
1163   /// value is the pattern to match, the second pattern is the result to
1164   /// emit.
1165   std::vector<PatternToMatch> PatternsToMatch;
1166
1167   TypeSetByHwMode LegalVTS;
1168
1169   using PatternRewriterFn = std::function<void (TreePattern *)>;
1170   PatternRewriterFn PatternRewriter;
1171
1172   unsigned NumScopes = 0;
1173
1174 public:
1175   CodeGenDAGPatterns(RecordKeeper &R,
1176                      PatternRewriterFn PatternRewriter = nullptr);
1177
1178   CodeGenTarget &getTargetInfo() { return Target; }
1179   const CodeGenTarget &getTargetInfo() const { return Target; }
1180   const TypeSetByHwMode &getLegalTypes() const { return LegalVTS; }
1181
1182   Record *getSDNodeNamed(const std::string &Name) const;
1183
1184   const SDNodeInfo &getSDNodeInfo(Record *R) const {
1185     auto F = SDNodes.find(R);
1186     assert(F != SDNodes.end() && "Unknown node!");
1187     return F->second;
1188   }
1189
1190   // Node transformation lookups.
1191   typedef std::pair<Record*, std::string> NodeXForm;
1192   const NodeXForm &getSDNodeTransform(Record *R) const {
1193     auto F = SDNodeXForms.find(R);
1194     assert(F != SDNodeXForms.end() && "Invalid transform!");
1195     return F->second;
1196   }
1197
1198   typedef std::map<Record*, NodeXForm, LessRecordByID>::const_iterator
1199           nx_iterator;
1200   nx_iterator nx_begin() const { return SDNodeXForms.begin(); }
1201   nx_iterator nx_end() const { return SDNodeXForms.end(); }
1202
1203
1204   const ComplexPattern &getComplexPattern(Record *R) const {
1205     auto F = ComplexPatterns.find(R);
1206     assert(F != ComplexPatterns.end() && "Unknown addressing mode!");
1207     return F->second;
1208   }
1209
1210   const CodeGenIntrinsic &getIntrinsic(Record *R) const {
1211     for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
1212       if (Intrinsics[i].TheDef == R) return Intrinsics[i];
1213     for (unsigned i = 0, e = TgtIntrinsics.size(); i != e; ++i)
1214       if (TgtIntrinsics[i].TheDef == R) return TgtIntrinsics[i];
1215     llvm_unreachable("Unknown intrinsic!");
1216   }
1217
1218   const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const {
1219     if (IID-1 < Intrinsics.size())
1220       return Intrinsics[IID-1];
1221     if (IID-Intrinsics.size()-1 < TgtIntrinsics.size())
1222       return TgtIntrinsics[IID-Intrinsics.size()-1];
1223     llvm_unreachable("Bad intrinsic ID!");
1224   }
1225
1226   unsigned getIntrinsicID(Record *R) const {
1227     for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
1228       if (Intrinsics[i].TheDef == R) return i;
1229     for (unsigned i = 0, e = TgtIntrinsics.size(); i != e; ++i)
1230       if (TgtIntrinsics[i].TheDef == R) return i + Intrinsics.size();
1231     llvm_unreachable("Unknown intrinsic!");
1232   }
1233
1234   const DAGDefaultOperand &getDefaultOperand(Record *R) const {
1235     auto F = DefaultOperands.find(R);
1236     assert(F != DefaultOperands.end() &&"Isn't an analyzed default operand!");
1237     return F->second;
1238   }
1239
1240   // Pattern Fragment information.
1241   TreePattern *getPatternFragment(Record *R) const {
1242     auto F = PatternFragments.find(R);
1243     assert(F != PatternFragments.end() && "Invalid pattern fragment request!");
1244     return F->second.get();
1245   }
1246   TreePattern *getPatternFragmentIfRead(Record *R) const {
1247     auto F = PatternFragments.find(R);
1248     if (F == PatternFragments.end())
1249       return nullptr;
1250     return F->second.get();
1251   }
1252
1253   typedef std::map<Record *, std::unique_ptr<TreePattern>,
1254                    LessRecordByID>::const_iterator pf_iterator;
1255   pf_iterator pf_begin() const { return PatternFragments.begin(); }
1256   pf_iterator pf_end() const { return PatternFragments.end(); }
1257   iterator_range<pf_iterator> ptfs() const { return PatternFragments; }
1258
1259   // Patterns to match information.
1260   typedef std::vector<PatternToMatch>::const_iterator ptm_iterator;
1261   ptm_iterator ptm_begin() const { return PatternsToMatch.begin(); }
1262   ptm_iterator ptm_end() const { return PatternsToMatch.end(); }
1263   iterator_range<ptm_iterator> ptms() const { return PatternsToMatch; }
1264
1265   /// Parse the Pattern for an instruction, and insert the result in DAGInsts.
1266   typedef std::map<Record*, DAGInstruction, LessRecordByID> DAGInstMap;
1267   void parseInstructionPattern(
1268       CodeGenInstruction &CGI, ListInit *Pattern,
1269       DAGInstMap &DAGInsts);
1270
1271   const DAGInstruction &getInstruction(Record *R) const {
1272     auto F = Instructions.find(R);
1273     assert(F != Instructions.end() && "Unknown instruction!");
1274     return F->second;
1275   }
1276
1277   Record *get_intrinsic_void_sdnode() const {
1278     return intrinsic_void_sdnode;
1279   }
1280   Record *get_intrinsic_w_chain_sdnode() const {
1281     return intrinsic_w_chain_sdnode;
1282   }
1283   Record *get_intrinsic_wo_chain_sdnode() const {
1284     return intrinsic_wo_chain_sdnode;
1285   }
1286
1287   bool hasTargetIntrinsics() { return !TgtIntrinsics.empty(); }
1288
1289   unsigned allocateScope() { return ++NumScopes; }
1290
1291   bool operandHasDefault(Record *Op) const {
1292     return Op->isSubClassOf("OperandWithDefaultOps") &&
1293       !getDefaultOperand(Op).DefaultOps.empty();
1294   }
1295
1296 private:
1297   void ParseNodeInfo();
1298   void ParseNodeTransforms();
1299   void ParseComplexPatterns();
1300   void ParsePatternFragments(bool OutFrags = false);
1301   void ParseDefaultOperands();
1302   void ParseInstructions();
1303   void ParsePatterns();
1304   void ExpandHwModeBasedTypes();
1305   void InferInstructionFlags();
1306   void GenerateVariants();
1307   void VerifyInstructionFlags();
1308
1309   std::vector<Predicate> makePredList(ListInit *L);
1310
1311   void ParseOnePattern(Record *TheDef,
1312                        TreePattern &Pattern, TreePattern &Result,
1313                        const std::vector<Record *> &InstImpResults);
1314   void AddPatternToMatch(TreePattern *Pattern, PatternToMatch &&PTM);
1315   void FindPatternInputsAndOutputs(
1316       TreePattern &I, TreePatternNodePtr Pat,
1317       std::map<std::string, TreePatternNodePtr> &InstInputs,
1318       MapVector<std::string, TreePatternNodePtr,
1319                 std::map<std::string, unsigned>> &InstResults,
1320       std::vector<Record *> &InstImpResults);
1321 };
1322
1323
1324 inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode *N,
1325                                              TreePattern &TP) const {
1326     bool MadeChange = false;
1327     for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i)
1328       MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP);
1329     return MadeChange;
1330   }
1331
1332 } // end namespace llvm
1333
1334 #endif