llvm/utils/TableGen/CodeGenDAGPatterns.h

   1 //===- CodeGenDAGPatterns.h - Read DAG patterns from .td file ---*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file declares the CodeGenDAGPatterns class, which is used to read and
  10 // represent the patterns present in a .td file for instructions.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
  15 #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
  16
  17 #include "CodeGenIntrinsics.h"
  18 #include "CodeGenTarget.h"
  19 #include "SDNodeProperties.h"
  20 #include "llvm/ADT/MapVector.h"
  21 #include "llvm/ADT/SmallVector.h"
  22 #include "llvm/ADT/StringMap.h"
  23 #include "llvm/ADT/StringSet.h"
  24 #include "llvm/Support/ErrorHandling.h"
  25 #include "llvm/Support/MathExtras.h"
  26 #include <algorithm>
  27 #include <array>
  28 #include <functional>
  29 #include <map>
  30 #include <numeric>
  31 #include <set>
  32 #include <vector>
  33
  34 namespace llvm {
  35
  36 class Record;
  37 class Init;
  38 class ListInit;
  39 class DagInit;
  40 class SDNodeInfo;
  41 class TreePattern;
  42 class TreePatternNode;
  43 class CodeGenDAGPatterns;
  44
  45 /// Shared pointer for TreePatternNode.
  46 using TreePatternNodePtr = std::shared_ptr<TreePatternNode>;
  47
  48 /// This represents a set of MVTs. Since the underlying type for the MVT
  49 /// is uint8_t, there are at most 256 values. To reduce the number of memory
  50 /// allocations and deallocations, represent the set as a sequence of bits.
  51 /// To reduce the allocations even further, make MachineValueTypeSet own
  52 /// the storage and use std::array as the bit container.
  53 struct MachineValueTypeSet {
  54   static_assert(std::is_same<std::underlying_type<MVT::SimpleValueType>::type,
  55                              uint8_t>::value,
  56                 "Change uint8_t here to the SimpleValueType's type");
  57   static unsigned constexpr Capacity = std::numeric_limits<uint8_t>::max()+1;
  58   using WordType = uint64_t;
  59   static unsigned constexpr WordWidth = CHAR_BIT*sizeof(WordType);
  60   static unsigned constexpr NumWords = Capacity/WordWidth;
  61   static_assert(NumWords*WordWidth == Capacity,
  62                 "Capacity should be a multiple of WordWidth");
  63
  64   LLVM_ATTRIBUTE_ALWAYS_INLINE
  65   MachineValueTypeSet() {
  66     clear();
  67   }
  68
  69   LLVM_ATTRIBUTE_ALWAYS_INLINE
  70   unsigned size() const {
  71     unsigned Count = 0;
  72     for (WordType W : Words)
  73       Count += countPopulation(W);
  74     return Count;
  75   }
  76   LLVM_ATTRIBUTE_ALWAYS_INLINE
  77   void clear() {
  78     std::memset(Words.data(), 0, NumWords*sizeof(WordType));
  79   }
  80   LLVM_ATTRIBUTE_ALWAYS_INLINE
  81   bool empty() const {
  82     for (WordType W : Words)
  83       if (W != 0)
  84         return false;
  85     return true;
  86   }
  87   LLVM_ATTRIBUTE_ALWAYS_INLINE
  88   unsigned count(MVT T) const {
  89     return (Words[T.SimpleTy / WordWidth] >> (T.SimpleTy % WordWidth)) & 1;
  90   }
  91   std::pair<MachineValueTypeSet&,bool> insert(MVT T) {
  92     bool V = count(T.SimpleTy);
  93     Words[T.SimpleTy / WordWidth] |= WordType(1) << (T.SimpleTy % WordWidth);
  94     return {*this, V};
  95   }
  96   MachineValueTypeSet &insert(const MachineValueTypeSet &S) {
  97     for (unsigned i = 0; i != NumWords; ++i)
  98       Words[i] |= S.Words[i];
  99     return *this;
 100   }
 101   LLVM_ATTRIBUTE_ALWAYS_INLINE
 102   void erase(MVT T) {
 103     Words[T.SimpleTy / WordWidth] &= ~(WordType(1) << (T.SimpleTy % WordWidth));
 104   }
 105
 106   struct const_iterator {
 107     // Some implementations of the C++ library require these traits to be
 108     // defined.
 109     using iterator_category = std::forward_iterator_tag;
 110     using value_type = MVT;
 111     using difference_type = ptrdiff_t;
 112     using pointer = const MVT*;
 113     using reference = const MVT&;
 114
 115     LLVM_ATTRIBUTE_ALWAYS_INLINE
 116     MVT operator*() const {
 117       assert(Pos != Capacity);
 118       return MVT::SimpleValueType(Pos);
 119     }
 120     LLVM_ATTRIBUTE_ALWAYS_INLINE
 121     const_iterator(const MachineValueTypeSet *S, bool End) : Set(S) {
 122       Pos = End ? Capacity : find_from_pos(0);
 123     }
 124     LLVM_ATTRIBUTE_ALWAYS_INLINE
 125     const_iterator &operator++() {
 126       assert(Pos != Capacity);
 127       Pos = find_from_pos(Pos+1);
 128       return *this;
 129     }
 130
 131     LLVM_ATTRIBUTE_ALWAYS_INLINE
 132     bool operator==(const const_iterator &It) const {
 133       return Set == It.Set && Pos == It.Pos;
 134     }
 135     LLVM_ATTRIBUTE_ALWAYS_INLINE
 136     bool operator!=(const const_iterator &It) const {
 137       return !operator==(It);
 138     }
 139
 140   private:
 141     unsigned find_from_pos(unsigned P) const {
 142       unsigned SkipWords = P / WordWidth;
 143       unsigned SkipBits = P % WordWidth;
 144       unsigned Count = SkipWords * WordWidth;
 145
 146       // If P is in the middle of a word, process it manually here, because
 147       // the trailing bits need to be masked off to use findFirstSet.
 148       if (SkipBits != 0) {
 149         WordType W = Set->Words[SkipWords];
 150         W &= maskLeadingOnes<WordType>(WordWidth-SkipBits);
 151         if (W != 0)
 152           return Count + findFirstSet(W);
 153         Count += WordWidth;
 154         SkipWords++;
 155       }
 156
 157       for (unsigned i = SkipWords; i != NumWords; ++i) {
 158         WordType W = Set->Words[i];
 159         if (W != 0)
 160           return Count + findFirstSet(W);
 161         Count += WordWidth;
 162       }
 163       return Capacity;
 164     }
 165
 166     const MachineValueTypeSet *Set;
 167     unsigned Pos;
 168   };
 169
 170   LLVM_ATTRIBUTE_ALWAYS_INLINE
 171   const_iterator begin() const { return const_iterator(this, false); }
 172   LLVM_ATTRIBUTE_ALWAYS_INLINE
 173   const_iterator end()   const { return const_iterator(this, true); }
 174
 175   LLVM_ATTRIBUTE_ALWAYS_INLINE
 176   bool operator==(const MachineValueTypeSet &S) const {
 177     return Words == S.Words;
 178   }
 179   LLVM_ATTRIBUTE_ALWAYS_INLINE
 180   bool operator!=(const MachineValueTypeSet &S) const {
 181     return !operator==(S);
 182   }
 183
 184 private:
 185   friend struct const_iterator;
 186   std::array<WordType,NumWords> Words;
 187 };
 188
 189 struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
 190   using SetType = MachineValueTypeSet;
 191   SmallVector<unsigned, 16> AddrSpaces;
 192
 193   TypeSetByHwMode() = default;
 194   TypeSetByHwMode(const TypeSetByHwMode &VTS) = default;
 195   TypeSetByHwMode &operator=(const TypeSetByHwMode &) = default;
 196   TypeSetByHwMode(MVT::SimpleValueType VT)
 197     : TypeSetByHwMode(ValueTypeByHwMode(VT)) {}
 198   TypeSetByHwMode(ValueTypeByHwMode VT)
 199     : TypeSetByHwMode(ArrayRef<ValueTypeByHwMode>(&VT, 1)) {}
 200   TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList);
 201
 202   SetType &getOrCreate(unsigned Mode) {
 203     return Map[Mode];
 204   }
 205
 206   bool isValueTypeByHwMode(bool AllowEmpty) const;
 207   ValueTypeByHwMode getValueTypeByHwMode() const;
 208
 209   LLVM_ATTRIBUTE_ALWAYS_INLINE
 210   bool isMachineValueType() const {
 211     return isDefaultOnly() && Map.begin()->second.size() == 1;
 212   }
 213
 214   LLVM_ATTRIBUTE_ALWAYS_INLINE
 215   MVT getMachineValueType() const {
 216     assert(isMachineValueType());
 217     return *Map.begin()->second.begin();
 218   }
 219
 220   bool isPossible() const;
 221
 222   LLVM_ATTRIBUTE_ALWAYS_INLINE
 223   bool isDefaultOnly() const {
 224     return Map.size() == 1 && Map.begin()->first == DefaultMode;
 225   }
 226
 227   bool isPointer() const {
 228     return getValueTypeByHwMode().isPointer();
 229   }
 230
 231   unsigned getPtrAddrSpace() const {
 232     assert(isPointer());
 233     return getValueTypeByHwMode().PtrAddrSpace;
 234   }
 235
 236   bool insert(const ValueTypeByHwMode &VVT);
 237   bool constrain(const TypeSetByHwMode &VTS);
 238   template <typename Predicate> bool constrain(Predicate P);
 239   template <typename Predicate>
 240   bool assign_if(const TypeSetByHwMode &VTS, Predicate P);
 241
 242   void writeToStream(raw_ostream &OS) const;
 243   static void writeToStream(const SetType &S, raw_ostream &OS);
 244
 245   bool operator==(const TypeSetByHwMode &VTS) const;
 246   bool operator!=(const TypeSetByHwMode &VTS) const { return !(*this == VTS); }
 247
 248   void dump() const;
 249   bool validate() const;
 250
 251 private:
 252   unsigned PtrAddrSpace = std::numeric_limits<unsigned>::max();
 253   /// Intersect two sets. Return true if anything has changed.
 254   bool intersect(SetType &Out, const SetType &In);
 255 };
 256
 257 raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T);
 258
 259 struct TypeInfer {
 260   TypeInfer(TreePattern &T) : TP(T), ForceMode(0) {}
 261
 262   bool isConcrete(const TypeSetByHwMode &VTS, bool AllowEmpty) const {
 263     return VTS.isValueTypeByHwMode(AllowEmpty);
 264   }
 265   ValueTypeByHwMode getConcrete(const TypeSetByHwMode &VTS,
 266                                 bool AllowEmpty) const {
 267     assert(VTS.isValueTypeByHwMode(AllowEmpty));
 268     return VTS.getValueTypeByHwMode();
 269   }
 270
 271   /// The protocol in the following functions (Merge*, force*, Enforce*,
 272   /// expand*) is to return "true" if a change has been made, "false"
 273   /// otherwise.
 274
 275   bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In);
 276   bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) {
 277     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
 278   }
 279   bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) {
 280     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
 281   }
 282
 283   /// Reduce the set \p Out to have at most one element for each mode.
 284   bool forceArbitrary(TypeSetByHwMode &Out);
 285
 286   /// The following four functions ensure that upon return the set \p Out
 287   /// will only contain types of the specified kind: integer, floating-point,
 288   /// scalar, or vector.
 289   /// If \p Out is empty, all legal types of the specified kind will be added
 290   /// to it. Otherwise, all types that are not of the specified kind will be
 291   /// removed from \p Out.
 292   bool EnforceInteger(TypeSetByHwMode &Out);
 293   bool EnforceFloatingPoint(TypeSetByHwMode &Out);
 294   bool EnforceScalar(TypeSetByHwMode &Out);
 295   bool EnforceVector(TypeSetByHwMode &Out);
 296
 297   /// If \p Out is empty, fill it with all legal types. Otherwise, leave it
 298   /// unchanged.
 299   bool EnforceAny(TypeSetByHwMode &Out);
 300   /// Make sure that for each type in \p Small, there exists a larger type
 301   /// in \p Big.
 302   bool EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big);
 303   /// 1. Ensure that for each type T in \p Vec, T is a vector type, and that
 304   ///    for each type U in \p Elem, U is a scalar type.
 305   /// 2. Ensure that for each (scalar) type U in \p Elem, there exists a
 306   ///    (vector) type T in \p Vec, such that U is the element type of T.
 307   bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, TypeSetByHwMode &Elem);
 308   bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
 309                               const ValueTypeByHwMode &VVT);
 310   /// Ensure that for each type T in \p Sub, T is a vector type, and there
 311   /// exists a type U in \p Vec such that U is a vector type with the same
 312   /// element type as T and at least as many elements as T.
 313   bool EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
 314                                     TypeSetByHwMode &Sub);
 315   /// 1. Ensure that \p V has a scalar type iff \p W has a scalar type.
 316   /// 2. Ensure that for each vector type T in \p V, there exists a vector
 317   ///    type U in \p W, such that T and U have the same number of elements.
 318   /// 3. Ensure that for each vector type U in \p W, there exists a vector
 319   ///    type T in \p V, such that T and U have the same number of elements
 320   ///    (reverse of 2).
 321   bool EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W);
 322   /// 1. Ensure that for each type T in \p A, there exists a type U in \p B,
 323   ///    such that T and U have equal size in bits.
 324   /// 2. Ensure that for each type U in \p B, there exists a type T in \p A
 325   ///    such that T and U have equal size in bits (reverse of 1).
 326   bool EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B);
 327
 328   /// For each overloaded type (i.e. of form *Any), replace it with the
 329   /// corresponding subset of legal, specific types.
 330   void expandOverloads(TypeSetByHwMode &VTS);
 331   void expandOverloads(TypeSetByHwMode::SetType &Out,
 332                        const TypeSetByHwMode::SetType &Legal);
 333
 334   struct ValidateOnExit {
 335     ValidateOnExit(TypeSetByHwMode &T, TypeInfer &TI) : Infer(TI), VTS(T) {}
 336   #ifndef NDEBUG
 337     ~ValidateOnExit();
 338   #else
 339     ~ValidateOnExit() {}  // Empty destructor with NDEBUG.
 340   #endif
 341     TypeInfer &Infer;
 342     TypeSetByHwMode &VTS;
 343   };
 344
 345   struct SuppressValidation {
 346     SuppressValidation(TypeInfer &TI) : Infer(TI), SavedValidate(TI.Validate) {
 347       Infer.Validate = false;
 348     }
 349     ~SuppressValidation() {
 350       Infer.Validate = SavedValidate;
 351     }
 352     TypeInfer &Infer;
 353     bool SavedValidate;
 354   };
 355
 356   TreePattern &TP;
 357   unsigned ForceMode;     // Mode to use when set.
 358   bool CodeGen = false;   // Set during generation of matcher code.
 359   bool Validate = true;   // Indicate whether to validate types.
 360
 361 private:
 362   const TypeSetByHwMode &getLegalTypes();
 363
 364   /// Cached legal types (in default mode).
 365   bool LegalTypesCached = false;
 366   TypeSetByHwMode LegalCache;
 367 };
 368
 369 /// Set type used to track multiply used variables in patterns
 370 typedef StringSet<> MultipleUseVarSet;
 371
 372 /// SDTypeConstraint - This is a discriminated union of constraints,
 373 /// corresponding to the SDTypeConstraint tablegen class in Target.td.
 374 struct SDTypeConstraint {
 375   SDTypeConstraint(Record *R, const CodeGenHwModes &CGH);
 376
 377   unsigned OperandNo;   // The operand # this constraint applies to.
 378   enum {
 379     SDTCisVT, SDTCisPtrTy, SDTCisInt, SDTCisFP, SDTCisVec, SDTCisSameAs,
 380     SDTCisVTSmallerThanOp, SDTCisOpSmallerThanOp, SDTCisEltOfVec,
 381     SDTCisSubVecOfVec, SDTCVecEltisVT, SDTCisSameNumEltsAs, SDTCisSameSizeAs
 382   } ConstraintType;
 383
 384   union {   // The discriminated union.
 385     struct {
 386       unsigned OtherOperandNum;
 387     } SDTCisSameAs_Info;
 388     struct {
 389       unsigned OtherOperandNum;
 390     } SDTCisVTSmallerThanOp_Info;
 391     struct {
 392       unsigned BigOperandNum;
 393     } SDTCisOpSmallerThanOp_Info;
 394     struct {
 395       unsigned OtherOperandNum;
 396     } SDTCisEltOfVec_Info;
 397     struct {
 398       unsigned OtherOperandNum;
 399     } SDTCisSubVecOfVec_Info;
 400     struct {
 401       unsigned OtherOperandNum;
 402     } SDTCisSameNumEltsAs_Info;
 403     struct {
 404       unsigned OtherOperandNum;
 405     } SDTCisSameSizeAs_Info;
 406   } x;
 407
 408   // The VT for SDTCisVT and SDTCVecEltisVT.
 409   // Must not be in the union because it has a non-trivial destructor.
 410   ValueTypeByHwMode VVT;
 411
 412   /// ApplyTypeConstraint - Given a node in a pattern, apply this type
 413   /// constraint to the nodes operands.  This returns true if it makes a
 414   /// change, false otherwise.  If a type contradiction is found, an error
 415   /// is flagged.
 416   bool ApplyTypeConstraint(TreePatternNode *N, const SDNodeInfo &NodeInfo,
 417                            TreePattern &TP) const;
 418 };
 419
 420 /// ScopedName - A name of a node associated with a "scope" that indicates
 421 /// the context (e.g. instance of Pattern or PatFrag) in which the name was
 422 /// used. This enables substitution of pattern fragments while keeping track
 423 /// of what name(s) were originally given to various nodes in the tree.
 424 class ScopedName {
 425   unsigned Scope;
 426   std::string Identifier;
 427 public:
 428   ScopedName(unsigned Scope, StringRef Identifier)
 429       : Scope(Scope), Identifier(std::string(Identifier)) {
 430     assert(Scope != 0 &&
 431            "Scope == 0 is used to indicate predicates without arguments");
 432   }
 433
 434   unsigned getScope() const { return Scope; }
 435   const std::string &getIdentifier() const { return Identifier; }
 436
 437   bool operator==(const ScopedName &o) const;
 438   bool operator!=(const ScopedName &o) const;
 439 };
 440
 441 /// SDNodeInfo - One of these records is created for each SDNode instance in
 442 /// the target .td file.  This represents the various dag nodes we will be
 443 /// processing.
 444 class SDNodeInfo {
 445   Record *Def;
 446   StringRef EnumName;
 447   StringRef SDClassName;
 448   unsigned Properties;
 449   unsigned NumResults;
 450   int NumOperands;
 451   std::vector<SDTypeConstraint> TypeConstraints;
 452 public:
 453   // Parse the specified record.
 454   SDNodeInfo(Record *R, const CodeGenHwModes &CGH);
 455
 456   unsigned getNumResults() const { return NumResults; }
 457
 458   /// getNumOperands - This is the number of operands required or -1 if
 459   /// variadic.
 460   int getNumOperands() const { return NumOperands; }
 461   Record *getRecord() const { return Def; }
 462   StringRef getEnumName() const { return EnumName; }
 463   StringRef getSDClassName() const { return SDClassName; }
 464
 465   const std::vector<SDTypeConstraint> &getTypeConstraints() const {
 466     return TypeConstraints;
 467   }
 468
 469   /// getKnownType - If the type constraints on this node imply a fixed type
 470   /// (e.g. all stores return void, etc), then return it as an
 471   /// MVT::SimpleValueType.  Otherwise, return MVT::Other.
 472   MVT::SimpleValueType getKnownType(unsigned ResNo) const;
 473
 474   /// hasProperty - Return true if this node has the specified property.
 475   ///
 476   bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }
 477
 478   /// ApplyTypeConstraints - Given a node in a pattern, apply the type
 479   /// constraints for this node to the operands of the node.  This returns
 480   /// true if it makes a change, false otherwise.  If a type contradiction is
 481   /// found, an error is flagged.
 482   bool ApplyTypeConstraints(TreePatternNode *N, TreePattern &TP) const;
 483 };
 484
 485 /// TreePredicateFn - This is an abstraction that represents the predicates on
 486 /// a PatFrag node.  This is a simple one-word wrapper around a pointer to
 487 /// provide nice accessors.
 488 class TreePredicateFn {
 489   /// PatFragRec - This is the TreePattern for the PatFrag that we
 490   /// originally came from.
 491   TreePattern *PatFragRec;
 492 public:
 493   /// TreePredicateFn constructor.  Here 'N' is a subclass of PatFrag.
 494   TreePredicateFn(TreePattern *N);
 495
 496
 497   TreePattern *getOrigPatFragRecord() const { return PatFragRec; }
 498
 499   /// isAlwaysTrue - Return true if this is a noop predicate.
 500   bool isAlwaysTrue() const;
 501
 502   bool isImmediatePattern() const { return hasImmCode(); }
 503
 504   /// getImmediatePredicateCode - Return the code that evaluates this pattern if
 505   /// this is an immediate predicate.  It is an error to call this on a
 506   /// non-immediate pattern.
 507   std::string getImmediatePredicateCode() const {
 508     std::string Result = getImmCode();
 509     assert(!Result.empty() && "Isn't an immediate pattern!");
 510     return Result;
 511   }
 512
 513   bool operator==(const TreePredicateFn &RHS) const {
 514     return PatFragRec == RHS.PatFragRec;
 515   }
 516
 517   bool operator!=(const TreePredicateFn &RHS) const { return !(*this == RHS); }
 518
 519   /// Return the name to use in the generated code to reference this, this is
 520   /// "Predicate_foo" if from a pattern fragment "foo".
 521   std::string getFnName() const;
 522
 523   /// getCodeToRunOnSDNode - Return the code for the function body that
 524   /// evaluates this predicate.  The argument is expected to be in "Node",
 525   /// not N.  This handles casting and conversion to a concrete node type as
 526   /// appropriate.
 527   std::string getCodeToRunOnSDNode() const;
 528
 529   /// Get the data type of the argument to getImmediatePredicateCode().
 530   StringRef getImmType() const;
 531
 532   /// Get a string that describes the type returned by getImmType() but is
 533   /// usable as part of an identifier.
 534   StringRef getImmTypeIdentifier() const;
 535
 536   // Predicate code uses the PatFrag's captured operands.
 537   bool usesOperands() const;
 538
 539   // Is the desired predefined predicate for a load?
 540   bool isLoad() const;
 541   // Is the desired predefined predicate for a store?
 542   bool isStore() const;
 543   // Is the desired predefined predicate for an atomic?
 544   bool isAtomic() const;
 545
 546   /// Is this predicate the predefined unindexed load predicate?
 547   /// Is this predicate the predefined unindexed store predicate?
 548   bool isUnindexed() const;
 549   /// Is this predicate the predefined non-extending load predicate?
 550   bool isNonExtLoad() const;
 551   /// Is this predicate the predefined any-extend load predicate?
 552   bool isAnyExtLoad() const;
 553   /// Is this predicate the predefined sign-extend load predicate?
 554   bool isSignExtLoad() const;
 555   /// Is this predicate the predefined zero-extend load predicate?
 556   bool isZeroExtLoad() const;
 557   /// Is this predicate the predefined non-truncating store predicate?
 558   bool isNonTruncStore() const;
 559   /// Is this predicate the predefined truncating store predicate?
 560   bool isTruncStore() const;
 561
 562   /// Is this predicate the predefined monotonic atomic predicate?
 563   bool isAtomicOrderingMonotonic() const;
 564   /// Is this predicate the predefined acquire atomic predicate?
 565   bool isAtomicOrderingAcquire() const;
 566   /// Is this predicate the predefined release atomic predicate?
 567   bool isAtomicOrderingRelease() const;
 568   /// Is this predicate the predefined acquire-release atomic predicate?
 569   bool isAtomicOrderingAcquireRelease() const;
 570   /// Is this predicate the predefined sequentially consistent atomic predicate?
 571   bool isAtomicOrderingSequentiallyConsistent() const;
 572
 573   /// Is this predicate the predefined acquire-or-stronger atomic predicate?
 574   bool isAtomicOrderingAcquireOrStronger() const;
 575   /// Is this predicate the predefined weaker-than-acquire atomic predicate?
 576   bool isAtomicOrderingWeakerThanAcquire() const;
 577
 578   /// Is this predicate the predefined release-or-stronger atomic predicate?
 579   bool isAtomicOrderingReleaseOrStronger() const;
 580   /// Is this predicate the predefined weaker-than-release atomic predicate?
 581   bool isAtomicOrderingWeakerThanRelease() const;
 582
 583   /// If non-null, indicates that this predicate is a predefined memory VT
 584   /// predicate for a load/store and returns the ValueType record for the memory VT.
 585   Record *getMemoryVT() const;
 586   /// If non-null, indicates that this predicate is a predefined memory VT
 587   /// predicate (checking only the scalar type) for load/store and returns the
 588   /// ValueType record for the memory VT.
 589   Record *getScalarMemoryVT() const;
 590
 591   ListInit *getAddressSpaces() const;
 592   int64_t getMinAlignment() const;
 593
 594   // If true, indicates that GlobalISel-based C++ code was supplied.
 595   bool hasGISelPredicateCode() const;
 596   std::string getGISelPredicateCode() const;
 597
 598 private:
 599   bool hasPredCode() const;
 600   bool hasImmCode() const;
 601   std::string getPredCode() const;
 602   std::string getImmCode() const;
 603   bool immCodeUsesAPInt() const;
 604   bool immCodeUsesAPFloat() const;
 605
 606   bool isPredefinedPredicateEqualTo(StringRef Field, bool Value) const;
 607 };
 608
 609 struct TreePredicateCall {
 610   TreePredicateFn Fn;
 611
 612   // Scope -- unique identifier for retrieving named arguments. 0 is used when
 613   // the predicate does not use named arguments.
 614   unsigned Scope;
 615
 616   TreePredicateCall(const TreePredicateFn &Fn, unsigned Scope)
 617     : Fn(Fn), Scope(Scope) {}
 618
 619   bool operator==(const TreePredicateCall &o) const {
 620     return Fn == o.Fn && Scope == o.Scope;
 621   }
 622   bool operator!=(const TreePredicateCall &o) const {
 623     return !(*this == o);
 624   }
 625 };
 626
 627 class TreePatternNode {
 628   /// The type of each node result.  Before and during type inference, each
 629   /// result may be a set of possible types.  After (successful) type inference,
 630   /// each is a single concrete type.
 631   std::vector<TypeSetByHwMode> Types;
 632
 633   /// The index of each result in results of the pattern.
 634   std::vector<unsigned> ResultPerm;
 635
 636   /// Operator - The Record for the operator if this is an interior node (not
 637   /// a leaf).
 638   Record *Operator;
 639
 640   /// Val - The init value (e.g. the "GPRC" record, or "7") for a leaf.
 641   ///
 642   Init *Val;
 643
 644   /// Name - The name given to this node with the :$foo notation.
 645   ///
 646   std::string Name;
 647
 648   std::vector<ScopedName> NamesAsPredicateArg;
 649
 650   /// PredicateCalls - The predicate functions to execute on this node to check
 651   /// for a match.  If this list is empty, no predicate is involved.
 652   std::vector<TreePredicateCall> PredicateCalls;
 653
 654   /// TransformFn - The transformation function to execute on this node before
 655   /// it can be substituted into the resulting instruction on a pattern match.
 656   Record *TransformFn;
 657
 658   std::vector<TreePatternNodePtr> Children;
 659
 660 public:
 661   TreePatternNode(Record *Op, std::vector<TreePatternNodePtr> Ch,
 662                   unsigned NumResults)
 663       : Operator(Op), Val(nullptr), TransformFn(nullptr),
 664         Children(std::move(Ch)) {
 665     Types.resize(NumResults);
 666     ResultPerm.resize(NumResults);
 667     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
 668   }
 669   TreePatternNode(Init *val, unsigned NumResults)    // leaf ctor
 670     : Operator(nullptr), Val(val), TransformFn(nullptr) {
 671     Types.resize(NumResults);
 672     ResultPerm.resize(NumResults);
 673     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
 674   }
 675
 676   bool hasName() const { return !Name.empty(); }
 677   const std::string &getName() const { return Name; }
 678   void setName(StringRef N) { Name.assign(N.begin(), N.end()); }
 679
 680   const std::vector<ScopedName> &getNamesAsPredicateArg() const {
 681     return NamesAsPredicateArg;
 682   }
 683   void setNamesAsPredicateArg(const std::vector<ScopedName>& Names) {
 684     NamesAsPredicateArg = Names;
 685   }
 686   void addNameAsPredicateArg(const ScopedName &N) {
 687     NamesAsPredicateArg.push_back(N);
 688   }
 689
 690   bool isLeaf() const { return Val != nullptr; }
 691
 692   // Type accessors.
 693   unsigned getNumTypes() const { return Types.size(); }
 694   ValueTypeByHwMode getType(unsigned ResNo) const {
 695     return Types[ResNo].getValueTypeByHwMode();
 696   }
 697   const std::vector<TypeSetByHwMode> &getExtTypes() const { return Types; }
 698   const TypeSetByHwMode &getExtType(unsigned ResNo) const {
 699     return Types[ResNo];
 700   }
 701   TypeSetByHwMode &getExtType(unsigned ResNo) { return Types[ResNo]; }
 702   void setType(unsigned ResNo, const TypeSetByHwMode &T) { Types[ResNo] = T; }
 703   MVT::SimpleValueType getSimpleType(unsigned ResNo) const {
 704     return Types[ResNo].getMachineValueType().SimpleTy;
 705   }
 706
 707   bool hasConcreteType(unsigned ResNo) const {
 708     return Types[ResNo].isValueTypeByHwMode(false);
 709   }
 710   bool isTypeCompletelyUnknown(unsigned ResNo, TreePattern &TP) const {
 711     return Types[ResNo].empty();
 712   }
 713
 714   unsigned getNumResults() const { return ResultPerm.size(); }
 715   unsigned getResultIndex(unsigned ResNo) const { return ResultPerm[ResNo]; }
 716   void setResultIndex(unsigned ResNo, unsigned RI) { ResultPerm[ResNo] = RI; }
 717
 718   Init *getLeafValue() const { assert(isLeaf()); return Val; }
 719   Record *getOperator() const { assert(!isLeaf()); return Operator; }
 720
 721   unsigned getNumChildren() const { return Children.size(); }
 722   TreePatternNode *getChild(unsigned N) const { return Children[N].get(); }
 723   const TreePatternNodePtr &getChildShared(unsigned N) const {
 724     return Children[N];
 725   }
 726   void setChild(unsigned i, TreePatternNodePtr N) { Children[i] = N; }
 727
 728   /// hasChild - Return true if N is any of our children.
 729   bool hasChild(const TreePatternNode *N) const {
 730     for (unsigned i = 0, e = Children.size(); i != e; ++i)
 731       if (Children[i].get() == N)
 732         return true;
 733     return false;
 734   }
 735
 736   bool hasProperTypeByHwMode() const;
 737   bool hasPossibleType() const;
 738   bool setDefaultMode(unsigned Mode);
 739
 740   bool hasAnyPredicate() const { return !PredicateCalls.empty(); }
 741
 742   const std::vector<TreePredicateCall> &getPredicateCalls() const {
 743     return PredicateCalls;
 744   }
 745   void clearPredicateCalls() { PredicateCalls.clear(); }
 746   void setPredicateCalls(const std::vector<TreePredicateCall> &Calls) {
 747     assert(PredicateCalls.empty() && "Overwriting non-empty predicate list!");
 748     PredicateCalls = Calls;
 749   }
 750   void addPredicateCall(const TreePredicateCall &Call) {
 751     assert(!Call.Fn.isAlwaysTrue() && "Empty predicate string!");
 752     assert(!is_contained(PredicateCalls, Call) && "predicate applied recursively");
 753     PredicateCalls.push_back(Call);
 754   }
 755   void addPredicateCall(const TreePredicateFn &Fn, unsigned Scope) {
 756     assert((Scope != 0) == Fn.usesOperands());
 757     addPredicateCall(TreePredicateCall(Fn, Scope));
 758   }
 759
 760   Record *getTransformFn() const { return TransformFn; }
 761   void setTransformFn(Record *Fn) { TransformFn = Fn; }
 762
 763   /// getIntrinsicInfo - If this node corresponds to an intrinsic, return the
 764   /// CodeGenIntrinsic information for it, otherwise return a null pointer.
 765   const CodeGenIntrinsic *getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const;
 766
 767   /// getComplexPatternInfo - If this node corresponds to a ComplexPattern,
 768   /// return the ComplexPattern information, otherwise return null.
 769   const ComplexPattern *
 770   getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const;
 771
 772   /// Returns the number of MachineInstr operands that would be produced by this
 773   /// node if it mapped directly to an output Instruction's
 774   /// operand. ComplexPattern specifies this explicitly; MIOperandInfo gives it
 775   /// for Operands; otherwise 1.
 776   unsigned getNumMIResults(const CodeGenDAGPatterns &CGP) const;
 777
 778   /// NodeHasProperty - Return true if this node has the specified property.
 779   bool NodeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const;
 780
 781   /// TreeHasProperty - Return true if any node in this tree has the specified
 782   /// property.
 783   bool TreeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const;
 784
 785   /// isCommutativeIntrinsic - Return true if the node is an intrinsic which is
 786   /// marked isCommutative.
 787   bool isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const;
 788
 789   void print(raw_ostream &OS) const;
 790   void dump() const;
 791
 792 public:   // Higher level manipulation routines.
 793
 794   /// clone - Return a new copy of this tree.
 795   ///
 796   TreePatternNodePtr clone() const;
 797
 798   /// RemoveAllTypes - Recursively strip all the types of this tree.
 799   void RemoveAllTypes();
 800
 801   /// isIsomorphicTo - Return true if this node is recursively isomorphic to
 802   /// the specified node.  For this comparison, all of the state of the node
 803   /// is considered, except for the assigned name.  Nodes with differing names
 804   /// that are otherwise identical are considered isomorphic.
 805   bool isIsomorphicTo(const TreePatternNode *N,
 806                       const MultipleUseVarSet &DepVars) const;
 807
 808   /// SubstituteFormalArguments - Replace the formal arguments in this tree
 809   /// with actual values specified by ArgMap.
 810   void
 811   SubstituteFormalArguments(std::map<std::string, TreePatternNodePtr> &ArgMap);
 812
 813   /// InlinePatternFragments - If this pattern refers to any pattern
 814   /// fragments, return the set of inlined versions (this can be more than
 815   /// one if a PatFrags record has multiple alternatives).
 816   void InlinePatternFragments(TreePatternNodePtr T,
 817                               TreePattern &TP,
 818                               std::vector<TreePatternNodePtr> &OutAlternatives);
 819
 820   /// ApplyTypeConstraints - Apply all of the type constraints relevant to
 821   /// this node and its children in the tree.  This returns true if it makes a
 822   /// change, false otherwise.  If a type contradiction is found, flag an error.
 823   bool ApplyTypeConstraints(TreePattern &TP, bool NotRegisters);
 824
 825   /// UpdateNodeType - Set the node type of N to VT if VT contains
 826   /// information.  If N already contains a conflicting type, then flag an
 827   /// error.  This returns true if any information was updated.
 828   ///
 829   bool UpdateNodeType(unsigned ResNo, const TypeSetByHwMode &InTy,
 830                       TreePattern &TP);
 831   bool UpdateNodeType(unsigned ResNo, MVT::SimpleValueType InTy,
 832                       TreePattern &TP);
 833   bool UpdateNodeType(unsigned ResNo, ValueTypeByHwMode InTy,
 834                       TreePattern &TP);
 835
 836   // Update node type with types inferred from an instruction operand or result
 837   // def from the ins/outs lists.
 838   // Return true if the type changed.
 839   bool UpdateNodeTypeFromInst(unsigned ResNo, Record *Operand, TreePattern &TP);
 840
 841   /// ContainsUnresolvedType - Return true if this tree contains any
 842   /// unresolved types.
 843   bool ContainsUnresolvedType(TreePattern &TP) const;
 844
 845   /// canPatternMatch - If it is impossible for this pattern to match on this
 846   /// target, fill in Reason and return false.  Otherwise, return true.
 847   bool canPatternMatch(std::string &Reason, const CodeGenDAGPatterns &CDP);
 848 };
 849
 850 inline raw_ostream &operator<<(raw_ostream &OS, const TreePatternNode &TPN) {
 851   TPN.print(OS);
 852   return OS;
 853 }
 854
 855
 856 /// TreePattern - Represent a pattern, used for instructions, pattern
 857 /// fragments, etc.
 858 ///
 859 class TreePattern {
 860   /// Trees - The list of pattern trees which corresponds to this pattern.
 861   /// Note that PatFrag's only have a single tree.
 862   ///
 863   std::vector<TreePatternNodePtr> Trees;
 864
 865   /// NamedNodes - This is all of the nodes that have names in the trees in this
 866   /// pattern.
 867   StringMap<SmallVector<TreePatternNode *, 1>> NamedNodes;
 868
 869   /// TheRecord - The actual TableGen record corresponding to this pattern.
 870   ///
 871   Record *TheRecord;
 872
 873   /// Args - This is a list of all of the arguments to this pattern (for
 874   /// PatFrag patterns), which are the 'node' markers in this pattern.
 875   std::vector<std::string> Args;
 876
 877   /// CDP - the top-level object coordinating this madness.
 878   ///
 879   CodeGenDAGPatterns &CDP;
 880
 881   /// isInputPattern - True if this is an input pattern, something to match.
 882   /// False if this is an output pattern, something to emit.
 883   bool isInputPattern;
 884
 885   /// hasError - True if the currently processed nodes have unresolvable types
 886   /// or other non-fatal errors
 887   bool HasError;
 888
 889   /// It's important that the usage of operands in ComplexPatterns is
 890   /// consistent: each named operand can be defined by at most one
 891   /// ComplexPattern. This records the ComplexPattern instance and the operand
 892   /// number for each operand encountered in a ComplexPattern to aid in that
 893   /// check.
 894   StringMap<std::pair<Record *, unsigned>> ComplexPatternOperands;
 895
 896   TypeInfer Infer;
 897
 898 public:
 899
 900   /// TreePattern constructor - Parse the specified DagInits into the
 901   /// current record.
 902   TreePattern(Record *TheRec, ListInit *RawPat, bool isInput,
 903               CodeGenDAGPatterns &ise);
 904   TreePattern(Record *TheRec, DagInit *Pat, bool isInput,
 905               CodeGenDAGPatterns &ise);
 906   TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput,
 907               CodeGenDAGPatterns &ise);
 908
 909   /// getTrees - Return the tree patterns which corresponds to this pattern.
 910   ///
 911   const std::vector<TreePatternNodePtr> &getTrees() const { return Trees; }
 912   unsigned getNumTrees() const { return Trees.size(); }
 913   const TreePatternNodePtr &getTree(unsigned i) const { return Trees[i]; }
 914   void setTree(unsigned i, TreePatternNodePtr Tree) { Trees[i] = Tree; }
 915   const TreePatternNodePtr &getOnlyTree() const {
 916     assert(Trees.size() == 1 && "Doesn't have exactly one pattern!");
 917     return Trees[0];
 918   }
 919
 920   const StringMap<SmallVector<TreePatternNode *, 1>> &getNamedNodesMap() {
 921     if (NamedNodes.empty())
 922       ComputeNamedNodes();
 923     return NamedNodes;
 924   }
 925
 926   /// getRecord - Return the actual TableGen record corresponding to this
 927   /// pattern.
 928   ///
 929   Record *getRecord() const { return TheRecord; }
 930
 931   unsigned getNumArgs() const { return Args.size(); }
 932   const std::string &getArgName(unsigned i) const {
 933     assert(i < Args.size() && "Argument reference out of range!");
 934     return Args[i];
 935   }
 936   std::vector<std::string> &getArgList() { return Args; }
 937
 938   CodeGenDAGPatterns &getDAGPatterns() const { return CDP; }
 939
 940   /// InlinePatternFragments - If this pattern refers to any pattern
 941   /// fragments, inline them into place, giving us a pattern without any
 942   /// PatFrags references.  This may increase the number of trees in the
 943   /// pattern if a PatFrags has multiple alternatives.
 944   void InlinePatternFragments() {
 945     std::vector<TreePatternNodePtr> Copy = Trees;
 946     Trees.clear();
 947     for (unsigned i = 0, e = Copy.size(); i != e; ++i)
 948       Copy[i]->InlinePatternFragments(Copy[i], *this, Trees);
 949   }
 950
 951   /// InferAllTypes - Infer/propagate as many types throughout the expression
 952   /// patterns as possible.  Return true if all types are inferred, false
 953   /// otherwise.  Bail out if a type contradiction is found.
 954   bool InferAllTypes(
 955       const StringMap<SmallVector<TreePatternNode *, 1>> *NamedTypes = nullptr);
 956
 957   /// error - If this is the first error in the current resolution step,
 958   /// print it and set the error flag.  Otherwise, continue silently.
 959   void error(const Twine &Msg);
 960   bool hasError() const {
 961     return HasError;
 962   }
 963   void resetError() {
 964     HasError = false;
 965   }
 966
 967   TypeInfer &getInfer() { return Infer; }
 968
 969   void print(raw_ostream &OS) const;
 970   void dump() const;
 971
 972 private:
 973   TreePatternNodePtr ParseTreePattern(Init *DI, StringRef OpName);
 974   void ComputeNamedNodes();
 975   void ComputeNamedNodes(TreePatternNode *N);
 976 };
 977
 978
 979 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
 980                                             const TypeSetByHwMode &InTy,
 981                                             TreePattern &TP) {
 982   TypeSetByHwMode VTS(InTy);
 983   TP.getInfer().expandOverloads(VTS);
 984   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
 985 }
 986
 987 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
 988                                             MVT::SimpleValueType InTy,
 989                                             TreePattern &TP) {
 990   TypeSetByHwMode VTS(InTy);
 991   TP.getInfer().expandOverloads(VTS);
 992   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
 993 }
 994
 995 inline bool TreePatternNode::UpdateNodeType(unsigned ResNo,
 996                                             ValueTypeByHwMode InTy,
 997                                             TreePattern &TP) {
 998   TypeSetByHwMode VTS(InTy);
 999   TP.getInfer().expandOverloads(VTS);
1000   return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS);
1001 }
1002
1003
1004 /// DAGDefaultOperand - One of these is created for each OperandWithDefaultOps
1005 /// that has a set ExecuteAlways / DefaultOps field.
1006 struct DAGDefaultOperand {
1007   std::vector<TreePatternNodePtr> DefaultOps;
1008 };
1009
1010 class DAGInstruction {
1011   std::vector<Record*> Results;
1012   std::vector<Record*> Operands;
1013   std::vector<Record*> ImpResults;
1014   TreePatternNodePtr SrcPattern;
1015   TreePatternNodePtr ResultPattern;
1016
1017 public:
1018   DAGInstruction(const std::vector<Record*> &results,
1019                  const std::vector<Record*> &operands,
1020                  const std::vector<Record*> &impresults,
1021                  TreePatternNodePtr srcpattern = nullptr,
1022                  TreePatternNodePtr resultpattern = nullptr)
1023     : Results(results), Operands(operands), ImpResults(impresults),
1024       SrcPattern(srcpattern), ResultPattern(resultpattern) {}
1025
1026   unsigned getNumResults() const { return Results.size(); }
1027   unsigned getNumOperands() const { return Operands.size(); }
1028   unsigned getNumImpResults() const { return ImpResults.size(); }
1029   const std::vector<Record*>& getImpResults() const { return ImpResults; }
1030
1031   Record *getResult(unsigned RN) const {
1032     assert(RN < Results.size());
1033     return Results[RN];
1034   }
1035
1036   Record *getOperand(unsigned ON) const {
1037     assert(ON < Operands.size());
1038     return Operands[ON];
1039   }
1040
1041   Record *getImpResult(unsigned RN) const {
1042     assert(RN < ImpResults.size());
1043     return ImpResults[RN];
1044   }
1045
1046   TreePatternNodePtr getSrcPattern() const { return SrcPattern; }
1047   TreePatternNodePtr getResultPattern() const { return ResultPattern; }
1048 };
1049
1050 /// PatternToMatch - Used by CodeGenDAGPatterns to keep tab of patterns
1051 /// processed to produce isel.
1052 class PatternToMatch {
1053   Record          *SrcRecord;   // Originating Record for the pattern.
1054   ListInit        *Predicates;  // Top level predicate conditions to match.
1055   TreePatternNodePtr SrcPattern;      // Source pattern to match.
1056   TreePatternNodePtr DstPattern;      // Resulting pattern.
1057   std::vector<Record*> Dstregs; // Physical register defs being matched.
1058   std::string      HwModeFeatures;
1059   int              AddedComplexity; // Add to matching pattern complexity.
1060   unsigned         ID;          // Unique ID for the record.
1061   unsigned         ForceMode;   // Force this mode in type inference when set.
1062
1063 public:
1064   PatternToMatch(Record *srcrecord, ListInit *preds, TreePatternNodePtr src,
1065                  TreePatternNodePtr dst, std::vector<Record *> dstregs,
1066                  int complexity, unsigned uid, unsigned setmode = 0,
1067                  const Twine &hwmodefeatures = "")
1068       : SrcRecord(srcrecord), Predicates(preds), SrcPattern(src),
1069         DstPattern(dst), Dstregs(std::move(dstregs)),
1070         HwModeFeatures(hwmodefeatures.str()), AddedComplexity(complexity),
1071         ID(uid), ForceMode(setmode) {}
1072
1073   Record          *getSrcRecord()  const { return SrcRecord; }
1074   ListInit        *getPredicates() const { return Predicates; }
1075   TreePatternNode *getSrcPattern() const { return SrcPattern.get(); }
1076   TreePatternNodePtr getSrcPatternShared() const { return SrcPattern; }
1077   TreePatternNode *getDstPattern() const { return DstPattern.get(); }
1078   TreePatternNodePtr getDstPatternShared() const { return DstPattern; }
1079   const std::vector<Record*> &getDstRegs() const { return Dstregs; }
1080   StringRef   getHwModeFeatures() const { return HwModeFeatures; }
1081   int         getAddedComplexity() const { return AddedComplexity; }
1082   unsigned getID() const { return ID; }
1083   unsigned getForceMode() const { return ForceMode; }
1084
1085   std::string getPredicateCheck() const;
1086   void getPredicateRecords(SmallVectorImpl<Record *> &PredicateRecs) const;
1087
1088   /// Compute the complexity metric for the input pattern.  This roughly
1089   /// corresponds to the number of nodes that are covered.
1090   int getPatternComplexity(const CodeGenDAGPatterns &CGP) const;
1091 };
1092
1093 class CodeGenDAGPatterns {
1094   RecordKeeper &Records;
1095   CodeGenTarget Target;
1096   CodeGenIntrinsicTable Intrinsics;
1097
1098   std::map<Record*, SDNodeInfo, LessRecordByID> SDNodes;
1099   std::map<Record*, std::pair<Record*, std::string>, LessRecordByID>
1100       SDNodeXForms;
1101   std::map<Record*, ComplexPattern, LessRecordByID> ComplexPatterns;
1102   std::map<Record *, std::unique_ptr<TreePattern>, LessRecordByID>
1103       PatternFragments;
1104   std::map<Record*, DAGDefaultOperand, LessRecordByID> DefaultOperands;
1105   std::map<Record*, DAGInstruction, LessRecordByID> Instructions;
1106
1107   // Specific SDNode definitions:
1108   Record *intrinsic_void_sdnode;
1109   Record *intrinsic_w_chain_sdnode, *intrinsic_wo_chain_sdnode;
1110
1111   /// PatternsToMatch - All of the things we are matching on the DAG.  The first
1112   /// value is the pattern to match, the second pattern is the result to
1113   /// emit.
1114   std::vector<PatternToMatch> PatternsToMatch;
1115
1116   TypeSetByHwMode LegalVTS;
1117
1118   using PatternRewriterFn = std::function<void (TreePattern *)>;
1119   PatternRewriterFn PatternRewriter;
1120
1121   unsigned NumScopes = 0;
1122
1123 public:
1124   CodeGenDAGPatterns(RecordKeeper &R,
1125                      PatternRewriterFn PatternRewriter = nullptr);
1126
1127   CodeGenTarget &getTargetInfo() { return Target; }
1128   const CodeGenTarget &getTargetInfo() const { return Target; }
1129   const TypeSetByHwMode &getLegalTypes() const { return LegalVTS; }
1130
1131   Record *getSDNodeNamed(StringRef Name) const;
1132
1133   const SDNodeInfo &getSDNodeInfo(Record *R) const {
1134     auto F = SDNodes.find(R);
1135     assert(F != SDNodes.end() && "Unknown node!");
1136     return F->second;
1137   }
1138
1139   // Node transformation lookups.
1140   typedef std::pair<Record*, std::string> NodeXForm;
1141   const NodeXForm &getSDNodeTransform(Record *R) const {
1142     auto F = SDNodeXForms.find(R);
1143     assert(F != SDNodeXForms.end() && "Invalid transform!");
1144     return F->second;
1145   }
1146
1147   const ComplexPattern &getComplexPattern(Record *R) const {
1148     auto F = ComplexPatterns.find(R);
1149     assert(F != ComplexPatterns.end() && "Unknown addressing mode!");
1150     return F->second;
1151   }
1152
1153   const CodeGenIntrinsic &getIntrinsic(Record *R) const {
1154     for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
1155       if (Intrinsics[i].TheDef == R) return Intrinsics[i];
1156     llvm_unreachable("Unknown intrinsic!");
1157   }
1158
1159   const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const {
1160     if (IID-1 < Intrinsics.size())
1161       return Intrinsics[IID-1];
1162     llvm_unreachable("Bad intrinsic ID!");
1163   }
1164
1165   unsigned getIntrinsicID(Record *R) const {
1166     for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
1167       if (Intrinsics[i].TheDef == R) return i;
1168     llvm_unreachable("Unknown intrinsic!");
1169   }
1170
1171   const DAGDefaultOperand &getDefaultOperand(Record *R) const {
1172     auto F = DefaultOperands.find(R);
1173     assert(F != DefaultOperands.end() &&"Isn't an analyzed default operand!");
1174     return F->second;
1175   }
1176
1177   // Pattern Fragment information.
1178   TreePattern *getPatternFragment(Record *R) const {
1179     auto F = PatternFragments.find(R);
1180     assert(F != PatternFragments.end() && "Invalid pattern fragment request!");
1181     return F->second.get();
1182   }
1183   TreePattern *getPatternFragmentIfRead(Record *R) const {
1184     auto F = PatternFragments.find(R);
1185     if (F == PatternFragments.end())
1186       return nullptr;
1187     return F->second.get();
1188   }
1189
1190   typedef std::map<Record *, std::unique_ptr<TreePattern>,
1191                    LessRecordByID>::const_iterator pf_iterator;
1192   pf_iterator pf_begin() const { return PatternFragments.begin(); }
1193   pf_iterator pf_end() const { return PatternFragments.end(); }
1194   iterator_range<pf_iterator> ptfs() const { return PatternFragments; }
1195
1196   // Patterns to match information.
1197   typedef std::vector<PatternToMatch>::const_iterator ptm_iterator;
1198   ptm_iterator ptm_begin() const { return PatternsToMatch.begin(); }
1199   ptm_iterator ptm_end() const { return PatternsToMatch.end(); }
1200   iterator_range<ptm_iterator> ptms() const { return PatternsToMatch; }
1201
1202   /// Parse the Pattern for an instruction, and insert the result in DAGInsts.
1203   typedef std::map<Record*, DAGInstruction, LessRecordByID> DAGInstMap;
1204   void parseInstructionPattern(
1205       CodeGenInstruction &CGI, ListInit *Pattern,
1206       DAGInstMap &DAGInsts);
1207
1208   const DAGInstruction &getInstruction(Record *R) const {
1209     auto F = Instructions.find(R);
1210     assert(F != Instructions.end() && "Unknown instruction!");
1211     return F->second;
1212   }
1213
1214   Record *get_intrinsic_void_sdnode() const {
1215     return intrinsic_void_sdnode;
1216   }
1217   Record *get_intrinsic_w_chain_sdnode() const {
1218     return intrinsic_w_chain_sdnode;
1219   }
1220   Record *get_intrinsic_wo_chain_sdnode() const {
1221     return intrinsic_wo_chain_sdnode;
1222   }
1223
1224   unsigned allocateScope() { return ++NumScopes; }
1225
1226   bool operandHasDefault(Record *Op) const {
1227     return Op->isSubClassOf("OperandWithDefaultOps") &&
1228       !getDefaultOperand(Op).DefaultOps.empty();
1229   }
1230
1231 private:
1232   void ParseNodeInfo();
1233   void ParseNodeTransforms();
1234   void ParseComplexPatterns();
1235   void ParsePatternFragments(bool OutFrags = false);
1236   void ParseDefaultOperands();
1237   void ParseInstructions();
1238   void ParsePatterns();
1239   void ExpandHwModeBasedTypes();
1240   void InferInstructionFlags();
1241   void GenerateVariants();
1242   void VerifyInstructionFlags();
1243
1244   void ParseOnePattern(Record *TheDef,
1245                        TreePattern &Pattern, TreePattern &Result,
1246                        const std::vector<Record *> &InstImpResults);
1247   void AddPatternToMatch(TreePattern *Pattern, PatternToMatch &&PTM);
1248   void FindPatternInputsAndOutputs(
1249       TreePattern &I, TreePatternNodePtr Pat,
1250       std::map<std::string, TreePatternNodePtr> &InstInputs,
1251       MapVector<std::string, TreePatternNodePtr,
1252                 std::map<std::string, unsigned>> &InstResults,
1253       std::vector<Record *> &InstImpResults);
1254 };
1255
1256
1257 inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode *N,
1258                                              TreePattern &TP) const {
1259     bool MadeChange = false;
1260     for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i)
1261       MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP);
1262     return MadeChange;
1263   }
1264
1265 } // end namespace llvm
1266
1267 #endif