clang/utils/TableGen/SveEmitter.cpp

   1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
   2 //
   3 //  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 //  See https://llvm.org/LICENSE.txt for license information.
   5 //  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
  10 // a declaration and definition of each function specified by the ARM C/C++
  11 // Language Extensions (ACLE).
  12 //
  13 // For details, visit:
  14 //  https://developer.arm.com/architectures/system-architectures/software-standards/acle
  15 //
  16 // Each SVE instruction is implemented in terms of 1 or more functions which
  17 // are suffixed with the element type of the input vectors.  Functions may be
  18 // implemented in terms of generic vector operations such as +, *, -, etc. or
  19 // by calling a __builtin_-prefixed function which will be handled by clang's
  20 // CodeGen library.
  21 //
  22 // See also the documentation in include/clang/Basic/arm_sve.td.
  23 //
  24 //===----------------------------------------------------------------------===//
  25
  26 #include "llvm/ADT/STLExtras.h"
  27 #include "llvm/ADT/StringMap.h"
  28 #include "llvm/ADT/ArrayRef.h"
  29 #include "llvm/ADT/StringExtras.h"
  30 #include "llvm/TableGen/Record.h"
  31 #include "llvm/TableGen/Error.h"
  32 #include <string>
  33 #include <sstream>
  34 #include <set>
  35 #include <cctype>
  36 #include <tuple>
  37
  38 using namespace llvm;
  39
  40 enum ClassKind {
  41   ClassNone,
  42   ClassS,     // signed/unsigned, e.g., "_s8", "_u8" suffix
  43   ClassG,     // Overloaded name without type suffix
  44 };
  45
  46 using TypeSpec = std::string;
  47
  48 namespace {
  49
  50 class ImmCheck {
  51   unsigned Arg;
  52   unsigned Kind;
  53   unsigned ElementSizeInBits;
  54
  55 public:
  56   ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
  57       : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
  58   ImmCheck(const ImmCheck &Other) = default;
  59   ~ImmCheck() = default;
  60
  61   unsigned getArg() const { return Arg; }
  62   unsigned getKind() const { return Kind; }
  63   unsigned getElementSizeInBits() const { return ElementSizeInBits; }
  64 };
  65
  66 class SVEType {
  67   TypeSpec TS;
  68   bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
  69   bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
  70       Svcount;
  71   unsigned Bitwidth, ElementBitwidth, NumVectors;
  72
  73 public:
  74   SVEType() : SVEType(TypeSpec(), 'v') {}
  75
  76   SVEType(TypeSpec TS, char CharMod, unsigned NumVectors = 1)
  77       : TS(TS), Float(false), Signed(true), Immediate(false), Void(false),
  78         Constant(false), Pointer(false), BFloat(false), DefaultType(false),
  79         IsScalable(true), Predicate(false), PredicatePattern(false),
  80         PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
  81         NumVectors(NumVectors) {
  82     if (!TS.empty())
  83       applyTypespec();
  84     applyModifier(CharMod);
  85   }
  86
  87   bool isPointer() const { return Pointer; }
  88   bool isVoidPointer() const { return Pointer && Void; }
  89   bool isSigned() const { return Signed; }
  90   bool isImmediate() const { return Immediate; }
  91   bool isScalar() const { return NumVectors == 0; }
  92   bool isVector() const { return NumVectors > 0; }
  93   bool isScalableVector() const { return isVector() && IsScalable; }
  94   bool isChar() const { return ElementBitwidth == 8; }
  95   bool isVoid() const { return Void & !Pointer; }
  96   bool isDefault() const { return DefaultType; }
  97   bool isFloat() const { return Float && !BFloat; }
  98   bool isBFloat() const { return BFloat && !Float; }
  99   bool isFloatingPoint() const { return Float || BFloat; }
 100   bool isInteger() const {
 101     return !isFloatingPoint() && !Predicate && !Svcount;
 102   }
 103   bool isScalarPredicate() const {
 104     return !isFloatingPoint() && Predicate && NumVectors == 0;
 105   }
 106   bool isPredicateVector() const { return Predicate; }
 107   bool isPredicatePattern() const { return PredicatePattern; }
 108   bool isPrefetchOp() const { return PrefetchOp; }
 109   bool isSvcount() const { return Svcount; }
 110   bool isConstant() const { return Constant; }
 111   unsigned getElementSizeInBits() const { return ElementBitwidth; }
 112   unsigned getNumVectors() const { return NumVectors; }
 113
 114   unsigned getNumElements() const {
 115     assert(ElementBitwidth != ~0U);
 116     return Bitwidth / ElementBitwidth;
 117   }
 118   unsigned getSizeInBits() const {
 119     return Bitwidth;
 120   }
 121
 122   /// Return the string representation of a type, which is an encoded
 123   /// string for passing to the BUILTIN() macro in Builtins.def.
 124   std::string builtin_str() const;
 125
 126   /// Return the C/C++ string representation of a type for use in the
 127   /// arm_sve.h header file.
 128   std::string str() const;
 129
 130 private:
 131   /// Creates the type based on the typespec string in TS.
 132   void applyTypespec();
 133
 134   /// Applies a prototype modifier to the type.
 135   void applyModifier(char Mod);
 136 };
 137
 138
 139 class SVEEmitter;
 140
 141 /// The main grunt class. This represents an instantiation of an intrinsic with
 142 /// a particular typespec and prototype.
 143 class Intrinsic {
 144   /// The unmangled name.
 145   std::string Name;
 146
 147   /// The name of the corresponding LLVM IR intrinsic.
 148   std::string LLVMName;
 149
 150   /// Intrinsic prototype.
 151   std::string Proto;
 152
 153   /// The base type spec for this intrinsic.
 154   TypeSpec BaseTypeSpec;
 155
 156   /// The base class kind. Most intrinsics use ClassS, which has full type
 157   /// info for integers (_s32/_u32), or ClassG which is used for overloaded
 158   /// intrinsics.
 159   ClassKind Class;
 160
 161   /// The architectural #ifdef guard.
 162   std::string Guard;
 163
 164   // The merge suffix such as _m, _x or _z.
 165   std::string MergeSuffix;
 166
 167   /// The types of return value [0] and parameters [1..].
 168   std::vector<SVEType> Types;
 169
 170   /// The "base type", which is VarType('d', BaseTypeSpec).
 171   SVEType BaseType;
 172
 173   uint64_t Flags;
 174
 175   SmallVector<ImmCheck, 2> ImmChecks;
 176
 177 public:
 178   Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
 179             StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
 180             uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
 181             ClassKind Class, SVEEmitter &Emitter, StringRef Guard);
 182
 183   ~Intrinsic()=default;
 184
 185   std::string getName() const { return Name; }
 186   std::string getLLVMName() const { return LLVMName; }
 187   std::string getProto() const { return Proto; }
 188   TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
 189   SVEType getBaseType() const { return BaseType; }
 190
 191   StringRef getGuard() const { return Guard; }
 192   ClassKind getClassKind() const { return Class; }
 193
 194   SVEType getReturnType() const { return Types[0]; }
 195   ArrayRef<SVEType> getTypes() const { return Types; }
 196   SVEType getParamType(unsigned I) const { return Types[I + 1]; }
 197   unsigned getNumParams() const {
 198     return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
 199   }
 200
 201   uint64_t getFlags() const { return Flags; }
 202   bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
 203
 204   ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
 205
 206   /// Return the type string for a BUILTIN() macro in Builtins.def.
 207   std::string getBuiltinTypeStr();
 208
 209   /// Return the name, mangled with type information. The name is mangled for
 210   /// ClassS, so will add type suffixes such as _u32/_s32.
 211   std::string getMangledName() const { return mangleName(ClassS); }
 212
 213   /// As above, but mangles the LLVM name instead.
 214   std::string getMangledLLVMName() const { return mangleLLVMName(); }
 215
 216   /// Returns true if the intrinsic is overloaded, in that it should also generate
 217   /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
 218   /// 'svld1_u32(..)'.
 219   static bool isOverloadedIntrinsic(StringRef Name) {
 220     auto BrOpen = Name.find('[');
 221     auto BrClose = Name.find(']');
 222     return BrOpen != std::string::npos && BrClose != std::string::npos;
 223   }
 224
 225   /// Return true if the intrinsic takes a splat operand.
 226   bool hasSplat() const {
 227     // These prototype modifiers are described in arm_sve.td.
 228     return Proto.find_first_of("ajfrKLR@") != std::string::npos;
 229   }
 230
 231   /// Return the parameter index of the splat operand.
 232   unsigned getSplatIdx() const {
 233     unsigned I = 1, Param = 0;
 234     for (; I < Proto.size(); ++I, ++Param) {
 235       if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
 236           Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
 237           Proto[I] == 'R' || Proto[I] == '@')
 238         break;
 239
 240       // Multivector modifier can be skipped
 241       if (Proto[I] == '.')
 242         I += 2;
 243     }
 244     assert(I != Proto.size() && "Prototype has no splat operand");
 245     return Param;
 246   }
 247
 248   /// Emits the intrinsic declaration to the ostream.
 249   void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter) const;
 250
 251 private:
 252   std::string getMergeSuffix() const { return MergeSuffix; }
 253   std::string mangleName(ClassKind LocalCK) const;
 254   std::string mangleLLVMName() const;
 255   std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
 256                                    std::string Proto) const;
 257 };
 258
 259 class SVEEmitter {
 260 private:
 261   // The reinterpret builtins are generated separately because they
 262   // need the cross product of all types (121 functions in total),
 263   // which is inconvenient to specify in the arm_sve.td file or
 264   // generate in CGBuiltin.cpp.
 265   struct ReinterpretTypeInfo {
 266     const char *Suffix;
 267     const char *Type;
 268     const char *BuiltinType;
 269   };
 270   SmallVector<ReinterpretTypeInfo, 12> Reinterprets = {
 271       {"s8", "svint8_t", "q16Sc"},   {"s16", "svint16_t", "q8Ss"},
 272       {"s32", "svint32_t", "q4Si"},  {"s64", "svint64_t", "q2SWi"},
 273       {"u8", "svuint8_t", "q16Uc"},  {"u16", "svuint16_t", "q8Us"},
 274       {"u32", "svuint32_t", "q4Ui"}, {"u64", "svuint64_t", "q2UWi"},
 275       {"f16", "svfloat16_t", "q8h"}, {"bf16", "svbfloat16_t", "q8y"},
 276       {"f32", "svfloat32_t", "q4f"}, {"f64", "svfloat64_t", "q2d"}};
 277
 278   RecordKeeper &Records;
 279   llvm::StringMap<uint64_t> EltTypes;
 280   llvm::StringMap<uint64_t> MemEltTypes;
 281   llvm::StringMap<uint64_t> FlagTypes;
 282   llvm::StringMap<uint64_t> MergeTypes;
 283   llvm::StringMap<uint64_t> ImmCheckTypes;
 284
 285 public:
 286   SVEEmitter(RecordKeeper &R) : Records(R) {
 287     for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
 288       EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
 289     for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
 290       MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
 291     for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
 292       FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
 293     for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
 294       MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
 295     for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
 296       ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
 297   }
 298
 299   /// Returns the enum value for the immcheck type
 300   unsigned getEnumValueForImmCheck(StringRef C) const {
 301     auto It = ImmCheckTypes.find(C);
 302     if (It != ImmCheckTypes.end())
 303       return It->getValue();
 304     llvm_unreachable("Unsupported imm check");
 305   }
 306
 307   /// Returns the enum value for the flag type
 308   uint64_t getEnumValueForFlag(StringRef C) const {
 309     auto Res = FlagTypes.find(C);
 310     if (Res != FlagTypes.end())
 311       return Res->getValue();
 312     llvm_unreachable("Unsupported flag");
 313   }
 314
 315   // Returns the SVETypeFlags for a given value and mask.
 316   uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
 317     auto It = FlagTypes.find(MaskName);
 318     if (It != FlagTypes.end()) {
 319       uint64_t Mask = It->getValue();
 320       unsigned Shift = llvm::countr_zero(Mask);
 321       assert(Shift < 64 && "Mask value produced an invalid shift value");
 322       return (V << Shift) & Mask;
 323     }
 324     llvm_unreachable("Unsupported flag");
 325   }
 326
 327   // Returns the SVETypeFlags for the given element type.
 328   uint64_t encodeEltType(StringRef EltName) {
 329     auto It = EltTypes.find(EltName);
 330     if (It != EltTypes.end())
 331       return encodeFlag(It->getValue(), "EltTypeMask");
 332     llvm_unreachable("Unsupported EltType");
 333   }
 334
 335   // Returns the SVETypeFlags for the given memory element type.
 336   uint64_t encodeMemoryElementType(uint64_t MT) {
 337     return encodeFlag(MT, "MemEltTypeMask");
 338   }
 339
 340   // Returns the SVETypeFlags for the given merge type.
 341   uint64_t encodeMergeType(uint64_t MT) {
 342     return encodeFlag(MT, "MergeTypeMask");
 343   }
 344
 345   // Returns the SVETypeFlags for the given splat operand.
 346   unsigned encodeSplatOperand(unsigned SplatIdx) {
 347     assert(SplatIdx < 7 && "SplatIdx out of encodable range");
 348     return encodeFlag(SplatIdx + 1, "SplatOperandMask");
 349   }
 350
 351   // Returns the SVETypeFlags value for the given SVEType.
 352   uint64_t encodeTypeFlags(const SVEType &T);
 353
 354   /// Emit arm_sve.h.
 355   void createHeader(raw_ostream &o);
 356
 357   /// Emit all the __builtin prototypes and code needed by Sema.
 358   void createBuiltins(raw_ostream &o);
 359
 360   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
 361   void createCodeGenMap(raw_ostream &o);
 362
 363   /// Emit all the range checks for the immediates.
 364   void createRangeChecks(raw_ostream &o);
 365
 366   /// Create the SVETypeFlags used in CGBuiltins
 367   void createTypeFlags(raw_ostream &o);
 368
 369   /// Emit arm_sme.h.
 370   void createSMEHeader(raw_ostream &o);
 371
 372   /// Emit all the SME __builtin prototypes and code needed by Sema.
 373   void createSMEBuiltins(raw_ostream &o);
 374
 375   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
 376   void createSMECodeGenMap(raw_ostream &o);
 377
 378   /// Emit all the range checks for the immediates.
 379   void createSMERangeChecks(raw_ostream &o);
 380
 381   /// Create intrinsic and add it to \p Out
 382   void createIntrinsic(Record *R,
 383                        SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
 384 };
 385
 386 } // end anonymous namespace
 387
 388
 389 //===----------------------------------------------------------------------===//
 390 // Type implementation
 391 //===----------------------------------------------------------------------===//
 392
 393 std::string SVEType::builtin_str() const {
 394   std::string S;
 395   if (isVoid())
 396     return "v";
 397
 398   if (isScalarPredicate())
 399     return "b";
 400
 401   if (isSvcount())
 402     return "Qa";
 403
 404   if (isVoidPointer())
 405     S += "v";
 406   else if (!isFloatingPoint())
 407     switch (ElementBitwidth) {
 408     case 1: S += "b"; break;
 409     case 8: S += "c"; break;
 410     case 16: S += "s"; break;
 411     case 32: S += "i"; break;
 412     case 64: S += "Wi"; break;
 413     case 128: S += "LLLi"; break;
 414     default: llvm_unreachable("Unhandled case!");
 415     }
 416   else if (isFloat())
 417     switch (ElementBitwidth) {
 418     case 16: S += "h"; break;
 419     case 32: S += "f"; break;
 420     case 64: S += "d"; break;
 421     default: llvm_unreachable("Unhandled case!");
 422     }
 423   else if (isBFloat()) {
 424     assert(ElementBitwidth == 16 && "Not a valid BFloat.");
 425     S += "y";
 426   }
 427
 428   if (!isFloatingPoint()) {
 429     if ((isChar() || isPointer()) && !isVoidPointer()) {
 430       // Make chars and typed pointers explicitly signed.
 431       if (Signed)
 432         S = "S" + S;
 433       else if (!Signed)
 434         S = "U" + S;
 435     } else if (!isVoidPointer() && !Signed) {
 436       S = "U" + S;
 437     }
 438   }
 439
 440   // Constant indices are "int", but have the "constant expression" modifier.
 441   if (isImmediate()) {
 442     assert(!isFloat() && "fp immediates are not supported");
 443     S = "I" + S;
 444   }
 445
 446   if (isScalar()) {
 447     if (Constant) S += "C";
 448     if (Pointer) S += "*";
 449     return S;
 450   }
 451
 452   assert(isScalableVector() && "Unsupported type");
 453   return "q" + utostr(getNumElements() * NumVectors) + S;
 454 }
 455
 456 std::string SVEType::str() const {
 457   if (isPredicatePattern())
 458     return "enum svpattern";
 459
 460   if (isPrefetchOp())
 461     return "enum svprfop";
 462
 463   std::string S;
 464   if (Void)
 465     S += "void";
 466   else {
 467     if (isScalableVector() || isSvcount())
 468       S += "sv";
 469     if (!Signed && !isFloatingPoint())
 470       S += "u";
 471
 472     if (Float)
 473       S += "float";
 474     else if (isSvcount())
 475       S += "count";
 476     else if (isScalarPredicate() || isPredicateVector())
 477       S += "bool";
 478     else if (isBFloat())
 479       S += "bfloat";
 480     else
 481       S += "int";
 482
 483     if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
 484       S += utostr(ElementBitwidth);
 485     if (!isScalableVector() && isVector())
 486       S += "x" + utostr(getNumElements());
 487     if (NumVectors > 1)
 488       S += "x" + utostr(NumVectors);
 489     if (!isScalarPredicate())
 490       S += "_t";
 491   }
 492
 493   if (Constant)
 494     S += " const";
 495   if (Pointer)
 496     S += " *";
 497
 498   return S;
 499 }
 500 void SVEType::applyTypespec() {
 501   for (char I : TS) {
 502     switch (I) {
 503     case 'Q':
 504       Svcount = true;
 505       break;
 506     case 'P':
 507       Predicate = true;
 508       break;
 509     case 'U':
 510       Signed = false;
 511       break;
 512     case 'c':
 513       ElementBitwidth = 8;
 514       break;
 515     case 's':
 516       ElementBitwidth = 16;
 517       break;
 518     case 'i':
 519       ElementBitwidth = 32;
 520       break;
 521     case 'l':
 522       ElementBitwidth = 64;
 523       break;
 524     case 'q':
 525       ElementBitwidth = 128;
 526       break;
 527     case 'h':
 528       Float = true;
 529       ElementBitwidth = 16;
 530       break;
 531     case 'f':
 532       Float = true;
 533       ElementBitwidth = 32;
 534       break;
 535     case 'd':
 536       Float = true;
 537       ElementBitwidth = 64;
 538       break;
 539     case 'b':
 540       BFloat = true;
 541       Float = false;
 542       ElementBitwidth = 16;
 543       break;
 544     default:
 545       llvm_unreachable("Unhandled type code!");
 546     }
 547   }
 548   assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
 549 }
 550
 551 void SVEType::applyModifier(char Mod) {
 552   switch (Mod) {
 553   case 'v':
 554     Void = true;
 555     break;
 556   case 'd':
 557     DefaultType = true;
 558     break;
 559   case 'c':
 560     Constant = true;
 561     [[fallthrough]];
 562   case 'p':
 563     Pointer = true;
 564     Bitwidth = ElementBitwidth;
 565     NumVectors = 0;
 566     break;
 567   case 'e':
 568     Signed = false;
 569     ElementBitwidth /= 2;
 570     break;
 571   case 'h':
 572     ElementBitwidth /= 2;
 573     break;
 574   case 'q':
 575     ElementBitwidth /= 4;
 576     break;
 577   case 'b':
 578     Signed = false;
 579     Float = false;
 580     BFloat = false;
 581     ElementBitwidth /= 4;
 582     break;
 583   case 'o':
 584     ElementBitwidth *= 4;
 585     break;
 586   case 'P':
 587     Signed = true;
 588     Float = false;
 589     BFloat = false;
 590     Predicate = true;
 591     Svcount = false;
 592     Bitwidth = 16;
 593     ElementBitwidth = 1;
 594     break;
 595   case 's':
 596   case 'a':
 597     Bitwidth = ElementBitwidth;
 598     NumVectors = 0;
 599     break;
 600   case 'R':
 601     ElementBitwidth /= 2;
 602     NumVectors = 0;
 603     break;
 604   case 'r':
 605     ElementBitwidth /= 4;
 606     NumVectors = 0;
 607     break;
 608   case '@':
 609     Signed = false;
 610     Float = false;
 611     BFloat = false;
 612     ElementBitwidth /= 4;
 613     NumVectors = 0;
 614     break;
 615   case 'K':
 616     Signed = true;
 617     Float = false;
 618     BFloat = false;
 619     Bitwidth = ElementBitwidth;
 620     NumVectors = 0;
 621     break;
 622   case 'L':
 623     Signed = false;
 624     Float = false;
 625     BFloat = false;
 626     Bitwidth = ElementBitwidth;
 627     NumVectors = 0;
 628     break;
 629   case 'u':
 630     Predicate = false;
 631     Svcount = false;
 632     Signed = false;
 633     Float = false;
 634     BFloat = false;
 635     break;
 636   case 'x':
 637     Predicate = false;
 638     Svcount = false;
 639     Signed = true;
 640     Float = false;
 641     BFloat = false;
 642     break;
 643   case 'i':
 644     Predicate = false;
 645     Svcount = false;
 646     Float = false;
 647     BFloat = false;
 648     ElementBitwidth = Bitwidth = 64;
 649     NumVectors = 0;
 650     Signed = false;
 651     Immediate = true;
 652     break;
 653   case 'I':
 654     Predicate = false;
 655     Svcount = false;
 656     Float = false;
 657     BFloat = false;
 658     ElementBitwidth = Bitwidth = 32;
 659     NumVectors = 0;
 660     Signed = true;
 661     Immediate = true;
 662     PredicatePattern = true;
 663     break;
 664   case 'J':
 665     Predicate = false;
 666     Svcount = false;
 667     Float = false;
 668     BFloat = false;
 669     ElementBitwidth = Bitwidth = 32;
 670     NumVectors = 0;
 671     Signed = true;
 672     Immediate = true;
 673     PrefetchOp = true;
 674     break;
 675   case 'k':
 676     Predicate = false;
 677     Svcount = false;
 678     Signed = true;
 679     Float = false;
 680     BFloat = false;
 681     ElementBitwidth = Bitwidth = 32;
 682     NumVectors = 0;
 683     break;
 684   case 'l':
 685     Predicate = false;
 686     Svcount = false;
 687     Signed = true;
 688     Float = false;
 689     BFloat = false;
 690     ElementBitwidth = Bitwidth = 64;
 691     NumVectors = 0;
 692     break;
 693   case 'm':
 694     Predicate = false;
 695     Svcount = false;
 696     Signed = false;
 697     Float = false;
 698     BFloat = false;
 699     ElementBitwidth = Bitwidth = 32;
 700     NumVectors = 0;
 701     break;
 702   case 'n':
 703     Predicate = false;
 704     Svcount = false;
 705     Signed = false;
 706     Float = false;
 707     BFloat = false;
 708     ElementBitwidth = Bitwidth = 64;
 709     NumVectors = 0;
 710     break;
 711   case 'w':
 712     ElementBitwidth = 64;
 713     break;
 714   case 'j':
 715     ElementBitwidth = Bitwidth = 64;
 716     NumVectors = 0;
 717     break;
 718   case 'f':
 719     Signed = false;
 720     ElementBitwidth = Bitwidth = 64;
 721     NumVectors = 0;
 722     break;
 723   case 'g':
 724     Signed = false;
 725     Float = false;
 726     BFloat = false;
 727     ElementBitwidth = 64;
 728     break;
 729   case 't':
 730     Signed = true;
 731     Float = false;
 732     BFloat = false;
 733     ElementBitwidth = 32;
 734     break;
 735   case 'z':
 736     Signed = false;
 737     Float = false;
 738     BFloat = false;
 739     ElementBitwidth = 32;
 740     break;
 741   case 'O':
 742     Predicate = false;
 743     Svcount = false;
 744     Float = true;
 745     ElementBitwidth = 16;
 746     break;
 747   case 'M':
 748     Predicate = false;
 749     Svcount = false;
 750     Float = true;
 751     BFloat = false;
 752     ElementBitwidth = 32;
 753     break;
 754   case 'N':
 755     Predicate = false;
 756     Svcount = false;
 757     Float = true;
 758     ElementBitwidth = 64;
 759     break;
 760   case 'Q':
 761     Constant = true;
 762     Pointer = true;
 763     Void = true;
 764     NumVectors = 0;
 765     break;
 766   case 'S':
 767     Constant = true;
 768     Pointer = true;
 769     ElementBitwidth = Bitwidth = 8;
 770     NumVectors = 0;
 771     Signed = true;
 772     break;
 773   case 'W':
 774     Constant = true;
 775     Pointer = true;
 776     ElementBitwidth = Bitwidth = 8;
 777     NumVectors = 0;
 778     Signed = false;
 779     break;
 780   case 'T':
 781     Constant = true;
 782     Pointer = true;
 783     ElementBitwidth = Bitwidth = 16;
 784     NumVectors = 0;
 785     Signed = true;
 786     break;
 787   case 'X':
 788     Constant = true;
 789     Pointer = true;
 790     ElementBitwidth = Bitwidth = 16;
 791     NumVectors = 0;
 792     Signed = false;
 793     break;
 794   case 'Y':
 795     Constant = true;
 796     Pointer = true;
 797     ElementBitwidth = Bitwidth = 32;
 798     NumVectors = 0;
 799     Signed = false;
 800     break;
 801   case 'U':
 802     Constant = true;
 803     Pointer = true;
 804     ElementBitwidth = Bitwidth = 32;
 805     NumVectors = 0;
 806     Signed = true;
 807     break;
 808   case '%':
 809     Pointer = true;
 810     Void = true;
 811     NumVectors = 0;
 812     break;
 813   case 'A':
 814     Pointer = true;
 815     ElementBitwidth = Bitwidth = 8;
 816     NumVectors = 0;
 817     Signed = true;
 818     break;
 819   case 'B':
 820     Pointer = true;
 821     ElementBitwidth = Bitwidth = 16;
 822     NumVectors = 0;
 823     Signed = true;
 824     break;
 825   case 'C':
 826     Pointer = true;
 827     ElementBitwidth = Bitwidth = 32;
 828     NumVectors = 0;
 829     Signed = true;
 830     break;
 831   case 'D':
 832     Pointer = true;
 833     ElementBitwidth = Bitwidth = 64;
 834     NumVectors = 0;
 835     Signed = true;
 836     break;
 837   case 'E':
 838     Pointer = true;
 839     ElementBitwidth = Bitwidth = 8;
 840     NumVectors = 0;
 841     Signed = false;
 842     break;
 843   case 'F':
 844     Pointer = true;
 845     ElementBitwidth = Bitwidth = 16;
 846     NumVectors = 0;
 847     Signed = false;
 848     break;
 849   case 'G':
 850     Pointer = true;
 851     ElementBitwidth = Bitwidth = 32;
 852     NumVectors = 0;
 853     Signed = false;
 854     break;
 855   case '$':
 856     Predicate = false;
 857     Svcount = false;
 858     Float = false;
 859     BFloat = true;
 860     ElementBitwidth = 16;
 861     break;
 862   case '}':
 863     Predicate = false;
 864     Signed = true;
 865     Svcount = true;
 866     NumVectors = 0;
 867     Float = false;
 868     BFloat = false;
 869     break;
 870   case '.':
 871     llvm_unreachable(". is never a type in itself");
 872     break;
 873   default:
 874     llvm_unreachable("Unhandled character!");
 875   }
 876 }
 877
 878 /// Returns the modifier and number of vectors for the given operand \p Op.
 879 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
 880   for (unsigned P = 0; !Proto.empty(); ++P) {
 881     unsigned NumVectors = 1;
 882     unsigned CharsToSkip = 1;
 883     char Mod = Proto[0];
 884     if (Mod == '2' || Mod == '3' || Mod == '4') {
 885       NumVectors = Mod - '0';
 886       Mod = 'd';
 887       if (Proto.size() > 1 && Proto[1] == '.') {
 888         Mod = Proto[2];
 889         CharsToSkip = 3;
 890       }
 891     }
 892
 893     if (P == Op)
 894       return {Mod, NumVectors};
 895
 896     Proto = Proto.drop_front(CharsToSkip);
 897   }
 898   llvm_unreachable("Unexpected Op");
 899 }
 900
 901 //===----------------------------------------------------------------------===//
 902 // Intrinsic implementation
 903 //===----------------------------------------------------------------------===//
 904
 905 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
 906                      StringRef MergeSuffix, uint64_t MemoryElementTy,
 907                      StringRef LLVMName, uint64_t Flags,
 908                      ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
 909                      SVEEmitter &Emitter, StringRef Guard)
 910     : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
 911       BaseTypeSpec(BT), Class(Class), Guard(Guard.str()),
 912       MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
 913       ImmChecks(Checks.begin(), Checks.end()) {
 914   // Types[0] is the return value.
 915   for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
 916     char Mod;
 917     unsigned NumVectors;
 918     std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
 919     SVEType T(BaseTypeSpec, Mod, NumVectors);
 920     Types.push_back(T);
 921
 922     // Add range checks for immediates
 923     if (I > 0) {
 924       if (T.isPredicatePattern())
 925         ImmChecks.emplace_back(
 926             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
 927       else if (T.isPrefetchOp())
 928         ImmChecks.emplace_back(
 929             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
 930     }
 931   }
 932
 933   // Set flags based on properties
 934   this->Flags |= Emitter.encodeTypeFlags(BaseType);
 935   this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
 936   this->Flags |= Emitter.encodeMergeType(MergeTy);
 937   if (hasSplat())
 938     this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
 939 }
 940
 941 std::string Intrinsic::getBuiltinTypeStr() {
 942   std::string S = getReturnType().builtin_str();
 943   for (unsigned I = 0; I < getNumParams(); ++I)
 944     S += getParamType(I).builtin_str();
 945
 946   return S;
 947 }
 948
 949 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
 950                                             std::string Proto) const {
 951   std::string Ret = Name;
 952   while (Ret.find('{') != std::string::npos) {
 953     size_t Pos = Ret.find('{');
 954     size_t End = Ret.find('}');
 955     unsigned NumChars = End - Pos + 1;
 956     assert(NumChars == 3 && "Unexpected template argument");
 957
 958     SVEType T;
 959     char C = Ret[Pos+1];
 960     switch(C) {
 961     default:
 962       llvm_unreachable("Unknown predication specifier");
 963     case 'd':
 964       T = SVEType(TS, 'd');
 965       break;
 966     case '0':
 967     case '1':
 968     case '2':
 969     case '3':
 970       T = SVEType(TS, Proto[C - '0']);
 971       break;
 972     }
 973
 974     // Replace templated arg with the right suffix (e.g. u32)
 975     std::string TypeCode;
 976     if (T.isInteger())
 977       TypeCode = T.isSigned() ? 's' : 'u';
 978     else if (T.isSvcount())
 979       TypeCode = 'c';
 980     else if (T.isPredicateVector())
 981       TypeCode = 'b';
 982     else if (T.isBFloat())
 983       TypeCode = "bf";
 984     else
 985       TypeCode = 'f';
 986     Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
 987   }
 988
 989   return Ret;
 990 }
 991
 992 std::string Intrinsic::mangleLLVMName() const {
 993   std::string S = getLLVMName();
 994
 995   // Replace all {d} like expressions with e.g. 'u32'
 996   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
 997 }
 998
 999 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1000   std::string S = getName();
1001
1002   if (LocalCK == ClassG) {
1003     // Remove the square brackets and everything in between.
1004     while (S.find('[') != std::string::npos) {
1005       auto Start = S.find('[');
1006       auto End = S.find(']');
1007       S.erase(Start, (End-Start)+1);
1008     }
1009   } else {
1010     // Remove the square brackets.
1011     while (S.find('[') != std::string::npos) {
1012       auto BrPos = S.find('[');
1013       if (BrPos != std::string::npos)
1014         S.erase(BrPos, 1);
1015       BrPos = S.find(']');
1016       if (BrPos != std::string::npos)
1017         S.erase(BrPos, 1);
1018     }
1019   }
1020
1021   // Replace all {d} like expressions with e.g. 'u32'
1022   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
1023          getMergeSuffix();
1024 }
1025
1026 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter) const {
1027   bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1028
1029   std::string FullName = mangleName(ClassS);
1030   std::string ProtoName = mangleName(getClassKind());
1031   std::string SMEAttrs = "";
1032
1033   if (Flags & Emitter.getEnumValueForFlag("IsStreaming"))
1034     SMEAttrs += ", arm_streaming";
1035   if (Flags & Emitter.getEnumValueForFlag("IsStreamingCompatible"))
1036     SMEAttrs += ", arm_streaming_compatible";
1037   if (Flags & Emitter.getEnumValueForFlag("IsSharedZA"))
1038     SMEAttrs += ", arm_shared_za";
1039   if (Flags & Emitter.getEnumValueForFlag("IsPreservesZA"))
1040     SMEAttrs += ", arm_preserves_za";
1041
1042   OS << (IsOverloaded ? "__aio " : "__ai ")
1043      << "__attribute__((__clang_arm_builtin_alias("
1044      << (SMEAttrs.empty() ? "__builtin_sve_" : "__builtin_sme_")
1045      << FullName << ")";
1046   if (!SMEAttrs.empty())
1047     OS << SMEAttrs;
1048   OS << "))\n";
1049
1050   OS << getTypes()[0].str() << " " << ProtoName << "(";
1051   for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1052     if (I != 0)
1053       OS << ", ";
1054     OS << getTypes()[I + 1].str();
1055   }
1056   OS << ");\n";
1057 }
1058
1059 //===----------------------------------------------------------------------===//
1060 // SVEEmitter implementation
1061 //===----------------------------------------------------------------------===//
1062 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1063   if (T.isFloat()) {
1064     switch (T.getElementSizeInBits()) {
1065     case 16:
1066       return encodeEltType("EltTyFloat16");
1067     case 32:
1068       return encodeEltType("EltTyFloat32");
1069     case 64:
1070       return encodeEltType("EltTyFloat64");
1071     default:
1072       llvm_unreachable("Unhandled float element bitwidth!");
1073     }
1074   }
1075
1076   if (T.isBFloat()) {
1077     assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1078     return encodeEltType("EltTyBFloat16");
1079   }
1080
1081   if (T.isPredicateVector() || T.isSvcount()) {
1082     switch (T.getElementSizeInBits()) {
1083     case 8:
1084       return encodeEltType("EltTyBool8");
1085     case 16:
1086       return encodeEltType("EltTyBool16");
1087     case 32:
1088       return encodeEltType("EltTyBool32");
1089     case 64:
1090       return encodeEltType("EltTyBool64");
1091     default:
1092       llvm_unreachable("Unhandled predicate element bitwidth!");
1093     }
1094   }
1095
1096   switch (T.getElementSizeInBits()) {
1097   case 8:
1098     return encodeEltType("EltTyInt8");
1099   case 16:
1100     return encodeEltType("EltTyInt16");
1101   case 32:
1102     return encodeEltType("EltTyInt32");
1103   case 64:
1104     return encodeEltType("EltTyInt64");
1105   case 128:
1106     return encodeEltType("EltTyInt128");
1107   default:
1108     llvm_unreachable("Unhandled integer element bitwidth!");
1109   }
1110 }
1111
1112 void SVEEmitter::createIntrinsic(
1113     Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1114   StringRef Name = R->getValueAsString("Name");
1115   StringRef Proto = R->getValueAsString("Prototype");
1116   StringRef Types = R->getValueAsString("Types");
1117   StringRef Guard = R->getValueAsString("TargetGuard");
1118   StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1119   uint64_t Merge = R->getValueAsInt("Merge");
1120   StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1121   uint64_t MemEltType = R->getValueAsInt("MemEltType");
1122   std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
1123   std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
1124
1125   int64_t Flags = 0;
1126   for (auto FlagRec : FlagsList)
1127     Flags |= FlagRec->getValueAsInt("Value");
1128
1129   // Create a dummy TypeSpec for non-overloaded builtins.
1130   if (Types.empty()) {
1131     assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1132            "Expect TypeSpec for overloaded builtin!");
1133     Types = "i";
1134   }
1135
1136   // Extract type specs from string
1137   SmallVector<TypeSpec, 8> TypeSpecs;
1138   TypeSpec Acc;
1139   for (char I : Types) {
1140     Acc.push_back(I);
1141     if (islower(I)) {
1142       TypeSpecs.push_back(TypeSpec(Acc));
1143       Acc.clear();
1144     }
1145   }
1146
1147   // Remove duplicate type specs.
1148   llvm::sort(TypeSpecs);
1149   TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1150                   TypeSpecs.end());
1151
1152   // Create an Intrinsic for each type spec.
1153   for (auto TS : TypeSpecs) {
1154     // Collate a list of range/option checks for the immediates.
1155     SmallVector<ImmCheck, 2> ImmChecks;
1156     for (auto *R : ImmCheckList) {
1157       int64_t Arg = R->getValueAsInt("Arg");
1158       int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
1159       int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
1160       assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
1161
1162       unsigned ElementSizeInBits = 0;
1163       char Mod;
1164       unsigned NumVectors;
1165       std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
1166       if (EltSizeArg >= 0)
1167         ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1168       ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
1169     }
1170
1171     Out.push_back(std::make_unique<Intrinsic>(
1172         Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1173         TS, ClassS, *this, Guard));
1174
1175     // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1176     if (Intrinsic::isOverloadedIntrinsic(Name))
1177       Out.push_back(std::make_unique<Intrinsic>(
1178           Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1179           ImmChecks, TS, ClassG, *this, Guard));
1180   }
1181 }
1182
1183 void SVEEmitter::createHeader(raw_ostream &OS) {
1184   OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1185         "-----------------------------------===\n"
1186         " *\n"
1187         " *\n"
1188         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1189         "Exceptions.\n"
1190         " * See https://llvm.org/LICENSE.txt for license information.\n"
1191         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1192         " *\n"
1193         " *===-----------------------------------------------------------------"
1194         "------===\n"
1195         " */\n\n";
1196
1197   OS << "#ifndef __ARM_SVE_H\n";
1198   OS << "#define __ARM_SVE_H\n\n";
1199
1200   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1201   OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1202   OS << "#endif\n";
1203
1204   OS << "#include <stdint.h>\n\n";
1205   OS << "#ifdef  __cplusplus\n";
1206   OS << "extern \"C\" {\n";
1207   OS << "#else\n";
1208   OS << "#include <stdbool.h>\n";
1209   OS << "#endif\n\n";
1210
1211   OS << "typedef __fp16 float16_t;\n";
1212   OS << "typedef float float32_t;\n";
1213   OS << "typedef double float64_t;\n";
1214
1215   OS << "typedef __SVInt8_t svint8_t;\n";
1216   OS << "typedef __SVInt16_t svint16_t;\n";
1217   OS << "typedef __SVInt32_t svint32_t;\n";
1218   OS << "typedef __SVInt64_t svint64_t;\n";
1219   OS << "typedef __SVUint8_t svuint8_t;\n";
1220   OS << "typedef __SVUint16_t svuint16_t;\n";
1221   OS << "typedef __SVUint32_t svuint32_t;\n";
1222   OS << "typedef __SVUint64_t svuint64_t;\n";
1223   OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1224
1225   OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1226
1227   OS << "#include <arm_bf16.h>\n";
1228
1229   OS << "typedef __SVFloat32_t svfloat32_t;\n";
1230   OS << "typedef __SVFloat64_t svfloat64_t;\n";
1231   OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1232   OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1233   OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1234   OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1235   OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1236   OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1237   OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1238   OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1239   OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1240   OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1241   OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1242   OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1243   OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1244   OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1245   OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1246   OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1247   OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1248   OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1249   OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1250   OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1251   OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1252   OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1253   OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1254   OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1255   OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1256   OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1257   OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1258   OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1259   OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1260   OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1261   OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1262   OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1263   OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1264   OS << "typedef __SVBool_t  svbool_t;\n";
1265   OS << "typedef __clang_svboolx2_t  svboolx2_t;\n";
1266   OS << "typedef __clang_svboolx4_t  svboolx4_t;\n\n";
1267
1268   OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1269   OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1270   OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1271
1272   OS << "typedef __SVCount_t svcount_t;\n\n";
1273
1274   OS << "enum svpattern\n";
1275   OS << "{\n";
1276   OS << "  SV_POW2 = 0,\n";
1277   OS << "  SV_VL1 = 1,\n";
1278   OS << "  SV_VL2 = 2,\n";
1279   OS << "  SV_VL3 = 3,\n";
1280   OS << "  SV_VL4 = 4,\n";
1281   OS << "  SV_VL5 = 5,\n";
1282   OS << "  SV_VL6 = 6,\n";
1283   OS << "  SV_VL7 = 7,\n";
1284   OS << "  SV_VL8 = 8,\n";
1285   OS << "  SV_VL16 = 9,\n";
1286   OS << "  SV_VL32 = 10,\n";
1287   OS << "  SV_VL64 = 11,\n";
1288   OS << "  SV_VL128 = 12,\n";
1289   OS << "  SV_VL256 = 13,\n";
1290   OS << "  SV_MUL4 = 29,\n";
1291   OS << "  SV_MUL3 = 30,\n";
1292   OS << "  SV_ALL = 31\n";
1293   OS << "};\n\n";
1294
1295   OS << "enum svprfop\n";
1296   OS << "{\n";
1297   OS << "  SV_PLDL1KEEP = 0,\n";
1298   OS << "  SV_PLDL1STRM = 1,\n";
1299   OS << "  SV_PLDL2KEEP = 2,\n";
1300   OS << "  SV_PLDL2STRM = 3,\n";
1301   OS << "  SV_PLDL3KEEP = 4,\n";
1302   OS << "  SV_PLDL3STRM = 5,\n";
1303   OS << "  SV_PSTL1KEEP = 8,\n";
1304   OS << "  SV_PSTL1STRM = 9,\n";
1305   OS << "  SV_PSTL2KEEP = 10,\n";
1306   OS << "  SV_PSTL2STRM = 11,\n";
1307   OS << "  SV_PSTL3KEEP = 12,\n";
1308   OS << "  SV_PSTL3STRM = 13\n";
1309   OS << "};\n\n";
1310
1311   OS << "/* Function attributes */\n";
1312   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1313         "__nodebug__))\n\n";
1314   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1315         "__nodebug__, __overloadable__))\n\n";
1316
1317   // Add reinterpret functions.
1318   for (auto ShortForm : { false, true } )
1319     for (const ReinterpretTypeInfo &From : Reinterprets)
1320       for (const ReinterpretTypeInfo &To : Reinterprets) {
1321         if (ShortForm) {
1322           OS << "__aio __attribute__((target(\"sve\"))) " << From.Type
1323              << " svreinterpret_" << From.Suffix;
1324           OS << "(" << To.Type << " op) __arm_streaming_compatible {\n";
1325           OS << "  return __builtin_sve_reinterpret_" << From.Suffix << "_"
1326              << To.Suffix << "(op);\n";
1327           OS << "}\n\n";
1328         } else
1329           OS << "#define svreinterpret_" << From.Suffix << "_" << To.Suffix
1330              << "(...) __builtin_sve_reinterpret_" << From.Suffix << "_"
1331              << To.Suffix << "(__VA_ARGS__)\n";
1332       }
1333
1334   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1335   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1336   for (auto *R : RV)
1337     createIntrinsic(R, Defs);
1338
1339   // Sort intrinsics in header file by following order/priority:
1340   // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1341   // - Class (is intrinsic overloaded or not)
1342   // - Intrinsic name
1343   std::stable_sort(
1344       Defs.begin(), Defs.end(), [](const std::unique_ptr<Intrinsic> &A,
1345                                    const std::unique_ptr<Intrinsic> &B) {
1346         auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1347           return std::make_tuple(I->getGuard(), (unsigned)I->getClassKind(), I->getName());
1348         };
1349         return ToTuple(A) < ToTuple(B);
1350       });
1351
1352   // Actually emit the intrinsic declarations.
1353   for (auto &I : Defs)
1354     I->emitIntrinsic(OS, *this);
1355
1356   OS << "#define svcvtnt_bf16_x      svcvtnt_bf16_m\n";
1357   OS << "#define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m\n";
1358
1359   OS << "#define svcvtnt_f16_x      svcvtnt_f16_m\n";
1360   OS << "#define svcvtnt_f16_f32_x  svcvtnt_f16_f32_m\n";
1361   OS << "#define svcvtnt_f32_x      svcvtnt_f32_m\n";
1362   OS << "#define svcvtnt_f32_f64_x  svcvtnt_f32_f64_m\n\n";
1363
1364   OS << "#define svcvtxnt_f32_x     svcvtxnt_f32_m\n";
1365   OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1366
1367   OS << "#ifdef __cplusplus\n";
1368   OS << "} // extern \"C\"\n";
1369   OS << "#endif\n\n";
1370   OS << "#undef __ai\n\n";
1371   OS << "#undef __aio\n\n";
1372   OS << "#endif /* __ARM_SVE_H */\n";
1373 }
1374
1375 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1376   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1377   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1378   for (auto *R : RV)
1379     createIntrinsic(R, Defs);
1380
1381   // The mappings must be sorted based on BuiltinID.
1382   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1383                       const std::unique_ptr<Intrinsic> &B) {
1384     return A->getMangledName() < B->getMangledName();
1385   });
1386
1387   OS << "#ifdef GET_SVE_BUILTINS\n";
1388   for (auto &Def : Defs) {
1389     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1390     // declarations only live in the header file.
1391     if (Def->getClassKind() != ClassG)
1392       OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1393          << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1394          << "\")\n";
1395   }
1396
1397   // Add reinterpret builtins
1398   for (const ReinterpretTypeInfo &From : Reinterprets)
1399     for (const ReinterpretTypeInfo &To : Reinterprets)
1400       OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << From.Suffix << "_"
1401          << To.Suffix << +", \"" << From.BuiltinType << To.BuiltinType
1402          << "\", \"n\", \"sve\")\n";
1403
1404   OS << "#endif\n\n";
1405 }
1406
1407 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1408   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1409   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1410   for (auto *R : RV)
1411     createIntrinsic(R, Defs);
1412
1413   // The mappings must be sorted based on BuiltinID.
1414   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1415                       const std::unique_ptr<Intrinsic> &B) {
1416     return A->getMangledName() < B->getMangledName();
1417   });
1418
1419   OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1420   for (auto &Def : Defs) {
1421     // Builtins only exist for non-overloaded intrinsics, overloaded
1422     // declarations only live in the header file.
1423     if (Def->getClassKind() == ClassG)
1424       continue;
1425
1426     uint64_t Flags = Def->getFlags();
1427     auto FlagString = std::to_string(Flags);
1428
1429     std::string LLVMName = Def->getMangledLLVMName();
1430     std::string Builtin = Def->getMangledName();
1431     if (!LLVMName.empty())
1432       OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1433          << "),\n";
1434     else
1435       OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1436   }
1437   OS << "#endif\n\n";
1438 }
1439
1440 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1441   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1442   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1443   for (auto *R : RV)
1444     createIntrinsic(R, Defs);
1445
1446   // The mappings must be sorted based on BuiltinID.
1447   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1448                       const std::unique_ptr<Intrinsic> &B) {
1449     return A->getMangledName() < B->getMangledName();
1450   });
1451
1452
1453   OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1454
1455   // Ensure these are only emitted once.
1456   std::set<std::string> Emitted;
1457
1458   for (auto &Def : Defs) {
1459     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1460         Def->getImmChecks().empty())
1461       continue;
1462
1463     OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1464     for (auto &Check : Def->getImmChecks())
1465       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1466          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1467     OS << "  break;\n";
1468
1469     Emitted.insert(Def->getMangledName());
1470   }
1471
1472   OS << "#endif\n\n";
1473 }
1474
1475 /// Create the SVETypeFlags used in CGBuiltins
1476 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1477   OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1478   for (auto &KV : FlagTypes)
1479     OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1480   OS << "#endif\n\n";
1481
1482   OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1483   for (auto &KV : EltTypes)
1484     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1485   OS << "#endif\n\n";
1486
1487   OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1488   for (auto &KV : MemEltTypes)
1489     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1490   OS << "#endif\n\n";
1491
1492   OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1493   for (auto &KV : MergeTypes)
1494     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1495   OS << "#endif\n\n";
1496
1497   OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
1498   for (auto &KV : ImmCheckTypes)
1499     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1500   OS << "#endif\n\n";
1501 }
1502
1503 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1504   OS << "/*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics "
1505         "------===\n"
1506         " *\n"
1507         " *\n"
1508         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1509         "Exceptions.\n"
1510         " * See https://llvm.org/LICENSE.txt for license information.\n"
1511         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1512         " *\n"
1513         " *===-----------------------------------------------------------------"
1514         "------===\n"
1515         " */\n\n";
1516
1517   OS << "#ifndef __ARM_SME_H\n";
1518   OS << "#define __ARM_SME_H\n\n";
1519
1520   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1521   OS << "#error \"Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h\"\n";
1522   OS << "#endif\n";
1523
1524   OS << "#include <arm_sve.h> \n\n";
1525
1526   OS << "/* Function attributes */\n";
1527   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1528         "__nodebug__))\n\n";
1529   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1530         "__nodebug__, __overloadable__))\n\n";
1531
1532   OS << "#ifdef  __cplusplus\n";
1533   OS << "extern \"C\" {\n";
1534   OS << "#endif\n\n";
1535
1536   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1537   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1538   for (auto *R : RV)
1539     createIntrinsic(R, Defs);
1540
1541   // Sort intrinsics in header file by following order/priority similar to SVE:
1542   // - Architectural guard
1543   // - Class (is intrinsic overloaded or not)
1544   // - Intrinsic name
1545   std::stable_sort(Defs.begin(), Defs.end(),
1546                    [](const std::unique_ptr<Intrinsic> &A,
1547                       const std::unique_ptr<Intrinsic> &B) {
1548                      auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1549                        return std::make_tuple(I->getGuard(),
1550                                               (unsigned)I->getClassKind(),
1551                                               I->getName());
1552                      };
1553                      return ToTuple(A) < ToTuple(B);
1554                    });
1555
1556   // Actually emit the intrinsic declaration.
1557   for (auto &I : Defs) {
1558     I->emitIntrinsic(OS, *this);
1559   }
1560
1561   OS << "#ifdef __cplusplus\n";
1562   OS << "} // extern \"C\"\n";
1563   OS << "#endif\n\n";
1564   OS << "#undef __ai\n\n";
1565   OS << "#endif /* __ARM_SME_H */\n";
1566 }
1567
1568 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1569   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1570   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1571   for (auto *R : RV) {
1572     createIntrinsic(R, Defs);
1573   }
1574
1575   // The mappings must be sorted based on BuiltinID.
1576   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1577                       const std::unique_ptr<Intrinsic> &B) {
1578     return A->getMangledName() < B->getMangledName();
1579   });
1580
1581   OS << "#ifdef GET_SME_BUILTINS\n";
1582   for (auto &Def : Defs) {
1583     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1584     // declarations only live in the header file.
1585     if (Def->getClassKind() != ClassG)
1586       OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
1587          << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1588          << "\")\n";
1589   }
1590
1591   OS << "#endif\n\n";
1592 }
1593
1594 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1595   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1596   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1597   for (auto *R : RV) {
1598     createIntrinsic(R, Defs);
1599   }
1600
1601   // The mappings must be sorted based on BuiltinID.
1602   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1603                       const std::unique_ptr<Intrinsic> &B) {
1604     return A->getMangledName() < B->getMangledName();
1605   });
1606
1607   OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1608   for (auto &Def : Defs) {
1609     // Builtins only exist for non-overloaded intrinsics, overloaded
1610     // declarations only live in the header file.
1611     if (Def->getClassKind() == ClassG)
1612       continue;
1613
1614     uint64_t Flags = Def->getFlags();
1615     auto FlagString = std::to_string(Flags);
1616
1617     std::string LLVMName = Def->getLLVMName();
1618     std::string Builtin = Def->getMangledName();
1619     if (!LLVMName.empty())
1620       OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1621          << "),\n";
1622     else
1623       OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1624   }
1625   OS << "#endif\n\n";
1626 }
1627
1628 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1629   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1630   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1631   for (auto *R : RV) {
1632     createIntrinsic(R, Defs);
1633   }
1634
1635   // The mappings must be sorted based on BuiltinID.
1636   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1637                       const std::unique_ptr<Intrinsic> &B) {
1638     return A->getMangledName() < B->getMangledName();
1639   });
1640
1641
1642   OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1643
1644   // Ensure these are only emitted once.
1645   std::set<std::string> Emitted;
1646
1647   for (auto &Def : Defs) {
1648     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1649         Def->getImmChecks().empty())
1650       continue;
1651
1652     OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1653     for (auto &Check : Def->getImmChecks())
1654       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1655          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1656     OS << "  break;\n";
1657
1658     Emitted.insert(Def->getMangledName());
1659   }
1660
1661   OS << "#endif\n\n";
1662 }
1663
1664 namespace clang {
1665 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
1666   SVEEmitter(Records).createHeader(OS);
1667 }
1668
1669 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1670   SVEEmitter(Records).createBuiltins(OS);
1671 }
1672
1673 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1674   SVEEmitter(Records).createCodeGenMap(OS);
1675 }
1676
1677 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1678   SVEEmitter(Records).createRangeChecks(OS);
1679 }
1680
1681 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
1682   SVEEmitter(Records).createTypeFlags(OS);
1683 }
1684
1685 void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
1686   SVEEmitter(Records).createSMEHeader(OS);
1687 }
1688
1689 void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1690   SVEEmitter(Records).createSMEBuiltins(OS);
1691 }
1692
1693 void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1694   SVEEmitter(Records).createSMECodeGenMap(OS);
1695 }
1696
1697 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1698   SVEEmitter(Records).createSMERangeChecks(OS);
1699 }
1700 } // End namespace clang