?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[libc] Switch to using the generic `<gpuintrin.h>` implementations (#121810)
[llvm-project.git] / clang / utils / TableGen / SveEmitter.cpp
blob97b768db3a31355e52daa4650b45a11e0ef7d993
1 //===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors. Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringMap.h"
30 #include "llvm/TableGen/AArch64ImmCheck.h"
31 #include "llvm/TableGen/Error.h"
32 #include "llvm/TableGen/Record.h"
33 #include <array>
34 #include <cctype>
35 #include <set>
36 #include <string>
37 #include <tuple>
38
39 using namespace llvm;
40
41 enum ClassKind {
42 ClassNone,
43 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
44 ClassG, // Overloaded name without type suffix
45 };
46
47 enum class ACLEKind { SVE, SME };
48
49 using TypeSpec = std::string;
50
51 namespace {
52 class SVEType {
53
54 enum TypeKind {
55 Invalid,
56 Void,
57 Float,
58 SInt,
59 UInt,
60 BFloat16,
61 MFloat8,
62 Svcount,
63 PrefetchOp,
64 PredicatePattern,
65 Predicate,
66 Fpm
67 };
68 TypeKind Kind;
69 bool Immediate, Constant, Pointer, DefaultType, IsScalable;
70 unsigned Bitwidth, ElementBitwidth, NumVectors;
71
72 public:
73 SVEType() : SVEType("", 'v') {}
74
75 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
76 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
77 DefaultType(false), IsScalable(true), Bitwidth(128),
78 ElementBitwidth(~0U), NumVectors(NumVectors) {
79 if (!TS.empty())
80 applyTypespec(TS);
81 applyModifier(CharMod);
82 }
83
84 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
85 NumVectors = NumV;
86 }
87
88 bool isPointer() const { return Pointer; }
89 bool isConstant() const { return Constant; }
90 bool isImmediate() const { return Immediate; }
91 bool isScalar() const { return NumVectors == 0; }
92 bool isVector() const { return NumVectors > 0; }
93 bool isScalableVector() const { return isVector() && IsScalable; }
94 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
95 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
96 bool isVoid() const { return Kind == Void; }
97 bool isDefault() const { return DefaultType; }
98 bool isFloat() const { return Kind == Float; }
99 bool isBFloat() const { return Kind == BFloat16; }
100 bool isMFloat() const { return Kind == MFloat8; }
101 bool isFloatingPoint() const {
102 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
103 }
104 bool isInteger() const { return Kind == SInt || Kind == UInt; }
105 bool isSignedInteger() const { return Kind == SInt; }
106 bool isUnsignedInteger() const { return Kind == UInt; }
107 bool isScalarPredicate() const {
108 return Kind == Predicate && NumVectors == 0;
109 }
110 bool isPredicate() const { return Kind == Predicate; }
111 bool isPredicatePattern() const { return Kind == PredicatePattern; }
112 bool isPrefetchOp() const { return Kind == PrefetchOp; }
113 bool isSvcount() const { return Kind == Svcount; }
114 bool isFpm() const { return Kind == Fpm; }
115 bool isInvalid() const { return Kind == Invalid; }
116 unsigned getElementSizeInBits() const { return ElementBitwidth; }
117 unsigned getNumVectors() const { return NumVectors; }
118
119 unsigned getNumElements() const {
120 assert(ElementBitwidth != ~0U);
121 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
122 }
123 unsigned getSizeInBits() const {
124 return Bitwidth;
125 }
126
127 /// Return the string representation of a type, which is an encoded
128 /// string for passing to the BUILTIN() macro in Builtins.def.
129 std::string builtin_str() const;
130
131 /// Return the C/C++ string representation of a type for use in the
132 /// arm_sve.h header file.
133 std::string str() const;
134
135 private:
136 /// Creates the type based on the typespec string in TS.
137 void applyTypespec(StringRef TS);
138
139 /// Applies a prototype modifier to the type.
140 void applyModifier(char Mod);
141
142 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
143 std::string builtinBaseType() const;
144 };
145
146 class SVEEmitter;
147
148 /// The main grunt class. This represents an instantiation of an intrinsic with
149 /// a particular typespec and prototype.
150 class Intrinsic {
151 /// The unmangled name.
152 std::string Name;
153
154 /// The name of the corresponding LLVM IR intrinsic.
155 std::string LLVMName;
156
157 /// Intrinsic prototype.
158 std::string Proto;
159
160 /// The base type spec for this intrinsic.
161 TypeSpec BaseTypeSpec;
162
163 /// The base class kind. Most intrinsics use ClassS, which has full type
164 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
165 /// intrinsics.
166 ClassKind Class;
167
168 /// The architectural #ifdef guard.
169 std::string SVEGuard, SMEGuard;
170
171 // The merge suffix such as _m, _x or _z.
172 std::string MergeSuffix;
173
174 /// The types of return value [0] and parameters [1..].
175 std::vector<SVEType> Types;
176
177 /// The "base type", which is VarType('d', BaseTypeSpec).
178 SVEType BaseType;
179
180 uint64_t Flags;
181
182 SmallVector<ImmCheck, 2> ImmChecks;
183
184 public:
185 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
186 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
187 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
188 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
189 StringRef SMEGuard);
190
191 ~Intrinsic()=default;
192
193 std::string getName() const { return Name; }
194 std::string getLLVMName() const { return LLVMName; }
195 std::string getProto() const { return Proto; }
196 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
197 SVEType getBaseType() const { return BaseType; }
198
199 StringRef getSVEGuard() const { return SVEGuard; }
200 StringRef getSMEGuard() const { return SMEGuard; }
201 void printGuard(raw_ostream &OS) const {
202 if (!SVEGuard.empty() && SMEGuard.empty())
203 OS << SVEGuard;
204 else if (SVEGuard.empty() && !SMEGuard.empty())
205 OS << SMEGuard;
206 else {
207 if (SVEGuard.find(",") != std::string::npos ||
208 SVEGuard.find("|") != std::string::npos)
209 OS << "(" << SVEGuard << ")";
210 else
211 OS << SVEGuard;
212 OS << "|";
213 if (SMEGuard.find(",") != std::string::npos ||
214 SMEGuard.find("|") != std::string::npos)
215 OS << "(" << SMEGuard << ")";
216 else
217 OS << SMEGuard;
218 }
219 }
220 ClassKind getClassKind() const { return Class; }
221
222 SVEType getReturnType() const { return Types[0]; }
223 ArrayRef<SVEType> getTypes() const { return Types; }
224 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
225 unsigned getNumParams() const {
226 return Proto.size() - (2 * count(Proto, '.')) - 1;
227 }
228
229 uint64_t getFlags() const { return Flags; }
230 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
231
232 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
233
234 /// Return the type string for a BUILTIN() macro in Builtins.def.
235 std::string getBuiltinTypeStr();
236
237 /// Return the name, mangled with type information. The name is mangled for
238 /// ClassS, so will add type suffixes such as _u32/_s32.
239 std::string getMangledName() const { return mangleName(ClassS); }
240
241 /// As above, but mangles the LLVM name instead.
242 std::string getMangledLLVMName() const { return mangleLLVMName(); }
243
244 /// Returns true if the intrinsic is overloaded, in that it should also generate
245 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
246 /// 'svld1_u32(..)'.
247 static bool isOverloadedIntrinsic(StringRef Name) {
248 auto BrOpen = Name.find('[');
249 auto BrClose = Name.find(']');
250 return BrOpen != std::string::npos && BrClose != std::string::npos;
251 }
252
253 /// Return true if the intrinsic takes a splat operand.
254 bool hasSplat() const {
255 // These prototype modifiers are described in arm_sve.td.
256 return Proto.find_first_of("ajfrKLR@!") != std::string::npos;
257 }
258
259 /// Return the parameter index of the splat operand.
260 unsigned getSplatIdx() const {
261 unsigned I = 1, Param = 0;
262 for (; I < Proto.size(); ++I, ++Param) {
263 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
264 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
265 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
266 break;
267
268 // Multivector modifier can be skipped
269 if (Proto[I] == '.')
270 I += 2;
271 }
272 assert(I != Proto.size() && "Prototype has no splat operand");
273 return Param;
274 }
275
276 /// Emits the intrinsic declaration to the ostream.
277 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
278
279 private:
280 std::string getMergeSuffix() const { return MergeSuffix; }
281 std::string mangleName(ClassKind LocalCK) const;
282 std::string mangleLLVMName() const;
283 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
284 std::string Proto) const;
285 };
286
287 class SVEEmitter {
288 private:
289 // The reinterpret builtins are generated separately because they
290 // need the cross product of all types (121 functions in total),
291 // which is inconvenient to specify in the arm_sve.td file or
292 // generate in CGBuiltin.cpp.
293 struct ReinterpretTypeInfo {
294 SVEType BaseType;
295 const char *Suffix;
296 };
297
298 static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
299
300 const RecordKeeper &Records;
301 StringMap<uint64_t> EltTypes;
302 StringMap<uint64_t> MemEltTypes;
303 StringMap<uint64_t> FlagTypes;
304 StringMap<uint64_t> MergeTypes;
305 StringMap<uint64_t> ImmCheckTypes;
306
307 public:
308 SVEEmitter(const RecordKeeper &R) : Records(R) {
309 for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
310 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
311 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
312 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
313 for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
314 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
315 for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
316 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
317 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
318 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
319 }
320
321 /// Returns the enum value for the immcheck type
322 unsigned getEnumValueForImmCheck(StringRef C) const {
323 auto It = ImmCheckTypes.find(C);
324 if (It != ImmCheckTypes.end())
325 return It->getValue();
326 llvm_unreachable("Unsupported imm check");
327 }
328
329 /// Returns the enum value for the flag type
330 uint64_t getEnumValueForFlag(StringRef C) const {
331 auto Res = FlagTypes.find(C);
332 if (Res != FlagTypes.end())
333 return Res->getValue();
334 llvm_unreachable("Unsupported flag");
335 }
336
337 // Returns the SVETypeFlags for a given value and mask.
338 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
339 auto It = FlagTypes.find(MaskName);
340 if (It != FlagTypes.end()) {
341 uint64_t Mask = It->getValue();
342 unsigned Shift = countr_zero(Mask);
343 assert(Shift < 64 && "Mask value produced an invalid shift value");
344 return (V << Shift) & Mask;
345 }
346 llvm_unreachable("Unsupported flag");
347 }
348
349 // Returns the SVETypeFlags for the given element type.
350 uint64_t encodeEltType(StringRef EltName) {
351 auto It = EltTypes.find(EltName);
352 if (It != EltTypes.end())
353 return encodeFlag(It->getValue(), "EltTypeMask");
354 llvm_unreachable("Unsupported EltType");
355 }
356
357 // Returns the SVETypeFlags for the given memory element type.
358 uint64_t encodeMemoryElementType(uint64_t MT) {
359 return encodeFlag(MT, "MemEltTypeMask");
360 }
361
362 // Returns the SVETypeFlags for the given merge type.
363 uint64_t encodeMergeType(uint64_t MT) {
364 return encodeFlag(MT, "MergeTypeMask");
365 }
366
367 // Returns the SVETypeFlags for the given splat operand.
368 unsigned encodeSplatOperand(unsigned SplatIdx) {
369 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
370 return encodeFlag(SplatIdx + 1, "SplatOperandMask");
371 }
372
373 // Returns the SVETypeFlags value for the given SVEType.
374 uint64_t encodeTypeFlags(const SVEType &T);
375
376 /// Emit arm_sve.h.
377 void createHeader(raw_ostream &o);
378
379 // Emits core intrinsics in both arm_sme.h and arm_sve.h
380 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
381 ACLEKind Kind);
382
383 /// Emit all the __builtin prototypes and code needed by Sema.
384 void createBuiltins(raw_ostream &o);
385
386 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
387 void createCodeGenMap(raw_ostream &o);
388
389 /// Emit all the range checks for the immediates.
390 void createRangeChecks(raw_ostream &o);
391
392 // Emit all the ImmCheckTypes to arm_immcheck_types.inc
393 void createImmCheckTypes(raw_ostream &OS);
394
395 /// Create the SVETypeFlags used in CGBuiltins
396 void createTypeFlags(raw_ostream &o);
397
398 /// Emit arm_sme.h.
399 void createSMEHeader(raw_ostream &o);
400
401 /// Emit all the SME __builtin prototypes and code needed by Sema.
402 void createSMEBuiltins(raw_ostream &o);
403
404 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
405 void createSMECodeGenMap(raw_ostream &o);
406
407 /// Create a table for a builtin's requirement for PSTATE.SM.
408 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
409
410 /// Emit all the range checks for the immediates.
411 void createSMERangeChecks(raw_ostream &o);
412
413 /// Create a table for a builtin's requirement for PSTATE.ZA.
414 void createBuiltinZAState(raw_ostream &OS);
415
416 /// Create intrinsic and add it to \p Out
417 void createIntrinsic(const Record *R,
418 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
419 };
420
421 const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
422 {{{SVEType("c", 'd'), "s8"},
423 {SVEType("Uc", 'd'), "u8"},
424 {SVEType("s", 'd'), "s16"},
425 {SVEType("Us", 'd'), "u16"},
426 {SVEType("i", 'd'), "s32"},
427 {SVEType("Ui", 'd'), "u32"},
428 {SVEType("l", 'd'), "s64"},
429 {SVEType("Ul", 'd'), "u64"},
430 {SVEType("h", 'd'), "f16"},
431 {SVEType("b", 'd'), "bf16"},
432 {SVEType("f", 'd'), "f32"},
433 {SVEType("d", 'd'), "f64"}}};
434
435 } // end anonymous namespace
436
437 //===----------------------------------------------------------------------===//
438 // Type implementation
439 //===----------------------------------------------------------------------===//
440
441 std::string SVEType::builtinBaseType() const {
442 switch (Kind) {
443 case TypeKind::Void:
444 return "v";
445 case TypeKind::Svcount:
446 return "Qa";
447 case TypeKind::PrefetchOp:
448 case TypeKind::PredicatePattern:
449 return "i";
450 case TypeKind::Fpm:
451 return "Wi";
452 case TypeKind::Predicate:
453 return "b";
454 case TypeKind::BFloat16:
455 assert(ElementBitwidth == 16 && "Invalid BFloat16!");
456 return "y";
457 case TypeKind::MFloat8:
458 assert(ElementBitwidth == 8 && "Invalid MFloat8!");
459 return "c";
460 case TypeKind::Float:
461 switch (ElementBitwidth) {
462 case 16:
463 return "h";
464 case 32:
465 return "f";
466 case 64:
467 return "d";
468 default:
469 llvm_unreachable("Unhandled float width!");
470 }
471 case TypeKind::SInt:
472 case TypeKind::UInt:
473 switch (ElementBitwidth) {
474 case 1:
475 return "b";
476 case 8:
477 return "c";
478 case 16:
479 return "s";
480 case 32:
481 return "i";
482 case 64:
483 return "Wi";
484 case 128:
485 return "LLLi";
486 default:
487 llvm_unreachable("Unhandled bitwidth!");
488 }
489 case TypeKind::Invalid:
490 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
491 }
492 llvm_unreachable("Unhandled TypeKind!");
493 }
494
495 std::string SVEType::builtin_str() const {
496 std::string Prefix;
497
498 if (isScalableVector())
499 Prefix = "q" + llvm::utostr(getNumElements() * NumVectors);
500 else if (isFixedLengthVector())
501 Prefix = "V" + llvm::utostr(getNumElements() * NumVectors);
502 else if (isImmediate()) {
503 assert(!isFloatingPoint() && "fp immediates are not supported");
504 Prefix = "I";
505 }
506
507 // Make chars and integer pointers explicitly signed.
508 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
509 Prefix += "S";
510 else if (isUnsignedInteger())
511 Prefix += "U";
512
513 std::string BuiltinStr = Prefix + builtinBaseType();
514 if (isConstant())
515 BuiltinStr += "C";
516 if (isPointer())
517 BuiltinStr += "*";
518
519 return BuiltinStr;
520 }
521
522 std::string SVEType::str() const {
523 std::string TypeStr;
524
525 switch (Kind) {
526 case TypeKind::PrefetchOp:
527 return "enum svprfop";
528 case TypeKind::PredicatePattern:
529 return "enum svpattern";
530 case TypeKind::Fpm:
531 TypeStr += "fpm";
532 break;
533 case TypeKind::Void:
534 TypeStr += "void";
535 break;
536 case TypeKind::Float:
537 TypeStr += "float" + llvm::utostr(ElementBitwidth);
538 break;
539 case TypeKind::Svcount:
540 TypeStr += "svcount";
541 break;
542 case TypeKind::Predicate:
543 TypeStr += "bool";
544 break;
545 case TypeKind::BFloat16:
546 TypeStr += "bfloat16";
547 break;
548 case TypeKind::MFloat8:
549 TypeStr += "mfloat8";
550 break;
551 case TypeKind::SInt:
552 TypeStr += "int" + llvm::utostr(ElementBitwidth);
553 break;
554 case TypeKind::UInt:
555 TypeStr += "uint" + llvm::utostr(ElementBitwidth);
556 break;
557 case TypeKind::Invalid:
558 llvm_unreachable("Attempting to resolve type name from Invalid type!");
559 }
560
561 if (isFixedLengthVector())
562 TypeStr += "x" + llvm::utostr(getNumElements());
563 else if (isScalableVector())
564 TypeStr = "sv" + TypeStr;
565
566 if (NumVectors > 1)
567 TypeStr += "x" + llvm::utostr(NumVectors);
568 if (!isScalarPredicate() && !isVoid())
569 TypeStr += "_t";
570 if (isConstant())
571 TypeStr += " const";
572 if (isPointer())
573 TypeStr += " *";
574
575 return TypeStr;
576 }
577
578 void SVEType::applyTypespec(StringRef TS) {
579 for (char I : TS) {
580 switch (I) {
581 case 'Q':
582 assert(isInvalid() && "Unexpected use of typespec modifier");
583 Kind = Svcount;
584 break;
585 case 'P':
586 assert(isInvalid() && "Unexpected use of typespec modifier");
587 Kind = Predicate;
588 break;
589 case 'U':
590 assert(isInvalid() && "Unexpected use of typespec modifier");
591 Kind = UInt;
592 break;
593 case 'c':
594 Kind = isInvalid() ? SInt : Kind;
595 ElementBitwidth = 8;
596 break;
597 case 's':
598 Kind = isInvalid() ? SInt : Kind;
599 ElementBitwidth = 16;
600 break;
601 case 'i':
602 Kind = isInvalid() ? SInt : Kind;
603 ElementBitwidth = 32;
604 break;
605 case 'l':
606 Kind = isInvalid() ? SInt : Kind;
607 ElementBitwidth = 64;
608 break;
609 case 'q':
610 Kind = isInvalid() ? SInt : Kind;
611 ElementBitwidth = 128;
612 break;
613 case 'h':
614 assert(isInvalid() && "Unexpected use of typespec modifier");
615 Kind = Float;
616 ElementBitwidth = 16;
617 break;
618 case 'f':
619 assert(isInvalid() && "Unexpected use of typespec modifier");
620 Kind = Float;
621 ElementBitwidth = 32;
622 break;
623 case 'd':
624 assert(isInvalid() && "Unexpected use of typespec modifier");
625 Kind = Float;
626 ElementBitwidth = 64;
627 break;
628 case 'b':
629 assert(isInvalid() && "Unexpected use of typespec modifier");
630 Kind = BFloat16;
631 ElementBitwidth = 16;
632 break;
633 case 'm':
634 assert(isInvalid() && "Unexpected use of typespec modifier");
635 Kind = MFloat8;
636 ElementBitwidth = 8;
637 break;
638 default:
639 llvm_unreachable("Unhandled type code!");
640 }
641 }
642 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
643 }
644
645 void SVEType::applyModifier(char Mod) {
646 switch (Mod) {
647 case 'v':
648 Kind = Void;
649 NumVectors = 0;
650 break;
651 case 'd':
652 DefaultType = true;
653 break;
654 case 'c':
655 Constant = true;
656 [[fallthrough]];
657 case 'p':
658 Pointer = true;
659 Bitwidth = ElementBitwidth;
660 NumVectors = 0;
661 break;
662 case 'e':
663 Kind = UInt;
664 ElementBitwidth /= 2;
665 break;
666 case 'h':
667 ElementBitwidth /= 2;
668 break;
669 case 'q':
670 ElementBitwidth /= 4;
671 break;
672 case 'b':
673 Kind = UInt;
674 ElementBitwidth /= 4;
675 break;
676 case 'o':
677 ElementBitwidth *= 4;
678 break;
679 case 'P':
680 Kind = Predicate;
681 Bitwidth = 16;
682 ElementBitwidth = 1;
683 break;
684 case '{':
685 IsScalable = false;
686 Bitwidth = 128;
687 NumVectors = 1;
688 break;
689 case 's':
690 case 'a':
691 Bitwidth = ElementBitwidth;
692 NumVectors = 0;
693 break;
694 case 'R':
695 ElementBitwidth /= 2;
696 NumVectors = 0;
697 break;
698 case 'r':
699 ElementBitwidth /= 4;
700 NumVectors = 0;
701 break;
702 case '@':
703 Kind = UInt;
704 ElementBitwidth /= 4;
705 NumVectors = 0;
706 break;
707 case 'K':
708 Kind = SInt;
709 Bitwidth = ElementBitwidth;
710 NumVectors = 0;
711 break;
712 case 'L':
713 Kind = UInt;
714 Bitwidth = ElementBitwidth;
715 NumVectors = 0;
716 break;
717 case 'u':
718 Kind = UInt;
719 break;
720 case 'x':
721 Kind = SInt;
722 break;
723 case 'i':
724 Kind = UInt;
725 ElementBitwidth = Bitwidth = 64;
726 NumVectors = 0;
727 Immediate = true;
728 break;
729 case 'I':
730 Kind = PredicatePattern;
731 ElementBitwidth = Bitwidth = 32;
732 NumVectors = 0;
733 Immediate = true;
734 break;
735 case 'J':
736 Kind = PrefetchOp;
737 ElementBitwidth = Bitwidth = 32;
738 NumVectors = 0;
739 Immediate = true;
740 break;
741 case 'k':
742 Kind = SInt;
743 ElementBitwidth = Bitwidth = 32;
744 NumVectors = 0;
745 break;
746 case 'l':
747 Kind = SInt;
748 ElementBitwidth = Bitwidth = 64;
749 NumVectors = 0;
750 break;
751 case 'm':
752 Kind = UInt;
753 ElementBitwidth = Bitwidth = 32;
754 NumVectors = 0;
755 break;
756 case '>':
757 Kind = Fpm;
758 ElementBitwidth = Bitwidth = 64;
759 NumVectors = 0;
760 break;
761 case 'n':
762 Kind = UInt;
763 ElementBitwidth = Bitwidth = 64;
764 NumVectors = 0;
765 break;
766 case 'w':
767 ElementBitwidth = 64;
768 break;
769 case 'j':
770 ElementBitwidth = Bitwidth = 64;
771 NumVectors = 0;
772 break;
773 case 'f':
774 Kind = UInt;
775 ElementBitwidth = Bitwidth = 64;
776 NumVectors = 0;
777 break;
778 case 'g':
779 Kind = UInt;
780 ElementBitwidth = 64;
781 break;
782 case '#':
783 Kind = SInt;
784 ElementBitwidth = 64;
785 break;
786 case '[':
787 Kind = UInt;
788 ElementBitwidth = 8;
789 break;
790 case 't':
791 Kind = SInt;
792 ElementBitwidth = 32;
793 break;
794 case 'z':
795 Kind = UInt;
796 ElementBitwidth = 32;
797 break;
798 case 'O':
799 Kind = Float;
800 ElementBitwidth = 16;
801 break;
802 case 'M':
803 Kind = Float;
804 ElementBitwidth = 32;
805 break;
806 case 'N':
807 Kind = Float;
808 ElementBitwidth = 64;
809 break;
810 case 'Q':
811 Kind = Void;
812 Constant = true;
813 Pointer = true;
814 NumVectors = 0;
815 break;
816 case 'S':
817 Kind = SInt;
818 Constant = true;
819 Pointer = true;
820 ElementBitwidth = Bitwidth = 8;
821 NumVectors = 0;
822 break;
823 case 'W':
824 Kind = UInt;
825 Constant = true;
826 Pointer = true;
827 ElementBitwidth = Bitwidth = 8;
828 NumVectors = 0;
829 break;
830 case 'T':
831 Kind = SInt;
832 Constant = true;
833 Pointer = true;
834 ElementBitwidth = Bitwidth = 16;
835 NumVectors = 0;
836 break;
837 case 'X':
838 Kind = UInt;
839 Constant = true;
840 Pointer = true;
841 ElementBitwidth = Bitwidth = 16;
842 NumVectors = 0;
843 break;
844 case 'Y':
845 Kind = UInt;
846 Constant = true;
847 Pointer = true;
848 ElementBitwidth = Bitwidth = 32;
849 NumVectors = 0;
850 break;
851 case 'U':
852 Kind = SInt;
853 Constant = true;
854 Pointer = true;
855 ElementBitwidth = Bitwidth = 32;
856 NumVectors = 0;
857 break;
858 case '%':
859 Kind = Void;
860 Pointer = true;
861 NumVectors = 0;
862 break;
863 case 'A':
864 Kind = SInt;
865 Pointer = true;
866 ElementBitwidth = Bitwidth = 8;
867 NumVectors = 0;
868 break;
869 case 'B':
870 Kind = SInt;
871 Pointer = true;
872 ElementBitwidth = Bitwidth = 16;
873 NumVectors = 0;
874 break;
875 case 'C':
876 Kind = SInt;
877 Pointer = true;
878 ElementBitwidth = Bitwidth = 32;
879 NumVectors = 0;
880 break;
881 case 'D':
882 Kind = SInt;
883 Pointer = true;
884 ElementBitwidth = Bitwidth = 64;
885 NumVectors = 0;
886 break;
887 case 'E':
888 Kind = UInt;
889 Pointer = true;
890 ElementBitwidth = Bitwidth = 8;
891 NumVectors = 0;
892 break;
893 case 'F':
894 Kind = UInt;
895 Pointer = true;
896 ElementBitwidth = Bitwidth = 16;
897 NumVectors = 0;
898 break;
899 case 'G':
900 Kind = UInt;
901 Pointer = true;
902 ElementBitwidth = Bitwidth = 32;
903 NumVectors = 0;
904 break;
905 case '$':
906 Kind = BFloat16;
907 ElementBitwidth = 16;
908 break;
909 case '}':
910 Kind = Svcount;
911 NumVectors = 0;
912 break;
913 case '~':
914 Kind = MFloat8;
915 ElementBitwidth = 8;
916 break;
917 case '!':
918 Kind = MFloat8;
919 Bitwidth = ElementBitwidth = 8;
920 NumVectors = 0;
921 break;
922 case '.':
923 llvm_unreachable(". is never a type in itself");
924 break;
925 default:
926 llvm_unreachable("Unhandled character!");
927 }
928 }
929
930 /// Returns the modifier and number of vectors for the given operand \p Op.
931 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
932 for (unsigned P = 0; !Proto.empty(); ++P) {
933 unsigned NumVectors = 1;
934 unsigned CharsToSkip = 1;
935 char Mod = Proto[0];
936 if (Mod == '2' || Mod == '3' || Mod == '4') {
937 NumVectors = Mod - '0';
938 Mod = 'd';
939 if (Proto.size() > 1 && Proto[1] == '.') {
940 Mod = Proto[2];
941 CharsToSkip = 3;
942 }
943 }
944
945 if (P == Op)
946 return {Mod, NumVectors};
947
948 Proto = Proto.drop_front(CharsToSkip);
949 }
950 llvm_unreachable("Unexpected Op");
951 }
952
953 //===----------------------------------------------------------------------===//
954 // Intrinsic implementation
955 //===----------------------------------------------------------------------===//
956
957 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
958 StringRef MergeSuffix, uint64_t MemoryElementTy,
959 StringRef LLVMName, uint64_t Flags,
960 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
961 SVEEmitter &Emitter, StringRef SVEGuard,
962 StringRef SMEGuard)
963 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
964 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
965 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
966
967 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
968 if (Guard.contains('|'))
969 return Base.str() + ",(" + Guard.str() + ")";
970 if (Guard.empty() || Guard == Base || Guard.starts_with(Base.str() + ","))
971 return Guard.str();
972 return Base.str() + "," + Guard.str();
973 };
974
975 this->SVEGuard = FormatGuard(SVEGuard, "sve");
976 this->SMEGuard = FormatGuard(SMEGuard, "sme");
977
978 // Types[0] is the return value.
979 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
980 char Mod;
981 unsigned NumVectors;
982 std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
983 SVEType T(BaseTypeSpec, Mod, NumVectors);
984 Types.push_back(T);
985
986 // Add range checks for immediates
987 if (I > 0) {
988 if (T.isPredicatePattern())
989 ImmChecks.emplace_back(
990 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
991 else if (T.isPrefetchOp())
992 ImmChecks.emplace_back(
993 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
994 }
995 }
996
997 // Set flags based on properties
998 this->Flags |= Emitter.encodeTypeFlags(BaseType);
999 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
1000 this->Flags |= Emitter.encodeMergeType(MergeTy);
1001 if (hasSplat())
1002 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
1003 }
1004
1005 std::string Intrinsic::getBuiltinTypeStr() {
1006 std::string S = getReturnType().builtin_str();
1007 for (unsigned I = 0; I < getNumParams(); ++I)
1008 S += getParamType(I).builtin_str();
1009
1010 return S;
1011 }
1012
1013 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1014 std::string Proto) const {
1015 std::string Ret = Name;
1016 while (Ret.find('{') != std::string::npos) {
1017 size_t Pos = Ret.find('{');
1018 size_t End = Ret.find('}');
1019 unsigned NumChars = End - Pos + 1;
1020 assert(NumChars == 3 && "Unexpected template argument");
1021
1022 SVEType T;
1023 char C = Ret[Pos+1];
1024 switch(C) {
1025 default:
1026 llvm_unreachable("Unknown predication specifier");
1027 case 'd':
1028 T = SVEType(TS, 'd');
1029 break;
1030 case '0':
1031 case '1':
1032 case '2':
1033 case '3':
1034 T = SVEType(TS, Proto[C - '0']);
1035 break;
1036 }
1037
1038 // Replace templated arg with the right suffix (e.g. u32)
1039 std::string TypeCode;
1040
1041 if (T.isSignedInteger())
1042 TypeCode = 's';
1043 else if (T.isUnsignedInteger())
1044 TypeCode = 'u';
1045 else if (T.isSvcount())
1046 TypeCode = 'c';
1047 else if (T.isPredicate())
1048 TypeCode = 'b';
1049 else if (T.isBFloat())
1050 TypeCode = "bf";
1051 else if (T.isMFloat())
1052 TypeCode = "mfp";
1053 else
1054 TypeCode = 'f';
1055 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
1056 }
1057
1058 return Ret;
1059 }
1060
1061 std::string Intrinsic::mangleLLVMName() const {
1062 std::string S = getLLVMName();
1063
1064 // Replace all {d} like expressions with e.g. 'u32'
1065 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
1066 }
1067
1068 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1069 std::string S = getName();
1070
1071 if (LocalCK == ClassG) {
1072 // Remove the square brackets and everything in between.
1073 while (S.find('[') != std::string::npos) {
1074 auto Start = S.find('[');
1075 auto End = S.find(']');
1076 S.erase(Start, (End-Start)+1);
1077 }
1078 } else {
1079 // Remove the square brackets.
1080 while (S.find('[') != std::string::npos) {
1081 auto BrPos = S.find('[');
1082 if (BrPos != std::string::npos)
1083 S.erase(BrPos, 1);
1084 BrPos = S.find(']');
1085 if (BrPos != std::string::npos)
1086 S.erase(BrPos, 1);
1087 }
1088 }
1089
1090 // Replace all {d} like expressions with e.g. 'u32'
1091 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
1092 getMergeSuffix();
1093 }
1094
1095 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1096 ACLEKind Kind) const {
1097 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1098
1099 std::string FullName = mangleName(ClassS);
1100 std::string ProtoName = mangleName(getClassKind());
1101 OS << (IsOverloaded ? "__aio " : "__ai ")
1102 << "__attribute__((__clang_arm_builtin_alias(";
1103
1104 switch (Kind) {
1105 case ACLEKind::SME:
1106 OS << "__builtin_sme_" << FullName << ")";
1107 break;
1108 case ACLEKind::SVE:
1109 OS << "__builtin_sve_" << FullName << ")";
1110 break;
1111 }
1112
1113 OS << "))\n";
1114
1115 OS << getTypes()[0].str() << " " << ProtoName << "(";
1116 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1117 if (I != 0)
1118 OS << ", ";
1119 OS << getTypes()[I + 1].str();
1120 }
1121 OS << ");\n";
1122 }
1123
1124 //===----------------------------------------------------------------------===//
1125 // SVEEmitter implementation
1126 //===----------------------------------------------------------------------===//
1127 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1128 if (T.isFloat()) {
1129 switch (T.getElementSizeInBits()) {
1130 case 16:
1131 return encodeEltType("EltTyFloat16");
1132 case 32:
1133 return encodeEltType("EltTyFloat32");
1134 case 64:
1135 return encodeEltType("EltTyFloat64");
1136 default:
1137 llvm_unreachable("Unhandled float element bitwidth!");
1138 }
1139 }
1140
1141 if (T.isBFloat()) {
1142 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1143 return encodeEltType("EltTyBFloat16");
1144 }
1145
1146 if (T.isMFloat()) {
1147 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1148 return encodeEltType("EltTyMFloat8");
1149 }
1150
1151 if (T.isPredicate() || T.isSvcount()) {
1152 switch (T.getElementSizeInBits()) {
1153 case 8:
1154 return encodeEltType("EltTyBool8");
1155 case 16:
1156 return encodeEltType("EltTyBool16");
1157 case 32:
1158 return encodeEltType("EltTyBool32");
1159 case 64:
1160 return encodeEltType("EltTyBool64");
1161 default:
1162 llvm_unreachable("Unhandled predicate element bitwidth!");
1163 }
1164 }
1165
1166 switch (T.getElementSizeInBits()) {
1167 case 8:
1168 return encodeEltType("EltTyInt8");
1169 case 16:
1170 return encodeEltType("EltTyInt16");
1171 case 32:
1172 return encodeEltType("EltTyInt32");
1173 case 64:
1174 return encodeEltType("EltTyInt64");
1175 case 128:
1176 return encodeEltType("EltTyInt128");
1177 default:
1178 llvm_unreachable("Unhandled integer element bitwidth!");
1179 }
1180 }
1181
1182 void SVEEmitter::createIntrinsic(
1183 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1184 StringRef Name = R->getValueAsString("Name");
1185 StringRef Proto = R->getValueAsString("Prototype");
1186 StringRef Types = R->getValueAsString("Types");
1187 StringRef SVEGuard = R->getValueAsString("SVETargetGuard");
1188 StringRef SMEGuard = R->getValueAsString("SMETargetGuard");
1189 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1190 uint64_t Merge = R->getValueAsInt("Merge");
1191 StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1192 uint64_t MemEltType = R->getValueAsInt("MemEltType");
1193
1194 int64_t Flags = 0;
1195 for (const Record *FlagRec : R->getValueAsListOfDefs("Flags"))
1196 Flags |= FlagRec->getValueAsInt("Value");
1197
1198 // Create a dummy TypeSpec for non-overloaded builtins.
1199 if (Types.empty()) {
1200 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1201 "Expect TypeSpec for overloaded builtin!");
1202 Types = "i";
1203 }
1204
1205 // Extract type specs from string
1206 SmallVector<TypeSpec, 8> TypeSpecs;
1207 TypeSpec Acc;
1208 for (char I : Types) {
1209 Acc.push_back(I);
1210 if (islower(I)) {
1211 TypeSpecs.push_back(TypeSpec(Acc));
1212 Acc.clear();
1213 }
1214 }
1215
1216 // Remove duplicate type specs.
1217 sort(TypeSpecs);
1218 TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1219 TypeSpecs.end());
1220
1221 // Create an Intrinsic for each type spec.
1222 for (auto TS : TypeSpecs) {
1223 // Collate a list of range/option checks for the immediates.
1224 SmallVector<ImmCheck, 2> ImmChecks;
1225 for (const Record *ImmR : R->getValueAsListOfDefs("ImmChecks")) {
1226 int64_t ArgIdx = ImmR->getValueAsInt("ImmArgIdx");
1227 int64_t EltSizeArgIdx = ImmR->getValueAsInt("TypeContextArgIdx");
1228 int64_t Kind = ImmR->getValueAsDef("Kind")->getValueAsInt("Value");
1229 assert(ArgIdx >= 0 && Kind >= 0 &&
1230 "ImmArgIdx and Kind must be nonnegative");
1231
1232 unsigned ElementSizeInBits = 0;
1233 auto [Mod, NumVectors] = getProtoModifier(Proto, EltSizeArgIdx + 1);
1234 if (EltSizeArgIdx >= 0)
1235 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1236 ImmChecks.push_back(ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1237 }
1238
1239 Out.push_back(std::make_unique<Intrinsic>(
1240 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1241 TS, ClassS, *this, SVEGuard, SMEGuard));
1242
1243 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1244 if (Intrinsic::isOverloadedIntrinsic(Name))
1245 Out.push_back(std::make_unique<Intrinsic>(
1246 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1247 ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));
1248 }
1249 }
1250
1251 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1252 SVEEmitter &Emitter,
1253 ACLEKind Kind) {
1254 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1255 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1256 for (auto *R : RV)
1257 createIntrinsic(R, Defs);
1258
1259 // Sort intrinsics in header file by following order/priority:
1260 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1261 // - Class (is intrinsic overloaded or not)
1262 // - Intrinsic name
1263 std::stable_sort(Defs.begin(), Defs.end(),
1264 [](const std::unique_ptr<Intrinsic> &A,
1265 const std::unique_ptr<Intrinsic> &B) {
1266 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1267 return std::make_tuple(
1268 I->getSVEGuard().str() + I->getSMEGuard().str(),
1269 (unsigned)I->getClassKind(), I->getName());
1270 };
1271 return ToTuple(A) < ToTuple(B);
1272 });
1273
1274 // Actually emit the intrinsic declarations.
1275 for (auto &I : Defs)
1276 I->emitIntrinsic(OS, Emitter, Kind);
1277 }
1278
1279 void SVEEmitter::createHeader(raw_ostream &OS) {
1280 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1281 "-----------------------------------===\n"
1282 " *\n"
1283 " *\n"
1284 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1285 "Exceptions.\n"
1286 " * See https://llvm.org/LICENSE.txt for license information.\n"
1287 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1288 " *\n"
1289 " *===-----------------------------------------------------------------"
1290 "------===\n"
1291 " */\n\n";
1292
1293 OS << "#ifndef __ARM_SVE_H\n";
1294 OS << "#define __ARM_SVE_H\n\n";
1295
1296 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1297 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1298 OS << "#endif\n";
1299
1300 OS << "#include <stdint.h>\n\n";
1301 OS << "#ifdef __cplusplus\n";
1302 OS << "extern \"C\" {\n";
1303 OS << "#else\n";
1304 OS << "#include <stdbool.h>\n";
1305 OS << "#endif\n\n";
1306
1307 OS << "typedef __fp16 float16_t;\n";
1308 OS << "typedef float float32_t;\n";
1309 OS << "typedef double float64_t;\n";
1310
1311 OS << "typedef __SVInt8_t svint8_t;\n";
1312 OS << "typedef __SVInt16_t svint16_t;\n";
1313 OS << "typedef __SVInt32_t svint32_t;\n";
1314 OS << "typedef __SVInt64_t svint64_t;\n";
1315 OS << "typedef __SVUint8_t svuint8_t;\n";
1316 OS << "typedef __SVUint16_t svuint16_t;\n";
1317 OS << "typedef __SVUint32_t svuint32_t;\n";
1318 OS << "typedef __SVUint64_t svuint64_t;\n";
1319 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1320
1321 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1322
1323 OS << "#include <arm_bf16.h>\n";
1324 OS << "#include <arm_vector_types.h>\n";
1325
1326 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1327
1328 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1329 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1330 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1331 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1332 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1333 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1334 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1335 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1336 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1337 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1338 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1339 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1340 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1341 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1342 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1343 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1344 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1345 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1346 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1347 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1348 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1349 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1350 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1351 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1352 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1353 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1354 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1355 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1356 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1357 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1358 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1359 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1360 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1361 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1362 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1363 OS << "typedef __SVBool_t svbool_t;\n";
1364 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1365 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1366
1367 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1368 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1369 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1370
1371 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1372 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1373 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1374
1375 OS << "typedef __SVCount_t svcount_t;\n\n";
1376
1377 OS << "enum svpattern\n";
1378 OS << "{\n";
1379 OS << " SV_POW2 = 0,\n";
1380 OS << " SV_VL1 = 1,\n";
1381 OS << " SV_VL2 = 2,\n";
1382 OS << " SV_VL3 = 3,\n";
1383 OS << " SV_VL4 = 4,\n";
1384 OS << " SV_VL5 = 5,\n";
1385 OS << " SV_VL6 = 6,\n";
1386 OS << " SV_VL7 = 7,\n";
1387 OS << " SV_VL8 = 8,\n";
1388 OS << " SV_VL16 = 9,\n";
1389 OS << " SV_VL32 = 10,\n";
1390 OS << " SV_VL64 = 11,\n";
1391 OS << " SV_VL128 = 12,\n";
1392 OS << " SV_VL256 = 13,\n";
1393 OS << " SV_MUL4 = 29,\n";
1394 OS << " SV_MUL3 = 30,\n";
1395 OS << " SV_ALL = 31\n";
1396 OS << "};\n\n";
1397
1398 OS << "enum svprfop\n";
1399 OS << "{\n";
1400 OS << " SV_PLDL1KEEP = 0,\n";
1401 OS << " SV_PLDL1STRM = 1,\n";
1402 OS << " SV_PLDL2KEEP = 2,\n";
1403 OS << " SV_PLDL2STRM = 3,\n";
1404 OS << " SV_PLDL3KEEP = 4,\n";
1405 OS << " SV_PLDL3STRM = 5,\n";
1406 OS << " SV_PSTL1KEEP = 8,\n";
1407 OS << " SV_PSTL1STRM = 9,\n";
1408 OS << " SV_PSTL2KEEP = 10,\n";
1409 OS << " SV_PSTL2STRM = 11,\n";
1410 OS << " SV_PSTL3KEEP = 12,\n";
1411 OS << " SV_PSTL3STRM = 13\n";
1412 OS << "};\n\n";
1413
1414 OS << "/* Function attributes */\n";
1415 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1416 "__nodebug__))\n\n";
1417 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1418 "__nodebug__, __overloadable__))\n\n";
1419
1420 // Add reinterpret functions.
1421 for (auto [N, Suffix] :
1422 std::initializer_list<std::pair<unsigned, const char *>>{
1423 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1424 for (auto ShortForm : {false, true})
1425 for (const ReinterpretTypeInfo &To : Reinterprets) {
1426 SVEType ToV(To.BaseType, N);
1427 for (const ReinterpretTypeInfo &From : Reinterprets) {
1428 SVEType FromV(From.BaseType, N);
1429 OS << "__aio "
1430 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1431 "reinterpret_"
1432 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1433 << ToV.str() << " svreinterpret_" << To.Suffix;
1434 if (!ShortForm)
1435 OS << "_" << From.Suffix << Suffix;
1436 OS << "(" << FromV.str() << " op);\n";
1437 }
1438 }
1439 }
1440
1441 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
1442
1443 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1444 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1445
1446 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1447 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1448 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1449 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1450
1451 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1452 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1453
1454 OS << "#ifdef __cplusplus\n";
1455 OS << "} // extern \"C\"\n";
1456 OS << "#endif\n\n";
1457 OS << "#undef __ai\n\n";
1458 OS << "#undef __aio\n\n";
1459 OS << "#endif /* __ARM_SVE_H */\n";
1460 }
1461
1462 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1463 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1464 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1465 for (auto *R : RV)
1466 createIntrinsic(R, Defs);
1467
1468 // The mappings must be sorted based on BuiltinID.
1469 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1470 const std::unique_ptr<Intrinsic> &B) {
1471 return A->getMangledName() < B->getMangledName();
1472 });
1473
1474 OS << "#ifdef GET_SVE_BUILTINS\n";
1475 for (auto &Def : Defs) {
1476 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1477 // declarations only live in the header file.
1478 if (Def->getClassKind() != ClassG) {
1479 OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1480 << Def->getBuiltinTypeStr() << "\", \"n\", \"";
1481 Def->printGuard(OS);
1482 OS << "\")\n";
1483 }
1484 }
1485
1486 // Add reinterpret functions.
1487 for (auto [N, Suffix] :
1488 std::initializer_list<std::pair<unsigned, const char *>>{
1489 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1490 for (const ReinterpretTypeInfo &To : Reinterprets) {
1491 SVEType ToV(To.BaseType, N);
1492 for (const ReinterpretTypeInfo &From : Reinterprets) {
1493 SVEType FromV(From.BaseType, N);
1494 OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
1495 << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
1496 << FromV.builtin_str() << "\", \"n\", \"sme|sve\")\n";
1497 }
1498 }
1499 }
1500
1501 OS << "#endif\n\n";
1502 }
1503
1504 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1505 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1506 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1507 for (auto *R : RV)
1508 createIntrinsic(R, Defs);
1509
1510 // The mappings must be sorted based on BuiltinID.
1511 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1512 const std::unique_ptr<Intrinsic> &B) {
1513 return A->getMangledName() < B->getMangledName();
1514 });
1515
1516 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1517 for (auto &Def : Defs) {
1518 // Builtins only exist for non-overloaded intrinsics, overloaded
1519 // declarations only live in the header file.
1520 if (Def->getClassKind() == ClassG)
1521 continue;
1522
1523 uint64_t Flags = Def->getFlags();
1524 auto FlagString = std::to_string(Flags);
1525
1526 std::string LLVMName = Def->getMangledLLVMName();
1527 std::string Builtin = Def->getMangledName();
1528 if (!LLVMName.empty())
1529 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1530 << "),\n";
1531 else
1532 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1533 }
1534 OS << "#endif\n\n";
1535 }
1536
1537 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1538 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1539 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1540 for (auto *R : RV)
1541 createIntrinsic(R, Defs);
1542
1543 // The mappings must be sorted based on BuiltinID.
1544 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1545 const std::unique_ptr<Intrinsic> &B) {
1546 return A->getMangledName() < B->getMangledName();
1547 });
1548
1549 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1550
1551 // Ensure these are only emitted once.
1552 std::set<std::string> Emitted;
1553
1554 for (auto &Def : Defs) {
1555 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1556 Def->getImmChecks().empty())
1557 continue;
1558
1559 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1560 for (auto &Check : Def->getImmChecks())
1561 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1562 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1563 OS << " break;\n";
1564
1565 Emitted.insert(Def->getMangledName());
1566 }
1567
1568 OS << "#endif\n\n";
1569 }
1570
1571 /// Create the SVETypeFlags used in CGBuiltins
1572 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1573 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1574 for (auto &KV : FlagTypes)
1575 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1576 OS << "#endif\n\n";
1577
1578 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1579 for (auto &KV : EltTypes)
1580 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1581 OS << "#endif\n\n";
1582
1583 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1584 for (auto &KV : MemEltTypes)
1585 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1586 OS << "#endif\n\n";
1587
1588 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1589 for (auto &KV : MergeTypes)
1590 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1591 OS << "#endif\n\n";
1592 }
1593
1594 void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1595 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1596 for (auto &KV : ImmCheckTypes)
1597 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1598 OS << "#endif\n\n";
1599 }
1600
1601 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1602 OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1603 "------===\n"
1604 " *\n"
1605 " *\n"
1606 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1607 "Exceptions.\n"
1608 " * See https://llvm.org/LICENSE.txt for license information.\n"
1609 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1610 " *\n"
1611 " *===-----------------------------------------------------------------"
1612 "------===\n"
1613 " */\n\n";
1614
1615 OS << "#ifndef __ARM_SME_H\n";
1616 OS << "#define __ARM_SME_H\n\n";
1617
1618 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1619 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1620 OS << "#endif\n";
1621
1622 OS << "#include <arm_sve.h>\n\n";
1623 OS << "#include <stddef.h>\n\n";
1624
1625 OS << "/* Function attributes */\n";
1626 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1627 "__nodebug__))\n\n";
1628 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1629 "__nodebug__, __overloadable__))\n\n";
1630
1631 OS << "#ifdef __cplusplus\n";
1632 OS << "extern \"C\" {\n";
1633 OS << "#endif\n\n";
1634
1635 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1636
1637 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1638 OS << " uint64_t x0, x1;\n";
1639 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1640 OS << " return x0 & (1ULL << 63);\n";
1641 OS << "}\n\n";
1642
1643 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1644 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1645 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1646 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1647
1648 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1649 "__arm_streaming_compatible __arm_out(\"za\") "
1650 "{ }\n\n";
1651
1652 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
1653
1654 OS << "#ifdef __cplusplus\n";
1655 OS << "} // extern \"C\"\n";
1656 OS << "#endif\n\n";
1657 OS << "#undef __ai\n\n";
1658 OS << "#endif /* __ARM_SME_H */\n";
1659 }
1660
1661 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1662 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1663 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1664 for (auto *R : RV) {
1665 createIntrinsic(R, Defs);
1666 }
1667
1668 // The mappings must be sorted based on BuiltinID.
1669 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1670 const std::unique_ptr<Intrinsic> &B) {
1671 return A->getMangledName() < B->getMangledName();
1672 });
1673
1674 OS << "#ifdef GET_SME_BUILTINS\n";
1675 for (auto &Def : Defs) {
1676 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1677 // declarations only live in the header file.
1678 if (Def->getClassKind() != ClassG) {
1679 OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
1680 << Def->getBuiltinTypeStr() << "\", \"n\", \"";
1681 Def->printGuard(OS);
1682 OS << "\")\n";
1683 }
1684 }
1685
1686 OS << "#endif\n\n";
1687 }
1688
1689 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1690 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1691 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1692 for (auto *R : RV) {
1693 createIntrinsic(R, Defs);
1694 }
1695
1696 // The mappings must be sorted based on BuiltinID.
1697 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1698 const std::unique_ptr<Intrinsic> &B) {
1699 return A->getMangledName() < B->getMangledName();
1700 });
1701
1702 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1703 for (auto &Def : Defs) {
1704 // Builtins only exist for non-overloaded intrinsics, overloaded
1705 // declarations only live in the header file.
1706 if (Def->getClassKind() == ClassG)
1707 continue;
1708
1709 uint64_t Flags = Def->getFlags();
1710 auto FlagString = std::to_string(Flags);
1711
1712 std::string LLVMName = Def->getLLVMName();
1713 std::string Builtin = Def->getMangledName();
1714 if (!LLVMName.empty())
1715 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1716 << "),\n";
1717 else
1718 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1719 }
1720 OS << "#endif\n\n";
1721 }
1722
1723 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1724 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1725 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1726 for (auto *R : RV) {
1727 createIntrinsic(R, Defs);
1728 }
1729
1730 // The mappings must be sorted based on BuiltinID.
1731 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1732 const std::unique_ptr<Intrinsic> &B) {
1733 return A->getMangledName() < B->getMangledName();
1734 });
1735
1736 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1737
1738 // Ensure these are only emitted once.
1739 std::set<std::string> Emitted;
1740
1741 for (auto &Def : Defs) {
1742 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1743 Def->getImmChecks().empty())
1744 continue;
1745
1746 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1747 for (auto &Check : Def->getImmChecks())
1748 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1749 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1750 << "));\n";
1751 OS << " break;\n";
1752
1753 Emitted.insert(Def->getMangledName());
1754 }
1755
1756 OS << "#endif\n\n";
1757 }
1758
1759 void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1760 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1761 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1762 for (auto *R : RV)
1763 createIntrinsic(R, Defs);
1764
1765 std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1766 for (auto &Def : Defs) {
1767 std::string Key;
1768 auto AddToKey = [&Key](const std::string &S) -> void {
1769 Key = Key.empty() ? S : (Key + " | " + S);
1770 };
1771
1772 if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
1773 AddToKey("ArmInZA");
1774 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
1775 AddToKey("ArmOutZA");
1776 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
1777 AddToKey("ArmInOutZA");
1778
1779 if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
1780 AddToKey("ArmInZT0");
1781 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
1782 AddToKey("ArmOutZT0");
1783 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
1784 AddToKey("ArmInOutZT0");
1785
1786 if (!Key.empty())
1787 IntrinsicsPerState[Key].insert(Def->getMangledName());
1788 }
1789
1790 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1791 for (auto &KV : IntrinsicsPerState) {
1792 for (StringRef Name : KV.second)
1793 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1794 OS << " return " << KV.first << ";\n";
1795 }
1796 OS << "#endif\n\n";
1797 }
1798
1799 void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1800 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1801 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1802 for (auto *R : RV)
1803 createIntrinsic(R, Defs);
1804
1805 StringRef ExtensionKind;
1806 switch (Kind) {
1807 case ACLEKind::SME:
1808 ExtensionKind = "SME";
1809 break;
1810 case ACLEKind::SVE:
1811 ExtensionKind = "SVE";
1812 break;
1813 }
1814
1815 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1816
1817 StringMap<std::set<std::string>> StreamingMap;
1818
1819 uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
1820 uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");
1821 uint64_t IsStreamingCompatibleFlag =
1822 getEnumValueForFlag("IsStreamingCompatible");
1823
1824 for (auto &Def : Defs) {
1825 if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
1826 !Def->getSMEGuard().empty())
1827 report_fatal_error("Missing VerifyRuntimeMode flag");
1828
1829 if (Def->isFlagSet(IsStreamingFlag))
1830 StreamingMap["ArmStreaming"].insert(Def->getMangledName());
1831 else if (Def->isFlagSet(VerifyRuntimeMode))
1832 StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());
1833 else if (Def->isFlagSet(IsStreamingCompatibleFlag))
1834 StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
1835 else
1836 StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
1837 }
1838
1839 for (auto BuiltinType : StreamingMap.keys()) {
1840 for (auto Name : StreamingMap[BuiltinType]) {
1841 OS << "case " << ExtensionKind << "::BI__builtin_"
1842 << ExtensionKind.lower() << "_";
1843 OS << Name << ":\n";
1844 }
1845 OS << " BuiltinType = " << BuiltinType << ";\n";
1846 OS << " break;\n";
1847 }
1848
1849 OS << "#endif\n\n";
1850 }
1851
1852 namespace clang {
1853 void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
1854 SVEEmitter(Records).createHeader(OS);
1855 }
1856
1857 void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1858 SVEEmitter(Records).createBuiltins(OS);
1859 }
1860
1861 void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1862 SVEEmitter(Records).createCodeGenMap(OS);
1863 }
1864
1865 void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1866 SVEEmitter(Records).createRangeChecks(OS);
1867 }
1868
1869 void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
1870 SVEEmitter(Records).createTypeFlags(OS);
1871 }
1872
1873 void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
1874 SVEEmitter(Records).createImmCheckTypes(OS);
1875 }
1876
1877 void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1878 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
1879 }
1880
1881 void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
1882 SVEEmitter(Records).createSMEHeader(OS);
1883 }
1884
1885 void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1886 SVEEmitter(Records).createSMEBuiltins(OS);
1887 }
1888
1889 void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1890 SVEEmitter(Records).createSMECodeGenMap(OS);
1891 }
1892
1893 void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1894 SVEEmitter(Records).createSMERangeChecks(OS);
1895 }
1896
1897 void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1898 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
1899 }
1900
1901 void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
1902 SVEEmitter(Records).createBuiltinZAState(OS);
1903 }
1904 } // End namespace clang
LLVM monorepo