?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[Flang] remove whole-archive option for AIX linker (#76039)
[llvm-project.git] / clang / utils / TableGen / SveEmitter.cpp
blob311c6b09dc790135cc774aaf15ac0be784cc5c63
1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors. Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringMap.h"
30 #include "llvm/TableGen/Error.h"
31 #include "llvm/TableGen/Record.h"
32 #include <array>
33 #include <cctype>
34 #include <set>
35 #include <sstream>
36 #include <string>
37 #include <tuple>
38
39 using namespace llvm;
40
41 enum ClassKind {
42 ClassNone,
43 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
44 ClassG, // Overloaded name without type suffix
45 };
46
47 enum class ACLEKind { SVE, SME };
48
49 using TypeSpec = std::string;
50
51 namespace {
52
53 class ImmCheck {
54 unsigned Arg;
55 unsigned Kind;
56 unsigned ElementSizeInBits;
57
58 public:
59 ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
60 : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
61 ImmCheck(const ImmCheck &Other) = default;
62 ~ImmCheck() = default;
63
64 unsigned getArg() const { return Arg; }
65 unsigned getKind() const { return Kind; }
66 unsigned getElementSizeInBits() const { return ElementSizeInBits; }
67 };
68
69 class SVEType {
70 bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
71 bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
72 Svcount;
73 unsigned Bitwidth, ElementBitwidth, NumVectors;
74
75 public:
76 SVEType() : SVEType("", 'v') {}
77
78 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
79 : Float(false), Signed(true), Immediate(false), Void(false),
80 Constant(false), Pointer(false), BFloat(false), DefaultType(false),
81 IsScalable(true), Predicate(false), PredicatePattern(false),
82 PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
83 NumVectors(NumVectors) {
84 if (!TS.empty())
85 applyTypespec(TS);
86 applyModifier(CharMod);
87 }
88
89 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
90 NumVectors = NumV;
91 }
92
93 bool isPointer() const { return Pointer; }
94 bool isVoidPointer() const { return Pointer && Void; }
95 bool isSigned() const { return Signed; }
96 bool isImmediate() const { return Immediate; }
97 bool isScalar() const { return NumVectors == 0; }
98 bool isVector() const { return NumVectors > 0; }
99 bool isScalableVector() const { return isVector() && IsScalable; }
100 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
101 bool isChar() const { return ElementBitwidth == 8; }
102 bool isVoid() const { return Void & !Pointer; }
103 bool isDefault() const { return DefaultType; }
104 bool isFloat() const { return Float && !BFloat; }
105 bool isBFloat() const { return BFloat && !Float; }
106 bool isFloatingPoint() const { return Float || BFloat; }
107 bool isInteger() const {
108 return !isFloatingPoint() && !Predicate && !Svcount;
109 }
110 bool isScalarPredicate() const {
111 return !isFloatingPoint() && Predicate && NumVectors == 0;
112 }
113 bool isPredicateVector() const { return Predicate; }
114 bool isPredicatePattern() const { return PredicatePattern; }
115 bool isPrefetchOp() const { return PrefetchOp; }
116 bool isSvcount() const { return Svcount; }
117 bool isConstant() const { return Constant; }
118 unsigned getElementSizeInBits() const { return ElementBitwidth; }
119 unsigned getNumVectors() const { return NumVectors; }
120
121 unsigned getNumElements() const {
122 assert(ElementBitwidth != ~0U);
123 return Bitwidth / ElementBitwidth;
124 }
125 unsigned getSizeInBits() const {
126 return Bitwidth;
127 }
128
129 /// Return the string representation of a type, which is an encoded
130 /// string for passing to the BUILTIN() macro in Builtins.def.
131 std::string builtin_str() const;
132
133 /// Return the C/C++ string representation of a type for use in the
134 /// arm_sve.h header file.
135 std::string str() const;
136
137 private:
138 /// Creates the type based on the typespec string in TS.
139 void applyTypespec(StringRef TS);
140
141 /// Applies a prototype modifier to the type.
142 void applyModifier(char Mod);
143 };
144
145 class SVEEmitter;
146
147 /// The main grunt class. This represents an instantiation of an intrinsic with
148 /// a particular typespec and prototype.
149 class Intrinsic {
150 /// The unmangled name.
151 std::string Name;
152
153 /// The name of the corresponding LLVM IR intrinsic.
154 std::string LLVMName;
155
156 /// Intrinsic prototype.
157 std::string Proto;
158
159 /// The base type spec for this intrinsic.
160 TypeSpec BaseTypeSpec;
161
162 /// The base class kind. Most intrinsics use ClassS, which has full type
163 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
164 /// intrinsics.
165 ClassKind Class;
166
167 /// The architectural #ifdef guard.
168 std::string Guard;
169
170 // The merge suffix such as _m, _x or _z.
171 std::string MergeSuffix;
172
173 /// The types of return value [0] and parameters [1..].
174 std::vector<SVEType> Types;
175
176 /// The "base type", which is VarType('d', BaseTypeSpec).
177 SVEType BaseType;
178
179 uint64_t Flags;
180
181 SmallVector<ImmCheck, 2> ImmChecks;
182
183 public:
184 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
185 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
186 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
187 ClassKind Class, SVEEmitter &Emitter, StringRef Guard);
188
189 ~Intrinsic()=default;
190
191 std::string getName() const { return Name; }
192 std::string getLLVMName() const { return LLVMName; }
193 std::string getProto() const { return Proto; }
194 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
195 SVEType getBaseType() const { return BaseType; }
196
197 StringRef getGuard() const { return Guard; }
198 ClassKind getClassKind() const { return Class; }
199
200 SVEType getReturnType() const { return Types[0]; }
201 ArrayRef<SVEType> getTypes() const { return Types; }
202 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
203 unsigned getNumParams() const {
204 return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
205 }
206
207 uint64_t getFlags() const { return Flags; }
208 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
209
210 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
211
212 /// Return the type string for a BUILTIN() macro in Builtins.def.
213 std::string getBuiltinTypeStr();
214
215 /// Return the name, mangled with type information. The name is mangled for
216 /// ClassS, so will add type suffixes such as _u32/_s32.
217 std::string getMangledName() const { return mangleName(ClassS); }
218
219 /// As above, but mangles the LLVM name instead.
220 std::string getMangledLLVMName() const { return mangleLLVMName(); }
221
222 /// Returns true if the intrinsic is overloaded, in that it should also generate
223 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
224 /// 'svld1_u32(..)'.
225 static bool isOverloadedIntrinsic(StringRef Name) {
226 auto BrOpen = Name.find('[');
227 auto BrClose = Name.find(']');
228 return BrOpen != std::string::npos && BrClose != std::string::npos;
229 }
230
231 /// Return true if the intrinsic takes a splat operand.
232 bool hasSplat() const {
233 // These prototype modifiers are described in arm_sve.td.
234 return Proto.find_first_of("ajfrKLR@") != std::string::npos;
235 }
236
237 /// Return the parameter index of the splat operand.
238 unsigned getSplatIdx() const {
239 unsigned I = 1, Param = 0;
240 for (; I < Proto.size(); ++I, ++Param) {
241 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
242 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
243 Proto[I] == 'R' || Proto[I] == '@')
244 break;
245
246 // Multivector modifier can be skipped
247 if (Proto[I] == '.')
248 I += 2;
249 }
250 assert(I != Proto.size() && "Prototype has no splat operand");
251 return Param;
252 }
253
254 /// Emits the intrinsic declaration to the ostream.
255 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
256
257 private:
258 std::string getMergeSuffix() const { return MergeSuffix; }
259 std::string mangleName(ClassKind LocalCK) const;
260 std::string mangleLLVMName() const;
261 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
262 std::string Proto) const;
263 };
264
265 class SVEEmitter {
266 private:
267 // The reinterpret builtins are generated separately because they
268 // need the cross product of all types (121 functions in total),
269 // which is inconvenient to specify in the arm_sve.td file or
270 // generate in CGBuiltin.cpp.
271 struct ReinterpretTypeInfo {
272 SVEType BaseType;
273 const char *Suffix;
274 };
275
276 static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
277
278 RecordKeeper &Records;
279 llvm::StringMap<uint64_t> EltTypes;
280 llvm::StringMap<uint64_t> MemEltTypes;
281 llvm::StringMap<uint64_t> FlagTypes;
282 llvm::StringMap<uint64_t> MergeTypes;
283 llvm::StringMap<uint64_t> ImmCheckTypes;
284
285 public:
286 SVEEmitter(RecordKeeper &R) : Records(R) {
287 for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
288 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
289 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
290 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
291 for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
292 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
293 for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
294 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
295 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
296 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
297 }
298
299 /// Returns the enum value for the immcheck type
300 unsigned getEnumValueForImmCheck(StringRef C) const {
301 auto It = ImmCheckTypes.find(C);
302 if (It != ImmCheckTypes.end())
303 return It->getValue();
304 llvm_unreachable("Unsupported imm check");
305 }
306
307 /// Returns the enum value for the flag type
308 uint64_t getEnumValueForFlag(StringRef C) const {
309 auto Res = FlagTypes.find(C);
310 if (Res != FlagTypes.end())
311 return Res->getValue();
312 llvm_unreachable("Unsupported flag");
313 }
314
315 // Returns the SVETypeFlags for a given value and mask.
316 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
317 auto It = FlagTypes.find(MaskName);
318 if (It != FlagTypes.end()) {
319 uint64_t Mask = It->getValue();
320 unsigned Shift = llvm::countr_zero(Mask);
321 assert(Shift < 64 && "Mask value produced an invalid shift value");
322 return (V << Shift) & Mask;
323 }
324 llvm_unreachable("Unsupported flag");
325 }
326
327 // Returns the SVETypeFlags for the given element type.
328 uint64_t encodeEltType(StringRef EltName) {
329 auto It = EltTypes.find(EltName);
330 if (It != EltTypes.end())
331 return encodeFlag(It->getValue(), "EltTypeMask");
332 llvm_unreachable("Unsupported EltType");
333 }
334
335 // Returns the SVETypeFlags for the given memory element type.
336 uint64_t encodeMemoryElementType(uint64_t MT) {
337 return encodeFlag(MT, "MemEltTypeMask");
338 }
339
340 // Returns the SVETypeFlags for the given merge type.
341 uint64_t encodeMergeType(uint64_t MT) {
342 return encodeFlag(MT, "MergeTypeMask");
343 }
344
345 // Returns the SVETypeFlags for the given splat operand.
346 unsigned encodeSplatOperand(unsigned SplatIdx) {
347 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
348 return encodeFlag(SplatIdx + 1, "SplatOperandMask");
349 }
350
351 // Returns the SVETypeFlags value for the given SVEType.
352 uint64_t encodeTypeFlags(const SVEType &T);
353
354 /// Emit arm_sve.h.
355 void createHeader(raw_ostream &o);
356
357 // Emits core intrinsics in both arm_sme.h and arm_sve.h
358 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
359 ACLEKind Kind);
360
361 /// Emit all the __builtin prototypes and code needed by Sema.
362 void createBuiltins(raw_ostream &o);
363
364 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
365 void createCodeGenMap(raw_ostream &o);
366
367 /// Emit all the range checks for the immediates.
368 void createRangeChecks(raw_ostream &o);
369
370 /// Create the SVETypeFlags used in CGBuiltins
371 void createTypeFlags(raw_ostream &o);
372
373 /// Emit arm_sme.h.
374 void createSMEHeader(raw_ostream &o);
375
376 /// Emit all the SME __builtin prototypes and code needed by Sema.
377 void createSMEBuiltins(raw_ostream &o);
378
379 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
380 void createSMECodeGenMap(raw_ostream &o);
381
382 /// Create a table for a builtin's requirement for PSTATE.SM.
383 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
384
385 /// Emit all the range checks for the immediates.
386 void createSMERangeChecks(raw_ostream &o);
387
388 /// Create a table for a builtin's requirement for PSTATE.ZA.
389 void createBuiltinZAState(raw_ostream &OS);
390
391 /// Create intrinsic and add it to \p Out
392 void createIntrinsic(Record *R,
393 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
394 };
395
396 const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
397 {{{SVEType("c", 'd'), "s8"},
398 {SVEType("Uc", 'd'), "u8"},
399 {SVEType("s", 'd'), "s16"},
400 {SVEType("Us", 'd'), "u16"},
401 {SVEType("i", 'd'), "s32"},
402 {SVEType("Ui", 'd'), "u32"},
403 {SVEType("l", 'd'), "s64"},
404 {SVEType("Ul", 'd'), "u64"},
405 {SVEType("h", 'd'), "f16"},
406 {SVEType("b", 'd'), "bf16"},
407 {SVEType("f", 'd'), "f32"},
408 {SVEType("d", 'd'), "f64"}}};
409
410 } // end anonymous namespace
411
412
413 //===----------------------------------------------------------------------===//
414 // Type implementation
415 //===----------------------------------------------------------------------===//
416
417 std::string SVEType::builtin_str() const {
418 std::string S;
419 if (isVoid())
420 return "v";
421
422 if (isScalarPredicate())
423 return "b";
424
425 if (isSvcount())
426 return "Qa";
427
428 if (isVoidPointer())
429 S += "v";
430 else if (!isFloatingPoint())
431 switch (ElementBitwidth) {
432 case 1: S += "b"; break;
433 case 8: S += "c"; break;
434 case 16: S += "s"; break;
435 case 32: S += "i"; break;
436 case 64: S += "Wi"; break;
437 case 128: S += "LLLi"; break;
438 default: llvm_unreachable("Unhandled case!");
439 }
440 else if (isFloat())
441 switch (ElementBitwidth) {
442 case 16: S += "h"; break;
443 case 32: S += "f"; break;
444 case 64: S += "d"; break;
445 default: llvm_unreachable("Unhandled case!");
446 }
447 else if (isBFloat()) {
448 assert(ElementBitwidth == 16 && "Not a valid BFloat.");
449 S += "y";
450 }
451
452 if (!isFloatingPoint()) {
453 if ((isChar() || isPointer()) && !isVoidPointer()) {
454 // Make chars and typed pointers explicitly signed.
455 if (Signed)
456 S = "S" + S;
457 else if (!Signed)
458 S = "U" + S;
459 } else if (!isVoidPointer() && !Signed) {
460 S = "U" + S;
461 }
462 }
463
464 // Constant indices are "int", but have the "constant expression" modifier.
465 if (isImmediate()) {
466 assert(!isFloat() && "fp immediates are not supported");
467 S = "I" + S;
468 }
469
470 if (isScalar()) {
471 if (Constant) S += "C";
472 if (Pointer) S += "*";
473 return S;
474 }
475
476 if (isFixedLengthVector())
477 return "V" + utostr(getNumElements() * NumVectors) + S;
478 return "q" + utostr(getNumElements() * NumVectors) + S;
479 }
480
481 std::string SVEType::str() const {
482 if (isPredicatePattern())
483 return "enum svpattern";
484
485 if (isPrefetchOp())
486 return "enum svprfop";
487
488 std::string S;
489 if (Void)
490 S += "void";
491 else {
492 if (isScalableVector() || isSvcount())
493 S += "sv";
494 if (!Signed && !isFloatingPoint())
495 S += "u";
496
497 if (Float)
498 S += "float";
499 else if (isSvcount())
500 S += "count";
501 else if (isScalarPredicate() || isPredicateVector())
502 S += "bool";
503 else if (isBFloat())
504 S += "bfloat";
505 else
506 S += "int";
507
508 if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
509 S += utostr(ElementBitwidth);
510 if (isFixedLengthVector())
511 S += "x" + utostr(getNumElements());
512 if (NumVectors > 1)
513 S += "x" + utostr(NumVectors);
514 if (!isScalarPredicate())
515 S += "_t";
516 }
517
518 if (Constant)
519 S += " const";
520 if (Pointer)
521 S += " *";
522
523 return S;
524 }
525
526 void SVEType::applyTypespec(StringRef TS) {
527 for (char I : TS) {
528 switch (I) {
529 case 'Q':
530 Svcount = true;
531 break;
532 case 'P':
533 Predicate = true;
534 break;
535 case 'U':
536 Signed = false;
537 break;
538 case 'c':
539 ElementBitwidth = 8;
540 break;
541 case 's':
542 ElementBitwidth = 16;
543 break;
544 case 'i':
545 ElementBitwidth = 32;
546 break;
547 case 'l':
548 ElementBitwidth = 64;
549 break;
550 case 'q':
551 ElementBitwidth = 128;
552 break;
553 case 'h':
554 Float = true;
555 ElementBitwidth = 16;
556 break;
557 case 'f':
558 Float = true;
559 ElementBitwidth = 32;
560 break;
561 case 'd':
562 Float = true;
563 ElementBitwidth = 64;
564 break;
565 case 'b':
566 BFloat = true;
567 Float = false;
568 ElementBitwidth = 16;
569 break;
570 default:
571 llvm_unreachable("Unhandled type code!");
572 }
573 }
574 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
575 }
576
577 void SVEType::applyModifier(char Mod) {
578 switch (Mod) {
579 case 'v':
580 Void = true;
581 break;
582 case 'd':
583 DefaultType = true;
584 break;
585 case 'c':
586 Constant = true;
587 [[fallthrough]];
588 case 'p':
589 Pointer = true;
590 Bitwidth = ElementBitwidth;
591 NumVectors = 0;
592 break;
593 case 'e':
594 Signed = false;
595 ElementBitwidth /= 2;
596 break;
597 case 'h':
598 ElementBitwidth /= 2;
599 break;
600 case 'q':
601 ElementBitwidth /= 4;
602 break;
603 case 'b':
604 Signed = false;
605 Float = false;
606 BFloat = false;
607 ElementBitwidth /= 4;
608 break;
609 case 'o':
610 ElementBitwidth *= 4;
611 break;
612 case 'P':
613 Signed = true;
614 Float = false;
615 BFloat = false;
616 Predicate = true;
617 Svcount = false;
618 Bitwidth = 16;
619 ElementBitwidth = 1;
620 break;
621 case '{':
622 IsScalable = false;
623 Bitwidth = 128;
624 NumVectors = 1;
625 break;
626 case 's':
627 case 'a':
628 Bitwidth = ElementBitwidth;
629 NumVectors = 0;
630 break;
631 case 'R':
632 ElementBitwidth /= 2;
633 NumVectors = 0;
634 break;
635 case 'r':
636 ElementBitwidth /= 4;
637 NumVectors = 0;
638 break;
639 case '@':
640 Signed = false;
641 Float = false;
642 BFloat = false;
643 ElementBitwidth /= 4;
644 NumVectors = 0;
645 break;
646 case 'K':
647 Signed = true;
648 Float = false;
649 BFloat = false;
650 Bitwidth = ElementBitwidth;
651 NumVectors = 0;
652 break;
653 case 'L':
654 Signed = false;
655 Float = false;
656 BFloat = false;
657 Bitwidth = ElementBitwidth;
658 NumVectors = 0;
659 break;
660 case 'u':
661 Predicate = false;
662 Svcount = false;
663 Signed = false;
664 Float = false;
665 BFloat = false;
666 break;
667 case 'x':
668 Predicate = false;
669 Svcount = false;
670 Signed = true;
671 Float = false;
672 BFloat = false;
673 break;
674 case 'i':
675 Predicate = false;
676 Svcount = false;
677 Float = false;
678 BFloat = false;
679 ElementBitwidth = Bitwidth = 64;
680 NumVectors = 0;
681 Signed = false;
682 Immediate = true;
683 break;
684 case 'I':
685 Predicate = false;
686 Svcount = false;
687 Float = false;
688 BFloat = false;
689 ElementBitwidth = Bitwidth = 32;
690 NumVectors = 0;
691 Signed = true;
692 Immediate = true;
693 PredicatePattern = true;
694 break;
695 case 'J':
696 Predicate = false;
697 Svcount = false;
698 Float = false;
699 BFloat = false;
700 ElementBitwidth = Bitwidth = 32;
701 NumVectors = 0;
702 Signed = true;
703 Immediate = true;
704 PrefetchOp = true;
705 break;
706 case 'k':
707 Predicate = false;
708 Svcount = false;
709 Signed = true;
710 Float = false;
711 BFloat = false;
712 ElementBitwidth = Bitwidth = 32;
713 NumVectors = 0;
714 break;
715 case 'l':
716 Predicate = false;
717 Svcount = false;
718 Signed = true;
719 Float = false;
720 BFloat = false;
721 ElementBitwidth = Bitwidth = 64;
722 NumVectors = 0;
723 break;
724 case 'm':
725 Predicate = false;
726 Svcount = false;
727 Signed = false;
728 Float = false;
729 BFloat = false;
730 ElementBitwidth = Bitwidth = 32;
731 NumVectors = 0;
732 break;
733 case 'n':
734 Predicate = false;
735 Svcount = false;
736 Signed = false;
737 Float = false;
738 BFloat = false;
739 ElementBitwidth = Bitwidth = 64;
740 NumVectors = 0;
741 break;
742 case 'w':
743 ElementBitwidth = 64;
744 break;
745 case 'j':
746 ElementBitwidth = Bitwidth = 64;
747 NumVectors = 0;
748 break;
749 case 'f':
750 Signed = false;
751 ElementBitwidth = Bitwidth = 64;
752 NumVectors = 0;
753 break;
754 case 'g':
755 Signed = false;
756 Float = false;
757 BFloat = false;
758 ElementBitwidth = 64;
759 break;
760 case '[':
761 Signed = false;
762 Float = false;
763 BFloat = false;
764 ElementBitwidth = 8;
765 break;
766 case 't':
767 Signed = true;
768 Float = false;
769 BFloat = false;
770 ElementBitwidth = 32;
771 break;
772 case 'z':
773 Signed = false;
774 Float = false;
775 BFloat = false;
776 ElementBitwidth = 32;
777 break;
778 case 'O':
779 Predicate = false;
780 Svcount = false;
781 Float = true;
782 ElementBitwidth = 16;
783 break;
784 case 'M':
785 Predicate = false;
786 Svcount = false;
787 Float = true;
788 BFloat = false;
789 ElementBitwidth = 32;
790 break;
791 case 'N':
792 Predicate = false;
793 Svcount = false;
794 Float = true;
795 ElementBitwidth = 64;
796 break;
797 case 'Q':
798 Constant = true;
799 Pointer = true;
800 Void = true;
801 NumVectors = 0;
802 break;
803 case 'S':
804 Constant = true;
805 Pointer = true;
806 ElementBitwidth = Bitwidth = 8;
807 NumVectors = 0;
808 Signed = true;
809 break;
810 case 'W':
811 Constant = true;
812 Pointer = true;
813 ElementBitwidth = Bitwidth = 8;
814 NumVectors = 0;
815 Signed = false;
816 break;
817 case 'T':
818 Constant = true;
819 Pointer = true;
820 ElementBitwidth = Bitwidth = 16;
821 NumVectors = 0;
822 Signed = true;
823 break;
824 case 'X':
825 Constant = true;
826 Pointer = true;
827 ElementBitwidth = Bitwidth = 16;
828 NumVectors = 0;
829 Signed = false;
830 break;
831 case 'Y':
832 Constant = true;
833 Pointer = true;
834 ElementBitwidth = Bitwidth = 32;
835 NumVectors = 0;
836 Signed = false;
837 break;
838 case 'U':
839 Constant = true;
840 Pointer = true;
841 ElementBitwidth = Bitwidth = 32;
842 NumVectors = 0;
843 Signed = true;
844 break;
845 case '%':
846 Pointer = true;
847 Void = true;
848 NumVectors = 0;
849 break;
850 case 'A':
851 Pointer = true;
852 ElementBitwidth = Bitwidth = 8;
853 NumVectors = 0;
854 Signed = true;
855 break;
856 case 'B':
857 Pointer = true;
858 ElementBitwidth = Bitwidth = 16;
859 NumVectors = 0;
860 Signed = true;
861 break;
862 case 'C':
863 Pointer = true;
864 ElementBitwidth = Bitwidth = 32;
865 NumVectors = 0;
866 Signed = true;
867 break;
868 case 'D':
869 Pointer = true;
870 ElementBitwidth = Bitwidth = 64;
871 NumVectors = 0;
872 Signed = true;
873 break;
874 case 'E':
875 Pointer = true;
876 ElementBitwidth = Bitwidth = 8;
877 NumVectors = 0;
878 Signed = false;
879 break;
880 case 'F':
881 Pointer = true;
882 ElementBitwidth = Bitwidth = 16;
883 NumVectors = 0;
884 Signed = false;
885 break;
886 case 'G':
887 Pointer = true;
888 ElementBitwidth = Bitwidth = 32;
889 NumVectors = 0;
890 Signed = false;
891 break;
892 case '$':
893 Predicate = false;
894 Svcount = false;
895 Float = false;
896 BFloat = true;
897 ElementBitwidth = 16;
898 break;
899 case '}':
900 Predicate = false;
901 Signed = true;
902 Svcount = true;
903 NumVectors = 0;
904 Float = false;
905 BFloat = false;
906 break;
907 case '.':
908 llvm_unreachable(". is never a type in itself");
909 break;
910 default:
911 llvm_unreachable("Unhandled character!");
912 }
913 }
914
915 /// Returns the modifier and number of vectors for the given operand \p Op.
916 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
917 for (unsigned P = 0; !Proto.empty(); ++P) {
918 unsigned NumVectors = 1;
919 unsigned CharsToSkip = 1;
920 char Mod = Proto[0];
921 if (Mod == '2' || Mod == '3' || Mod == '4') {
922 NumVectors = Mod - '0';
923 Mod = 'd';
924 if (Proto.size() > 1 && Proto[1] == '.') {
925 Mod = Proto[2];
926 CharsToSkip = 3;
927 }
928 }
929
930 if (P == Op)
931 return {Mod, NumVectors};
932
933 Proto = Proto.drop_front(CharsToSkip);
934 }
935 llvm_unreachable("Unexpected Op");
936 }
937
938 //===----------------------------------------------------------------------===//
939 // Intrinsic implementation
940 //===----------------------------------------------------------------------===//
941
942 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
943 StringRef MergeSuffix, uint64_t MemoryElementTy,
944 StringRef LLVMName, uint64_t Flags,
945 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
946 SVEEmitter &Emitter, StringRef Guard)
947 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
948 BaseTypeSpec(BT), Class(Class), Guard(Guard.str()),
949 MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
950 ImmChecks(Checks.begin(), Checks.end()) {
951 // Types[0] is the return value.
952 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
953 char Mod;
954 unsigned NumVectors;
955 std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
956 SVEType T(BaseTypeSpec, Mod, NumVectors);
957 Types.push_back(T);
958
959 // Add range checks for immediates
960 if (I > 0) {
961 if (T.isPredicatePattern())
962 ImmChecks.emplace_back(
963 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
964 else if (T.isPrefetchOp())
965 ImmChecks.emplace_back(
966 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
967 }
968 }
969
970 // Set flags based on properties
971 this->Flags |= Emitter.encodeTypeFlags(BaseType);
972 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
973 this->Flags |= Emitter.encodeMergeType(MergeTy);
974 if (hasSplat())
975 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
976 }
977
978 std::string Intrinsic::getBuiltinTypeStr() {
979 std::string S = getReturnType().builtin_str();
980 for (unsigned I = 0; I < getNumParams(); ++I)
981 S += getParamType(I).builtin_str();
982
983 return S;
984 }
985
986 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
987 std::string Proto) const {
988 std::string Ret = Name;
989 while (Ret.find('{') != std::string::npos) {
990 size_t Pos = Ret.find('{');
991 size_t End = Ret.find('}');
992 unsigned NumChars = End - Pos + 1;
993 assert(NumChars == 3 && "Unexpected template argument");
994
995 SVEType T;
996 char C = Ret[Pos+1];
997 switch(C) {
998 default:
999 llvm_unreachable("Unknown predication specifier");
1000 case 'd':
1001 T = SVEType(TS, 'd');
1002 break;
1003 case '0':
1004 case '1':
1005 case '2':
1006 case '3':
1007 T = SVEType(TS, Proto[C - '0']);
1008 break;
1009 }
1010
1011 // Replace templated arg with the right suffix (e.g. u32)
1012 std::string TypeCode;
1013 if (T.isInteger())
1014 TypeCode = T.isSigned() ? 's' : 'u';
1015 else if (T.isSvcount())
1016 TypeCode = 'c';
1017 else if (T.isPredicateVector())
1018 TypeCode = 'b';
1019 else if (T.isBFloat())
1020 TypeCode = "bf";
1021 else
1022 TypeCode = 'f';
1023 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
1024 }
1025
1026 return Ret;
1027 }
1028
1029 std::string Intrinsic::mangleLLVMName() const {
1030 std::string S = getLLVMName();
1031
1032 // Replace all {d} like expressions with e.g. 'u32'
1033 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
1034 }
1035
1036 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1037 std::string S = getName();
1038
1039 if (LocalCK == ClassG) {
1040 // Remove the square brackets and everything in between.
1041 while (S.find('[') != std::string::npos) {
1042 auto Start = S.find('[');
1043 auto End = S.find(']');
1044 S.erase(Start, (End-Start)+1);
1045 }
1046 } else {
1047 // Remove the square brackets.
1048 while (S.find('[') != std::string::npos) {
1049 auto BrPos = S.find('[');
1050 if (BrPos != std::string::npos)
1051 S.erase(BrPos, 1);
1052 BrPos = S.find(']');
1053 if (BrPos != std::string::npos)
1054 S.erase(BrPos, 1);
1055 }
1056 }
1057
1058 // Replace all {d} like expressions with e.g. 'u32'
1059 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
1060 getMergeSuffix();
1061 }
1062
1063 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1064 ACLEKind Kind) const {
1065 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1066
1067 std::string FullName = mangleName(ClassS);
1068 std::string ProtoName = mangleName(getClassKind());
1069 std::string SMEAttrs = "";
1070
1071 if (Flags & Emitter.getEnumValueForFlag("IsStreaming"))
1072 SMEAttrs += ", arm_streaming";
1073 if (Flags & Emitter.getEnumValueForFlag("IsStreamingCompatible"))
1074 SMEAttrs += ", arm_streaming_compatible";
1075 if (Flags & Emitter.getEnumValueForFlag("IsSharedZA"))
1076 SMEAttrs += ", arm_shared_za";
1077 if (Flags & Emitter.getEnumValueForFlag("IsPreservesZA"))
1078 SMEAttrs += ", arm_preserves_za";
1079
1080 OS << (IsOverloaded ? "__aio " : "__ai ")
1081 << "__attribute__((__clang_arm_builtin_alias(";
1082
1083 switch (Kind) {
1084 case ACLEKind::SME:
1085 OS << "__builtin_sme_" << FullName << ")";
1086 break;
1087 case ACLEKind::SVE:
1088 OS << "__builtin_sve_" << FullName << ")";
1089 break;
1090 }
1091
1092 if (!SMEAttrs.empty())
1093 OS << SMEAttrs;
1094 OS << "))\n";
1095
1096 OS << getTypes()[0].str() << " " << ProtoName << "(";
1097 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1098 if (I != 0)
1099 OS << ", ";
1100 OS << getTypes()[I + 1].str();
1101 }
1102 OS << ");\n";
1103 }
1104
1105 //===----------------------------------------------------------------------===//
1106 // SVEEmitter implementation
1107 //===----------------------------------------------------------------------===//
1108 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1109 if (T.isFloat()) {
1110 switch (T.getElementSizeInBits()) {
1111 case 16:
1112 return encodeEltType("EltTyFloat16");
1113 case 32:
1114 return encodeEltType("EltTyFloat32");
1115 case 64:
1116 return encodeEltType("EltTyFloat64");
1117 default:
1118 llvm_unreachable("Unhandled float element bitwidth!");
1119 }
1120 }
1121
1122 if (T.isBFloat()) {
1123 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1124 return encodeEltType("EltTyBFloat16");
1125 }
1126
1127 if (T.isPredicateVector() || T.isSvcount()) {
1128 switch (T.getElementSizeInBits()) {
1129 case 8:
1130 return encodeEltType("EltTyBool8");
1131 case 16:
1132 return encodeEltType("EltTyBool16");
1133 case 32:
1134 return encodeEltType("EltTyBool32");
1135 case 64:
1136 return encodeEltType("EltTyBool64");
1137 default:
1138 llvm_unreachable("Unhandled predicate element bitwidth!");
1139 }
1140 }
1141
1142 switch (T.getElementSizeInBits()) {
1143 case 8:
1144 return encodeEltType("EltTyInt8");
1145 case 16:
1146 return encodeEltType("EltTyInt16");
1147 case 32:
1148 return encodeEltType("EltTyInt32");
1149 case 64:
1150 return encodeEltType("EltTyInt64");
1151 case 128:
1152 return encodeEltType("EltTyInt128");
1153 default:
1154 llvm_unreachable("Unhandled integer element bitwidth!");
1155 }
1156 }
1157
1158 void SVEEmitter::createIntrinsic(
1159 Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1160 StringRef Name = R->getValueAsString("Name");
1161 StringRef Proto = R->getValueAsString("Prototype");
1162 StringRef Types = R->getValueAsString("Types");
1163 StringRef Guard = R->getValueAsString("TargetGuard");
1164 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1165 uint64_t Merge = R->getValueAsInt("Merge");
1166 StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1167 uint64_t MemEltType = R->getValueAsInt("MemEltType");
1168 std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
1169 std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
1170
1171 int64_t Flags = 0;
1172 for (auto FlagRec : FlagsList)
1173 Flags |= FlagRec->getValueAsInt("Value");
1174
1175 // Create a dummy TypeSpec for non-overloaded builtins.
1176 if (Types.empty()) {
1177 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1178 "Expect TypeSpec for overloaded builtin!");
1179 Types = "i";
1180 }
1181
1182 // Extract type specs from string
1183 SmallVector<TypeSpec, 8> TypeSpecs;
1184 TypeSpec Acc;
1185 for (char I : Types) {
1186 Acc.push_back(I);
1187 if (islower(I)) {
1188 TypeSpecs.push_back(TypeSpec(Acc));
1189 Acc.clear();
1190 }
1191 }
1192
1193 // Remove duplicate type specs.
1194 llvm::sort(TypeSpecs);
1195 TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1196 TypeSpecs.end());
1197
1198 // Create an Intrinsic for each type spec.
1199 for (auto TS : TypeSpecs) {
1200 // Collate a list of range/option checks for the immediates.
1201 SmallVector<ImmCheck, 2> ImmChecks;
1202 for (auto *R : ImmCheckList) {
1203 int64_t Arg = R->getValueAsInt("Arg");
1204 int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
1205 int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
1206 assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
1207
1208 unsigned ElementSizeInBits = 0;
1209 char Mod;
1210 unsigned NumVectors;
1211 std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
1212 if (EltSizeArg >= 0)
1213 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1214 ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
1215 }
1216
1217 Out.push_back(std::make_unique<Intrinsic>(
1218 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1219 TS, ClassS, *this, Guard));
1220
1221 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1222 if (Intrinsic::isOverloadedIntrinsic(Name))
1223 Out.push_back(std::make_unique<Intrinsic>(
1224 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1225 ImmChecks, TS, ClassG, *this, Guard));
1226 }
1227 }
1228
1229 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1230 SVEEmitter &Emitter,
1231 ACLEKind Kind) {
1232 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1233 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1234 for (auto *R : RV)
1235 createIntrinsic(R, Defs);
1236
1237 // Sort intrinsics in header file by following order/priority:
1238 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1239 // - Class (is intrinsic overloaded or not)
1240 // - Intrinsic name
1241 std::stable_sort(Defs.begin(), Defs.end(),
1242 [](const std::unique_ptr<Intrinsic> &A,
1243 const std::unique_ptr<Intrinsic> &B) {
1244 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1245 return std::make_tuple(I->getGuard(),
1246 (unsigned)I->getClassKind(),
1247 I->getName());
1248 };
1249 return ToTuple(A) < ToTuple(B);
1250 });
1251
1252 // Actually emit the intrinsic declarations.
1253 for (auto &I : Defs)
1254 I->emitIntrinsic(OS, Emitter, Kind);
1255 }
1256
1257 void SVEEmitter::createHeader(raw_ostream &OS) {
1258 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1259 "-----------------------------------===\n"
1260 " *\n"
1261 " *\n"
1262 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1263 "Exceptions.\n"
1264 " * See https://llvm.org/LICENSE.txt for license information.\n"
1265 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1266 " *\n"
1267 " *===-----------------------------------------------------------------"
1268 "------===\n"
1269 " */\n\n";
1270
1271 OS << "#ifndef __ARM_SVE_H\n";
1272 OS << "#define __ARM_SVE_H\n\n";
1273
1274 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1275 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1276 OS << "#endif\n";
1277
1278 OS << "#include <stdint.h>\n\n";
1279 OS << "#ifdef __cplusplus\n";
1280 OS << "extern \"C\" {\n";
1281 OS << "#else\n";
1282 OS << "#include <stdbool.h>\n";
1283 OS << "#endif\n\n";
1284
1285 OS << "typedef __fp16 float16_t;\n";
1286 OS << "typedef float float32_t;\n";
1287 OS << "typedef double float64_t;\n";
1288
1289 OS << "typedef __SVInt8_t svint8_t;\n";
1290 OS << "typedef __SVInt16_t svint16_t;\n";
1291 OS << "typedef __SVInt32_t svint32_t;\n";
1292 OS << "typedef __SVInt64_t svint64_t;\n";
1293 OS << "typedef __SVUint8_t svuint8_t;\n";
1294 OS << "typedef __SVUint16_t svuint16_t;\n";
1295 OS << "typedef __SVUint32_t svuint32_t;\n";
1296 OS << "typedef __SVUint64_t svuint64_t;\n";
1297 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1298
1299 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1300
1301 OS << "#include <arm_bf16.h>\n";
1302 OS << "#include <arm_vector_types.h>\n";
1303
1304 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1305 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1306 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1307 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1308 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1309 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1310 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1311 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1312 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1313 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1314 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1315 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1316 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1317 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1318 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1319 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1320 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1321 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1322 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1323 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1324 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1325 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1326 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1327 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1328 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1329 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1330 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1331 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1332 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1333 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1334 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1335 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1336 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1337 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1338 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1339 OS << "typedef __SVBool_t svbool_t;\n";
1340 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1341 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1342
1343 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1344 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1345 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1346
1347 OS << "typedef __SVCount_t svcount_t;\n\n";
1348
1349 OS << "enum svpattern\n";
1350 OS << "{\n";
1351 OS << " SV_POW2 = 0,\n";
1352 OS << " SV_VL1 = 1,\n";
1353 OS << " SV_VL2 = 2,\n";
1354 OS << " SV_VL3 = 3,\n";
1355 OS << " SV_VL4 = 4,\n";
1356 OS << " SV_VL5 = 5,\n";
1357 OS << " SV_VL6 = 6,\n";
1358 OS << " SV_VL7 = 7,\n";
1359 OS << " SV_VL8 = 8,\n";
1360 OS << " SV_VL16 = 9,\n";
1361 OS << " SV_VL32 = 10,\n";
1362 OS << " SV_VL64 = 11,\n";
1363 OS << " SV_VL128 = 12,\n";
1364 OS << " SV_VL256 = 13,\n";
1365 OS << " SV_MUL4 = 29,\n";
1366 OS << " SV_MUL3 = 30,\n";
1367 OS << " SV_ALL = 31\n";
1368 OS << "};\n\n";
1369
1370 OS << "enum svprfop\n";
1371 OS << "{\n";
1372 OS << " SV_PLDL1KEEP = 0,\n";
1373 OS << " SV_PLDL1STRM = 1,\n";
1374 OS << " SV_PLDL2KEEP = 2,\n";
1375 OS << " SV_PLDL2STRM = 3,\n";
1376 OS << " SV_PLDL3KEEP = 4,\n";
1377 OS << " SV_PLDL3STRM = 5,\n";
1378 OS << " SV_PSTL1KEEP = 8,\n";
1379 OS << " SV_PSTL1STRM = 9,\n";
1380 OS << " SV_PSTL2KEEP = 10,\n";
1381 OS << " SV_PSTL2STRM = 11,\n";
1382 OS << " SV_PSTL3KEEP = 12,\n";
1383 OS << " SV_PSTL3STRM = 13\n";
1384 OS << "};\n\n";
1385
1386 OS << "/* Function attributes */\n";
1387 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1388 "__nodebug__))\n\n";
1389 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1390 "__nodebug__, __overloadable__))\n\n";
1391
1392 // Add reinterpret functions.
1393 for (auto [N, Suffix] :
1394 std::initializer_list<std::pair<unsigned, const char *>>{
1395 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1396 for (auto ShortForm : {false, true})
1397 for (const ReinterpretTypeInfo &To : Reinterprets) {
1398 SVEType ToV(To.BaseType, N);
1399 for (const ReinterpretTypeInfo &From : Reinterprets) {
1400 SVEType FromV(From.BaseType, N);
1401 if (ShortForm) {
1402 OS << "__aio __attribute__((target(\"sve\"))) " << ToV.str()
1403 << " svreinterpret_" << To.Suffix;
1404 OS << "(" << FromV.str() << " op) __arm_streaming_compatible {\n";
1405 OS << " return __builtin_sve_reinterpret_" << To.Suffix << "_"
1406 << From.Suffix << Suffix << "(op);\n";
1407 OS << "}\n\n";
1408 } else
1409 OS << "#define svreinterpret_" << To.Suffix << "_" << From.Suffix
1410 << Suffix << "(...) __builtin_sve_reinterpret_" << To.Suffix
1411 << "_" << From.Suffix << Suffix << "(__VA_ARGS__)\n";
1412 }
1413 }
1414 }
1415
1416 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
1417
1418 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1419 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1420
1421 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1422 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1423 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1424 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1425
1426 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1427 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1428
1429 OS << "#ifdef __cplusplus\n";
1430 OS << "} // extern \"C\"\n";
1431 OS << "#endif\n\n";
1432 OS << "#undef __ai\n\n";
1433 OS << "#undef __aio\n\n";
1434 OS << "#endif /* __ARM_SVE_H */\n";
1435 }
1436
1437 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1438 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1439 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1440 for (auto *R : RV)
1441 createIntrinsic(R, Defs);
1442
1443 // The mappings must be sorted based on BuiltinID.
1444 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1445 const std::unique_ptr<Intrinsic> &B) {
1446 return A->getMangledName() < B->getMangledName();
1447 });
1448
1449 OS << "#ifdef GET_SVE_BUILTINS\n";
1450 for (auto &Def : Defs) {
1451 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1452 // declarations only live in the header file.
1453 if (Def->getClassKind() != ClassG)
1454 OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1455 << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1456 << "\")\n";
1457 }
1458
1459 // Add reinterpret functions.
1460 for (auto [N, Suffix] :
1461 std::initializer_list<std::pair<unsigned, const char *>>{
1462 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1463 for (const ReinterpretTypeInfo &To : Reinterprets) {
1464 SVEType ToV(To.BaseType, N);
1465 for (const ReinterpretTypeInfo &From : Reinterprets) {
1466 SVEType FromV(From.BaseType, N);
1467 OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
1468 << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
1469 << FromV.builtin_str() << "\", \"n\", \"sve\")\n";
1470 }
1471 }
1472 }
1473
1474 OS << "#endif\n\n";
1475 }
1476
1477 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1478 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1479 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1480 for (auto *R : RV)
1481 createIntrinsic(R, Defs);
1482
1483 // The mappings must be sorted based on BuiltinID.
1484 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1485 const std::unique_ptr<Intrinsic> &B) {
1486 return A->getMangledName() < B->getMangledName();
1487 });
1488
1489 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1490 for (auto &Def : Defs) {
1491 // Builtins only exist for non-overloaded intrinsics, overloaded
1492 // declarations only live in the header file.
1493 if (Def->getClassKind() == ClassG)
1494 continue;
1495
1496 uint64_t Flags = Def->getFlags();
1497 auto FlagString = std::to_string(Flags);
1498
1499 std::string LLVMName = Def->getMangledLLVMName();
1500 std::string Builtin = Def->getMangledName();
1501 if (!LLVMName.empty())
1502 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1503 << "),\n";
1504 else
1505 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1506 }
1507 OS << "#endif\n\n";
1508 }
1509
1510 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1511 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1512 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1513 for (auto *R : RV)
1514 createIntrinsic(R, Defs);
1515
1516 // The mappings must be sorted based on BuiltinID.
1517 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1518 const std::unique_ptr<Intrinsic> &B) {
1519 return A->getMangledName() < B->getMangledName();
1520 });
1521
1522
1523 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1524
1525 // Ensure these are only emitted once.
1526 std::set<std::string> Emitted;
1527
1528 for (auto &Def : Defs) {
1529 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1530 Def->getImmChecks().empty())
1531 continue;
1532
1533 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1534 for (auto &Check : Def->getImmChecks())
1535 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1536 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1537 OS << " break;\n";
1538
1539 Emitted.insert(Def->getMangledName());
1540 }
1541
1542 OS << "#endif\n\n";
1543 }
1544
1545 /// Create the SVETypeFlags used in CGBuiltins
1546 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1547 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1548 for (auto &KV : FlagTypes)
1549 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1550 OS << "#endif\n\n";
1551
1552 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1553 for (auto &KV : EltTypes)
1554 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1555 OS << "#endif\n\n";
1556
1557 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1558 for (auto &KV : MemEltTypes)
1559 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1560 OS << "#endif\n\n";
1561
1562 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1563 for (auto &KV : MergeTypes)
1564 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1565 OS << "#endif\n\n";
1566
1567 OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
1568 for (auto &KV : ImmCheckTypes)
1569 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1570 OS << "#endif\n\n";
1571 }
1572
1573 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1574 OS << "/*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics "
1575 "------===\n"
1576 " *\n"
1577 " *\n"
1578 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1579 "Exceptions.\n"
1580 " * See https://llvm.org/LICENSE.txt for license information.\n"
1581 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1582 " *\n"
1583 " *===-----------------------------------------------------------------"
1584 "------===\n"
1585 " */\n\n";
1586
1587 OS << "#ifndef __ARM_SME_H\n";
1588 OS << "#define __ARM_SME_H\n\n";
1589
1590 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1591 OS << "#error \"Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h\"\n";
1592 OS << "#endif\n";
1593
1594 OS << "#include <arm_sve.h>\n\n";
1595
1596 OS << "/* Function attributes */\n";
1597 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1598 "__nodebug__))\n\n";
1599 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1600 "__nodebug__, __overloadable__))\n\n";
1601
1602 OS << "#ifdef __cplusplus\n";
1603 OS << "extern \"C\" {\n";
1604 OS << "#endif\n\n";
1605
1606 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
1607
1608 OS << "#ifdef __cplusplus\n";
1609 OS << "} // extern \"C\"\n";
1610 OS << "#endif\n\n";
1611 OS << "#undef __ai\n\n";
1612 OS << "#endif /* __ARM_SME_H */\n";
1613 }
1614
1615 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1616 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1617 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1618 for (auto *R : RV) {
1619 createIntrinsic(R, Defs);
1620 }
1621
1622 // The mappings must be sorted based on BuiltinID.
1623 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1624 const std::unique_ptr<Intrinsic> &B) {
1625 return A->getMangledName() < B->getMangledName();
1626 });
1627
1628 OS << "#ifdef GET_SME_BUILTINS\n";
1629 for (auto &Def : Defs) {
1630 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1631 // declarations only live in the header file.
1632 if (Def->getClassKind() != ClassG)
1633 OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
1634 << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
1635 << "\")\n";
1636 }
1637
1638 OS << "#endif\n\n";
1639 }
1640
1641 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1642 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1643 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1644 for (auto *R : RV) {
1645 createIntrinsic(R, Defs);
1646 }
1647
1648 // The mappings must be sorted based on BuiltinID.
1649 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1650 const std::unique_ptr<Intrinsic> &B) {
1651 return A->getMangledName() < B->getMangledName();
1652 });
1653
1654 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1655 for (auto &Def : Defs) {
1656 // Builtins only exist for non-overloaded intrinsics, overloaded
1657 // declarations only live in the header file.
1658 if (Def->getClassKind() == ClassG)
1659 continue;
1660
1661 uint64_t Flags = Def->getFlags();
1662 auto FlagString = std::to_string(Flags);
1663
1664 std::string LLVMName = Def->getLLVMName();
1665 std::string Builtin = Def->getMangledName();
1666 if (!LLVMName.empty())
1667 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1668 << "),\n";
1669 else
1670 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1671 }
1672 OS << "#endif\n\n";
1673 }
1674
1675 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1676 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1677 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1678 for (auto *R : RV) {
1679 createIntrinsic(R, Defs);
1680 }
1681
1682 // The mappings must be sorted based on BuiltinID.
1683 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1684 const std::unique_ptr<Intrinsic> &B) {
1685 return A->getMangledName() < B->getMangledName();
1686 });
1687
1688
1689 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1690
1691 // Ensure these are only emitted once.
1692 std::set<std::string> Emitted;
1693
1694 for (auto &Def : Defs) {
1695 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1696 Def->getImmChecks().empty())
1697 continue;
1698
1699 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1700 for (auto &Check : Def->getImmChecks())
1701 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1702 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1703 OS << " break;\n";
1704
1705 Emitted.insert(Def->getMangledName());
1706 }
1707
1708 OS << "#endif\n\n";
1709 }
1710
1711 void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1712 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1713 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1714 for (auto *R : RV)
1715 createIntrinsic(R, Defs);
1716
1717 std::map<bool, std::set<std::string>> DefsZAState;
1718
1719 uint64_t IsSharedZAFlag = getEnumValueForFlag("IsSharedZA");
1720 for (auto &Def : Defs) {
1721 bool HasZAState = Def->isFlagSet(IsSharedZAFlag);
1722 DefsZAState[HasZAState].insert(Def->getMangledName());
1723 }
1724
1725 OS << "#ifdef GET_SME_BUILTIN_HAS_ZA_STATE\n";
1726
1727 for (auto HasZA : {true, false}) {
1728 auto Names = DefsZAState[HasZA];
1729 for (auto Name : Names)
1730 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1731 OS << " return " << (HasZA ? "true" : "false") << ";\n";
1732 }
1733 OS << "#endif\n\n";
1734 }
1735
1736 void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1737 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1738 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1739 for (auto *R : RV)
1740 createIntrinsic(R, Defs);
1741
1742 StringRef ExtensionKind;
1743 switch (Kind) {
1744 case ACLEKind::SME:
1745 ExtensionKind = "SME";
1746 break;
1747 case ACLEKind::SVE:
1748 ExtensionKind = "SVE";
1749 break;
1750 }
1751
1752 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1753
1754 llvm::StringMap<std::set<std::string>> StreamingMap;
1755
1756 uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
1757 uint64_t IsStreamingCompatibleFlag =
1758 getEnumValueForFlag("IsStreamingCompatible");
1759 for (auto &Def : Defs) {
1760 if (Def->isFlagSet(IsStreamingFlag))
1761 StreamingMap["ArmStreaming"].insert(Def->getMangledName());
1762 else if (Def->isFlagSet(IsStreamingCompatibleFlag))
1763 StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
1764 else
1765 StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
1766 }
1767
1768 for (auto BuiltinType : StreamingMap.keys()) {
1769 for (auto Name : StreamingMap[BuiltinType]) {
1770 OS << "case " << ExtensionKind << "::BI__builtin_"
1771 << ExtensionKind.lower() << "_";
1772 OS << Name << ":\n";
1773 }
1774 OS << " BuiltinType = " << BuiltinType << ";\n";
1775 OS << " break;\n";
1776 }
1777
1778 OS << "#endif\n\n";
1779 }
1780
1781 namespace clang {
1782 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
1783 SVEEmitter(Records).createHeader(OS);
1784 }
1785
1786 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1787 SVEEmitter(Records).createBuiltins(OS);
1788 }
1789
1790 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1791 SVEEmitter(Records).createCodeGenMap(OS);
1792 }
1793
1794 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1795 SVEEmitter(Records).createRangeChecks(OS);
1796 }
1797
1798 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
1799 SVEEmitter(Records).createTypeFlags(OS);
1800 }
1801
1802 void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1803 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
1804 }
1805
1806 void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
1807 SVEEmitter(Records).createSMEHeader(OS);
1808 }
1809
1810 void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1811 SVEEmitter(Records).createSMEBuiltins(OS);
1812 }
1813
1814 void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1815 SVEEmitter(Records).createSMECodeGenMap(OS);
1816 }
1817
1818 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1819 SVEEmitter(Records).createSMERangeChecks(OS);
1820 }
1821
1822 void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1823 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
1824 }
1825
1826 void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
1827 SVEEmitter(Records).createBuiltinZAState(OS);
1828 }
1829 } // End namespace clang
LLVM monorepo