1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // OpDocGen uses the description of operations to generate documentation for the
12 //===----------------------------------------------------------------------===//
14 #include "DialectGenUtilities.h"
15 #include "DocGenUtilities.h"
16 #include "OpGenHelpers.h"
17 #include "mlir/Support/IndentedOstream.h"
18 #include "mlir/TableGen/AttrOrTypeDef.h"
19 #include "mlir/TableGen/Attribute.h"
20 #include "mlir/TableGen/GenInfo.h"
21 #include "mlir/TableGen/Operator.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/SetVector.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/FormatVariadic.h"
28 #include "llvm/Support/Regex.h"
29 #include "llvm/Support/Signals.h"
30 #include "llvm/TableGen/Error.h"
31 #include "llvm/TableGen/Record.h"
32 #include "llvm/TableGen/TableGenBackend.h"
37 //===----------------------------------------------------------------------===//
38 // Commandline Options
39 //===----------------------------------------------------------------------===//
40 static llvm::cl::OptionCategory
41 docCat("Options for -gen-(attrdef|typedef|enum|op|dialect)-doc");
42 llvm::cl::opt
<std::string
>
43 stripPrefix("strip-prefix",
44 llvm::cl::desc("Strip prefix of the fully qualified names"),
45 llvm::cl::init("::mlir::"), llvm::cl::cat(docCat
));
46 llvm::cl::opt
<bool> allowHugoSpecificFeatures(
47 "allow-hugo-specific-features",
48 llvm::cl::desc("Allows using features specific to Hugo"),
49 llvm::cl::init(false), llvm::cl::cat(docCat
));
53 using namespace mlir::tblgen
;
54 using mlir::tblgen::Operator
;
56 void mlir::tblgen::emitSummary(StringRef summary
, raw_ostream
&os
) {
57 if (!summary
.empty()) {
58 llvm::StringRef trimmed
= summary
.trim();
59 char first
= std::toupper(trimmed
.front());
60 llvm::StringRef rest
= trimmed
.drop_front();
61 os
<< "\n_" << first
<< rest
<< "_\n\n";
65 // Emit the description by aligning the text to the left per line (e.g.,
66 // removing the minimum indentation across the block).
68 // This expects that the description in the tablegen file is already formatted
69 // in a way the user wanted but has some additional indenting due to being
70 // nested in the op definition.
71 void mlir::tblgen::emitDescription(StringRef description
, raw_ostream
&os
) {
72 raw_indented_ostream
ros(os
);
73 ros
.printReindented(description
.rtrim(" \t"));
76 void mlir::tblgen::emitDescriptionComment(StringRef description
,
77 raw_ostream
&os
, StringRef prefix
) {
78 if (description
.empty())
80 raw_indented_ostream
ros(os
);
81 StringRef trimmed
= description
.rtrim(" \t");
82 ros
.printReindented(trimmed
, (Twine(prefix
) + "/// ").str());
83 if (!trimmed
.ends_with("\n"))
87 // Emits `str` with trailing newline if not empty.
88 static void emitIfNotEmpty(StringRef str
, raw_ostream
&os
) {
90 emitDescription(str
, os
);
95 /// Emit the given named constraint.
97 static void emitNamedConstraint(const T
&it
, raw_ostream
&os
) {
99 os
<< "| `" << it
.name
<< "`";
101 os
<< "«unnamed»";
102 os
<< " | " << it
.constraint
.getSummary() << "\n";
105 //===----------------------------------------------------------------------===//
106 // Operation Documentation
107 //===----------------------------------------------------------------------===//
109 /// Emit the assembly format of an operation.
110 static void emitAssemblyFormat(StringRef opName
, StringRef format
,
112 os
<< "\nSyntax:\n\n```\noperation ::= `" << opName
<< "` ";
114 // Print the assembly format aligned.
115 unsigned indent
= strlen("operation ::= ");
116 std::pair
<StringRef
, StringRef
> split
= format
.split('\n');
117 os
<< split
.first
.trim() << "\n";
119 split
= split
.second
.split('\n');
120 StringRef formatChunk
= split
.first
.trim();
121 if (!formatChunk
.empty())
122 os
.indent(indent
) << formatChunk
<< "\n";
123 } while (!split
.second
.empty());
127 /// Place `text` between backticks so that the Markdown processor renders it as
129 static std::string
backticks(const std::string
&text
) {
130 return '`' + text
+ '`';
133 static void emitOpTraitsDoc(const Operator
&op
, raw_ostream
&os
) {
134 // TODO: We should link to the trait/documentation of it. That also means we
135 // should add descriptions to traits that can be queried.
136 // Collect using set to sort effects, interfaces & traits.
137 std::set
<std::string
> effects
, interfaces
, traits
;
138 for (auto &trait
: op
.getTraits()) {
139 if (isa
<PredTrait
>(&trait
))
142 std::string name
= trait
.getDef().getName().str();
143 StringRef ref
= name
;
144 StringRef traitName
= trait
.getDef().getValueAsString("trait");
145 traitName
.consume_back("::Trait");
146 traitName
.consume_back("::Impl");
147 if (ref
.starts_with("anonymous_"))
148 name
= traitName
.str();
149 if (isa
<InterfaceTrait
>(&trait
)) {
150 if (trait
.getDef().isSubClassOf("SideEffectsTraitBase")) {
151 auto effectName
= trait
.getDef().getValueAsString("baseEffectName");
152 effectName
.consume_front("::");
153 effectName
.consume_front("mlir::");
154 std::string effectStr
;
155 llvm::raw_string_ostream
os(effectStr
);
156 os
<< effectName
<< "{";
157 auto list
= trait
.getDef().getValueAsListOfDefs("effects");
158 llvm::interleaveComma(list
, os
, [&](Record
*rec
) {
159 StringRef effect
= rec
->getValueAsString("effect");
160 effect
.consume_front("::");
161 effect
.consume_front("mlir::");
162 os
<< effect
<< " on " << rec
->getValueAsString("resource");
165 effects
.insert(backticks(os
.str()));
166 name
.append(llvm::formatv(" ({0})", traitName
).str());
168 interfaces
.insert(backticks(name
));
172 traits
.insert(backticks(name
));
174 if (!traits
.empty()) {
175 llvm::interleaveComma(traits
, os
<< "\nTraits: ");
178 if (!interfaces
.empty()) {
179 llvm::interleaveComma(interfaces
, os
<< "\nInterfaces: ");
182 if (!effects
.empty()) {
183 llvm::interleaveComma(effects
, os
<< "\nEffects: ");
188 static StringRef
resolveAttrDescription(const Attribute
&attr
) {
189 StringRef description
= attr
.getDescription();
190 if (description
.empty())
191 return attr
.getBaseAttr().getDescription();
195 static void emitOpDoc(const Operator
&op
, raw_ostream
&os
) {
196 std::string classNameStr
= op
.getQualCppClassName();
197 StringRef className
= classNameStr
;
198 (void)className
.consume_front(stripPrefix
);
199 os
<< llvm::formatv("### `{0}` ({1})\n", op
.getOperationName(), className
);
201 // Emit the summary, syntax, and description if present.
203 emitSummary(op
.getSummary(), os
);
204 if (op
.hasAssemblyFormat())
205 emitAssemblyFormat(op
.getOperationName(), op
.getAssemblyFormat().trim(),
207 if (op
.hasDescription())
208 mlir::tblgen::emitDescription(op
.getDescription(), os
);
210 emitOpTraitsDoc(op
, os
);
213 if (op
.getNumAttributes() != 0) {
214 os
<< "\n#### Attributes:\n\n";
215 // Note: This table is HTML rather than markdown so the attribute's
216 // description can appear in an expandable region. The description may be
217 // multiple lines, which is not supported in a markdown table cell.
220 os
<< "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
221 for (const auto &it
: op
.getAttributes()) {
222 StringRef storageType
= it
.attr
.getStorageType();
223 // Name and storage type.
225 os
<< "<td><code>" << it
.name
<< "</code></td><td>" << storageType
227 StringRef description
= resolveAttrDescription(it
.attr
);
228 if (allowHugoSpecificFeatures
&& !description
.empty()) {
229 // Expandable description.
230 // This appears as just the summary, but when clicked shows the full
232 os
<< "<details>" << "<summary>" << it
.attr
.getSummary() << "</summary>"
233 << "{{% markdown %}}" << description
<< "{{% /markdown %}}"
236 // Fallback: Single-line summary.
237 os
<< it
.attr
.getSummary();
239 os
<< "</td></tr>\n";
244 // Emit each of the operands.
245 if (op
.getNumOperands() != 0) {
246 os
<< "\n#### Operands:\n\n";
247 os
<< "| Operand | Description |\n"
248 << "| :-----: | ----------- |\n";
249 for (const auto &it
: op
.getOperands())
250 emitNamedConstraint(it
, os
);
254 if (op
.getNumResults() != 0) {
255 os
<< "\n#### Results:\n\n";
256 os
<< "| Result | Description |\n"
257 << "| :----: | ----------- |\n";
258 for (const auto &it
: op
.getResults())
259 emitNamedConstraint(it
, os
);
263 if (op
.getNumSuccessors() != 0) {
264 os
<< "\n#### Successors:\n\n";
265 os
<< "| Successor | Description |\n"
266 << "| :-------: | ----------- |\n";
267 for (const auto &it
: op
.getSuccessors())
268 emitNamedConstraint(it
, os
);
274 static void emitSourceLink(StringRef inputFilename
, raw_ostream
&os
) {
275 size_t pathBegin
= inputFilename
.find("mlir/include/mlir/");
276 if (pathBegin
== StringRef::npos
)
279 StringRef inputFromMlirInclude
= inputFilename
.substr(pathBegin
);
281 os
<< "[source](https://github.com/llvm/llvm-project/blob/main/"
282 << inputFromMlirInclude
<< ")\n\n";
285 static void emitOpDoc(const RecordKeeper
&recordKeeper
, raw_ostream
&os
) {
286 auto opDefs
= getRequestedOpDefinitions(recordKeeper
);
288 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
289 emitSourceLink(recordKeeper
.getInputFilename(), os
);
290 for (const llvm::Record
*opDef
: opDefs
)
291 emitOpDoc(Operator(opDef
), os
);
294 //===----------------------------------------------------------------------===//
295 // Attribute Documentation
296 //===----------------------------------------------------------------------===//
298 static void emitAttrDoc(const Attribute
&attr
, raw_ostream
&os
) {
299 os
<< "### " << attr
.getSummary() << "\n\n";
300 emitDescription(attr
.getDescription(), os
);
304 //===----------------------------------------------------------------------===//
305 // Type Documentation
306 //===----------------------------------------------------------------------===//
308 static void emitTypeDoc(const Type
&type
, raw_ostream
&os
) {
309 os
<< "### " << type
.getSummary() << "\n\n";
310 emitDescription(type
.getDescription(), os
);
314 //===----------------------------------------------------------------------===//
315 // TypeDef Documentation
316 //===----------------------------------------------------------------------===//
318 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef
&def
,
320 ArrayRef
<AttrOrTypeParameter
> parameters
= def
.getParameters();
321 char prefix
= isa
<AttrDef
>(def
) ? '#' : '!';
322 if (parameters
.empty()) {
323 os
<< "\nSyntax: `" << prefix
<< def
.getDialect().getName() << "."
324 << def
.getMnemonic() << "`\n";
328 os
<< "\nSyntax:\n\n```\n"
329 << prefix
<< def
.getDialect().getName() << "." << def
.getMnemonic()
331 for (const auto &it
: llvm::enumerate(parameters
)) {
332 const AttrOrTypeParameter
¶m
= it
.value();
333 os
<< " " << param
.getSyntax();
334 if (it
.index() < (parameters
.size() - 1))
336 os
<< " # " << param
.getName() << "\n";
341 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef
&def
, raw_ostream
&os
) {
342 os
<< llvm::formatv("### {0}\n", def
.getCppClassName());
344 // Emit the summary if present.
345 if (def
.hasSummary())
346 os
<< "\n" << def
.getSummary() << "\n";
348 // Emit the syntax if present.
349 if (def
.getMnemonic() && !def
.hasCustomAssemblyFormat())
350 emitAttrOrTypeDefAssemblyFormat(def
, os
);
352 // Emit the description if present.
353 if (def
.hasDescription()) {
355 mlir::tblgen::emitDescription(def
.getDescription(), os
);
358 // Emit parameter documentation.
359 ArrayRef
<AttrOrTypeParameter
> parameters
= def
.getParameters();
360 if (!parameters
.empty()) {
361 os
<< "\n#### Parameters:\n\n";
362 os
<< "| Parameter | C++ type | Description |\n"
363 << "| :-------: | :-------: | ----------- |\n";
364 for (const auto &it
: parameters
) {
365 auto desc
= it
.getSummary();
366 os
<< "| " << it
.getName() << " | `" << it
.getCppType() << "` | "
367 << (desc
? *desc
: "") << " |\n";
374 static void emitAttrOrTypeDefDoc(const RecordKeeper
&recordKeeper
,
375 raw_ostream
&os
, StringRef recordTypeName
) {
376 std::vector
<llvm::Record
*> defs
=
377 recordKeeper
.getAllDerivedDefinitions(recordTypeName
);
379 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
380 for (const llvm::Record
*def
: defs
)
381 emitAttrOrTypeDefDoc(AttrOrTypeDef(def
), os
);
384 //===----------------------------------------------------------------------===//
385 // Enum Documentation
386 //===----------------------------------------------------------------------===//
388 static void emitEnumDoc(const EnumAttr
&def
, raw_ostream
&os
) {
389 os
<< llvm::formatv("### {0}\n", def
.getEnumClassName());
391 // Emit the summary if present.
392 if (!def
.getSummary().empty())
393 os
<< "\n" << def
.getSummary() << "\n";
395 // Emit case documentation.
396 std::vector
<EnumAttrCase
> cases
= def
.getAllCases();
397 os
<< "\n#### Cases:\n\n";
398 os
<< "| Symbol | Value | String |\n"
399 << "| :----: | :---: | ------ |\n";
400 for (const auto &it
: cases
) {
401 os
<< "| " << it
.getSymbol() << " | `" << it
.getValue() << "` | "
402 << it
.getStr() << " |\n";
408 static void emitEnumDoc(const RecordKeeper
&recordKeeper
, raw_ostream
&os
) {
409 std::vector
<llvm::Record
*> defs
=
410 recordKeeper
.getAllDerivedDefinitions("EnumAttr");
412 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
413 for (const llvm::Record
*def
: defs
)
414 emitEnumDoc(EnumAttr(def
), os
);
417 //===----------------------------------------------------------------------===//
418 // Dialect Documentation
419 //===----------------------------------------------------------------------===//
422 const Dialect
&getDialect() const { return ops
.front().getDialect(); }
424 // Returns the summary description of the section.
425 std::string summary
= "";
427 // Returns the description of the section.
428 StringRef description
= "";
430 // Instances inside the section.
431 std::vector
<Operator
> ops
;
434 static void maybeNest(bool nest
, llvm::function_ref
<void(raw_ostream
&os
)> fn
,
437 llvm::raw_string_ostream
ss(str
);
439 for (StringRef x
: llvm::split(ss
.str(), "\n")) {
440 if (nest
&& x
.starts_with("#"))
446 static void emitBlock(ArrayRef
<Attribute
> attributes
, StringRef inputFilename
,
447 ArrayRef
<AttrDef
> attrDefs
, ArrayRef
<OpDocGroup
> ops
,
448 ArrayRef
<Type
> types
, ArrayRef
<TypeDef
> typeDefs
,
449 ArrayRef
<EnumAttr
> enums
, raw_ostream
&os
) {
451 os
<< "## Operations\n\n";
452 emitSourceLink(inputFilename
, os
);
453 for (const OpDocGroup
&grouping
: ops
) {
454 bool nested
= !grouping
.summary
.empty();
457 [&](raw_ostream
&os
) {
459 os
<< "## " << StringRef(grouping
.summary
).trim() << "\n\n";
460 emitDescription(grouping
.description
, os
);
463 for (const Operator
&op
: grouping
.ops
) {
471 if (!attributes
.empty()) {
472 os
<< "## Attribute constraints\n\n";
473 for (const Attribute
&attr
: attributes
)
474 emitAttrDoc(attr
, os
);
477 if (!attrDefs
.empty()) {
478 os
<< "## Attributes\n\n";
479 for (const AttrDef
&def
: attrDefs
)
480 emitAttrOrTypeDefDoc(def
, os
);
483 // TODO: Add link between use and def for types
484 if (!types
.empty()) {
485 os
<< "## Type constraints\n\n";
486 for (const Type
&type
: types
)
487 emitTypeDoc(type
, os
);
490 if (!typeDefs
.empty()) {
491 os
<< "## Types\n\n";
492 for (const TypeDef
&def
: typeDefs
)
493 emitAttrOrTypeDefDoc(def
, os
);
496 if (!enums
.empty()) {
497 os
<< "## Enums\n\n";
498 for (const EnumAttr
&def
: enums
)
499 emitEnumDoc(def
, os
);
503 static void emitDialectDoc(const Dialect
&dialect
, StringRef inputFilename
,
504 ArrayRef
<Attribute
> attributes
,
505 ArrayRef
<AttrDef
> attrDefs
, ArrayRef
<OpDocGroup
> ops
,
506 ArrayRef
<Type
> types
, ArrayRef
<TypeDef
> typeDefs
,
507 ArrayRef
<EnumAttr
> enums
, raw_ostream
&os
) {
508 os
<< "# '" << dialect
.getName() << "' Dialect\n\n";
509 emitIfNotEmpty(dialect
.getSummary(), os
);
510 emitIfNotEmpty(dialect
.getDescription(), os
);
512 // Generate a TOC marker except if description already contains one.
513 llvm::Regex
r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline
);
514 if (!r
.match(dialect
.getDescription()))
517 emitBlock(attributes
, inputFilename
, attrDefs
, ops
, types
, typeDefs
, enums
,
521 static bool emitDialectDoc(const RecordKeeper
&recordKeeper
, raw_ostream
&os
) {
522 std::vector
<Record
*> dialectDefs
=
523 recordKeeper
.getAllDerivedDefinitionsIfDefined("Dialect");
524 SmallVector
<Dialect
> dialects(dialectDefs
.begin(), dialectDefs
.end());
525 std::optional
<Dialect
> dialect
= findDialectToGenerate(dialects
);
529 std::vector
<Record
*> opDefs
= getRequestedOpDefinitions(recordKeeper
);
530 std::vector
<Record
*> attrDefs
=
531 recordKeeper
.getAllDerivedDefinitionsIfDefined("DialectAttr");
532 std::vector
<Record
*> typeDefs
=
533 recordKeeper
.getAllDerivedDefinitionsIfDefined("DialectType");
534 std::vector
<Record
*> typeDefDefs
=
535 recordKeeper
.getAllDerivedDefinitionsIfDefined("TypeDef");
536 std::vector
<Record
*> attrDefDefs
=
537 recordKeeper
.getAllDerivedDefinitionsIfDefined("AttrDef");
538 std::vector
<Record
*> enumDefs
=
539 recordKeeper
.getAllDerivedDefinitionsIfDefined("EnumAttrInfo");
541 std::vector
<Attribute
> dialectAttrs
;
542 std::vector
<AttrDef
> dialectAttrDefs
;
543 std::vector
<OpDocGroup
> dialectOps
;
544 std::vector
<Type
> dialectTypes
;
545 std::vector
<TypeDef
> dialectTypeDefs
;
546 std::vector
<EnumAttr
> dialectEnums
;
548 llvm::SmallDenseSet
<Record
*> seen
;
549 auto addIfNotSeen
= [&](llvm::Record
*record
, const auto &def
, auto &vec
) {
550 if (seen
.insert(record
).second
) {
556 auto addIfInDialect
= [&](llvm::Record
*record
, const auto &def
, auto &vec
) {
557 return def
.getDialect() == *dialect
&& addIfNotSeen(record
, def
, vec
);
560 SmallDenseMap
<Record
*, OpDocGroup
> opDocGroup
;
562 for (Record
*def
: attrDefDefs
)
563 addIfInDialect(def
, AttrDef(def
), dialectAttrDefs
);
564 for (Record
*def
: attrDefs
)
565 addIfInDialect(def
, Attribute(def
), dialectAttrs
);
566 for (Record
*def
: opDefs
) {
567 if (Record
*group
= def
->getValueAsOptionalDef("opDocGroup")) {
568 OpDocGroup
&op
= opDocGroup
[group
];
569 addIfInDialect(def
, Operator(def
), op
.ops
);
572 op
.ops
.emplace_back(def
);
573 addIfInDialect(def
, op
, dialectOps
);
577 recordKeeper
.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
578 if (opDocGroup
[rec
].ops
.empty())
580 opDocGroup
[rec
].summary
= rec
->getValueAsString("summary");
581 opDocGroup
[rec
].description
= rec
->getValueAsString("description");
582 dialectOps
.push_back(opDocGroup
[rec
]);
584 for (Record
*def
: typeDefDefs
)
585 addIfInDialect(def
, TypeDef(def
), dialectTypeDefs
);
586 for (Record
*def
: typeDefs
)
587 addIfInDialect(def
, Type(def
), dialectTypes
);
588 dialectEnums
.reserve(enumDefs
.size());
589 for (Record
*def
: enumDefs
)
590 addIfNotSeen(def
, EnumAttr(def
), dialectEnums
);
592 // Sort alphabetically ignorning dialect for ops and section name for
594 // TODO: The sorting order could be revised, currently attempting to sort of
595 // keep in alphabetical order.
596 std::sort(dialectOps
.begin(), dialectOps
.end(),
597 [](const OpDocGroup
&lhs
, const OpDocGroup
&rhs
) {
598 auto getDesc
= [](const OpDocGroup
&arg
) -> StringRef
{
599 if (!arg
.summary
.empty())
601 return arg
.ops
.front().getDef().getValueAsString("opName");
603 return getDesc(lhs
).compare_insensitive(getDesc(rhs
)) < 0;
606 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
607 emitDialectDoc(*dialect
, recordKeeper
.getInputFilename(), dialectAttrs
,
608 dialectAttrDefs
, dialectOps
, dialectTypes
, dialectTypeDefs
,
613 //===----------------------------------------------------------------------===//
615 //===----------------------------------------------------------------------===//
617 static mlir::GenRegistration
618 genAttrRegister("gen-attrdef-doc",
619 "Generate dialect attribute documentation",
620 [](const RecordKeeper
&records
, raw_ostream
&os
) {
621 emitAttrOrTypeDefDoc(records
, os
, "AttrDef");
625 static mlir::GenRegistration
626 genOpRegister("gen-op-doc", "Generate dialect documentation",
627 [](const RecordKeeper
&records
, raw_ostream
&os
) {
628 emitOpDoc(records
, os
);
632 static mlir::GenRegistration
633 genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
634 [](const RecordKeeper
&records
, raw_ostream
&os
) {
635 emitAttrOrTypeDefDoc(records
, os
, "TypeDef");
639 static mlir::GenRegistration
640 genEnumRegister("gen-enum-doc", "Generate dialect enum documentation",
641 [](const RecordKeeper
&records
, raw_ostream
&os
) {
642 emitEnumDoc(records
, os
);
646 static mlir::GenRegistration
647 genRegister("gen-dialect-doc", "Generate dialect documentation",
648 [](const RecordKeeper
&records
, raw_ostream
&os
) {
649 return emitDialectDoc(records
, os
);