1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // OpDocGen uses the description of operations to generate documentation for the
12 //===----------------------------------------------------------------------===//
14 #include "DialectGenUtilities.h"
15 #include "DocGenUtilities.h"
16 #include "OpGenHelpers.h"
17 #include "mlir/Support/IndentedOstream.h"
18 #include "mlir/TableGen/AttrOrTypeDef.h"
19 #include "mlir/TableGen/Attribute.h"
20 #include "mlir/TableGen/GenInfo.h"
21 #include "mlir/TableGen/Operator.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/SetVector.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/FormatVariadic.h"
28 #include "llvm/Support/Regex.h"
29 #include "llvm/Support/Signals.h"
30 #include "llvm/TableGen/Error.h"
31 #include "llvm/TableGen/Record.h"
32 #include "llvm/TableGen/TableGenBackend.h"
39 using namespace mlir::tblgen
;
40 using mlir::tblgen::Operator
;
42 //===----------------------------------------------------------------------===//
43 // Commandline Options
44 //===----------------------------------------------------------------------===//
45 static cl::OptionCategory
46 docCat("Options for -gen-(attrdef|typedef|enum|op|dialect)-doc");
48 stripPrefix("strip-prefix",
49 cl::desc("Strip prefix of the fully qualified names"),
50 cl::init("::mlir::"), cl::cat(docCat
));
51 cl::opt
<bool> allowHugoSpecificFeatures(
52 "allow-hugo-specific-features",
53 cl::desc("Allows using features specific to Hugo"), cl::init(false),
56 void mlir::tblgen::emitSummary(StringRef summary
, raw_ostream
&os
) {
57 if (!summary
.empty()) {
58 StringRef trimmed
= summary
.trim();
59 char first
= std::toupper(trimmed
.front());
60 StringRef rest
= trimmed
.drop_front();
61 os
<< "\n_" << first
<< rest
<< "_\n\n";
65 // Emit the description by aligning the text to the left per line (e.g.,
66 // removing the minimum indentation across the block).
68 // This expects that the description in the tablegen file is already formatted
69 // in a way the user wanted but has some additional indenting due to being
70 // nested in the op definition.
71 void mlir::tblgen::emitDescription(StringRef description
, raw_ostream
&os
) {
72 raw_indented_ostream
ros(os
);
73 StringRef trimmed
= description
.rtrim(" \t");
74 ros
.printReindented(trimmed
);
75 if (!trimmed
.ends_with("\n"))
79 void mlir::tblgen::emitDescriptionComment(StringRef description
,
80 raw_ostream
&os
, StringRef prefix
) {
81 if (description
.empty())
83 raw_indented_ostream
ros(os
);
84 StringRef trimmed
= description
.rtrim(" \t");
85 ros
.printReindented(trimmed
, (Twine(prefix
) + "/// ").str());
86 if (!trimmed
.ends_with("\n"))
90 // Emits `str` with trailing newline if not empty.
91 static void emitIfNotEmpty(StringRef str
, raw_ostream
&os
) {
93 emitDescription(str
, os
);
98 /// Emit the given named constraint.
100 static void emitNamedConstraint(const T
&it
, raw_ostream
&os
) {
101 if (!it
.name
.empty())
102 os
<< "| `" << it
.name
<< "`";
104 os
<< "«unnamed»";
105 os
<< " | " << it
.constraint
.getSummary() << "\n";
108 //===----------------------------------------------------------------------===//
109 // Operation Documentation
110 //===----------------------------------------------------------------------===//
112 /// Emit the assembly format of an operation.
113 static void emitAssemblyFormat(StringRef opName
, StringRef format
,
115 os
<< "\nSyntax:\n\n```\noperation ::= `" << opName
<< "` ";
117 // Print the assembly format aligned.
118 unsigned indent
= strlen("operation ::= ");
119 std::pair
<StringRef
, StringRef
> split
= format
.split('\n');
120 os
<< split
.first
.trim() << "\n";
122 split
= split
.second
.split('\n');
123 StringRef formatChunk
= split
.first
.trim();
124 if (!formatChunk
.empty())
125 os
.indent(indent
) << formatChunk
<< "\n";
126 } while (!split
.second
.empty());
130 /// Place `text` between backticks so that the Markdown processor renders it as
132 static std::string
backticks(const std::string
&text
) {
133 return '`' + text
+ '`';
136 static void emitOpTraitsDoc(const Operator
&op
, raw_ostream
&os
) {
137 // TODO: We should link to the trait/documentation of it. That also means we
138 // should add descriptions to traits that can be queried.
139 // Collect using set to sort effects, interfaces & traits.
140 std::set
<std::string
> effects
, interfaces
, traits
;
141 for (auto &trait
: op
.getTraits()) {
142 if (isa
<PredTrait
>(&trait
))
145 std::string name
= trait
.getDef().getName().str();
146 StringRef ref
= name
;
147 StringRef traitName
= trait
.getDef().getValueAsString("trait");
148 traitName
.consume_back("::Trait");
149 traitName
.consume_back("::Impl");
150 if (ref
.starts_with("anonymous_"))
151 name
= traitName
.str();
152 if (isa
<InterfaceTrait
>(&trait
)) {
153 if (trait
.getDef().isSubClassOf("SideEffectsTraitBase")) {
154 auto effectName
= trait
.getDef().getValueAsString("baseEffectName");
155 effectName
.consume_front("::");
156 effectName
.consume_front("mlir::");
157 std::string effectStr
;
158 raw_string_ostream
os(effectStr
);
159 os
<< effectName
<< "{";
160 auto list
= trait
.getDef().getValueAsListOfDefs("effects");
161 interleaveComma(list
, os
, [&](const Record
*rec
) {
162 StringRef effect
= rec
->getValueAsString("effect");
163 effect
.consume_front("::");
164 effect
.consume_front("mlir::");
165 os
<< effect
<< " on " << rec
->getValueAsString("resource");
168 effects
.insert(backticks(effectStr
));
169 name
.append(formatv(" ({0})", traitName
).str());
171 interfaces
.insert(backticks(name
));
175 traits
.insert(backticks(name
));
177 if (!traits
.empty()) {
178 interleaveComma(traits
, os
<< "\nTraits: ");
181 if (!interfaces
.empty()) {
182 interleaveComma(interfaces
, os
<< "\nInterfaces: ");
185 if (!effects
.empty()) {
186 interleaveComma(effects
, os
<< "\nEffects: ");
191 static StringRef
resolveAttrDescription(const Attribute
&attr
) {
192 StringRef description
= attr
.getDescription();
193 if (description
.empty())
194 return attr
.getBaseAttr().getDescription();
198 static void emitOpDoc(const Operator
&op
, raw_ostream
&os
) {
199 std::string classNameStr
= op
.getQualCppClassName();
200 StringRef className
= classNameStr
;
201 (void)className
.consume_front(stripPrefix
);
202 os
<< formatv("### `{0}` ({1})\n", op
.getOperationName(), className
);
204 // Emit the summary, syntax, and description if present.
206 emitSummary(op
.getSummary(), os
);
207 if (op
.hasAssemblyFormat())
208 emitAssemblyFormat(op
.getOperationName(), op
.getAssemblyFormat().trim(),
210 if (op
.hasDescription())
211 mlir::tblgen::emitDescription(op
.getDescription(), os
);
213 emitOpTraitsDoc(op
, os
);
216 if (op
.getNumAttributes() != 0) {
217 os
<< "\n#### Attributes:\n\n";
218 // Note: This table is HTML rather than markdown so the attribute's
219 // description can appear in an expandable region. The description may be
220 // multiple lines, which is not supported in a markdown table cell.
223 os
<< "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
224 for (const auto &it
: op
.getAttributes()) {
225 StringRef storageType
= it
.attr
.getStorageType();
226 // Name and storage type.
228 os
<< "<td><code>" << it
.name
<< "</code></td><td>" << storageType
230 StringRef description
= resolveAttrDescription(it
.attr
);
231 if (allowHugoSpecificFeatures
&& !description
.empty()) {
232 // Expandable description.
233 // This appears as just the summary, but when clicked shows the full
235 os
<< "<details>" << "<summary>" << it
.attr
.getSummary() << "</summary>"
236 << "{{% markdown %}}" << description
<< "{{% /markdown %}}"
239 // Fallback: Single-line summary.
240 os
<< it
.attr
.getSummary();
242 os
<< "</td></tr>\n";
247 // Emit each of the operands.
248 if (op
.getNumOperands() != 0) {
249 os
<< "\n#### Operands:\n\n";
250 os
<< "| Operand | Description |\n"
251 << "| :-----: | ----------- |\n";
252 for (const auto &it
: op
.getOperands())
253 emitNamedConstraint(it
, os
);
257 if (op
.getNumResults() != 0) {
258 os
<< "\n#### Results:\n\n";
259 os
<< "| Result | Description |\n"
260 << "| :----: | ----------- |\n";
261 for (const auto &it
: op
.getResults())
262 emitNamedConstraint(it
, os
);
266 if (op
.getNumSuccessors() != 0) {
267 os
<< "\n#### Successors:\n\n";
268 os
<< "| Successor | Description |\n"
269 << "| :-------: | ----------- |\n";
270 for (const auto &it
: op
.getSuccessors())
271 emitNamedConstraint(it
, os
);
277 static void emitSourceLink(StringRef inputFilename
, raw_ostream
&os
) {
278 size_t pathBegin
= inputFilename
.find("mlir/include/mlir/");
279 if (pathBegin
== StringRef::npos
)
282 StringRef inputFromMlirInclude
= inputFilename
.substr(pathBegin
);
284 os
<< "[source](https://github.com/llvm/llvm-project/blob/main/"
285 << inputFromMlirInclude
<< ")\n\n";
288 static void emitOpDoc(const RecordKeeper
&records
, raw_ostream
&os
) {
289 auto opDefs
= getRequestedOpDefinitions(records
);
291 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
292 emitSourceLink(records
.getInputFilename(), os
);
293 for (const Record
*opDef
: opDefs
)
294 emitOpDoc(Operator(opDef
), os
);
297 //===----------------------------------------------------------------------===//
298 // Attribute Documentation
299 //===----------------------------------------------------------------------===//
301 static void emitAttrDoc(const Attribute
&attr
, raw_ostream
&os
) {
302 os
<< "### " << attr
.getSummary() << "\n\n";
303 emitDescription(attr
.getDescription(), os
);
307 //===----------------------------------------------------------------------===//
308 // Type Documentation
309 //===----------------------------------------------------------------------===//
311 static void emitTypeDoc(const Type
&type
, raw_ostream
&os
) {
312 os
<< "### " << type
.getSummary() << "\n\n";
313 emitDescription(type
.getDescription(), os
);
317 //===----------------------------------------------------------------------===//
318 // TypeDef Documentation
319 //===----------------------------------------------------------------------===//
321 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef
&def
,
323 ArrayRef
<AttrOrTypeParameter
> parameters
= def
.getParameters();
324 char prefix
= isa
<AttrDef
>(def
) ? '#' : '!';
325 if (parameters
.empty()) {
326 os
<< "\nSyntax: `" << prefix
<< def
.getDialect().getName() << "."
327 << def
.getMnemonic() << "`\n";
331 os
<< "\nSyntax:\n\n```\n"
332 << prefix
<< def
.getDialect().getName() << "." << def
.getMnemonic()
334 for (const auto &it
: llvm::enumerate(parameters
)) {
335 const AttrOrTypeParameter
¶m
= it
.value();
336 os
<< " " << param
.getSyntax();
337 if (it
.index() < (parameters
.size() - 1))
339 os
<< " # " << param
.getName() << "\n";
344 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef
&def
, raw_ostream
&os
) {
345 os
<< formatv("### {0}\n", def
.getCppClassName());
347 // Emit the summary if present.
348 if (def
.hasSummary())
349 os
<< "\n" << def
.getSummary() << "\n";
351 // Emit the syntax if present.
352 if (def
.getMnemonic() && !def
.hasCustomAssemblyFormat())
353 emitAttrOrTypeDefAssemblyFormat(def
, os
);
355 // Emit the description if present.
356 if (def
.hasDescription()) {
358 mlir::tblgen::emitDescription(def
.getDescription(), os
);
361 // Emit parameter documentation.
362 ArrayRef
<AttrOrTypeParameter
> parameters
= def
.getParameters();
363 if (!parameters
.empty()) {
364 os
<< "\n#### Parameters:\n\n";
365 os
<< "| Parameter | C++ type | Description |\n"
366 << "| :-------: | :-------: | ----------- |\n";
367 for (const auto &it
: parameters
) {
368 auto desc
= it
.getSummary();
369 os
<< "| " << it
.getName() << " | `" << it
.getCppType() << "` | "
370 << (desc
? *desc
: "") << " |\n";
377 static void emitAttrOrTypeDefDoc(const RecordKeeper
&records
, raw_ostream
&os
,
378 StringRef recordTypeName
) {
379 auto defs
= records
.getAllDerivedDefinitions(recordTypeName
);
381 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
382 for (const Record
*def
: defs
)
383 emitAttrOrTypeDefDoc(AttrOrTypeDef(def
), os
);
386 //===----------------------------------------------------------------------===//
387 // Enum Documentation
388 //===----------------------------------------------------------------------===//
390 static void emitEnumDoc(const EnumAttr
&def
, raw_ostream
&os
) {
391 os
<< formatv("### {0}\n", def
.getEnumClassName());
393 // Emit the summary if present.
394 if (!def
.getSummary().empty())
395 os
<< "\n" << def
.getSummary() << "\n";
397 // Emit case documentation.
398 std::vector
<EnumAttrCase
> cases
= def
.getAllCases();
399 os
<< "\n#### Cases:\n\n";
400 os
<< "| Symbol | Value | String |\n"
401 << "| :----: | :---: | ------ |\n";
402 for (const auto &it
: cases
) {
403 os
<< "| " << it
.getSymbol() << " | `" << it
.getValue() << "` | "
404 << it
.getStr() << " |\n";
410 static void emitEnumDoc(const RecordKeeper
&records
, raw_ostream
&os
) {
411 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
412 for (const Record
*def
: records
.getAllDerivedDefinitions("EnumAttr"))
413 emitEnumDoc(EnumAttr(def
), os
);
416 //===----------------------------------------------------------------------===//
417 // Dialect Documentation
418 //===----------------------------------------------------------------------===//
421 const Dialect
&getDialect() const { return ops
.front().getDialect(); }
423 // Returns the summary description of the section.
424 std::string summary
= "";
426 // Returns the description of the section.
427 StringRef description
= "";
429 // Instances inside the section.
430 std::vector
<Operator
> ops
;
433 static void maybeNest(bool nest
, llvm::function_ref
<void(raw_ostream
&os
)> fn
,
436 raw_string_ostream
ss(str
);
438 for (StringRef x
: llvm::split(str
, "\n")) {
439 if (nest
&& x
.starts_with("#"))
445 static void emitBlock(ArrayRef
<Attribute
> attributes
, StringRef inputFilename
,
446 ArrayRef
<AttrDef
> attrDefs
, ArrayRef
<OpDocGroup
> ops
,
447 ArrayRef
<Type
> types
, ArrayRef
<TypeDef
> typeDefs
,
448 ArrayRef
<EnumAttr
> enums
, raw_ostream
&os
) {
450 os
<< "## Operations\n\n";
451 emitSourceLink(inputFilename
, os
);
452 for (const OpDocGroup
&grouping
: ops
) {
453 bool nested
= !grouping
.summary
.empty();
456 [&](raw_ostream
&os
) {
458 os
<< "## " << StringRef(grouping
.summary
).trim() << "\n\n";
459 emitDescription(grouping
.description
, os
);
462 for (const Operator
&op
: grouping
.ops
) {
470 if (!attributes
.empty()) {
471 os
<< "## Attribute constraints\n\n";
472 for (const Attribute
&attr
: attributes
)
473 emitAttrDoc(attr
, os
);
476 if (!attrDefs
.empty()) {
477 os
<< "## Attributes\n\n";
478 for (const AttrDef
&def
: attrDefs
)
479 emitAttrOrTypeDefDoc(def
, os
);
482 // TODO: Add link between use and def for types
483 if (!types
.empty()) {
484 os
<< "## Type constraints\n\n";
485 for (const Type
&type
: types
)
486 emitTypeDoc(type
, os
);
489 if (!typeDefs
.empty()) {
490 os
<< "## Types\n\n";
491 for (const TypeDef
&def
: typeDefs
)
492 emitAttrOrTypeDefDoc(def
, os
);
495 if (!enums
.empty()) {
496 os
<< "## Enums\n\n";
497 for (const EnumAttr
&def
: enums
)
498 emitEnumDoc(def
, os
);
502 static void emitDialectDoc(const Dialect
&dialect
, StringRef inputFilename
,
503 ArrayRef
<Attribute
> attributes
,
504 ArrayRef
<AttrDef
> attrDefs
, ArrayRef
<OpDocGroup
> ops
,
505 ArrayRef
<Type
> types
, ArrayRef
<TypeDef
> typeDefs
,
506 ArrayRef
<EnumAttr
> enums
, raw_ostream
&os
) {
507 os
<< "# '" << dialect
.getName() << "' Dialect\n\n";
508 emitIfNotEmpty(dialect
.getSummary(), os
);
509 emitIfNotEmpty(dialect
.getDescription(), os
);
511 // Generate a TOC marker except if description already contains one.
512 Regex
r("^[[:space:]]*\\[TOC\\]$", Regex::RegexFlags::Newline
);
513 if (!r
.match(dialect
.getDescription()))
516 emitBlock(attributes
, inputFilename
, attrDefs
, ops
, types
, typeDefs
, enums
,
520 static bool emitDialectDoc(const RecordKeeper
&records
, raw_ostream
&os
) {
521 auto dialectDefs
= records
.getAllDerivedDefinitionsIfDefined("Dialect");
522 SmallVector
<Dialect
> dialects(dialectDefs
.begin(), dialectDefs
.end());
523 std::optional
<Dialect
> dialect
= findDialectToGenerate(dialects
);
527 std::vector
<const Record
*> opDefs
= getRequestedOpDefinitions(records
);
528 auto attrDefs
= records
.getAllDerivedDefinitionsIfDefined("DialectAttr");
529 auto typeDefs
= records
.getAllDerivedDefinitionsIfDefined("DialectType");
530 auto typeDefDefs
= records
.getAllDerivedDefinitionsIfDefined("TypeDef");
531 auto attrDefDefs
= records
.getAllDerivedDefinitionsIfDefined("AttrDef");
532 auto enumDefs
= records
.getAllDerivedDefinitionsIfDefined("EnumAttrInfo");
534 std::vector
<Attribute
> dialectAttrs
;
535 std::vector
<AttrDef
> dialectAttrDefs
;
536 std::vector
<OpDocGroup
> dialectOps
;
537 std::vector
<Type
> dialectTypes
;
538 std::vector
<TypeDef
> dialectTypeDefs
;
539 std::vector
<EnumAttr
> dialectEnums
;
541 SmallDenseSet
<const Record
*> seen
;
542 auto addIfNotSeen
= [&](const Record
*record
, const auto &def
, auto &vec
) {
543 if (seen
.insert(record
).second
) {
549 auto addIfInDialect
= [&](const Record
*record
, const auto &def
, auto &vec
) {
550 return def
.getDialect() == *dialect
&& addIfNotSeen(record
, def
, vec
);
553 SmallDenseMap
<const Record
*, OpDocGroup
> opDocGroup
;
555 for (const Record
*def
: attrDefDefs
)
556 addIfInDialect(def
, AttrDef(def
), dialectAttrDefs
);
557 for (const Record
*def
: attrDefs
)
558 addIfInDialect(def
, Attribute(def
), dialectAttrs
);
559 for (const Record
*def
: opDefs
) {
560 if (const Record
*group
= def
->getValueAsOptionalDef("opDocGroup")) {
561 OpDocGroup
&op
= opDocGroup
[group
];
562 addIfInDialect(def
, Operator(def
), op
.ops
);
565 op
.ops
.emplace_back(def
);
566 addIfInDialect(def
, op
, dialectOps
);
569 for (const Record
*rec
:
570 records
.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
571 if (opDocGroup
[rec
].ops
.empty())
573 opDocGroup
[rec
].summary
= rec
->getValueAsString("summary");
574 opDocGroup
[rec
].description
= rec
->getValueAsString("description");
575 dialectOps
.push_back(opDocGroup
[rec
]);
577 for (const Record
*def
: typeDefDefs
)
578 addIfInDialect(def
, TypeDef(def
), dialectTypeDefs
);
579 for (const Record
*def
: typeDefs
)
580 addIfInDialect(def
, Type(def
), dialectTypes
);
581 dialectEnums
.reserve(enumDefs
.size());
582 for (const Record
*def
: enumDefs
)
583 addIfNotSeen(def
, EnumAttr(def
), dialectEnums
);
585 // Sort alphabetically ignorning dialect for ops and section name for
587 // TODO: The sorting order could be revised, currently attempting to sort of
588 // keep in alphabetical order.
589 std::sort(dialectOps
.begin(), dialectOps
.end(),
590 [](const OpDocGroup
&lhs
, const OpDocGroup
&rhs
) {
591 auto getDesc
= [](const OpDocGroup
&arg
) -> StringRef
{
592 if (!arg
.summary
.empty())
594 return arg
.ops
.front().getDef().getValueAsString("opName");
596 return getDesc(lhs
).compare_insensitive(getDesc(rhs
)) < 0;
599 os
<< "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
600 emitDialectDoc(*dialect
, records
.getInputFilename(), dialectAttrs
,
601 dialectAttrDefs
, dialectOps
, dialectTypes
, dialectTypeDefs
,
606 //===----------------------------------------------------------------------===//
608 //===----------------------------------------------------------------------===//
610 static mlir::GenRegistration
611 genAttrRegister("gen-attrdef-doc",
612 "Generate dialect attribute documentation",
613 [](const RecordKeeper
&records
, raw_ostream
&os
) {
614 emitAttrOrTypeDefDoc(records
, os
, "AttrDef");
618 static mlir::GenRegistration
619 genOpRegister("gen-op-doc", "Generate dialect documentation",
620 [](const RecordKeeper
&records
, raw_ostream
&os
) {
621 emitOpDoc(records
, os
);
625 static mlir::GenRegistration
626 genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
627 [](const RecordKeeper
&records
, raw_ostream
&os
) {
628 emitAttrOrTypeDefDoc(records
, os
, "TypeDef");
632 static mlir::GenRegistration
633 genEnumRegister("gen-enum-doc", "Generate dialect enum documentation",
634 [](const RecordKeeper
&records
, raw_ostream
&os
) {
635 emitEnumDoc(records
, os
);
639 static mlir::GenRegistration
640 genRegister("gen-dialect-doc", "Generate dialect documentation",
641 [](const RecordKeeper
&records
, raw_ostream
&os
) {
642 return emitDialectDoc(records
, os
);