mlir/tools/mlir-tblgen/OpDocGen.cpp

   1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // OpDocGen uses the description of operations to generate documentation for the
  10 // operations.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "DialectGenUtilities.h"
  15 #include "DocGenUtilities.h"
  16 #include "OpGenHelpers.h"
  17 #include "mlir/Support/IndentedOstream.h"
  18 #include "mlir/TableGen/AttrOrTypeDef.h"
  19 #include "mlir/TableGen/GenInfo.h"
  20 #include "mlir/TableGen/Operator.h"
  21 #include "llvm/ADT/DenseMap.h"
  22 #include "llvm/ADT/SetVector.h"
  23 #include "llvm/ADT/StringExtras.h"
  24 #include "llvm/ADT/StringRef.h"
  25 #include "llvm/Support/CommandLine.h"
  26 #include "llvm/Support/FormatVariadic.h"
  27 #include "llvm/Support/Regex.h"
  28 #include "llvm/Support/Signals.h"
  29 #include "llvm/TableGen/Error.h"
  30 #include "llvm/TableGen/Record.h"
  31 #include "llvm/TableGen/TableGenBackend.h"
  32
  33 #include <set>
  34 #include <string>
  35
  36 //===----------------------------------------------------------------------===//
  37 // Commandline Options
  38 //===----------------------------------------------------------------------===//
  39 static llvm::cl::OptionCategory
  40     docCat("Options for -gen-(attrdef|typedef|op|dialect)-doc");
  41 llvm::cl::opt<std::string>
  42     stripPrefix("strip-prefix",
  43                 llvm::cl::desc("Strip prefix of the fully qualified names"),
  44                 llvm::cl::init("::mlir::"), llvm::cl::cat(docCat));
  45 llvm::cl::opt<bool> allowHugoSpecificFeatures(
  46     "allow-hugo-specific-features",
  47     llvm::cl::desc("Allows using features specific to Hugo"),
  48     llvm::cl::init(false), llvm::cl::cat(docCat));
  49
  50 using namespace llvm;
  51 using namespace mlir;
  52 using namespace mlir::tblgen;
  53 using mlir::tblgen::Operator;
  54
  55 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
  56   if (!summary.empty()) {
  57     llvm::StringRef trimmed = summary.trim();
  58     char first = std::toupper(trimmed.front());
  59     llvm::StringRef rest = trimmed.drop_front();
  60     os << "\n_" << first << rest << "_\n\n";
  61   }
  62 }
  63
  64 // Emit the description by aligning the text to the left per line (e.g.,
  65 // removing the minimum indentation across the block).
  66 //
  67 // This expects that the description in the tablegen file is already formatted
  68 // in a way the user wanted but has some additional indenting due to being
  69 // nested in the op definition.
  70 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
  71   raw_indented_ostream ros(os);
  72   ros.printReindented(description.rtrim(" \t"));
  73 }
  74
  75 void mlir::tblgen::emitDescriptionComment(StringRef description,
  76                                           raw_ostream &os, StringRef prefix) {
  77   if (description.empty())
  78     return;
  79   raw_indented_ostream ros(os);
  80   StringRef trimmed = description.rtrim(" \t");
  81   ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
  82   if (!trimmed.endswith("\n"))
  83     ros << "\n";
  84 }
  85
  86 // Emits `str` with trailing newline if not empty.
  87 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
  88   if (!str.empty()) {
  89     emitDescription(str, os);
  90     os << "\n";
  91   }
  92 }
  93
  94 /// Emit the given named constraint.
  95 template <typename T>
  96 static void emitNamedConstraint(const T &it, raw_ostream &os) {
  97   if (!it.name.empty())
  98     os << "| `" << it.name << "`";
  99   else
 100     os << "&laquo;unnamed&raquo;";
 101   os << " | " << it.constraint.getSummary() << "\n";
 102 }
 103
 104 //===----------------------------------------------------------------------===//
 105 // Operation Documentation
 106 //===----------------------------------------------------------------------===//
 107
 108 /// Emit the assembly format of an operation.
 109 static void emitAssemblyFormat(StringRef opName, StringRef format,
 110                                raw_ostream &os) {
 111   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
 112
 113   // Print the assembly format aligned.
 114   unsigned indent = strlen("operation ::= ");
 115   std::pair<StringRef, StringRef> split = format.split('\n');
 116   os << split.first.trim() << "\n";
 117   do {
 118     split = split.second.split('\n');
 119     StringRef formatChunk = split.first.trim();
 120     if (!formatChunk.empty())
 121       os.indent(indent) << formatChunk << "\n";
 122   } while (!split.second.empty());
 123   os << "```\n\n";
 124 }
 125
 126 static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
 127   // TODO: We should link to the trait/documentation of it. That also means we
 128   // should add descriptions to traits that can be queried.
 129   // Collect using set to sort effects, interfaces & traits.
 130   std::set<std::string> effects, interfaces, traits;
 131   for (auto &trait : op.getTraits()) {
 132     if (isa<PredTrait>(&trait))
 133       continue;
 134
 135     std::string name = trait.getDef().getName().str();
 136     StringRef ref = name;
 137     StringRef traitName = trait.getDef().getValueAsString("trait");
 138     traitName.consume_back("::Trait");
 139     traitName.consume_back("::Impl");
 140     if (ref.startswith("anonymous_"))
 141       name = traitName.str();
 142     if (isa<InterfaceTrait>(&trait)) {
 143       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
 144         auto effectName = trait.getDef().getValueAsString("baseEffectName");
 145         effectName.consume_front("::");
 146         effectName.consume_front("mlir::");
 147         std::string effectStr;
 148         llvm::raw_string_ostream os(effectStr);
 149         os << effectName << "{";
 150         auto list = trait.getDef().getValueAsListOfDefs("effects");
 151         llvm::interleaveComma(list, os, [&](Record *rec) {
 152           StringRef effect = rec->getValueAsString("effect");
 153           effect.consume_front("::");
 154           effect.consume_front("mlir::");
 155           os << effect << " on " << rec->getValueAsString("resource");
 156         });
 157         os << "}";
 158         effects.insert(os.str());
 159         name.append(llvm::formatv(" ({0})", traitName).str());
 160       }
 161       interfaces.insert(name);
 162       continue;
 163     }
 164
 165     traits.insert(name);
 166   }
 167   if (!traits.empty()) {
 168     llvm::interleaveComma(traits, os << "\nTraits: ");
 169     os << "\n";
 170   }
 171   if (!interfaces.empty()) {
 172     llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
 173     os << "\n";
 174   }
 175   if (!effects.empty()) {
 176     llvm::interleaveComma(effects, os << "\nEffects: ");
 177     os << "\n";
 178   }
 179 }
 180
 181 static StringRef resolveAttrDescription(const Attribute &attr) {
 182   StringRef description = attr.getDescription();
 183   if (description.empty())
 184     return attr.getBaseAttr().getDescription();
 185   return description;
 186 }
 187
 188 static void emitOpDoc(const Operator &op, raw_ostream &os) {
 189   std::string classNameStr = op.getQualCppClassName();
 190   StringRef className = classNameStr;
 191   (void)className.consume_front(stripPrefix);
 192   os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(), className);
 193
 194   // Emit the summary, syntax, and description if present.
 195   if (op.hasSummary())
 196     emitSummary(op.getSummary(), os);
 197   if (op.hasAssemblyFormat())
 198     emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),
 199                        os);
 200   if (op.hasDescription())
 201     mlir::tblgen::emitDescription(op.getDescription(), os);
 202
 203   emitOpTraitsDoc(op, os);
 204
 205   // Emit attributes.
 206   if (op.getNumAttributes() != 0) {
 207     os << "\n#### Attributes:\n\n";
 208     // Note: This table is HTML rather than markdown so the attribute's
 209     // description can appear in an expandable region. The description may be
 210     // multiple lines, which is not supported in a markdown table cell.
 211     os << "<table>\n";
 212     // Header.
 213     os << "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
 214     for (const auto &it : op.getAttributes()) {
 215       StringRef storageType = it.attr.getStorageType();
 216       // Name and storage type.
 217       os << "<tr>";
 218       os << "<td><code>" << it.name << "</code></td><td>" << storageType
 219          << "</td><td>";
 220       StringRef description = resolveAttrDescription(it.attr);
 221       if (allowHugoSpecificFeatures && !description.empty()) {
 222         // Expandable description.
 223         // This appears as just the summary, but when clicked shows the full
 224         // description.
 225         os << "<details>"
 226            << "<summary>" << it.attr.getSummary() << "</summary>"
 227            << "{{% markdown %}}" << description << "{{% /markdown %}}"
 228            << "</details>";
 229       } else {
 230         // Fallback: Single-line summary.
 231         os << it.attr.getSummary();
 232       }
 233       os << "</td></tr>\n";
 234     }
 235     os << "</table>\n";
 236   }
 237
 238   // Emit each of the operands.
 239   if (op.getNumOperands() != 0) {
 240     os << "\n#### Operands:\n\n";
 241     os << "| Operand | Description |\n"
 242        << "| :-----: | ----------- |\n";
 243     for (const auto &it : op.getOperands())
 244       emitNamedConstraint(it, os);
 245   }
 246
 247   // Emit results.
 248   if (op.getNumResults() != 0) {
 249     os << "\n#### Results:\n\n";
 250     os << "| Result | Description |\n"
 251        << "| :----: | ----------- |\n";
 252     for (const auto &it : op.getResults())
 253       emitNamedConstraint(it, os);
 254   }
 255
 256   // Emit successors.
 257   if (op.getNumSuccessors() != 0) {
 258     os << "\n#### Successors:\n\n";
 259     os << "| Successor | Description |\n"
 260        << "| :-------: | ----------- |\n";
 261     for (const auto &it : op.getSuccessors())
 262       emitNamedConstraint(it, os);
 263   }
 264
 265   os << "\n";
 266 }
 267
 268 static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
 269   auto opDefs = getRequestedOpDefinitions(recordKeeper);
 270
 271   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
 272   for (const llvm::Record *opDef : opDefs)
 273     emitOpDoc(Operator(opDef), os);
 274 }
 275
 276 //===----------------------------------------------------------------------===//
 277 // Attribute Documentation
 278 //===----------------------------------------------------------------------===//
 279
 280 static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
 281   os << "### " << attr.getSummary() << "\n\n";
 282   emitDescription(attr.getDescription(), os);
 283   os << "\n\n";
 284 }
 285
 286 //===----------------------------------------------------------------------===//
 287 // Type Documentation
 288 //===----------------------------------------------------------------------===//
 289
 290 static void emitTypeDoc(const Type &type, raw_ostream &os) {
 291   os << "### " << type.getSummary() << "\n\n";
 292   emitDescription(type.getDescription(), os);
 293   os << "\n\n";
 294 }
 295
 296 //===----------------------------------------------------------------------===//
 297 // TypeDef Documentation
 298 //===----------------------------------------------------------------------===//
 299
 300 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
 301                                             raw_ostream &os) {
 302   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
 303   char prefix = isa<AttrDef>(def) ? '#' : '!';
 304   if (parameters.empty()) {
 305     os << "\nSyntax: `" << prefix << def.getDialect().getName() << "."
 306        << def.getMnemonic() << "`\n";
 307     return;
 308   }
 309
 310   os << "\nSyntax:\n\n```\n"
 311      << prefix << def.getDialect().getName() << "." << def.getMnemonic()
 312      << "<\n";
 313   for (const auto &it : llvm::enumerate(parameters)) {
 314     const AttrOrTypeParameter &param = it.value();
 315     os << "  " << param.getSyntax();
 316     if (it.index() < (parameters.size() - 1))
 317       os << ",";
 318     os << "   # " << param.getName() << "\n";
 319   }
 320   os << ">\n```\n";
 321 }
 322
 323 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
 324   os << llvm::formatv("### {0}\n", def.getCppClassName());
 325
 326   // Emit the summary if present.
 327   if (def.hasSummary())
 328     os << "\n" << def.getSummary() << "\n";
 329
 330   // Emit the syntax if present.
 331   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
 332     emitAttrOrTypeDefAssemblyFormat(def, os);
 333
 334   // Emit the description if present.
 335   if (def.hasDescription()) {
 336     os << "\n";
 337     mlir::tblgen::emitDescription(def.getDescription(), os);
 338   }
 339
 340   // Emit parameter documentation.
 341   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
 342   if (!parameters.empty()) {
 343     os << "\n#### Parameters:\n\n";
 344     os << "| Parameter | C++ type | Description |\n"
 345        << "| :-------: | :-------: | ----------- |\n";
 346     for (const auto &it : parameters) {
 347       auto desc = it.getSummary();
 348       os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
 349          << (desc ? *desc : "") << " |\n";
 350     }
 351   }
 352
 353   os << "\n";
 354 }
 355
 356 static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
 357                                  raw_ostream &os, StringRef recordTypeName) {
 358   std::vector<llvm::Record *> defs =
 359       recordKeeper.getAllDerivedDefinitions(recordTypeName);
 360
 361   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
 362   for (const llvm::Record *def : defs)
 363     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
 364 }
 365
 366 //===----------------------------------------------------------------------===//
 367 // Dialect Documentation
 368 //===----------------------------------------------------------------------===//
 369
 370 struct OpDocGroup {
 371   const Dialect &getDialect() const { return ops.front().getDialect(); }
 372
 373   // Returns the summary description of the section.
 374   std::string summary = "";
 375
 376   // Returns the description of the section.
 377   StringRef description = "";
 378
 379   // Instances inside the section.
 380   std::vector<Operator> ops;
 381 };
 382
 383 static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,
 384                       raw_ostream &os) {
 385   std::string str;
 386   llvm::raw_string_ostream ss(str);
 387   fn(ss);
 388   for (StringRef x : llvm::split(ss.str(), "\n")) {
 389     if (nest && x.starts_with("#"))
 390       os << "#";
 391     os << x << "\n";
 392   }
 393 }
 394
 395 static void emitBlock(ArrayRef<Attribute> attributes,
 396                       ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
 397                       ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
 398                       raw_ostream &os) {
 399   if (!ops.empty()) {
 400     os << "## Operation definition\n\n";
 401     for (const OpDocGroup &grouping : ops) {
 402       bool nested = !grouping.summary.empty();
 403       maybeNest(
 404           nested,
 405           [&](raw_ostream &os) {
 406             if (nested) {
 407               os << "## " << StringRef(grouping.summary).trim() << "\n\n";
 408               emitDescription(grouping.description, os);
 409               os << "\n\n";
 410             }
 411             for (const Operator &op : grouping.ops) {
 412               emitOpDoc(op, os);
 413             }
 414           },
 415           os);
 416     }
 417   }
 418
 419   if (!attributes.empty()) {
 420     os << "## Attribute constraint definition\n\n";
 421     for (const Attribute &attr : attributes)
 422       emitAttrDoc(attr, os);
 423   }
 424
 425   if (!attrDefs.empty()) {
 426     os << "## Attribute definition\n\n";
 427     for (const AttrDef &def : attrDefs)
 428       emitAttrOrTypeDefDoc(def, os);
 429   }
 430
 431   // TODO: Add link between use and def for types
 432   if (!types.empty()) {
 433     os << "## Type constraint definition\n\n";
 434     for (const Type &type : types)
 435       emitTypeDoc(type, os);
 436   }
 437
 438   if (!typeDefs.empty()) {
 439     os << "## Type definition\n\n";
 440     for (const TypeDef &def : typeDefs)
 441       emitAttrOrTypeDefDoc(def, os);
 442   }
 443 }
 444
 445 static void emitDialectDoc(const Dialect &dialect,
 446                            ArrayRef<Attribute> attributes,
 447                            ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
 448                            ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
 449                            raw_ostream &os) {
 450   os << "# '" << dialect.getName() << "' Dialect\n\n";
 451   emitIfNotEmpty(dialect.getSummary(), os);
 452   emitIfNotEmpty(dialect.getDescription(), os);
 453
 454   // Generate a TOC marker except if description already contains one.
 455   llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
 456   if (!r.match(dialect.getDescription()))
 457     os << "[TOC]\n\n";
 458
 459   emitBlock(attributes, attrDefs, ops, types, typeDefs, os);
 460 }
 461
 462 static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
 463   std::vector<Record *> dialectDefs =
 464       recordKeeper.getAllDerivedDefinitionsIfDefined("Dialect");
 465   SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());
 466   std::optional<Dialect> dialect = findDialectToGenerate(dialects);
 467   if (!dialect)
 468     return true;
 469
 470   std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
 471   std::vector<Record *> attrDefs =
 472       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectAttr");
 473   std::vector<Record *> typeDefs =
 474       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
 475   std::vector<Record *> typeDefDefs =
 476       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
 477   std::vector<Record *> attrDefDefs =
 478       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
 479
 480   std::vector<Attribute> dialectAttrs;
 481   std::vector<AttrDef> dialectAttrDefs;
 482   std::vector<OpDocGroup> dialectOps;
 483   std::vector<Type> dialectTypes;
 484   std::vector<TypeDef> dialectTypeDefs;
 485
 486   llvm::SmallDenseSet<Record *> seen;
 487   auto addIfInDialect = [&](llvm::Record *record, const auto &def, auto &vec) {
 488     if (seen.insert(record).second && def.getDialect() == *dialect) {
 489       vec.push_back(def);
 490       return true;
 491     }
 492     return false;
 493   };
 494
 495   SmallDenseMap<Record *, OpDocGroup> opDocGroup;
 496
 497   for (Record *def : attrDefDefs)
 498     addIfInDialect(def, AttrDef(def), dialectAttrDefs);
 499   for (Record *def : attrDefs)
 500     addIfInDialect(def, Attribute(def), dialectAttrs);
 501   for (Record *def : opDefs) {
 502     if (Record *group = def->getValueAsOptionalDef("opDocGroup")) {
 503       OpDocGroup &op = opDocGroup[group];
 504       addIfInDialect(def, Operator(def), op.ops);
 505     } else {
 506       OpDocGroup op;
 507       op.ops.emplace_back(def);
 508       addIfInDialect(def, op, dialectOps);
 509     }
 510   }
 511   for (Record *rec :
 512        recordKeeper.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
 513     if (opDocGroup[rec].ops.empty())
 514       continue;
 515     opDocGroup[rec].summary = rec->getValueAsString("summary");
 516     opDocGroup[rec].description = rec->getValueAsString("description");
 517     dialectOps.push_back(opDocGroup[rec]);
 518   }
 519   for (Record *def : typeDefDefs)
 520     addIfInDialect(def, TypeDef(def), dialectTypeDefs);
 521   for (Record *def : typeDefs)
 522     addIfInDialect(def, Type(def), dialectTypes);
 523
 524   // Sort alphabetically ignorning dialect for ops and section name for
 525   // sections.
 526   // TODO: The sorting order could be revised, currently attempting to sort of
 527   // keep in alphabetical order.
 528   std::sort(dialectOps.begin(), dialectOps.end(),
 529             [](const OpDocGroup &lhs, const OpDocGroup &rhs) {
 530               auto getDesc = [](const OpDocGroup &arg) -> StringRef {
 531                 if (!arg.summary.empty())
 532                   return arg.summary;
 533                 return arg.ops.front().getDef().getValueAsString("opName");
 534               };
 535               return getDesc(lhs).compare_insensitive(getDesc(rhs)) < 0;
 536             });
 537
 538   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
 539   emitDialectDoc(*dialect, dialectAttrs, dialectAttrDefs, dialectOps,
 540                  dialectTypes, dialectTypeDefs, os);
 541   return false;
 542 }
 543
 544 //===----------------------------------------------------------------------===//
 545 // Gen Registration
 546 //===----------------------------------------------------------------------===//
 547
 548 static mlir::GenRegistration
 549     genAttrRegister("gen-attrdef-doc",
 550                     "Generate dialect attribute documentation",
 551                     [](const RecordKeeper &records, raw_ostream &os) {
 552                       emitAttrOrTypeDefDoc(records, os, "AttrDef");
 553                       return false;
 554                     });
 555
 556 static mlir::GenRegistration
 557     genOpRegister("gen-op-doc", "Generate dialect documentation",
 558                   [](const RecordKeeper &records, raw_ostream &os) {
 559                     emitOpDoc(records, os);
 560                     return false;
 561                   });
 562
 563 static mlir::GenRegistration
 564     genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
 565                     [](const RecordKeeper &records, raw_ostream &os) {
 566                       emitAttrOrTypeDefDoc(records, os, "TypeDef");
 567                       return false;
 568                     });
 569
 570 static mlir::GenRegistration
 571     genRegister("gen-dialect-doc", "Generate dialect documentation",
 572                 [](const RecordKeeper &records, raw_ostream &os) {
 573                   return emitDialectDoc(records, os);
 574                 });