1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // OpDocGen uses the description of operations to generate documentation for the
10 // operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "DialectGenUtilities.h"
15 #include "DocGenUtilities.h"
16 #include "OpGenHelpers.h"
17 #include "mlir/Support/IndentedOstream.h"
18 #include "mlir/TableGen/AttrOrTypeDef.h"
19 #include "mlir/TableGen/GenInfo.h"
20 #include "mlir/TableGen/Operator.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/FormatVariadic.h"
26 #include "llvm/Support/Regex.h"
27 #include "llvm/Support/Signals.h"
28 #include "llvm/TableGen/Error.h"
29 #include "llvm/TableGen/Record.h"
30 #include "llvm/TableGen/TableGenBackend.h"
31 
32 #include <set>
33 
34 using namespace llvm;
35 using namespace mlir;
36 using namespace mlir::tblgen;
37 
38 using mlir::tblgen::Operator;
39 
40 // Emit the description by aligning the text to the left per line (e.g.,
41 // removing the minimum indentation across the block).
42 //
43 // This expects that the description in the tablegen file is already formatted
44 // in a way the user wanted but has some additional indenting due to being
45 // nested in the op definition.
emitDescription(StringRef description,raw_ostream & os)46 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
47   raw_indented_ostream ros(os);
48   ros.printReindented(description.rtrim(" \t"));
49 }
50 
51 // Emits `str` with trailing newline if not empty.
emitIfNotEmpty(StringRef str,raw_ostream & os)52 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
53   if (!str.empty()) {
54     emitDescription(str, os);
55     os << "\n";
56   }
57 }
58 
59 /// Emit the given named constraint.
60 template <typename T>
emitNamedConstraint(const T & it,raw_ostream & os)61 static void emitNamedConstraint(const T &it, raw_ostream &os) {
62   if (!it.name.empty())
63     os << "| `" << it.name << "`";
64   else
65     os << "&laquo;unnamed&raquo;";
66   os << " | " << it.constraint.getSummary() << "\n";
67 }
68 
69 //===----------------------------------------------------------------------===//
70 // Operation Documentation
71 //===----------------------------------------------------------------------===//
72 
73 /// Emit the assembly format of an operation.
emitAssemblyFormat(StringRef opName,StringRef format,raw_ostream & os)74 static void emitAssemblyFormat(StringRef opName, StringRef format,
75                                raw_ostream &os) {
76   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
77 
78   // Print the assembly format aligned.
79   unsigned indent = strlen("operation ::= ");
80   std::pair<StringRef, StringRef> split = format.split('\n');
81   os << split.first.trim() << "\n";
82   do {
83     split = split.second.split('\n');
84     StringRef formatChunk = split.first.trim();
85     if (!formatChunk.empty())
86       os.indent(indent) << formatChunk << "\n";
87   } while (!split.second.empty());
88   os << "```\n\n";
89 }
90 
emitOpTraitsDoc(const Operator & op,raw_ostream & os)91 static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
92   // TODO: We should link to the trait/documentation of it. That also means we
93   // should add descriptions to traits that can be queried.
94   // Collect using set to sort effects, interfaces & traits.
95   std::set<std::string> effects, interfaces, traits;
96   for (auto &trait : op.getTraits()) {
97     if (isa<PredTrait>(&trait))
98       continue;
99 
100     std::string name = trait.getDef().getName().str();
101     StringRef ref = name;
102     StringRef traitName = trait.getDef().getValueAsString("trait");
103     traitName.consume_back("::Trait");
104     traitName.consume_back("::Impl");
105     if (ref.startswith("anonymous_"))
106       name = traitName.str();
107     if (isa<InterfaceTrait>(&trait)) {
108       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
109         auto effectName = trait.getDef().getValueAsString("baseEffectName");
110         effectName.consume_front("::");
111         effectName.consume_front("mlir::");
112         std::string effectStr;
113         llvm::raw_string_ostream os(effectStr);
114         os << effectName << "{";
115         auto list = trait.getDef().getValueAsListOfDefs("effects");
116         llvm::interleaveComma(list, os, [&](Record *rec) {
117           StringRef effect = rec->getValueAsString("effect");
118           effect.consume_front("::");
119           effect.consume_front("mlir::");
120           os << effect << " on " << rec->getValueAsString("resource");
121         });
122         os << "}";
123         effects.insert(os.str());
124         name.append(llvm::formatv(" ({0})", traitName).str());
125       }
126       interfaces.insert(name);
127       continue;
128     }
129 
130     traits.insert(name);
131   }
132   if (!traits.empty()) {
133     llvm::interleaveComma(traits, os << "\nTraits: ");
134     os << "\n";
135   }
136   if (!interfaces.empty()) {
137     llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
138     os << "\n";
139   }
140   if (!effects.empty()) {
141     llvm::interleaveComma(effects, os << "\nEffects: ");
142     os << "\n";
143   }
144 }
145 
emitOpDoc(const Operator & op,raw_ostream & os)146 static void emitOpDoc(const Operator &op, raw_ostream &os) {
147   os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(),
148                       op.getQualCppClassName());
149 
150   // Emit the summary, syntax, and description if present.
151   if (op.hasSummary())
152     os << "\n" << op.getSummary() << "\n\n";
153   if (op.hasAssemblyFormat())
154     emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),
155                        os);
156   if (op.hasDescription())
157     mlir::tblgen::emitDescription(op.getDescription(), os);
158 
159   emitOpTraitsDoc(op, os);
160 
161   // Emit attributes.
162   if (op.getNumAttributes() != 0) {
163     // TODO: Attributes are only documented by TableGen name, with no further
164     // info. This should be improved.
165     os << "\n#### Attributes:\n\n";
166     os << "| Attribute | MLIR Type | Description |\n"
167        << "| :-------: | :-------: | ----------- |\n";
168     for (const auto &it : op.getAttributes()) {
169       StringRef storageType = it.attr.getStorageType();
170       os << "| `" << it.name << "` | " << storageType << " | "
171          << it.attr.getSummary() << "\n";
172     }
173   }
174 
175   // Emit each of the operands.
176   if (op.getNumOperands() != 0) {
177     os << "\n#### Operands:\n\n";
178     os << "| Operand | Description |\n"
179        << "| :-----: | ----------- |\n";
180     for (const auto &it : op.getOperands())
181       emitNamedConstraint(it, os);
182   }
183 
184   // Emit results.
185   if (op.getNumResults() != 0) {
186     os << "\n#### Results:\n\n";
187     os << "| Result | Description |\n"
188        << "| :----: | ----------- |\n";
189     for (const auto &it : op.getResults())
190       emitNamedConstraint(it, os);
191   }
192 
193   // Emit successors.
194   if (op.getNumSuccessors() != 0) {
195     os << "\n#### Successors:\n\n";
196     os << "| Successor | Description |\n"
197        << "| :-------: | ----------- |\n";
198     for (const auto &it : op.getSuccessors())
199       emitNamedConstraint(it, os);
200   }
201 
202   os << "\n";
203 }
204 
emitOpDoc(const RecordKeeper & recordKeeper,raw_ostream & os)205 static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
206   auto opDefs = getRequestedOpDefinitions(recordKeeper);
207 
208   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
209   for (const llvm::Record *opDef : opDefs)
210     emitOpDoc(Operator(opDef), os);
211 }
212 
213 //===----------------------------------------------------------------------===//
214 // Attribute Documentation
215 //===----------------------------------------------------------------------===//
216 
emitAttrDoc(const Attribute & attr,raw_ostream & os)217 static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
218   os << "### " << attr.getSummary() << "\n\n";
219   emitDescription(attr.getDescription(), os);
220   os << "\n\n";
221 }
222 
223 //===----------------------------------------------------------------------===//
224 // Type Documentation
225 //===----------------------------------------------------------------------===//
226 
emitTypeDoc(const Type & type,raw_ostream & os)227 static void emitTypeDoc(const Type &type, raw_ostream &os) {
228   os << "### " << type.getSummary() << "\n\n";
229   emitDescription(type.getDescription(), os);
230   os << "\n\n";
231 }
232 
233 //===----------------------------------------------------------------------===//
234 // TypeDef Documentation
235 //===----------------------------------------------------------------------===//
236 
emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef & def,raw_ostream & os)237 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
238                                             raw_ostream &os) {
239   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
240   if (parameters.empty()) {
241     os << "\nSyntax: `!" << def.getDialect().getName() << "."
242        << def.getMnemonic() << "`\n";
243     return;
244   }
245 
246   os << "\nSyntax:\n\n```\n!" << def.getDialect().getName() << "."
247      << def.getMnemonic() << "<\n";
248   for (const auto &it : llvm::enumerate(parameters)) {
249     const AttrOrTypeParameter &param = it.value();
250     os << "  " << param.getSyntax();
251     if (it.index() < (parameters.size() - 1))
252       os << ",";
253     os << "   # " << param.getName() << "\n";
254   }
255   os << ">\n```\n";
256 }
257 
emitAttrOrTypeDefDoc(const AttrOrTypeDef & def,raw_ostream & os)258 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
259   os << llvm::formatv("### {0}\n", def.getCppClassName());
260 
261   // Emit the summary if present.
262   if (def.hasSummary())
263     os << "\n" << def.getSummary() << "\n";
264 
265   // Emit the syntax if present.
266   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
267     emitAttrOrTypeDefAssemblyFormat(def, os);
268 
269   // Emit the description if present.
270   if (def.hasDescription()) {
271     os << "\n";
272     mlir::tblgen::emitDescription(def.getDescription(), os);
273   }
274 
275   // Emit parameter documentation.
276   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
277   if (!parameters.empty()) {
278     os << "\n#### Parameters:\n\n";
279     os << "| Parameter | C++ type | Description |\n"
280        << "| :-------: | :-------: | ----------- |\n";
281     for (const auto &it : parameters) {
282       auto desc = it.getSummary();
283       os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
284          << (desc ? *desc : "") << " |\n";
285     }
286   }
287 
288   os << "\n";
289 }
290 
emitAttrOrTypeDefDoc(const RecordKeeper & recordKeeper,raw_ostream & os,StringRef recordTypeName)291 static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
292                                  raw_ostream &os, StringRef recordTypeName) {
293   std::vector<llvm::Record *> defs =
294       recordKeeper.getAllDerivedDefinitions(recordTypeName);
295 
296   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
297   for (const llvm::Record *def : defs)
298     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
299 }
300 
301 //===----------------------------------------------------------------------===//
302 // Dialect Documentation
303 //===----------------------------------------------------------------------===//
304 
emitDialectDoc(const Dialect & dialect,ArrayRef<Attribute> attributes,ArrayRef<AttrDef> attrDefs,ArrayRef<Operator> ops,ArrayRef<Type> types,ArrayRef<TypeDef> typeDefs,raw_ostream & os)305 static void emitDialectDoc(const Dialect &dialect,
306                            ArrayRef<Attribute> attributes,
307                            ArrayRef<AttrDef> attrDefs, ArrayRef<Operator> ops,
308                            ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
309                            raw_ostream &os) {
310   os << "# '" << dialect.getName() << "' Dialect\n\n";
311   emitIfNotEmpty(dialect.getSummary(), os);
312   emitIfNotEmpty(dialect.getDescription(), os);
313 
314   // Generate a TOC marker except if description already contains one.
315   llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
316   if (!r.match(dialect.getDescription()))
317     os << "[TOC]\n\n";
318 
319   if (!ops.empty()) {
320     os << "## Operation definition\n\n";
321     for (const Operator &op : ops)
322       emitOpDoc(op, os);
323   }
324 
325   if (!attributes.empty()) {
326     os << "## Attribute constraint definition\n\n";
327     for (const Attribute &attr : attributes)
328       emitAttrDoc(attr, os);
329   }
330 
331   if (!attrDefs.empty()) {
332     os << "## Attribute definition\n\n";
333     for (const AttrDef &def : attrDefs)
334       emitAttrOrTypeDefDoc(def, os);
335   }
336 
337   // TODO: Add link between use and def for types
338   if (!types.empty()) {
339     os << "## Type constraint definition\n\n";
340     for (const Type &type : types)
341       emitTypeDoc(type, os);
342   }
343 
344   if (!typeDefs.empty()) {
345     os << "## Type definition\n\n";
346     for (const TypeDef &def : typeDefs)
347       emitAttrOrTypeDefDoc(def, os);
348   }
349 }
350 
emitDialectDoc(const RecordKeeper & recordKeeper,raw_ostream & os)351 static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
352   std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
353   std::vector<Record *> attrDefs =
354       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectAttr");
355   std::vector<Record *> typeDefs =
356       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
357   std::vector<Record *> typeDefDefs =
358       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
359   std::vector<Record *> attrDefDefs =
360       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
361 
362   llvm::SetVector<Dialect, SmallVector<Dialect>, std::set<Dialect>>
363       dialectsWithDocs;
364 
365   llvm::StringMap<std::vector<Attribute>> dialectAttrs;
366   llvm::StringMap<std::vector<AttrDef>> dialectAttrDefs;
367   llvm::StringMap<std::vector<Operator>> dialectOps;
368   llvm::StringMap<std::vector<Type>> dialectTypes;
369   llvm::StringMap<std::vector<TypeDef>> dialectTypeDefs;
370   llvm::SmallDenseSet<Record *> seen;
371   for (Record *attrDef : attrDefDefs) {
372     AttrDef attr(attrDef);
373     dialectAttrDefs[attr.getDialect().getName()].push_back(attr);
374     dialectsWithDocs.insert(attr.getDialect());
375     seen.insert(attrDef);
376   }
377   for (Record *attrDef : attrDefs) {
378     if (seen.count(attrDef))
379       continue;
380     Attribute attr(attrDef);
381     if (const Dialect &dialect = attr.getDialect()) {
382       dialectAttrs[dialect.getName()].push_back(attr);
383       dialectsWithDocs.insert(dialect);
384     }
385   }
386   for (Record *opDef : opDefs) {
387     Operator op(opDef);
388     dialectOps[op.getDialect().getName()].push_back(op);
389     dialectsWithDocs.insert(op.getDialect());
390   }
391   for (Record *typeDef : typeDefDefs) {
392     TypeDef type(typeDef);
393     dialectTypeDefs[type.getDialect().getName()].push_back(type);
394     dialectsWithDocs.insert(type.getDialect());
395     seen.insert(typeDef);
396   }
397   for (Record *typeDef : typeDefs) {
398     if (seen.count(typeDef))
399       continue;
400     Type type(typeDef);
401     if (const Dialect &dialect = type.getDialect()) {
402       dialectTypes[dialect.getName()].push_back(type);
403       dialectsWithDocs.insert(dialect);
404     }
405   }
406 
407   Optional<Dialect> dialect =
408       findDialectToGenerate(dialectsWithDocs.getArrayRef());
409   if (!dialect)
410     return true;
411 
412   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
413   StringRef dialectName = dialect->getName();
414   emitDialectDoc(*dialect, dialectAttrs[dialectName],
415                  dialectAttrDefs[dialectName], dialectOps[dialectName],
416                  dialectTypes[dialectName], dialectTypeDefs[dialectName], os);
417   return false;
418 }
419 
420 //===----------------------------------------------------------------------===//
421 // Gen Registration
422 //===----------------------------------------------------------------------===//
423 
424 static mlir::GenRegistration
425     genAttrRegister("gen-attrdef-doc",
426                     "Generate dialect attribute documentation",
__anond306c54e0202(const RecordKeeper &records, raw_ostream &os) 427                     [](const RecordKeeper &records, raw_ostream &os) {
428                       emitAttrOrTypeDefDoc(records, os, "AttrDef");
429                       return false;
430                     });
431 
432 static mlir::GenRegistration
433     genOpRegister("gen-op-doc", "Generate dialect documentation",
__anond306c54e0302(const RecordKeeper &records, raw_ostream &os) 434                   [](const RecordKeeper &records, raw_ostream &os) {
435                     emitOpDoc(records, os);
436                     return false;
437                   });
438 
439 static mlir::GenRegistration
440     genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
__anond306c54e0402(const RecordKeeper &records, raw_ostream &os) 441                     [](const RecordKeeper &records, raw_ostream &os) {
442                       emitAttrOrTypeDefDoc(records, os, "TypeDef");
443                       return false;
444                     });
445 
446 static mlir::GenRegistration
447     genRegister("gen-dialect-doc", "Generate dialect documentation",
__anond306c54e0502(const RecordKeeper &records, raw_ostream &os) 448                 [](const RecordKeeper &records, raw_ostream &os) {
449                   return emitDialectDoc(records, os);
450                 });
451