1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // OpDocGen uses the description of operations to generate documentation for the
10 // operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "DocGenUtilities.h"
15 #include "OpGenHelpers.h"
16 #include "mlir/Support/IndentedOstream.h"
17 #include "mlir/TableGen/AttrOrTypeDef.h"
18 #include "mlir/TableGen/GenInfo.h"
19 #include "mlir/TableGen/Operator.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/StringExtras.h"
22 #include "llvm/Support/CommandLine.h"
23 #include "llvm/Support/FormatVariadic.h"
24 #include "llvm/Support/Regex.h"
25 #include "llvm/Support/Signals.h"
26 #include "llvm/TableGen/Error.h"
27 #include "llvm/TableGen/Record.h"
28 #include "llvm/TableGen/TableGenBackend.h"
29 
30 #include <set>
31 
32 using namespace llvm;
33 using namespace mlir;
34 using namespace mlir::tblgen;
35 
36 using mlir::tblgen::Operator;
37 
38 extern llvm::cl::opt<std::string> selectedDialect;
39 
40 // Emit the description by aligning the text to the left per line (e.g.,
41 // removing the minimum indentation across the block).
42 //
43 // This expects that the description in the tablegen file is already formatted
44 // in a way the user wanted but has some additional indenting due to being
45 // nested in the op definition.
46 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
47   raw_indented_ostream ros(os);
48   ros.printReindented(description.rtrim(" \t"));
49 }
50 
51 // Emits `str` with trailing newline if not empty.
52 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
53   if (!str.empty()) {
54     emitDescription(str, os);
55     os << "\n";
56   }
57 }
58 
59 /// Emit the given named constraint.
60 template <typename T>
61 static void emitNamedConstraint(const T &it, raw_ostream &os) {
62   if (!it.name.empty())
63     os << "| `" << it.name << "`";
64   else
65     os << "&laquo;unnamed&raquo;";
66   os << " | " << it.constraint.getSummary() << "\n";
67 }
68 
69 //===----------------------------------------------------------------------===//
70 // Operation Documentation
71 //===----------------------------------------------------------------------===//
72 
73 /// Emit the assembly format of an operation.
74 static void emitAssemblyFormat(StringRef opName, StringRef format,
75                                raw_ostream &os) {
76   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
77 
78   // Print the assembly format aligned.
79   unsigned indent = strlen("operation ::= ");
80   std::pair<StringRef, StringRef> split = format.split('\n');
81   os << split.first.trim() << "\n";
82   do {
83     split = split.second.split('\n');
84     StringRef formatChunk = split.first.trim();
85     if (!formatChunk.empty())
86       os.indent(indent) << formatChunk << "\n";
87   } while (!split.second.empty());
88   os << "```\n\n";
89 }
90 
91 static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
92   // TODO: We should link to the trait/documentation of it. That also means we
93   // should add descriptions to traits that can be queried.
94   // Collect using set to sort effects, interfaces & traits.
95   std::set<std::string> effects, interfaces, traits;
96   for (auto &trait : op.getTraits()) {
97     if (isa<PredTrait>(&trait))
98       continue;
99 
100     std::string name = trait.getDef().getName().str();
101     StringRef ref = name;
102     StringRef traitName = trait.getDef().getValueAsString("trait");
103     traitName.consume_back("::Trait");
104     traitName.consume_back("::Impl");
105     if (ref.startswith("anonymous_"))
106       name = traitName.str();
107     if (isa<InterfaceTrait>(&trait)) {
108       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
109         auto effectName = trait.getDef().getValueAsString("baseEffectName");
110         effectName.consume_front("::");
111         effectName.consume_front("mlir::");
112         std::string effectStr;
113         llvm::raw_string_ostream os(effectStr);
114         os << effectName << "{";
115         auto list = trait.getDef().getValueAsListOfDefs("effects");
116         llvm::interleaveComma(list, os, [&](Record *rec) {
117           StringRef effect = rec->getValueAsString("effect");
118           effect.consume_front("::");
119           effect.consume_front("mlir::");
120           os << effect << " on " << rec->getValueAsString("resource");
121         });
122         os << "}";
123         effects.insert(os.str());
124         name.append(llvm::formatv(" ({0})", traitName).str());
125       }
126       interfaces.insert(name);
127       continue;
128     }
129 
130     traits.insert(name);
131   }
132   if (!traits.empty()) {
133     llvm::interleaveComma(traits, os << "\nTraits: ");
134     os << "\n";
135   }
136   if (!interfaces.empty()) {
137     llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
138     os << "\n";
139   }
140   if (!effects.empty()) {
141     llvm::interleaveComma(effects, os << "\nEffects: ");
142     os << "\n";
143   }
144 }
145 
146 static void emitOpDoc(const Operator &op, raw_ostream &os) {
147   os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(),
148                       op.getQualCppClassName());
149 
150   // Emit the summary, syntax, and description if present.
151   if (op.hasSummary())
152     os << "\n" << op.getSummary() << "\n\n";
153   if (op.hasAssemblyFormat())
154     emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),
155                        os);
156   if (op.hasDescription())
157     mlir::tblgen::emitDescription(op.getDescription(), os);
158 
159   emitOpTraitsDoc(op, os);
160 
161   // Emit attributes.
162   if (op.getNumAttributes() != 0) {
163     // TODO: Attributes are only documented by TableGen name, with no further
164     // info. This should be improved.
165     os << "\n#### Attributes:\n\n";
166     os << "| Attribute | MLIR Type | Description |\n"
167        << "| :-------: | :-------: | ----------- |\n";
168     for (const auto &it : op.getAttributes()) {
169       StringRef storageType = it.attr.getStorageType();
170       os << "| `" << it.name << "` | " << storageType << " | "
171          << it.attr.getSummary() << "\n";
172     }
173   }
174 
175   // Emit each of the operands.
176   if (op.getNumOperands() != 0) {
177     os << "\n#### Operands:\n\n";
178     os << "| Operand | Description |\n"
179        << "| :-----: | ----------- |\n";
180     for (const auto &it : op.getOperands())
181       emitNamedConstraint(it, os);
182   }
183 
184   // Emit results.
185   if (op.getNumResults() != 0) {
186     os << "\n#### Results:\n\n";
187     os << "| Result | Description |\n"
188        << "| :----: | ----------- |\n";
189     for (const auto &it : op.getResults())
190       emitNamedConstraint(it, os);
191   }
192 
193   // Emit successors.
194   if (op.getNumSuccessors() != 0) {
195     os << "\n#### Successors:\n\n";
196     os << "| Successor | Description |\n"
197        << "| :-------: | ----------- |\n";
198     for (const auto &it : op.getSuccessors())
199       emitNamedConstraint(it, os);
200   }
201 
202   os << "\n";
203 }
204 
205 static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
206   auto opDefs = getRequestedOpDefinitions(recordKeeper);
207 
208   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
209   for (const llvm::Record *opDef : opDefs)
210     emitOpDoc(Operator(opDef), os);
211 }
212 
213 //===----------------------------------------------------------------------===//
214 // Type Documentation
215 //===----------------------------------------------------------------------===//
216 
217 static void emitTypeDoc(const Type &type, raw_ostream &os) {
218   os << "### " << type.getSummary() << "\n";
219   emitDescription(type.getDescription(), os);
220   os << "\n";
221 }
222 
223 //===----------------------------------------------------------------------===//
224 // TypeDef Documentation
225 //===----------------------------------------------------------------------===//
226 
227 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
228                                             raw_ostream &os) {
229   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
230   if (parameters.empty()) {
231     os << "\nSyntax: `!" << def.getDialect().getName() << "."
232        << def.getMnemonic() << "`\n";
233     return;
234   }
235 
236   os << "\nSyntax:\n\n```\n!" << def.getDialect().getName() << "."
237      << def.getMnemonic() << "<\n";
238   for (const auto &it : llvm::enumerate(parameters)) {
239     const AttrOrTypeParameter &param = it.value();
240     os << "  " << param.getSyntax();
241     if (it.index() < (parameters.size() - 1))
242       os << ",";
243     os << "   # " << param.getName() << "\n";
244   }
245   os << ">\n```\n";
246 }
247 
248 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
249   os << llvm::formatv("### {0}\n", def.getCppClassName());
250 
251   // Emit the summary if present.
252   if (def.hasSummary())
253     os << "\n" << def.getSummary() << "\n";
254 
255   // Emit the syntax if present.
256   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
257     emitAttrOrTypeDefAssemblyFormat(def, os);
258 
259   // Emit the description if present.
260   if (def.hasDescription()) {
261     os << "\n";
262     mlir::tblgen::emitDescription(def.getDescription(), os);
263   }
264 
265   // Emit parameter documentation.
266   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
267   if (!parameters.empty()) {
268     os << "\n#### Parameters:\n\n";
269     os << "| Parameter | C++ type | Description |\n"
270        << "| :-------: | :-------: | ----------- |\n";
271     for (const auto &it : parameters) {
272       auto desc = it.getSummary();
273       os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
274          << (desc ? *desc : "") << " |\n";
275     }
276   }
277 
278   os << "\n";
279 }
280 
281 static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
282                                  raw_ostream &os, StringRef recordTypeName) {
283   std::vector<llvm::Record *> defs =
284       recordKeeper.getAllDerivedDefinitions(recordTypeName);
285 
286   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
287   for (const llvm::Record *def : defs)
288     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
289 }
290 
291 //===----------------------------------------------------------------------===//
292 // Dialect Documentation
293 //===----------------------------------------------------------------------===//
294 
295 static void emitDialectDoc(const Dialect &dialect, ArrayRef<AttrDef> attrDefs,
296                            ArrayRef<Operator> ops, ArrayRef<Type> types,
297                            ArrayRef<TypeDef> typeDefs, raw_ostream &os) {
298   if (selectedDialect.getNumOccurrences() &&
299       dialect.getName() != selectedDialect)
300     return;
301   os << "# '" << dialect.getName() << "' Dialect\n\n";
302   emitIfNotEmpty(dialect.getSummary(), os);
303   emitIfNotEmpty(dialect.getDescription(), os);
304 
305   // Generate a TOC marker except if description already contains one.
306   llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
307   if (!r.match(dialect.getDescription()))
308     os << "[TOC]\n\n";
309 
310   if (!attrDefs.empty()) {
311     os << "## Attribute definition\n\n";
312     for (const AttrDef &def : attrDefs)
313       emitAttrOrTypeDefDoc(def, os);
314   }
315 
316   // TODO: Add link between use and def for types
317   if (!types.empty()) {
318     os << "## Type constraint definition\n\n";
319     for (const Type &type : types)
320       emitTypeDoc(type, os);
321   }
322 
323   if (!ops.empty()) {
324     os << "## Operation definition\n\n";
325     for (const Operator &op : ops)
326       emitOpDoc(op, os);
327   }
328 
329   if (!typeDefs.empty()) {
330     os << "## Type definition\n\n";
331     for (const TypeDef &def : typeDefs)
332       emitAttrOrTypeDefDoc(def, os);
333   }
334 }
335 
336 static void emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
337   std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
338   std::vector<Record *> typeDefs =
339       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
340   std::vector<Record *> typeDefDefs =
341       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
342   std::vector<Record *> attrDefDefs =
343       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
344 
345   std::set<Dialect> dialectsWithDocs;
346 
347   llvm::StringMap<std::vector<AttrDef>> dialectAttrDefs;
348   llvm::StringMap<std::vector<Operator>> dialectOps;
349   llvm::StringMap<std::vector<Type>> dialectTypes;
350   llvm::StringMap<std::vector<TypeDef>> dialectTypeDefs;
351   for (auto *attrDef : attrDefDefs) {
352     AttrDef attr(attrDef);
353     dialectAttrDefs[attr.getDialect().getName()].push_back(attr);
354     dialectsWithDocs.insert(attr.getDialect());
355   }
356   for (auto *opDef : opDefs) {
357     Operator op(opDef);
358     dialectOps[op.getDialect().getName()].push_back(op);
359     dialectsWithDocs.insert(op.getDialect());
360   }
361   for (auto *typeDef : typeDefs) {
362     Type type(typeDef);
363     if (auto dialect = type.getDialect())
364       dialectTypes[dialect.getName()].push_back(type);
365   }
366   for (auto *typeDef : typeDefDefs) {
367     TypeDef type(typeDef);
368     dialectTypeDefs[type.getDialect().getName()].push_back(type);
369     dialectsWithDocs.insert(type.getDialect());
370   }
371 
372   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
373   for (const Dialect &dialect : dialectsWithDocs) {
374     StringRef dialectName = dialect.getName();
375     emitDialectDoc(dialect, dialectAttrDefs[dialectName],
376                    dialectOps[dialectName], dialectTypes[dialectName],
377                    dialectTypeDefs[dialectName], os);
378   }
379 }
380 
381 //===----------------------------------------------------------------------===//
382 // Gen Registration
383 //===----------------------------------------------------------------------===//
384 
385 static mlir::GenRegistration
386     genAttrRegister("gen-attrdef-doc",
387                     "Generate dialect attribute documentation",
388                     [](const RecordKeeper &records, raw_ostream &os) {
389                       emitAttrOrTypeDefDoc(records, os, "AttrDef");
390                       return false;
391                     });
392 
393 static mlir::GenRegistration
394     genOpRegister("gen-op-doc", "Generate dialect documentation",
395                   [](const RecordKeeper &records, raw_ostream &os) {
396                     emitOpDoc(records, os);
397                     return false;
398                   });
399 
400 static mlir::GenRegistration
401     genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
402                     [](const RecordKeeper &records, raw_ostream &os) {
403                       emitAttrOrTypeDefDoc(records, os, "TypeDef");
404                       return false;
405                     });
406 
407 static mlir::GenRegistration
408     genRegister("gen-dialect-doc", "Generate dialect documentation",
409                 [](const RecordKeeper &records, raw_ostream &os) {
410                   emitDialectDoc(records, os);
411                   return false;
412                 });
413