1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "clang/Sema/LoopHint.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
24                               const llvm::DebugLoc &StartLoc,
25                               const llvm::DebugLoc &EndLoc) {
26 
27   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
28       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
29       Attrs.UnrollAndJamCount == 0 &&
30       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
31       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
32       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
33       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
34       !EndLoc)
35     return nullptr;
36 
37   SmallVector<Metadata *, 4> Args;
38   // Reserve operand 0 for loop id self reference.
39   auto TempNode = MDNode::getTemporary(Ctx, None);
40   Args.push_back(TempNode.get());
41 
42   // If we have a valid start debug location for the loop, add it.
43   if (StartLoc) {
44     Args.push_back(StartLoc.getAsMDNode());
45 
46     // If we also have a valid end debug location for the loop, add it.
47     if (EndLoc)
48       Args.push_back(EndLoc.getAsMDNode());
49   }
50 
51   // Setting vectorize.width
52   if (Attrs.VectorizeWidth > 0) {
53     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
54                         ConstantAsMetadata::get(ConstantInt::get(
55                             Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
56     Args.push_back(MDNode::get(Ctx, Vals));
57   }
58 
59   // Setting interleave.count
60   if (Attrs.InterleaveCount > 0) {
61     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
62                         ConstantAsMetadata::get(ConstantInt::get(
63                             Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
64     Args.push_back(MDNode::get(Ctx, Vals));
65   }
66 
67   // Setting unroll.count
68   if (Attrs.UnrollCount > 0) {
69     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
70                         ConstantAsMetadata::get(ConstantInt::get(
71                             Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
72     Args.push_back(MDNode::get(Ctx, Vals));
73   }
74 
75   // Setting unroll_and_jam.count
76   if (Attrs.UnrollAndJamCount > 0) {
77     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
78                         ConstantAsMetadata::get(ConstantInt::get(
79                             Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
80     Args.push_back(MDNode::get(Ctx, Vals));
81   }
82 
83   // Setting vectorize.enable
84   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
85     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
86                         ConstantAsMetadata::get(ConstantInt::get(
87                             Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
88                                                    LoopAttributes::Enable)))};
89     Args.push_back(MDNode::get(Ctx, Vals));
90   }
91 
92   // Setting unroll.full or unroll.disable
93   if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
94     std::string Name;
95     if (Attrs.UnrollEnable == LoopAttributes::Enable)
96       Name = "llvm.loop.unroll.enable";
97     else if (Attrs.UnrollEnable == LoopAttributes::Full)
98       Name = "llvm.loop.unroll.full";
99     else
100       Name = "llvm.loop.unroll.disable";
101     Metadata *Vals[] = {MDString::get(Ctx, Name)};
102     Args.push_back(MDNode::get(Ctx, Vals));
103   }
104 
105   // Setting unroll_and_jam.full or unroll_and_jam.disable
106   if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
107     std::string Name;
108     if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
109       Name = "llvm.loop.unroll_and_jam.enable";
110     else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
111       Name = "llvm.loop.unroll_and_jam.full";
112     else
113       Name = "llvm.loop.unroll_and_jam.disable";
114     Metadata *Vals[] = {MDString::get(Ctx, Name)};
115     Args.push_back(MDNode::get(Ctx, Vals));
116   }
117 
118   if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
120                         ConstantAsMetadata::get(ConstantInt::get(
121                             Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
122                                                    LoopAttributes::Enable)))};
123     Args.push_back(MDNode::get(Ctx, Vals));
124   }
125 
126   // Set the first operand to itself.
127   MDNode *LoopID = MDNode::get(Ctx, Args);
128   LoopID->replaceOperandWith(0, LoopID);
129   return LoopID;
130 }
131 
132 LoopAttributes::LoopAttributes(bool IsParallel)
133     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
134       UnrollEnable(LoopAttributes::Unspecified),
135       UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
136       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
137       DistributeEnable(LoopAttributes::Unspecified) {}
138 
139 void LoopAttributes::clear() {
140   IsParallel = false;
141   VectorizeWidth = 0;
142   InterleaveCount = 0;
143   UnrollCount = 0;
144   UnrollAndJamCount = 0;
145   VectorizeEnable = LoopAttributes::Unspecified;
146   UnrollEnable = LoopAttributes::Unspecified;
147   UnrollAndJamEnable = LoopAttributes::Unspecified;
148   DistributeEnable = LoopAttributes::Unspecified;
149 }
150 
151 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
152                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
153     : LoopID(nullptr), Header(Header), Attrs(Attrs) {
154   LoopID = createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc);
155 }
156 
157 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
158                          const llvm::DebugLoc &EndLoc) {
159   Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
160   // Clear the attributes so nested loops do not inherit them.
161   StagedAttrs.clear();
162 }
163 
164 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
165                          ArrayRef<const clang::Attr *> Attrs,
166                          const llvm::DebugLoc &StartLoc,
167                          const llvm::DebugLoc &EndLoc) {
168 
169   // Identify loop hint attributes from Attrs.
170   for (const auto *Attr : Attrs) {
171     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
172     const OpenCLUnrollHintAttr *OpenCLHint =
173         dyn_cast<OpenCLUnrollHintAttr>(Attr);
174 
175     // Skip non loop hint attributes
176     if (!LH && !OpenCLHint) {
177       continue;
178     }
179 
180     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
181     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
182     unsigned ValueInt = 1;
183     // Translate opencl_unroll_hint attribute argument to
184     // equivalent LoopHintAttr enums.
185     // OpenCL v2.0 s6.11.5:
186     // 0 - full unroll (no argument).
187     // 1 - disable unroll.
188     // other positive integer n - unroll by n.
189     if (OpenCLHint) {
190       ValueInt = OpenCLHint->getUnrollHint();
191       if (ValueInt == 0) {
192         State = LoopHintAttr::Full;
193       } else if (ValueInt != 1) {
194         Option = LoopHintAttr::UnrollCount;
195         State = LoopHintAttr::Numeric;
196       }
197     } else if (LH) {
198       auto *ValueExpr = LH->getValue();
199       if (ValueExpr) {
200         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
201         ValueInt = ValueAPS.getSExtValue();
202       }
203 
204       Option = LH->getOption();
205       State = LH->getState();
206     }
207     switch (State) {
208     case LoopHintAttr::Disable:
209       switch (Option) {
210       case LoopHintAttr::Vectorize:
211         // Disable vectorization by specifying a width of 1.
212         setVectorizeWidth(1);
213         break;
214       case LoopHintAttr::Interleave:
215         // Disable interleaving by speciyfing a count of 1.
216         setInterleaveCount(1);
217         break;
218       case LoopHintAttr::Unroll:
219         setUnrollState(LoopAttributes::Disable);
220         break;
221       case LoopHintAttr::UnrollAndJam:
222         setUnrollAndJamState(LoopAttributes::Disable);
223         break;
224       case LoopHintAttr::Distribute:
225         setDistributeState(false);
226         break;
227       case LoopHintAttr::UnrollCount:
228       case LoopHintAttr::UnrollAndJamCount:
229       case LoopHintAttr::VectorizeWidth:
230       case LoopHintAttr::InterleaveCount:
231         llvm_unreachable("Options cannot be disabled.");
232         break;
233       }
234       break;
235     case LoopHintAttr::Enable:
236       switch (Option) {
237       case LoopHintAttr::Vectorize:
238       case LoopHintAttr::Interleave:
239         setVectorizeEnable(true);
240         break;
241       case LoopHintAttr::Unroll:
242         setUnrollState(LoopAttributes::Enable);
243         break;
244       case LoopHintAttr::UnrollAndJam:
245         setUnrollAndJamState(LoopAttributes::Enable);
246         break;
247       case LoopHintAttr::Distribute:
248         setDistributeState(true);
249         break;
250       case LoopHintAttr::UnrollCount:
251       case LoopHintAttr::UnrollAndJamCount:
252       case LoopHintAttr::VectorizeWidth:
253       case LoopHintAttr::InterleaveCount:
254         llvm_unreachable("Options cannot enabled.");
255         break;
256       }
257       break;
258     case LoopHintAttr::AssumeSafety:
259       switch (Option) {
260       case LoopHintAttr::Vectorize:
261       case LoopHintAttr::Interleave:
262         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
263         setParallel(true);
264         setVectorizeEnable(true);
265         break;
266       case LoopHintAttr::Unroll:
267       case LoopHintAttr::UnrollAndJam:
268       case LoopHintAttr::UnrollCount:
269       case LoopHintAttr::UnrollAndJamCount:
270       case LoopHintAttr::VectorizeWidth:
271       case LoopHintAttr::InterleaveCount:
272       case LoopHintAttr::Distribute:
273         llvm_unreachable("Options cannot be used to assume mem safety.");
274         break;
275       }
276       break;
277     case LoopHintAttr::Full:
278       switch (Option) {
279       case LoopHintAttr::Unroll:
280         setUnrollState(LoopAttributes::Full);
281         break;
282       case LoopHintAttr::UnrollAndJam:
283         setUnrollAndJamState(LoopAttributes::Full);
284         break;
285       case LoopHintAttr::Vectorize:
286       case LoopHintAttr::Interleave:
287       case LoopHintAttr::UnrollCount:
288       case LoopHintAttr::UnrollAndJamCount:
289       case LoopHintAttr::VectorizeWidth:
290       case LoopHintAttr::InterleaveCount:
291       case LoopHintAttr::Distribute:
292         llvm_unreachable("Options cannot be used with 'full' hint.");
293         break;
294       }
295       break;
296     case LoopHintAttr::Numeric:
297       switch (Option) {
298       case LoopHintAttr::VectorizeWidth:
299         setVectorizeWidth(ValueInt);
300         break;
301       case LoopHintAttr::InterleaveCount:
302         setInterleaveCount(ValueInt);
303         break;
304       case LoopHintAttr::UnrollCount:
305         setUnrollCount(ValueInt);
306         break;
307       case LoopHintAttr::UnrollAndJamCount:
308         setUnrollAndJamCount(ValueInt);
309         break;
310       case LoopHintAttr::Unroll:
311       case LoopHintAttr::UnrollAndJam:
312       case LoopHintAttr::Vectorize:
313       case LoopHintAttr::Interleave:
314       case LoopHintAttr::Distribute:
315         llvm_unreachable("Options cannot be assigned a value.");
316         break;
317       }
318       break;
319     }
320   }
321 
322   /// Stage the attributes.
323   push(Header, StartLoc, EndLoc);
324 }
325 
326 void LoopInfoStack::pop() {
327   assert(!Active.empty() && "No active loops to pop");
328   Active.pop_back();
329 }
330 
331 void LoopInfoStack::InsertHelper(Instruction *I) const {
332   if (!hasInfo())
333     return;
334 
335   const LoopInfo &L = getInfo();
336   if (!L.getLoopID())
337     return;
338 
339   if (I->isTerminator()) {
340     for (BasicBlock *Succ : successors(I))
341       if (Succ == L.getHeader()) {
342         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
343         break;
344       }
345     return;
346   }
347 
348   if (I->mayReadOrWriteMemory()) {
349     SmallVector<Metadata *, 2> ParallelLoopIDs;
350     for (const LoopInfo &AL : Active)
351       if (AL.getAttributes().IsParallel)
352         ParallelLoopIDs.push_back(AL.getLoopID());
353 
354     MDNode *ParallelMD = nullptr;
355     if (ParallelLoopIDs.size() == 1)
356       ParallelMD = cast<MDNode>(ParallelLoopIDs[0]);
357     else if (ParallelLoopIDs.size() >= 2)
358       ParallelMD = MDNode::get(I->getContext(), ParallelLoopIDs);
359     I->setMetadata("llvm.mem.parallel_loop_access", ParallelMD);
360   }
361 }
362