1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "clang/Sema/LoopHint.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
20 using namespace llvm;
21 
22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
23                               const llvm::DebugLoc &StartLoc,
24                               const llvm::DebugLoc &EndLoc) {
25 
26   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
27       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
28       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
29       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
30       Attrs.DistributeEnable == LoopAttributes::Unspecified &&
31       !StartLoc && !EndLoc)
32     return nullptr;
33 
34   SmallVector<Metadata *, 4> Args;
35   // Reserve operand 0 for loop id self reference.
36   auto TempNode = MDNode::getTemporary(Ctx, None);
37   Args.push_back(TempNode.get());
38 
39   // If we have a valid start debug location for the loop, add it.
40   if (StartLoc) {
41     Args.push_back(StartLoc.getAsMDNode());
42 
43     // If we also have a valid end debug location for the loop, add it.
44     if (EndLoc)
45       Args.push_back(EndLoc.getAsMDNode());
46   }
47 
48   // Setting vectorize.width
49   if (Attrs.VectorizeWidth > 0) {
50     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
51                         ConstantAsMetadata::get(ConstantInt::get(
52                             Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
53     Args.push_back(MDNode::get(Ctx, Vals));
54   }
55 
56   // Setting interleave.count
57   if (Attrs.InterleaveCount > 0) {
58     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
59                         ConstantAsMetadata::get(ConstantInt::get(
60                             Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
61     Args.push_back(MDNode::get(Ctx, Vals));
62   }
63 
64   // Setting interleave.count
65   if (Attrs.UnrollCount > 0) {
66     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
67                         ConstantAsMetadata::get(ConstantInt::get(
68                             Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
69     Args.push_back(MDNode::get(Ctx, Vals));
70   }
71 
72   // Setting vectorize.enable
73   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
74     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
75                         ConstantAsMetadata::get(ConstantInt::get(
76                             Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
77                                                    LoopAttributes::Enable)))};
78     Args.push_back(MDNode::get(Ctx, Vals));
79   }
80 
81   // Setting unroll.full or unroll.disable
82   if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
83     std::string Name;
84     if (Attrs.UnrollEnable == LoopAttributes::Enable)
85       Name = "llvm.loop.unroll.enable";
86     else if (Attrs.UnrollEnable == LoopAttributes::Full)
87       Name = "llvm.loop.unroll.full";
88     else
89       Name = "llvm.loop.unroll.disable";
90     Metadata *Vals[] = {MDString::get(Ctx, Name)};
91     Args.push_back(MDNode::get(Ctx, Vals));
92   }
93 
94   if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
95     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
96                         ConstantAsMetadata::get(ConstantInt::get(
97                             Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
98                                                    LoopAttributes::Enable)))};
99     Args.push_back(MDNode::get(Ctx, Vals));
100   }
101 
102   // Set the first operand to itself.
103   MDNode *LoopID = MDNode::get(Ctx, Args);
104   LoopID->replaceOperandWith(0, LoopID);
105   return LoopID;
106 }
107 
108 LoopAttributes::LoopAttributes(bool IsParallel)
109     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
110       UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
111       InterleaveCount(0), UnrollCount(0),
112       DistributeEnable(LoopAttributes::Unspecified) {}
113 
114 void LoopAttributes::clear() {
115   IsParallel = false;
116   VectorizeWidth = 0;
117   InterleaveCount = 0;
118   UnrollCount = 0;
119   VectorizeEnable = LoopAttributes::Unspecified;
120   UnrollEnable = LoopAttributes::Unspecified;
121   DistributeEnable = LoopAttributes::Unspecified;
122 }
123 
124 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
125                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
126     : LoopID(nullptr), Header(Header), Attrs(Attrs) {
127   LoopID = createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc);
128 }
129 
130 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
131                          const llvm::DebugLoc &EndLoc) {
132   Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
133   // Clear the attributes so nested loops do not inherit them.
134   StagedAttrs.clear();
135 }
136 
137 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
138                          ArrayRef<const clang::Attr *> Attrs,
139                          const llvm::DebugLoc &StartLoc,
140                          const llvm::DebugLoc &EndLoc) {
141 
142   // Identify loop hint attributes from Attrs.
143   for (const auto *Attr : Attrs) {
144     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
145     const OpenCLUnrollHintAttr *OpenCLHint =
146         dyn_cast<OpenCLUnrollHintAttr>(Attr);
147 
148     // Skip non loop hint attributes
149     if (!LH && !OpenCLHint) {
150       continue;
151     }
152 
153     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
154     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
155     unsigned ValueInt = 1;
156     // Translate opencl_unroll_hint attribute argument to
157     // equivalent LoopHintAttr enums.
158     // OpenCL v2.0 s6.11.5:
159     // 0 - full unroll (no argument).
160     // 1 - disable unroll.
161     // other positive integer n - unroll by n.
162     if (OpenCLHint) {
163       ValueInt = OpenCLHint->getUnrollHint();
164       if (ValueInt == 0) {
165         State = LoopHintAttr::Full;
166       } else if (ValueInt != 1) {
167         Option = LoopHintAttr::UnrollCount;
168         State = LoopHintAttr::Numeric;
169       }
170     } else if (LH) {
171       auto *ValueExpr = LH->getValue();
172       if (ValueExpr) {
173         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
174         ValueInt = ValueAPS.getSExtValue();
175       }
176 
177       Option = LH->getOption();
178       State = LH->getState();
179     }
180     switch (State) {
181     case LoopHintAttr::Disable:
182       switch (Option) {
183       case LoopHintAttr::Vectorize:
184         // Disable vectorization by specifying a width of 1.
185         setVectorizeWidth(1);
186         break;
187       case LoopHintAttr::Interleave:
188         // Disable interleaving by speciyfing a count of 1.
189         setInterleaveCount(1);
190         break;
191       case LoopHintAttr::Unroll:
192         setUnrollState(LoopAttributes::Disable);
193         break;
194       case LoopHintAttr::Distribute:
195         setDistributeState(false);
196         break;
197       case LoopHintAttr::UnrollCount:
198       case LoopHintAttr::VectorizeWidth:
199       case LoopHintAttr::InterleaveCount:
200         llvm_unreachable("Options cannot be disabled.");
201         break;
202       }
203       break;
204     case LoopHintAttr::Enable:
205       switch (Option) {
206       case LoopHintAttr::Vectorize:
207       case LoopHintAttr::Interleave:
208         setVectorizeEnable(true);
209         break;
210       case LoopHintAttr::Unroll:
211         setUnrollState(LoopAttributes::Enable);
212         break;
213       case LoopHintAttr::Distribute:
214         setDistributeState(true);
215         break;
216       case LoopHintAttr::UnrollCount:
217       case LoopHintAttr::VectorizeWidth:
218       case LoopHintAttr::InterleaveCount:
219         llvm_unreachable("Options cannot enabled.");
220         break;
221       }
222       break;
223     case LoopHintAttr::AssumeSafety:
224       switch (Option) {
225       case LoopHintAttr::Vectorize:
226       case LoopHintAttr::Interleave:
227         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
228         setParallel(true);
229         setVectorizeEnable(true);
230         break;
231       case LoopHintAttr::Unroll:
232       case LoopHintAttr::UnrollCount:
233       case LoopHintAttr::VectorizeWidth:
234       case LoopHintAttr::InterleaveCount:
235       case LoopHintAttr::Distribute:
236         llvm_unreachable("Options cannot be used to assume mem safety.");
237         break;
238       }
239       break;
240     case LoopHintAttr::Full:
241       switch (Option) {
242       case LoopHintAttr::Unroll:
243         setUnrollState(LoopAttributes::Full);
244         break;
245       case LoopHintAttr::Vectorize:
246       case LoopHintAttr::Interleave:
247       case LoopHintAttr::UnrollCount:
248       case LoopHintAttr::VectorizeWidth:
249       case LoopHintAttr::InterleaveCount:
250       case LoopHintAttr::Distribute:
251         llvm_unreachable("Options cannot be used with 'full' hint.");
252         break;
253       }
254       break;
255     case LoopHintAttr::Numeric:
256       switch (Option) {
257       case LoopHintAttr::VectorizeWidth:
258         setVectorizeWidth(ValueInt);
259         break;
260       case LoopHintAttr::InterleaveCount:
261         setInterleaveCount(ValueInt);
262         break;
263       case LoopHintAttr::UnrollCount:
264         setUnrollCount(ValueInt);
265         break;
266       case LoopHintAttr::Unroll:
267       case LoopHintAttr::Vectorize:
268       case LoopHintAttr::Interleave:
269       case LoopHintAttr::Distribute:
270         llvm_unreachable("Options cannot be assigned a value.");
271         break;
272       }
273       break;
274     }
275   }
276 
277   /// Stage the attributes.
278   push(Header, StartLoc, EndLoc);
279 }
280 
281 void LoopInfoStack::pop() {
282   assert(!Active.empty() && "No active loops to pop");
283   Active.pop_back();
284 }
285 
286 void LoopInfoStack::InsertHelper(Instruction *I) const {
287   if (!hasInfo())
288     return;
289 
290   const LoopInfo &L = getInfo();
291   if (!L.getLoopID())
292     return;
293 
294   if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
295     for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
296       if (TI->getSuccessor(i) == L.getHeader()) {
297         TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
298         break;
299       }
300     return;
301   }
302 
303   if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
304     I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
305 }
306