1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "clang/Sema/LoopHint.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
20 using namespace llvm;
21 
22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
23                               llvm::DebugLoc Location) {
24 
25   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
26       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
27       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
28       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
29       !Location)
30     return nullptr;
31 
32   SmallVector<Metadata *, 4> Args;
33   // Reserve operand 0 for loop id self reference.
34   auto TempNode = MDNode::getTemporary(Ctx, None);
35   Args.push_back(TempNode.get());
36 
37   // If we have a valid debug location for the loop, add it.
38   if (Location)
39     Args.push_back(Location.getAsMDNode());
40 
41   // Setting vectorize.width
42   if (Attrs.VectorizeWidth > 0) {
43     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
44                         ConstantAsMetadata::get(ConstantInt::get(
45                             Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
46     Args.push_back(MDNode::get(Ctx, Vals));
47   }
48 
49   // Setting interleave.count
50   if (Attrs.InterleaveCount > 0) {
51     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
52                         ConstantAsMetadata::get(ConstantInt::get(
53                             Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
54     Args.push_back(MDNode::get(Ctx, Vals));
55   }
56 
57   // Setting interleave.count
58   if (Attrs.UnrollCount > 0) {
59     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
60                         ConstantAsMetadata::get(ConstantInt::get(
61                             Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
62     Args.push_back(MDNode::get(Ctx, Vals));
63   }
64 
65   // Setting vectorize.enable
66   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
67     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
68                         ConstantAsMetadata::get(ConstantInt::get(
69                             Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
70                                                    LoopAttributes::Enable)))};
71     Args.push_back(MDNode::get(Ctx, Vals));
72   }
73 
74   // Setting unroll.full or unroll.disable
75   if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
76     std::string Name;
77     if (Attrs.UnrollEnable == LoopAttributes::Enable)
78       Name = "llvm.loop.unroll.enable";
79     else if (Attrs.UnrollEnable == LoopAttributes::Full)
80       Name = "llvm.loop.unroll.full";
81     else
82       Name = "llvm.loop.unroll.disable";
83     Metadata *Vals[] = {MDString::get(Ctx, Name)};
84     Args.push_back(MDNode::get(Ctx, Vals));
85   }
86 
87   // Set the first operand to itself.
88   MDNode *LoopID = MDNode::get(Ctx, Args);
89   LoopID->replaceOperandWith(0, LoopID);
90   return LoopID;
91 }
92 
93 LoopAttributes::LoopAttributes(bool IsParallel)
94     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
95       UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
96       InterleaveCount(0), UnrollCount(0) {}
97 
98 void LoopAttributes::clear() {
99   IsParallel = false;
100   VectorizeWidth = 0;
101   InterleaveCount = 0;
102   UnrollCount = 0;
103   VectorizeEnable = LoopAttributes::Unspecified;
104   UnrollEnable = LoopAttributes::Unspecified;
105 }
106 
107 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
108                    llvm::DebugLoc Location)
109     : LoopID(nullptr), Header(Header), Attrs(Attrs) {
110   LoopID = createMetadata(Header->getContext(), Attrs, Location);
111 }
112 
113 void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) {
114   Active.push_back(LoopInfo(Header, StagedAttrs, Location));
115   // Clear the attributes so nested loops do not inherit them.
116   StagedAttrs.clear();
117 }
118 
119 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
120                          ArrayRef<const clang::Attr *> Attrs,
121                          llvm::DebugLoc Location) {
122 
123   // Identify loop hint attributes from Attrs.
124   for (const auto *Attr : Attrs) {
125     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
126     const OpenCLUnrollHintAttr *OpenCLHint =
127         dyn_cast<OpenCLUnrollHintAttr>(Attr);
128 
129     // Skip non loop hint attributes
130     if (!LH && !OpenCLHint) {
131       continue;
132     }
133 
134     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
135     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
136     unsigned ValueInt = 1;
137     // Translate opencl_unroll_hint attribute argument to
138     // equivalent LoopHintAttr enums.
139     // OpenCL v2.0 s6.11.5:
140     // 0 - full unroll (no argument).
141     // 1 - disable unroll.
142     // other positive integer n - unroll by n.
143     if (OpenCLHint) {
144       ValueInt = OpenCLHint->getUnrollHint();
145       if (ValueInt == 0) {
146         State = LoopHintAttr::Full;
147       } else if (ValueInt != 1) {
148         Option = LoopHintAttr::UnrollCount;
149         State = LoopHintAttr::Numeric;
150       }
151     } else if (LH) {
152       auto *ValueExpr = LH->getValue();
153       if (ValueExpr) {
154         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
155         ValueInt = ValueAPS.getSExtValue();
156       }
157 
158       Option = LH->getOption();
159       State = LH->getState();
160     }
161     switch (State) {
162     case LoopHintAttr::Disable:
163       switch (Option) {
164       case LoopHintAttr::Vectorize:
165         // Disable vectorization by specifying a width of 1.
166         setVectorizeWidth(1);
167         break;
168       case LoopHintAttr::Interleave:
169         // Disable interleaving by speciyfing a count of 1.
170         setInterleaveCount(1);
171         break;
172       case LoopHintAttr::Unroll:
173         setUnrollState(LoopAttributes::Disable);
174         break;
175       case LoopHintAttr::UnrollCount:
176       case LoopHintAttr::VectorizeWidth:
177       case LoopHintAttr::InterleaveCount:
178         llvm_unreachable("Options cannot be disabled.");
179         break;
180       }
181       break;
182     case LoopHintAttr::Enable:
183       switch (Option) {
184       case LoopHintAttr::Vectorize:
185       case LoopHintAttr::Interleave:
186         setVectorizeEnable(true);
187         break;
188       case LoopHintAttr::Unroll:
189         setUnrollState(LoopAttributes::Enable);
190         break;
191       case LoopHintAttr::UnrollCount:
192       case LoopHintAttr::VectorizeWidth:
193       case LoopHintAttr::InterleaveCount:
194         llvm_unreachable("Options cannot enabled.");
195         break;
196       }
197       break;
198     case LoopHintAttr::AssumeSafety:
199       switch (Option) {
200       case LoopHintAttr::Vectorize:
201       case LoopHintAttr::Interleave:
202         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
203         setParallel(true);
204         setVectorizeEnable(true);
205         break;
206       case LoopHintAttr::Unroll:
207       case LoopHintAttr::UnrollCount:
208       case LoopHintAttr::VectorizeWidth:
209       case LoopHintAttr::InterleaveCount:
210         llvm_unreachable("Options cannot be used to assume mem safety.");
211         break;
212       }
213       break;
214     case LoopHintAttr::Full:
215       switch (Option) {
216       case LoopHintAttr::Unroll:
217         setUnrollState(LoopAttributes::Full);
218         break;
219       case LoopHintAttr::Vectorize:
220       case LoopHintAttr::Interleave:
221       case LoopHintAttr::UnrollCount:
222       case LoopHintAttr::VectorizeWidth:
223       case LoopHintAttr::InterleaveCount:
224         llvm_unreachable("Options cannot be used with 'full' hint.");
225         break;
226       }
227       break;
228     case LoopHintAttr::Numeric:
229       switch (Option) {
230       case LoopHintAttr::VectorizeWidth:
231         setVectorizeWidth(ValueInt);
232         break;
233       case LoopHintAttr::InterleaveCount:
234         setInterleaveCount(ValueInt);
235         break;
236       case LoopHintAttr::UnrollCount:
237         setUnrollCount(ValueInt);
238         break;
239       case LoopHintAttr::Unroll:
240       case LoopHintAttr::Vectorize:
241       case LoopHintAttr::Interleave:
242         llvm_unreachable("Options cannot be assigned a value.");
243         break;
244       }
245       break;
246     }
247   }
248 
249   /// Stage the attributes.
250   push(Header, Location);
251 }
252 
253 void LoopInfoStack::pop() {
254   assert(!Active.empty() && "No active loops to pop");
255   Active.pop_back();
256 }
257 
258 void LoopInfoStack::InsertHelper(Instruction *I) const {
259   if (!hasInfo())
260     return;
261 
262   const LoopInfo &L = getInfo();
263   if (!L.getLoopID())
264     return;
265 
266   if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
267     for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
268       if (TI->getSuccessor(i) == L.getHeader()) {
269         TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
270         break;
271       }
272     return;
273   }
274 
275   if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
276     I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
277 }
278