1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25   LLVMContext &Ctx = Header->getContext();
26   SmallVector<Metadata *, 4> NewLoopProperties;
27   NewLoopProperties.push_back(nullptr);
28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31   LoopID->replaceOperandWith(0, LoopID);
32   return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36                                            ArrayRef<Metadata *> LoopProperties,
37                                            bool &HasUserTransforms) {
38   LLVMContext &Ctx = Header->getContext();
39 
40   Optional<bool> Enabled;
41   if (Attrs.PipelineDisabled)
42     Enabled = false;
43   else if (Attrs.PipelineInitiationInterval != 0)
44     Enabled = true;
45 
46   if (Enabled != true) {
47     SmallVector<Metadata *, 4> NewLoopProperties;
48     if (Enabled == false) {
49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50       NewLoopProperties.push_back(
51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52                             ConstantAsMetadata::get(ConstantInt::get(
53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
54       LoopProperties = NewLoopProperties;
55     }
56     return createLoopPropertiesMetadata(LoopProperties);
57   }
58 
59   SmallVector<Metadata *, 4> Args;
60   Args.push_back(nullptr);
61   Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63   if (Attrs.PipelineInitiationInterval > 0) {
64     Metadata *Vals[] = {
65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66         ConstantAsMetadata::get(ConstantInt::get(
67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68     Args.push_back(MDNode::get(Ctx, Vals));
69   }
70 
71   // No follow-up: This is the last transformation.
72 
73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74   LoopID->replaceOperandWith(0, LoopID);
75   HasUserTransforms = true;
76   return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81                                       ArrayRef<Metadata *> LoopProperties,
82                                       bool &HasUserTransforms) {
83   LLVMContext &Ctx = Header->getContext();
84 
85   Optional<bool> Enabled;
86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
87     Enabled = false;
88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
89     Enabled = None;
90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91            Attrs.UnrollCount != 0)
92     Enabled = true;
93 
94   if (Enabled != true) {
95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96     // if unrolling is disabled.
97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98   }
99 
100   SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102   // Apply all loop properties to the unrolled loop.
103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105   // Don't unroll an already unrolled loop.
106   FollowupLoopProperties.push_back(
107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109   bool FollowupHasTransforms = false;
110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111                                               FollowupHasTransforms);
112 
113   SmallVector<Metadata *, 4> Args;
114   Args.push_back(nullptr);
115   Args.append(LoopProperties.begin(), LoopProperties.end());
116 
117   // Setting unroll.count
118   if (Attrs.UnrollCount > 0) {
119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120                         ConstantAsMetadata::get(ConstantInt::get(
121                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122     Args.push_back(MDNode::get(Ctx, Vals));
123   }
124 
125   // Setting unroll.full or unroll.disable
126   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128     Args.push_back(MDNode::get(Ctx, Vals));
129   }
130 
131   if (FollowupHasTransforms)
132     Args.push_back(MDNode::get(
133         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134 
135   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136   LoopID->replaceOperandWith(0, LoopID);
137   HasUserTransforms = true;
138   return LoopID;
139 }
140 
141 MDNode *
142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143                                      ArrayRef<Metadata *> LoopProperties,
144                                      bool &HasUserTransforms) {
145   LLVMContext &Ctx = Header->getContext();
146 
147   Optional<bool> Enabled;
148   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
149     Enabled = false;
150   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151            Attrs.UnrollAndJamCount != 0)
152     Enabled = true;
153 
154   if (Enabled != true) {
155     SmallVector<Metadata *, 4> NewLoopProperties;
156     if (Enabled == false) {
157       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158       NewLoopProperties.push_back(MDNode::get(
159           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160       LoopProperties = NewLoopProperties;
161     }
162     return createPartialUnrollMetadata(Attrs, LoopProperties,
163                                        HasUserTransforms);
164   }
165 
166   SmallVector<Metadata *, 4> FollowupLoopProperties;
167   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168   FollowupLoopProperties.push_back(
169       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170 
171   bool FollowupHasTransforms = false;
172   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173                                                  FollowupHasTransforms);
174 
175   SmallVector<Metadata *, 4> Args;
176   Args.push_back(nullptr);
177   Args.append(LoopProperties.begin(), LoopProperties.end());
178 
179   // Setting unroll_and_jam.count
180   if (Attrs.UnrollAndJamCount > 0) {
181     Metadata *Vals[] = {
182         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184                                                  Attrs.UnrollAndJamCount))};
185     Args.push_back(MDNode::get(Ctx, Vals));
186   }
187 
188   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
189     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190     Args.push_back(MDNode::get(Ctx, Vals));
191   }
192 
193   if (FollowupHasTransforms)
194     Args.push_back(MDNode::get(
195         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196               Followup}));
197 
198   if (UnrollAndJamInnerFollowup)
199     Args.push_back(MDNode::get(
200         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201               UnrollAndJamInnerFollowup}));
202 
203   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204   LoopID->replaceOperandWith(0, LoopID);
205   HasUserTransforms = true;
206   return LoopID;
207 }
208 
209 MDNode *
210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211                                       ArrayRef<Metadata *> LoopProperties,
212                                       bool &HasUserTransforms) {
213   LLVMContext &Ctx = Header->getContext();
214 
215   Optional<bool> Enabled;
216   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
217     Enabled = false;
218   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
219            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
220            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
221     Enabled = true;
222 
223   if (Enabled != true) {
224     SmallVector<Metadata *, 4> NewLoopProperties;
225     if (Enabled == false) {
226       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
227       NewLoopProperties.push_back(
228           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
229                             ConstantAsMetadata::get(ConstantInt::get(
230                                 llvm::Type::getInt1Ty(Ctx), 0))}));
231       LoopProperties = NewLoopProperties;
232     }
233     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
234   }
235 
236   // Apply all loop properties to the vectorized loop.
237   SmallVector<Metadata *, 4> FollowupLoopProperties;
238   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
239 
240   // Don't vectorize an already vectorized loop.
241   FollowupLoopProperties.push_back(
242       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
243 
244   bool FollowupHasTransforms = false;
245   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
246                                                 FollowupHasTransforms);
247 
248   SmallVector<Metadata *, 4> Args;
249   Args.push_back(nullptr);
250   Args.append(LoopProperties.begin(), LoopProperties.end());
251 
252   // Setting vectorize.predicate
253   bool IsVectorPredicateEnabled = false;
254   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
255       Attrs.VectorizeEnable != LoopAttributes::Disable &&
256       Attrs.VectorizeWidth < 1) {
257 
258     IsVectorPredicateEnabled =
259         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
260 
261     Metadata *Vals[] = {
262         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
263         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264                                                  IsVectorPredicateEnabled))};
265     Args.push_back(MDNode::get(Ctx, Vals));
266   }
267 
268   // Setting vectorize.width
269   if (Attrs.VectorizeWidth > 0) {
270     Metadata *Vals[] = {
271         MDString::get(Ctx, "llvm.loop.vectorize.width"),
272         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
273                                                  Attrs.VectorizeWidth))};
274     Args.push_back(MDNode::get(Ctx, Vals));
275   }
276 
277   // Setting interleave.count
278   if (Attrs.InterleaveCount > 0) {
279     Metadata *Vals[] = {
280         MDString::get(Ctx, "llvm.loop.interleave.count"),
281         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
282                                                  Attrs.InterleaveCount))};
283     Args.push_back(MDNode::get(Ctx, Vals));
284   }
285 
286   // vectorize.enable is set if:
287   // 1) loop hint vectorize.enable is set, or
288   // 2) it is implied when vectorize.predicate is set, or
289   // 3) it is implied when vectorize.width is set.
290   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
291       IsVectorPredicateEnabled ||
292       Attrs.VectorizeWidth > 1 ) {
293     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
294     Args.push_back(
295         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
296                           ConstantAsMetadata::get(ConstantInt::get(
297                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
298   }
299 
300   if (FollowupHasTransforms)
301     Args.push_back(MDNode::get(
302         Ctx,
303         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
304 
305   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
306   LoopID->replaceOperandWith(0, LoopID);
307   HasUserTransforms = true;
308   return LoopID;
309 }
310 
311 MDNode *
312 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
313                                        ArrayRef<Metadata *> LoopProperties,
314                                        bool &HasUserTransforms) {
315   LLVMContext &Ctx = Header->getContext();
316 
317   Optional<bool> Enabled;
318   if (Attrs.DistributeEnable == LoopAttributes::Disable)
319     Enabled = false;
320   if (Attrs.DistributeEnable == LoopAttributes::Enable)
321     Enabled = true;
322 
323   if (Enabled != true) {
324     SmallVector<Metadata *, 4> NewLoopProperties;
325     if (Enabled == false) {
326       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
327       NewLoopProperties.push_back(
328           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
329                             ConstantAsMetadata::get(ConstantInt::get(
330                                 llvm::Type::getInt1Ty(Ctx), 0))}));
331       LoopProperties = NewLoopProperties;
332     }
333     return createLoopVectorizeMetadata(Attrs, LoopProperties,
334                                        HasUserTransforms);
335   }
336 
337   bool FollowupHasTransforms = false;
338   MDNode *Followup =
339       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
340 
341   SmallVector<Metadata *, 4> Args;
342   Args.push_back(nullptr);
343   Args.append(LoopProperties.begin(), LoopProperties.end());
344 
345   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
346                       ConstantAsMetadata::get(ConstantInt::get(
347                           llvm::Type::getInt1Ty(Ctx),
348                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
349   Args.push_back(MDNode::get(Ctx, Vals));
350 
351   if (FollowupHasTransforms)
352     Args.push_back(MDNode::get(
353         Ctx,
354         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
355 
356   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
357   LoopID->replaceOperandWith(0, LoopID);
358   HasUserTransforms = true;
359   return LoopID;
360 }
361 
362 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
363                                            ArrayRef<Metadata *> LoopProperties,
364                                            bool &HasUserTransforms) {
365   LLVMContext &Ctx = Header->getContext();
366 
367   Optional<bool> Enabled;
368   if (Attrs.UnrollEnable == LoopAttributes::Disable)
369     Enabled = false;
370   else if (Attrs.UnrollEnable == LoopAttributes::Full)
371     Enabled = true;
372 
373   if (Enabled != true) {
374     SmallVector<Metadata *, 4> NewLoopProperties;
375     if (Enabled == false) {
376       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
377       NewLoopProperties.push_back(
378           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
379       LoopProperties = NewLoopProperties;
380     }
381     return createLoopDistributeMetadata(Attrs, LoopProperties,
382                                         HasUserTransforms);
383   }
384 
385   SmallVector<Metadata *, 4> Args;
386   Args.push_back(nullptr);
387   Args.append(LoopProperties.begin(), LoopProperties.end());
388   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
389 
390   // No follow-up: there is no loop after full unrolling.
391   // TODO: Warn if there are transformations after full unrolling.
392 
393   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
394   LoopID->replaceOperandWith(0, LoopID);
395   HasUserTransforms = true;
396   return LoopID;
397 }
398 
399 MDNode *LoopInfo::createMetadata(
400     const LoopAttributes &Attrs,
401     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
402     bool &HasUserTransforms) {
403   SmallVector<Metadata *, 3> LoopProperties;
404 
405   // If we have a valid start debug location for the loop, add it.
406   if (StartLoc) {
407     LoopProperties.push_back(StartLoc.getAsMDNode());
408 
409     // If we also have a valid end debug location for the loop, add it.
410     if (EndLoc)
411       LoopProperties.push_back(EndLoc.getAsMDNode());
412   }
413 
414   LLVMContext &Ctx = Header->getContext();
415   if (Attrs.MustProgress)
416     LoopProperties.push_back(
417         MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
418 
419   assert(!!AccGroup == Attrs.IsParallel &&
420          "There must be an access group iff the loop is parallel");
421   if (Attrs.IsParallel) {
422     LoopProperties.push_back(MDNode::get(
423         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
424   }
425 
426   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
427                         AdditionalLoopProperties.end());
428   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
429 }
430 
431 LoopAttributes::LoopAttributes(bool IsParallel)
432     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
433       UnrollEnable(LoopAttributes::Unspecified),
434       UnrollAndJamEnable(LoopAttributes::Unspecified),
435       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
436       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
437       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
438       PipelineInitiationInterval(0), MustProgress(false) {}
439 
440 void LoopAttributes::clear() {
441   IsParallel = false;
442   VectorizeWidth = 0;
443   InterleaveCount = 0;
444   UnrollCount = 0;
445   UnrollAndJamCount = 0;
446   VectorizeEnable = LoopAttributes::Unspecified;
447   UnrollEnable = LoopAttributes::Unspecified;
448   UnrollAndJamEnable = LoopAttributes::Unspecified;
449   VectorizePredicateEnable = LoopAttributes::Unspecified;
450   DistributeEnable = LoopAttributes::Unspecified;
451   PipelineDisabled = false;
452   PipelineInitiationInterval = 0;
453   MustProgress = false;
454 }
455 
456 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
457                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
458                    LoopInfo *Parent)
459     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
460       Parent(Parent) {
461 
462   if (Attrs.IsParallel) {
463     // Create an access group for this loop.
464     LLVMContext &Ctx = Header->getContext();
465     AccGroup = MDNode::getDistinct(Ctx, {});
466   }
467 
468   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
469       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
470       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
471       Attrs.PipelineInitiationInterval == 0 &&
472       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
473       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
474       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
475       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
476       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
477       !EndLoc && !Attrs.MustProgress)
478     return;
479 
480   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
481 }
482 
483 void LoopInfo::finish() {
484   // We did not annotate the loop body instructions because there are no
485   // attributes for this loop.
486   if (!TempLoopID)
487     return;
488 
489   MDNode *LoopID;
490   LoopAttributes CurLoopAttr = Attrs;
491   LLVMContext &Ctx = Header->getContext();
492 
493   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
494                  Parent->Attrs.UnrollAndJamCount != 0)) {
495     // Parent unroll-and-jams this loop.
496     // Split the transformations in those that happens before the unroll-and-jam
497     // and those after.
498 
499     LoopAttributes BeforeJam, AfterJam;
500 
501     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
502 
503     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
504     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
505     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
506     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
507     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
508 
509     switch (Attrs.UnrollEnable) {
510     case LoopAttributes::Unspecified:
511     case LoopAttributes::Disable:
512       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
513       AfterJam.UnrollEnable = Attrs.UnrollEnable;
514       break;
515     case LoopAttributes::Full:
516       BeforeJam.UnrollEnable = LoopAttributes::Full;
517       break;
518     case LoopAttributes::Enable:
519       AfterJam.UnrollEnable = LoopAttributes::Enable;
520       break;
521     }
522 
523     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
524     AfterJam.UnrollCount = Attrs.UnrollCount;
525     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
526     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
527 
528     // If this loop is subject of an unroll-and-jam by the parent loop, and has
529     // an unroll-and-jam annotation itself, we have to decide whether to first
530     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
531     // UnrollAndJam pass processes loops from inner to outer, so we apply the
532     // inner first.
533     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
534     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
535 
536     // Set the inner followup metadata to process by the outer loop. Only
537     // consider the first inner loop.
538     if (!Parent->UnrollAndJamInnerFollowup) {
539       // Splitting the attributes into a BeforeJam and an AfterJam part will
540       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
541       // to be forwarded to the AfterJam part. We detect the situation here and
542       // add it manually.
543       SmallVector<Metadata *, 1> BeforeLoopProperties;
544       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
545           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
546           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
547         BeforeLoopProperties.push_back(
548             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
549 
550       bool InnerFollowupHasTransform = false;
551       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
552                                              InnerFollowupHasTransform);
553       if (InnerFollowupHasTransform)
554         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
555     }
556 
557     CurLoopAttr = BeforeJam;
558   }
559 
560   bool HasUserTransforms = false;
561   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
562   TempLoopID->replaceAllUsesWith(LoopID);
563 }
564 
565 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
566                          const llvm::DebugLoc &EndLoc) {
567   Active.emplace_back(
568       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
569                    Active.empty() ? nullptr : Active.back().get()));
570   // Clear the attributes so nested loops do not inherit them.
571   StagedAttrs.clear();
572 }
573 
574 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
575                          const clang::CodeGenOptions &CGOpts,
576                          ArrayRef<const clang::Attr *> Attrs,
577                          const llvm::DebugLoc &StartLoc,
578                          const llvm::DebugLoc &EndLoc, bool MustProgress) {
579   // Identify loop hint attributes from Attrs.
580   for (const auto *Attr : Attrs) {
581     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
582     const OpenCLUnrollHintAttr *OpenCLHint =
583         dyn_cast<OpenCLUnrollHintAttr>(Attr);
584 
585     // Skip non loop hint attributes
586     if (!LH && !OpenCLHint) {
587       continue;
588     }
589 
590     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
591     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
592     unsigned ValueInt = 1;
593     // Translate opencl_unroll_hint attribute argument to
594     // equivalent LoopHintAttr enums.
595     // OpenCL v2.0 s6.11.5:
596     // 0 - enable unroll (no argument).
597     // 1 - disable unroll.
598     // other positive integer n - unroll by n.
599     if (OpenCLHint) {
600       ValueInt = OpenCLHint->getUnrollHint();
601       if (ValueInt == 0) {
602         State = LoopHintAttr::Enable;
603       } else if (ValueInt != 1) {
604         Option = LoopHintAttr::UnrollCount;
605         State = LoopHintAttr::Numeric;
606       }
607     } else if (LH) {
608       auto *ValueExpr = LH->getValue();
609       if (ValueExpr) {
610         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
611         ValueInt = ValueAPS.getSExtValue();
612       }
613 
614       Option = LH->getOption();
615       State = LH->getState();
616     }
617     switch (State) {
618     case LoopHintAttr::Disable:
619       switch (Option) {
620       case LoopHintAttr::Vectorize:
621         // Disable vectorization by specifying a width of 1.
622         setVectorizeWidth(1);
623         break;
624       case LoopHintAttr::Interleave:
625         // Disable interleaving by speciyfing a count of 1.
626         setInterleaveCount(1);
627         break;
628       case LoopHintAttr::Unroll:
629         setUnrollState(LoopAttributes::Disable);
630         break;
631       case LoopHintAttr::UnrollAndJam:
632         setUnrollAndJamState(LoopAttributes::Disable);
633         break;
634       case LoopHintAttr::VectorizePredicate:
635         setVectorizePredicateState(LoopAttributes::Disable);
636         break;
637       case LoopHintAttr::Distribute:
638         setDistributeState(false);
639         break;
640       case LoopHintAttr::PipelineDisabled:
641         setPipelineDisabled(true);
642         break;
643       case LoopHintAttr::UnrollCount:
644       case LoopHintAttr::UnrollAndJamCount:
645       case LoopHintAttr::VectorizeWidth:
646       case LoopHintAttr::InterleaveCount:
647       case LoopHintAttr::PipelineInitiationInterval:
648         llvm_unreachable("Options cannot be disabled.");
649         break;
650       }
651       break;
652     case LoopHintAttr::Enable:
653       switch (Option) {
654       case LoopHintAttr::Vectorize:
655       case LoopHintAttr::Interleave:
656         setVectorizeEnable(true);
657         break;
658       case LoopHintAttr::Unroll:
659         setUnrollState(LoopAttributes::Enable);
660         break;
661       case LoopHintAttr::UnrollAndJam:
662         setUnrollAndJamState(LoopAttributes::Enable);
663         break;
664       case LoopHintAttr::VectorizePredicate:
665         setVectorizePredicateState(LoopAttributes::Enable);
666         break;
667       case LoopHintAttr::Distribute:
668         setDistributeState(true);
669         break;
670       case LoopHintAttr::UnrollCount:
671       case LoopHintAttr::UnrollAndJamCount:
672       case LoopHintAttr::VectorizeWidth:
673       case LoopHintAttr::InterleaveCount:
674       case LoopHintAttr::PipelineDisabled:
675       case LoopHintAttr::PipelineInitiationInterval:
676         llvm_unreachable("Options cannot enabled.");
677         break;
678       }
679       break;
680     case LoopHintAttr::AssumeSafety:
681       switch (Option) {
682       case LoopHintAttr::Vectorize:
683       case LoopHintAttr::Interleave:
684         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
685         setParallel(true);
686         setVectorizeEnable(true);
687         break;
688       case LoopHintAttr::Unroll:
689       case LoopHintAttr::UnrollAndJam:
690       case LoopHintAttr::VectorizePredicate:
691       case LoopHintAttr::UnrollCount:
692       case LoopHintAttr::UnrollAndJamCount:
693       case LoopHintAttr::VectorizeWidth:
694       case LoopHintAttr::InterleaveCount:
695       case LoopHintAttr::Distribute:
696       case LoopHintAttr::PipelineDisabled:
697       case LoopHintAttr::PipelineInitiationInterval:
698         llvm_unreachable("Options cannot be used to assume mem safety.");
699         break;
700       }
701       break;
702     case LoopHintAttr::Full:
703       switch (Option) {
704       case LoopHintAttr::Unroll:
705         setUnrollState(LoopAttributes::Full);
706         break;
707       case LoopHintAttr::UnrollAndJam:
708         setUnrollAndJamState(LoopAttributes::Full);
709         break;
710       case LoopHintAttr::Vectorize:
711       case LoopHintAttr::Interleave:
712       case LoopHintAttr::UnrollCount:
713       case LoopHintAttr::UnrollAndJamCount:
714       case LoopHintAttr::VectorizeWidth:
715       case LoopHintAttr::InterleaveCount:
716       case LoopHintAttr::Distribute:
717       case LoopHintAttr::PipelineDisabled:
718       case LoopHintAttr::PipelineInitiationInterval:
719       case LoopHintAttr::VectorizePredicate:
720         llvm_unreachable("Options cannot be used with 'full' hint.");
721         break;
722       }
723       break;
724     case LoopHintAttr::Numeric:
725       switch (Option) {
726       case LoopHintAttr::VectorizeWidth:
727         setVectorizeWidth(ValueInt);
728         break;
729       case LoopHintAttr::InterleaveCount:
730         setInterleaveCount(ValueInt);
731         break;
732       case LoopHintAttr::UnrollCount:
733         setUnrollCount(ValueInt);
734         break;
735       case LoopHintAttr::UnrollAndJamCount:
736         setUnrollAndJamCount(ValueInt);
737         break;
738       case LoopHintAttr::PipelineInitiationInterval:
739         setPipelineInitiationInterval(ValueInt);
740         break;
741       case LoopHintAttr::Unroll:
742       case LoopHintAttr::UnrollAndJam:
743       case LoopHintAttr::VectorizePredicate:
744       case LoopHintAttr::Vectorize:
745       case LoopHintAttr::Interleave:
746       case LoopHintAttr::Distribute:
747       case LoopHintAttr::PipelineDisabled:
748         llvm_unreachable("Options cannot be assigned a value.");
749         break;
750       }
751       break;
752     }
753   }
754 
755   setMustProgress(MustProgress);
756 
757   if (CGOpts.OptimizationLevel > 0)
758     // Disable unrolling for the loop, if unrolling is disabled (via
759     // -fno-unroll-loops) and no pragmas override the decision.
760     if (!CGOpts.UnrollLoops &&
761         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
762          StagedAttrs.UnrollCount == 0))
763       setUnrollState(LoopAttributes::Disable);
764 
765   /// Stage the attributes.
766   push(Header, StartLoc, EndLoc);
767 }
768 
769 void LoopInfoStack::pop() {
770   assert(!Active.empty() && "No active loops to pop");
771   Active.back()->finish();
772   Active.pop_back();
773 }
774 
775 void LoopInfoStack::InsertHelper(Instruction *I) const {
776   if (I->mayReadOrWriteMemory()) {
777     SmallVector<Metadata *, 4> AccessGroups;
778     for (const auto &AL : Active) {
779       // Here we assume that every loop that has an access group is parallel.
780       if (MDNode *Group = AL->getAccessGroup())
781         AccessGroups.push_back(Group);
782     }
783     MDNode *UnionMD = nullptr;
784     if (AccessGroups.size() == 1)
785       UnionMD = cast<MDNode>(AccessGroups[0]);
786     else if (AccessGroups.size() >= 2)
787       UnionMD = MDNode::get(I->getContext(), AccessGroups);
788     I->setMetadata("llvm.access.group", UnionMD);
789   }
790 
791   if (!hasInfo())
792     return;
793 
794   const LoopInfo &L = getInfo();
795   if (!L.getLoopID())
796     return;
797 
798   if (I->isTerminator()) {
799     for (BasicBlock *Succ : successors(I))
800       if (Succ == L.getHeader()) {
801         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
802         break;
803       }
804     return;
805   }
806 }
807