1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "llvm/IR/BasicBlock.h"
14 #include "llvm/IR/CFG.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
20 using namespace llvm;
21 
22 MDNode *
23 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
24   LLVMContext &Ctx = Header->getContext();
25   SmallVector<Metadata *, 4> NewLoopProperties;
26   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
27   NewLoopProperties.push_back(TempNode.get());
28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31   LoopID->replaceOperandWith(0, LoopID);
32   return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36                                            ArrayRef<Metadata *> LoopProperties,
37                                            bool &HasUserTransforms) {
38   LLVMContext &Ctx = Header->getContext();
39 
40   Optional<bool> Enabled;
41   if (Attrs.PipelineDisabled)
42     Enabled = false;
43   else if (Attrs.PipelineInitiationInterval != 0)
44     Enabled = true;
45 
46   if (Enabled != true) {
47     SmallVector<Metadata *, 4> NewLoopProperties;
48     if (Enabled == false) {
49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50       NewLoopProperties.push_back(
51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52                             ConstantAsMetadata::get(ConstantInt::get(
53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
54       LoopProperties = NewLoopProperties;
55     }
56     return createLoopPropertiesMetadata(LoopProperties);
57   }
58 
59   SmallVector<Metadata *, 4> Args;
60   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
61   Args.push_back(TempNode.get());
62   Args.append(LoopProperties.begin(), LoopProperties.end());
63 
64   if (Attrs.PipelineInitiationInterval > 0) {
65     Metadata *Vals[] = {
66         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
67         ConstantAsMetadata::get(ConstantInt::get(
68             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
69     Args.push_back(MDNode::get(Ctx, Vals));
70   }
71 
72   // No follow-up: This is the last transformation.
73 
74   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
75   LoopID->replaceOperandWith(0, LoopID);
76   HasUserTransforms = true;
77   return LoopID;
78 }
79 
80 MDNode *
81 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
82                                       ArrayRef<Metadata *> LoopProperties,
83                                       bool &HasUserTransforms) {
84   LLVMContext &Ctx = Header->getContext();
85 
86   Optional<bool> Enabled;
87   if (Attrs.UnrollEnable == LoopAttributes::Disable)
88     Enabled = false;
89   else if (Attrs.UnrollEnable == LoopAttributes::Full)
90     Enabled = None;
91   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
92            Attrs.UnrollCount != 0)
93     Enabled = true;
94 
95   if (Enabled != true) {
96     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
97     // if unrolling is disabled.
98     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
99   }
100 
101   SmallVector<Metadata *, 4> FollowupLoopProperties;
102 
103   // Apply all loop properties to the unrolled loop.
104   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
105 
106   // Don't unroll an already unrolled loop.
107   FollowupLoopProperties.push_back(
108       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
109 
110   bool FollowupHasTransforms = false;
111   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
112                                               FollowupHasTransforms);
113 
114   SmallVector<Metadata *, 4> Args;
115   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
116   Args.push_back(TempNode.get());
117   Args.append(LoopProperties.begin(), LoopProperties.end());
118 
119   // Setting unroll.count
120   if (Attrs.UnrollCount > 0) {
121     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
122                         ConstantAsMetadata::get(ConstantInt::get(
123                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
124     Args.push_back(MDNode::get(Ctx, Vals));
125   }
126 
127   // Setting unroll.full or unroll.disable
128   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
129     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
130     Args.push_back(MDNode::get(Ctx, Vals));
131   }
132 
133   if (FollowupHasTransforms)
134     Args.push_back(MDNode::get(
135         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
136 
137   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
138   LoopID->replaceOperandWith(0, LoopID);
139   HasUserTransforms = true;
140   return LoopID;
141 }
142 
143 MDNode *
144 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
145                                      ArrayRef<Metadata *> LoopProperties,
146                                      bool &HasUserTransforms) {
147   LLVMContext &Ctx = Header->getContext();
148 
149   Optional<bool> Enabled;
150   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
151     Enabled = false;
152   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
153            Attrs.UnrollAndJamCount != 0)
154     Enabled = true;
155 
156   if (Enabled != true) {
157     SmallVector<Metadata *, 4> NewLoopProperties;
158     if (Enabled == false) {
159       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
160       NewLoopProperties.push_back(MDNode::get(
161           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
162       LoopProperties = NewLoopProperties;
163     }
164     return createPartialUnrollMetadata(Attrs, LoopProperties,
165                                        HasUserTransforms);
166   }
167 
168   SmallVector<Metadata *, 4> FollowupLoopProperties;
169   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
170   FollowupLoopProperties.push_back(
171       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
172 
173   bool FollowupHasTransforms = false;
174   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
175                                                  FollowupHasTransforms);
176 
177   SmallVector<Metadata *, 4> Args;
178   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
179   Args.push_back(TempNode.get());
180   Args.append(LoopProperties.begin(), LoopProperties.end());
181 
182   // Setting unroll_and_jam.count
183   if (Attrs.UnrollAndJamCount > 0) {
184     Metadata *Vals[] = {
185         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
186         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
187                                                  Attrs.UnrollAndJamCount))};
188     Args.push_back(MDNode::get(Ctx, Vals));
189   }
190 
191   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
192     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
193     Args.push_back(MDNode::get(Ctx, Vals));
194   }
195 
196   if (FollowupHasTransforms)
197     Args.push_back(MDNode::get(
198         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
199               Followup}));
200 
201   if (UnrollAndJamInnerFollowup)
202     Args.push_back(MDNode::get(
203         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
204               UnrollAndJamInnerFollowup}));
205 
206   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
207   LoopID->replaceOperandWith(0, LoopID);
208   HasUserTransforms = true;
209   return LoopID;
210 }
211 
212 MDNode *
213 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
214                                       ArrayRef<Metadata *> LoopProperties,
215                                       bool &HasUserTransforms) {
216   LLVMContext &Ctx = Header->getContext();
217 
218   Optional<bool> Enabled;
219   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
220     Enabled = false;
221   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
222            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
223            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
224     Enabled = true;
225 
226   if (Enabled != true) {
227     SmallVector<Metadata *, 4> NewLoopProperties;
228     if (Enabled == false) {
229       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
230       NewLoopProperties.push_back(
231           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
232                             ConstantAsMetadata::get(ConstantInt::get(
233                                 llvm::Type::getInt1Ty(Ctx), 0))}));
234       LoopProperties = NewLoopProperties;
235     }
236     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
237   }
238 
239   // Apply all loop properties to the vectorized loop.
240   SmallVector<Metadata *, 4> FollowupLoopProperties;
241   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
242 
243   // Don't vectorize an already vectorized loop.
244   FollowupLoopProperties.push_back(
245       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
246 
247   bool FollowupHasTransforms = false;
248   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
249                                                 FollowupHasTransforms);
250 
251   SmallVector<Metadata *, 4> Args;
252   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
253   Args.push_back(TempNode.get());
254   Args.append(LoopProperties.begin(), LoopProperties.end());
255 
256   // Setting vectorize.predicate
257   bool IsVectorPredicateEnabled = false;
258   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
259       Attrs.VectorizeEnable != LoopAttributes::Disable &&
260       Attrs.VectorizeWidth < 1) {
261 
262     IsVectorPredicateEnabled =
263         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
264 
265     Metadata *Vals[] = {
266         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
267         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
268                                                  IsVectorPredicateEnabled))};
269     Args.push_back(MDNode::get(Ctx, Vals));
270   }
271 
272   // Setting vectorize.width
273   if (Attrs.VectorizeWidth > 0) {
274     Metadata *Vals[] = {
275         MDString::get(Ctx, "llvm.loop.vectorize.width"),
276         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
277                                                  Attrs.VectorizeWidth))};
278     Args.push_back(MDNode::get(Ctx, Vals));
279   }
280 
281   // Setting interleave.count
282   if (Attrs.InterleaveCount > 0) {
283     Metadata *Vals[] = {
284         MDString::get(Ctx, "llvm.loop.interleave.count"),
285         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
286                                                  Attrs.InterleaveCount))};
287     Args.push_back(MDNode::get(Ctx, Vals));
288   }
289 
290   // vectorize.enable is set if:
291   // 1) loop hint vectorize.enable is set, or
292   // 2) it is implied when vectorize.predicate is set, or
293   // 3) it is implied when vectorize.width is set.
294   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
295       IsVectorPredicateEnabled ||
296       Attrs.VectorizeWidth > 1 ) {
297     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
298     Args.push_back(
299         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
300                           ConstantAsMetadata::get(ConstantInt::get(
301                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
302   }
303 
304   if (FollowupHasTransforms)
305     Args.push_back(MDNode::get(
306         Ctx,
307         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
308 
309   MDNode *LoopID = MDNode::get(Ctx, Args);
310   LoopID->replaceOperandWith(0, LoopID);
311   HasUserTransforms = true;
312   return LoopID;
313 }
314 
315 MDNode *
316 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
317                                        ArrayRef<Metadata *> LoopProperties,
318                                        bool &HasUserTransforms) {
319   LLVMContext &Ctx = Header->getContext();
320 
321   Optional<bool> Enabled;
322   if (Attrs.DistributeEnable == LoopAttributes::Disable)
323     Enabled = false;
324   if (Attrs.DistributeEnable == LoopAttributes::Enable)
325     Enabled = true;
326 
327   if (Enabled != true) {
328     SmallVector<Metadata *, 4> NewLoopProperties;
329     if (Enabled == false) {
330       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
331       NewLoopProperties.push_back(
332           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
333                             ConstantAsMetadata::get(ConstantInt::get(
334                                 llvm::Type::getInt1Ty(Ctx), 0))}));
335       LoopProperties = NewLoopProperties;
336     }
337     return createLoopVectorizeMetadata(Attrs, LoopProperties,
338                                        HasUserTransforms);
339   }
340 
341   bool FollowupHasTransforms = false;
342   MDNode *Followup =
343       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
344 
345   SmallVector<Metadata *, 4> Args;
346   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
347   Args.push_back(TempNode.get());
348   Args.append(LoopProperties.begin(), LoopProperties.end());
349 
350   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
351                       ConstantAsMetadata::get(ConstantInt::get(
352                           llvm::Type::getInt1Ty(Ctx),
353                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
354   Args.push_back(MDNode::get(Ctx, Vals));
355 
356   if (FollowupHasTransforms)
357     Args.push_back(MDNode::get(
358         Ctx,
359         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
360 
361   MDNode *LoopID = MDNode::get(Ctx, Args);
362   LoopID->replaceOperandWith(0, LoopID);
363   HasUserTransforms = true;
364   return LoopID;
365 }
366 
367 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
368                                            ArrayRef<Metadata *> LoopProperties,
369                                            bool &HasUserTransforms) {
370   LLVMContext &Ctx = Header->getContext();
371 
372   Optional<bool> Enabled;
373   if (Attrs.UnrollEnable == LoopAttributes::Disable)
374     Enabled = false;
375   else if (Attrs.UnrollEnable == LoopAttributes::Full)
376     Enabled = true;
377 
378   if (Enabled != true) {
379     SmallVector<Metadata *, 4> NewLoopProperties;
380     if (Enabled == false) {
381       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
382       NewLoopProperties.push_back(
383           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
384       LoopProperties = NewLoopProperties;
385     }
386     return createLoopDistributeMetadata(Attrs, LoopProperties,
387                                         HasUserTransforms);
388   }
389 
390   SmallVector<Metadata *, 4> Args;
391   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
392   Args.push_back(TempNode.get());
393   Args.append(LoopProperties.begin(), LoopProperties.end());
394   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
395 
396   // No follow-up: there is no loop after full unrolling.
397   // TODO: Warn if there are transformations after full unrolling.
398 
399   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
400   LoopID->replaceOperandWith(0, LoopID);
401   HasUserTransforms = true;
402   return LoopID;
403 }
404 
405 MDNode *LoopInfo::createMetadata(
406     const LoopAttributes &Attrs,
407     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
408     bool &HasUserTransforms) {
409   SmallVector<Metadata *, 3> LoopProperties;
410 
411   // If we have a valid start debug location for the loop, add it.
412   if (StartLoc) {
413     LoopProperties.push_back(StartLoc.getAsMDNode());
414 
415     // If we also have a valid end debug location for the loop, add it.
416     if (EndLoc)
417       LoopProperties.push_back(EndLoc.getAsMDNode());
418   }
419 
420   assert(!!AccGroup == Attrs.IsParallel &&
421          "There must be an access group iff the loop is parallel");
422   if (Attrs.IsParallel) {
423     LLVMContext &Ctx = Header->getContext();
424     LoopProperties.push_back(MDNode::get(
425         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
426   }
427 
428   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
429                         AdditionalLoopProperties.end());
430   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
431 }
432 
433 LoopAttributes::LoopAttributes(bool IsParallel)
434     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
435       UnrollEnable(LoopAttributes::Unspecified),
436       UnrollAndJamEnable(LoopAttributes::Unspecified),
437       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
438       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
439       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
440       PipelineInitiationInterval(0) {}
441 
442 void LoopAttributes::clear() {
443   IsParallel = false;
444   VectorizeWidth = 0;
445   InterleaveCount = 0;
446   UnrollCount = 0;
447   UnrollAndJamCount = 0;
448   VectorizeEnable = LoopAttributes::Unspecified;
449   UnrollEnable = LoopAttributes::Unspecified;
450   UnrollAndJamEnable = LoopAttributes::Unspecified;
451   VectorizePredicateEnable = LoopAttributes::Unspecified;
452   DistributeEnable = LoopAttributes::Unspecified;
453   PipelineDisabled = false;
454   PipelineInitiationInterval = 0;
455 }
456 
457 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
458                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
459                    LoopInfo *Parent)
460     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
461       Parent(Parent) {
462 
463   if (Attrs.IsParallel) {
464     // Create an access group for this loop.
465     LLVMContext &Ctx = Header->getContext();
466     AccGroup = MDNode::getDistinct(Ctx, {});
467   }
468 
469   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
470       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
471       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
472       Attrs.PipelineInitiationInterval == 0 &&
473       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
474       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
475       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
476       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
477       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
478       !EndLoc)
479     return;
480 
481   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
482 }
483 
484 void LoopInfo::finish() {
485   // We did not annotate the loop body instructions because there are no
486   // attributes for this loop.
487   if (!TempLoopID)
488     return;
489 
490   MDNode *LoopID;
491   LoopAttributes CurLoopAttr = Attrs;
492   LLVMContext &Ctx = Header->getContext();
493 
494   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
495                  Parent->Attrs.UnrollAndJamCount != 0)) {
496     // Parent unroll-and-jams this loop.
497     // Split the transformations in those that happens before the unroll-and-jam
498     // and those after.
499 
500     LoopAttributes BeforeJam, AfterJam;
501 
502     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
503 
504     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
505     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
506     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
507     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
508     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
509 
510     switch (Attrs.UnrollEnable) {
511     case LoopAttributes::Unspecified:
512     case LoopAttributes::Disable:
513       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
514       AfterJam.UnrollEnable = Attrs.UnrollEnable;
515       break;
516     case LoopAttributes::Full:
517       BeforeJam.UnrollEnable = LoopAttributes::Full;
518       break;
519     case LoopAttributes::Enable:
520       AfterJam.UnrollEnable = LoopAttributes::Enable;
521       break;
522     }
523 
524     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
525     AfterJam.UnrollCount = Attrs.UnrollCount;
526     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
527     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
528 
529     // If this loop is subject of an unroll-and-jam by the parent loop, and has
530     // an unroll-and-jam annotation itself, we have to decide whether to first
531     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
532     // UnrollAndJam pass processes loops from inner to outer, so we apply the
533     // inner first.
534     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
535     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
536 
537     // Set the inner followup metadata to process by the outer loop. Only
538     // consider the first inner loop.
539     if (!Parent->UnrollAndJamInnerFollowup) {
540       // Splitting the attributes into a BeforeJam and an AfterJam part will
541       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
542       // to be forwarded to the AfterJam part. We detect the situation here and
543       // add it manually.
544       SmallVector<Metadata *, 1> BeforeLoopProperties;
545       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
546           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
547           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
548         BeforeLoopProperties.push_back(
549             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
550 
551       bool InnerFollowupHasTransform = false;
552       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
553                                              InnerFollowupHasTransform);
554       if (InnerFollowupHasTransform)
555         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
556     }
557 
558     CurLoopAttr = BeforeJam;
559   }
560 
561   bool HasUserTransforms = false;
562   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
563   TempLoopID->replaceAllUsesWith(LoopID);
564 }
565 
566 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
567                          const llvm::DebugLoc &EndLoc) {
568   Active.emplace_back(
569       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
570                    Active.empty() ? nullptr : Active.back().get()));
571   // Clear the attributes so nested loops do not inherit them.
572   StagedAttrs.clear();
573 }
574 
575 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
576                          ArrayRef<const clang::Attr *> Attrs,
577                          const llvm::DebugLoc &StartLoc,
578                          const llvm::DebugLoc &EndLoc) {
579 
580   // Identify loop hint attributes from Attrs.
581   for (const auto *Attr : Attrs) {
582     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
583     const OpenCLUnrollHintAttr *OpenCLHint =
584         dyn_cast<OpenCLUnrollHintAttr>(Attr);
585 
586     // Skip non loop hint attributes
587     if (!LH && !OpenCLHint) {
588       continue;
589     }
590 
591     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
592     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
593     unsigned ValueInt = 1;
594     // Translate opencl_unroll_hint attribute argument to
595     // equivalent LoopHintAttr enums.
596     // OpenCL v2.0 s6.11.5:
597     // 0 - enable unroll (no argument).
598     // 1 - disable unroll.
599     // other positive integer n - unroll by n.
600     if (OpenCLHint) {
601       ValueInt = OpenCLHint->getUnrollHint();
602       if (ValueInt == 0) {
603         State = LoopHintAttr::Enable;
604       } else if (ValueInt != 1) {
605         Option = LoopHintAttr::UnrollCount;
606         State = LoopHintAttr::Numeric;
607       }
608     } else if (LH) {
609       auto *ValueExpr = LH->getValue();
610       if (ValueExpr) {
611         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
612         ValueInt = ValueAPS.getSExtValue();
613       }
614 
615       Option = LH->getOption();
616       State = LH->getState();
617     }
618     switch (State) {
619     case LoopHintAttr::Disable:
620       switch (Option) {
621       case LoopHintAttr::Vectorize:
622         // Disable vectorization by specifying a width of 1.
623         setVectorizeWidth(1);
624         break;
625       case LoopHintAttr::Interleave:
626         // Disable interleaving by speciyfing a count of 1.
627         setInterleaveCount(1);
628         break;
629       case LoopHintAttr::Unroll:
630         setUnrollState(LoopAttributes::Disable);
631         break;
632       case LoopHintAttr::UnrollAndJam:
633         setUnrollAndJamState(LoopAttributes::Disable);
634         break;
635       case LoopHintAttr::VectorizePredicate:
636         setVectorizePredicateState(LoopAttributes::Disable);
637         break;
638       case LoopHintAttr::Distribute:
639         setDistributeState(false);
640         break;
641       case LoopHintAttr::PipelineDisabled:
642         setPipelineDisabled(true);
643         break;
644       case LoopHintAttr::UnrollCount:
645       case LoopHintAttr::UnrollAndJamCount:
646       case LoopHintAttr::VectorizeWidth:
647       case LoopHintAttr::InterleaveCount:
648       case LoopHintAttr::PipelineInitiationInterval:
649         llvm_unreachable("Options cannot be disabled.");
650         break;
651       }
652       break;
653     case LoopHintAttr::Enable:
654       switch (Option) {
655       case LoopHintAttr::Vectorize:
656       case LoopHintAttr::Interleave:
657         setVectorizeEnable(true);
658         break;
659       case LoopHintAttr::Unroll:
660         setUnrollState(LoopAttributes::Enable);
661         break;
662       case LoopHintAttr::UnrollAndJam:
663         setUnrollAndJamState(LoopAttributes::Enable);
664         break;
665       case LoopHintAttr::VectorizePredicate:
666         setVectorizePredicateState(LoopAttributes::Enable);
667         break;
668       case LoopHintAttr::Distribute:
669         setDistributeState(true);
670         break;
671       case LoopHintAttr::UnrollCount:
672       case LoopHintAttr::UnrollAndJamCount:
673       case LoopHintAttr::VectorizeWidth:
674       case LoopHintAttr::InterleaveCount:
675       case LoopHintAttr::PipelineDisabled:
676       case LoopHintAttr::PipelineInitiationInterval:
677         llvm_unreachable("Options cannot enabled.");
678         break;
679       }
680       break;
681     case LoopHintAttr::AssumeSafety:
682       switch (Option) {
683       case LoopHintAttr::Vectorize:
684       case LoopHintAttr::Interleave:
685         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
686         setParallel(true);
687         setVectorizeEnable(true);
688         break;
689       case LoopHintAttr::Unroll:
690       case LoopHintAttr::UnrollAndJam:
691       case LoopHintAttr::VectorizePredicate:
692       case LoopHintAttr::UnrollCount:
693       case LoopHintAttr::UnrollAndJamCount:
694       case LoopHintAttr::VectorizeWidth:
695       case LoopHintAttr::InterleaveCount:
696       case LoopHintAttr::Distribute:
697       case LoopHintAttr::PipelineDisabled:
698       case LoopHintAttr::PipelineInitiationInterval:
699         llvm_unreachable("Options cannot be used to assume mem safety.");
700         break;
701       }
702       break;
703     case LoopHintAttr::Full:
704       switch (Option) {
705       case LoopHintAttr::Unroll:
706         setUnrollState(LoopAttributes::Full);
707         break;
708       case LoopHintAttr::UnrollAndJam:
709         setUnrollAndJamState(LoopAttributes::Full);
710         break;
711       case LoopHintAttr::Vectorize:
712       case LoopHintAttr::Interleave:
713       case LoopHintAttr::UnrollCount:
714       case LoopHintAttr::UnrollAndJamCount:
715       case LoopHintAttr::VectorizeWidth:
716       case LoopHintAttr::InterleaveCount:
717       case LoopHintAttr::Distribute:
718       case LoopHintAttr::PipelineDisabled:
719       case LoopHintAttr::PipelineInitiationInterval:
720       case LoopHintAttr::VectorizePredicate:
721         llvm_unreachable("Options cannot be used with 'full' hint.");
722         break;
723       }
724       break;
725     case LoopHintAttr::Numeric:
726       switch (Option) {
727       case LoopHintAttr::VectorizeWidth:
728         setVectorizeWidth(ValueInt);
729         break;
730       case LoopHintAttr::InterleaveCount:
731         setInterleaveCount(ValueInt);
732         break;
733       case LoopHintAttr::UnrollCount:
734         setUnrollCount(ValueInt);
735         break;
736       case LoopHintAttr::UnrollAndJamCount:
737         setUnrollAndJamCount(ValueInt);
738         break;
739       case LoopHintAttr::PipelineInitiationInterval:
740         setPipelineInitiationInterval(ValueInt);
741         break;
742       case LoopHintAttr::Unroll:
743       case LoopHintAttr::UnrollAndJam:
744       case LoopHintAttr::VectorizePredicate:
745       case LoopHintAttr::Vectorize:
746       case LoopHintAttr::Interleave:
747       case LoopHintAttr::Distribute:
748       case LoopHintAttr::PipelineDisabled:
749         llvm_unreachable("Options cannot be assigned a value.");
750         break;
751       }
752       break;
753     }
754   }
755 
756   /// Stage the attributes.
757   push(Header, StartLoc, EndLoc);
758 }
759 
760 void LoopInfoStack::pop() {
761   assert(!Active.empty() && "No active loops to pop");
762   Active.back()->finish();
763   Active.pop_back();
764 }
765 
766 void LoopInfoStack::InsertHelper(Instruction *I) const {
767   if (I->mayReadOrWriteMemory()) {
768     SmallVector<Metadata *, 4> AccessGroups;
769     for (const auto &AL : Active) {
770       // Here we assume that every loop that has an access group is parallel.
771       if (MDNode *Group = AL->getAccessGroup())
772         AccessGroups.push_back(Group);
773     }
774     MDNode *UnionMD = nullptr;
775     if (AccessGroups.size() == 1)
776       UnionMD = cast<MDNode>(AccessGroups[0]);
777     else if (AccessGroups.size() >= 2)
778       UnionMD = MDNode::get(I->getContext(), AccessGroups);
779     I->setMetadata("llvm.access.group", UnionMD);
780   }
781 
782   if (!hasInfo())
783     return;
784 
785   const LoopInfo &L = getInfo();
786   if (!L.getLoopID())
787     return;
788 
789   if (I->isTerminator()) {
790     for (BasicBlock *Succ : successors(I))
791       if (Succ == L.getHeader()) {
792         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
793         break;
794       }
795     return;
796   }
797 }
798