1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/CFG.h"
14 #include "llvm/IR/Constants.h"
15 #include "llvm/IR/InstrTypes.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/Metadata.h"
18 using namespace clang::CodeGen;
19 using namespace llvm;
20 
21 MDNode *
22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23   LLVMContext &Ctx = Header->getContext();
24   SmallVector<Metadata *, 4> NewLoopProperties;
25   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26   NewLoopProperties.push_back(TempNode.get());
27   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28 
29   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30   LoopID->replaceOperandWith(0, LoopID);
31   return LoopID;
32 }
33 
34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35                                            ArrayRef<Metadata *> LoopProperties,
36                                            bool &HasUserTransforms) {
37   LLVMContext &Ctx = Header->getContext();
38 
39   Optional<bool> Enabled;
40   if (Attrs.PipelineDisabled)
41     Enabled = false;
42   else if (Attrs.PipelineInitiationInterval != 0)
43     Enabled = true;
44 
45   if (Enabled != true) {
46     SmallVector<Metadata *, 4> NewLoopProperties;
47     if (Enabled == false) {
48       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49       NewLoopProperties.push_back(
50           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51                             ConstantAsMetadata::get(ConstantInt::get(
52                                 llvm::Type::getInt1Ty(Ctx), 1))}));
53       LoopProperties = NewLoopProperties;
54     }
55     return createLoopPropertiesMetadata(LoopProperties);
56   }
57 
58   SmallVector<Metadata *, 4> Args;
59   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60   Args.push_back(TempNode.get());
61   Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63   if (Attrs.PipelineInitiationInterval > 0) {
64     Metadata *Vals[] = {
65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66         ConstantAsMetadata::get(ConstantInt::get(
67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68     Args.push_back(MDNode::get(Ctx, Vals));
69   }
70 
71   // No follow-up: This is the last transformation.
72 
73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74   LoopID->replaceOperandWith(0, LoopID);
75   HasUserTransforms = true;
76   return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81                                       ArrayRef<Metadata *> LoopProperties,
82                                       bool &HasUserTransforms) {
83   LLVMContext &Ctx = Header->getContext();
84 
85   Optional<bool> Enabled;
86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
87     Enabled = false;
88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
89     Enabled = None;
90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91            Attrs.UnrollCount != 0)
92     Enabled = true;
93 
94   if (Enabled != true) {
95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96     // if unrolling is disabled.
97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98   }
99 
100   SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102   // Apply all loop properties to the unrolled loop.
103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105   // Don't unroll an already unrolled loop.
106   FollowupLoopProperties.push_back(
107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109   bool FollowupHasTransforms = false;
110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111                                               FollowupHasTransforms);
112 
113   SmallVector<Metadata *, 4> Args;
114   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115   Args.push_back(TempNode.get());
116   Args.append(LoopProperties.begin(), LoopProperties.end());
117 
118   // Setting unroll.count
119   if (Attrs.UnrollCount > 0) {
120     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121                         ConstantAsMetadata::get(ConstantInt::get(
122                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123     Args.push_back(MDNode::get(Ctx, Vals));
124   }
125 
126   // Setting unroll.full or unroll.disable
127   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129     Args.push_back(MDNode::get(Ctx, Vals));
130   }
131 
132   if (FollowupHasTransforms)
133     Args.push_back(MDNode::get(
134         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135 
136   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137   LoopID->replaceOperandWith(0, LoopID);
138   HasUserTransforms = true;
139   return LoopID;
140 }
141 
142 MDNode *
143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144                                      ArrayRef<Metadata *> LoopProperties,
145                                      bool &HasUserTransforms) {
146   LLVMContext &Ctx = Header->getContext();
147 
148   Optional<bool> Enabled;
149   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150     Enabled = false;
151   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152            Attrs.UnrollAndJamCount != 0)
153     Enabled = true;
154 
155   if (Enabled != true) {
156     SmallVector<Metadata *, 4> NewLoopProperties;
157     if (Enabled == false) {
158       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159       NewLoopProperties.push_back(MDNode::get(
160           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161       LoopProperties = NewLoopProperties;
162     }
163     return createPartialUnrollMetadata(Attrs, LoopProperties,
164                                        HasUserTransforms);
165   }
166 
167   SmallVector<Metadata *, 4> FollowupLoopProperties;
168   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169   FollowupLoopProperties.push_back(
170       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171 
172   bool FollowupHasTransforms = false;
173   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174                                                  FollowupHasTransforms);
175 
176   SmallVector<Metadata *, 4> Args;
177   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178   Args.push_back(TempNode.get());
179   Args.append(LoopProperties.begin(), LoopProperties.end());
180 
181   // Setting unroll_and_jam.count
182   if (Attrs.UnrollAndJamCount > 0) {
183     Metadata *Vals[] = {
184         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186                                                  Attrs.UnrollAndJamCount))};
187     Args.push_back(MDNode::get(Ctx, Vals));
188   }
189 
190   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
191     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192     Args.push_back(MDNode::get(Ctx, Vals));
193   }
194 
195   if (FollowupHasTransforms)
196     Args.push_back(MDNode::get(
197         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198               Followup}));
199 
200   if (UnrollAndJamInnerFollowup)
201     Args.push_back(MDNode::get(
202         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203               UnrollAndJamInnerFollowup}));
204 
205   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206   LoopID->replaceOperandWith(0, LoopID);
207   HasUserTransforms = true;
208   return LoopID;
209 }
210 
211 MDNode *
212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213                                       ArrayRef<Metadata *> LoopProperties,
214                                       bool &HasUserTransforms) {
215   LLVMContext &Ctx = Header->getContext();
216 
217   Optional<bool> Enabled;
218   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
219     Enabled = false;
220   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
221            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
222            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
223     Enabled = true;
224 
225   if (Enabled != true) {
226     SmallVector<Metadata *, 4> NewLoopProperties;
227     if (Enabled == false) {
228       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229       NewLoopProperties.push_back(
230           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231                             ConstantAsMetadata::get(ConstantInt::get(
232                                 llvm::Type::getInt1Ty(Ctx), 0))}));
233       LoopProperties = NewLoopProperties;
234     }
235     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236   }
237 
238   // Apply all loop properties to the vectorized loop.
239   SmallVector<Metadata *, 4> FollowupLoopProperties;
240   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241 
242   // Don't vectorize an already vectorized loop.
243   FollowupLoopProperties.push_back(
244       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245 
246   bool FollowupHasTransforms = false;
247   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248                                                 FollowupHasTransforms);
249 
250   SmallVector<Metadata *, 4> Args;
251   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252   Args.push_back(TempNode.get());
253   Args.append(LoopProperties.begin(), LoopProperties.end());
254 
255   // Setting vectorize.predicate
256   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
257     Metadata *Vals[] = {
258         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
259         ConstantAsMetadata::get(ConstantInt::get(
260             llvm::Type::getInt1Ty(Ctx),
261             (Attrs.VectorizePredicateEnable == LoopAttributes::Enable)))};
262     Args.push_back(MDNode::get(Ctx, Vals));
263   }
264 
265   // Setting vectorize.width
266   if (Attrs.VectorizeWidth > 0) {
267     Metadata *Vals[] = {
268         MDString::get(Ctx, "llvm.loop.vectorize.width"),
269         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
270                                                  Attrs.VectorizeWidth))};
271     Args.push_back(MDNode::get(Ctx, Vals));
272   }
273 
274   // Setting interleave.count
275   if (Attrs.InterleaveCount > 0) {
276     Metadata *Vals[] = {
277         MDString::get(Ctx, "llvm.loop.interleave.count"),
278         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
279                                                  Attrs.InterleaveCount))};
280     Args.push_back(MDNode::get(Ctx, Vals));
281   }
282 
283   // Setting vectorize.enable
284   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
285     Metadata *Vals[] = {
286         MDString::get(Ctx, "llvm.loop.vectorize.enable"),
287         ConstantAsMetadata::get(ConstantInt::get(
288             llvm::Type::getInt1Ty(Ctx),
289             (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
290     Args.push_back(MDNode::get(Ctx, Vals));
291   }
292 
293   if (FollowupHasTransforms)
294     Args.push_back(MDNode::get(
295         Ctx,
296         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
297 
298   MDNode *LoopID = MDNode::get(Ctx, Args);
299   LoopID->replaceOperandWith(0, LoopID);
300   HasUserTransforms = true;
301   return LoopID;
302 }
303 
304 MDNode *
305 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
306                                        ArrayRef<Metadata *> LoopProperties,
307                                        bool &HasUserTransforms) {
308   LLVMContext &Ctx = Header->getContext();
309 
310   Optional<bool> Enabled;
311   if (Attrs.DistributeEnable == LoopAttributes::Disable)
312     Enabled = false;
313   if (Attrs.DistributeEnable == LoopAttributes::Enable)
314     Enabled = true;
315 
316   if (Enabled != true) {
317     SmallVector<Metadata *, 4> NewLoopProperties;
318     if (Enabled == false) {
319       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
320       NewLoopProperties.push_back(
321           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
322                             ConstantAsMetadata::get(ConstantInt::get(
323                                 llvm::Type::getInt1Ty(Ctx), 0))}));
324       LoopProperties = NewLoopProperties;
325     }
326     return createLoopVectorizeMetadata(Attrs, LoopProperties,
327                                        HasUserTransforms);
328   }
329 
330   bool FollowupHasTransforms = false;
331   MDNode *Followup =
332       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
333 
334   SmallVector<Metadata *, 4> Args;
335   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
336   Args.push_back(TempNode.get());
337   Args.append(LoopProperties.begin(), LoopProperties.end());
338 
339   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
340                       ConstantAsMetadata::get(ConstantInt::get(
341                           llvm::Type::getInt1Ty(Ctx),
342                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
343   Args.push_back(MDNode::get(Ctx, Vals));
344 
345   if (FollowupHasTransforms)
346     Args.push_back(MDNode::get(
347         Ctx,
348         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
349 
350   MDNode *LoopID = MDNode::get(Ctx, Args);
351   LoopID->replaceOperandWith(0, LoopID);
352   HasUserTransforms = true;
353   return LoopID;
354 }
355 
356 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
357                                            ArrayRef<Metadata *> LoopProperties,
358                                            bool &HasUserTransforms) {
359   LLVMContext &Ctx = Header->getContext();
360 
361   Optional<bool> Enabled;
362   if (Attrs.UnrollEnable == LoopAttributes::Disable)
363     Enabled = false;
364   else if (Attrs.UnrollEnable == LoopAttributes::Full)
365     Enabled = true;
366 
367   if (Enabled != true) {
368     SmallVector<Metadata *, 4> NewLoopProperties;
369     if (Enabled == false) {
370       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
371       NewLoopProperties.push_back(
372           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
373       LoopProperties = NewLoopProperties;
374     }
375     return createLoopDistributeMetadata(Attrs, LoopProperties,
376                                         HasUserTransforms);
377   }
378 
379   SmallVector<Metadata *, 4> Args;
380   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
381   Args.push_back(TempNode.get());
382   Args.append(LoopProperties.begin(), LoopProperties.end());
383   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
384 
385   // No follow-up: there is no loop after full unrolling.
386   // TODO: Warn if there are transformations after full unrolling.
387 
388   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
389   LoopID->replaceOperandWith(0, LoopID);
390   HasUserTransforms = true;
391   return LoopID;
392 }
393 
394 MDNode *LoopInfo::createMetadata(
395     const LoopAttributes &Attrs,
396     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
397     bool &HasUserTransforms) {
398   SmallVector<Metadata *, 3> LoopProperties;
399 
400   // If we have a valid start debug location for the loop, add it.
401   if (StartLoc) {
402     LoopProperties.push_back(StartLoc.getAsMDNode());
403 
404     // If we also have a valid end debug location for the loop, add it.
405     if (EndLoc)
406       LoopProperties.push_back(EndLoc.getAsMDNode());
407   }
408 
409   assert(!!AccGroup == Attrs.IsParallel &&
410          "There must be an access group iff the loop is parallel");
411   if (Attrs.IsParallel) {
412     LLVMContext &Ctx = Header->getContext();
413     LoopProperties.push_back(MDNode::get(
414         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
415   }
416 
417   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
418                         AdditionalLoopProperties.end());
419   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
420 }
421 
422 LoopAttributes::LoopAttributes(bool IsParallel)
423     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
424       UnrollEnable(LoopAttributes::Unspecified),
425       UnrollAndJamEnable(LoopAttributes::Unspecified),
426       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
427       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
428       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
429       PipelineInitiationInterval(0) {}
430 
431 void LoopAttributes::clear() {
432   IsParallel = false;
433   VectorizeWidth = 0;
434   InterleaveCount = 0;
435   UnrollCount = 0;
436   UnrollAndJamCount = 0;
437   VectorizeEnable = LoopAttributes::Unspecified;
438   UnrollEnable = LoopAttributes::Unspecified;
439   UnrollAndJamEnable = LoopAttributes::Unspecified;
440   VectorizePredicateEnable = LoopAttributes::Unspecified;
441   DistributeEnable = LoopAttributes::Unspecified;
442   PipelineDisabled = false;
443   PipelineInitiationInterval = 0;
444 }
445 
446 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
447                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
448                    LoopInfo *Parent)
449     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
450       Parent(Parent) {
451 
452   if (Attrs.IsParallel) {
453     // Create an access group for this loop.
454     LLVMContext &Ctx = Header->getContext();
455     AccGroup = MDNode::getDistinct(Ctx, {});
456   }
457 
458   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
459       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
460       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
461       Attrs.PipelineInitiationInterval == 0 &&
462       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
463       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
464       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
465       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
466       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
467       !EndLoc)
468     return;
469 
470   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
471 }
472 
473 void LoopInfo::finish() {
474   // We did not annotate the loop body instructions because there are no
475   // attributes for this loop.
476   if (!TempLoopID)
477     return;
478 
479   MDNode *LoopID;
480   LoopAttributes CurLoopAttr = Attrs;
481   LLVMContext &Ctx = Header->getContext();
482 
483   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
484                  Parent->Attrs.UnrollAndJamCount != 0)) {
485     // Parent unroll-and-jams this loop.
486     // Split the transformations in those that happens before the unroll-and-jam
487     // and those after.
488 
489     LoopAttributes BeforeJam, AfterJam;
490 
491     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
492 
493     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
494     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
495     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
496     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
497     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
498 
499     switch (Attrs.UnrollEnable) {
500     case LoopAttributes::Unspecified:
501     case LoopAttributes::Disable:
502       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
503       AfterJam.UnrollEnable = Attrs.UnrollEnable;
504       break;
505     case LoopAttributes::Full:
506       BeforeJam.UnrollEnable = LoopAttributes::Full;
507       break;
508     case LoopAttributes::Enable:
509       AfterJam.UnrollEnable = LoopAttributes::Enable;
510       break;
511     }
512 
513     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
514     AfterJam.UnrollCount = Attrs.UnrollCount;
515     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
516     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
517 
518     // If this loop is subject of an unroll-and-jam by the parent loop, and has
519     // an unroll-and-jam annotation itself, we have to decide whether to first
520     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
521     // UnrollAndJam pass processes loops from inner to outer, so we apply the
522     // inner first.
523     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
524     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
525 
526     // Set the inner followup metadata to process by the outer loop. Only
527     // consider the first inner loop.
528     if (!Parent->UnrollAndJamInnerFollowup) {
529       // Splitting the attributes into a BeforeJam and an AfterJam part will
530       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
531       // to be forwarded to the AfterJam part. We detect the situation here and
532       // add it manually.
533       SmallVector<Metadata *, 1> BeforeLoopProperties;
534       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
535           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
536           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
537         BeforeLoopProperties.push_back(
538             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
539 
540       bool InnerFollowupHasTransform = false;
541       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
542                                              InnerFollowupHasTransform);
543       if (InnerFollowupHasTransform)
544         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
545     }
546 
547     CurLoopAttr = BeforeJam;
548   }
549 
550   bool HasUserTransforms = false;
551   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
552   TempLoopID->replaceAllUsesWith(LoopID);
553 }
554 
555 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
556                          const llvm::DebugLoc &EndLoc) {
557   Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
558                             Active.empty() ? nullptr : &Active.back()));
559   // Clear the attributes so nested loops do not inherit them.
560   StagedAttrs.clear();
561 }
562 
563 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
564                          ArrayRef<const clang::Attr *> Attrs,
565                          const llvm::DebugLoc &StartLoc,
566                          const llvm::DebugLoc &EndLoc) {
567 
568   // Identify loop hint attributes from Attrs.
569   for (const auto *Attr : Attrs) {
570     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
571     const OpenCLUnrollHintAttr *OpenCLHint =
572         dyn_cast<OpenCLUnrollHintAttr>(Attr);
573 
574     // Skip non loop hint attributes
575     if (!LH && !OpenCLHint) {
576       continue;
577     }
578 
579     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
580     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
581     unsigned ValueInt = 1;
582     // Translate opencl_unroll_hint attribute argument to
583     // equivalent LoopHintAttr enums.
584     // OpenCL v2.0 s6.11.5:
585     // 0 - enable unroll (no argument).
586     // 1 - disable unroll.
587     // other positive integer n - unroll by n.
588     if (OpenCLHint) {
589       ValueInt = OpenCLHint->getUnrollHint();
590       if (ValueInt == 0) {
591         State = LoopHintAttr::Enable;
592       } else if (ValueInt != 1) {
593         Option = LoopHintAttr::UnrollCount;
594         State = LoopHintAttr::Numeric;
595       }
596     } else if (LH) {
597       auto *ValueExpr = LH->getValue();
598       if (ValueExpr) {
599         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
600         ValueInt = ValueAPS.getSExtValue();
601       }
602 
603       Option = LH->getOption();
604       State = LH->getState();
605     }
606     switch (State) {
607     case LoopHintAttr::Disable:
608       switch (Option) {
609       case LoopHintAttr::Vectorize:
610         // Disable vectorization by specifying a width of 1.
611         setVectorizeWidth(1);
612         break;
613       case LoopHintAttr::Interleave:
614         // Disable interleaving by speciyfing a count of 1.
615         setInterleaveCount(1);
616         break;
617       case LoopHintAttr::Unroll:
618         setUnrollState(LoopAttributes::Disable);
619         break;
620       case LoopHintAttr::UnrollAndJam:
621         setUnrollAndJamState(LoopAttributes::Disable);
622         break;
623       case LoopHintAttr::VectorizePredicate:
624         setVectorizePredicateState(LoopAttributes::Disable);
625         break;
626       case LoopHintAttr::Distribute:
627         setDistributeState(false);
628         break;
629       case LoopHintAttr::PipelineDisabled:
630         setPipelineDisabled(true);
631         break;
632       case LoopHintAttr::UnrollCount:
633       case LoopHintAttr::UnrollAndJamCount:
634       case LoopHintAttr::VectorizeWidth:
635       case LoopHintAttr::InterleaveCount:
636       case LoopHintAttr::PipelineInitiationInterval:
637         llvm_unreachable("Options cannot be disabled.");
638         break;
639       }
640       break;
641     case LoopHintAttr::Enable:
642       switch (Option) {
643       case LoopHintAttr::Vectorize:
644       case LoopHintAttr::Interleave:
645         setVectorizeEnable(true);
646         break;
647       case LoopHintAttr::Unroll:
648         setUnrollState(LoopAttributes::Enable);
649         break;
650       case LoopHintAttr::UnrollAndJam:
651         setUnrollAndJamState(LoopAttributes::Enable);
652         break;
653       case LoopHintAttr::VectorizePredicate:
654         setVectorizePredicateState(LoopAttributes::Enable);
655         break;
656       case LoopHintAttr::Distribute:
657         setDistributeState(true);
658         break;
659       case LoopHintAttr::UnrollCount:
660       case LoopHintAttr::UnrollAndJamCount:
661       case LoopHintAttr::VectorizeWidth:
662       case LoopHintAttr::InterleaveCount:
663       case LoopHintAttr::PipelineDisabled:
664       case LoopHintAttr::PipelineInitiationInterval:
665         llvm_unreachable("Options cannot enabled.");
666         break;
667       }
668       break;
669     case LoopHintAttr::AssumeSafety:
670       switch (Option) {
671       case LoopHintAttr::Vectorize:
672       case LoopHintAttr::Interleave:
673         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
674         setParallel(true);
675         setVectorizeEnable(true);
676         break;
677       case LoopHintAttr::Unroll:
678       case LoopHintAttr::UnrollAndJam:
679       case LoopHintAttr::VectorizePredicate:
680       case LoopHintAttr::UnrollCount:
681       case LoopHintAttr::UnrollAndJamCount:
682       case LoopHintAttr::VectorizeWidth:
683       case LoopHintAttr::InterleaveCount:
684       case LoopHintAttr::Distribute:
685       case LoopHintAttr::PipelineDisabled:
686       case LoopHintAttr::PipelineInitiationInterval:
687         llvm_unreachable("Options cannot be used to assume mem safety.");
688         break;
689       }
690       break;
691     case LoopHintAttr::Full:
692       switch (Option) {
693       case LoopHintAttr::Unroll:
694         setUnrollState(LoopAttributes::Full);
695         break;
696       case LoopHintAttr::UnrollAndJam:
697         setUnrollAndJamState(LoopAttributes::Full);
698         break;
699       case LoopHintAttr::Vectorize:
700       case LoopHintAttr::Interleave:
701       case LoopHintAttr::UnrollCount:
702       case LoopHintAttr::UnrollAndJamCount:
703       case LoopHintAttr::VectorizeWidth:
704       case LoopHintAttr::InterleaveCount:
705       case LoopHintAttr::Distribute:
706       case LoopHintAttr::PipelineDisabled:
707       case LoopHintAttr::PipelineInitiationInterval:
708       case LoopHintAttr::VectorizePredicate:
709         llvm_unreachable("Options cannot be used with 'full' hint.");
710         break;
711       }
712       break;
713     case LoopHintAttr::Numeric:
714       switch (Option) {
715       case LoopHintAttr::VectorizeWidth:
716         setVectorizeWidth(ValueInt);
717         break;
718       case LoopHintAttr::InterleaveCount:
719         setInterleaveCount(ValueInt);
720         break;
721       case LoopHintAttr::UnrollCount:
722         setUnrollCount(ValueInt);
723         break;
724       case LoopHintAttr::UnrollAndJamCount:
725         setUnrollAndJamCount(ValueInt);
726         break;
727       case LoopHintAttr::PipelineInitiationInterval:
728         setPipelineInitiationInterval(ValueInt);
729         break;
730       case LoopHintAttr::Unroll:
731       case LoopHintAttr::UnrollAndJam:
732       case LoopHintAttr::VectorizePredicate:
733       case LoopHintAttr::Vectorize:
734       case LoopHintAttr::Interleave:
735       case LoopHintAttr::Distribute:
736       case LoopHintAttr::PipelineDisabled:
737         llvm_unreachable("Options cannot be assigned a value.");
738         break;
739       }
740       break;
741     }
742   }
743 
744   /// Stage the attributes.
745   push(Header, StartLoc, EndLoc);
746 }
747 
748 void LoopInfoStack::pop() {
749   assert(!Active.empty() && "No active loops to pop");
750   Active.back().finish();
751   Active.pop_back();
752 }
753 
754 void LoopInfoStack::InsertHelper(Instruction *I) const {
755   if (I->mayReadOrWriteMemory()) {
756     SmallVector<Metadata *, 4> AccessGroups;
757     for (const LoopInfo &AL : Active) {
758       // Here we assume that every loop that has an access group is parallel.
759       if (MDNode *Group = AL.getAccessGroup())
760         AccessGroups.push_back(Group);
761     }
762     MDNode *UnionMD = nullptr;
763     if (AccessGroups.size() == 1)
764       UnionMD = cast<MDNode>(AccessGroups[0]);
765     else if (AccessGroups.size() >= 2)
766       UnionMD = MDNode::get(I->getContext(), AccessGroups);
767     I->setMetadata("llvm.access.group", UnionMD);
768   }
769 
770   if (!hasInfo())
771     return;
772 
773   const LoopInfo &L = getInfo();
774   if (!L.getLoopID())
775     return;
776 
777   if (I->isTerminator()) {
778     for (BasicBlock *Succ : successors(I))
779       if (Succ == L.getHeader()) {
780         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
781         break;
782       }
783     return;
784   }
785 }
786