1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This provides a class for OpenMP runtime code generation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGOpenMPRuntime.h"
14 #include "CGCXXABI.h"
15 #include "CGCleanup.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenFunction.h"
18 #include "clang/AST/Attr.h"
19 #include "clang/AST/Decl.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/Basic/BitmaskEnum.h"
22 #include "clang/CodeGen/ConstantInitBuilder.h"
23 #include "llvm/ADT/ArrayRef.h"
24 #include "llvm/ADT/SetOperations.h"
25 #include "llvm/Bitcode/BitcodeReader.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/GlobalValue.h"
28 #include "llvm/IR/Value.h"
29 #include "llvm/Support/Format.h"
30 #include "llvm/Support/raw_ostream.h"
31 #include <cassert>
32 
33 using namespace clang;
34 using namespace CodeGen;
35 using namespace llvm::omp;
36 
37 namespace {
38 /// Base class for handling code generation inside OpenMP regions.
39 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
40 public:
41   /// Kinds of OpenMP regions used in codegen.
42   enum CGOpenMPRegionKind {
43     /// Region with outlined function for standalone 'parallel'
44     /// directive.
45     ParallelOutlinedRegion,
46     /// Region with outlined function for standalone 'task' directive.
47     TaskOutlinedRegion,
48     /// Region for constructs that do not require function outlining,
49     /// like 'for', 'sections', 'atomic' etc. directives.
50     InlinedRegion,
51     /// Region with outlined function for standalone 'target' directive.
52     TargetRegion,
53   };
54 
55   CGOpenMPRegionInfo(const CapturedStmt &CS,
56                      const CGOpenMPRegionKind RegionKind,
57                      const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
58                      bool HasCancel)
59       : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
60         CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
61 
62   CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
63                      const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
64                      bool HasCancel)
65       : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
66         Kind(Kind), HasCancel(HasCancel) {}
67 
68   /// Get a variable or parameter for storing global thread id
69   /// inside OpenMP construct.
70   virtual const VarDecl *getThreadIDVariable() const = 0;
71 
72   /// Emit the captured statement body.
73   void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
74 
75   /// Get an LValue for the current ThreadID variable.
76   /// \return LValue for thread id variable. This LValue always has type int32*.
77   virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
78 
79   virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
80 
81   CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
82 
83   OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
84 
85   bool hasCancel() const { return HasCancel; }
86 
87   static bool classof(const CGCapturedStmtInfo *Info) {
88     return Info->getKind() == CR_OpenMP;
89   }
90 
91   ~CGOpenMPRegionInfo() override = default;
92 
93 protected:
94   CGOpenMPRegionKind RegionKind;
95   RegionCodeGenTy CodeGen;
96   OpenMPDirectiveKind Kind;
97   bool HasCancel;
98 };
99 
100 /// API for captured statement code generation in OpenMP constructs.
101 class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
102 public:
103   CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
104                              const RegionCodeGenTy &CodeGen,
105                              OpenMPDirectiveKind Kind, bool HasCancel,
106                              StringRef HelperName)
107       : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
108                            HasCancel),
109         ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
110     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
111   }
112 
113   /// Get a variable or parameter for storing global thread id
114   /// inside OpenMP construct.
115   const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
116 
117   /// Get the name of the capture helper.
118   StringRef getHelperName() const override { return HelperName; }
119 
120   static bool classof(const CGCapturedStmtInfo *Info) {
121     return CGOpenMPRegionInfo::classof(Info) &&
122            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
123                ParallelOutlinedRegion;
124   }
125 
126 private:
127   /// A variable or parameter storing global thread id for OpenMP
128   /// constructs.
129   const VarDecl *ThreadIDVar;
130   StringRef HelperName;
131 };
132 
133 /// API for captured statement code generation in OpenMP constructs.
134 class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
135 public:
136   class UntiedTaskActionTy final : public PrePostActionTy {
137     bool Untied;
138     const VarDecl *PartIDVar;
139     const RegionCodeGenTy UntiedCodeGen;
140     llvm::SwitchInst *UntiedSwitch = nullptr;
141 
142   public:
143     UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
144                        const RegionCodeGenTy &UntiedCodeGen)
145         : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
146     void Enter(CodeGenFunction &CGF) override {
147       if (Untied) {
148         // Emit task switching point.
149         LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
150             CGF.GetAddrOfLocalVar(PartIDVar),
151             PartIDVar->getType()->castAs<PointerType>());
152         llvm::Value *Res =
153             CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
154         llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
155         UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
156         CGF.EmitBlock(DoneBB);
157         CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
158         CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
159         UntiedSwitch->addCase(CGF.Builder.getInt32(0),
160                               CGF.Builder.GetInsertBlock());
161         emitUntiedSwitch(CGF);
162       }
163     }
164     void emitUntiedSwitch(CodeGenFunction &CGF) const {
165       if (Untied) {
166         LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
167             CGF.GetAddrOfLocalVar(PartIDVar),
168             PartIDVar->getType()->castAs<PointerType>());
169         CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
170                               PartIdLVal);
171         UntiedCodeGen(CGF);
172         CodeGenFunction::JumpDest CurPoint =
173             CGF.getJumpDestInCurrentScope(".untied.next.");
174         CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
175         CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
176         UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
177                               CGF.Builder.GetInsertBlock());
178         CGF.EmitBranchThroughCleanup(CurPoint);
179         CGF.EmitBlock(CurPoint.getBlock());
180       }
181     }
182     unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
183   };
184   CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
185                                  const VarDecl *ThreadIDVar,
186                                  const RegionCodeGenTy &CodeGen,
187                                  OpenMPDirectiveKind Kind, bool HasCancel,
188                                  const UntiedTaskActionTy &Action)
189       : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
190         ThreadIDVar(ThreadIDVar), Action(Action) {
191     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
192   }
193 
194   /// Get a variable or parameter for storing global thread id
195   /// inside OpenMP construct.
196   const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
197 
198   /// Get an LValue for the current ThreadID variable.
199   LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
200 
201   /// Get the name of the capture helper.
202   StringRef getHelperName() const override { return ".omp_outlined."; }
203 
204   void emitUntiedSwitch(CodeGenFunction &CGF) override {
205     Action.emitUntiedSwitch(CGF);
206   }
207 
208   static bool classof(const CGCapturedStmtInfo *Info) {
209     return CGOpenMPRegionInfo::classof(Info) &&
210            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
211                TaskOutlinedRegion;
212   }
213 
214 private:
215   /// A variable or parameter storing global thread id for OpenMP
216   /// constructs.
217   const VarDecl *ThreadIDVar;
218   /// Action for emitting code for untied tasks.
219   const UntiedTaskActionTy &Action;
220 };
221 
222 /// API for inlined captured statement code generation in OpenMP
223 /// constructs.
224 class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
225 public:
226   CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
227                             const RegionCodeGenTy &CodeGen,
228                             OpenMPDirectiveKind Kind, bool HasCancel)
229       : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
230         OldCSI(OldCSI),
231         OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
232 
233   // Retrieve the value of the context parameter.
234   llvm::Value *getContextValue() const override {
235     if (OuterRegionInfo)
236       return OuterRegionInfo->getContextValue();
237     llvm_unreachable("No context value for inlined OpenMP region");
238   }
239 
240   void setContextValue(llvm::Value *V) override {
241     if (OuterRegionInfo) {
242       OuterRegionInfo->setContextValue(V);
243       return;
244     }
245     llvm_unreachable("No context value for inlined OpenMP region");
246   }
247 
248   /// Lookup the captured field decl for a variable.
249   const FieldDecl *lookup(const VarDecl *VD) const override {
250     if (OuterRegionInfo)
251       return OuterRegionInfo->lookup(VD);
252     // If there is no outer outlined region,no need to lookup in a list of
253     // captured variables, we can use the original one.
254     return nullptr;
255   }
256 
257   FieldDecl *getThisFieldDecl() const override {
258     if (OuterRegionInfo)
259       return OuterRegionInfo->getThisFieldDecl();
260     return nullptr;
261   }
262 
263   /// Get a variable or parameter for storing global thread id
264   /// inside OpenMP construct.
265   const VarDecl *getThreadIDVariable() const override {
266     if (OuterRegionInfo)
267       return OuterRegionInfo->getThreadIDVariable();
268     return nullptr;
269   }
270 
271   /// Get an LValue for the current ThreadID variable.
272   LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
273     if (OuterRegionInfo)
274       return OuterRegionInfo->getThreadIDVariableLValue(CGF);
275     llvm_unreachable("No LValue for inlined OpenMP construct");
276   }
277 
278   /// Get the name of the capture helper.
279   StringRef getHelperName() const override {
280     if (auto *OuterRegionInfo = getOldCSI())
281       return OuterRegionInfo->getHelperName();
282     llvm_unreachable("No helper name for inlined OpenMP construct");
283   }
284 
285   void emitUntiedSwitch(CodeGenFunction &CGF) override {
286     if (OuterRegionInfo)
287       OuterRegionInfo->emitUntiedSwitch(CGF);
288   }
289 
290   CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
291 
292   static bool classof(const CGCapturedStmtInfo *Info) {
293     return CGOpenMPRegionInfo::classof(Info) &&
294            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
295   }
296 
297   ~CGOpenMPInlinedRegionInfo() override = default;
298 
299 private:
300   /// CodeGen info about outer OpenMP region.
301   CodeGenFunction::CGCapturedStmtInfo *OldCSI;
302   CGOpenMPRegionInfo *OuterRegionInfo;
303 };
304 
305 /// API for captured statement code generation in OpenMP target
306 /// constructs. For this captures, implicit parameters are used instead of the
307 /// captured fields. The name of the target region has to be unique in a given
308 /// application so it is provided by the client, because only the client has
309 /// the information to generate that.
310 class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
311 public:
312   CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
313                            const RegionCodeGenTy &CodeGen, StringRef HelperName)
314       : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
315                            /*HasCancel=*/false),
316         HelperName(HelperName) {}
317 
318   /// This is unused for target regions because each starts executing
319   /// with a single thread.
320   const VarDecl *getThreadIDVariable() const override { return nullptr; }
321 
322   /// Get the name of the capture helper.
323   StringRef getHelperName() const override { return HelperName; }
324 
325   static bool classof(const CGCapturedStmtInfo *Info) {
326     return CGOpenMPRegionInfo::classof(Info) &&
327            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
328   }
329 
330 private:
331   StringRef HelperName;
332 };
333 
334 static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
335   llvm_unreachable("No codegen for expressions");
336 }
337 /// API for generation of expressions captured in a innermost OpenMP
338 /// region.
339 class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
340 public:
341   CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
342       : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
343                                   OMPD_unknown,
344                                   /*HasCancel=*/false),
345         PrivScope(CGF) {
346     // Make sure the globals captured in the provided statement are local by
347     // using the privatization logic. We assume the same variable is not
348     // captured more than once.
349     for (const auto &C : CS.captures()) {
350       if (!C.capturesVariable() && !C.capturesVariableByCopy())
351         continue;
352 
353       const VarDecl *VD = C.getCapturedVar();
354       if (VD->isLocalVarDeclOrParm())
355         continue;
356 
357       DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
358                       /*RefersToEnclosingVariableOrCapture=*/false,
359                       VD->getType().getNonReferenceType(), VK_LValue,
360                       C.getLocation());
361       PrivScope.addPrivate(
362           VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
363     }
364     (void)PrivScope.Privatize();
365   }
366 
367   /// Lookup the captured field decl for a variable.
368   const FieldDecl *lookup(const VarDecl *VD) const override {
369     if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
370       return FD;
371     return nullptr;
372   }
373 
374   /// Emit the captured statement body.
375   void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
376     llvm_unreachable("No body for expressions");
377   }
378 
379   /// Get a variable or parameter for storing global thread id
380   /// inside OpenMP construct.
381   const VarDecl *getThreadIDVariable() const override {
382     llvm_unreachable("No thread id for expressions");
383   }
384 
385   /// Get the name of the capture helper.
386   StringRef getHelperName() const override {
387     llvm_unreachable("No helper name for expressions");
388   }
389 
390   static bool classof(const CGCapturedStmtInfo *Info) { return false; }
391 
392 private:
393   /// Private scope to capture global variables.
394   CodeGenFunction::OMPPrivateScope PrivScope;
395 };
396 
397 /// RAII for emitting code of OpenMP constructs.
398 class InlinedOpenMPRegionRAII {
399   CodeGenFunction &CGF;
400   llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
401   FieldDecl *LambdaThisCaptureField = nullptr;
402   const CodeGen::CGBlockInfo *BlockInfo = nullptr;
403 
404 public:
405   /// Constructs region for combined constructs.
406   /// \param CodeGen Code generation sequence for combined directives. Includes
407   /// a list of functions used for code generation of implicitly inlined
408   /// regions.
409   InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
410                           OpenMPDirectiveKind Kind, bool HasCancel)
411       : CGF(CGF) {
412     // Start emission for the construct.
413     CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
414         CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
415     std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
416     LambdaThisCaptureField = CGF.LambdaThisCaptureField;
417     CGF.LambdaThisCaptureField = nullptr;
418     BlockInfo = CGF.BlockInfo;
419     CGF.BlockInfo = nullptr;
420   }
421 
422   ~InlinedOpenMPRegionRAII() {
423     // Restore original CapturedStmtInfo only if we're done with code emission.
424     auto *OldCSI =
425         cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
426     delete CGF.CapturedStmtInfo;
427     CGF.CapturedStmtInfo = OldCSI;
428     std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
429     CGF.LambdaThisCaptureField = LambdaThisCaptureField;
430     CGF.BlockInfo = BlockInfo;
431   }
432 };
433 
434 /// Values for bit flags used in the ident_t to describe the fields.
435 /// All enumeric elements are named and described in accordance with the code
436 /// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
437 enum OpenMPLocationFlags : unsigned {
438   /// Use trampoline for internal microtask.
439   OMP_IDENT_IMD = 0x01,
440   /// Use c-style ident structure.
441   OMP_IDENT_KMPC = 0x02,
442   /// Atomic reduction option for kmpc_reduce.
443   OMP_ATOMIC_REDUCE = 0x10,
444   /// Explicit 'barrier' directive.
445   OMP_IDENT_BARRIER_EXPL = 0x20,
446   /// Implicit barrier in code.
447   OMP_IDENT_BARRIER_IMPL = 0x40,
448   /// Implicit barrier in 'for' directive.
449   OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
450   /// Implicit barrier in 'sections' directive.
451   OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
452   /// Implicit barrier in 'single' directive.
453   OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
454   /// Call of __kmp_for_static_init for static loop.
455   OMP_IDENT_WORK_LOOP = 0x200,
456   /// Call of __kmp_for_static_init for sections.
457   OMP_IDENT_WORK_SECTIONS = 0x400,
458   /// Call of __kmp_for_static_init for distribute.
459   OMP_IDENT_WORK_DISTRIBUTE = 0x800,
460   LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
461 };
462 
463 namespace {
464 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
465 /// Values for bit flags for marking which requires clauses have been used.
466 enum OpenMPOffloadingRequiresDirFlags : int64_t {
467   /// flag undefined.
468   OMP_REQ_UNDEFINED               = 0x000,
469   /// no requires clause present.
470   OMP_REQ_NONE                    = 0x001,
471   /// reverse_offload clause.
472   OMP_REQ_REVERSE_OFFLOAD         = 0x002,
473   /// unified_address clause.
474   OMP_REQ_UNIFIED_ADDRESS         = 0x004,
475   /// unified_shared_memory clause.
476   OMP_REQ_UNIFIED_SHARED_MEMORY   = 0x008,
477   /// dynamic_allocators clause.
478   OMP_REQ_DYNAMIC_ALLOCATORS      = 0x010,
479   LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
480 };
481 
482 enum OpenMPOffloadingReservedDeviceIDs {
483   /// Device ID if the device was not defined, runtime should get it
484   /// from environment variables in the spec.
485   OMP_DEVICEID_UNDEF = -1,
486 };
487 } // anonymous namespace
488 
489 /// Describes ident structure that describes a source location.
490 /// All descriptions are taken from
491 /// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
492 /// Original structure:
493 /// typedef struct ident {
494 ///    kmp_int32 reserved_1;   /**<  might be used in Fortran;
495 ///                                  see above  */
496 ///    kmp_int32 flags;        /**<  also f.flags; KMP_IDENT_xxx flags;
497 ///                                  KMP_IDENT_KMPC identifies this union
498 ///                                  member  */
499 ///    kmp_int32 reserved_2;   /**<  not really used in Fortran any more;
500 ///                                  see above */
501 ///#if USE_ITT_BUILD
502 ///                            /*  but currently used for storing
503 ///                                region-specific ITT */
504 ///                            /*  contextual information. */
505 ///#endif /* USE_ITT_BUILD */
506 ///    kmp_int32 reserved_3;   /**< source[4] in Fortran, do not use for
507 ///                                 C++  */
508 ///    char const *psource;    /**< String describing the source location.
509 ///                            The string is composed of semi-colon separated
510 //                             fields which describe the source file,
511 ///                            the function and a pair of line numbers that
512 ///                            delimit the construct.
513 ///                             */
514 /// } ident_t;
515 enum IdentFieldIndex {
516   /// might be used in Fortran
517   IdentField_Reserved_1,
518   /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
519   IdentField_Flags,
520   /// Not really used in Fortran any more
521   IdentField_Reserved_2,
522   /// Source[4] in Fortran, do not use for C++
523   IdentField_Reserved_3,
524   /// String describing the source location. The string is composed of
525   /// semi-colon separated fields which describe the source file, the function
526   /// and a pair of line numbers that delimit the construct.
527   IdentField_PSource
528 };
529 
530 /// Schedule types for 'omp for' loops (these enumerators are taken from
531 /// the enum sched_type in kmp.h).
532 enum OpenMPSchedType {
533   /// Lower bound for default (unordered) versions.
534   OMP_sch_lower = 32,
535   OMP_sch_static_chunked = 33,
536   OMP_sch_static = 34,
537   OMP_sch_dynamic_chunked = 35,
538   OMP_sch_guided_chunked = 36,
539   OMP_sch_runtime = 37,
540   OMP_sch_auto = 38,
541   /// static with chunk adjustment (e.g., simd)
542   OMP_sch_static_balanced_chunked = 45,
543   /// Lower bound for 'ordered' versions.
544   OMP_ord_lower = 64,
545   OMP_ord_static_chunked = 65,
546   OMP_ord_static = 66,
547   OMP_ord_dynamic_chunked = 67,
548   OMP_ord_guided_chunked = 68,
549   OMP_ord_runtime = 69,
550   OMP_ord_auto = 70,
551   OMP_sch_default = OMP_sch_static,
552   /// dist_schedule types
553   OMP_dist_sch_static_chunked = 91,
554   OMP_dist_sch_static = 92,
555   /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
556   /// Set if the monotonic schedule modifier was present.
557   OMP_sch_modifier_monotonic = (1 << 29),
558   /// Set if the nonmonotonic schedule modifier was present.
559   OMP_sch_modifier_nonmonotonic = (1 << 30),
560 };
561 
562 enum OpenMPRTLFunction {
563   /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
564   /// kmpc_micro microtask, ...);
565   OMPRTL__kmpc_fork_call,
566   /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
567   /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
568   OMPRTL__kmpc_threadprivate_cached,
569   /// Call to void __kmpc_threadprivate_register( ident_t *,
570   /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
571   OMPRTL__kmpc_threadprivate_register,
572   // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
573   OMPRTL__kmpc_global_thread_num,
574   // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
575   // kmp_critical_name *crit);
576   OMPRTL__kmpc_critical,
577   // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
578   // global_tid, kmp_critical_name *crit, uintptr_t hint);
579   OMPRTL__kmpc_critical_with_hint,
580   // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
581   // kmp_critical_name *crit);
582   OMPRTL__kmpc_end_critical,
583   // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
584   // global_tid);
585   OMPRTL__kmpc_cancel_barrier,
586   // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
587   OMPRTL__kmpc_barrier,
588   // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
589   OMPRTL__kmpc_for_static_fini,
590   // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
591   // global_tid);
592   OMPRTL__kmpc_serialized_parallel,
593   // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
594   // global_tid);
595   OMPRTL__kmpc_end_serialized_parallel,
596   // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
597   // kmp_int32 num_threads);
598   OMPRTL__kmpc_push_num_threads,
599   // Call to void __kmpc_flush(ident_t *loc);
600   OMPRTL__kmpc_flush,
601   // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
602   OMPRTL__kmpc_master,
603   // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
604   OMPRTL__kmpc_end_master,
605   // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
606   // int end_part);
607   OMPRTL__kmpc_omp_taskyield,
608   // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
609   OMPRTL__kmpc_single,
610   // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
611   OMPRTL__kmpc_end_single,
612   // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
613   // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
614   // kmp_routine_entry_t *task_entry);
615   OMPRTL__kmpc_omp_task_alloc,
616   // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
617   // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
618   // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
619   // kmp_int64 device_id);
620   OMPRTL__kmpc_omp_target_task_alloc,
621   // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
622   // new_task);
623   OMPRTL__kmpc_omp_task,
624   // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
625   // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
626   // kmp_int32 didit);
627   OMPRTL__kmpc_copyprivate,
628   // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
629   // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
630   // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
631   OMPRTL__kmpc_reduce,
632   // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
633   // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
634   // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
635   // *lck);
636   OMPRTL__kmpc_reduce_nowait,
637   // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
638   // kmp_critical_name *lck);
639   OMPRTL__kmpc_end_reduce,
640   // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
641   // kmp_critical_name *lck);
642   OMPRTL__kmpc_end_reduce_nowait,
643   // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
644   // kmp_task_t * new_task);
645   OMPRTL__kmpc_omp_task_begin_if0,
646   // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
647   // kmp_task_t * new_task);
648   OMPRTL__kmpc_omp_task_complete_if0,
649   // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
650   OMPRTL__kmpc_ordered,
651   // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
652   OMPRTL__kmpc_end_ordered,
653   // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
654   // global_tid);
655   OMPRTL__kmpc_omp_taskwait,
656   // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
657   OMPRTL__kmpc_taskgroup,
658   // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
659   OMPRTL__kmpc_end_taskgroup,
660   // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
661   // int proc_bind);
662   OMPRTL__kmpc_push_proc_bind,
663   // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
664   // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
665   // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
666   OMPRTL__kmpc_omp_task_with_deps,
667   // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
668   // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
669   // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
670   OMPRTL__kmpc_omp_wait_deps,
671   // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
672   // global_tid, kmp_int32 cncl_kind);
673   OMPRTL__kmpc_cancellationpoint,
674   // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
675   // kmp_int32 cncl_kind);
676   OMPRTL__kmpc_cancel,
677   // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
678   // kmp_int32 num_teams, kmp_int32 thread_limit);
679   OMPRTL__kmpc_push_num_teams,
680   // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
681   // microtask, ...);
682   OMPRTL__kmpc_fork_teams,
683   // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
684   // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
685   // sched, kmp_uint64 grainsize, void *task_dup);
686   OMPRTL__kmpc_taskloop,
687   // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
688   // num_dims, struct kmp_dim *dims);
689   OMPRTL__kmpc_doacross_init,
690   // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
691   OMPRTL__kmpc_doacross_fini,
692   // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
693   // *vec);
694   OMPRTL__kmpc_doacross_post,
695   // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
696   // *vec);
697   OMPRTL__kmpc_doacross_wait,
698   // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
699   // *data);
700   OMPRTL__kmpc_task_reduction_init,
701   // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
702   // *d);
703   OMPRTL__kmpc_task_reduction_get_th_data,
704   // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
705   OMPRTL__kmpc_alloc,
706   // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
707   OMPRTL__kmpc_free,
708 
709   //
710   // Offloading related calls
711   //
712   // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
713   // size);
714   OMPRTL__kmpc_push_target_tripcount,
715   // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
716   // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
717   // *arg_types);
718   OMPRTL__tgt_target,
719   // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
720   // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
721   // *arg_types);
722   OMPRTL__tgt_target_nowait,
723   // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
724   // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
725   // *arg_types, int32_t num_teams, int32_t thread_limit);
726   OMPRTL__tgt_target_teams,
727   // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
728   // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
729   // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
730   OMPRTL__tgt_target_teams_nowait,
731   // Call to void __tgt_register_requires(int64_t flags);
732   OMPRTL__tgt_register_requires,
733   // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
734   OMPRTL__tgt_register_lib,
735   // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
736   OMPRTL__tgt_unregister_lib,
737   // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
738   // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
739   OMPRTL__tgt_target_data_begin,
740   // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
741   // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
742   // *arg_types);
743   OMPRTL__tgt_target_data_begin_nowait,
744   // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
745   // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
746   OMPRTL__tgt_target_data_end,
747   // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
748   // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
749   // *arg_types);
750   OMPRTL__tgt_target_data_end_nowait,
751   // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
752   // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
753   OMPRTL__tgt_target_data_update,
754   // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
755   // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
756   // *arg_types);
757   OMPRTL__tgt_target_data_update_nowait,
758   // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
759   OMPRTL__tgt_mapper_num_components,
760   // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
761   // *base, void *begin, int64_t size, int64_t type);
762   OMPRTL__tgt_push_mapper_component,
763 };
764 
765 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP
766 /// region.
767 class CleanupTy final : public EHScopeStack::Cleanup {
768   PrePostActionTy *Action;
769 
770 public:
771   explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
772   void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
773     if (!CGF.HaveInsertPoint())
774       return;
775     Action->Exit(CGF);
776   }
777 };
778 
779 } // anonymous namespace
780 
781 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
782   CodeGenFunction::RunCleanupsScope Scope(CGF);
783   if (PrePostAction) {
784     CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
785     Callback(CodeGen, CGF, *PrePostAction);
786   } else {
787     PrePostActionTy Action;
788     Callback(CodeGen, CGF, Action);
789   }
790 }
791 
792 /// Check if the combiner is a call to UDR combiner and if it is so return the
793 /// UDR decl used for reduction.
794 static const OMPDeclareReductionDecl *
795 getReductionInit(const Expr *ReductionOp) {
796   if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
797     if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
798       if (const auto *DRE =
799               dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
800         if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
801           return DRD;
802   return nullptr;
803 }
804 
805 static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
806                                              const OMPDeclareReductionDecl *DRD,
807                                              const Expr *InitOp,
808                                              Address Private, Address Original,
809                                              QualType Ty) {
810   if (DRD->getInitializer()) {
811     std::pair<llvm::Function *, llvm::Function *> Reduction =
812         CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
813     const auto *CE = cast<CallExpr>(InitOp);
814     const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
815     const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
816     const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
817     const auto *LHSDRE =
818         cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
819     const auto *RHSDRE =
820         cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
821     CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
822     PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
823                             [=]() { return Private; });
824     PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
825                             [=]() { return Original; });
826     (void)PrivateScope.Privatize();
827     RValue Func = RValue::get(Reduction.second);
828     CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
829     CGF.EmitIgnoredExpr(InitOp);
830   } else {
831     llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
832     std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
833     auto *GV = new llvm::GlobalVariable(
834         CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
835         llvm::GlobalValue::PrivateLinkage, Init, Name);
836     LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
837     RValue InitRVal;
838     switch (CGF.getEvaluationKind(Ty)) {
839     case TEK_Scalar:
840       InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
841       break;
842     case TEK_Complex:
843       InitRVal =
844           RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
845       break;
846     case TEK_Aggregate:
847       InitRVal = RValue::getAggregate(LV.getAddress(CGF));
848       break;
849     }
850     OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
851     CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
852     CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
853                          /*IsInitializer=*/false);
854   }
855 }
856 
857 /// Emit initialization of arrays of complex types.
858 /// \param DestAddr Address of the array.
859 /// \param Type Type of array.
860 /// \param Init Initial expression of array.
861 /// \param SrcAddr Address of the original array.
862 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
863                                  QualType Type, bool EmitDeclareReductionInit,
864                                  const Expr *Init,
865                                  const OMPDeclareReductionDecl *DRD,
866                                  Address SrcAddr = Address::invalid()) {
867   // Perform element-by-element initialization.
868   QualType ElementTy;
869 
870   // Drill down to the base element type on both arrays.
871   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
872   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
873   DestAddr =
874       CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
875   if (DRD)
876     SrcAddr =
877         CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
878 
879   llvm::Value *SrcBegin = nullptr;
880   if (DRD)
881     SrcBegin = SrcAddr.getPointer();
882   llvm::Value *DestBegin = DestAddr.getPointer();
883   // Cast from pointer to array type to pointer to single element.
884   llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
885   // The basic structure here is a while-do loop.
886   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
887   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
888   llvm::Value *IsEmpty =
889       CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
890   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
891 
892   // Enter the loop body, making that address the current address.
893   llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
894   CGF.EmitBlock(BodyBB);
895 
896   CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
897 
898   llvm::PHINode *SrcElementPHI = nullptr;
899   Address SrcElementCurrent = Address::invalid();
900   if (DRD) {
901     SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
902                                           "omp.arraycpy.srcElementPast");
903     SrcElementPHI->addIncoming(SrcBegin, EntryBB);
904     SrcElementCurrent =
905         Address(SrcElementPHI,
906                 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
907   }
908   llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
909       DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
910   DestElementPHI->addIncoming(DestBegin, EntryBB);
911   Address DestElementCurrent =
912       Address(DestElementPHI,
913               DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
914 
915   // Emit copy.
916   {
917     CodeGenFunction::RunCleanupsScope InitScope(CGF);
918     if (EmitDeclareReductionInit) {
919       emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
920                                        SrcElementCurrent, ElementTy);
921     } else
922       CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
923                            /*IsInitializer=*/false);
924   }
925 
926   if (DRD) {
927     // Shift the address forward by one element.
928     llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
929         SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
930     SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
931   }
932 
933   // Shift the address forward by one element.
934   llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
935       DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
936   // Check whether we've reached the end.
937   llvm::Value *Done =
938       CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
939   CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
940   DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
941 
942   // Done.
943   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
944 }
945 
946 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
947   return CGF.EmitOMPSharedLValue(E);
948 }
949 
950 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
951                                             const Expr *E) {
952   if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
953     return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
954   return LValue();
955 }
956 
957 void ReductionCodeGen::emitAggregateInitialization(
958     CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
959     const OMPDeclareReductionDecl *DRD) {
960   // Emit VarDecl with copy init for arrays.
961   // Get the address of the original variable captured in current
962   // captured region.
963   const auto *PrivateVD =
964       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
965   bool EmitDeclareReductionInit =
966       DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
967   EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
968                        EmitDeclareReductionInit,
969                        EmitDeclareReductionInit ? ClausesData[N].ReductionOp
970                                                 : PrivateVD->getInit(),
971                        DRD, SharedLVal.getAddress(CGF));
972 }
973 
974 ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
975                                    ArrayRef<const Expr *> Privates,
976                                    ArrayRef<const Expr *> ReductionOps) {
977   ClausesData.reserve(Shareds.size());
978   SharedAddresses.reserve(Shareds.size());
979   Sizes.reserve(Shareds.size());
980   BaseDecls.reserve(Shareds.size());
981   auto IPriv = Privates.begin();
982   auto IRed = ReductionOps.begin();
983   for (const Expr *Ref : Shareds) {
984     ClausesData.emplace_back(Ref, *IPriv, *IRed);
985     std::advance(IPriv, 1);
986     std::advance(IRed, 1);
987   }
988 }
989 
990 void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
991   assert(SharedAddresses.size() == N &&
992          "Number of generated lvalues must be exactly N.");
993   LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
994   LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
995   SharedAddresses.emplace_back(First, Second);
996 }
997 
998 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
999   const auto *PrivateVD =
1000       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1001   QualType PrivateType = PrivateVD->getType();
1002   bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1003   if (!PrivateType->isVariablyModifiedType()) {
1004     Sizes.emplace_back(
1005         CGF.getTypeSize(
1006             SharedAddresses[N].first.getType().getNonReferenceType()),
1007         nullptr);
1008     return;
1009   }
1010   llvm::Value *Size;
1011   llvm::Value *SizeInChars;
1012   auto *ElemType = cast<llvm::PointerType>(
1013                        SharedAddresses[N].first.getPointer(CGF)->getType())
1014                        ->getElementType();
1015   auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1016   if (AsArraySection) {
1017     Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1018                                      SharedAddresses[N].first.getPointer(CGF));
1019     Size = CGF.Builder.CreateNUWAdd(
1020         Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1021     SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1022   } else {
1023     SizeInChars = CGF.getTypeSize(
1024         SharedAddresses[N].first.getType().getNonReferenceType());
1025     Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1026   }
1027   Sizes.emplace_back(SizeInChars, Size);
1028   CodeGenFunction::OpaqueValueMapping OpaqueMap(
1029       CGF,
1030       cast<OpaqueValueExpr>(
1031           CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1032       RValue::get(Size));
1033   CGF.EmitVariablyModifiedType(PrivateType);
1034 }
1035 
1036 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1037                                          llvm::Value *Size) {
1038   const auto *PrivateVD =
1039       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1040   QualType PrivateType = PrivateVD->getType();
1041   if (!PrivateType->isVariablyModifiedType()) {
1042     assert(!Size && !Sizes[N].second &&
1043            "Size should be nullptr for non-variably modified reduction "
1044            "items.");
1045     return;
1046   }
1047   CodeGenFunction::OpaqueValueMapping OpaqueMap(
1048       CGF,
1049       cast<OpaqueValueExpr>(
1050           CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1051       RValue::get(Size));
1052   CGF.EmitVariablyModifiedType(PrivateType);
1053 }
1054 
1055 void ReductionCodeGen::emitInitialization(
1056     CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1057     llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1058   assert(SharedAddresses.size() > N && "No variable was generated");
1059   const auto *PrivateVD =
1060       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1061   const OMPDeclareReductionDecl *DRD =
1062       getReductionInit(ClausesData[N].ReductionOp);
1063   QualType PrivateType = PrivateVD->getType();
1064   PrivateAddr = CGF.Builder.CreateElementBitCast(
1065       PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1066   QualType SharedType = SharedAddresses[N].first.getType();
1067   SharedLVal = CGF.MakeAddrLValue(
1068       CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
1069                                        CGF.ConvertTypeForMem(SharedType)),
1070       SharedType, SharedAddresses[N].first.getBaseInfo(),
1071       CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1072   if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1073     emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1074   } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1075     emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1076                                      PrivateAddr, SharedLVal.getAddress(CGF),
1077                                      SharedLVal.getType());
1078   } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1079              !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1080     CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1081                          PrivateVD->getType().getQualifiers(),
1082                          /*IsInitializer=*/false);
1083   }
1084 }
1085 
1086 bool ReductionCodeGen::needCleanups(unsigned N) {
1087   const auto *PrivateVD =
1088       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1089   QualType PrivateType = PrivateVD->getType();
1090   QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1091   return DTorKind != QualType::DK_none;
1092 }
1093 
1094 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1095                                     Address PrivateAddr) {
1096   const auto *PrivateVD =
1097       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1098   QualType PrivateType = PrivateVD->getType();
1099   QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1100   if (needCleanups(N)) {
1101     PrivateAddr = CGF.Builder.CreateElementBitCast(
1102         PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1103     CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1104   }
1105 }
1106 
1107 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1108                           LValue BaseLV) {
1109   BaseTy = BaseTy.getNonReferenceType();
1110   while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1111          !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1112     if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1113       BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
1114     } else {
1115       LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
1116       BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1117     }
1118     BaseTy = BaseTy->getPointeeType();
1119   }
1120   return CGF.MakeAddrLValue(
1121       CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
1122                                        CGF.ConvertTypeForMem(ElTy)),
1123       BaseLV.getType(), BaseLV.getBaseInfo(),
1124       CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1125 }
1126 
1127 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1128                           llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1129                           llvm::Value *Addr) {
1130   Address Tmp = Address::invalid();
1131   Address TopTmp = Address::invalid();
1132   Address MostTopTmp = Address::invalid();
1133   BaseTy = BaseTy.getNonReferenceType();
1134   while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1135          !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1136     Tmp = CGF.CreateMemTemp(BaseTy);
1137     if (TopTmp.isValid())
1138       CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1139     else
1140       MostTopTmp = Tmp;
1141     TopTmp = Tmp;
1142     BaseTy = BaseTy->getPointeeType();
1143   }
1144   llvm::Type *Ty = BaseLVType;
1145   if (Tmp.isValid())
1146     Ty = Tmp.getElementType();
1147   Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1148   if (Tmp.isValid()) {
1149     CGF.Builder.CreateStore(Addr, Tmp);
1150     return MostTopTmp;
1151   }
1152   return Address(Addr, BaseLVAlignment);
1153 }
1154 
1155 static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1156   const VarDecl *OrigVD = nullptr;
1157   if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1158     const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1159     while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1160       Base = TempOASE->getBase()->IgnoreParenImpCasts();
1161     while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1162       Base = TempASE->getBase()->IgnoreParenImpCasts();
1163     DE = cast<DeclRefExpr>(Base);
1164     OrigVD = cast<VarDecl>(DE->getDecl());
1165   } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1166     const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1167     while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1168       Base = TempASE->getBase()->IgnoreParenImpCasts();
1169     DE = cast<DeclRefExpr>(Base);
1170     OrigVD = cast<VarDecl>(DE->getDecl());
1171   }
1172   return OrigVD;
1173 }
1174 
1175 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1176                                                Address PrivateAddr) {
1177   const DeclRefExpr *DE;
1178   if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1179     BaseDecls.emplace_back(OrigVD);
1180     LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1181     LValue BaseLValue =
1182         loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1183                     OriginalBaseLValue);
1184     llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1185         BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1186     llvm::Value *PrivatePointer =
1187         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1188             PrivateAddr.getPointer(),
1189             SharedAddresses[N].first.getAddress(CGF).getType());
1190     llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1191     return castToBase(CGF, OrigVD->getType(),
1192                       SharedAddresses[N].first.getType(),
1193                       OriginalBaseLValue.getAddress(CGF).getType(),
1194                       OriginalBaseLValue.getAlignment(), Ptr);
1195   }
1196   BaseDecls.emplace_back(
1197       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1198   return PrivateAddr;
1199 }
1200 
1201 bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1202   const OMPDeclareReductionDecl *DRD =
1203       getReductionInit(ClausesData[N].ReductionOp);
1204   return DRD && DRD->getInitializer();
1205 }
1206 
1207 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1208   return CGF.EmitLoadOfPointerLValue(
1209       CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1210       getThreadIDVariable()->getType()->castAs<PointerType>());
1211 }
1212 
1213 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1214   if (!CGF.HaveInsertPoint())
1215     return;
1216   // 1.2.2 OpenMP Language Terminology
1217   // Structured block - An executable statement with a single entry at the
1218   // top and a single exit at the bottom.
1219   // The point of exit cannot be a branch out of the structured block.
1220   // longjmp() and throw() must not violate the entry/exit criteria.
1221   CGF.EHStack.pushTerminate();
1222   CodeGen(CGF);
1223   CGF.EHStack.popTerminate();
1224 }
1225 
1226 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1227     CodeGenFunction &CGF) {
1228   return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1229                             getThreadIDVariable()->getType(),
1230                             AlignmentSource::Decl);
1231 }
1232 
1233 static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1234                                        QualType FieldTy) {
1235   auto *Field = FieldDecl::Create(
1236       C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1237       C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1238       /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1239   Field->setAccess(AS_public);
1240   DC->addDecl(Field);
1241   return Field;
1242 }
1243 
1244 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1245                                  StringRef Separator)
1246     : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1247       OffloadEntriesInfoManager(CGM) {
1248   ASTContext &C = CGM.getContext();
1249   RecordDecl *RD = C.buildImplicitRecord("ident_t");
1250   QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1251   RD->startDefinition();
1252   // reserved_1
1253   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1254   // flags
1255   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1256   // reserved_2
1257   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1258   // reserved_3
1259   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1260   // psource
1261   addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1262   RD->completeDefinition();
1263   IdentQTy = C.getRecordType(RD);
1264   IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1265   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1266 
1267   loadOffloadInfoMetadata();
1268 }
1269 
1270 bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1271                                             const GlobalDecl &OldGD,
1272                                             llvm::GlobalValue *OrigAddr,
1273                                             bool IsForDefinition) {
1274   // Emit at least a definition for the aliasee if the the address of the
1275   // original function is requested.
1276   if (IsForDefinition || OrigAddr)
1277     (void)CGM.GetAddrOfGlobal(NewGD);
1278   StringRef NewMangledName = CGM.getMangledName(NewGD);
1279   llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1280   if (Addr && !Addr->isDeclaration()) {
1281     const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1282     const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD);
1283     llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1284 
1285     // Create a reference to the named value.  This ensures that it is emitted
1286     // if a deferred decl.
1287     llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1288 
1289     // Create the new alias itself, but don't set a name yet.
1290     auto *GA =
1291         llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1292 
1293     if (OrigAddr) {
1294       assert(OrigAddr->isDeclaration() && "Expected declaration");
1295 
1296       GA->takeName(OrigAddr);
1297       OrigAddr->replaceAllUsesWith(
1298           llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1299       OrigAddr->eraseFromParent();
1300     } else {
1301       GA->setName(CGM.getMangledName(OldGD));
1302     }
1303 
1304     // Set attributes which are particular to an alias; this is a
1305     // specialization of the attributes which may be set on a global function.
1306     if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1307         D->isWeakImported())
1308       GA->setLinkage(llvm::Function::WeakAnyLinkage);
1309 
1310     CGM.SetCommonAttributes(OldGD, GA);
1311     return true;
1312   }
1313   return false;
1314 }
1315 
1316 void CGOpenMPRuntime::clear() {
1317   InternalVars.clear();
1318   // Clean non-target variable declarations possibly used only in debug info.
1319   for (const auto &Data : EmittedNonTargetVariables) {
1320     if (!Data.getValue().pointsToAliveValue())
1321       continue;
1322     auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1323     if (!GV)
1324       continue;
1325     if (!GV->isDeclaration() || GV->getNumUses() > 0)
1326       continue;
1327     GV->eraseFromParent();
1328   }
1329   // Emit aliases for the deferred aliasees.
1330   for (const auto &Pair : DeferredVariantFunction) {
1331     StringRef MangledName = CGM.getMangledName(Pair.second.second);
1332     llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1333     // If not able to emit alias, just emit original declaration.
1334     (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1335                                 /*IsForDefinition=*/false);
1336   }
1337 }
1338 
1339 std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1340   SmallString<128> Buffer;
1341   llvm::raw_svector_ostream OS(Buffer);
1342   StringRef Sep = FirstSeparator;
1343   for (StringRef Part : Parts) {
1344     OS << Sep << Part;
1345     Sep = Separator;
1346   }
1347   return OS.str();
1348 }
1349 
1350 static llvm::Function *
1351 emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1352                           const Expr *CombinerInitializer, const VarDecl *In,
1353                           const VarDecl *Out, bool IsCombiner) {
1354   // void .omp_combiner.(Ty *in, Ty *out);
1355   ASTContext &C = CGM.getContext();
1356   QualType PtrTy = C.getPointerType(Ty).withRestrict();
1357   FunctionArgList Args;
1358   ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1359                                /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1360   ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1361                               /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1362   Args.push_back(&OmpOutParm);
1363   Args.push_back(&OmpInParm);
1364   const CGFunctionInfo &FnInfo =
1365       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1366   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1367   std::string Name = CGM.getOpenMPRuntime().getName(
1368       {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1369   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1370                                     Name, &CGM.getModule());
1371   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1372   if (CGM.getLangOpts().Optimize) {
1373     Fn->removeFnAttr(llvm::Attribute::NoInline);
1374     Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1375     Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1376   }
1377   CodeGenFunction CGF(CGM);
1378   // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1379   // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1380   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1381                     Out->getLocation());
1382   CodeGenFunction::OMPPrivateScope Scope(CGF);
1383   Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1384   Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1385     return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1386         .getAddress(CGF);
1387   });
1388   Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1389   Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1390     return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1391         .getAddress(CGF);
1392   });
1393   (void)Scope.Privatize();
1394   if (!IsCombiner && Out->hasInit() &&
1395       !CGF.isTrivialInitializer(Out->getInit())) {
1396     CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1397                          Out->getType().getQualifiers(),
1398                          /*IsInitializer=*/true);
1399   }
1400   if (CombinerInitializer)
1401     CGF.EmitIgnoredExpr(CombinerInitializer);
1402   Scope.ForceCleanup();
1403   CGF.FinishFunction();
1404   return Fn;
1405 }
1406 
1407 void CGOpenMPRuntime::emitUserDefinedReduction(
1408     CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1409   if (UDRMap.count(D) > 0)
1410     return;
1411   llvm::Function *Combiner = emitCombinerOrInitializer(
1412       CGM, D->getType(), D->getCombiner(),
1413       cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1414       cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1415       /*IsCombiner=*/true);
1416   llvm::Function *Initializer = nullptr;
1417   if (const Expr *Init = D->getInitializer()) {
1418     Initializer = emitCombinerOrInitializer(
1419         CGM, D->getType(),
1420         D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1421                                                                      : nullptr,
1422         cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1423         cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1424         /*IsCombiner=*/false);
1425   }
1426   UDRMap.try_emplace(D, Combiner, Initializer);
1427   if (CGF) {
1428     auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1429     Decls.second.push_back(D);
1430   }
1431 }
1432 
1433 std::pair<llvm::Function *, llvm::Function *>
1434 CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1435   auto I = UDRMap.find(D);
1436   if (I != UDRMap.end())
1437     return I->second;
1438   emitUserDefinedReduction(/*CGF=*/nullptr, D);
1439   return UDRMap.lookup(D);
1440 }
1441 
1442 static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1443     CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1444     const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1445     const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1446   assert(ThreadIDVar->getType()->isPointerType() &&
1447          "thread id variable must be of type kmp_int32 *");
1448   CodeGenFunction CGF(CGM, true);
1449   bool HasCancel = false;
1450   if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1451     HasCancel = OPD->hasCancel();
1452   else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1453     HasCancel = OPSD->hasCancel();
1454   else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1455     HasCancel = OPFD->hasCancel();
1456   else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1457     HasCancel = OPFD->hasCancel();
1458   else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1459     HasCancel = OPFD->hasCancel();
1460   else if (const auto *OPFD =
1461                dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1462     HasCancel = OPFD->hasCancel();
1463   else if (const auto *OPFD =
1464                dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1465     HasCancel = OPFD->hasCancel();
1466   CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1467                                     HasCancel, OutlinedHelperName);
1468   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1469   return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1470 }
1471 
1472 llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1473     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1474     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1475   const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1476   return emitParallelOrTeamsOutlinedFunction(
1477       CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1478 }
1479 
1480 llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1481     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1482     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1483   const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1484   return emitParallelOrTeamsOutlinedFunction(
1485       CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1486 }
1487 
1488 llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1489     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1490     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1491     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1492     bool Tied, unsigned &NumberOfParts) {
1493   auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1494                                               PrePostActionTy &) {
1495     llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1496     llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1497     llvm::Value *TaskArgs[] = {
1498         UpLoc, ThreadID,
1499         CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1500                                     TaskTVar->getType()->castAs<PointerType>())
1501             .getPointer(CGF)};
1502     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1503   };
1504   CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1505                                                             UntiedCodeGen);
1506   CodeGen.setAction(Action);
1507   assert(!ThreadIDVar->getType()->isPointerType() &&
1508          "thread id variable must be of type kmp_int32 for tasks");
1509   const OpenMPDirectiveKind Region =
1510       isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1511                                                       : OMPD_task;
1512   const CapturedStmt *CS = D.getCapturedStmt(Region);
1513   const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1514   CodeGenFunction CGF(CGM, true);
1515   CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1516                                         InnermostKind,
1517                                         TD ? TD->hasCancel() : false, Action);
1518   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1519   llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1520   if (!Tied)
1521     NumberOfParts = Action.getNumberOfParts();
1522   return Res;
1523 }
1524 
1525 static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1526                              const RecordDecl *RD, const CGRecordLayout &RL,
1527                              ArrayRef<llvm::Constant *> Data) {
1528   llvm::StructType *StructTy = RL.getLLVMType();
1529   unsigned PrevIdx = 0;
1530   ConstantInitBuilder CIBuilder(CGM);
1531   auto DI = Data.begin();
1532   for (const FieldDecl *FD : RD->fields()) {
1533     unsigned Idx = RL.getLLVMFieldNo(FD);
1534     // Fill the alignment.
1535     for (unsigned I = PrevIdx; I < Idx; ++I)
1536       Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1537     PrevIdx = Idx + 1;
1538     Fields.add(*DI);
1539     ++DI;
1540   }
1541 }
1542 
1543 template <class... As>
1544 static llvm::GlobalVariable *
1545 createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1546                    ArrayRef<llvm::Constant *> Data, const Twine &Name,
1547                    As &&... Args) {
1548   const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1549   const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1550   ConstantInitBuilder CIBuilder(CGM);
1551   ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1552   buildStructValue(Fields, CGM, RD, RL, Data);
1553   return Fields.finishAndCreateGlobal(
1554       Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1555       std::forward<As>(Args)...);
1556 }
1557 
1558 template <typename T>
1559 static void
1560 createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1561                                          ArrayRef<llvm::Constant *> Data,
1562                                          T &Parent) {
1563   const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1564   const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1565   ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1566   buildStructValue(Fields, CGM, RD, RL, Data);
1567   Fields.finishAndAddTo(Parent);
1568 }
1569 
1570 Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1571   CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1572   unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1573   FlagsTy FlagsKey(Flags, Reserved2Flags);
1574   llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1575   if (!Entry) {
1576     if (!DefaultOpenMPPSource) {
1577       // Initialize default location for psource field of ident_t structure of
1578       // all ident_t objects. Format is ";file;function;line;column;;".
1579       // Taken from
1580       // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1581       DefaultOpenMPPSource =
1582           CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1583       DefaultOpenMPPSource =
1584           llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1585     }
1586 
1587     llvm::Constant *Data[] = {
1588         llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1589         llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1590         llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1591         llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1592     llvm::GlobalValue *DefaultOpenMPLocation =
1593         createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1594                            llvm::GlobalValue::PrivateLinkage);
1595     DefaultOpenMPLocation->setUnnamedAddr(
1596         llvm::GlobalValue::UnnamedAddr::Global);
1597 
1598     OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1599   }
1600   return Address(Entry, Align);
1601 }
1602 
1603 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1604                                              bool AtCurrentPoint) {
1605   auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1606   assert(!Elem.second.ServiceInsertPt && "Insert point is set already.");
1607 
1608   llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1609   if (AtCurrentPoint) {
1610     Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1611         Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1612   } else {
1613     Elem.second.ServiceInsertPt =
1614         new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1615     Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1616   }
1617 }
1618 
1619 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1620   auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1621   if (Elem.second.ServiceInsertPt) {
1622     llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1623     Elem.second.ServiceInsertPt = nullptr;
1624     Ptr->eraseFromParent();
1625   }
1626 }
1627 
1628 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1629                                                  SourceLocation Loc,
1630                                                  unsigned Flags) {
1631   Flags |= OMP_IDENT_KMPC;
1632   // If no debug info is generated - return global default location.
1633   if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1634       Loc.isInvalid())
1635     return getOrCreateDefaultLocation(Flags).getPointer();
1636 
1637   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1638 
1639   CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1640   Address LocValue = Address::invalid();
1641   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1642   if (I != OpenMPLocThreadIDMap.end())
1643     LocValue = Address(I->second.DebugLoc, Align);
1644 
1645   // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1646   // GetOpenMPThreadID was called before this routine.
1647   if (!LocValue.isValid()) {
1648     // Generate "ident_t .kmpc_loc.addr;"
1649     Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1650     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1651     Elem.second.DebugLoc = AI.getPointer();
1652     LocValue = AI;
1653 
1654     if (!Elem.second.ServiceInsertPt)
1655       setLocThreadIdInsertPt(CGF);
1656     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1657     CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1658     CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1659                              CGF.getTypeSize(IdentQTy));
1660   }
1661 
1662   // char **psource = &.kmpc_loc_<flags>.addr.psource;
1663   LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1664   auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1665   LValue PSource =
1666       CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1667 
1668   llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1669   if (OMPDebugLoc == nullptr) {
1670     SmallString<128> Buffer2;
1671     llvm::raw_svector_ostream OS2(Buffer2);
1672     // Build debug location
1673     PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1674     OS2 << ";" << PLoc.getFilename() << ";";
1675     if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1676       OS2 << FD->getQualifiedNameAsString();
1677     OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1678     OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1679     OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1680   }
1681   // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1682   CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1683 
1684   // Our callers always pass this to a runtime function, so for
1685   // convenience, go ahead and return a naked pointer.
1686   return LocValue.getPointer();
1687 }
1688 
1689 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1690                                           SourceLocation Loc) {
1691   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1692 
1693   llvm::Value *ThreadID = nullptr;
1694   // Check whether we've already cached a load of the thread id in this
1695   // function.
1696   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1697   if (I != OpenMPLocThreadIDMap.end()) {
1698     ThreadID = I->second.ThreadID;
1699     if (ThreadID != nullptr)
1700       return ThreadID;
1701   }
1702   // If exceptions are enabled, do not use parameter to avoid possible crash.
1703   if (auto *OMPRegionInfo =
1704           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1705     if (OMPRegionInfo->getThreadIDVariable()) {
1706       // Check if this an outlined function with thread id passed as argument.
1707       LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1708       llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1709       if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1710           !CGF.getLangOpts().CXXExceptions ||
1711           CGF.Builder.GetInsertBlock() == TopBlock ||
1712           !isa<llvm::Instruction>(LVal.getPointer(CGF)) ||
1713           cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1714               TopBlock ||
1715           cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1716               CGF.Builder.GetInsertBlock()) {
1717         ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1718         // If value loaded in entry block, cache it and use it everywhere in
1719         // function.
1720         if (CGF.Builder.GetInsertBlock() == TopBlock) {
1721           auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1722           Elem.second.ThreadID = ThreadID;
1723         }
1724         return ThreadID;
1725       }
1726     }
1727   }
1728 
1729   // This is not an outlined function region - need to call __kmpc_int32
1730   // kmpc_global_thread_num(ident_t *loc).
1731   // Generate thread id value and cache this value for use across the
1732   // function.
1733   auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1734   if (!Elem.second.ServiceInsertPt)
1735     setLocThreadIdInsertPt(CGF);
1736   CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1737   CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1738   llvm::CallInst *Call = CGF.Builder.CreateCall(
1739       createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1740       emitUpdateLocation(CGF, Loc));
1741   Call->setCallingConv(CGF.getRuntimeCC());
1742   Elem.second.ThreadID = Call;
1743   return Call;
1744 }
1745 
1746 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1747   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1748   if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1749     clearLocThreadIdInsertPt(CGF);
1750     OpenMPLocThreadIDMap.erase(CGF.CurFn);
1751   }
1752   if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1753     for(auto *D : FunctionUDRMap[CGF.CurFn])
1754       UDRMap.erase(D);
1755     FunctionUDRMap.erase(CGF.CurFn);
1756   }
1757   auto I = FunctionUDMMap.find(CGF.CurFn);
1758   if (I != FunctionUDMMap.end()) {
1759     for(auto *D : I->second)
1760       UDMMap.erase(D);
1761     FunctionUDMMap.erase(I);
1762   }
1763 }
1764 
1765 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1766   return IdentTy->getPointerTo();
1767 }
1768 
1769 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1770   if (!Kmpc_MicroTy) {
1771     // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1772     llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1773                                  llvm::PointerType::getUnqual(CGM.Int32Ty)};
1774     Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1775   }
1776   return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1777 }
1778 
1779 llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1780   llvm::FunctionCallee RTLFn = nullptr;
1781   switch (static_cast<OpenMPRTLFunction>(Function)) {
1782   case OMPRTL__kmpc_fork_call: {
1783     // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1784     // microtask, ...);
1785     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1786                                 getKmpc_MicroPointerTy()};
1787     auto *FnTy =
1788         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1789     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1790     if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1791       if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1792         llvm::LLVMContext &Ctx = F->getContext();
1793         llvm::MDBuilder MDB(Ctx);
1794         // Annotate the callback behavior of the __kmpc_fork_call:
1795         //  - The callback callee is argument number 2 (microtask).
1796         //  - The first two arguments of the callback callee are unknown (-1).
1797         //  - All variadic arguments to the __kmpc_fork_call are passed to the
1798         //    callback callee.
1799         F->addMetadata(
1800             llvm::LLVMContext::MD_callback,
1801             *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1802                                         2, {-1, -1},
1803                                         /* VarArgsArePassed */ true)}));
1804       }
1805     }
1806     break;
1807   }
1808   case OMPRTL__kmpc_global_thread_num: {
1809     // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1810     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1811     auto *FnTy =
1812         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1813     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1814     break;
1815   }
1816   case OMPRTL__kmpc_threadprivate_cached: {
1817     // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1818     // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1819     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1820                                 CGM.VoidPtrTy, CGM.SizeTy,
1821                                 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1822     auto *FnTy =
1823         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1824     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1825     break;
1826   }
1827   case OMPRTL__kmpc_critical: {
1828     // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1829     // kmp_critical_name *crit);
1830     llvm::Type *TypeParams[] = {
1831         getIdentTyPointerTy(), CGM.Int32Ty,
1832         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1833     auto *FnTy =
1834         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1835     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1836     break;
1837   }
1838   case OMPRTL__kmpc_critical_with_hint: {
1839     // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1840     // kmp_critical_name *crit, uintptr_t hint);
1841     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1842                                 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1843                                 CGM.IntPtrTy};
1844     auto *FnTy =
1845         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1846     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1847     break;
1848   }
1849   case OMPRTL__kmpc_threadprivate_register: {
1850     // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1851     // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1852     // typedef void *(*kmpc_ctor)(void *);
1853     auto *KmpcCtorTy =
1854         llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1855                                 /*isVarArg*/ false)->getPointerTo();
1856     // typedef void *(*kmpc_cctor)(void *, void *);
1857     llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1858     auto *KmpcCopyCtorTy =
1859         llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1860                                 /*isVarArg*/ false)
1861             ->getPointerTo();
1862     // typedef void (*kmpc_dtor)(void *);
1863     auto *KmpcDtorTy =
1864         llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1865             ->getPointerTo();
1866     llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1867                               KmpcCopyCtorTy, KmpcDtorTy};
1868     auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1869                                         /*isVarArg*/ false);
1870     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1871     break;
1872   }
1873   case OMPRTL__kmpc_end_critical: {
1874     // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1875     // kmp_critical_name *crit);
1876     llvm::Type *TypeParams[] = {
1877         getIdentTyPointerTy(), CGM.Int32Ty,
1878         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1879     auto *FnTy =
1880         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1881     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1882     break;
1883   }
1884   case OMPRTL__kmpc_cancel_barrier: {
1885     // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1886     // global_tid);
1887     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1888     auto *FnTy =
1889         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1890     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1891     break;
1892   }
1893   case OMPRTL__kmpc_barrier: {
1894     // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1895     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1896     auto *FnTy =
1897         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1898     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1899     break;
1900   }
1901   case OMPRTL__kmpc_for_static_fini: {
1902     // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1903     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1904     auto *FnTy =
1905         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1906     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1907     break;
1908   }
1909   case OMPRTL__kmpc_push_num_threads: {
1910     // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1911     // kmp_int32 num_threads)
1912     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1913                                 CGM.Int32Ty};
1914     auto *FnTy =
1915         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1916     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1917     break;
1918   }
1919   case OMPRTL__kmpc_serialized_parallel: {
1920     // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1921     // global_tid);
1922     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1923     auto *FnTy =
1924         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1925     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1926     break;
1927   }
1928   case OMPRTL__kmpc_end_serialized_parallel: {
1929     // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1930     // global_tid);
1931     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1932     auto *FnTy =
1933         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1934     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1935     break;
1936   }
1937   case OMPRTL__kmpc_flush: {
1938     // Build void __kmpc_flush(ident_t *loc);
1939     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1940     auto *FnTy =
1941         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1942     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1943     break;
1944   }
1945   case OMPRTL__kmpc_master: {
1946     // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1947     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1948     auto *FnTy =
1949         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1950     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1951     break;
1952   }
1953   case OMPRTL__kmpc_end_master: {
1954     // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1955     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1956     auto *FnTy =
1957         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1958     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1959     break;
1960   }
1961   case OMPRTL__kmpc_omp_taskyield: {
1962     // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1963     // int end_part);
1964     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1965     auto *FnTy =
1966         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1967     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1968     break;
1969   }
1970   case OMPRTL__kmpc_single: {
1971     // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1972     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1973     auto *FnTy =
1974         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1975     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1976     break;
1977   }
1978   case OMPRTL__kmpc_end_single: {
1979     // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1980     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1981     auto *FnTy =
1982         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1983     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1984     break;
1985   }
1986   case OMPRTL__kmpc_omp_task_alloc: {
1987     // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1988     // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1989     // kmp_routine_entry_t *task_entry);
1990     assert(KmpRoutineEntryPtrTy != nullptr &&
1991            "Type kmp_routine_entry_t must be created.");
1992     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1993                                 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1994     // Return void * and then cast to particular kmp_task_t type.
1995     auto *FnTy =
1996         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1997     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1998     break;
1999   }
2000   case OMPRTL__kmpc_omp_target_task_alloc: {
2001     // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
2002     // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2003     // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
2004     assert(KmpRoutineEntryPtrTy != nullptr &&
2005            "Type kmp_routine_entry_t must be created.");
2006     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2007                                 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
2008                                 CGM.Int64Ty};
2009     // Return void * and then cast to particular kmp_task_t type.
2010     auto *FnTy =
2011         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2012     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2013     break;
2014   }
2015   case OMPRTL__kmpc_omp_task: {
2016     // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2017     // *new_task);
2018     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2019                                 CGM.VoidPtrTy};
2020     auto *FnTy =
2021         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2022     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2023     break;
2024   }
2025   case OMPRTL__kmpc_copyprivate: {
2026     // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2027     // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2028     // kmp_int32 didit);
2029     llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2030     auto *CpyFnTy =
2031         llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2032     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2033                                 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2034                                 CGM.Int32Ty};
2035     auto *FnTy =
2036         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2037     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2038     break;
2039   }
2040   case OMPRTL__kmpc_reduce: {
2041     // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2042     // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2043     // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2044     llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2045     auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2046                                                /*isVarArg=*/false);
2047     llvm::Type *TypeParams[] = {
2048         getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2049         CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2050         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2051     auto *FnTy =
2052         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2053     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2054     break;
2055   }
2056   case OMPRTL__kmpc_reduce_nowait: {
2057     // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2058     // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2059     // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2060     // *lck);
2061     llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2062     auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2063                                                /*isVarArg=*/false);
2064     llvm::Type *TypeParams[] = {
2065         getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2066         CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2067         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2068     auto *FnTy =
2069         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2070     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2071     break;
2072   }
2073   case OMPRTL__kmpc_end_reduce: {
2074     // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2075     // kmp_critical_name *lck);
2076     llvm::Type *TypeParams[] = {
2077         getIdentTyPointerTy(), CGM.Int32Ty,
2078         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2079     auto *FnTy =
2080         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2081     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2082     break;
2083   }
2084   case OMPRTL__kmpc_end_reduce_nowait: {
2085     // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2086     // kmp_critical_name *lck);
2087     llvm::Type *TypeParams[] = {
2088         getIdentTyPointerTy(), CGM.Int32Ty,
2089         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2090     auto *FnTy =
2091         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2092     RTLFn =
2093         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2094     break;
2095   }
2096   case OMPRTL__kmpc_omp_task_begin_if0: {
2097     // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2098     // *new_task);
2099     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2100                                 CGM.VoidPtrTy};
2101     auto *FnTy =
2102         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2103     RTLFn =
2104         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2105     break;
2106   }
2107   case OMPRTL__kmpc_omp_task_complete_if0: {
2108     // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2109     // *new_task);
2110     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2111                                 CGM.VoidPtrTy};
2112     auto *FnTy =
2113         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2114     RTLFn = CGM.CreateRuntimeFunction(FnTy,
2115                                       /*Name=*/"__kmpc_omp_task_complete_if0");
2116     break;
2117   }
2118   case OMPRTL__kmpc_ordered: {
2119     // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2120     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2121     auto *FnTy =
2122         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2123     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2124     break;
2125   }
2126   case OMPRTL__kmpc_end_ordered: {
2127     // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2128     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2129     auto *FnTy =
2130         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2131     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2132     break;
2133   }
2134   case OMPRTL__kmpc_omp_taskwait: {
2135     // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2136     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2137     auto *FnTy =
2138         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2139     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2140     break;
2141   }
2142   case OMPRTL__kmpc_taskgroup: {
2143     // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2144     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2145     auto *FnTy =
2146         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2147     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2148     break;
2149   }
2150   case OMPRTL__kmpc_end_taskgroup: {
2151     // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2152     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2153     auto *FnTy =
2154         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2155     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2156     break;
2157   }
2158   case OMPRTL__kmpc_push_proc_bind: {
2159     // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2160     // int proc_bind)
2161     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2162     auto *FnTy =
2163         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2164     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2165     break;
2166   }
2167   case OMPRTL__kmpc_omp_task_with_deps: {
2168     // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2169     // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2170     // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2171     llvm::Type *TypeParams[] = {
2172         getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2173         CGM.VoidPtrTy,         CGM.Int32Ty, CGM.VoidPtrTy};
2174     auto *FnTy =
2175         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2176     RTLFn =
2177         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2178     break;
2179   }
2180   case OMPRTL__kmpc_omp_wait_deps: {
2181     // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2182     // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2183     // kmp_depend_info_t *noalias_dep_list);
2184     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2185                                 CGM.Int32Ty,           CGM.VoidPtrTy,
2186                                 CGM.Int32Ty,           CGM.VoidPtrTy};
2187     auto *FnTy =
2188         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2189     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2190     break;
2191   }
2192   case OMPRTL__kmpc_cancellationpoint: {
2193     // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2194     // global_tid, kmp_int32 cncl_kind)
2195     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2196     auto *FnTy =
2197         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2198     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2199     break;
2200   }
2201   case OMPRTL__kmpc_cancel: {
2202     // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2203     // kmp_int32 cncl_kind)
2204     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2205     auto *FnTy =
2206         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2207     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2208     break;
2209   }
2210   case OMPRTL__kmpc_push_num_teams: {
2211     // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2212     // kmp_int32 num_teams, kmp_int32 num_threads)
2213     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2214         CGM.Int32Ty};
2215     auto *FnTy =
2216         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2217     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2218     break;
2219   }
2220   case OMPRTL__kmpc_fork_teams: {
2221     // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2222     // microtask, ...);
2223     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2224                                 getKmpc_MicroPointerTy()};
2225     auto *FnTy =
2226         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2227     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2228     if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2229       if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2230         llvm::LLVMContext &Ctx = F->getContext();
2231         llvm::MDBuilder MDB(Ctx);
2232         // Annotate the callback behavior of the __kmpc_fork_teams:
2233         //  - The callback callee is argument number 2 (microtask).
2234         //  - The first two arguments of the callback callee are unknown (-1).
2235         //  - All variadic arguments to the __kmpc_fork_teams are passed to the
2236         //    callback callee.
2237         F->addMetadata(
2238             llvm::LLVMContext::MD_callback,
2239             *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2240                                         2, {-1, -1},
2241                                         /* VarArgsArePassed */ true)}));
2242       }
2243     }
2244     break;
2245   }
2246   case OMPRTL__kmpc_taskloop: {
2247     // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2248     // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2249     // sched, kmp_uint64 grainsize, void *task_dup);
2250     llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2251                                 CGM.IntTy,
2252                                 CGM.VoidPtrTy,
2253                                 CGM.IntTy,
2254                                 CGM.Int64Ty->getPointerTo(),
2255                                 CGM.Int64Ty->getPointerTo(),
2256                                 CGM.Int64Ty,
2257                                 CGM.IntTy,
2258                                 CGM.IntTy,
2259                                 CGM.Int64Ty,
2260                                 CGM.VoidPtrTy};
2261     auto *FnTy =
2262         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2263     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2264     break;
2265   }
2266   case OMPRTL__kmpc_doacross_init: {
2267     // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2268     // num_dims, struct kmp_dim *dims);
2269     llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2270                                 CGM.Int32Ty,
2271                                 CGM.Int32Ty,
2272                                 CGM.VoidPtrTy};
2273     auto *FnTy =
2274         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2275     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2276     break;
2277   }
2278   case OMPRTL__kmpc_doacross_fini: {
2279     // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2280     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2281     auto *FnTy =
2282         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2283     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2284     break;
2285   }
2286   case OMPRTL__kmpc_doacross_post: {
2287     // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2288     // *vec);
2289     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2290                                 CGM.Int64Ty->getPointerTo()};
2291     auto *FnTy =
2292         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2293     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2294     break;
2295   }
2296   case OMPRTL__kmpc_doacross_wait: {
2297     // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2298     // *vec);
2299     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2300                                 CGM.Int64Ty->getPointerTo()};
2301     auto *FnTy =
2302         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2303     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2304     break;
2305   }
2306   case OMPRTL__kmpc_task_reduction_init: {
2307     // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2308     // *data);
2309     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2310     auto *FnTy =
2311         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2312     RTLFn =
2313         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2314     break;
2315   }
2316   case OMPRTL__kmpc_task_reduction_get_th_data: {
2317     // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2318     // *d);
2319     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2320     auto *FnTy =
2321         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2322     RTLFn = CGM.CreateRuntimeFunction(
2323         FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2324     break;
2325   }
2326   case OMPRTL__kmpc_alloc: {
2327     // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2328     // al); omp_allocator_handle_t type is void *.
2329     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2330     auto *FnTy =
2331         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2332     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2333     break;
2334   }
2335   case OMPRTL__kmpc_free: {
2336     // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2337     // al); omp_allocator_handle_t type is void *.
2338     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2339     auto *FnTy =
2340         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2341     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2342     break;
2343   }
2344   case OMPRTL__kmpc_push_target_tripcount: {
2345     // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2346     // size);
2347     llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2348     llvm::FunctionType *FnTy =
2349         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2350     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2351     break;
2352   }
2353   case OMPRTL__tgt_target: {
2354     // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2355     // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2356     // *arg_types);
2357     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2358                                 CGM.VoidPtrTy,
2359                                 CGM.Int32Ty,
2360                                 CGM.VoidPtrPtrTy,
2361                                 CGM.VoidPtrPtrTy,
2362                                 CGM.Int64Ty->getPointerTo(),
2363                                 CGM.Int64Ty->getPointerTo()};
2364     auto *FnTy =
2365         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2366     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2367     break;
2368   }
2369   case OMPRTL__tgt_target_nowait: {
2370     // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2371     // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2372     // int64_t *arg_types);
2373     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2374                                 CGM.VoidPtrTy,
2375                                 CGM.Int32Ty,
2376                                 CGM.VoidPtrPtrTy,
2377                                 CGM.VoidPtrPtrTy,
2378                                 CGM.Int64Ty->getPointerTo(),
2379                                 CGM.Int64Ty->getPointerTo()};
2380     auto *FnTy =
2381         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2382     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2383     break;
2384   }
2385   case OMPRTL__tgt_target_teams: {
2386     // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2387     // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2388     // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2389     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2390                                 CGM.VoidPtrTy,
2391                                 CGM.Int32Ty,
2392                                 CGM.VoidPtrPtrTy,
2393                                 CGM.VoidPtrPtrTy,
2394                                 CGM.Int64Ty->getPointerTo(),
2395                                 CGM.Int64Ty->getPointerTo(),
2396                                 CGM.Int32Ty,
2397                                 CGM.Int32Ty};
2398     auto *FnTy =
2399         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2400     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2401     break;
2402   }
2403   case OMPRTL__tgt_target_teams_nowait: {
2404     // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2405     // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2406     // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2407     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2408                                 CGM.VoidPtrTy,
2409                                 CGM.Int32Ty,
2410                                 CGM.VoidPtrPtrTy,
2411                                 CGM.VoidPtrPtrTy,
2412                                 CGM.Int64Ty->getPointerTo(),
2413                                 CGM.Int64Ty->getPointerTo(),
2414                                 CGM.Int32Ty,
2415                                 CGM.Int32Ty};
2416     auto *FnTy =
2417         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2418     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2419     break;
2420   }
2421   case OMPRTL__tgt_register_requires: {
2422     // Build void __tgt_register_requires(int64_t flags);
2423     llvm::Type *TypeParams[] = {CGM.Int64Ty};
2424     auto *FnTy =
2425         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2426     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2427     break;
2428   }
2429   case OMPRTL__tgt_register_lib: {
2430     // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2431     QualType ParamTy =
2432         CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2433     llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2434     auto *FnTy =
2435         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2436     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2437     break;
2438   }
2439   case OMPRTL__tgt_unregister_lib: {
2440     // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2441     QualType ParamTy =
2442         CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2443     llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2444     auto *FnTy =
2445         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2446     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2447     break;
2448   }
2449   case OMPRTL__tgt_target_data_begin: {
2450     // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2451     // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2452     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2453                                 CGM.Int32Ty,
2454                                 CGM.VoidPtrPtrTy,
2455                                 CGM.VoidPtrPtrTy,
2456                                 CGM.Int64Ty->getPointerTo(),
2457                                 CGM.Int64Ty->getPointerTo()};
2458     auto *FnTy =
2459         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2460     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2461     break;
2462   }
2463   case OMPRTL__tgt_target_data_begin_nowait: {
2464     // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2465     // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2466     // *arg_types);
2467     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2468                                 CGM.Int32Ty,
2469                                 CGM.VoidPtrPtrTy,
2470                                 CGM.VoidPtrPtrTy,
2471                                 CGM.Int64Ty->getPointerTo(),
2472                                 CGM.Int64Ty->getPointerTo()};
2473     auto *FnTy =
2474         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2475     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2476     break;
2477   }
2478   case OMPRTL__tgt_target_data_end: {
2479     // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2480     // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2481     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2482                                 CGM.Int32Ty,
2483                                 CGM.VoidPtrPtrTy,
2484                                 CGM.VoidPtrPtrTy,
2485                                 CGM.Int64Ty->getPointerTo(),
2486                                 CGM.Int64Ty->getPointerTo()};
2487     auto *FnTy =
2488         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2489     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2490     break;
2491   }
2492   case OMPRTL__tgt_target_data_end_nowait: {
2493     // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2494     // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2495     // *arg_types);
2496     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2497                                 CGM.Int32Ty,
2498                                 CGM.VoidPtrPtrTy,
2499                                 CGM.VoidPtrPtrTy,
2500                                 CGM.Int64Ty->getPointerTo(),
2501                                 CGM.Int64Ty->getPointerTo()};
2502     auto *FnTy =
2503         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2504     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2505     break;
2506   }
2507   case OMPRTL__tgt_target_data_update: {
2508     // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2509     // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2510     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2511                                 CGM.Int32Ty,
2512                                 CGM.VoidPtrPtrTy,
2513                                 CGM.VoidPtrPtrTy,
2514                                 CGM.Int64Ty->getPointerTo(),
2515                                 CGM.Int64Ty->getPointerTo()};
2516     auto *FnTy =
2517         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2518     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2519     break;
2520   }
2521   case OMPRTL__tgt_target_data_update_nowait: {
2522     // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2523     // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2524     // *arg_types);
2525     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2526                                 CGM.Int32Ty,
2527                                 CGM.VoidPtrPtrTy,
2528                                 CGM.VoidPtrPtrTy,
2529                                 CGM.Int64Ty->getPointerTo(),
2530                                 CGM.Int64Ty->getPointerTo()};
2531     auto *FnTy =
2532         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2533     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2534     break;
2535   }
2536   case OMPRTL__tgt_mapper_num_components: {
2537     // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2538     llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2539     auto *FnTy =
2540         llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2541     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2542     break;
2543   }
2544   case OMPRTL__tgt_push_mapper_component: {
2545     // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2546     // *base, void *begin, int64_t size, int64_t type);
2547     llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2548                                 CGM.Int64Ty, CGM.Int64Ty};
2549     auto *FnTy =
2550         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2551     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2552     break;
2553   }
2554   }
2555   assert(RTLFn && "Unable to find OpenMP runtime function");
2556   return RTLFn;
2557 }
2558 
2559 llvm::FunctionCallee
2560 CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2561   assert((IVSize == 32 || IVSize == 64) &&
2562          "IV size is not compatible with the omp runtime");
2563   StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2564                                             : "__kmpc_for_static_init_4u")
2565                                 : (IVSigned ? "__kmpc_for_static_init_8"
2566                                             : "__kmpc_for_static_init_8u");
2567   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2568   auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2569   llvm::Type *TypeParams[] = {
2570     getIdentTyPointerTy(),                     // loc
2571     CGM.Int32Ty,                               // tid
2572     CGM.Int32Ty,                               // schedtype
2573     llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2574     PtrTy,                                     // p_lower
2575     PtrTy,                                     // p_upper
2576     PtrTy,                                     // p_stride
2577     ITy,                                       // incr
2578     ITy                                        // chunk
2579   };
2580   auto *FnTy =
2581       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2582   return CGM.CreateRuntimeFunction(FnTy, Name);
2583 }
2584 
2585 llvm::FunctionCallee
2586 CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2587   assert((IVSize == 32 || IVSize == 64) &&
2588          "IV size is not compatible with the omp runtime");
2589   StringRef Name =
2590       IVSize == 32
2591           ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2592           : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2593   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2594   llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2595                                CGM.Int32Ty,           // tid
2596                                CGM.Int32Ty,           // schedtype
2597                                ITy,                   // lower
2598                                ITy,                   // upper
2599                                ITy,                   // stride
2600                                ITy                    // chunk
2601   };
2602   auto *FnTy =
2603       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2604   return CGM.CreateRuntimeFunction(FnTy, Name);
2605 }
2606 
2607 llvm::FunctionCallee
2608 CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2609   assert((IVSize == 32 || IVSize == 64) &&
2610          "IV size is not compatible with the omp runtime");
2611   StringRef Name =
2612       IVSize == 32
2613           ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2614           : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2615   llvm::Type *TypeParams[] = {
2616       getIdentTyPointerTy(), // loc
2617       CGM.Int32Ty,           // tid
2618   };
2619   auto *FnTy =
2620       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2621   return CGM.CreateRuntimeFunction(FnTy, Name);
2622 }
2623 
2624 llvm::FunctionCallee
2625 CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2626   assert((IVSize == 32 || IVSize == 64) &&
2627          "IV size is not compatible with the omp runtime");
2628   StringRef Name =
2629       IVSize == 32
2630           ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2631           : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2632   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2633   auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2634   llvm::Type *TypeParams[] = {
2635     getIdentTyPointerTy(),                     // loc
2636     CGM.Int32Ty,                               // tid
2637     llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2638     PtrTy,                                     // p_lower
2639     PtrTy,                                     // p_upper
2640     PtrTy                                      // p_stride
2641   };
2642   auto *FnTy =
2643       llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2644   return CGM.CreateRuntimeFunction(FnTy, Name);
2645 }
2646 
2647 /// Obtain information that uniquely identifies a target entry. This
2648 /// consists of the file and device IDs as well as line number associated with
2649 /// the relevant entry source location.
2650 static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2651                                      unsigned &DeviceID, unsigned &FileID,
2652                                      unsigned &LineNum) {
2653   SourceManager &SM = C.getSourceManager();
2654 
2655   // The loc should be always valid and have a file ID (the user cannot use
2656   // #pragma directives in macros)
2657 
2658   assert(Loc.isValid() && "Source location is expected to be always valid.");
2659 
2660   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2661   assert(PLoc.isValid() && "Source location is expected to be always valid.");
2662 
2663   llvm::sys::fs::UniqueID ID;
2664   if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2665     SM.getDiagnostics().Report(diag::err_cannot_open_file)
2666         << PLoc.getFilename() << EC.message();
2667 
2668   DeviceID = ID.getDevice();
2669   FileID = ID.getFile();
2670   LineNum = PLoc.getLine();
2671 }
2672 
2673 Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2674   if (CGM.getLangOpts().OpenMPSimd)
2675     return Address::invalid();
2676   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2677       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2678   if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2679               (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2680                HasRequiresUnifiedSharedMemory))) {
2681     SmallString<64> PtrName;
2682     {
2683       llvm::raw_svector_ostream OS(PtrName);
2684       OS << CGM.getMangledName(GlobalDecl(VD));
2685       if (!VD->isExternallyVisible()) {
2686         unsigned DeviceID, FileID, Line;
2687         getTargetEntryUniqueInfo(CGM.getContext(),
2688                                  VD->getCanonicalDecl()->getBeginLoc(),
2689                                  DeviceID, FileID, Line);
2690         OS << llvm::format("_%x", FileID);
2691       }
2692       OS << "_decl_tgt_ref_ptr";
2693     }
2694     llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2695     if (!Ptr) {
2696       QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2697       Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2698                                         PtrName);
2699 
2700       auto *GV = cast<llvm::GlobalVariable>(Ptr);
2701       GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2702 
2703       if (!CGM.getLangOpts().OpenMPIsDevice)
2704         GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2705       registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2706     }
2707     return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2708   }
2709   return Address::invalid();
2710 }
2711 
2712 llvm::Constant *
2713 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2714   assert(!CGM.getLangOpts().OpenMPUseTLS ||
2715          !CGM.getContext().getTargetInfo().isTLSSupported());
2716   // Lookup the entry, lazily creating it if necessary.
2717   std::string Suffix = getName({"cache", ""});
2718   return getOrCreateInternalVariable(
2719       CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2720 }
2721 
2722 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2723                                                 const VarDecl *VD,
2724                                                 Address VDAddr,
2725                                                 SourceLocation Loc) {
2726   if (CGM.getLangOpts().OpenMPUseTLS &&
2727       CGM.getContext().getTargetInfo().isTLSSupported())
2728     return VDAddr;
2729 
2730   llvm::Type *VarTy = VDAddr.getElementType();
2731   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2732                          CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2733                                                        CGM.Int8PtrTy),
2734                          CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2735                          getOrCreateThreadPrivateCache(VD)};
2736   return Address(CGF.EmitRuntimeCall(
2737       createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2738                  VDAddr.getAlignment());
2739 }
2740 
2741 void CGOpenMPRuntime::emitThreadPrivateVarInit(
2742     CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2743     llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2744   // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2745   // library.
2746   llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2747   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2748                       OMPLoc);
2749   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2750   // to register constructor/destructor for variable.
2751   llvm::Value *Args[] = {
2752       OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2753       Ctor, CopyCtor, Dtor};
2754   CGF.EmitRuntimeCall(
2755       createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2756 }
2757 
2758 llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2759     const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2760     bool PerformInit, CodeGenFunction *CGF) {
2761   if (CGM.getLangOpts().OpenMPUseTLS &&
2762       CGM.getContext().getTargetInfo().isTLSSupported())
2763     return nullptr;
2764 
2765   VD = VD->getDefinition(CGM.getContext());
2766   if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) {
2767     QualType ASTTy = VD->getType();
2768 
2769     llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2770     const Expr *Init = VD->getAnyInitializer();
2771     if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2772       // Generate function that re-emits the declaration's initializer into the
2773       // threadprivate copy of the variable VD
2774       CodeGenFunction CtorCGF(CGM);
2775       FunctionArgList Args;
2776       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2777                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2778                             ImplicitParamDecl::Other);
2779       Args.push_back(&Dst);
2780 
2781       const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2782           CGM.getContext().VoidPtrTy, Args);
2783       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2784       std::string Name = getName({"__kmpc_global_ctor_", ""});
2785       llvm::Function *Fn =
2786           CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2787       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2788                             Args, Loc, Loc);
2789       llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2790           CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2791           CGM.getContext().VoidPtrTy, Dst.getLocation());
2792       Address Arg = Address(ArgVal, VDAddr.getAlignment());
2793       Arg = CtorCGF.Builder.CreateElementBitCast(
2794           Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2795       CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2796                                /*IsInitializer=*/true);
2797       ArgVal = CtorCGF.EmitLoadOfScalar(
2798           CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2799           CGM.getContext().VoidPtrTy, Dst.getLocation());
2800       CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2801       CtorCGF.FinishFunction();
2802       Ctor = Fn;
2803     }
2804     if (VD->getType().isDestructedType() != QualType::DK_none) {
2805       // Generate function that emits destructor call for the threadprivate copy
2806       // of the variable VD
2807       CodeGenFunction DtorCGF(CGM);
2808       FunctionArgList Args;
2809       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2810                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2811                             ImplicitParamDecl::Other);
2812       Args.push_back(&Dst);
2813 
2814       const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2815           CGM.getContext().VoidTy, Args);
2816       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2817       std::string Name = getName({"__kmpc_global_dtor_", ""});
2818       llvm::Function *Fn =
2819           CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2820       auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2821       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2822                             Loc, Loc);
2823       // Create a scope with an artificial location for the body of this function.
2824       auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2825       llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2826           DtorCGF.GetAddrOfLocalVar(&Dst),
2827           /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2828       DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2829                           DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2830                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2831       DtorCGF.FinishFunction();
2832       Dtor = Fn;
2833     }
2834     // Do not emit init function if it is not required.
2835     if (!Ctor && !Dtor)
2836       return nullptr;
2837 
2838     llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2839     auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2840                                                /*isVarArg=*/false)
2841                            ->getPointerTo();
2842     // Copying constructor for the threadprivate variable.
2843     // Must be NULL - reserved by runtime, but currently it requires that this
2844     // parameter is always NULL. Otherwise it fires assertion.
2845     CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2846     if (Ctor == nullptr) {
2847       auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2848                                              /*isVarArg=*/false)
2849                          ->getPointerTo();
2850       Ctor = llvm::Constant::getNullValue(CtorTy);
2851     }
2852     if (Dtor == nullptr) {
2853       auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2854                                              /*isVarArg=*/false)
2855                          ->getPointerTo();
2856       Dtor = llvm::Constant::getNullValue(DtorTy);
2857     }
2858     if (!CGF) {
2859       auto *InitFunctionTy =
2860           llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2861       std::string Name = getName({"__omp_threadprivate_init_", ""});
2862       llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2863           InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2864       CodeGenFunction InitCGF(CGM);
2865       FunctionArgList ArgList;
2866       InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2867                             CGM.getTypes().arrangeNullaryFunction(), ArgList,
2868                             Loc, Loc);
2869       emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2870       InitCGF.FinishFunction();
2871       return InitFunction;
2872     }
2873     emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2874   }
2875   return nullptr;
2876 }
2877 
2878 bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2879                                                      llvm::GlobalVariable *Addr,
2880                                                      bool PerformInit) {
2881   if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2882       !CGM.getLangOpts().OpenMPIsDevice)
2883     return false;
2884   Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2885       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2886   if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2887       (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2888        HasRequiresUnifiedSharedMemory))
2889     return CGM.getLangOpts().OpenMPIsDevice;
2890   VD = VD->getDefinition(CGM.getContext());
2891   if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2892     return CGM.getLangOpts().OpenMPIsDevice;
2893 
2894   QualType ASTTy = VD->getType();
2895 
2896   SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2897   // Produce the unique prefix to identify the new target regions. We use
2898   // the source location of the variable declaration which we know to not
2899   // conflict with any target region.
2900   unsigned DeviceID;
2901   unsigned FileID;
2902   unsigned Line;
2903   getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2904   SmallString<128> Buffer, Out;
2905   {
2906     llvm::raw_svector_ostream OS(Buffer);
2907     OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2908        << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2909   }
2910 
2911   const Expr *Init = VD->getAnyInitializer();
2912   if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2913     llvm::Constant *Ctor;
2914     llvm::Constant *ID;
2915     if (CGM.getLangOpts().OpenMPIsDevice) {
2916       // Generate function that re-emits the declaration's initializer into
2917       // the threadprivate copy of the variable VD
2918       CodeGenFunction CtorCGF(CGM);
2919 
2920       const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2921       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2922       llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2923           FTy, Twine(Buffer, "_ctor"), FI, Loc);
2924       auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2925       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2926                             FunctionArgList(), Loc, Loc);
2927       auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2928       CtorCGF.EmitAnyExprToMem(Init,
2929                                Address(Addr, CGM.getContext().getDeclAlign(VD)),
2930                                Init->getType().getQualifiers(),
2931                                /*IsInitializer=*/true);
2932       CtorCGF.FinishFunction();
2933       Ctor = Fn;
2934       ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2935       CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2936     } else {
2937       Ctor = new llvm::GlobalVariable(
2938           CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2939           llvm::GlobalValue::PrivateLinkage,
2940           llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2941       ID = Ctor;
2942     }
2943 
2944     // Register the information for the entry associated with the constructor.
2945     Out.clear();
2946     OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2947         DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2948         ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2949   }
2950   if (VD->getType().isDestructedType() != QualType::DK_none) {
2951     llvm::Constant *Dtor;
2952     llvm::Constant *ID;
2953     if (CGM.getLangOpts().OpenMPIsDevice) {
2954       // Generate function that emits destructor call for the threadprivate
2955       // copy of the variable VD
2956       CodeGenFunction DtorCGF(CGM);
2957 
2958       const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2959       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2960       llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2961           FTy, Twine(Buffer, "_dtor"), FI, Loc);
2962       auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2963       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2964                             FunctionArgList(), Loc, Loc);
2965       // Create a scope with an artificial location for the body of this
2966       // function.
2967       auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2968       DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2969                           ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2970                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2971       DtorCGF.FinishFunction();
2972       Dtor = Fn;
2973       ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2974       CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2975     } else {
2976       Dtor = new llvm::GlobalVariable(
2977           CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2978           llvm::GlobalValue::PrivateLinkage,
2979           llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2980       ID = Dtor;
2981     }
2982     // Register the information for the entry associated with the destructor.
2983     Out.clear();
2984     OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2985         DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2986         ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2987   }
2988   return CGM.getLangOpts().OpenMPIsDevice;
2989 }
2990 
2991 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2992                                                           QualType VarType,
2993                                                           StringRef Name) {
2994   std::string Suffix = getName({"artificial", ""});
2995   std::string CacheSuffix = getName({"cache", ""});
2996   llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2997   llvm::Value *GAddr =
2998       getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2999   llvm::Value *Args[] = {
3000       emitUpdateLocation(CGF, SourceLocation()),
3001       getThreadID(CGF, SourceLocation()),
3002       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
3003       CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
3004                                 /*isSigned=*/false),
3005       getOrCreateInternalVariable(
3006           CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
3007   return Address(
3008       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3009           CGF.EmitRuntimeCall(
3010               createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3011           VarLVType->getPointerTo(/*AddrSpace=*/0)),
3012       CGM.getPointerAlign());
3013 }
3014 
3015 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
3016                                    const RegionCodeGenTy &ThenGen,
3017                                    const RegionCodeGenTy &ElseGen) {
3018   CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3019 
3020   // If the condition constant folds and can be elided, try to avoid emitting
3021   // the condition and the dead arm of the if/else.
3022   bool CondConstant;
3023   if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
3024     if (CondConstant)
3025       ThenGen(CGF);
3026     else
3027       ElseGen(CGF);
3028     return;
3029   }
3030 
3031   // Otherwise, the condition did not fold, or we couldn't elide it.  Just
3032   // emit the conditional branch.
3033   llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3034   llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3035   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3036   CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3037 
3038   // Emit the 'then' code.
3039   CGF.EmitBlock(ThenBlock);
3040   ThenGen(CGF);
3041   CGF.EmitBranch(ContBlock);
3042   // Emit the 'else' code if present.
3043   // There is no need to emit line number for unconditional branch.
3044   (void)ApplyDebugLocation::CreateEmpty(CGF);
3045   CGF.EmitBlock(ElseBlock);
3046   ElseGen(CGF);
3047   // There is no need to emit line number for unconditional branch.
3048   (void)ApplyDebugLocation::CreateEmpty(CGF);
3049   CGF.EmitBranch(ContBlock);
3050   // Emit the continuation block for code after the if.
3051   CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3052 }
3053 
3054 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3055                                        llvm::Function *OutlinedFn,
3056                                        ArrayRef<llvm::Value *> CapturedVars,
3057                                        const Expr *IfCond) {
3058   if (!CGF.HaveInsertPoint())
3059     return;
3060   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3061   auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3062                                                      PrePostActionTy &) {
3063     // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3064     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3065     llvm::Value *Args[] = {
3066         RTLoc,
3067         CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3068         CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3069     llvm::SmallVector<llvm::Value *, 16> RealArgs;
3070     RealArgs.append(std::begin(Args), std::end(Args));
3071     RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3072 
3073     llvm::FunctionCallee RTLFn =
3074         RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3075     CGF.EmitRuntimeCall(RTLFn, RealArgs);
3076   };
3077   auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3078                                                           PrePostActionTy &) {
3079     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3080     llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3081     // Build calls:
3082     // __kmpc_serialized_parallel(&Loc, GTid);
3083     llvm::Value *Args[] = {RTLoc, ThreadID};
3084     CGF.EmitRuntimeCall(
3085         RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3086 
3087     // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
3088     Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
3089     Address ZeroAddrBound =
3090         CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3091                                          /*Name=*/".bound.zero.addr");
3092     CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
3093     llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3094     // ThreadId for serialized parallels is 0.
3095     OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
3096     OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
3097     OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3098     RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3099 
3100     // __kmpc_end_serialized_parallel(&Loc, GTid);
3101     llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3102     CGF.EmitRuntimeCall(
3103         RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3104         EndArgs);
3105   };
3106   if (IfCond) {
3107     emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3108   } else {
3109     RegionCodeGenTy ThenRCG(ThenGen);
3110     ThenRCG(CGF);
3111   }
3112 }
3113 
3114 // If we're inside an (outlined) parallel region, use the region info's
3115 // thread-ID variable (it is passed in a first argument of the outlined function
3116 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3117 // regular serial code region, get thread ID by calling kmp_int32
3118 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3119 // return the address of that temp.
3120 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3121                                              SourceLocation Loc) {
3122   if (auto *OMPRegionInfo =
3123           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3124     if (OMPRegionInfo->getThreadIDVariable())
3125       return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3126 
3127   llvm::Value *ThreadID = getThreadID(CGF, Loc);
3128   QualType Int32Ty =
3129       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3130   Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3131   CGF.EmitStoreOfScalar(ThreadID,
3132                         CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3133 
3134   return ThreadIDTemp;
3135 }
3136 
3137 llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3138     llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3139   SmallString<256> Buffer;
3140   llvm::raw_svector_ostream Out(Buffer);
3141   Out << Name;
3142   StringRef RuntimeName = Out.str();
3143   auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3144   if (Elem.second) {
3145     assert(Elem.second->getType()->getPointerElementType() == Ty &&
3146            "OMP internal variable has different type than requested");
3147     return &*Elem.second;
3148   }
3149 
3150   return Elem.second = new llvm::GlobalVariable(
3151              CGM.getModule(), Ty, /*IsConstant*/ false,
3152              llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3153              Elem.first(), /*InsertBefore=*/nullptr,
3154              llvm::GlobalValue::NotThreadLocal, AddressSpace);
3155 }
3156 
3157 llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3158   std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3159   std::string Name = getName({Prefix, "var"});
3160   return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3161 }
3162 
3163 namespace {
3164 /// Common pre(post)-action for different OpenMP constructs.
3165 class CommonActionTy final : public PrePostActionTy {
3166   llvm::FunctionCallee EnterCallee;
3167   ArrayRef<llvm::Value *> EnterArgs;
3168   llvm::FunctionCallee ExitCallee;
3169   ArrayRef<llvm::Value *> ExitArgs;
3170   bool Conditional;
3171   llvm::BasicBlock *ContBlock = nullptr;
3172 
3173 public:
3174   CommonActionTy(llvm::FunctionCallee EnterCallee,
3175                  ArrayRef<llvm::Value *> EnterArgs,
3176                  llvm::FunctionCallee ExitCallee,
3177                  ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3178       : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3179         ExitArgs(ExitArgs), Conditional(Conditional) {}
3180   void Enter(CodeGenFunction &CGF) override {
3181     llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3182     if (Conditional) {
3183       llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3184       auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3185       ContBlock = CGF.createBasicBlock("omp_if.end");
3186       // Generate the branch (If-stmt)
3187       CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3188       CGF.EmitBlock(ThenBlock);
3189     }
3190   }
3191   void Done(CodeGenFunction &CGF) {
3192     // Emit the rest of blocks/branches
3193     CGF.EmitBranch(ContBlock);
3194     CGF.EmitBlock(ContBlock, true);
3195   }
3196   void Exit(CodeGenFunction &CGF) override {
3197     CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3198   }
3199 };
3200 } // anonymous namespace
3201 
3202 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3203                                          StringRef CriticalName,
3204                                          const RegionCodeGenTy &CriticalOpGen,
3205                                          SourceLocation Loc, const Expr *Hint) {
3206   // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3207   // CriticalOpGen();
3208   // __kmpc_end_critical(ident_t *, gtid, Lock);
3209   // Prepare arguments and build a call to __kmpc_critical
3210   if (!CGF.HaveInsertPoint())
3211     return;
3212   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3213                          getCriticalRegionLock(CriticalName)};
3214   llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3215                                                 std::end(Args));
3216   if (Hint) {
3217     EnterArgs.push_back(CGF.Builder.CreateIntCast(
3218         CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3219   }
3220   CommonActionTy Action(
3221       createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
3222                                  : OMPRTL__kmpc_critical),
3223       EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3224   CriticalOpGen.setAction(Action);
3225   emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3226 }
3227 
3228 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3229                                        const RegionCodeGenTy &MasterOpGen,
3230                                        SourceLocation Loc) {
3231   if (!CGF.HaveInsertPoint())
3232     return;
3233   // if(__kmpc_master(ident_t *, gtid)) {
3234   //   MasterOpGen();
3235   //   __kmpc_end_master(ident_t *, gtid);
3236   // }
3237   // Prepare arguments and build a call to __kmpc_master
3238   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3239   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3240                         createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3241                         /*Conditional=*/true);
3242   MasterOpGen.setAction(Action);
3243   emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3244   Action.Done(CGF);
3245 }
3246 
3247 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3248                                         SourceLocation Loc) {
3249   if (!CGF.HaveInsertPoint())
3250     return;
3251   // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3252   llvm::Value *Args[] = {
3253       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3254       llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3255   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3256   if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3257     Region->emitUntiedSwitch(CGF);
3258 }
3259 
3260 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3261                                           const RegionCodeGenTy &TaskgroupOpGen,
3262                                           SourceLocation Loc) {
3263   if (!CGF.HaveInsertPoint())
3264     return;
3265   // __kmpc_taskgroup(ident_t *, gtid);
3266   // TaskgroupOpGen();
3267   // __kmpc_end_taskgroup(ident_t *, gtid);
3268   // Prepare arguments and build a call to __kmpc_taskgroup
3269   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3270   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3271                         createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3272                         Args);
3273   TaskgroupOpGen.setAction(Action);
3274   emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3275 }
3276 
3277 /// Given an array of pointers to variables, project the address of a
3278 /// given variable.
3279 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3280                                       unsigned Index, const VarDecl *Var) {
3281   // Pull out the pointer to the variable.
3282   Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3283   llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3284 
3285   Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3286   Addr = CGF.Builder.CreateElementBitCast(
3287       Addr, CGF.ConvertTypeForMem(Var->getType()));
3288   return Addr;
3289 }
3290 
3291 static llvm::Value *emitCopyprivateCopyFunction(
3292     CodeGenModule &CGM, llvm::Type *ArgsType,
3293     ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3294     ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3295     SourceLocation Loc) {
3296   ASTContext &C = CGM.getContext();
3297   // void copy_func(void *LHSArg, void *RHSArg);
3298   FunctionArgList Args;
3299   ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3300                            ImplicitParamDecl::Other);
3301   ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3302                            ImplicitParamDecl::Other);
3303   Args.push_back(&LHSArg);
3304   Args.push_back(&RHSArg);
3305   const auto &CGFI =
3306       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3307   std::string Name =
3308       CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3309   auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3310                                     llvm::GlobalValue::InternalLinkage, Name,
3311                                     &CGM.getModule());
3312   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3313   Fn->setDoesNotRecurse();
3314   CodeGenFunction CGF(CGM);
3315   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3316   // Dest = (void*[n])(LHSArg);
3317   // Src = (void*[n])(RHSArg);
3318   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3319       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3320       ArgsType), CGF.getPointerAlign());
3321   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3322       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3323       ArgsType), CGF.getPointerAlign());
3324   // *(Type0*)Dst[0] = *(Type0*)Src[0];
3325   // *(Type1*)Dst[1] = *(Type1*)Src[1];
3326   // ...
3327   // *(Typen*)Dst[n] = *(Typen*)Src[n];
3328   for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3329     const auto *DestVar =
3330         cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3331     Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3332 
3333     const auto *SrcVar =
3334         cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3335     Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3336 
3337     const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3338     QualType Type = VD->getType();
3339     CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3340   }
3341   CGF.FinishFunction();
3342   return Fn;
3343 }
3344 
3345 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3346                                        const RegionCodeGenTy &SingleOpGen,
3347                                        SourceLocation Loc,
3348                                        ArrayRef<const Expr *> CopyprivateVars,
3349                                        ArrayRef<const Expr *> SrcExprs,
3350                                        ArrayRef<const Expr *> DstExprs,
3351                                        ArrayRef<const Expr *> AssignmentOps) {
3352   if (!CGF.HaveInsertPoint())
3353     return;
3354   assert(CopyprivateVars.size() == SrcExprs.size() &&
3355          CopyprivateVars.size() == DstExprs.size() &&
3356          CopyprivateVars.size() == AssignmentOps.size());
3357   ASTContext &C = CGM.getContext();
3358   // int32 did_it = 0;
3359   // if(__kmpc_single(ident_t *, gtid)) {
3360   //   SingleOpGen();
3361   //   __kmpc_end_single(ident_t *, gtid);
3362   //   did_it = 1;
3363   // }
3364   // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3365   // <copy_func>, did_it);
3366 
3367   Address DidIt = Address::invalid();
3368   if (!CopyprivateVars.empty()) {
3369     // int32 did_it = 0;
3370     QualType KmpInt32Ty =
3371         C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3372     DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3373     CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3374   }
3375   // Prepare arguments and build a call to __kmpc_single
3376   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3377   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3378                         createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3379                         /*Conditional=*/true);
3380   SingleOpGen.setAction(Action);
3381   emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3382   if (DidIt.isValid()) {
3383     // did_it = 1;
3384     CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3385   }
3386   Action.Done(CGF);
3387   // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3388   // <copy_func>, did_it);
3389   if (DidIt.isValid()) {
3390     llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3391     QualType CopyprivateArrayTy = C.getConstantArrayType(
3392         C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3393         /*IndexTypeQuals=*/0);
3394     // Create a list of all private variables for copyprivate.
3395     Address CopyprivateList =
3396         CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3397     for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3398       Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3399       CGF.Builder.CreateStore(
3400           CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3401               CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
3402               CGF.VoidPtrTy),
3403           Elem);
3404     }
3405     // Build function that copies private values from single region to all other
3406     // threads in the corresponding parallel region.
3407     llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3408         CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3409         CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3410     llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3411     Address CL =
3412       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3413                                                       CGF.VoidPtrTy);
3414     llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3415     llvm::Value *Args[] = {
3416         emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3417         getThreadID(CGF, Loc),        // i32 <gtid>
3418         BufSize,                      // size_t <buf_size>
3419         CL.getPointer(),              // void *<copyprivate list>
3420         CpyFn,                        // void (*) (void *, void *) <copy_func>
3421         DidItVal                      // i32 did_it
3422     };
3423     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3424   }
3425 }
3426 
3427 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3428                                         const RegionCodeGenTy &OrderedOpGen,
3429                                         SourceLocation Loc, bool IsThreads) {
3430   if (!CGF.HaveInsertPoint())
3431     return;
3432   // __kmpc_ordered(ident_t *, gtid);
3433   // OrderedOpGen();
3434   // __kmpc_end_ordered(ident_t *, gtid);
3435   // Prepare arguments and build a call to __kmpc_ordered
3436   if (IsThreads) {
3437     llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3438     CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3439                           createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3440                           Args);
3441     OrderedOpGen.setAction(Action);
3442     emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3443     return;
3444   }
3445   emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3446 }
3447 
3448 unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3449   unsigned Flags;
3450   if (Kind == OMPD_for)
3451     Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3452   else if (Kind == OMPD_sections)
3453     Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3454   else if (Kind == OMPD_single)
3455     Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3456   else if (Kind == OMPD_barrier)
3457     Flags = OMP_IDENT_BARRIER_EXPL;
3458   else
3459     Flags = OMP_IDENT_BARRIER_IMPL;
3460   return Flags;
3461 }
3462 
3463 void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3464     CodeGenFunction &CGF, const OMPLoopDirective &S,
3465     OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3466   // Check if the loop directive is actually a doacross loop directive. In this
3467   // case choose static, 1 schedule.
3468   if (llvm::any_of(
3469           S.getClausesOfKind<OMPOrderedClause>(),
3470           [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
3471     ScheduleKind = OMPC_SCHEDULE_static;
3472     // Chunk size is 1 in this case.
3473     llvm::APInt ChunkSize(32, 1);
3474     ChunkExpr = IntegerLiteral::Create(
3475         CGF.getContext(), ChunkSize,
3476         CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3477         SourceLocation());
3478   }
3479 }
3480 
3481 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3482                                       OpenMPDirectiveKind Kind, bool EmitChecks,
3483                                       bool ForceSimpleCall) {
3484   if (!CGF.HaveInsertPoint())
3485     return;
3486   // Build call __kmpc_cancel_barrier(loc, thread_id);
3487   // Build call __kmpc_barrier(loc, thread_id);
3488   unsigned Flags = getDefaultFlagsForBarriers(Kind);
3489   // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3490   // thread_id);
3491   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3492                          getThreadID(CGF, Loc)};
3493   if (auto *OMPRegionInfo =
3494           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3495     if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3496       llvm::Value *Result = CGF.EmitRuntimeCall(
3497           createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3498       if (EmitChecks) {
3499         // if (__kmpc_cancel_barrier()) {
3500         //   exit from construct;
3501         // }
3502         llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3503         llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3504         llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3505         CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3506         CGF.EmitBlock(ExitBB);
3507         //   exit from construct;
3508         CodeGenFunction::JumpDest CancelDestination =
3509             CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3510         CGF.EmitBranchThroughCleanup(CancelDestination);
3511         CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3512       }
3513       return;
3514     }
3515   }
3516   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3517 }
3518 
3519 /// Map the OpenMP loop schedule to the runtime enumeration.
3520 static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3521                                           bool Chunked, bool Ordered) {
3522   switch (ScheduleKind) {
3523   case OMPC_SCHEDULE_static:
3524     return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3525                    : (Ordered ? OMP_ord_static : OMP_sch_static);
3526   case OMPC_SCHEDULE_dynamic:
3527     return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3528   case OMPC_SCHEDULE_guided:
3529     return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3530   case OMPC_SCHEDULE_runtime:
3531     return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3532   case OMPC_SCHEDULE_auto:
3533     return Ordered ? OMP_ord_auto : OMP_sch_auto;
3534   case OMPC_SCHEDULE_unknown:
3535     assert(!Chunked && "chunk was specified but schedule kind not known");
3536     return Ordered ? OMP_ord_static : OMP_sch_static;
3537   }
3538   llvm_unreachable("Unexpected runtime schedule");
3539 }
3540 
3541 /// Map the OpenMP distribute schedule to the runtime enumeration.
3542 static OpenMPSchedType
3543 getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3544   // only static is allowed for dist_schedule
3545   return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3546 }
3547 
3548 bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3549                                          bool Chunked) const {
3550   OpenMPSchedType Schedule =
3551       getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3552   return Schedule == OMP_sch_static;
3553 }
3554 
3555 bool CGOpenMPRuntime::isStaticNonchunked(
3556     OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3557   OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3558   return Schedule == OMP_dist_sch_static;
3559 }
3560 
3561 bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3562                                       bool Chunked) const {
3563   OpenMPSchedType Schedule =
3564       getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3565   return Schedule == OMP_sch_static_chunked;
3566 }
3567 
3568 bool CGOpenMPRuntime::isStaticChunked(
3569     OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3570   OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3571   return Schedule == OMP_dist_sch_static_chunked;
3572 }
3573 
3574 bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3575   OpenMPSchedType Schedule =
3576       getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3577   assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
3578   return Schedule != OMP_sch_static;
3579 }
3580 
3581 static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3582                                   OpenMPScheduleClauseModifier M1,
3583                                   OpenMPScheduleClauseModifier M2) {
3584   int Modifier = 0;
3585   switch (M1) {
3586   case OMPC_SCHEDULE_MODIFIER_monotonic:
3587     Modifier = OMP_sch_modifier_monotonic;
3588     break;
3589   case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3590     Modifier = OMP_sch_modifier_nonmonotonic;
3591     break;
3592   case OMPC_SCHEDULE_MODIFIER_simd:
3593     if (Schedule == OMP_sch_static_chunked)
3594       Schedule = OMP_sch_static_balanced_chunked;
3595     break;
3596   case OMPC_SCHEDULE_MODIFIER_last:
3597   case OMPC_SCHEDULE_MODIFIER_unknown:
3598     break;
3599   }
3600   switch (M2) {
3601   case OMPC_SCHEDULE_MODIFIER_monotonic:
3602     Modifier = OMP_sch_modifier_monotonic;
3603     break;
3604   case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3605     Modifier = OMP_sch_modifier_nonmonotonic;
3606     break;
3607   case OMPC_SCHEDULE_MODIFIER_simd:
3608     if (Schedule == OMP_sch_static_chunked)
3609       Schedule = OMP_sch_static_balanced_chunked;
3610     break;
3611   case OMPC_SCHEDULE_MODIFIER_last:
3612   case OMPC_SCHEDULE_MODIFIER_unknown:
3613     break;
3614   }
3615   // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3616   // If the static schedule kind is specified or if the ordered clause is
3617   // specified, and if the nonmonotonic modifier is not specified, the effect is
3618   // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3619   // modifier is specified, the effect is as if the nonmonotonic modifier is
3620   // specified.
3621   if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
3622     if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
3623           Schedule == OMP_sch_static_balanced_chunked ||
3624           Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
3625           Schedule == OMP_dist_sch_static_chunked ||
3626           Schedule == OMP_dist_sch_static))
3627       Modifier = OMP_sch_modifier_nonmonotonic;
3628   }
3629   return Schedule | Modifier;
3630 }
3631 
3632 void CGOpenMPRuntime::emitForDispatchInit(
3633     CodeGenFunction &CGF, SourceLocation Loc,
3634     const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3635     bool Ordered, const DispatchRTInput &DispatchValues) {
3636   if (!CGF.HaveInsertPoint())
3637     return;
3638   OpenMPSchedType Schedule = getRuntimeSchedule(
3639       ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3640   assert(Ordered ||
3641          (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
3642           Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
3643           Schedule != OMP_sch_static_balanced_chunked));
3644   // Call __kmpc_dispatch_init(
3645   //          ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3646   //          kmp_int[32|64] lower, kmp_int[32|64] upper,
3647   //          kmp_int[32|64] stride, kmp_int[32|64] chunk);
3648 
3649   // If the Chunk was not specified in the clause - use default value 1.
3650   llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3651                                             : CGF.Builder.getIntN(IVSize, 1);
3652   llvm::Value *Args[] = {
3653       emitUpdateLocation(CGF, Loc),
3654       getThreadID(CGF, Loc),
3655       CGF.Builder.getInt32(addMonoNonMonoModifier(
3656           CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3657       DispatchValues.LB,                                     // Lower
3658       DispatchValues.UB,                                     // Upper
3659       CGF.Builder.getIntN(IVSize, 1),                        // Stride
3660       Chunk                                                  // Chunk
3661   };
3662   CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3663 }
3664 
3665 static void emitForStaticInitCall(
3666     CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3667     llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3668     OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3669     const CGOpenMPRuntime::StaticRTInput &Values) {
3670   if (!CGF.HaveInsertPoint())
3671     return;
3672 
3673   assert(!Values.Ordered);
3674   assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
3675          Schedule == OMP_sch_static_balanced_chunked ||
3676          Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
3677          Schedule == OMP_dist_sch_static ||
3678          Schedule == OMP_dist_sch_static_chunked);
3679 
3680   // Call __kmpc_for_static_init(
3681   //          ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3682   //          kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3683   //          kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3684   //          kmp_int[32|64] incr, kmp_int[32|64] chunk);
3685   llvm::Value *Chunk = Values.Chunk;
3686   if (Chunk == nullptr) {
3687     assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
3688             Schedule == OMP_dist_sch_static) &&
3689            "expected static non-chunked schedule");
3690     // If the Chunk was not specified in the clause - use default value 1.
3691     Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3692   } else {
3693     assert((Schedule == OMP_sch_static_chunked ||
3694             Schedule == OMP_sch_static_balanced_chunked ||
3695             Schedule == OMP_ord_static_chunked ||
3696             Schedule == OMP_dist_sch_static_chunked) &&
3697            "expected static chunked schedule");
3698   }
3699   llvm::Value *Args[] = {
3700       UpdateLocation,
3701       ThreadId,
3702       CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3703                                                   M2)), // Schedule type
3704       Values.IL.getPointer(),                           // &isLastIter
3705       Values.LB.getPointer(),                           // &LB
3706       Values.UB.getPointer(),                           // &UB
3707       Values.ST.getPointer(),                           // &Stride
3708       CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
3709       Chunk                                             // Chunk
3710   };
3711   CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3712 }
3713 
3714 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3715                                         SourceLocation Loc,
3716                                         OpenMPDirectiveKind DKind,
3717                                         const OpenMPScheduleTy &ScheduleKind,
3718                                         const StaticRTInput &Values) {
3719   OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3720       ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3721   assert(isOpenMPWorksharingDirective(DKind) &&
3722          "Expected loop-based or sections-based directive.");
3723   llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3724                                              isOpenMPLoopDirective(DKind)
3725                                                  ? OMP_IDENT_WORK_LOOP
3726                                                  : OMP_IDENT_WORK_SECTIONS);
3727   llvm::Value *ThreadId = getThreadID(CGF, Loc);
3728   llvm::FunctionCallee StaticInitFunction =
3729       createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3730   emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3731                         ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3732 }
3733 
3734 void CGOpenMPRuntime::emitDistributeStaticInit(
3735     CodeGenFunction &CGF, SourceLocation Loc,
3736     OpenMPDistScheduleClauseKind SchedKind,
3737     const CGOpenMPRuntime::StaticRTInput &Values) {
3738   OpenMPSchedType ScheduleNum =
3739       getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3740   llvm::Value *UpdatedLocation =
3741       emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3742   llvm::Value *ThreadId = getThreadID(CGF, Loc);
3743   llvm::FunctionCallee StaticInitFunction =
3744       createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3745   emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3746                         ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3747                         OMPC_SCHEDULE_MODIFIER_unknown, Values);
3748 }
3749 
3750 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3751                                           SourceLocation Loc,
3752                                           OpenMPDirectiveKind DKind) {
3753   if (!CGF.HaveInsertPoint())
3754     return;
3755   // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3756   llvm::Value *Args[] = {
3757       emitUpdateLocation(CGF, Loc,
3758                          isOpenMPDistributeDirective(DKind)
3759                              ? OMP_IDENT_WORK_DISTRIBUTE
3760                              : isOpenMPLoopDirective(DKind)
3761                                    ? OMP_IDENT_WORK_LOOP
3762                                    : OMP_IDENT_WORK_SECTIONS),
3763       getThreadID(CGF, Loc)};
3764   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3765                       Args);
3766 }
3767 
3768 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3769                                                  SourceLocation Loc,
3770                                                  unsigned IVSize,
3771                                                  bool IVSigned) {
3772   if (!CGF.HaveInsertPoint())
3773     return;
3774   // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3775   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3776   CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3777 }
3778 
3779 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3780                                           SourceLocation Loc, unsigned IVSize,
3781                                           bool IVSigned, Address IL,
3782                                           Address LB, Address UB,
3783                                           Address ST) {
3784   // Call __kmpc_dispatch_next(
3785   //          ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3786   //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3787   //          kmp_int[32|64] *p_stride);
3788   llvm::Value *Args[] = {
3789       emitUpdateLocation(CGF, Loc),
3790       getThreadID(CGF, Loc),
3791       IL.getPointer(), // &isLastIter
3792       LB.getPointer(), // &Lower
3793       UB.getPointer(), // &Upper
3794       ST.getPointer()  // &Stride
3795   };
3796   llvm::Value *Call =
3797       CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3798   return CGF.EmitScalarConversion(
3799       Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3800       CGF.getContext().BoolTy, Loc);
3801 }
3802 
3803 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3804                                            llvm::Value *NumThreads,
3805                                            SourceLocation Loc) {
3806   if (!CGF.HaveInsertPoint())
3807     return;
3808   // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3809   llvm::Value *Args[] = {
3810       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3811       CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3812   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3813                       Args);
3814 }
3815 
3816 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3817                                          OpenMPProcBindClauseKind ProcBind,
3818                                          SourceLocation Loc) {
3819   if (!CGF.HaveInsertPoint())
3820     return;
3821   // Constants for proc bind value accepted by the runtime.
3822   enum ProcBindTy {
3823     ProcBindFalse = 0,
3824     ProcBindTrue,
3825     ProcBindMaster,
3826     ProcBindClose,
3827     ProcBindSpread,
3828     ProcBindIntel,
3829     ProcBindDefault
3830   } RuntimeProcBind;
3831   switch (ProcBind) {
3832   case OMPC_PROC_BIND_master:
3833     RuntimeProcBind = ProcBindMaster;
3834     break;
3835   case OMPC_PROC_BIND_close:
3836     RuntimeProcBind = ProcBindClose;
3837     break;
3838   case OMPC_PROC_BIND_spread:
3839     RuntimeProcBind = ProcBindSpread;
3840     break;
3841   case OMPC_PROC_BIND_unknown:
3842     llvm_unreachable("Unsupported proc_bind value.");
3843   }
3844   // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3845   llvm::Value *Args[] = {
3846       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3847       llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3848   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3849 }
3850 
3851 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3852                                 SourceLocation Loc) {
3853   if (!CGF.HaveInsertPoint())
3854     return;
3855   // Build call void __kmpc_flush(ident_t *loc)
3856   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3857                       emitUpdateLocation(CGF, Loc));
3858 }
3859 
3860 namespace {
3861 /// Indexes of fields for type kmp_task_t.
3862 enum KmpTaskTFields {
3863   /// List of shared variables.
3864   KmpTaskTShareds,
3865   /// Task routine.
3866   KmpTaskTRoutine,
3867   /// Partition id for the untied tasks.
3868   KmpTaskTPartId,
3869   /// Function with call of destructors for private variables.
3870   Data1,
3871   /// Task priority.
3872   Data2,
3873   /// (Taskloops only) Lower bound.
3874   KmpTaskTLowerBound,
3875   /// (Taskloops only) Upper bound.
3876   KmpTaskTUpperBound,
3877   /// (Taskloops only) Stride.
3878   KmpTaskTStride,
3879   /// (Taskloops only) Is last iteration flag.
3880   KmpTaskTLastIter,
3881   /// (Taskloops only) Reduction data.
3882   KmpTaskTReductions,
3883 };
3884 } // anonymous namespace
3885 
3886 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3887   return OffloadEntriesTargetRegion.empty() &&
3888          OffloadEntriesDeviceGlobalVar.empty();
3889 }
3890 
3891 /// Initialize target region entry.
3892 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3893     initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3894                                     StringRef ParentName, unsigned LineNum,
3895                                     unsigned Order) {
3896   assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3897                                              "only required for the device "
3898                                              "code generation.");
3899   OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3900       OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3901                                    OMPTargetRegionEntryTargetRegion);
3902   ++OffloadingEntriesNum;
3903 }
3904 
3905 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3906     registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3907                                   StringRef ParentName, unsigned LineNum,
3908                                   llvm::Constant *Addr, llvm::Constant *ID,
3909                                   OMPTargetRegionEntryKind Flags) {
3910   // If we are emitting code for a target, the entry is already initialized,
3911   // only has to be registered.
3912   if (CGM.getLangOpts().OpenMPIsDevice) {
3913     if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3914       unsigned DiagID = CGM.getDiags().getCustomDiagID(
3915           DiagnosticsEngine::Error,
3916           "Unable to find target region on line '%0' in the device code.");
3917       CGM.getDiags().Report(DiagID) << LineNum;
3918       return;
3919     }
3920     auto &Entry =
3921         OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3922     assert(Entry.isValid() && "Entry not initialized!");
3923     Entry.setAddress(Addr);
3924     Entry.setID(ID);
3925     Entry.setFlags(Flags);
3926   } else {
3927     OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3928     OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3929     ++OffloadingEntriesNum;
3930   }
3931 }
3932 
3933 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3934     unsigned DeviceID, unsigned FileID, StringRef ParentName,
3935     unsigned LineNum) const {
3936   auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3937   if (PerDevice == OffloadEntriesTargetRegion.end())
3938     return false;
3939   auto PerFile = PerDevice->second.find(FileID);
3940   if (PerFile == PerDevice->second.end())
3941     return false;
3942   auto PerParentName = PerFile->second.find(ParentName);
3943   if (PerParentName == PerFile->second.end())
3944     return false;
3945   auto PerLine = PerParentName->second.find(LineNum);
3946   if (PerLine == PerParentName->second.end())
3947     return false;
3948   // Fail if this entry is already registered.
3949   if (PerLine->second.getAddress() || PerLine->second.getID())
3950     return false;
3951   return true;
3952 }
3953 
3954 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3955     const OffloadTargetRegionEntryInfoActTy &Action) {
3956   // Scan all target region entries and perform the provided action.
3957   for (const auto &D : OffloadEntriesTargetRegion)
3958     for (const auto &F : D.second)
3959       for (const auto &P : F.second)
3960         for (const auto &L : P.second)
3961           Action(D.first, F.first, P.first(), L.first, L.second);
3962 }
3963 
3964 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3965     initializeDeviceGlobalVarEntryInfo(StringRef Name,
3966                                        OMPTargetGlobalVarEntryKind Flags,
3967                                        unsigned Order) {
3968   assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3969                                              "only required for the device "
3970                                              "code generation.");
3971   OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3972   ++OffloadingEntriesNum;
3973 }
3974 
3975 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3976     registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3977                                      CharUnits VarSize,
3978                                      OMPTargetGlobalVarEntryKind Flags,
3979                                      llvm::GlobalValue::LinkageTypes Linkage) {
3980   if (CGM.getLangOpts().OpenMPIsDevice) {
3981     auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3982     assert(Entry.isValid() && Entry.getFlags() == Flags &&
3983            "Entry not initialized!");
3984     assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3985            "Resetting with the new address.");
3986     if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
3987       if (Entry.getVarSize().isZero()) {
3988         Entry.setVarSize(VarSize);
3989         Entry.setLinkage(Linkage);
3990       }
3991       return;
3992     }
3993     Entry.setVarSize(VarSize);
3994     Entry.setLinkage(Linkage);
3995     Entry.setAddress(Addr);
3996   } else {
3997     if (hasDeviceGlobalVarEntryInfo(VarName)) {
3998       auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3999       assert(Entry.isValid() && Entry.getFlags() == Flags &&
4000              "Entry not initialized!");
4001       assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
4002              "Resetting with the new address.");
4003       if (Entry.getVarSize().isZero()) {
4004         Entry.setVarSize(VarSize);
4005         Entry.setLinkage(Linkage);
4006       }
4007       return;
4008     }
4009     OffloadEntriesDeviceGlobalVar.try_emplace(
4010         VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4011     ++OffloadingEntriesNum;
4012   }
4013 }
4014 
4015 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4016     actOnDeviceGlobalVarEntriesInfo(
4017         const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4018   // Scan all target region entries and perform the provided action.
4019   for (const auto &E : OffloadEntriesDeviceGlobalVar)
4020     Action(E.getKey(), E.getValue());
4021 }
4022 
4023 void CGOpenMPRuntime::createOffloadEntry(
4024     llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4025     llvm::GlobalValue::LinkageTypes Linkage) {
4026   StringRef Name = Addr->getName();
4027   llvm::Module &M = CGM.getModule();
4028   llvm::LLVMContext &C = M.getContext();
4029 
4030   // Create constant string with the name.
4031   llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4032 
4033   std::string StringName = getName({"omp_offloading", "entry_name"});
4034   auto *Str = new llvm::GlobalVariable(
4035       M, StrPtrInit->getType(), /*isConstant=*/true,
4036       llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4037   Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4038 
4039   llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4040                             llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4041                             llvm::ConstantInt::get(CGM.SizeTy, Size),
4042                             llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4043                             llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4044   std::string EntryName = getName({"omp_offloading", "entry", ""});
4045   llvm::GlobalVariable *Entry = createGlobalStruct(
4046       CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4047       Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4048 
4049   // The entry has to be created in the section the linker expects it to be.
4050   Entry->setSection("omp_offloading_entries");
4051 }
4052 
4053 void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4054   // Emit the offloading entries and metadata so that the device codegen side
4055   // can easily figure out what to emit. The produced metadata looks like
4056   // this:
4057   //
4058   // !omp_offload.info = !{!1, ...}
4059   //
4060   // Right now we only generate metadata for function that contain target
4061   // regions.
4062 
4063   // If we are in simd mode or there are no entries, we don't need to do
4064   // anything.
4065   if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty())
4066     return;
4067 
4068   llvm::Module &M = CGM.getModule();
4069   llvm::LLVMContext &C = M.getContext();
4070   SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4071                          SourceLocation, StringRef>,
4072               16>
4073       OrderedEntries(OffloadEntriesInfoManager.size());
4074   llvm::SmallVector<StringRef, 16> ParentFunctions(
4075       OffloadEntriesInfoManager.size());
4076 
4077   // Auxiliary methods to create metadata values and strings.
4078   auto &&GetMDInt = [this](unsigned V) {
4079     return llvm::ConstantAsMetadata::get(
4080         llvm::ConstantInt::get(CGM.Int32Ty, V));
4081   };
4082 
4083   auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4084 
4085   // Create the offloading info metadata node.
4086   llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4087 
4088   // Create function that emits metadata for each target region entry;
4089   auto &&TargetRegionMetadataEmitter =
4090       [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4091        &GetMDString](
4092           unsigned DeviceID, unsigned FileID, StringRef ParentName,
4093           unsigned Line,
4094           const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4095         // Generate metadata for target regions. Each entry of this metadata
4096         // contains:
4097         // - Entry 0 -> Kind of this type of metadata (0).
4098         // - Entry 1 -> Device ID of the file where the entry was identified.
4099         // - Entry 2 -> File ID of the file where the entry was identified.
4100         // - Entry 3 -> Mangled name of the function where the entry was
4101         // identified.
4102         // - Entry 4 -> Line in the file where the entry was identified.
4103         // - Entry 5 -> Order the entry was created.
4104         // The first element of the metadata node is the kind.
4105         llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4106                                  GetMDInt(FileID),      GetMDString(ParentName),
4107                                  GetMDInt(Line),        GetMDInt(E.getOrder())};
4108 
4109         SourceLocation Loc;
4110         for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4111                   E = CGM.getContext().getSourceManager().fileinfo_end();
4112              I != E; ++I) {
4113           if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4114               I->getFirst()->getUniqueID().getFile() == FileID) {
4115             Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4116                 I->getFirst(), Line, 1);
4117             break;
4118           }
4119         }
4120         // Save this entry in the right position of the ordered entries array.
4121         OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4122         ParentFunctions[E.getOrder()] = ParentName;
4123 
4124         // Add metadata to the named metadata node.
4125         MD->addOperand(llvm::MDNode::get(C, Ops));
4126       };
4127 
4128   OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4129       TargetRegionMetadataEmitter);
4130 
4131   // Create function that emits metadata for each device global variable entry;
4132   auto &&DeviceGlobalVarMetadataEmitter =
4133       [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4134        MD](StringRef MangledName,
4135            const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4136                &E) {
4137         // Generate metadata for global variables. Each entry of this metadata
4138         // contains:
4139         // - Entry 0 -> Kind of this type of metadata (1).
4140         // - Entry 1 -> Mangled name of the variable.
4141         // - Entry 2 -> Declare target kind.
4142         // - Entry 3 -> Order the entry was created.
4143         // The first element of the metadata node is the kind.
4144         llvm::Metadata *Ops[] = {
4145             GetMDInt(E.getKind()), GetMDString(MangledName),
4146             GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4147 
4148         // Save this entry in the right position of the ordered entries array.
4149         OrderedEntries[E.getOrder()] =
4150             std::make_tuple(&E, SourceLocation(), MangledName);
4151 
4152         // Add metadata to the named metadata node.
4153         MD->addOperand(llvm::MDNode::get(C, Ops));
4154       };
4155 
4156   OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4157       DeviceGlobalVarMetadataEmitter);
4158 
4159   for (const auto &E : OrderedEntries) {
4160     assert(std::get<0>(E) && "All ordered entries must exist!");
4161     if (const auto *CE =
4162             dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4163                 std::get<0>(E))) {
4164       if (!CE->getID() || !CE->getAddress()) {
4165         // Do not blame the entry if the parent funtion is not emitted.
4166         StringRef FnName = ParentFunctions[CE->getOrder()];
4167         if (!CGM.GetGlobalValue(FnName))
4168           continue;
4169         unsigned DiagID = CGM.getDiags().getCustomDiagID(
4170             DiagnosticsEngine::Error,
4171             "Offloading entry for target region in %0 is incorrect: either the "
4172             "address or the ID is invalid.");
4173         CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4174         continue;
4175       }
4176       createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4177                          CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4178     } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
4179                                              OffloadEntryInfoDeviceGlobalVar>(
4180                    std::get<0>(E))) {
4181       OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4182           static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4183               CE->getFlags());
4184       switch (Flags) {
4185       case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4186         if (CGM.getLangOpts().OpenMPIsDevice &&
4187             CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())
4188           continue;
4189         if (!CE->getAddress()) {
4190           unsigned DiagID = CGM.getDiags().getCustomDiagID(
4191               DiagnosticsEngine::Error, "Offloading entry for declare target "
4192                                         "variable %0 is incorrect: the "
4193                                         "address is invalid.");
4194           CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4195           continue;
4196         }
4197         // The vaiable has no definition - no need to add the entry.
4198         if (CE->getVarSize().isZero())
4199           continue;
4200         break;
4201       }
4202       case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4203         assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4204                 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&
4205                "Declaret target link address is set.");
4206         if (CGM.getLangOpts().OpenMPIsDevice)
4207           continue;
4208         if (!CE->getAddress()) {
4209           unsigned DiagID = CGM.getDiags().getCustomDiagID(
4210               DiagnosticsEngine::Error,
4211               "Offloading entry for declare target variable is incorrect: the "
4212               "address is invalid.");
4213           CGM.getDiags().Report(DiagID);
4214           continue;
4215         }
4216         break;
4217       }
4218       createOffloadEntry(CE->getAddress(), CE->getAddress(),
4219                          CE->getVarSize().getQuantity(), Flags,
4220                          CE->getLinkage());
4221     } else {
4222       llvm_unreachable("Unsupported entry kind.");
4223     }
4224   }
4225 }
4226 
4227 /// Loads all the offload entries information from the host IR
4228 /// metadata.
4229 void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4230   // If we are in target mode, load the metadata from the host IR. This code has
4231   // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4232 
4233   if (!CGM.getLangOpts().OpenMPIsDevice)
4234     return;
4235 
4236   if (CGM.getLangOpts().OMPHostIRFile.empty())
4237     return;
4238 
4239   auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4240   if (auto EC = Buf.getError()) {
4241     CGM.getDiags().Report(diag::err_cannot_open_file)
4242         << CGM.getLangOpts().OMPHostIRFile << EC.message();
4243     return;
4244   }
4245 
4246   llvm::LLVMContext C;
4247   auto ME = expectedToErrorOrAndEmitErrors(
4248       C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4249 
4250   if (auto EC = ME.getError()) {
4251     unsigned DiagID = CGM.getDiags().getCustomDiagID(
4252         DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4253     CGM.getDiags().Report(DiagID)
4254         << CGM.getLangOpts().OMPHostIRFile << EC.message();
4255     return;
4256   }
4257 
4258   llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4259   if (!MD)
4260     return;
4261 
4262   for (llvm::MDNode *MN : MD->operands()) {
4263     auto &&GetMDInt = [MN](unsigned Idx) {
4264       auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4265       return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4266     };
4267 
4268     auto &&GetMDString = [MN](unsigned Idx) {
4269       auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4270       return V->getString();
4271     };
4272 
4273     switch (GetMDInt(0)) {
4274     default:
4275       llvm_unreachable("Unexpected metadata!");
4276       break;
4277     case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4278         OffloadingEntryInfoTargetRegion:
4279       OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4280           /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4281           /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4282           /*Order=*/GetMDInt(5));
4283       break;
4284     case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4285         OffloadingEntryInfoDeviceGlobalVar:
4286       OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4287           /*MangledName=*/GetMDString(1),
4288           static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4289               /*Flags=*/GetMDInt(2)),
4290           /*Order=*/GetMDInt(3));
4291       break;
4292     }
4293   }
4294 }
4295 
4296 void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4297   if (!KmpRoutineEntryPtrTy) {
4298     // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4299     ASTContext &C = CGM.getContext();
4300     QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4301     FunctionProtoType::ExtProtoInfo EPI;
4302     KmpRoutineEntryPtrQTy = C.getPointerType(
4303         C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4304     KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4305   }
4306 }
4307 
4308 QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4309   // Make sure the type of the entry is already created. This is the type we
4310   // have to create:
4311   // struct __tgt_offload_entry{
4312   //   void      *addr;       // Pointer to the offload entry info.
4313   //                          // (function or global)
4314   //   char      *name;       // Name of the function or global.
4315   //   size_t     size;       // Size of the entry info (0 if it a function).
4316   //   int32_t    flags;      // Flags associated with the entry, e.g. 'link'.
4317   //   int32_t    reserved;   // Reserved, to use by the runtime library.
4318   // };
4319   if (TgtOffloadEntryQTy.isNull()) {
4320     ASTContext &C = CGM.getContext();
4321     RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4322     RD->startDefinition();
4323     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4324     addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4325     addFieldToRecordDecl(C, RD, C.getSizeType());
4326     addFieldToRecordDecl(
4327         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4328     addFieldToRecordDecl(
4329         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4330     RD->completeDefinition();
4331     RD->addAttr(PackedAttr::CreateImplicit(C));
4332     TgtOffloadEntryQTy = C.getRecordType(RD);
4333   }
4334   return TgtOffloadEntryQTy;
4335 }
4336 
4337 QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4338   // These are the types we need to build:
4339   // struct __tgt_device_image{
4340   // void   *ImageStart;       // Pointer to the target code start.
4341   // void   *ImageEnd;         // Pointer to the target code end.
4342   // // We also add the host entries to the device image, as it may be useful
4343   // // for the target runtime to have access to that information.
4344   // __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all
4345   //                                       // the entries.
4346   // __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4347   //                                       // entries (non inclusive).
4348   // };
4349   if (TgtDeviceImageQTy.isNull()) {
4350     ASTContext &C = CGM.getContext();
4351     RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4352     RD->startDefinition();
4353     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4354     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4355     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4356     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4357     RD->completeDefinition();
4358     TgtDeviceImageQTy = C.getRecordType(RD);
4359   }
4360   return TgtDeviceImageQTy;
4361 }
4362 
4363 QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4364   // struct __tgt_bin_desc{
4365   //   int32_t              NumDevices;      // Number of devices supported.
4366   //   __tgt_device_image   *DeviceImages;   // Arrays of device images
4367   //                                         // (one per device).
4368   //   __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all the
4369   //                                         // entries.
4370   //   __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4371   //                                         // entries (non inclusive).
4372   // };
4373   if (TgtBinaryDescriptorQTy.isNull()) {
4374     ASTContext &C = CGM.getContext();
4375     RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4376     RD->startDefinition();
4377     addFieldToRecordDecl(
4378         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4379     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4380     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4381     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4382     RD->completeDefinition();
4383     TgtBinaryDescriptorQTy = C.getRecordType(RD);
4384   }
4385   return TgtBinaryDescriptorQTy;
4386 }
4387 
4388 namespace {
4389 struct PrivateHelpersTy {
4390   PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4391                    const VarDecl *PrivateElemInit)
4392       : Original(Original), PrivateCopy(PrivateCopy),
4393         PrivateElemInit(PrivateElemInit) {}
4394   const VarDecl *Original;
4395   const VarDecl *PrivateCopy;
4396   const VarDecl *PrivateElemInit;
4397 };
4398 typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4399 } // anonymous namespace
4400 
4401 static RecordDecl *
4402 createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4403   if (!Privates.empty()) {
4404     ASTContext &C = CGM.getContext();
4405     // Build struct .kmp_privates_t. {
4406     //         /*  private vars  */
4407     //       };
4408     RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4409     RD->startDefinition();
4410     for (const auto &Pair : Privates) {
4411       const VarDecl *VD = Pair.second.Original;
4412       QualType Type = VD->getType().getNonReferenceType();
4413       FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4414       if (VD->hasAttrs()) {
4415         for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4416              E(VD->getAttrs().end());
4417              I != E; ++I)
4418           FD->addAttr(*I);
4419       }
4420     }
4421     RD->completeDefinition();
4422     return RD;
4423   }
4424   return nullptr;
4425 }
4426 
4427 static RecordDecl *
4428 createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4429                          QualType KmpInt32Ty,
4430                          QualType KmpRoutineEntryPointerQTy) {
4431   ASTContext &C = CGM.getContext();
4432   // Build struct kmp_task_t {
4433   //         void *              shareds;
4434   //         kmp_routine_entry_t routine;
4435   //         kmp_int32           part_id;
4436   //         kmp_cmplrdata_t data1;
4437   //         kmp_cmplrdata_t data2;
4438   // For taskloops additional fields:
4439   //         kmp_uint64          lb;
4440   //         kmp_uint64          ub;
4441   //         kmp_int64           st;
4442   //         kmp_int32           liter;
4443   //         void *              reductions;
4444   //       };
4445   RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4446   UD->startDefinition();
4447   addFieldToRecordDecl(C, UD, KmpInt32Ty);
4448   addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4449   UD->completeDefinition();
4450   QualType KmpCmplrdataTy = C.getRecordType(UD);
4451   RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4452   RD->startDefinition();
4453   addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4454   addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4455   addFieldToRecordDecl(C, RD, KmpInt32Ty);
4456   addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4457   addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4458   if (isOpenMPTaskLoopDirective(Kind)) {
4459     QualType KmpUInt64Ty =
4460         CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4461     QualType KmpInt64Ty =
4462         CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4463     addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4464     addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4465     addFieldToRecordDecl(C, RD, KmpInt64Ty);
4466     addFieldToRecordDecl(C, RD, KmpInt32Ty);
4467     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4468   }
4469   RD->completeDefinition();
4470   return RD;
4471 }
4472 
4473 static RecordDecl *
4474 createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4475                                      ArrayRef<PrivateDataTy> Privates) {
4476   ASTContext &C = CGM.getContext();
4477   // Build struct kmp_task_t_with_privates {
4478   //         kmp_task_t task_data;
4479   //         .kmp_privates_t. privates;
4480   //       };
4481   RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4482   RD->startDefinition();
4483   addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4484   if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4485     addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4486   RD->completeDefinition();
4487   return RD;
4488 }
4489 
4490 /// Emit a proxy function which accepts kmp_task_t as the second
4491 /// argument.
4492 /// \code
4493 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4494 ///   TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4495 ///   For taskloops:
4496 ///   tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4497 ///   tt->reductions, tt->shareds);
4498 ///   return 0;
4499 /// }
4500 /// \endcode
4501 static llvm::Function *
4502 emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4503                       OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4504                       QualType KmpTaskTWithPrivatesPtrQTy,
4505                       QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4506                       QualType SharedsPtrTy, llvm::Function *TaskFunction,
4507                       llvm::Value *TaskPrivatesMap) {
4508   ASTContext &C = CGM.getContext();
4509   FunctionArgList Args;
4510   ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4511                             ImplicitParamDecl::Other);
4512   ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4513                                 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4514                                 ImplicitParamDecl::Other);
4515   Args.push_back(&GtidArg);
4516   Args.push_back(&TaskTypeArg);
4517   const auto &TaskEntryFnInfo =
4518       CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4519   llvm::FunctionType *TaskEntryTy =
4520       CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4521   std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4522   auto *TaskEntry = llvm::Function::Create(
4523       TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4524   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4525   TaskEntry->setDoesNotRecurse();
4526   CodeGenFunction CGF(CGM);
4527   CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4528                     Loc, Loc);
4529 
4530   // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4531   // tt,
4532   // For taskloops:
4533   // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4534   // tt->task_data.shareds);
4535   llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4536       CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4537   LValue TDBase = CGF.EmitLoadOfPointerLValue(
4538       CGF.GetAddrOfLocalVar(&TaskTypeArg),
4539       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4540   const auto *KmpTaskTWithPrivatesQTyRD =
4541       cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4542   LValue Base =
4543       CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4544   const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4545   auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4546   LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4547   llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4548 
4549   auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4550   LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4551   llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4552       CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4553       CGF.ConvertTypeForMem(SharedsPtrTy));
4554 
4555   auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4556   llvm::Value *PrivatesParam;
4557   if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4558     LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4559     PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4560         PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
4561   } else {
4562     PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4563   }
4564 
4565   llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4566                                TaskPrivatesMap,
4567                                CGF.Builder
4568                                    .CreatePointerBitCastOrAddrSpaceCast(
4569                                        TDBase.getAddress(CGF), CGF.VoidPtrTy)
4570                                    .getPointer()};
4571   SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4572                                           std::end(CommonArgs));
4573   if (isOpenMPTaskLoopDirective(Kind)) {
4574     auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4575     LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4576     llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4577     auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4578     LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4579     llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4580     auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4581     LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4582     llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4583     auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4584     LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4585     llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4586     auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4587     LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4588     llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4589     CallArgs.push_back(LBParam);
4590     CallArgs.push_back(UBParam);
4591     CallArgs.push_back(StParam);
4592     CallArgs.push_back(LIParam);
4593     CallArgs.push_back(RParam);
4594   }
4595   CallArgs.push_back(SharedsParam);
4596 
4597   CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4598                                                   CallArgs);
4599   CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4600                              CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4601   CGF.FinishFunction();
4602   return TaskEntry;
4603 }
4604 
4605 static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4606                                             SourceLocation Loc,
4607                                             QualType KmpInt32Ty,
4608                                             QualType KmpTaskTWithPrivatesPtrQTy,
4609                                             QualType KmpTaskTWithPrivatesQTy) {
4610   ASTContext &C = CGM.getContext();
4611   FunctionArgList Args;
4612   ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4613                             ImplicitParamDecl::Other);
4614   ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4615                                 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4616                                 ImplicitParamDecl::Other);
4617   Args.push_back(&GtidArg);
4618   Args.push_back(&TaskTypeArg);
4619   const auto &DestructorFnInfo =
4620       CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4621   llvm::FunctionType *DestructorFnTy =
4622       CGM.getTypes().GetFunctionType(DestructorFnInfo);
4623   std::string Name =
4624       CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4625   auto *DestructorFn =
4626       llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4627                              Name, &CGM.getModule());
4628   CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4629                                     DestructorFnInfo);
4630   DestructorFn->setDoesNotRecurse();
4631   CodeGenFunction CGF(CGM);
4632   CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4633                     Args, Loc, Loc);
4634 
4635   LValue Base = CGF.EmitLoadOfPointerLValue(
4636       CGF.GetAddrOfLocalVar(&TaskTypeArg),
4637       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4638   const auto *KmpTaskTWithPrivatesQTyRD =
4639       cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4640   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4641   Base = CGF.EmitLValueForField(Base, *FI);
4642   for (const auto *Field :
4643        cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4644     if (QualType::DestructionKind DtorKind =
4645             Field->getType().isDestructedType()) {
4646       LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4647       CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
4648     }
4649   }
4650   CGF.FinishFunction();
4651   return DestructorFn;
4652 }
4653 
4654 /// Emit a privates mapping function for correct handling of private and
4655 /// firstprivate variables.
4656 /// \code
4657 /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4658 /// **noalias priv1,...,  <tyn> **noalias privn) {
4659 ///   *priv1 = &.privates.priv1;
4660 ///   ...;
4661 ///   *privn = &.privates.privn;
4662 /// }
4663 /// \endcode
4664 static llvm::Value *
4665 emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4666                                ArrayRef<const Expr *> PrivateVars,
4667                                ArrayRef<const Expr *> FirstprivateVars,
4668                                ArrayRef<const Expr *> LastprivateVars,
4669                                QualType PrivatesQTy,
4670                                ArrayRef<PrivateDataTy> Privates) {
4671   ASTContext &C = CGM.getContext();
4672   FunctionArgList Args;
4673   ImplicitParamDecl TaskPrivatesArg(
4674       C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4675       C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4676       ImplicitParamDecl::Other);
4677   Args.push_back(&TaskPrivatesArg);
4678   llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4679   unsigned Counter = 1;
4680   for (const Expr *E : PrivateVars) {
4681     Args.push_back(ImplicitParamDecl::Create(
4682         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4683         C.getPointerType(C.getPointerType(E->getType()))
4684             .withConst()
4685             .withRestrict(),
4686         ImplicitParamDecl::Other));
4687     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4688     PrivateVarsPos[VD] = Counter;
4689     ++Counter;
4690   }
4691   for (const Expr *E : FirstprivateVars) {
4692     Args.push_back(ImplicitParamDecl::Create(
4693         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4694         C.getPointerType(C.getPointerType(E->getType()))
4695             .withConst()
4696             .withRestrict(),
4697         ImplicitParamDecl::Other));
4698     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4699     PrivateVarsPos[VD] = Counter;
4700     ++Counter;
4701   }
4702   for (const Expr *E : LastprivateVars) {
4703     Args.push_back(ImplicitParamDecl::Create(
4704         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4705         C.getPointerType(C.getPointerType(E->getType()))
4706             .withConst()
4707             .withRestrict(),
4708         ImplicitParamDecl::Other));
4709     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4710     PrivateVarsPos[VD] = Counter;
4711     ++Counter;
4712   }
4713   const auto &TaskPrivatesMapFnInfo =
4714       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4715   llvm::FunctionType *TaskPrivatesMapTy =
4716       CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4717   std::string Name =
4718       CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4719   auto *TaskPrivatesMap = llvm::Function::Create(
4720       TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4721       &CGM.getModule());
4722   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4723                                     TaskPrivatesMapFnInfo);
4724   if (CGM.getLangOpts().Optimize) {
4725     TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4726     TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4727     TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4728   }
4729   CodeGenFunction CGF(CGM);
4730   CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4731                     TaskPrivatesMapFnInfo, Args, Loc, Loc);
4732 
4733   // *privi = &.privates.privi;
4734   LValue Base = CGF.EmitLoadOfPointerLValue(
4735       CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4736       TaskPrivatesArg.getType()->castAs<PointerType>());
4737   const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4738   Counter = 0;
4739   for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4740     LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4741     const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4742     LValue RefLVal =
4743         CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4744     LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4745         RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
4746     CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
4747     ++Counter;
4748   }
4749   CGF.FinishFunction();
4750   return TaskPrivatesMap;
4751 }
4752 
4753 /// Emit initialization for private variables in task-based directives.
4754 static void emitPrivatesInit(CodeGenFunction &CGF,
4755                              const OMPExecutableDirective &D,
4756                              Address KmpTaskSharedsPtr, LValue TDBase,
4757                              const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4758                              QualType SharedsTy, QualType SharedsPtrTy,
4759                              const OMPTaskDataTy &Data,
4760                              ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4761   ASTContext &C = CGF.getContext();
4762   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4763   LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4764   OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4765                                  ? OMPD_taskloop
4766                                  : OMPD_task;
4767   const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4768   CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4769   LValue SrcBase;
4770   bool IsTargetTask =
4771       isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4772       isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4773   // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4774   // PointersArray and SizesArray. The original variables for these arrays are
4775   // not captured and we get their addresses explicitly.
4776   if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4777       (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4778     SrcBase = CGF.MakeAddrLValue(
4779         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4780             KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4781         SharedsTy);
4782   }
4783   FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4784   for (const PrivateDataTy &Pair : Privates) {
4785     const VarDecl *VD = Pair.second.PrivateCopy;
4786     const Expr *Init = VD->getAnyInitializer();
4787     if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4788                              !CGF.isTrivialInitializer(Init)))) {
4789       LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4790       if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4791         const VarDecl *OriginalVD = Pair.second.Original;
4792         // Check if the variable is the target-based BasePointersArray,
4793         // PointersArray or SizesArray.
4794         LValue SharedRefLValue;
4795         QualType Type = PrivateLValue.getType();
4796         const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4797         if (IsTargetTask && !SharedField) {
4798           assert(isa<ImplicitParamDecl>(OriginalVD) &&
4799                  isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
4800                  cast<CapturedDecl>(OriginalVD->getDeclContext())
4801                          ->getNumParams() == 0 &&
4802                  isa<TranslationUnitDecl>(
4803                      cast<CapturedDecl>(OriginalVD->getDeclContext())
4804                          ->getDeclContext()) &&
4805                  "Expected artificial target data variable.");
4806           SharedRefLValue =
4807               CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4808         } else {
4809           SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4810           SharedRefLValue = CGF.MakeAddrLValue(
4811               Address(SharedRefLValue.getPointer(CGF),
4812                       C.getDeclAlign(OriginalVD)),
4813               SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4814               SharedRefLValue.getTBAAInfo());
4815         }
4816         if (Type->isArrayType()) {
4817           // Initialize firstprivate array.
4818           if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4819             // Perform simple memcpy.
4820             CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4821           } else {
4822             // Initialize firstprivate array using element-by-element
4823             // initialization.
4824             CGF.EmitOMPAggregateAssign(
4825                 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
4826                 Type,
4827                 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4828                                                   Address SrcElement) {
4829                   // Clean up any temporaries needed by the initialization.
4830                   CodeGenFunction::OMPPrivateScope InitScope(CGF);
4831                   InitScope.addPrivate(
4832                       Elem, [SrcElement]() -> Address { return SrcElement; });
4833                   (void)InitScope.Privatize();
4834                   // Emit initialization for single element.
4835                   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4836                       CGF, &CapturesInfo);
4837                   CGF.EmitAnyExprToMem(Init, DestElement,
4838                                        Init->getType().getQualifiers(),
4839                                        /*IsInitializer=*/false);
4840                 });
4841           }
4842         } else {
4843           CodeGenFunction::OMPPrivateScope InitScope(CGF);
4844           InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
4845             return SharedRefLValue.getAddress(CGF);
4846           });
4847           (void)InitScope.Privatize();
4848           CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4849           CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4850                              /*capturedByInit=*/false);
4851         }
4852       } else {
4853         CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4854       }
4855     }
4856     ++FI;
4857   }
4858 }
4859 
4860 /// Check if duplication function is required for taskloops.
4861 static bool checkInitIsRequired(CodeGenFunction &CGF,
4862                                 ArrayRef<PrivateDataTy> Privates) {
4863   bool InitRequired = false;
4864   for (const PrivateDataTy &Pair : Privates) {
4865     const VarDecl *VD = Pair.second.PrivateCopy;
4866     const Expr *Init = VD->getAnyInitializer();
4867     InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4868                                     !CGF.isTrivialInitializer(Init));
4869     if (InitRequired)
4870       break;
4871   }
4872   return InitRequired;
4873 }
4874 
4875 
4876 /// Emit task_dup function (for initialization of
4877 /// private/firstprivate/lastprivate vars and last_iter flag)
4878 /// \code
4879 /// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4880 /// lastpriv) {
4881 /// // setup lastprivate flag
4882 ///    task_dst->last = lastpriv;
4883 /// // could be constructor calls here...
4884 /// }
4885 /// \endcode
4886 static llvm::Value *
4887 emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4888                     const OMPExecutableDirective &D,
4889                     QualType KmpTaskTWithPrivatesPtrQTy,
4890                     const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4891                     const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4892                     QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4893                     ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4894   ASTContext &C = CGM.getContext();
4895   FunctionArgList Args;
4896   ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4897                            KmpTaskTWithPrivatesPtrQTy,
4898                            ImplicitParamDecl::Other);
4899   ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4900                            KmpTaskTWithPrivatesPtrQTy,
4901                            ImplicitParamDecl::Other);
4902   ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4903                                 ImplicitParamDecl::Other);
4904   Args.push_back(&DstArg);
4905   Args.push_back(&SrcArg);
4906   Args.push_back(&LastprivArg);
4907   const auto &TaskDupFnInfo =
4908       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4909   llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4910   std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4911   auto *TaskDup = llvm::Function::Create(
4912       TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4913   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4914   TaskDup->setDoesNotRecurse();
4915   CodeGenFunction CGF(CGM);
4916   CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4917                     Loc);
4918 
4919   LValue TDBase = CGF.EmitLoadOfPointerLValue(
4920       CGF.GetAddrOfLocalVar(&DstArg),
4921       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4922   // task_dst->liter = lastpriv;
4923   if (WithLastIter) {
4924     auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4925     LValue Base = CGF.EmitLValueForField(
4926         TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4927     LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4928     llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4929         CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4930     CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4931   }
4932 
4933   // Emit initial values for private copies (if any).
4934   assert(!Privates.empty());
4935   Address KmpTaskSharedsPtr = Address::invalid();
4936   if (!Data.FirstprivateVars.empty()) {
4937     LValue TDBase = CGF.EmitLoadOfPointerLValue(
4938         CGF.GetAddrOfLocalVar(&SrcArg),
4939         KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4940     LValue Base = CGF.EmitLValueForField(
4941         TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4942     KmpTaskSharedsPtr = Address(
4943         CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4944                                  Base, *std::next(KmpTaskTQTyRD->field_begin(),
4945                                                   KmpTaskTShareds)),
4946                              Loc),
4947         CGF.getNaturalTypeAlignment(SharedsTy));
4948   }
4949   emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4950                    SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4951   CGF.FinishFunction();
4952   return TaskDup;
4953 }
4954 
4955 /// Checks if destructor function is required to be generated.
4956 /// \return true if cleanups are required, false otherwise.
4957 static bool
4958 checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4959   bool NeedsCleanup = false;
4960   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4961   const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4962   for (const FieldDecl *FD : PrivateRD->fields()) {
4963     NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4964     if (NeedsCleanup)
4965       break;
4966   }
4967   return NeedsCleanup;
4968 }
4969 
4970 CGOpenMPRuntime::TaskResultTy
4971 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4972                               const OMPExecutableDirective &D,
4973                               llvm::Function *TaskFunction, QualType SharedsTy,
4974                               Address Shareds, const OMPTaskDataTy &Data) {
4975   ASTContext &C = CGM.getContext();
4976   llvm::SmallVector<PrivateDataTy, 4> Privates;
4977   // Aggregate privates and sort them by the alignment.
4978   auto I = Data.PrivateCopies.begin();
4979   for (const Expr *E : Data.PrivateVars) {
4980     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4981     Privates.emplace_back(
4982         C.getDeclAlign(VD),
4983         PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4984                          /*PrivateElemInit=*/nullptr));
4985     ++I;
4986   }
4987   I = Data.FirstprivateCopies.begin();
4988   auto IElemInitRef = Data.FirstprivateInits.begin();
4989   for (const Expr *E : Data.FirstprivateVars) {
4990     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4991     Privates.emplace_back(
4992         C.getDeclAlign(VD),
4993         PrivateHelpersTy(
4994             VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4995             cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4996     ++I;
4997     ++IElemInitRef;
4998   }
4999   I = Data.LastprivateCopies.begin();
5000   for (const Expr *E : Data.LastprivateVars) {
5001     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5002     Privates.emplace_back(
5003         C.getDeclAlign(VD),
5004         PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5005                          /*PrivateElemInit=*/nullptr));
5006     ++I;
5007   }
5008   llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5009     return L.first > R.first;
5010   });
5011   QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5012   // Build type kmp_routine_entry_t (if not built yet).
5013   emitKmpRoutineEntryT(KmpInt32Ty);
5014   // Build type kmp_task_t (if not built yet).
5015   if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5016     if (SavedKmpTaskloopTQTy.isNull()) {
5017       SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5018           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5019     }
5020     KmpTaskTQTy = SavedKmpTaskloopTQTy;
5021   } else {
5022     assert((D.getDirectiveKind() == OMPD_task ||
5023             isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
5024             isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
5025            "Expected taskloop, task or target directive");
5026     if (SavedKmpTaskTQTy.isNull()) {
5027       SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5028           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5029     }
5030     KmpTaskTQTy = SavedKmpTaskTQTy;
5031   }
5032   const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5033   // Build particular struct kmp_task_t for the given task.
5034   const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5035       createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5036   QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5037   QualType KmpTaskTWithPrivatesPtrQTy =
5038       C.getPointerType(KmpTaskTWithPrivatesQTy);
5039   llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
5040   llvm::Type *KmpTaskTWithPrivatesPtrTy =
5041       KmpTaskTWithPrivatesTy->getPointerTo();
5042   llvm::Value *KmpTaskTWithPrivatesTySize =
5043       CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
5044   QualType SharedsPtrTy = C.getPointerType(SharedsTy);
5045 
5046   // Emit initial values for private copies (if any).
5047   llvm::Value *TaskPrivatesMap = nullptr;
5048   llvm::Type *TaskPrivatesMapTy =
5049       std::next(TaskFunction->arg_begin(), 3)->getType();
5050   if (!Privates.empty()) {
5051     auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5052     TaskPrivatesMap = emitTaskPrivateMappingFunction(
5053         CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
5054         FI->getType(), Privates);
5055     TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5056         TaskPrivatesMap, TaskPrivatesMapTy);
5057   } else {
5058     TaskPrivatesMap = llvm::ConstantPointerNull::get(
5059         cast<llvm::PointerType>(TaskPrivatesMapTy));
5060   }
5061   // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
5062   // kmp_task_t *tt);
5063   llvm::Function *TaskEntry = emitProxyTaskFunction(
5064       CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5065       KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5066       TaskPrivatesMap);
5067 
5068   // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
5069   // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
5070   // kmp_routine_entry_t *task_entry);
5071   // Task flags. Format is taken from
5072   // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h,
5073   // description of kmp_tasking_flags struct.
5074   enum {
5075     TiedFlag = 0x1,
5076     FinalFlag = 0x2,
5077     DestructorsFlag = 0x8,
5078     PriorityFlag = 0x20
5079   };
5080   unsigned Flags = Data.Tied ? TiedFlag : 0;
5081   bool NeedsCleanup = false;
5082   if (!Privates.empty()) {
5083     NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
5084     if (NeedsCleanup)
5085       Flags = Flags | DestructorsFlag;
5086   }
5087   if (Data.Priority.getInt())
5088     Flags = Flags | PriorityFlag;
5089   llvm::Value *TaskFlags =
5090       Data.Final.getPointer()
5091           ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
5092                                      CGF.Builder.getInt32(FinalFlag),
5093                                      CGF.Builder.getInt32(/*C=*/0))
5094           : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
5095   TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
5096   llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
5097   SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
5098       getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5099       SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5100           TaskEntry, KmpRoutineEntryPtrTy)};
5101   llvm::Value *NewTask;
5102   if (D.hasClausesOfKind<OMPNowaitClause>()) {
5103     // Check if we have any device clause associated with the directive.
5104     const Expr *Device = nullptr;
5105     if (auto *C = D.getSingleClause<OMPDeviceClause>())
5106       Device = C->getDevice();
5107     // Emit device ID if any otherwise use default value.
5108     llvm::Value *DeviceID;
5109     if (Device)
5110       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5111                                            CGF.Int64Ty, /*isSigned=*/true);
5112     else
5113       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
5114     AllocArgs.push_back(DeviceID);
5115     NewTask = CGF.EmitRuntimeCall(
5116       createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs);
5117   } else {
5118     NewTask = CGF.EmitRuntimeCall(
5119       createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
5120   }
5121   llvm::Value *NewTaskNewTaskTTy =
5122       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5123           NewTask, KmpTaskTWithPrivatesPtrTy);
5124   LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
5125                                                KmpTaskTWithPrivatesQTy);
5126   LValue TDBase =
5127       CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
5128   // Fill the data in the resulting kmp_task_t record.
5129   // Copy shareds if there are any.
5130   Address KmpTaskSharedsPtr = Address::invalid();
5131   if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
5132     KmpTaskSharedsPtr =
5133         Address(CGF.EmitLoadOfScalar(
5134                     CGF.EmitLValueForField(
5135                         TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5136                                            KmpTaskTShareds)),
5137                     Loc),
5138                 CGF.getNaturalTypeAlignment(SharedsTy));
5139     LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
5140     LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
5141     CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
5142   }
5143   // Emit initial values for private copies (if any).
5144   TaskResultTy Result;
5145   if (!Privates.empty()) {
5146     emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5147                      SharedsTy, SharedsPtrTy, Data, Privates,
5148                      /*ForDup=*/false);
5149     if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
5150         (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
5151       Result.TaskDupFn = emitTaskDupFunction(
5152           CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5153           KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5154           /*WithLastIter=*/!Data.LastprivateVars.empty());
5155     }
5156   }
5157   // Fields of union "kmp_cmplrdata_t" for destructors and priority.
5158   enum { Priority = 0, Destructors = 1 };
5159   // Provide pointer to function with destructors for privates.
5160   auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5161   const RecordDecl *KmpCmplrdataUD =
5162       (*FI)->getType()->getAsUnionType()->getDecl();
5163   if (NeedsCleanup) {
5164     llvm::Value *DestructorFn = emitDestructorsFunction(
5165         CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5166         KmpTaskTWithPrivatesQTy);
5167     LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
5168     LValue DestructorsLV = CGF.EmitLValueForField(
5169         Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5170     CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5171                               DestructorFn, KmpRoutineEntryPtrTy),
5172                           DestructorsLV);
5173   }
5174   // Set priority.
5175   if (Data.Priority.getInt()) {
5176     LValue Data2LV = CGF.EmitLValueForField(
5177         TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5178     LValue PriorityLV = CGF.EmitLValueForField(
5179         Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
5180     CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
5181   }
5182   Result.NewTask = NewTask;
5183   Result.TaskEntry = TaskEntry;
5184   Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
5185   Result.TDBase = TDBase;
5186   Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
5187   return Result;
5188 }
5189 
5190 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
5191                                    const OMPExecutableDirective &D,
5192                                    llvm::Function *TaskFunction,
5193                                    QualType SharedsTy, Address Shareds,
5194                                    const Expr *IfCond,
5195                                    const OMPTaskDataTy &Data) {
5196   if (!CGF.HaveInsertPoint())
5197     return;
5198 
5199   TaskResultTy Result =
5200       emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5201   llvm::Value *NewTask = Result.NewTask;
5202   llvm::Function *TaskEntry = Result.TaskEntry;
5203   llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
5204   LValue TDBase = Result.TDBase;
5205   const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
5206   ASTContext &C = CGM.getContext();
5207   // Process list of dependences.
5208   Address DependenciesArray = Address::invalid();
5209   unsigned NumDependencies = Data.Dependences.size();
5210   if (NumDependencies) {
5211     // Dependence kind for RTL.
5212     enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5213     enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5214     RecordDecl *KmpDependInfoRD;
5215     QualType FlagsTy =
5216         C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
5217     llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
5218     if (KmpDependInfoTy.isNull()) {
5219       KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
5220       KmpDependInfoRD->startDefinition();
5221       addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5222       addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5223       addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5224       KmpDependInfoRD->completeDefinition();
5225       KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5226     } else {
5227       KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5228     }
5229     // Define type kmp_depend_info[<Dependences.size()>];
5230     QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5231         KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5232         nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
5233     // kmp_depend_info[<Dependences.size()>] deps;
5234     DependenciesArray =
5235         CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5236     for (unsigned I = 0; I < NumDependencies; ++I) {
5237       const Expr *E = Data.Dependences[I].second;
5238       LValue Addr = CGF.EmitLValue(E);
5239       llvm::Value *Size;
5240       QualType Ty = E->getType();
5241       if (const auto *ASE =
5242               dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5243         LValue UpAddrLVal =
5244             CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
5245         llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
5246             UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
5247         llvm::Value *LowIntPtr =
5248             CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy);
5249         llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5250         Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5251       } else {
5252         Size = CGF.getTypeSize(Ty);
5253       }
5254       LValue Base = CGF.MakeAddrLValue(
5255           CGF.Builder.CreateConstArrayGEP(DependenciesArray, I),
5256           KmpDependInfoTy);
5257       // deps[i].base_addr = &<Dependences[i].second>;
5258       LValue BaseAddrLVal = CGF.EmitLValueForField(
5259           Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5260       CGF.EmitStoreOfScalar(
5261           CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy),
5262           BaseAddrLVal);
5263       // deps[i].len = sizeof(<Dependences[i].second>);
5264       LValue LenLVal = CGF.EmitLValueForField(
5265           Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5266       CGF.EmitStoreOfScalar(Size, LenLVal);
5267       // deps[i].flags = <Dependences[i].first>;
5268       RTLDependenceKindTy DepKind;
5269       switch (Data.Dependences[I].first) {
5270       case OMPC_DEPEND_in:
5271         DepKind = DepIn;
5272         break;
5273       // Out and InOut dependencies must use the same code.
5274       case OMPC_DEPEND_out:
5275       case OMPC_DEPEND_inout:
5276         DepKind = DepInOut;
5277         break;
5278       case OMPC_DEPEND_mutexinoutset:
5279         DepKind = DepMutexInOutSet;
5280         break;
5281       case OMPC_DEPEND_source:
5282       case OMPC_DEPEND_sink:
5283       case OMPC_DEPEND_unknown:
5284         llvm_unreachable("Unknown task dependence type");
5285       }
5286       LValue FlagsLVal = CGF.EmitLValueForField(
5287           Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5288       CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5289                             FlagsLVal);
5290     }
5291     DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5292         CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy);
5293   }
5294 
5295   // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5296   // libcall.
5297   // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5298   // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5299   // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5300   // list is not empty
5301   llvm::Value *ThreadID = getThreadID(CGF, Loc);
5302   llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5303   llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5304   llvm::Value *DepTaskArgs[7];
5305   if (NumDependencies) {
5306     DepTaskArgs[0] = UpLoc;
5307     DepTaskArgs[1] = ThreadID;
5308     DepTaskArgs[2] = NewTask;
5309     DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5310     DepTaskArgs[4] = DependenciesArray.getPointer();
5311     DepTaskArgs[5] = CGF.Builder.getInt32(0);
5312     DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5313   }
5314   auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5315                         &TaskArgs,
5316                         &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5317     if (!Data.Tied) {
5318       auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5319       LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5320       CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5321     }
5322     if (NumDependencies) {
5323       CGF.EmitRuntimeCall(
5324           createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5325     } else {
5326       CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5327                           TaskArgs);
5328     }
5329     // Check if parent region is untied and build return for untied task;
5330     if (auto *Region =
5331             dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5332       Region->emitUntiedSwitch(CGF);
5333   };
5334 
5335   llvm::Value *DepWaitTaskArgs[6];
5336   if (NumDependencies) {
5337     DepWaitTaskArgs[0] = UpLoc;
5338     DepWaitTaskArgs[1] = ThreadID;
5339     DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5340     DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5341     DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5342     DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5343   }
5344   auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5345                         NumDependencies, &DepWaitTaskArgs,
5346                         Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5347     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5348     CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5349     // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5350     // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5351     // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5352     // is specified.
5353     if (NumDependencies)
5354       CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5355                           DepWaitTaskArgs);
5356     // Call proxy_task_entry(gtid, new_task);
5357     auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5358                       Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5359       Action.Enter(CGF);
5360       llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5361       CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5362                                                           OutlinedFnArgs);
5363     };
5364 
5365     // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5366     // kmp_task_t *new_task);
5367     // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5368     // kmp_task_t *new_task);
5369     RegionCodeGenTy RCG(CodeGen);
5370     CommonActionTy Action(
5371         RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5372         RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5373     RCG.setAction(Action);
5374     RCG(CGF);
5375   };
5376 
5377   if (IfCond) {
5378     emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5379   } else {
5380     RegionCodeGenTy ThenRCG(ThenCodeGen);
5381     ThenRCG(CGF);
5382   }
5383 }
5384 
5385 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5386                                        const OMPLoopDirective &D,
5387                                        llvm::Function *TaskFunction,
5388                                        QualType SharedsTy, Address Shareds,
5389                                        const Expr *IfCond,
5390                                        const OMPTaskDataTy &Data) {
5391   if (!CGF.HaveInsertPoint())
5392     return;
5393   TaskResultTy Result =
5394       emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5395   // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5396   // libcall.
5397   // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5398   // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5399   // sched, kmp_uint64 grainsize, void *task_dup);
5400   llvm::Value *ThreadID = getThreadID(CGF, Loc);
5401   llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5402   llvm::Value *IfVal;
5403   if (IfCond) {
5404     IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5405                                       /*isSigned=*/true);
5406   } else {
5407     IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5408   }
5409 
5410   LValue LBLVal = CGF.EmitLValueForField(
5411       Result.TDBase,
5412       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5413   const auto *LBVar =
5414       cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5415   CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF),
5416                        LBLVal.getQuals(),
5417                        /*IsInitializer=*/true);
5418   LValue UBLVal = CGF.EmitLValueForField(
5419       Result.TDBase,
5420       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5421   const auto *UBVar =
5422       cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5423   CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF),
5424                        UBLVal.getQuals(),
5425                        /*IsInitializer=*/true);
5426   LValue StLVal = CGF.EmitLValueForField(
5427       Result.TDBase,
5428       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5429   const auto *StVar =
5430       cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5431   CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF),
5432                        StLVal.getQuals(),
5433                        /*IsInitializer=*/true);
5434   // Store reductions address.
5435   LValue RedLVal = CGF.EmitLValueForField(
5436       Result.TDBase,
5437       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5438   if (Data.Reductions) {
5439     CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5440   } else {
5441     CGF.EmitNullInitialization(RedLVal.getAddress(CGF),
5442                                CGF.getContext().VoidPtrTy);
5443   }
5444   enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5445   llvm::Value *TaskArgs[] = {
5446       UpLoc,
5447       ThreadID,
5448       Result.NewTask,
5449       IfVal,
5450       LBLVal.getPointer(CGF),
5451       UBLVal.getPointer(CGF),
5452       CGF.EmitLoadOfScalar(StLVal, Loc),
5453       llvm::ConstantInt::getSigned(
5454           CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
5455       llvm::ConstantInt::getSigned(
5456           CGF.IntTy, Data.Schedule.getPointer()
5457                          ? Data.Schedule.getInt() ? NumTasks : Grainsize
5458                          : NoSchedule),
5459       Data.Schedule.getPointer()
5460           ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5461                                       /*isSigned=*/false)
5462           : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5463       Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5464                              Result.TaskDupFn, CGF.VoidPtrTy)
5465                        : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5466   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5467 }
5468 
5469 /// Emit reduction operation for each element of array (required for
5470 /// array sections) LHS op = RHS.
5471 /// \param Type Type of array.
5472 /// \param LHSVar Variable on the left side of the reduction operation
5473 /// (references element of array in original variable).
5474 /// \param RHSVar Variable on the right side of the reduction operation
5475 /// (references element of array in original variable).
5476 /// \param RedOpGen Generator of reduction operation with use of LHSVar and
5477 /// RHSVar.
5478 static void EmitOMPAggregateReduction(
5479     CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5480     const VarDecl *RHSVar,
5481     const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5482                                   const Expr *, const Expr *)> &RedOpGen,
5483     const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5484     const Expr *UpExpr = nullptr) {
5485   // Perform element-by-element initialization.
5486   QualType ElementTy;
5487   Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5488   Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5489 
5490   // Drill down to the base element type on both arrays.
5491   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5492   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5493 
5494   llvm::Value *RHSBegin = RHSAddr.getPointer();
5495   llvm::Value *LHSBegin = LHSAddr.getPointer();
5496   // Cast from pointer to array type to pointer to single element.
5497   llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5498   // The basic structure here is a while-do loop.
5499   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5500   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5501   llvm::Value *IsEmpty =
5502       CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5503   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5504 
5505   // Enter the loop body, making that address the current address.
5506   llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5507   CGF.EmitBlock(BodyBB);
5508 
5509   CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5510 
5511   llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5512       RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5513   RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5514   Address RHSElementCurrent =
5515       Address(RHSElementPHI,
5516               RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5517 
5518   llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5519       LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5520   LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5521   Address LHSElementCurrent =
5522       Address(LHSElementPHI,
5523               LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5524 
5525   // Emit copy.
5526   CodeGenFunction::OMPPrivateScope Scope(CGF);
5527   Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5528   Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5529   Scope.Privatize();
5530   RedOpGen(CGF, XExpr, EExpr, UpExpr);
5531   Scope.ForceCleanup();
5532 
5533   // Shift the address forward by one element.
5534   llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5535       LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5536   llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5537       RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5538   // Check whether we've reached the end.
5539   llvm::Value *Done =
5540       CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5541   CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5542   LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5543   RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5544 
5545   // Done.
5546   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5547 }
5548 
5549 /// Emit reduction combiner. If the combiner is a simple expression emit it as
5550 /// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5551 /// UDR combiner function.
5552 static void emitReductionCombiner(CodeGenFunction &CGF,
5553                                   const Expr *ReductionOp) {
5554   if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5555     if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5556       if (const auto *DRE =
5557               dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5558         if (const auto *DRD =
5559                 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5560           std::pair<llvm::Function *, llvm::Function *> Reduction =
5561               CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5562           RValue Func = RValue::get(Reduction.first);
5563           CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5564           CGF.EmitIgnoredExpr(ReductionOp);
5565           return;
5566         }
5567   CGF.EmitIgnoredExpr(ReductionOp);
5568 }
5569 
5570 llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5571     SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
5572     ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5573     ArrayRef<const Expr *> ReductionOps) {
5574   ASTContext &C = CGM.getContext();
5575 
5576   // void reduction_func(void *LHSArg, void *RHSArg);
5577   FunctionArgList Args;
5578   ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5579                            ImplicitParamDecl::Other);
5580   ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5581                            ImplicitParamDecl::Other);
5582   Args.push_back(&LHSArg);
5583   Args.push_back(&RHSArg);
5584   const auto &CGFI =
5585       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5586   std::string Name = getName({"omp", "reduction", "reduction_func"});
5587   auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5588                                     llvm::GlobalValue::InternalLinkage, Name,
5589                                     &CGM.getModule());
5590   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5591   Fn->setDoesNotRecurse();
5592   CodeGenFunction CGF(CGM);
5593   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5594 
5595   // Dst = (void*[n])(LHSArg);
5596   // Src = (void*[n])(RHSArg);
5597   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5598       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5599       ArgsType), CGF.getPointerAlign());
5600   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5601       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5602       ArgsType), CGF.getPointerAlign());
5603 
5604   //  ...
5605   //  *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5606   //  ...
5607   CodeGenFunction::OMPPrivateScope Scope(CGF);
5608   auto IPriv = Privates.begin();
5609   unsigned Idx = 0;
5610   for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5611     const auto *RHSVar =
5612         cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5613     Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5614       return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5615     });
5616     const auto *LHSVar =
5617         cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5618     Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5619       return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5620     });
5621     QualType PrivTy = (*IPriv)->getType();
5622     if (PrivTy->isVariablyModifiedType()) {
5623       // Get array size and emit VLA type.
5624       ++Idx;
5625       Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx);
5626       llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5627       const VariableArrayType *VLA =
5628           CGF.getContext().getAsVariableArrayType(PrivTy);
5629       const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5630       CodeGenFunction::OpaqueValueMapping OpaqueMap(
5631           CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5632       CGF.EmitVariablyModifiedType(PrivTy);
5633     }
5634   }
5635   Scope.Privatize();
5636   IPriv = Privates.begin();
5637   auto ILHS = LHSExprs.begin();
5638   auto IRHS = RHSExprs.begin();
5639   for (const Expr *E : ReductionOps) {
5640     if ((*IPriv)->getType()->isArrayType()) {
5641       // Emit reduction for array section.
5642       const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5643       const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5644       EmitOMPAggregateReduction(
5645           CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5646           [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5647             emitReductionCombiner(CGF, E);
5648           });
5649     } else {
5650       // Emit reduction for array subscript or single variable.
5651       emitReductionCombiner(CGF, E);
5652     }
5653     ++IPriv;
5654     ++ILHS;
5655     ++IRHS;
5656   }
5657   Scope.ForceCleanup();
5658   CGF.FinishFunction();
5659   return Fn;
5660 }
5661 
5662 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5663                                                   const Expr *ReductionOp,
5664                                                   const Expr *PrivateRef,
5665                                                   const DeclRefExpr *LHS,
5666                                                   const DeclRefExpr *RHS) {
5667   if (PrivateRef->getType()->isArrayType()) {
5668     // Emit reduction for array section.
5669     const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5670     const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5671     EmitOMPAggregateReduction(
5672         CGF, PrivateRef->getType(), LHSVar, RHSVar,
5673         [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5674           emitReductionCombiner(CGF, ReductionOp);
5675         });
5676   } else {
5677     // Emit reduction for array subscript or single variable.
5678     emitReductionCombiner(CGF, ReductionOp);
5679   }
5680 }
5681 
5682 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5683                                     ArrayRef<const Expr *> Privates,
5684                                     ArrayRef<const Expr *> LHSExprs,
5685                                     ArrayRef<const Expr *> RHSExprs,
5686                                     ArrayRef<const Expr *> ReductionOps,
5687                                     ReductionOptionsTy Options) {
5688   if (!CGF.HaveInsertPoint())
5689     return;
5690 
5691   bool WithNowait = Options.WithNowait;
5692   bool SimpleReduction = Options.SimpleReduction;
5693 
5694   // Next code should be emitted for reduction:
5695   //
5696   // static kmp_critical_name lock = { 0 };
5697   //
5698   // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5699   //  *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5700   //  ...
5701   //  *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5702   //  *(Type<n>-1*)rhs[<n>-1]);
5703   // }
5704   //
5705   // ...
5706   // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5707   // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5708   // RedList, reduce_func, &<lock>)) {
5709   // case 1:
5710   //  ...
5711   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5712   //  ...
5713   // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5714   // break;
5715   // case 2:
5716   //  ...
5717   //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5718   //  ...
5719   // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5720   // break;
5721   // default:;
5722   // }
5723   //
5724   // if SimpleReduction is true, only the next code is generated:
5725   //  ...
5726   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5727   //  ...
5728 
5729   ASTContext &C = CGM.getContext();
5730 
5731   if (SimpleReduction) {
5732     CodeGenFunction::RunCleanupsScope Scope(CGF);
5733     auto IPriv = Privates.begin();
5734     auto ILHS = LHSExprs.begin();
5735     auto IRHS = RHSExprs.begin();
5736     for (const Expr *E : ReductionOps) {
5737       emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5738                                   cast<DeclRefExpr>(*IRHS));
5739       ++IPriv;
5740       ++ILHS;
5741       ++IRHS;
5742     }
5743     return;
5744   }
5745 
5746   // 1. Build a list of reduction variables.
5747   // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5748   auto Size = RHSExprs.size();
5749   for (const Expr *E : Privates) {
5750     if (E->getType()->isVariablyModifiedType())
5751       // Reserve place for array size.
5752       ++Size;
5753   }
5754   llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5755   QualType ReductionArrayTy =
5756       C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
5757                              /*IndexTypeQuals=*/0);
5758   Address ReductionList =
5759       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5760   auto IPriv = Privates.begin();
5761   unsigned Idx = 0;
5762   for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5763     Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5764     CGF.Builder.CreateStore(
5765         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5766             CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy),
5767         Elem);
5768     if ((*IPriv)->getType()->isVariablyModifiedType()) {
5769       // Store array size.
5770       ++Idx;
5771       Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx);
5772       llvm::Value *Size = CGF.Builder.CreateIntCast(
5773           CGF.getVLASize(
5774                  CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5775               .NumElts,
5776           CGF.SizeTy, /*isSigned=*/false);
5777       CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5778                               Elem);
5779     }
5780   }
5781 
5782   // 2. Emit reduce_func().
5783   llvm::Function *ReductionFn = emitReductionFunction(
5784       Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5785       LHSExprs, RHSExprs, ReductionOps);
5786 
5787   // 3. Create static kmp_critical_name lock = { 0 };
5788   std::string Name = getName({"reduction"});
5789   llvm::Value *Lock = getCriticalRegionLock(Name);
5790 
5791   // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5792   // RedList, reduce_func, &<lock>);
5793   llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5794   llvm::Value *ThreadId = getThreadID(CGF, Loc);
5795   llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5796   llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5797       ReductionList.getPointer(), CGF.VoidPtrTy);
5798   llvm::Value *Args[] = {
5799       IdentTLoc,                             // ident_t *<loc>
5800       ThreadId,                              // i32 <gtid>
5801       CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5802       ReductionArrayTySize,                  // size_type sizeof(RedList)
5803       RL,                                    // void *RedList
5804       ReductionFn, // void (*) (void *, void *) <reduce_func>
5805       Lock         // kmp_critical_name *&<lock>
5806   };
5807   llvm::Value *Res = CGF.EmitRuntimeCall(
5808       createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5809                                        : OMPRTL__kmpc_reduce),
5810       Args);
5811 
5812   // 5. Build switch(res)
5813   llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5814   llvm::SwitchInst *SwInst =
5815       CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5816 
5817   // 6. Build case 1:
5818   //  ...
5819   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5820   //  ...
5821   // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5822   // break;
5823   llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5824   SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5825   CGF.EmitBlock(Case1BB);
5826 
5827   // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5828   llvm::Value *EndArgs[] = {
5829       IdentTLoc, // ident_t *<loc>
5830       ThreadId,  // i32 <gtid>
5831       Lock       // kmp_critical_name *&<lock>
5832   };
5833   auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5834                        CodeGenFunction &CGF, PrePostActionTy &Action) {
5835     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5836     auto IPriv = Privates.begin();
5837     auto ILHS = LHSExprs.begin();
5838     auto IRHS = RHSExprs.begin();
5839     for (const Expr *E : ReductionOps) {
5840       RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5841                                      cast<DeclRefExpr>(*IRHS));
5842       ++IPriv;
5843       ++ILHS;
5844       ++IRHS;
5845     }
5846   };
5847   RegionCodeGenTy RCG(CodeGen);
5848   CommonActionTy Action(
5849       nullptr, llvm::None,
5850       createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5851                                        : OMPRTL__kmpc_end_reduce),
5852       EndArgs);
5853   RCG.setAction(Action);
5854   RCG(CGF);
5855 
5856   CGF.EmitBranch(DefaultBB);
5857 
5858   // 7. Build case 2:
5859   //  ...
5860   //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5861   //  ...
5862   // break;
5863   llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5864   SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5865   CGF.EmitBlock(Case2BB);
5866 
5867   auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5868                              CodeGenFunction &CGF, PrePostActionTy &Action) {
5869     auto ILHS = LHSExprs.begin();
5870     auto IRHS = RHSExprs.begin();
5871     auto IPriv = Privates.begin();
5872     for (const Expr *E : ReductionOps) {
5873       const Expr *XExpr = nullptr;
5874       const Expr *EExpr = nullptr;
5875       const Expr *UpExpr = nullptr;
5876       BinaryOperatorKind BO = BO_Comma;
5877       if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5878         if (BO->getOpcode() == BO_Assign) {
5879           XExpr = BO->getLHS();
5880           UpExpr = BO->getRHS();
5881         }
5882       }
5883       // Try to emit update expression as a simple atomic.
5884       const Expr *RHSExpr = UpExpr;
5885       if (RHSExpr) {
5886         // Analyze RHS part of the whole expression.
5887         if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5888                 RHSExpr->IgnoreParenImpCasts())) {
5889           // If this is a conditional operator, analyze its condition for
5890           // min/max reduction operator.
5891           RHSExpr = ACO->getCond();
5892         }
5893         if (const auto *BORHS =
5894                 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5895           EExpr = BORHS->getRHS();
5896           BO = BORHS->getOpcode();
5897         }
5898       }
5899       if (XExpr) {
5900         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5901         auto &&AtomicRedGen = [BO, VD,
5902                                Loc](CodeGenFunction &CGF, const Expr *XExpr,
5903                                     const Expr *EExpr, const Expr *UpExpr) {
5904           LValue X = CGF.EmitLValue(XExpr);
5905           RValue E;
5906           if (EExpr)
5907             E = CGF.EmitAnyExpr(EExpr);
5908           CGF.EmitOMPAtomicSimpleUpdateExpr(
5909               X, E, BO, /*IsXLHSInRHSPart=*/true,
5910               llvm::AtomicOrdering::Monotonic, Loc,
5911               [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5912                 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5913                 PrivateScope.addPrivate(
5914                     VD, [&CGF, VD, XRValue, Loc]() {
5915                       Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5916                       CGF.emitOMPSimpleStore(
5917                           CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5918                           VD->getType().getNonReferenceType(), Loc);
5919                       return LHSTemp;
5920                     });
5921                 (void)PrivateScope.Privatize();
5922                 return CGF.EmitAnyExpr(UpExpr);
5923               });
5924         };
5925         if ((*IPriv)->getType()->isArrayType()) {
5926           // Emit atomic reduction for array section.
5927           const auto *RHSVar =
5928               cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5929           EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5930                                     AtomicRedGen, XExpr, EExpr, UpExpr);
5931         } else {
5932           // Emit atomic reduction for array subscript or single variable.
5933           AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5934         }
5935       } else {
5936         // Emit as a critical region.
5937         auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5938                                            const Expr *, const Expr *) {
5939           CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5940           std::string Name = RT.getName({"atomic_reduction"});
5941           RT.emitCriticalRegion(
5942               CGF, Name,
5943               [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5944                 Action.Enter(CGF);
5945                 emitReductionCombiner(CGF, E);
5946               },
5947               Loc);
5948         };
5949         if ((*IPriv)->getType()->isArrayType()) {
5950           const auto *LHSVar =
5951               cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5952           const auto *RHSVar =
5953               cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5954           EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5955                                     CritRedGen);
5956         } else {
5957           CritRedGen(CGF, nullptr, nullptr, nullptr);
5958         }
5959       }
5960       ++ILHS;
5961       ++IRHS;
5962       ++IPriv;
5963     }
5964   };
5965   RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5966   if (!WithNowait) {
5967     // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5968     llvm::Value *EndArgs[] = {
5969         IdentTLoc, // ident_t *<loc>
5970         ThreadId,  // i32 <gtid>
5971         Lock       // kmp_critical_name *&<lock>
5972     };
5973     CommonActionTy Action(nullptr, llvm::None,
5974                           createRuntimeFunction(OMPRTL__kmpc_end_reduce),
5975                           EndArgs);
5976     AtomicRCG.setAction(Action);
5977     AtomicRCG(CGF);
5978   } else {
5979     AtomicRCG(CGF);
5980   }
5981 
5982   CGF.EmitBranch(DefaultBB);
5983   CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5984 }
5985 
5986 /// Generates unique name for artificial threadprivate variables.
5987 /// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5988 static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5989                                       const Expr *Ref) {
5990   SmallString<256> Buffer;
5991   llvm::raw_svector_ostream Out(Buffer);
5992   const clang::DeclRefExpr *DE;
5993   const VarDecl *D = ::getBaseDecl(Ref, DE);
5994   if (!D)
5995     D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5996   D = D->getCanonicalDecl();
5997   std::string Name = CGM.getOpenMPRuntime().getName(
5998       {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5999   Out << Prefix << Name << "_"
6000       << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
6001   return Out.str();
6002 }
6003 
6004 /// Emits reduction initializer function:
6005 /// \code
6006 /// void @.red_init(void* %arg) {
6007 /// %0 = bitcast void* %arg to <type>*
6008 /// store <type> <init>, <type>* %0
6009 /// ret void
6010 /// }
6011 /// \endcode
6012 static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
6013                                            SourceLocation Loc,
6014                                            ReductionCodeGen &RCG, unsigned N) {
6015   ASTContext &C = CGM.getContext();
6016   FunctionArgList Args;
6017   ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6018                           ImplicitParamDecl::Other);
6019   Args.emplace_back(&Param);
6020   const auto &FnInfo =
6021       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6022   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6023   std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});
6024   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6025                                     Name, &CGM.getModule());
6026   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6027   Fn->setDoesNotRecurse();
6028   CodeGenFunction CGF(CGM);
6029   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6030   Address PrivateAddr = CGF.EmitLoadOfPointer(
6031       CGF.GetAddrOfLocalVar(&Param),
6032       C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6033   llvm::Value *Size = nullptr;
6034   // If the size of the reduction item is non-constant, load it from global
6035   // threadprivate variable.
6036   if (RCG.getSizes(N).second) {
6037     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6038         CGF, CGM.getContext().getSizeType(),
6039         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6040     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6041                                 CGM.getContext().getSizeType(), Loc);
6042   }
6043   RCG.emitAggregateType(CGF, N, Size);
6044   LValue SharedLVal;
6045   // If initializer uses initializer from declare reduction construct, emit a
6046   // pointer to the address of the original reduction item (reuired by reduction
6047   // initializer)
6048   if (RCG.usesReductionInitializer(N)) {
6049     Address SharedAddr =
6050         CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6051             CGF, CGM.getContext().VoidPtrTy,
6052             generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6053     SharedAddr = CGF.EmitLoadOfPointer(
6054         SharedAddr,
6055         CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
6056     SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
6057   } else {
6058     SharedLVal = CGF.MakeNaturalAlignAddrLValue(
6059         llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
6060         CGM.getContext().VoidPtrTy);
6061   }
6062   // Emit the initializer:
6063   // %0 = bitcast void* %arg to <type>*
6064   // store <type> <init>, <type>* %0
6065   RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal,
6066                          [](CodeGenFunction &) { return false; });
6067   CGF.FinishFunction();
6068   return Fn;
6069 }
6070 
6071 /// Emits reduction combiner function:
6072 /// \code
6073 /// void @.red_comb(void* %arg0, void* %arg1) {
6074 /// %lhs = bitcast void* %arg0 to <type>*
6075 /// %rhs = bitcast void* %arg1 to <type>*
6076 /// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
6077 /// store <type> %2, <type>* %lhs
6078 /// ret void
6079 /// }
6080 /// \endcode
6081 static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
6082                                            SourceLocation Loc,
6083                                            ReductionCodeGen &RCG, unsigned N,
6084                                            const Expr *ReductionOp,
6085                                            const Expr *LHS, const Expr *RHS,
6086                                            const Expr *PrivateRef) {
6087   ASTContext &C = CGM.getContext();
6088   const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
6089   const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
6090   FunctionArgList Args;
6091   ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
6092                                C.VoidPtrTy, ImplicitParamDecl::Other);
6093   ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6094                             ImplicitParamDecl::Other);
6095   Args.emplace_back(&ParamInOut);
6096   Args.emplace_back(&ParamIn);
6097   const auto &FnInfo =
6098       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6099   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6100   std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});
6101   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6102                                     Name, &CGM.getModule());
6103   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6104   Fn->setDoesNotRecurse();
6105   CodeGenFunction CGF(CGM);
6106   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6107   llvm::Value *Size = nullptr;
6108   // If the size of the reduction item is non-constant, load it from global
6109   // threadprivate variable.
6110   if (RCG.getSizes(N).second) {
6111     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6112         CGF, CGM.getContext().getSizeType(),
6113         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6114     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6115                                 CGM.getContext().getSizeType(), Loc);
6116   }
6117   RCG.emitAggregateType(CGF, N, Size);
6118   // Remap lhs and rhs variables to the addresses of the function arguments.
6119   // %lhs = bitcast void* %arg0 to <type>*
6120   // %rhs = bitcast void* %arg1 to <type>*
6121   CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6122   PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
6123     // Pull out the pointer to the variable.
6124     Address PtrAddr = CGF.EmitLoadOfPointer(
6125         CGF.GetAddrOfLocalVar(&ParamInOut),
6126         C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6127     return CGF.Builder.CreateElementBitCast(
6128         PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
6129   });
6130   PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
6131     // Pull out the pointer to the variable.
6132     Address PtrAddr = CGF.EmitLoadOfPointer(
6133         CGF.GetAddrOfLocalVar(&ParamIn),
6134         C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6135     return CGF.Builder.CreateElementBitCast(
6136         PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
6137   });
6138   PrivateScope.Privatize();
6139   // Emit the combiner body:
6140   // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6141   // store <type> %2, <type>* %lhs
6142   CGM.getOpenMPRuntime().emitSingleReductionCombiner(
6143       CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6144       cast<DeclRefExpr>(RHS));
6145   CGF.FinishFunction();
6146   return Fn;
6147 }
6148 
6149 /// Emits reduction finalizer function:
6150 /// \code
6151 /// void @.red_fini(void* %arg) {
6152 /// %0 = bitcast void* %arg to <type>*
6153 /// <destroy>(<type>* %0)
6154 /// ret void
6155 /// }
6156 /// \endcode
6157 static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6158                                            SourceLocation Loc,
6159                                            ReductionCodeGen &RCG, unsigned N) {
6160   if (!RCG.needCleanups(N))
6161     return nullptr;
6162   ASTContext &C = CGM.getContext();
6163   FunctionArgList Args;
6164   ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
6165                           ImplicitParamDecl::Other);
6166   Args.emplace_back(&Param);
6167   const auto &FnInfo =
6168       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
6169   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
6170   std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});
6171   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
6172                                     Name, &CGM.getModule());
6173   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
6174   Fn->setDoesNotRecurse();
6175   CodeGenFunction CGF(CGM);
6176   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
6177   Address PrivateAddr = CGF.EmitLoadOfPointer(
6178       CGF.GetAddrOfLocalVar(&Param),
6179       C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
6180   llvm::Value *Size = nullptr;
6181   // If the size of the reduction item is non-constant, load it from global
6182   // threadprivate variable.
6183   if (RCG.getSizes(N).second) {
6184     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6185         CGF, CGM.getContext().getSizeType(),
6186         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6187     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
6188                                 CGM.getContext().getSizeType(), Loc);
6189   }
6190   RCG.emitAggregateType(CGF, N, Size);
6191   // Emit the finalizer body:
6192   // <destroy>(<type>* %0)
6193   RCG.emitCleanups(CGF, N, PrivateAddr);
6194   CGF.FinishFunction(Loc);
6195   return Fn;
6196 }
6197 
6198 llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
6199     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
6200     ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6201   if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6202     return nullptr;
6203 
6204   // Build typedef struct:
6205   // kmp_task_red_input {
6206   //   void *reduce_shar; // shared reduction item
6207   //   size_t reduce_size; // size of data item
6208   //   void *reduce_init; // data initialization routine
6209   //   void *reduce_fini; // data finalization routine
6210   //   void *reduce_comb; // data combiner routine
6211   //   kmp_task_red_flags_t flags; // flags for additional info from compiler
6212   // } kmp_task_red_input_t;
6213   ASTContext &C = CGM.getContext();
6214   RecordDecl *RD = C.buildImplicitRecord("kmp_task_red_input_t");
6215   RD->startDefinition();
6216   const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6217   const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
6218   const FieldDecl *InitFD  = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6219   const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6220   const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
6221   const FieldDecl *FlagsFD = addFieldToRecordDecl(
6222       C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6223   RD->completeDefinition();
6224   QualType RDType = C.getRecordType(RD);
6225   unsigned Size = Data.ReductionVars.size();
6226   llvm::APInt ArraySize(/*numBits=*/64, Size);
6227   QualType ArrayRDType = C.getConstantArrayType(
6228       RDType, ArraySize, nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
6229   // kmp_task_red_input_t .rd_input.[Size];
6230   Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
6231   ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies,
6232                        Data.ReductionOps);
6233   for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6234     // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6235     llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
6236                            llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
6237     llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6238         TaskRedInput.getPointer(), Idxs,
6239         /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6240         ".rd_input.gep.");
6241     LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
6242     // ElemLVal.reduce_shar = &Shareds[Cnt];
6243     LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
6244     RCG.emitSharedLValue(CGF, Cnt);
6245     llvm::Value *CastedShared =
6246         CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF));
6247     CGF.EmitStoreOfScalar(CastedShared, SharedLVal);
6248     RCG.emitAggregateType(CGF, Cnt);
6249     llvm::Value *SizeValInChars;
6250     llvm::Value *SizeVal;
6251     std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
6252     // We use delayed creation/initialization for VLAs, array sections and
6253     // custom reduction initializations. It is required because runtime does not
6254     // provide the way to pass the sizes of VLAs/array sections to
6255     // initializer/combiner/finalizer functions and does not pass the pointer to
6256     // original reduction item to the initializer. Instead threadprivate global
6257     // variables are used to store these values and use them in the functions.
6258     bool DelayedCreation = !!SizeVal;
6259     SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
6260                                                /*isSigned=*/false);
6261     LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
6262     CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
6263     // ElemLVal.reduce_init = init;
6264     LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
6265     llvm::Value *InitAddr =
6266         CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt));
6267     CGF.EmitStoreOfScalar(InitAddr, InitLVal);
6268     DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
6269     // ElemLVal.reduce_fini = fini;
6270     LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
6271     llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
6272     llvm::Value *FiniAddr = Fini
6273                                 ? CGF.EmitCastToVoidPtr(Fini)
6274                                 : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
6275     CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
6276     // ElemLVal.reduce_comb = comb;
6277     LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
6278     llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction(
6279         CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
6280         RHSExprs[Cnt], Data.ReductionCopies[Cnt]));
6281     CGF.EmitStoreOfScalar(CombAddr, CombLVal);
6282     // ElemLVal.flags = 0;
6283     LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
6284     if (DelayedCreation) {
6285       CGF.EmitStoreOfScalar(
6286           llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true),
6287           FlagsLVal);
6288     } else
6289       CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF),
6290                                  FlagsLVal.getType());
6291   }
6292   // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void
6293   // *data);
6294   llvm::Value *Args[] = {
6295       CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6296                                 /*isSigned=*/true),
6297       llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6298       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
6299                                                       CGM.VoidPtrTy)};
6300   return CGF.EmitRuntimeCall(
6301       createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args);
6302 }
6303 
6304 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6305                                               SourceLocation Loc,
6306                                               ReductionCodeGen &RCG,
6307                                               unsigned N) {
6308   auto Sizes = RCG.getSizes(N);
6309   // Emit threadprivate global variable if the type is non-constant
6310   // (Sizes.second = nullptr).
6311   if (Sizes.second) {
6312     llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
6313                                                      /*isSigned=*/false);
6314     Address SizeAddr = getAddrOfArtificialThreadPrivate(
6315         CGF, CGM.getContext().getSizeType(),
6316         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6317     CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
6318   }
6319   // Store address of the original reduction item if custom initializer is used.
6320   if (RCG.usesReductionInitializer(N)) {
6321     Address SharedAddr = getAddrOfArtificialThreadPrivate(
6322         CGF, CGM.getContext().VoidPtrTy,
6323         generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6324     CGF.Builder.CreateStore(
6325         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6326             RCG.getSharedLValue(N).getPointer(CGF), CGM.VoidPtrTy),
6327         SharedAddr, /*IsVolatile=*/false);
6328   }
6329 }
6330 
6331 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6332                                               SourceLocation Loc,
6333                                               llvm::Value *ReductionsPtr,
6334                                               LValue SharedLVal) {
6335   // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6336   // *d);
6337   llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc),
6338                                                    CGM.IntTy,
6339                                                    /*isSigned=*/true),
6340                          ReductionsPtr,
6341                          CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6342                              SharedLVal.getPointer(CGF), CGM.VoidPtrTy)};
6343   return Address(
6344       CGF.EmitRuntimeCall(
6345           createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args),
6346       SharedLVal.getAlignment());
6347 }
6348 
6349 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
6350                                        SourceLocation Loc) {
6351   if (!CGF.HaveInsertPoint())
6352     return;
6353   // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6354   // global_tid);
6355   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
6356   // Ignore return result until untied tasks are supported.
6357   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
6358   if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
6359     Region->emitUntiedSwitch(CGF);
6360 }
6361 
6362 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6363                                            OpenMPDirectiveKind InnerKind,
6364                                            const RegionCodeGenTy &CodeGen,
6365                                            bool HasCancel) {
6366   if (!CGF.HaveInsertPoint())
6367     return;
6368   InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6369   CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6370 }
6371 
6372 namespace {
6373 enum RTCancelKind {
6374   CancelNoreq = 0,
6375   CancelParallel = 1,
6376   CancelLoop = 2,
6377   CancelSections = 3,
6378   CancelTaskgroup = 4
6379 };
6380 } // anonymous namespace
6381 
6382 static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6383   RTCancelKind CancelKind = CancelNoreq;
6384   if (CancelRegion == OMPD_parallel)
6385     CancelKind = CancelParallel;
6386   else if (CancelRegion == OMPD_for)
6387     CancelKind = CancelLoop;
6388   else if (CancelRegion == OMPD_sections)
6389     CancelKind = CancelSections;
6390   else {
6391     assert(CancelRegion == OMPD_taskgroup);
6392     CancelKind = CancelTaskgroup;
6393   }
6394   return CancelKind;
6395 }
6396 
6397 void CGOpenMPRuntime::emitCancellationPointCall(
6398     CodeGenFunction &CGF, SourceLocation Loc,
6399     OpenMPDirectiveKind CancelRegion) {
6400   if (!CGF.HaveInsertPoint())
6401     return;
6402   // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6403   // global_tid, kmp_int32 cncl_kind);
6404   if (auto *OMPRegionInfo =
6405           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6406     // For 'cancellation point taskgroup', the task region info may not have a
6407     // cancel. This may instead happen in another adjacent task.
6408     if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6409       llvm::Value *Args[] = {
6410           emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6411           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6412       // Ignore return result until untied tasks are supported.
6413       llvm::Value *Result = CGF.EmitRuntimeCall(
6414           createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
6415       // if (__kmpc_cancellationpoint()) {
6416       //   exit from construct;
6417       // }
6418       llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6419       llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6420       llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6421       CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6422       CGF.EmitBlock(ExitBB);
6423       // exit from construct;
6424       CodeGenFunction::JumpDest CancelDest =
6425           CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6426       CGF.EmitBranchThroughCleanup(CancelDest);
6427       CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6428     }
6429   }
6430 }
6431 
6432 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6433                                      const Expr *IfCond,
6434                                      OpenMPDirectiveKind CancelRegion) {
6435   if (!CGF.HaveInsertPoint())
6436     return;
6437   // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6438   // kmp_int32 cncl_kind);
6439   if (auto *OMPRegionInfo =
6440           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6441     auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
6442                                                         PrePostActionTy &) {
6443       CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6444       llvm::Value *Args[] = {
6445           RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6446           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6447       // Ignore return result until untied tasks are supported.
6448       llvm::Value *Result = CGF.EmitRuntimeCall(
6449           RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
6450       // if (__kmpc_cancel()) {
6451       //   exit from construct;
6452       // }
6453       llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6454       llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6455       llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6456       CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6457       CGF.EmitBlock(ExitBB);
6458       // exit from construct;
6459       CodeGenFunction::JumpDest CancelDest =
6460           CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6461       CGF.EmitBranchThroughCleanup(CancelDest);
6462       CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6463     };
6464     if (IfCond) {
6465       emitIfClause(CGF, IfCond, ThenGen,
6466                    [](CodeGenFunction &, PrePostActionTy &) {});
6467     } else {
6468       RegionCodeGenTy ThenRCG(ThenGen);
6469       ThenRCG(CGF);
6470     }
6471   }
6472 }
6473 
6474 void CGOpenMPRuntime::emitTargetOutlinedFunction(
6475     const OMPExecutableDirective &D, StringRef ParentName,
6476     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6477     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6478   assert(!ParentName.empty() && "Invalid target region parent name!");
6479   HasEmittedTargetRegion = true;
6480   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6481                                    IsOffloadEntry, CodeGen);
6482 }
6483 
6484 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6485     const OMPExecutableDirective &D, StringRef ParentName,
6486     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6487     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6488   // Create a unique name for the entry function using the source location
6489   // information of the current target region. The name will be something like:
6490   //
6491   // __omp_offloading_DD_FFFF_PP_lBB
6492   //
6493   // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
6494   // mangled name of the function that encloses the target region and BB is the
6495   // line number of the target region.
6496 
6497   unsigned DeviceID;
6498   unsigned FileID;
6499   unsigned Line;
6500   getTargetEntryUniqueInfo(CGM.getContext(), D.getBeginLoc(), DeviceID, FileID,
6501                            Line);
6502   SmallString<64> EntryFnName;
6503   {
6504     llvm::raw_svector_ostream OS(EntryFnName);
6505     OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
6506        << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
6507   }
6508 
6509   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6510 
6511   CodeGenFunction CGF(CGM, true);
6512   CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6513   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6514 
6515   OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
6516 
6517   // If this target outline function is not an offload entry, we don't need to
6518   // register it.
6519   if (!IsOffloadEntry)
6520     return;
6521 
6522   // The target region ID is used by the runtime library to identify the current
6523   // target region, so it only has to be unique and not necessarily point to
6524   // anything. It could be the pointer to the outlined function that implements
6525   // the target region, but we aren't using that so that the compiler doesn't
6526   // need to keep that, and could therefore inline the host function if proven
6527   // worthwhile during optimization. In the other hand, if emitting code for the
6528   // device, the ID has to be the function address so that it can retrieved from
6529   // the offloading entry and launched by the runtime library. We also mark the
6530   // outlined function to have external linkage in case we are emitting code for
6531   // the device, because these functions will be entry points to the device.
6532 
6533   if (CGM.getLangOpts().OpenMPIsDevice) {
6534     OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
6535     OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6536     OutlinedFn->setDSOLocal(false);
6537   } else {
6538     std::string Name = getName({EntryFnName, "region_id"});
6539     OutlinedFnID = new llvm::GlobalVariable(
6540         CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
6541         llvm::GlobalValue::WeakAnyLinkage,
6542         llvm::Constant::getNullValue(CGM.Int8Ty), Name);
6543   }
6544 
6545   // Register the information for the entry associated with this target region.
6546   OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
6547       DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6548       OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion);
6549 }
6550 
6551 /// Checks if the expression is constant or does not have non-trivial function
6552 /// calls.
6553 static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6554   // We can skip constant expressions.
6555   // We can skip expressions with trivial calls or simple expressions.
6556   return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) ||
6557           !E->hasNonTrivialCall(Ctx)) &&
6558          !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6559 }
6560 
6561 const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6562                                                     const Stmt *Body) {
6563   const Stmt *Child = Body->IgnoreContainers();
6564   while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {
6565     Child = nullptr;
6566     for (const Stmt *S : C->body()) {
6567       if (const auto *E = dyn_cast<Expr>(S)) {
6568         if (isTrivial(Ctx, E))
6569           continue;
6570       }
6571       // Some of the statements can be ignored.
6572       if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
6573           isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
6574         continue;
6575       // Analyze declarations.
6576       if (const auto *DS = dyn_cast<DeclStmt>(S)) {
6577         if (llvm::all_of(DS->decls(), [&Ctx](const Decl *D) {
6578               if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6579                   isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6580                   isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6581                   isa<UsingDirectiveDecl>(D) ||
6582                   isa<OMPDeclareReductionDecl>(D) ||
6583                   isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6584                 return true;
6585               const auto *VD = dyn_cast<VarDecl>(D);
6586               if (!VD)
6587                 return false;
6588               return VD->isConstexpr() ||
6589                      ((VD->getType().isTrivialType(Ctx) ||
6590                        VD->getType()->isReferenceType()) &&
6591                       (!VD->hasInit() || isTrivial(Ctx, VD->getInit())));
6592             }))
6593           continue;
6594       }
6595       // Found multiple children - cannot get the one child only.
6596       if (Child)
6597         return nullptr;
6598       Child = S;
6599     }
6600     if (Child)
6601       Child = Child->IgnoreContainers();
6602   }
6603   return Child;
6604 }
6605 
6606 /// Emit the number of teams for a target directive.  Inspect the num_teams
6607 /// clause associated with a teams construct combined or closely nested
6608 /// with the target directive.
6609 ///
6610 /// Emit a team of size one for directives such as 'target parallel' that
6611 /// have no associated teams construct.
6612 ///
6613 /// Otherwise, return nullptr.
6614 static llvm::Value *
6615 emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
6616                                const OMPExecutableDirective &D) {
6617   assert(!CGF.getLangOpts().OpenMPIsDevice &&
6618          "Clauses associated with the teams directive expected to be emitted "
6619          "only for the host!");
6620   OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6621   assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6622          "Expected target-based executable directive.");
6623   CGBuilderTy &Bld = CGF.Builder;
6624   switch (DirectiveKind) {
6625   case OMPD_target: {
6626     const auto *CS = D.getInnermostCapturedStmt();
6627     const auto *Body =
6628         CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6629     const Stmt *ChildStmt =
6630         CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body);
6631     if (const auto *NestedDir =
6632             dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6633       if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) {
6634         if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6635           CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6636           CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6637           const Expr *NumTeams =
6638               NestedDir->getSingleClause<OMPNumTeamsClause>()->getNumTeams();
6639           llvm::Value *NumTeamsVal =
6640               CGF.EmitScalarExpr(NumTeams,
6641                                  /*IgnoreResultAssign*/ true);
6642           return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6643                                    /*isSigned=*/true);
6644         }
6645         return Bld.getInt32(0);
6646       }
6647       if (isOpenMPParallelDirective(NestedDir->getDirectiveKind()) ||
6648           isOpenMPSimdDirective(NestedDir->getDirectiveKind()))
6649         return Bld.getInt32(1);
6650       return Bld.getInt32(0);
6651     }
6652     return nullptr;
6653   }
6654   case OMPD_target_teams:
6655   case OMPD_target_teams_distribute:
6656   case OMPD_target_teams_distribute_simd:
6657   case OMPD_target_teams_distribute_parallel_for:
6658   case OMPD_target_teams_distribute_parallel_for_simd: {
6659     if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6660       CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6661       const Expr *NumTeams =
6662           D.getSingleClause<OMPNumTeamsClause>()->getNumTeams();
6663       llvm::Value *NumTeamsVal =
6664           CGF.EmitScalarExpr(NumTeams,
6665                              /*IgnoreResultAssign*/ true);
6666       return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty,
6667                                /*isSigned=*/true);
6668     }
6669     return Bld.getInt32(0);
6670   }
6671   case OMPD_target_parallel:
6672   case OMPD_target_parallel_for:
6673   case OMPD_target_parallel_for_simd:
6674   case OMPD_target_simd:
6675     return Bld.getInt32(1);
6676   case OMPD_parallel:
6677   case OMPD_for:
6678   case OMPD_parallel_for:
6679   case OMPD_parallel_master:
6680   case OMPD_parallel_sections:
6681   case OMPD_for_simd:
6682   case OMPD_parallel_for_simd:
6683   case OMPD_cancel:
6684   case OMPD_cancellation_point:
6685   case OMPD_ordered:
6686   case OMPD_threadprivate:
6687   case OMPD_allocate:
6688   case OMPD_task:
6689   case OMPD_simd:
6690   case OMPD_sections:
6691   case OMPD_section:
6692   case OMPD_single:
6693   case OMPD_master:
6694   case OMPD_critical:
6695   case OMPD_taskyield:
6696   case OMPD_barrier:
6697   case OMPD_taskwait:
6698   case OMPD_taskgroup:
6699   case OMPD_atomic:
6700   case OMPD_flush:
6701   case OMPD_teams:
6702   case OMPD_target_data:
6703   case OMPD_target_exit_data:
6704   case OMPD_target_enter_data:
6705   case OMPD_distribute:
6706   case OMPD_distribute_simd:
6707   case OMPD_distribute_parallel_for:
6708   case OMPD_distribute_parallel_for_simd:
6709   case OMPD_teams_distribute:
6710   case OMPD_teams_distribute_simd:
6711   case OMPD_teams_distribute_parallel_for:
6712   case OMPD_teams_distribute_parallel_for_simd:
6713   case OMPD_target_update:
6714   case OMPD_declare_simd:
6715   case OMPD_declare_variant:
6716   case OMPD_declare_target:
6717   case OMPD_end_declare_target:
6718   case OMPD_declare_reduction:
6719   case OMPD_declare_mapper:
6720   case OMPD_taskloop:
6721   case OMPD_taskloop_simd:
6722   case OMPD_master_taskloop:
6723   case OMPD_master_taskloop_simd:
6724   case OMPD_parallel_master_taskloop:
6725   case OMPD_parallel_master_taskloop_simd:
6726   case OMPD_requires:
6727   case OMPD_unknown:
6728     break;
6729   }
6730   llvm_unreachable("Unexpected directive kind.");
6731 }
6732 
6733 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6734                                   llvm::Value *DefaultThreadLimitVal) {
6735   const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6736       CGF.getContext(), CS->getCapturedStmt());
6737   if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6738     if (isOpenMPParallelDirective(Dir->getDirectiveKind())) {
6739       llvm::Value *NumThreads = nullptr;
6740       llvm::Value *CondVal = nullptr;
6741       // Handle if clause. If if clause present, the number of threads is
6742       // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6743       if (Dir->hasClausesOfKind<OMPIfClause>()) {
6744         CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6745         CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6746         const OMPIfClause *IfClause = nullptr;
6747         for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6748           if (C->getNameModifier() == OMPD_unknown ||
6749               C->getNameModifier() == OMPD_parallel) {
6750             IfClause = C;
6751             break;
6752           }
6753         }
6754         if (IfClause) {
6755           const Expr *Cond = IfClause->getCondition();
6756           bool Result;
6757           if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6758             if (!Result)
6759               return CGF.Builder.getInt32(1);
6760           } else {
6761             CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange());
6762             if (const auto *PreInit =
6763                     cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) {
6764               for (const auto *I : PreInit->decls()) {
6765                 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6766                   CGF.EmitVarDecl(cast<VarDecl>(*I));
6767                 } else {
6768                   CodeGenFunction::AutoVarEmission Emission =
6769                       CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6770                   CGF.EmitAutoVarCleanups(Emission);
6771                 }
6772               }
6773             }
6774             CondVal = CGF.EvaluateExprAsBool(Cond);
6775           }
6776         }
6777       }
6778       // Check the value of num_threads clause iff if clause was not specified
6779       // or is not evaluated to false.
6780       if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6781         CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6782         CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6783         const auto *NumThreadsClause =
6784             Dir->getSingleClause<OMPNumThreadsClause>();
6785         CodeGenFunction::LexicalScope Scope(
6786             CGF, NumThreadsClause->getNumThreads()->getSourceRange());
6787         if (const auto *PreInit =
6788                 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6789           for (const auto *I : PreInit->decls()) {
6790             if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6791               CGF.EmitVarDecl(cast<VarDecl>(*I));
6792             } else {
6793               CodeGenFunction::AutoVarEmission Emission =
6794                   CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6795               CGF.EmitAutoVarCleanups(Emission);
6796             }
6797           }
6798         }
6799         NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads());
6800         NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty,
6801                                                /*isSigned=*/false);
6802         if (DefaultThreadLimitVal)
6803           NumThreads = CGF.Builder.CreateSelect(
6804               CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads),
6805               DefaultThreadLimitVal, NumThreads);
6806       } else {
6807         NumThreads = DefaultThreadLimitVal ? DefaultThreadLimitVal
6808                                            : CGF.Builder.getInt32(0);
6809       }
6810       // Process condition of the if clause.
6811       if (CondVal) {
6812         NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads,
6813                                               CGF.Builder.getInt32(1));
6814       }
6815       return NumThreads;
6816     }
6817     if (isOpenMPSimdDirective(Dir->getDirectiveKind()))
6818       return CGF.Builder.getInt32(1);
6819     return DefaultThreadLimitVal;
6820   }
6821   return DefaultThreadLimitVal ? DefaultThreadLimitVal
6822                                : CGF.Builder.getInt32(0);
6823 }
6824 
6825 /// Emit the number of threads for a target directive.  Inspect the
6826 /// thread_limit clause associated with a teams construct combined or closely
6827 /// nested with the target directive.
6828 ///
6829 /// Emit the num_threads clause for directives such as 'target parallel' that
6830 /// have no associated teams construct.
6831 ///
6832 /// Otherwise, return nullptr.
6833 static llvm::Value *
6834 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
6835                                  const OMPExecutableDirective &D) {
6836   assert(!CGF.getLangOpts().OpenMPIsDevice &&
6837          "Clauses associated with the teams directive expected to be emitted "
6838          "only for the host!");
6839   OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6840   assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6841          "Expected target-based executable directive.");
6842   CGBuilderTy &Bld = CGF.Builder;
6843   llvm::Value *ThreadLimitVal = nullptr;
6844   llvm::Value *NumThreadsVal = nullptr;
6845   switch (DirectiveKind) {
6846   case OMPD_target: {
6847     const CapturedStmt *CS = D.getInnermostCapturedStmt();
6848     if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6849       return NumThreads;
6850     const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6851         CGF.getContext(), CS->getCapturedStmt());
6852     if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6853       if (Dir->hasClausesOfKind<OMPThreadLimitClause>()) {
6854         CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6855         CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6856         const auto *ThreadLimitClause =
6857             Dir->getSingleClause<OMPThreadLimitClause>();
6858         CodeGenFunction::LexicalScope Scope(
6859             CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6860         if (const auto *PreInit =
6861                 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6862           for (const auto *I : PreInit->decls()) {
6863             if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6864               CGF.EmitVarDecl(cast<VarDecl>(*I));
6865             } else {
6866               CodeGenFunction::AutoVarEmission Emission =
6867                   CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
6868               CGF.EmitAutoVarCleanups(Emission);
6869             }
6870           }
6871         }
6872         llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
6873             ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
6874         ThreadLimitVal =
6875             Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
6876       }
6877       if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) &&
6878           !isOpenMPDistributeDirective(Dir->getDirectiveKind())) {
6879         CS = Dir->getInnermostCapturedStmt();
6880         const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6881             CGF.getContext(), CS->getCapturedStmt());
6882         Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6883       }
6884       if (Dir && isOpenMPDistributeDirective(Dir->getDirectiveKind()) &&
6885           !isOpenMPSimdDirective(Dir->getDirectiveKind())) {
6886         CS = Dir->getInnermostCapturedStmt();
6887         if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6888           return NumThreads;
6889       }
6890       if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind()))
6891         return Bld.getInt32(1);
6892     }
6893     return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6894   }
6895   case OMPD_target_teams: {
6896     if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6897       CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6898       const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6899       llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
6900           ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
6901       ThreadLimitVal =
6902           Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
6903     }
6904     const CapturedStmt *CS = D.getInnermostCapturedStmt();
6905     if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6906       return NumThreads;
6907     const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6908         CGF.getContext(), CS->getCapturedStmt());
6909     if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6910       if (Dir->getDirectiveKind() == OMPD_distribute) {
6911         CS = Dir->getInnermostCapturedStmt();
6912         if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal))
6913           return NumThreads;
6914       }
6915     }
6916     return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6917   }
6918   case OMPD_target_teams_distribute:
6919     if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6920       CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6921       const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6922       llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
6923           ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
6924       ThreadLimitVal =
6925           Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
6926     }
6927     return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal);
6928   case OMPD_target_parallel:
6929   case OMPD_target_parallel_for:
6930   case OMPD_target_parallel_for_simd:
6931   case OMPD_target_teams_distribute_parallel_for:
6932   case OMPD_target_teams_distribute_parallel_for_simd: {
6933     llvm::Value *CondVal = nullptr;
6934     // Handle if clause. If if clause present, the number of threads is
6935     // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6936     if (D.hasClausesOfKind<OMPIfClause>()) {
6937       const OMPIfClause *IfClause = nullptr;
6938       for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
6939         if (C->getNameModifier() == OMPD_unknown ||
6940             C->getNameModifier() == OMPD_parallel) {
6941           IfClause = C;
6942           break;
6943         }
6944       }
6945       if (IfClause) {
6946         const Expr *Cond = IfClause->getCondition();
6947         bool Result;
6948         if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) {
6949           if (!Result)
6950             return Bld.getInt32(1);
6951         } else {
6952           CodeGenFunction::RunCleanupsScope Scope(CGF);
6953           CondVal = CGF.EvaluateExprAsBool(Cond);
6954         }
6955       }
6956     }
6957     if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6958       CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6959       const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6960       llvm::Value *ThreadLimit = CGF.EmitScalarExpr(
6961           ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true);
6962       ThreadLimitVal =
6963           Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false);
6964     }
6965     if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
6966       CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6967       const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
6968       llvm::Value *NumThreads = CGF.EmitScalarExpr(
6969           NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true);
6970       NumThreadsVal =
6971           Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false);
6972       ThreadLimitVal = ThreadLimitVal
6973                            ? Bld.CreateSelect(Bld.CreateICmpULT(NumThreadsVal,
6974                                                                 ThreadLimitVal),
6975                                               NumThreadsVal, ThreadLimitVal)
6976                            : NumThreadsVal;
6977     }
6978     if (!ThreadLimitVal)
6979       ThreadLimitVal = Bld.getInt32(0);
6980     if (CondVal)
6981       return Bld.CreateSelect(CondVal, ThreadLimitVal, Bld.getInt32(1));
6982     return ThreadLimitVal;
6983   }
6984   case OMPD_target_teams_distribute_simd:
6985   case OMPD_target_simd:
6986     return Bld.getInt32(1);
6987   case OMPD_parallel:
6988   case OMPD_for:
6989   case OMPD_parallel_for:
6990   case OMPD_parallel_master:
6991   case OMPD_parallel_sections:
6992   case OMPD_for_simd:
6993   case OMPD_parallel_for_simd:
6994   case OMPD_cancel:
6995   case OMPD_cancellation_point:
6996   case OMPD_ordered:
6997   case OMPD_threadprivate:
6998   case OMPD_allocate:
6999   case OMPD_task:
7000   case OMPD_simd:
7001   case OMPD_sections:
7002   case OMPD_section:
7003   case OMPD_single:
7004   case OMPD_master:
7005   case OMPD_critical:
7006   case OMPD_taskyield:
7007   case OMPD_barrier:
7008   case OMPD_taskwait:
7009   case OMPD_taskgroup:
7010   case OMPD_atomic:
7011   case OMPD_flush:
7012   case OMPD_teams:
7013   case OMPD_target_data:
7014   case OMPD_target_exit_data:
7015   case OMPD_target_enter_data:
7016   case OMPD_distribute:
7017   case OMPD_distribute_simd:
7018   case OMPD_distribute_parallel_for:
7019   case OMPD_distribute_parallel_for_simd:
7020   case OMPD_teams_distribute:
7021   case OMPD_teams_distribute_simd:
7022   case OMPD_teams_distribute_parallel_for:
7023   case OMPD_teams_distribute_parallel_for_simd:
7024   case OMPD_target_update:
7025   case OMPD_declare_simd:
7026   case OMPD_declare_variant:
7027   case OMPD_declare_target:
7028   case OMPD_end_declare_target:
7029   case OMPD_declare_reduction:
7030   case OMPD_declare_mapper:
7031   case OMPD_taskloop:
7032   case OMPD_taskloop_simd:
7033   case OMPD_master_taskloop:
7034   case OMPD_master_taskloop_simd:
7035   case OMPD_parallel_master_taskloop:
7036   case OMPD_parallel_master_taskloop_simd:
7037   case OMPD_requires:
7038   case OMPD_unknown:
7039     break;
7040   }
7041   llvm_unreachable("Unsupported directive kind.");
7042 }
7043 
7044 namespace {
7045 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
7046 
7047 // Utility to handle information from clauses associated with a given
7048 // construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7049 // It provides a convenient interface to obtain the information and generate
7050 // code for that information.
7051 class MappableExprsHandler {
7052 public:
7053   /// Values for bit flags used to specify the mapping type for
7054   /// offloading.
7055   enum OpenMPOffloadMappingFlags : uint64_t {
7056     /// No flags
7057     OMP_MAP_NONE = 0x0,
7058     /// Allocate memory on the device and move data from host to device.
7059     OMP_MAP_TO = 0x01,
7060     /// Allocate memory on the device and move data from device to host.
7061     OMP_MAP_FROM = 0x02,
7062     /// Always perform the requested mapping action on the element, even
7063     /// if it was already mapped before.
7064     OMP_MAP_ALWAYS = 0x04,
7065     /// Delete the element from the device environment, ignoring the
7066     /// current reference count associated with the element.
7067     OMP_MAP_DELETE = 0x08,
7068     /// The element being mapped is a pointer-pointee pair; both the
7069     /// pointer and the pointee should be mapped.
7070     OMP_MAP_PTR_AND_OBJ = 0x10,
7071     /// This flags signals that the base address of an entry should be
7072     /// passed to the target kernel as an argument.
7073     OMP_MAP_TARGET_PARAM = 0x20,
7074     /// Signal that the runtime library has to return the device pointer
7075     /// in the current position for the data being mapped. Used when we have the
7076     /// use_device_ptr clause.
7077     OMP_MAP_RETURN_PARAM = 0x40,
7078     /// This flag signals that the reference being passed is a pointer to
7079     /// private data.
7080     OMP_MAP_PRIVATE = 0x80,
7081     /// Pass the element to the device by value.
7082     OMP_MAP_LITERAL = 0x100,
7083     /// Implicit map
7084     OMP_MAP_IMPLICIT = 0x200,
7085     /// Close is a hint to the runtime to allocate memory close to
7086     /// the target device.
7087     OMP_MAP_CLOSE = 0x400,
7088     /// The 16 MSBs of the flags indicate whether the entry is member of some
7089     /// struct/class.
7090     OMP_MAP_MEMBER_OF = 0xffff000000000000,
7091     LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ OMP_MAP_MEMBER_OF),
7092   };
7093 
7094   /// Get the offset of the OMP_MAP_MEMBER_OF field.
7095   static unsigned getFlagMemberOffset() {
7096     unsigned Offset = 0;
7097     for (uint64_t Remain = OMP_MAP_MEMBER_OF; !(Remain & 1);
7098          Remain = Remain >> 1)
7099       Offset++;
7100     return Offset;
7101   }
7102 
7103   /// Class that associates information with a base pointer to be passed to the
7104   /// runtime library.
7105   class BasePointerInfo {
7106     /// The base pointer.
7107     llvm::Value *Ptr = nullptr;
7108     /// The base declaration that refers to this device pointer, or null if
7109     /// there is none.
7110     const ValueDecl *DevPtrDecl = nullptr;
7111 
7112   public:
7113     BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
7114         : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
7115     llvm::Value *operator*() const { return Ptr; }
7116     const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
7117     void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
7118   };
7119 
7120   using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>;
7121   using MapValuesArrayTy = SmallVector<llvm::Value *, 4>;
7122   using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>;
7123 
7124   /// Map between a struct and the its lowest & highest elements which have been
7125   /// mapped.
7126   /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7127   ///                    HE(FieldIndex, Pointer)}
7128   struct StructRangeInfoTy {
7129     std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7130         0, Address::invalid()};
7131     std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7132         0, Address::invalid()};
7133     Address Base = Address::invalid();
7134   };
7135 
7136 private:
7137   /// Kind that defines how a device pointer has to be returned.
7138   struct MapInfo {
7139     OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7140     OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7141     ArrayRef<OpenMPMapModifierKind> MapModifiers;
7142     bool ReturnDevicePointer = false;
7143     bool IsImplicit = false;
7144 
7145     MapInfo() = default;
7146     MapInfo(
7147         OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7148         OpenMPMapClauseKind MapType,
7149         ArrayRef<OpenMPMapModifierKind> MapModifiers,
7150         bool ReturnDevicePointer, bool IsImplicit)
7151         : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7152           ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
7153   };
7154 
7155   /// If use_device_ptr is used on a pointer which is a struct member and there
7156   /// is no map information about it, then emission of that entry is deferred
7157   /// until the whole struct has been processed.
7158   struct DeferredDevicePtrEntryTy {
7159     const Expr *IE = nullptr;
7160     const ValueDecl *VD = nullptr;
7161 
7162     DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD)
7163         : IE(IE), VD(VD) {}
7164   };
7165 
7166   /// The target directive from where the mappable clauses were extracted. It
7167   /// is either a executable directive or a user-defined mapper directive.
7168   llvm::PointerUnion<const OMPExecutableDirective *,
7169                      const OMPDeclareMapperDecl *>
7170       CurDir;
7171 
7172   /// Function the directive is being generated for.
7173   CodeGenFunction &CGF;
7174 
7175   /// Set of all first private variables in the current directive.
7176   /// bool data is set to true if the variable is implicitly marked as
7177   /// firstprivate, false otherwise.
7178   llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7179 
7180   /// Map between device pointer declarations and their expression components.
7181   /// The key value for declarations in 'this' is null.
7182   llvm::DenseMap<
7183       const ValueDecl *,
7184       SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7185       DevPointersMap;
7186 
7187   llvm::Value *getExprTypeSize(const Expr *E) const {
7188     QualType ExprTy = E->getType().getCanonicalType();
7189 
7190     // Reference types are ignored for mapping purposes.
7191     if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7192       ExprTy = RefTy->getPointeeType().getCanonicalType();
7193 
7194     // Given that an array section is considered a built-in type, we need to
7195     // do the calculation based on the length of the section instead of relying
7196     // on CGF.getTypeSize(E->getType()).
7197     if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
7198       QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(
7199                             OAE->getBase()->IgnoreParenImpCasts())
7200                             .getCanonicalType();
7201 
7202       // If there is no length associated with the expression and lower bound is
7203       // not specified too, that means we are using the whole length of the
7204       // base.
7205       if (!OAE->getLength() && OAE->getColonLoc().isValid() &&
7206           !OAE->getLowerBound())
7207         return CGF.getTypeSize(BaseTy);
7208 
7209       llvm::Value *ElemSize;
7210       if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7211         ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
7212       } else {
7213         const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
7214         assert(ATy && "Expecting array type if not a pointer type.");
7215         ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
7216       }
7217 
7218       // If we don't have a length at this point, that is because we have an
7219       // array section with a single element.
7220       if (!OAE->getLength() && OAE->getColonLoc().isInvalid())
7221         return ElemSize;
7222 
7223       if (const Expr *LenExpr = OAE->getLength()) {
7224         llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
7225         LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
7226                                              CGF.getContext().getSizeType(),
7227                                              LenExpr->getExprLoc());
7228         return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
7229       }
7230       assert(!OAE->getLength() && OAE->getColonLoc().isValid() &&
7231              OAE->getLowerBound() && "expected array_section[lb:].");
7232       // Size = sizetype - lb * elemtype;
7233       llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
7234       llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
7235       LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
7236                                        CGF.getContext().getSizeType(),
7237                                        OAE->getLowerBound()->getExprLoc());
7238       LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
7239       llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
7240       llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
7241       LengthVal = CGF.Builder.CreateSelect(
7242           Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
7243       return LengthVal;
7244     }
7245     return CGF.getTypeSize(ExprTy);
7246   }
7247 
7248   /// Return the corresponding bits for a given map clause modifier. Add
7249   /// a flag marking the map as a pointer if requested. Add a flag marking the
7250   /// map as the first one of a series of maps that relate to the same map
7251   /// expression.
7252   OpenMPOffloadMappingFlags getMapTypeBits(
7253       OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7254       bool IsImplicit, bool AddPtrFlag, bool AddIsTargetParamFlag) const {
7255     OpenMPOffloadMappingFlags Bits =
7256         IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
7257     switch (MapType) {
7258     case OMPC_MAP_alloc:
7259     case OMPC_MAP_release:
7260       // alloc and release is the default behavior in the runtime library,  i.e.
7261       // if we don't pass any bits alloc/release that is what the runtime is
7262       // going to do. Therefore, we don't need to signal anything for these two
7263       // type modifiers.
7264       break;
7265     case OMPC_MAP_to:
7266       Bits |= OMP_MAP_TO;
7267       break;
7268     case OMPC_MAP_from:
7269       Bits |= OMP_MAP_FROM;
7270       break;
7271     case OMPC_MAP_tofrom:
7272       Bits |= OMP_MAP_TO | OMP_MAP_FROM;
7273       break;
7274     case OMPC_MAP_delete:
7275       Bits |= OMP_MAP_DELETE;
7276       break;
7277     case OMPC_MAP_unknown:
7278       llvm_unreachable("Unexpected map type!");
7279     }
7280     if (AddPtrFlag)
7281       Bits |= OMP_MAP_PTR_AND_OBJ;
7282     if (AddIsTargetParamFlag)
7283       Bits |= OMP_MAP_TARGET_PARAM;
7284     if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always)
7285         != MapModifiers.end())
7286       Bits |= OMP_MAP_ALWAYS;
7287     if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_close)
7288         != MapModifiers.end())
7289       Bits |= OMP_MAP_CLOSE;
7290     return Bits;
7291   }
7292 
7293   /// Return true if the provided expression is a final array section. A
7294   /// final array section, is one whose length can't be proved to be one.
7295   bool isFinalArraySectionExpression(const Expr *E) const {
7296     const auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
7297 
7298     // It is not an array section and therefore not a unity-size one.
7299     if (!OASE)
7300       return false;
7301 
7302     // An array section with no colon always refer to a single element.
7303     if (OASE->getColonLoc().isInvalid())
7304       return false;
7305 
7306     const Expr *Length = OASE->getLength();
7307 
7308     // If we don't have a length we have to check if the array has size 1
7309     // for this dimension. Also, we should always expect a length if the
7310     // base type is pointer.
7311     if (!Length) {
7312       QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType(
7313                              OASE->getBase()->IgnoreParenImpCasts())
7314                              .getCanonicalType();
7315       if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
7316         return ATy->getSize().getSExtValue() != 1;
7317       // If we don't have a constant dimension length, we have to consider
7318       // the current section as having any size, so it is not necessarily
7319       // unitary. If it happen to be unity size, that's user fault.
7320       return true;
7321     }
7322 
7323     // Check if the length evaluates to 1.
7324     Expr::EvalResult Result;
7325     if (!Length->EvaluateAsInt(Result, CGF.getContext()))
7326       return true; // Can have more that size 1.
7327 
7328     llvm::APSInt ConstLength = Result.Val.getInt();
7329     return ConstLength.getSExtValue() != 1;
7330   }
7331 
7332   /// Generate the base pointers, section pointers, sizes and map type
7333   /// bits for the provided map type, map modifier, and expression components.
7334   /// \a IsFirstComponent should be set to true if the provided set of
7335   /// components is the first associated with a capture.
7336   void generateInfoForComponentList(
7337       OpenMPMapClauseKind MapType,
7338       ArrayRef<OpenMPMapModifierKind> MapModifiers,
7339       OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7340       MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
7341       MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7342       StructRangeInfoTy &PartialStruct, bool IsFirstComponentList,
7343       bool IsImplicit,
7344       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7345           OverlappedElements = llvm::None) const {
7346     // The following summarizes what has to be generated for each map and the
7347     // types below. The generated information is expressed in this order:
7348     // base pointer, section pointer, size, flags
7349     // (to add to the ones that come from the map type and modifier).
7350     //
7351     // double d;
7352     // int i[100];
7353     // float *p;
7354     //
7355     // struct S1 {
7356     //   int i;
7357     //   float f[50];
7358     // }
7359     // struct S2 {
7360     //   int i;
7361     //   float f[50];
7362     //   S1 s;
7363     //   double *p;
7364     //   struct S2 *ps;
7365     // }
7366     // S2 s;
7367     // S2 *ps;
7368     //
7369     // map(d)
7370     // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
7371     //
7372     // map(i)
7373     // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
7374     //
7375     // map(i[1:23])
7376     // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
7377     //
7378     // map(p)
7379     // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7380     //
7381     // map(p[1:24])
7382     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
7383     //
7384     // map(s)
7385     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
7386     //
7387     // map(s.i)
7388     // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
7389     //
7390     // map(s.s.f)
7391     // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7392     //
7393     // map(s.p)
7394     // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
7395     //
7396     // map(to: s.p[:22])
7397     // &s, &(s.p), sizeof(double*), TARGET_PARAM (*)
7398     // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**)
7399     // &(s.p), &(s.p[0]), 22*sizeof(double),
7400     //   MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
7401     // (*) alloc space for struct members, only this is a target parameter
7402     // (**) map the pointer (nothing to be mapped in this example) (the compiler
7403     //      optimizes this entry out, same in the examples below)
7404     // (***) map the pointee (map: to)
7405     //
7406     // map(s.ps)
7407     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7408     //
7409     // map(from: s.ps->s.i)
7410     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7411     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7412     // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ  | FROM
7413     //
7414     // map(to: s.ps->ps)
7415     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7416     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7417     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ  | TO
7418     //
7419     // map(s.ps->ps->ps)
7420     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7421     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7422     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7423     // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7424     //
7425     // map(to: s.ps->ps->s.f[:22])
7426     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
7427     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
7428     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7429     // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7430     //
7431     // map(ps)
7432     // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7433     //
7434     // map(ps->i)
7435     // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7436     //
7437     // map(ps->s.f)
7438     // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7439     //
7440     // map(from: ps->p)
7441     // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7442     //
7443     // map(to: ps->p[:22])
7444     // ps, &(ps->p), sizeof(double*), TARGET_PARAM
7445     // ps, &(ps->p), sizeof(double*), MEMBER_OF(1)
7446     // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO
7447     //
7448     // map(ps->ps)
7449     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7450     //
7451     // map(from: ps->ps->s.i)
7452     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7453     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7454     // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7455     //
7456     // map(from: ps->ps->ps)
7457     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7458     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7459     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7460     //
7461     // map(ps->ps->ps->ps)
7462     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7463     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7464     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7465     // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
7466     //
7467     // map(to: ps->ps->ps->s.f[:22])
7468     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
7469     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
7470     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
7471     // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
7472     //
7473     // map(to: s.f[:22]) map(from: s.p[:33])
7474     // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) +
7475     //     sizeof(double*) (**), TARGET_PARAM
7476     // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7477     // &s, &(s.p), sizeof(double*), MEMBER_OF(1)
7478     // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM
7479     // (*) allocate contiguous space needed to fit all mapped members even if
7480     //     we allocate space for members not mapped (in this example,
7481     //     s.f[22..49] and s.s are not mapped, yet we must allocate space for
7482     //     them as well because they fall between &s.f[0] and &s.p)
7483     //
7484     // map(from: s.f[:22]) map(to: ps->p[:33])
7485     // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7486     // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7487     // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*)
7488     // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO
7489     // (*) the struct this entry pertains to is the 2nd element in the list of
7490     //     arguments, hence MEMBER_OF(2)
7491     //
7492     // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7493     // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM
7494     // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7495     // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7496     // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
7497     // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*)
7498     // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO
7499     // (*) the struct this entry pertains to is the 4th element in the list
7500     //     of arguments, hence MEMBER_OF(4)
7501 
7502     // Track if the map information being generated is the first for a capture.
7503     bool IsCaptureFirstInfo = IsFirstComponentList;
7504     // When the variable is on a declare target link or in a to clause with
7505     // unified memory, a reference is needed to hold the host/device address
7506     // of the variable.
7507     bool RequiresReference = false;
7508 
7509     // Scan the components from the base to the complete expression.
7510     auto CI = Components.rbegin();
7511     auto CE = Components.rend();
7512     auto I = CI;
7513 
7514     // Track if the map information being generated is the first for a list of
7515     // components.
7516     bool IsExpressionFirstInfo = true;
7517     Address BP = Address::invalid();
7518     const Expr *AssocExpr = I->getAssociatedExpression();
7519     const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
7520     const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
7521 
7522     if (isa<MemberExpr>(AssocExpr)) {
7523       // The base is the 'this' pointer. The content of the pointer is going
7524       // to be the base of the field being mapped.
7525       BP = CGF.LoadCXXThisAddress();
7526     } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7527                (OASE &&
7528                 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7529       BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF);
7530     } else {
7531       // The base is the reference to the variable.
7532       // BP = &Var.
7533       BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF);
7534       if (const auto *VD =
7535               dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7536         if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7537                 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7538           if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7539               (*Res == OMPDeclareTargetDeclAttr::MT_To &&
7540                CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
7541             RequiresReference = true;
7542             BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
7543           }
7544         }
7545       }
7546 
7547       // If the variable is a pointer and is being dereferenced (i.e. is not
7548       // the last component), the base has to be the pointer itself, not its
7549       // reference. References are ignored for mapping purposes.
7550       QualType Ty =
7551           I->getAssociatedDeclaration()->getType().getNonReferenceType();
7552       if (Ty->isAnyPointerType() && std::next(I) != CE) {
7553         BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
7554 
7555         // We do not need to generate individual map information for the
7556         // pointer, it can be associated with the combined storage.
7557         ++I;
7558       }
7559     }
7560 
7561     // Track whether a component of the list should be marked as MEMBER_OF some
7562     // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
7563     // in a component list should be marked as MEMBER_OF, all subsequent entries
7564     // do not belong to the base struct. E.g.
7565     // struct S2 s;
7566     // s.ps->ps->ps->f[:]
7567     //   (1) (2) (3) (4)
7568     // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
7569     // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
7570     // is the pointee of ps(2) which is not member of struct s, so it should not
7571     // be marked as such (it is still PTR_AND_OBJ).
7572     // The variable is initialized to false so that PTR_AND_OBJ entries which
7573     // are not struct members are not considered (e.g. array of pointers to
7574     // data).
7575     bool ShouldBeMemberOf = false;
7576 
7577     // Variable keeping track of whether or not we have encountered a component
7578     // in the component list which is a member expression. Useful when we have a
7579     // pointer or a final array section, in which case it is the previous
7580     // component in the list which tells us whether we have a member expression.
7581     // E.g. X.f[:]
7582     // While processing the final array section "[:]" it is "f" which tells us
7583     // whether we are dealing with a member of a declared struct.
7584     const MemberExpr *EncounteredME = nullptr;
7585 
7586     for (; I != CE; ++I) {
7587       // If the current component is member of a struct (parent struct) mark it.
7588       if (!EncounteredME) {
7589         EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
7590         // If we encounter a PTR_AND_OBJ entry from now on it should be marked
7591         // as MEMBER_OF the parent struct.
7592         if (EncounteredME)
7593           ShouldBeMemberOf = true;
7594       }
7595 
7596       auto Next = std::next(I);
7597 
7598       // We need to generate the addresses and sizes if this is the last
7599       // component, if the component is a pointer or if it is an array section
7600       // whose length can't be proved to be one. If this is a pointer, it
7601       // becomes the base address for the following components.
7602 
7603       // A final array section, is one whose length can't be proved to be one.
7604       bool IsFinalArraySection =
7605           isFinalArraySectionExpression(I->getAssociatedExpression());
7606 
7607       // Get information on whether the element is a pointer. Have to do a
7608       // special treatment for array sections given that they are built-in
7609       // types.
7610       const auto *OASE =
7611           dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
7612       bool IsPointer =
7613           (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE)
7614                        .getCanonicalType()
7615                        ->isAnyPointerType()) ||
7616           I->getAssociatedExpression()->getType()->isAnyPointerType();
7617 
7618       if (Next == CE || IsPointer || IsFinalArraySection) {
7619         // If this is not the last component, we expect the pointer to be
7620         // associated with an array expression or member expression.
7621         assert((Next == CE ||
7622                 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7623                 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7624                 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
7625                "Unexpected expression");
7626 
7627         Address LB = CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
7628                          .getAddress(CGF);
7629 
7630         // If this component is a pointer inside the base struct then we don't
7631         // need to create any entry for it - it will be combined with the object
7632         // it is pointing to into a single PTR_AND_OBJ entry.
7633         bool IsMemberPointer =
7634             IsPointer && EncounteredME &&
7635             (dyn_cast<MemberExpr>(I->getAssociatedExpression()) ==
7636              EncounteredME);
7637         if (!OverlappedElements.empty()) {
7638           // Handle base element with the info for overlapped elements.
7639           assert(!PartialStruct.Base.isValid() && "The base element is set.");
7640           assert(Next == CE &&
7641                  "Expected last element for the overlapped elements.");
7642           assert(!IsPointer &&
7643                  "Unexpected base element with the pointer type.");
7644           // Mark the whole struct as the struct that requires allocation on the
7645           // device.
7646           PartialStruct.LowestElem = {0, LB};
7647           CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
7648               I->getAssociatedExpression()->getType());
7649           Address HB = CGF.Builder.CreateConstGEP(
7650               CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LB,
7651                                                               CGF.VoidPtrTy),
7652               TypeSize.getQuantity() - 1);
7653           PartialStruct.HighestElem = {
7654               std::numeric_limits<decltype(
7655                   PartialStruct.HighestElem.first)>::max(),
7656               HB};
7657           PartialStruct.Base = BP;
7658           // Emit data for non-overlapped data.
7659           OpenMPOffloadMappingFlags Flags =
7660               OMP_MAP_MEMBER_OF |
7661               getMapTypeBits(MapType, MapModifiers, IsImplicit,
7662                              /*AddPtrFlag=*/false,
7663                              /*AddIsTargetParamFlag=*/false);
7664           LB = BP;
7665           llvm::Value *Size = nullptr;
7666           // Do bitcopy of all non-overlapped structure elements.
7667           for (OMPClauseMappableExprCommon::MappableExprComponentListRef
7668                    Component : OverlappedElements) {
7669             Address ComponentLB = Address::invalid();
7670             for (const OMPClauseMappableExprCommon::MappableComponent &MC :
7671                  Component) {
7672               if (MC.getAssociatedDeclaration()) {
7673                 ComponentLB =
7674                     CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
7675                         .getAddress(CGF);
7676                 Size = CGF.Builder.CreatePtrDiff(
7677                     CGF.EmitCastToVoidPtr(ComponentLB.getPointer()),
7678                     CGF.EmitCastToVoidPtr(LB.getPointer()));
7679                 break;
7680               }
7681             }
7682             BasePointers.push_back(BP.getPointer());
7683             Pointers.push_back(LB.getPointer());
7684             Sizes.push_back(CGF.Builder.CreateIntCast(Size, CGF.Int64Ty,
7685                                                       /*isSigned=*/true));
7686             Types.push_back(Flags);
7687             LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
7688           }
7689           BasePointers.push_back(BP.getPointer());
7690           Pointers.push_back(LB.getPointer());
7691           Size = CGF.Builder.CreatePtrDiff(
7692               CGF.EmitCastToVoidPtr(
7693                   CGF.Builder.CreateConstGEP(HB, 1).getPointer()),
7694               CGF.EmitCastToVoidPtr(LB.getPointer()));
7695           Sizes.push_back(
7696               CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7697           Types.push_back(Flags);
7698           break;
7699         }
7700         llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7701         if (!IsMemberPointer) {
7702           BasePointers.push_back(BP.getPointer());
7703           Pointers.push_back(LB.getPointer());
7704           Sizes.push_back(
7705               CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
7706 
7707           // We need to add a pointer flag for each map that comes from the
7708           // same expression except for the first one. We also need to signal
7709           // this map is the first one that relates with the current capture
7710           // (there is a set of entries for each capture).
7711           OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7712               MapType, MapModifiers, IsImplicit,
7713               !IsExpressionFirstInfo || RequiresReference,
7714               IsCaptureFirstInfo && !RequiresReference);
7715 
7716           if (!IsExpressionFirstInfo) {
7717             // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
7718             // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
7719             if (IsPointer)
7720               Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7721                          OMP_MAP_DELETE | OMP_MAP_CLOSE);
7722 
7723             if (ShouldBeMemberOf) {
7724               // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
7725               // should be later updated with the correct value of MEMBER_OF.
7726               Flags |= OMP_MAP_MEMBER_OF;
7727               // From now on, all subsequent PTR_AND_OBJ entries should not be
7728               // marked as MEMBER_OF.
7729               ShouldBeMemberOf = false;
7730             }
7731           }
7732 
7733           Types.push_back(Flags);
7734         }
7735 
7736         // If we have encountered a member expression so far, keep track of the
7737         // mapped member. If the parent is "*this", then the value declaration
7738         // is nullptr.
7739         if (EncounteredME) {
7740           const auto *FD = dyn_cast<FieldDecl>(EncounteredME->getMemberDecl());
7741           unsigned FieldIndex = FD->getFieldIndex();
7742 
7743           // Update info about the lowest and highest elements for this struct
7744           if (!PartialStruct.Base.isValid()) {
7745             PartialStruct.LowestElem = {FieldIndex, LB};
7746             PartialStruct.HighestElem = {FieldIndex, LB};
7747             PartialStruct.Base = BP;
7748           } else if (FieldIndex < PartialStruct.LowestElem.first) {
7749             PartialStruct.LowestElem = {FieldIndex, LB};
7750           } else if (FieldIndex > PartialStruct.HighestElem.first) {
7751             PartialStruct.HighestElem = {FieldIndex, LB};
7752           }
7753         }
7754 
7755         // If we have a final array section, we are done with this expression.
7756         if (IsFinalArraySection)
7757           break;
7758 
7759         // The pointer becomes the base for the next element.
7760         if (Next != CE)
7761           BP = LB;
7762 
7763         IsExpressionFirstInfo = false;
7764         IsCaptureFirstInfo = false;
7765       }
7766     }
7767   }
7768 
7769   /// Return the adjusted map modifiers if the declaration a capture refers to
7770   /// appears in a first-private clause. This is expected to be used only with
7771   /// directives that start with 'target'.
7772   MappableExprsHandler::OpenMPOffloadMappingFlags
7773   getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
7774     assert(Cap.capturesVariable() && "Expected capture by reference only!");
7775 
7776     // A first private variable captured by reference will use only the
7777     // 'private ptr' and 'map to' flag. Return the right flags if the captured
7778     // declaration is known as first-private in this handler.
7779     if (FirstPrivateDecls.count(Cap.getCapturedVar())) {
7780       if (Cap.getCapturedVar()->getType().isConstant(CGF.getContext()) &&
7781           Cap.getCaptureKind() == CapturedStmt::VCK_ByRef)
7782         return MappableExprsHandler::OMP_MAP_ALWAYS |
7783                MappableExprsHandler::OMP_MAP_TO;
7784       if (Cap.getCapturedVar()->getType()->isAnyPointerType())
7785         return MappableExprsHandler::OMP_MAP_TO |
7786                MappableExprsHandler::OMP_MAP_PTR_AND_OBJ;
7787       return MappableExprsHandler::OMP_MAP_PRIVATE |
7788              MappableExprsHandler::OMP_MAP_TO;
7789     }
7790     return MappableExprsHandler::OMP_MAP_TO |
7791            MappableExprsHandler::OMP_MAP_FROM;
7792   }
7793 
7794   static OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position) {
7795     // Rotate by getFlagMemberOffset() bits.
7796     return static_cast<OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
7797                                                   << getFlagMemberOffset());
7798   }
7799 
7800   static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7801                                      OpenMPOffloadMappingFlags MemberOfFlag) {
7802     // If the entry is PTR_AND_OBJ but has not been marked with the special
7803     // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
7804     // marked as MEMBER_OF.
7805     if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7806         ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7807       return;
7808 
7809     // Reset the placeholder value to prepare the flag for the assignment of the
7810     // proper MEMBER_OF value.
7811     Flags &= ~OMP_MAP_MEMBER_OF;
7812     Flags |= MemberOfFlag;
7813   }
7814 
7815   void getPlainLayout(const CXXRecordDecl *RD,
7816                       llvm::SmallVectorImpl<const FieldDecl *> &Layout,
7817                       bool AsBase) const {
7818     const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
7819 
7820     llvm::StructType *St =
7821         AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
7822 
7823     unsigned NumElements = St->getNumElements();
7824     llvm::SmallVector<
7825         llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7826         RecordLayout(NumElements);
7827 
7828     // Fill bases.
7829     for (const auto &I : RD->bases()) {
7830       if (I.isVirtual())
7831         continue;
7832       const auto *Base = I.getType()->getAsCXXRecordDecl();
7833       // Ignore empty bases.
7834       if (Base->isEmpty() || CGF.getContext()
7835                                  .getASTRecordLayout(Base)
7836                                  .getNonVirtualSize()
7837                                  .isZero())
7838         continue;
7839 
7840       unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base);
7841       RecordLayout[FieldIndex] = Base;
7842     }
7843     // Fill in virtual bases.
7844     for (const auto &I : RD->vbases()) {
7845       const auto *Base = I.getType()->getAsCXXRecordDecl();
7846       // Ignore empty bases.
7847       if (Base->isEmpty())
7848         continue;
7849       unsigned FieldIndex = RL.getVirtualBaseIndex(Base);
7850       if (RecordLayout[FieldIndex])
7851         continue;
7852       RecordLayout[FieldIndex] = Base;
7853     }
7854     // Fill in all the fields.
7855     assert(!RD->isUnion() && "Unexpected union.");
7856     for (const auto *Field : RD->fields()) {
7857       // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
7858       // will fill in later.)
7859       if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) {
7860         unsigned FieldIndex = RL.getLLVMFieldNo(Field);
7861         RecordLayout[FieldIndex] = Field;
7862       }
7863     }
7864     for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7865              &Data : RecordLayout) {
7866       if (Data.isNull())
7867         continue;
7868       if (const auto *Base = Data.dyn_cast<const CXXRecordDecl *>())
7869         getPlainLayout(Base, Layout, /*AsBase=*/true);
7870       else
7871         Layout.push_back(Data.get<const FieldDecl *>());
7872     }
7873   }
7874 
7875 public:
7876   MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
7877       : CurDir(&Dir), CGF(CGF) {
7878     // Extract firstprivate clause information.
7879     for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
7880       for (const auto *D : C->varlists())
7881         FirstPrivateDecls.try_emplace(
7882             cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit());
7883     // Extract device pointer clause information.
7884     for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
7885       for (auto L : C->component_lists())
7886         DevPointersMap[L.first].push_back(L.second);
7887   }
7888 
7889   /// Constructor for the declare mapper directive.
7890   MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
7891       : CurDir(&Dir), CGF(CGF) {}
7892 
7893   /// Generate code for the combined entry if we have a partially mapped struct
7894   /// and take care of the mapping flags of the arguments corresponding to
7895   /// individual struct members.
7896   void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
7897                          MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7898                          MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
7899                          const StructRangeInfoTy &PartialStruct) const {
7900     // Base is the base of the struct
7901     BasePointers.push_back(PartialStruct.Base.getPointer());
7902     // Pointer is the address of the lowest element
7903     llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
7904     Pointers.push_back(LB);
7905     // Size is (addr of {highest+1} element) - (addr of lowest element)
7906     llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
7907     llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1);
7908     llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
7909     llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
7910     llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr);
7911     llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty,
7912                                                   /*isSigned=*/false);
7913     Sizes.push_back(Size);
7914     // Map type is always TARGET_PARAM
7915     Types.push_back(OMP_MAP_TARGET_PARAM);
7916     // Remove TARGET_PARAM flag from the first element
7917     (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
7918 
7919     // All other current entries will be MEMBER_OF the combined entry
7920     // (except for PTR_AND_OBJ entries which do not have a placeholder value
7921     // 0xFFFF in the MEMBER_OF field).
7922     OpenMPOffloadMappingFlags MemberOfFlag =
7923         getMemberOfFlag(BasePointers.size() - 1);
7924     for (auto &M : CurTypes)
7925       setCorrectMemberOfFlag(M, MemberOfFlag);
7926   }
7927 
7928   /// Generate all the base pointers, section pointers, sizes and map
7929   /// types for the extracted mappable expressions. Also, for each item that
7930   /// relates with a device pointer, a pair of the relevant declaration and
7931   /// index where it occurs is appended to the device pointers info array.
7932   void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
7933                        MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7934                        MapFlagsArrayTy &Types) const {
7935     // We have to process the component lists that relate with the same
7936     // declaration in a single chunk so that we can generate the map flags
7937     // correctly. Therefore, we organize all lists in a map.
7938     llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
7939 
7940     // Helper function to fill the information map for the different supported
7941     // clauses.
7942     auto &&InfoGen = [&Info](
7943         const ValueDecl *D,
7944         OMPClauseMappableExprCommon::MappableExprComponentListRef L,
7945         OpenMPMapClauseKind MapType,
7946         ArrayRef<OpenMPMapModifierKind> MapModifiers,
7947         bool ReturnDevicePointer, bool IsImplicit) {
7948       const ValueDecl *VD =
7949           D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
7950       Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
7951                             IsImplicit);
7952     };
7953 
7954     assert(CurDir.is<const OMPExecutableDirective *>() &&
7955            "Expect a executable directive");
7956     const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
7957     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>())
7958       for (const auto L : C->component_lists()) {
7959         InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(),
7960             /*ReturnDevicePointer=*/false, C->isImplicit());
7961       }
7962     for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>())
7963       for (const auto L : C->component_lists()) {
7964         InfoGen(L.first, L.second, OMPC_MAP_to, llvm::None,
7965             /*ReturnDevicePointer=*/false, C->isImplicit());
7966       }
7967     for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>())
7968       for (const auto L : C->component_lists()) {
7969         InfoGen(L.first, L.second, OMPC_MAP_from, llvm::None,
7970             /*ReturnDevicePointer=*/false, C->isImplicit());
7971       }
7972 
7973     // Look at the use_device_ptr clause information and mark the existing map
7974     // entries as such. If there is no map information for an entry in the
7975     // use_device_ptr list, we create one with map type 'alloc' and zero size
7976     // section. It is the user fault if that was not mapped before. If there is
7977     // no map information and the pointer is a struct member, then we defer the
7978     // emission of that entry until the whole struct has been processed.
7979     llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
7980         DeferredInfo;
7981 
7982     for (const auto *C :
7983          CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) {
7984       for (const auto L : C->component_lists()) {
7985         assert(!L.second.empty() && "Not expecting empty list of components!");
7986         const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
7987         VD = cast<ValueDecl>(VD->getCanonicalDecl());
7988         const Expr *IE = L.second.back().getAssociatedExpression();
7989         // If the first component is a member expression, we have to look into
7990         // 'this', which maps to null in the map of map information. Otherwise
7991         // look directly for the information.
7992         auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7993 
7994         // We potentially have map information for this declaration already.
7995         // Look for the first set of components that refer to it.
7996         if (It != Info.end()) {
7997           auto CI = std::find_if(
7998               It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
7999                 return MI.Components.back().getAssociatedDeclaration() == VD;
8000               });
8001           // If we found a map entry, signal that the pointer has to be returned
8002           // and move on to the next declaration.
8003           if (CI != It->second.end()) {
8004             CI->ReturnDevicePointer = true;
8005             continue;
8006           }
8007         }
8008 
8009         // We didn't find any match in our map information - generate a zero
8010         // size array section - if the pointer is a struct member we defer this
8011         // action until the whole struct has been processed.
8012         if (isa<MemberExpr>(IE)) {
8013           // Insert the pointer into Info to be processed by
8014           // generateInfoForComponentList. Because it is a member pointer
8015           // without a pointee, no entry will be generated for it, therefore
8016           // we need to generate one after the whole struct has been processed.
8017           // Nonetheless, generateInfoForComponentList must be called to take
8018           // the pointer into account for the calculation of the range of the
8019           // partial struct.
8020           InfoGen(nullptr, L.second, OMPC_MAP_unknown, llvm::None,
8021                   /*ReturnDevicePointer=*/false, C->isImplicit());
8022           DeferredInfo[nullptr].emplace_back(IE, VD);
8023         } else {
8024           llvm::Value *Ptr =
8025               CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());
8026           BasePointers.emplace_back(Ptr, VD);
8027           Pointers.push_back(Ptr);
8028           Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8029           Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
8030         }
8031       }
8032     }
8033 
8034     for (const auto &M : Info) {
8035       // We need to know when we generate information for the first component
8036       // associated with a capture, because the mapping flags depend on it.
8037       bool IsFirstComponentList = true;
8038 
8039       // Temporary versions of arrays
8040       MapBaseValuesArrayTy CurBasePointers;
8041       MapValuesArrayTy CurPointers;
8042       MapValuesArrayTy CurSizes;
8043       MapFlagsArrayTy CurTypes;
8044       StructRangeInfoTy PartialStruct;
8045 
8046       for (const MapInfo &L : M.second) {
8047         assert(!L.Components.empty() &&
8048                "Not expecting declaration with no component lists.");
8049 
8050         // Remember the current base pointer index.
8051         unsigned CurrentBasePointersIdx = CurBasePointers.size();
8052         generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8053                                      CurBasePointers, CurPointers, CurSizes,
8054                                      CurTypes, PartialStruct,
8055                                      IsFirstComponentList, L.IsImplicit);
8056 
8057         // If this entry relates with a device pointer, set the relevant
8058         // declaration and add the 'return pointer' flag.
8059         if (L.ReturnDevicePointer) {
8060           assert(CurBasePointers.size() > CurrentBasePointersIdx &&
8061                  "Unexpected number of mapped base pointers.");
8062 
8063           const ValueDecl *RelevantVD =
8064               L.Components.back().getAssociatedDeclaration();
8065           assert(RelevantVD &&
8066                  "No relevant declaration related with device pointer??");
8067 
8068           CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
8069           CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
8070         }
8071         IsFirstComponentList = false;
8072       }
8073 
8074       // Append any pending zero-length pointers which are struct members and
8075       // used with use_device_ptr.
8076       auto CI = DeferredInfo.find(M.first);
8077       if (CI != DeferredInfo.end()) {
8078         for (const DeferredDevicePtrEntryTy &L : CI->second) {
8079           llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF);
8080           llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
8081               this->CGF.EmitLValue(L.IE), L.IE->getExprLoc());
8082           CurBasePointers.emplace_back(BasePtr, L.VD);
8083           CurPointers.push_back(Ptr);
8084           CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.Int64Ty));
8085           // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the placeholder
8086           // value MEMBER_OF=FFFF so that the entry is later updated with the
8087           // correct value of MEMBER_OF.
8088           CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
8089                              OMP_MAP_MEMBER_OF);
8090         }
8091       }
8092 
8093       // If there is an entry in PartialStruct it means we have a struct with
8094       // individual members mapped. Emit an extra combined entry.
8095       if (PartialStruct.Base.isValid())
8096         emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8097                           PartialStruct);
8098 
8099       // We need to append the results of this capture to what we already have.
8100       BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8101       Pointers.append(CurPointers.begin(), CurPointers.end());
8102       Sizes.append(CurSizes.begin(), CurSizes.end());
8103       Types.append(CurTypes.begin(), CurTypes.end());
8104     }
8105   }
8106 
8107   /// Generate all the base pointers, section pointers, sizes and map types for
8108   /// the extracted map clauses of user-defined mapper.
8109   void generateAllInfoForMapper(MapBaseValuesArrayTy &BasePointers,
8110                                 MapValuesArrayTy &Pointers,
8111                                 MapValuesArrayTy &Sizes,
8112                                 MapFlagsArrayTy &Types) const {
8113     assert(CurDir.is<const OMPDeclareMapperDecl *>() &&
8114            "Expect a declare mapper directive");
8115     const auto *CurMapperDir = CurDir.get<const OMPDeclareMapperDecl *>();
8116     // We have to process the component lists that relate with the same
8117     // declaration in a single chunk so that we can generate the map flags
8118     // correctly. Therefore, we organize all lists in a map.
8119     llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
8120 
8121     // Helper function to fill the information map for the different supported
8122     // clauses.
8123     auto &&InfoGen = [&Info](
8124         const ValueDecl *D,
8125         OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8126         OpenMPMapClauseKind MapType,
8127         ArrayRef<OpenMPMapModifierKind> MapModifiers,
8128         bool ReturnDevicePointer, bool IsImplicit) {
8129       const ValueDecl *VD =
8130           D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
8131       Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
8132                             IsImplicit);
8133     };
8134 
8135     for (const auto *C : CurMapperDir->clauselists()) {
8136       const auto *MC = cast<OMPMapClause>(C);
8137       for (const auto L : MC->component_lists()) {
8138         InfoGen(L.first, L.second, MC->getMapType(), MC->getMapTypeModifiers(),
8139                 /*ReturnDevicePointer=*/false, MC->isImplicit());
8140       }
8141     }
8142 
8143     for (const auto &M : Info) {
8144       // We need to know when we generate information for the first component
8145       // associated with a capture, because the mapping flags depend on it.
8146       bool IsFirstComponentList = true;
8147 
8148       // Temporary versions of arrays
8149       MapBaseValuesArrayTy CurBasePointers;
8150       MapValuesArrayTy CurPointers;
8151       MapValuesArrayTy CurSizes;
8152       MapFlagsArrayTy CurTypes;
8153       StructRangeInfoTy PartialStruct;
8154 
8155       for (const MapInfo &L : M.second) {
8156         assert(!L.Components.empty() &&
8157                "Not expecting declaration with no component lists.");
8158         generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8159                                      CurBasePointers, CurPointers, CurSizes,
8160                                      CurTypes, PartialStruct,
8161                                      IsFirstComponentList, L.IsImplicit);
8162         IsFirstComponentList = false;
8163       }
8164 
8165       // If there is an entry in PartialStruct it means we have a struct with
8166       // individual members mapped. Emit an extra combined entry.
8167       if (PartialStruct.Base.isValid())
8168         emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8169                           PartialStruct);
8170 
8171       // We need to append the results of this capture to what we already have.
8172       BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8173       Pointers.append(CurPointers.begin(), CurPointers.end());
8174       Sizes.append(CurSizes.begin(), CurSizes.end());
8175       Types.append(CurTypes.begin(), CurTypes.end());
8176     }
8177   }
8178 
8179   /// Emit capture info for lambdas for variables captured by reference.
8180   void generateInfoForLambdaCaptures(
8181       const ValueDecl *VD, llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers,
8182       MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8183       MapFlagsArrayTy &Types,
8184       llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
8185     const auto *RD = VD->getType()
8186                          .getCanonicalType()
8187                          .getNonReferenceType()
8188                          ->getAsCXXRecordDecl();
8189     if (!RD || !RD->isLambda())
8190       return;
8191     Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD));
8192     LValue VDLVal = CGF.MakeAddrLValue(
8193         VDAddr, VD->getType().getCanonicalType().getNonReferenceType());
8194     llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
8195     FieldDecl *ThisCapture = nullptr;
8196     RD->getCaptureFields(Captures, ThisCapture);
8197     if (ThisCapture) {
8198       LValue ThisLVal =
8199           CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture);
8200       LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture);
8201       LambdaPointers.try_emplace(ThisLVal.getPointer(CGF),
8202                                  VDLVal.getPointer(CGF));
8203       BasePointers.push_back(ThisLVal.getPointer(CGF));
8204       Pointers.push_back(ThisLValVal.getPointer(CGF));
8205       Sizes.push_back(
8206           CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8207                                     CGF.Int64Ty, /*isSigned=*/true));
8208       Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8209                       OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8210     }
8211     for (const LambdaCapture &LC : RD->captures()) {
8212       if (!LC.capturesVariable())
8213         continue;
8214       const VarDecl *VD = LC.getCapturedVar();
8215       if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
8216         continue;
8217       auto It = Captures.find(VD);
8218       assert(It != Captures.end() && "Found lambda capture without field.");
8219       LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second);
8220       if (LC.getCaptureKind() == LCK_ByRef) {
8221         LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second);
8222         LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8223                                    VDLVal.getPointer(CGF));
8224         BasePointers.push_back(VarLVal.getPointer(CGF));
8225         Pointers.push_back(VarLValVal.getPointer(CGF));
8226         Sizes.push_back(CGF.Builder.CreateIntCast(
8227             CGF.getTypeSize(
8228                 VD->getType().getCanonicalType().getNonReferenceType()),
8229             CGF.Int64Ty, /*isSigned=*/true));
8230       } else {
8231         RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation());
8232         LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8233                                    VDLVal.getPointer(CGF));
8234         BasePointers.push_back(VarLVal.getPointer(CGF));
8235         Pointers.push_back(VarRVal.getScalarVal());
8236         Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
8237       }
8238       Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8239                       OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8240     }
8241   }
8242 
8243   /// Set correct indices for lambdas captures.
8244   void adjustMemberOfForLambdaCaptures(
8245       const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8246       MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8247       MapFlagsArrayTy &Types) const {
8248     for (unsigned I = 0, E = Types.size(); I < E; ++I) {
8249       // Set correct member_of idx for all implicit lambda captures.
8250       if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8251                        OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT))
8252         continue;
8253       llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
8254       assert(BasePtr && "Unable to find base lambda address.");
8255       int TgtIdx = -1;
8256       for (unsigned J = I; J > 0; --J) {
8257         unsigned Idx = J - 1;
8258         if (Pointers[Idx] != BasePtr)
8259           continue;
8260         TgtIdx = Idx;
8261         break;
8262       }
8263       assert(TgtIdx != -1 && "Unable to find parent lambda.");
8264       // All other current entries will be MEMBER_OF the combined entry
8265       // (except for PTR_AND_OBJ entries which do not have a placeholder value
8266       // 0xFFFF in the MEMBER_OF field).
8267       OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx);
8268       setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8269     }
8270   }
8271 
8272   /// Generate the base pointers, section pointers, sizes and map types
8273   /// associated to a given capture.
8274   void generateInfoForCapture(const CapturedStmt::Capture *Cap,
8275                               llvm::Value *Arg,
8276                               MapBaseValuesArrayTy &BasePointers,
8277                               MapValuesArrayTy &Pointers,
8278                               MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
8279                               StructRangeInfoTy &PartialStruct) const {
8280     assert(!Cap->capturesVariableArrayType() &&
8281            "Not expecting to generate map info for a variable array type!");
8282 
8283     // We need to know when we generating information for the first component
8284     const ValueDecl *VD = Cap->capturesThis()
8285                               ? nullptr
8286                               : Cap->getCapturedVar()->getCanonicalDecl();
8287 
8288     // If this declaration appears in a is_device_ptr clause we just have to
8289     // pass the pointer by value. If it is a reference to a declaration, we just
8290     // pass its value.
8291     if (DevPointersMap.count(VD)) {
8292       BasePointers.emplace_back(Arg, VD);
8293       Pointers.push_back(Arg);
8294       Sizes.push_back(
8295           CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
8296                                     CGF.Int64Ty, /*isSigned=*/true));
8297       Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
8298       return;
8299     }
8300 
8301     using MapData =
8302         std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
8303                    OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, bool>;
8304     SmallVector<MapData, 4> DeclComponentLists;
8305     assert(CurDir.is<const OMPExecutableDirective *>() &&
8306            "Expect a executable directive");
8307     const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
8308     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
8309       for (const auto L : C->decl_component_lists(VD)) {
8310         assert(L.first == VD &&
8311                "We got information for the wrong declaration??");
8312         assert(!L.second.empty() &&
8313                "Not expecting declaration with no component lists.");
8314         DeclComponentLists.emplace_back(L.second, C->getMapType(),
8315                                         C->getMapTypeModifiers(),
8316                                         C->isImplicit());
8317       }
8318     }
8319 
8320     // Find overlapping elements (including the offset from the base element).
8321     llvm::SmallDenseMap<
8322         const MapData *,
8323         llvm::SmallVector<
8324             OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
8325         4>
8326         OverlappedData;
8327     size_t Count = 0;
8328     for (const MapData &L : DeclComponentLists) {
8329       OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8330       OpenMPMapClauseKind MapType;
8331       ArrayRef<OpenMPMapModifierKind> MapModifiers;
8332       bool IsImplicit;
8333       std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8334       ++Count;
8335       for (const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) {
8336         OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
8337         std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1;
8338         auto CI = Components.rbegin();
8339         auto CE = Components.rend();
8340         auto SI = Components1.rbegin();
8341         auto SE = Components1.rend();
8342         for (; CI != CE && SI != SE; ++CI, ++SI) {
8343           if (CI->getAssociatedExpression()->getStmtClass() !=
8344               SI->getAssociatedExpression()->getStmtClass())
8345             break;
8346           // Are we dealing with different variables/fields?
8347           if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8348             break;
8349         }
8350         // Found overlapping if, at least for one component, reached the head of
8351         // the components list.
8352         if (CI == CE || SI == SE) {
8353           assert((CI != CE || SI != SE) &&
8354                  "Unexpected full match of the mapping components.");
8355           const MapData &BaseData = CI == CE ? L : L1;
8356           OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
8357               SI == SE ? Components : Components1;
8358           auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8359           OverlappedElements.getSecond().push_back(SubData);
8360         }
8361       }
8362     }
8363     // Sort the overlapped elements for each item.
8364     llvm::SmallVector<const FieldDecl *, 4> Layout;
8365     if (!OverlappedData.empty()) {
8366       if (const auto *CRD =
8367               VD->getType().getCanonicalType()->getAsCXXRecordDecl())
8368         getPlainLayout(CRD, Layout, /*AsBase=*/false);
8369       else {
8370         const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl();
8371         Layout.append(RD->field_begin(), RD->field_end());
8372       }
8373     }
8374     for (auto &Pair : OverlappedData) {
8375       llvm::sort(
8376           Pair.getSecond(),
8377           [&Layout](
8378               OMPClauseMappableExprCommon::MappableExprComponentListRef First,
8379               OMPClauseMappableExprCommon::MappableExprComponentListRef
8380                   Second) {
8381             auto CI = First.rbegin();
8382             auto CE = First.rend();
8383             auto SI = Second.rbegin();
8384             auto SE = Second.rend();
8385             for (; CI != CE && SI != SE; ++CI, ++SI) {
8386               if (CI->getAssociatedExpression()->getStmtClass() !=
8387                   SI->getAssociatedExpression()->getStmtClass())
8388                 break;
8389               // Are we dealing with different variables/fields?
8390               if (CI->getAssociatedDeclaration() !=
8391                   SI->getAssociatedDeclaration())
8392                 break;
8393             }
8394 
8395             // Lists contain the same elements.
8396             if (CI == CE && SI == SE)
8397               return false;
8398 
8399             // List with less elements is less than list with more elements.
8400             if (CI == CE || SI == SE)
8401               return CI == CE;
8402 
8403             const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8404             const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8405             if (FD1->getParent() == FD2->getParent())
8406               return FD1->getFieldIndex() < FD2->getFieldIndex();
8407             const auto It =
8408                 llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) {
8409                   return FD == FD1 || FD == FD2;
8410                 });
8411             return *It == FD1;
8412           });
8413     }
8414 
8415     // Associated with a capture, because the mapping flags depend on it.
8416     // Go through all of the elements with the overlapped elements.
8417     for (const auto &Pair : OverlappedData) {
8418       const MapData &L = *Pair.getFirst();
8419       OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8420       OpenMPMapClauseKind MapType;
8421       ArrayRef<OpenMPMapModifierKind> MapModifiers;
8422       bool IsImplicit;
8423       std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8424       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
8425           OverlappedComponents = Pair.getSecond();
8426       bool IsFirstComponentList = true;
8427       generateInfoForComponentList(MapType, MapModifiers, Components,
8428                                    BasePointers, Pointers, Sizes, Types,
8429                                    PartialStruct, IsFirstComponentList,
8430                                    IsImplicit, OverlappedComponents);
8431     }
8432     // Go through other elements without overlapped elements.
8433     bool IsFirstComponentList = OverlappedData.empty();
8434     for (const MapData &L : DeclComponentLists) {
8435       OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8436       OpenMPMapClauseKind MapType;
8437       ArrayRef<OpenMPMapModifierKind> MapModifiers;
8438       bool IsImplicit;
8439       std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8440       auto It = OverlappedData.find(&L);
8441       if (It == OverlappedData.end())
8442         generateInfoForComponentList(MapType, MapModifiers, Components,
8443                                      BasePointers, Pointers, Sizes, Types,
8444                                      PartialStruct, IsFirstComponentList,
8445                                      IsImplicit);
8446       IsFirstComponentList = false;
8447     }
8448   }
8449 
8450   /// Generate the base pointers, section pointers, sizes and map types
8451   /// associated with the declare target link variables.
8452   void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
8453                                         MapValuesArrayTy &Pointers,
8454                                         MapValuesArrayTy &Sizes,
8455                                         MapFlagsArrayTy &Types) const {
8456     assert(CurDir.is<const OMPExecutableDirective *>() &&
8457            "Expect a executable directive");
8458     const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
8459     // Map other list items in the map clause which are not captured variables
8460     // but "declare target link" global variables.
8461     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
8462       for (const auto L : C->component_lists()) {
8463         if (!L.first)
8464           continue;
8465         const auto *VD = dyn_cast<VarDecl>(L.first);
8466         if (!VD)
8467           continue;
8468         llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8469             OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8470         if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8471             !Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
8472           continue;
8473         StructRangeInfoTy PartialStruct;
8474         generateInfoForComponentList(
8475             C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers,
8476             Pointers, Sizes, Types, PartialStruct,
8477             /*IsFirstComponentList=*/true, C->isImplicit());
8478         assert(!PartialStruct.Base.isValid() &&
8479                "No partial structs for declare target link expected.");
8480       }
8481     }
8482   }
8483 
8484   /// Generate the default map information for a given capture \a CI,
8485   /// record field declaration \a RI and captured value \a CV.
8486   void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
8487                               const FieldDecl &RI, llvm::Value *CV,
8488                               MapBaseValuesArrayTy &CurBasePointers,
8489                               MapValuesArrayTy &CurPointers,
8490                               MapValuesArrayTy &CurSizes,
8491                               MapFlagsArrayTy &CurMapTypes) const {
8492     bool IsImplicit = true;
8493     // Do the default mapping.
8494     if (CI.capturesThis()) {
8495       CurBasePointers.push_back(CV);
8496       CurPointers.push_back(CV);
8497       const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
8498       CurSizes.push_back(
8499           CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()),
8500                                     CGF.Int64Ty, /*isSigned=*/true));
8501       // Default map type.
8502       CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
8503     } else if (CI.capturesVariableByCopy()) {
8504       CurBasePointers.push_back(CV);
8505       CurPointers.push_back(CV);
8506       if (!RI.getType()->isAnyPointerType()) {
8507         // We have to signal to the runtime captures passed by value that are
8508         // not pointers.
8509         CurMapTypes.push_back(OMP_MAP_LITERAL);
8510         CurSizes.push_back(CGF.Builder.CreateIntCast(
8511             CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true));
8512       } else {
8513         // Pointers are implicitly mapped with a zero size and no flags
8514         // (other than first map that is added for all implicit maps).
8515         CurMapTypes.push_back(OMP_MAP_NONE);
8516         CurSizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
8517       }
8518       const VarDecl *VD = CI.getCapturedVar();
8519       auto I = FirstPrivateDecls.find(VD);
8520       if (I != FirstPrivateDecls.end())
8521         IsImplicit = I->getSecond();
8522     } else {
8523       assert(CI.capturesVariable() && "Expected captured reference.");
8524       const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
8525       QualType ElementType = PtrTy->getPointeeType();
8526       CurSizes.push_back(CGF.Builder.CreateIntCast(
8527           CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true));
8528       // The default map type for a scalar/complex type is 'to' because by
8529       // default the value doesn't have to be retrieved. For an aggregate
8530       // type, the default is 'tofrom'.
8531       CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
8532       const VarDecl *VD = CI.getCapturedVar();
8533       auto I = FirstPrivateDecls.find(VD);
8534       if (I != FirstPrivateDecls.end() &&
8535           VD->getType().isConstant(CGF.getContext())) {
8536         llvm::Constant *Addr =
8537             CGF.CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(CGF, VD);
8538         // Copy the value of the original variable to the new global copy.
8539         CGF.Builder.CreateMemCpy(
8540             CGF.MakeNaturalAlignAddrLValue(Addr, ElementType).getAddress(CGF),
8541             Address(CV, CGF.getContext().getTypeAlignInChars(ElementType)),
8542             CurSizes.back(), /*IsVolatile=*/false);
8543         // Use new global variable as the base pointers.
8544         CurBasePointers.push_back(Addr);
8545         CurPointers.push_back(Addr);
8546       } else {
8547         CurBasePointers.push_back(CV);
8548         if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {
8549           Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
8550               CV, ElementType, CGF.getContext().getDeclAlign(VD),
8551               AlignmentSource::Decl));
8552           CurPointers.push_back(PtrAddr.getPointer());
8553         } else {
8554           CurPointers.push_back(CV);
8555         }
8556       }
8557       if (I != FirstPrivateDecls.end())
8558         IsImplicit = I->getSecond();
8559     }
8560     // Every default map produces a single argument which is a target parameter.
8561     CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
8562 
8563     // Add flag stating this is an implicit map.
8564     if (IsImplicit)
8565       CurMapTypes.back() |= OMP_MAP_IMPLICIT;
8566   }
8567 };
8568 } // anonymous namespace
8569 
8570 /// Emit the arrays used to pass the captures and map information to the
8571 /// offloading runtime library. If there is no map or capture information,
8572 /// return nullptr by reference.
8573 static void
8574 emitOffloadingArrays(CodeGenFunction &CGF,
8575                      MappableExprsHandler::MapBaseValuesArrayTy &BasePointers,
8576                      MappableExprsHandler::MapValuesArrayTy &Pointers,
8577                      MappableExprsHandler::MapValuesArrayTy &Sizes,
8578                      MappableExprsHandler::MapFlagsArrayTy &MapTypes,
8579                      CGOpenMPRuntime::TargetDataInfo &Info) {
8580   CodeGenModule &CGM = CGF.CGM;
8581   ASTContext &Ctx = CGF.getContext();
8582 
8583   // Reset the array information.
8584   Info.clearArrayInfo();
8585   Info.NumberOfPtrs = BasePointers.size();
8586 
8587   if (Info.NumberOfPtrs) {
8588     // Detect if we have any capture size requiring runtime evaluation of the
8589     // size so that a constant array could be eventually used.
8590     bool hasRuntimeEvaluationCaptureSize = false;
8591     for (llvm::Value *S : Sizes)
8592       if (!isa<llvm::Constant>(S)) {
8593         hasRuntimeEvaluationCaptureSize = true;
8594         break;
8595       }
8596 
8597     llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
8598     QualType PointerArrayType = Ctx.getConstantArrayType(
8599         Ctx.VoidPtrTy, PointerNumAP, nullptr, ArrayType::Normal,
8600         /*IndexTypeQuals=*/0);
8601 
8602     Info.BasePointersArray =
8603         CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
8604     Info.PointersArray =
8605         CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
8606 
8607     // If we don't have any VLA types or other types that require runtime
8608     // evaluation, we can use a constant array for the map sizes, otherwise we
8609     // need to fill up the arrays as we do for the pointers.
8610     QualType Int64Ty =
8611         Ctx.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
8612     if (hasRuntimeEvaluationCaptureSize) {
8613       QualType SizeArrayType = Ctx.getConstantArrayType(
8614           Int64Ty, PointerNumAP, nullptr, ArrayType::Normal,
8615           /*IndexTypeQuals=*/0);
8616       Info.SizesArray =
8617           CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
8618     } else {
8619       // We expect all the sizes to be constant, so we collect them to create
8620       // a constant array.
8621       SmallVector<llvm::Constant *, 16> ConstSizes;
8622       for (llvm::Value *S : Sizes)
8623         ConstSizes.push_back(cast<llvm::Constant>(S));
8624 
8625       auto *SizesArrayInit = llvm::ConstantArray::get(
8626           llvm::ArrayType::get(CGM.Int64Ty, ConstSizes.size()), ConstSizes);
8627       std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
8628       auto *SizesArrayGbl = new llvm::GlobalVariable(
8629           CGM.getModule(), SizesArrayInit->getType(),
8630           /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8631           SizesArrayInit, Name);
8632       SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8633       Info.SizesArray = SizesArrayGbl;
8634     }
8635 
8636     // The map types are always constant so we don't need to generate code to
8637     // fill arrays. Instead, we create an array constant.
8638     SmallVector<uint64_t, 4> Mapping(MapTypes.size(), 0);
8639     llvm::copy(MapTypes, Mapping.begin());
8640     llvm::Constant *MapTypesArrayInit =
8641         llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping);
8642     std::string MaptypesName =
8643         CGM.getOpenMPRuntime().getName({"offload_maptypes"});
8644     auto *MapTypesArrayGbl = new llvm::GlobalVariable(
8645         CGM.getModule(), MapTypesArrayInit->getType(),
8646         /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
8647         MapTypesArrayInit, MaptypesName);
8648     MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8649     Info.MapTypesArray = MapTypesArrayGbl;
8650 
8651     for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
8652       llvm::Value *BPVal = *BasePointers[I];
8653       llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
8654           llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8655           Info.BasePointersArray, 0, I);
8656       BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8657           BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
8658       Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8659       CGF.Builder.CreateStore(BPVal, BPAddr);
8660 
8661       if (Info.requiresDevicePointerInfo())
8662         if (const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
8663           Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
8664 
8665       llvm::Value *PVal = Pointers[I];
8666       llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
8667           llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8668           Info.PointersArray, 0, I);
8669       P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8670           P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
8671       Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
8672       CGF.Builder.CreateStore(PVal, PAddr);
8673 
8674       if (hasRuntimeEvaluationCaptureSize) {
8675         llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
8676             llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
8677             Info.SizesArray,
8678             /*Idx0=*/0,
8679             /*Idx1=*/I);
8680         Address SAddr(S, Ctx.getTypeAlignInChars(Int64Ty));
8681         CGF.Builder.CreateStore(
8682             CGF.Builder.CreateIntCast(Sizes[I], CGM.Int64Ty, /*isSigned=*/true),
8683             SAddr);
8684       }
8685     }
8686   }
8687 }
8688 
8689 /// Emit the arguments to be passed to the runtime library based on the
8690 /// arrays of pointers, sizes and map types.
8691 static void emitOffloadingArraysArgument(
8692     CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
8693     llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg,
8694     llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
8695   CodeGenModule &CGM = CGF.CGM;
8696   if (Info.NumberOfPtrs) {
8697     BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8698         llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8699         Info.BasePointersArray,
8700         /*Idx0=*/0, /*Idx1=*/0);
8701     PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8702         llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
8703         Info.PointersArray,
8704         /*Idx0=*/0,
8705         /*Idx1=*/0);
8706     SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8707         llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs), Info.SizesArray,
8708         /*Idx0=*/0, /*Idx1=*/0);
8709     MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
8710         llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
8711         Info.MapTypesArray,
8712         /*Idx0=*/0,
8713         /*Idx1=*/0);
8714   } else {
8715     BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8716     PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
8717     SizesArrayArg = llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
8718     MapTypesArrayArg =
8719         llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
8720   }
8721 }
8722 
8723 /// Check for inner distribute directive.
8724 static const OMPExecutableDirective *
8725 getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
8726   const auto *CS = D.getInnermostCapturedStmt();
8727   const auto *Body =
8728       CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
8729   const Stmt *ChildStmt =
8730       CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
8731 
8732   if (const auto *NestedDir =
8733           dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8734     OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
8735     switch (D.getDirectiveKind()) {
8736     case OMPD_target:
8737       if (isOpenMPDistributeDirective(DKind))
8738         return NestedDir;
8739       if (DKind == OMPD_teams) {
8740         Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8741             /*IgnoreCaptured=*/true);
8742         if (!Body)
8743           return nullptr;
8744         ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
8745         if (const auto *NND =
8746                 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8747           DKind = NND->getDirectiveKind();
8748           if (isOpenMPDistributeDirective(DKind))
8749             return NND;
8750         }
8751       }
8752       return nullptr;
8753     case OMPD_target_teams:
8754       if (isOpenMPDistributeDirective(DKind))
8755         return NestedDir;
8756       return nullptr;
8757     case OMPD_target_parallel:
8758     case OMPD_target_simd:
8759     case OMPD_target_parallel_for:
8760     case OMPD_target_parallel_for_simd:
8761       return nullptr;
8762     case OMPD_target_teams_distribute:
8763     case OMPD_target_teams_distribute_simd:
8764     case OMPD_target_teams_distribute_parallel_for:
8765     case OMPD_target_teams_distribute_parallel_for_simd:
8766     case OMPD_parallel:
8767     case OMPD_for:
8768     case OMPD_parallel_for:
8769     case OMPD_parallel_master:
8770     case OMPD_parallel_sections:
8771     case OMPD_for_simd:
8772     case OMPD_parallel_for_simd:
8773     case OMPD_cancel:
8774     case OMPD_cancellation_point:
8775     case OMPD_ordered:
8776     case OMPD_threadprivate:
8777     case OMPD_allocate:
8778     case OMPD_task:
8779     case OMPD_simd:
8780     case OMPD_sections:
8781     case OMPD_section:
8782     case OMPD_single:
8783     case OMPD_master:
8784     case OMPD_critical:
8785     case OMPD_taskyield:
8786     case OMPD_barrier:
8787     case OMPD_taskwait:
8788     case OMPD_taskgroup:
8789     case OMPD_atomic:
8790     case OMPD_flush:
8791     case OMPD_teams:
8792     case OMPD_target_data:
8793     case OMPD_target_exit_data:
8794     case OMPD_target_enter_data:
8795     case OMPD_distribute:
8796     case OMPD_distribute_simd:
8797     case OMPD_distribute_parallel_for:
8798     case OMPD_distribute_parallel_for_simd:
8799     case OMPD_teams_distribute:
8800     case OMPD_teams_distribute_simd:
8801     case OMPD_teams_distribute_parallel_for:
8802     case OMPD_teams_distribute_parallel_for_simd:
8803     case OMPD_target_update:
8804     case OMPD_declare_simd:
8805     case OMPD_declare_variant:
8806     case OMPD_declare_target:
8807     case OMPD_end_declare_target:
8808     case OMPD_declare_reduction:
8809     case OMPD_declare_mapper:
8810     case OMPD_taskloop:
8811     case OMPD_taskloop_simd:
8812     case OMPD_master_taskloop:
8813     case OMPD_master_taskloop_simd:
8814     case OMPD_parallel_master_taskloop:
8815     case OMPD_parallel_master_taskloop_simd:
8816     case OMPD_requires:
8817     case OMPD_unknown:
8818       llvm_unreachable("Unexpected directive.");
8819     }
8820   }
8821 
8822   return nullptr;
8823 }
8824 
8825 /// Emit the user-defined mapper function. The code generation follows the
8826 /// pattern in the example below.
8827 /// \code
8828 /// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
8829 ///                                           void *base, void *begin,
8830 ///                                           int64_t size, int64_t type) {
8831 ///   // Allocate space for an array section first.
8832 ///   if (size > 1 && !maptype.IsDelete)
8833 ///     __tgt_push_mapper_component(rt_mapper_handle, base, begin,
8834 ///                                 size*sizeof(Ty), clearToFrom(type));
8835 ///   // Map members.
8836 ///   for (unsigned i = 0; i < size; i++) {
8837 ///     // For each component specified by this mapper:
8838 ///     for (auto c : all_components) {
8839 ///       if (c.hasMapper())
8840 ///         (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
8841 ///                       c.arg_type);
8842 ///       else
8843 ///         __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
8844 ///                                     c.arg_begin, c.arg_size, c.arg_type);
8845 ///     }
8846 ///   }
8847 ///   // Delete the array section.
8848 ///   if (size > 1 && maptype.IsDelete)
8849 ///     __tgt_push_mapper_component(rt_mapper_handle, base, begin,
8850 ///                                 size*sizeof(Ty), clearToFrom(type));
8851 /// }
8852 /// \endcode
8853 void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
8854                                             CodeGenFunction *CGF) {
8855   if (UDMMap.count(D) > 0)
8856     return;
8857   ASTContext &C = CGM.getContext();
8858   QualType Ty = D->getType();
8859   QualType PtrTy = C.getPointerType(Ty).withRestrict();
8860   QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
8861   auto *MapperVarDecl =
8862       cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl());
8863   SourceLocation Loc = D->getLocation();
8864   CharUnits ElementSize = C.getTypeSizeInChars(Ty);
8865 
8866   // Prepare mapper function arguments and attributes.
8867   ImplicitParamDecl HandleArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
8868                               C.VoidPtrTy, ImplicitParamDecl::Other);
8869   ImplicitParamDecl BaseArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
8870                             ImplicitParamDecl::Other);
8871   ImplicitParamDecl BeginArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
8872                              C.VoidPtrTy, ImplicitParamDecl::Other);
8873   ImplicitParamDecl SizeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty,
8874                             ImplicitParamDecl::Other);
8875   ImplicitParamDecl TypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty,
8876                             ImplicitParamDecl::Other);
8877   FunctionArgList Args;
8878   Args.push_back(&HandleArg);
8879   Args.push_back(&BaseArg);
8880   Args.push_back(&BeginArg);
8881   Args.push_back(&SizeArg);
8882   Args.push_back(&TypeArg);
8883   const CGFunctionInfo &FnInfo =
8884       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
8885   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
8886   SmallString<64> TyStr;
8887   llvm::raw_svector_ostream Out(TyStr);
8888   CGM.getCXXABI().getMangleContext().mangleTypeName(Ty, Out);
8889   std::string Name = getName({"omp_mapper", TyStr, D->getName()});
8890   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
8891                                     Name, &CGM.getModule());
8892   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
8893   Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
8894   // Start the mapper function code generation.
8895   CodeGenFunction MapperCGF(CGM);
8896   MapperCGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
8897   // Compute the starting and end addreses of array elements.
8898   llvm::Value *Size = MapperCGF.EmitLoadOfScalar(
8899       MapperCGF.GetAddrOfLocalVar(&SizeArg), /*Volatile=*/false,
8900       C.getPointerType(Int64Ty), Loc);
8901   llvm::Value *PtrBegin = MapperCGF.Builder.CreateBitCast(
8902       MapperCGF.GetAddrOfLocalVar(&BeginArg).getPointer(),
8903       CGM.getTypes().ConvertTypeForMem(C.getPointerType(PtrTy)));
8904   llvm::Value *PtrEnd = MapperCGF.Builder.CreateGEP(PtrBegin, Size);
8905   llvm::Value *MapType = MapperCGF.EmitLoadOfScalar(
8906       MapperCGF.GetAddrOfLocalVar(&TypeArg), /*Volatile=*/false,
8907       C.getPointerType(Int64Ty), Loc);
8908   // Prepare common arguments for array initiation and deletion.
8909   llvm::Value *Handle = MapperCGF.EmitLoadOfScalar(
8910       MapperCGF.GetAddrOfLocalVar(&HandleArg),
8911       /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
8912   llvm::Value *BaseIn = MapperCGF.EmitLoadOfScalar(
8913       MapperCGF.GetAddrOfLocalVar(&BaseArg),
8914       /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
8915   llvm::Value *BeginIn = MapperCGF.EmitLoadOfScalar(
8916       MapperCGF.GetAddrOfLocalVar(&BeginArg),
8917       /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc);
8918 
8919   // Emit array initiation if this is an array section and \p MapType indicates
8920   // that memory allocation is required.
8921   llvm::BasicBlock *HeadBB = MapperCGF.createBasicBlock("omp.arraymap.head");
8922   emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType,
8923                              ElementSize, HeadBB, /*IsInit=*/true);
8924 
8925   // Emit a for loop to iterate through SizeArg of elements and map all of them.
8926 
8927   // Emit the loop header block.
8928   MapperCGF.EmitBlock(HeadBB);
8929   llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.arraymap.body");
8930   llvm::BasicBlock *DoneBB = MapperCGF.createBasicBlock("omp.done");
8931   // Evaluate whether the initial condition is satisfied.
8932   llvm::Value *IsEmpty =
8933       MapperCGF.Builder.CreateICmpEQ(PtrBegin, PtrEnd, "omp.arraymap.isempty");
8934   MapperCGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
8935   llvm::BasicBlock *EntryBB = MapperCGF.Builder.GetInsertBlock();
8936 
8937   // Emit the loop body block.
8938   MapperCGF.EmitBlock(BodyBB);
8939   llvm::PHINode *PtrPHI = MapperCGF.Builder.CreatePHI(
8940       PtrBegin->getType(), 2, "omp.arraymap.ptrcurrent");
8941   PtrPHI->addIncoming(PtrBegin, EntryBB);
8942   Address PtrCurrent =
8943       Address(PtrPHI, MapperCGF.GetAddrOfLocalVar(&BeginArg)
8944                           .getAlignment()
8945                           .alignmentOfArrayElement(ElementSize));
8946   // Privatize the declared variable of mapper to be the current array element.
8947   CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
8948   Scope.addPrivate(MapperVarDecl, [&MapperCGF, PtrCurrent, PtrTy]() {
8949     return MapperCGF
8950         .EmitLoadOfPointerLValue(PtrCurrent, PtrTy->castAs<PointerType>())
8951         .getAddress(MapperCGF);
8952   });
8953   (void)Scope.Privatize();
8954 
8955   // Get map clause information. Fill up the arrays with all mapped variables.
8956   MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
8957   MappableExprsHandler::MapValuesArrayTy Pointers;
8958   MappableExprsHandler::MapValuesArrayTy Sizes;
8959   MappableExprsHandler::MapFlagsArrayTy MapTypes;
8960   MappableExprsHandler MEHandler(*D, MapperCGF);
8961   MEHandler.generateAllInfoForMapper(BasePointers, Pointers, Sizes, MapTypes);
8962 
8963   // Call the runtime API __tgt_mapper_num_components to get the number of
8964   // pre-existing components.
8965   llvm::Value *OffloadingArgs[] = {Handle};
8966   llvm::Value *PreviousSize = MapperCGF.EmitRuntimeCall(
8967       createRuntimeFunction(OMPRTL__tgt_mapper_num_components), OffloadingArgs);
8968   llvm::Value *ShiftedPreviousSize = MapperCGF.Builder.CreateShl(
8969       PreviousSize,
8970       MapperCGF.Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
8971 
8972   // Fill up the runtime mapper handle for all components.
8973   for (unsigned I = 0; I < BasePointers.size(); ++I) {
8974     llvm::Value *CurBaseArg = MapperCGF.Builder.CreateBitCast(
8975         *BasePointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
8976     llvm::Value *CurBeginArg = MapperCGF.Builder.CreateBitCast(
8977         Pointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
8978     llvm::Value *CurSizeArg = Sizes[I];
8979 
8980     // Extract the MEMBER_OF field from the map type.
8981     llvm::BasicBlock *MemberBB = MapperCGF.createBasicBlock("omp.member");
8982     MapperCGF.EmitBlock(MemberBB);
8983     llvm::Value *OriMapType = MapperCGF.Builder.getInt64(MapTypes[I]);
8984     llvm::Value *Member = MapperCGF.Builder.CreateAnd(
8985         OriMapType,
8986         MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_MEMBER_OF));
8987     llvm::BasicBlock *MemberCombineBB =
8988         MapperCGF.createBasicBlock("omp.member.combine");
8989     llvm::BasicBlock *TypeBB = MapperCGF.createBasicBlock("omp.type");
8990     llvm::Value *IsMember = MapperCGF.Builder.CreateIsNull(Member);
8991     MapperCGF.Builder.CreateCondBr(IsMember, TypeBB, MemberCombineBB);
8992     // Add the number of pre-existing components to the MEMBER_OF field if it
8993     // is valid.
8994     MapperCGF.EmitBlock(MemberCombineBB);
8995     llvm::Value *CombinedMember =
8996         MapperCGF.Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
8997     // Do nothing if it is not a member of previous components.
8998     MapperCGF.EmitBlock(TypeBB);
8999     llvm::PHINode *MemberMapType =
9000         MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.membermaptype");
9001     MemberMapType->addIncoming(OriMapType, MemberBB);
9002     MemberMapType->addIncoming(CombinedMember, MemberCombineBB);
9003 
9004     // Combine the map type inherited from user-defined mapper with that
9005     // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
9006     // bits of the \a MapType, which is the input argument of the mapper
9007     // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
9008     // bits of MemberMapType.
9009     // [OpenMP 5.0], 1.2.6. map-type decay.
9010     //        | alloc |  to   | from  | tofrom | release | delete
9011     // ----------------------------------------------------------
9012     // alloc  | alloc | alloc | alloc | alloc  | release | delete
9013     // to     | alloc |  to   | alloc |   to   | release | delete
9014     // from   | alloc | alloc | from  |  from  | release | delete
9015     // tofrom | alloc |  to   | from  | tofrom | release | delete
9016     llvm::Value *LeftToFrom = MapperCGF.Builder.CreateAnd(
9017         MapType,
9018         MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO |
9019                                    MappableExprsHandler::OMP_MAP_FROM));
9020     llvm::BasicBlock *AllocBB = MapperCGF.createBasicBlock("omp.type.alloc");
9021     llvm::BasicBlock *AllocElseBB =
9022         MapperCGF.createBasicBlock("omp.type.alloc.else");
9023     llvm::BasicBlock *ToBB = MapperCGF.createBasicBlock("omp.type.to");
9024     llvm::BasicBlock *ToElseBB = MapperCGF.createBasicBlock("omp.type.to.else");
9025     llvm::BasicBlock *FromBB = MapperCGF.createBasicBlock("omp.type.from");
9026     llvm::BasicBlock *EndBB = MapperCGF.createBasicBlock("omp.type.end");
9027     llvm::Value *IsAlloc = MapperCGF.Builder.CreateIsNull(LeftToFrom);
9028     MapperCGF.Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9029     // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
9030     MapperCGF.EmitBlock(AllocBB);
9031     llvm::Value *AllocMapType = MapperCGF.Builder.CreateAnd(
9032         MemberMapType,
9033         MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9034                                      MappableExprsHandler::OMP_MAP_FROM)));
9035     MapperCGF.Builder.CreateBr(EndBB);
9036     MapperCGF.EmitBlock(AllocElseBB);
9037     llvm::Value *IsTo = MapperCGF.Builder.CreateICmpEQ(
9038         LeftToFrom,
9039         MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO));
9040     MapperCGF.Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9041     // In case of to, clear OMP_MAP_FROM.
9042     MapperCGF.EmitBlock(ToBB);
9043     llvm::Value *ToMapType = MapperCGF.Builder.CreateAnd(
9044         MemberMapType,
9045         MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_FROM));
9046     MapperCGF.Builder.CreateBr(EndBB);
9047     MapperCGF.EmitBlock(ToElseBB);
9048     llvm::Value *IsFrom = MapperCGF.Builder.CreateICmpEQ(
9049         LeftToFrom,
9050         MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_FROM));
9051     MapperCGF.Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9052     // In case of from, clear OMP_MAP_TO.
9053     MapperCGF.EmitBlock(FromBB);
9054     llvm::Value *FromMapType = MapperCGF.Builder.CreateAnd(
9055         MemberMapType,
9056         MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_TO));
9057     // In case of tofrom, do nothing.
9058     MapperCGF.EmitBlock(EndBB);
9059     llvm::PHINode *CurMapType =
9060         MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.maptype");
9061     CurMapType->addIncoming(AllocMapType, AllocBB);
9062     CurMapType->addIncoming(ToMapType, ToBB);
9063     CurMapType->addIncoming(FromMapType, FromBB);
9064     CurMapType->addIncoming(MemberMapType, ToElseBB);
9065 
9066     // TODO: call the corresponding mapper function if a user-defined mapper is
9067     // associated with this map clause.
9068     // Call the runtime API __tgt_push_mapper_component to fill up the runtime
9069     // data structure.
9070     llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9071                                      CurSizeArg, CurMapType};
9072     MapperCGF.EmitRuntimeCall(
9073         createRuntimeFunction(OMPRTL__tgt_push_mapper_component),
9074         OffloadingArgs);
9075   }
9076 
9077   // Update the pointer to point to the next element that needs to be mapped,
9078   // and check whether we have mapped all elements.
9079   llvm::Value *PtrNext = MapperCGF.Builder.CreateConstGEP1_32(
9080       PtrPHI, /*Idx0=*/1, "omp.arraymap.next");
9081   PtrPHI->addIncoming(PtrNext, BodyBB);
9082   llvm::Value *IsDone =
9083       MapperCGF.Builder.CreateICmpEQ(PtrNext, PtrEnd, "omp.arraymap.isdone");
9084   llvm::BasicBlock *ExitBB = MapperCGF.createBasicBlock("omp.arraymap.exit");
9085   MapperCGF.Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9086 
9087   MapperCGF.EmitBlock(ExitBB);
9088   // Emit array deletion if this is an array section and \p MapType indicates
9089   // that deletion is required.
9090   emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType,
9091                              ElementSize, DoneBB, /*IsInit=*/false);
9092 
9093   // Emit the function exit block.
9094   MapperCGF.EmitBlock(DoneBB, /*IsFinished=*/true);
9095   MapperCGF.FinishFunction();
9096   UDMMap.try_emplace(D, Fn);
9097   if (CGF) {
9098     auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn);
9099     Decls.second.push_back(D);
9100   }
9101 }
9102 
9103 /// Emit the array initialization or deletion portion for user-defined mapper
9104 /// code generation. First, it evaluates whether an array section is mapped and
9105 /// whether the \a MapType instructs to delete this section. If \a IsInit is
9106 /// true, and \a MapType indicates to not delete this array, array
9107 /// initialization code is generated. If \a IsInit is false, and \a MapType
9108 /// indicates to not this array, array deletion code is generated.
9109 void CGOpenMPRuntime::emitUDMapperArrayInitOrDel(
9110     CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *Base,
9111     llvm::Value *Begin, llvm::Value *Size, llvm::Value *MapType,
9112     CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit) {
9113   StringRef Prefix = IsInit ? ".init" : ".del";
9114 
9115   // Evaluate if this is an array section.
9116   llvm::BasicBlock *IsDeleteBB =
9117       MapperCGF.createBasicBlock("omp.array" + Prefix + ".evaldelete");
9118   llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.array" + Prefix);
9119   llvm::Value *IsArray = MapperCGF.Builder.CreateICmpSGE(
9120       Size, MapperCGF.Builder.getInt64(1), "omp.arrayinit.isarray");
9121   MapperCGF.Builder.CreateCondBr(IsArray, IsDeleteBB, ExitBB);
9122 
9123   // Evaluate if we are going to delete this section.
9124   MapperCGF.EmitBlock(IsDeleteBB);
9125   llvm::Value *DeleteBit = MapperCGF.Builder.CreateAnd(
9126       MapType,
9127       MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_DELETE));
9128   llvm::Value *DeleteCond;
9129   if (IsInit) {
9130     DeleteCond = MapperCGF.Builder.CreateIsNull(
9131         DeleteBit, "omp.array" + Prefix + ".delete");
9132   } else {
9133     DeleteCond = MapperCGF.Builder.CreateIsNotNull(
9134         DeleteBit, "omp.array" + Prefix + ".delete");
9135   }
9136   MapperCGF.Builder.CreateCondBr(DeleteCond, BodyBB, ExitBB);
9137 
9138   MapperCGF.EmitBlock(BodyBB);
9139   // Get the array size by multiplying element size and element number (i.e., \p
9140   // Size).
9141   llvm::Value *ArraySize = MapperCGF.Builder.CreateNUWMul(
9142       Size, MapperCGF.Builder.getInt64(ElementSize.getQuantity()));
9143   // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
9144   // memory allocation/deletion purpose only.
9145   llvm::Value *MapTypeArg = MapperCGF.Builder.CreateAnd(
9146       MapType,
9147       MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9148                                    MappableExprsHandler::OMP_MAP_FROM)));
9149   // Call the runtime API __tgt_push_mapper_component to fill up the runtime
9150   // data structure.
9151   llvm::Value *OffloadingArgs[] = {Handle, Base, Begin, ArraySize, MapTypeArg};
9152   MapperCGF.EmitRuntimeCall(
9153       createRuntimeFunction(OMPRTL__tgt_push_mapper_component), OffloadingArgs);
9154 }
9155 
9156 void CGOpenMPRuntime::emitTargetNumIterationsCall(
9157     CodeGenFunction &CGF, const OMPExecutableDirective &D,
9158     llvm::Value *DeviceID,
9159     llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9160                                      const OMPLoopDirective &D)>
9161         SizeEmitter) {
9162   OpenMPDirectiveKind Kind = D.getDirectiveKind();
9163   const OMPExecutableDirective *TD = &D;
9164   // Get nested teams distribute kind directive, if any.
9165   if (!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind))
9166     TD = getNestedDistributeDirective(CGM.getContext(), D);
9167   if (!TD)
9168     return;
9169   const auto *LD = cast<OMPLoopDirective>(TD);
9170   auto &&CodeGen = [LD, DeviceID, SizeEmitter, this](CodeGenFunction &CGF,
9171                                                      PrePostActionTy &) {
9172     if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) {
9173       llvm::Value *Args[] = {DeviceID, NumIterations};
9174       CGF.EmitRuntimeCall(
9175           createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args);
9176     }
9177   };
9178   emitInlinedDirective(CGF, OMPD_unknown, CodeGen);
9179 }
9180 
9181 void CGOpenMPRuntime::emitTargetCall(
9182     CodeGenFunction &CGF, const OMPExecutableDirective &D,
9183     llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
9184     const Expr *Device,
9185     llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
9186                                      const OMPLoopDirective &D)>
9187         SizeEmitter) {
9188   if (!CGF.HaveInsertPoint())
9189     return;
9190 
9191   assert(OutlinedFn && "Invalid outlined function!");
9192 
9193   const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>();
9194   llvm::SmallVector<llvm::Value *, 16> CapturedVars;
9195   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
9196   auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
9197                                             PrePostActionTy &) {
9198     CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9199   };
9200   emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
9201 
9202   CodeGenFunction::OMPTargetDataInfo InputInfo;
9203   llvm::Value *MapTypesArray = nullptr;
9204   // Fill up the pointer arrays and transfer execution to the device.
9205   auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
9206                     &MapTypesArray, &CS, RequiresOuterTask, &CapturedVars,
9207                     SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
9208     // On top of the arrays that were filled up, the target offloading call
9209     // takes as arguments the device id as well as the host pointer. The host
9210     // pointer is used by the runtime library to identify the current target
9211     // region, so it only has to be unique and not necessarily point to
9212     // anything. It could be the pointer to the outlined function that
9213     // implements the target region, but we aren't using that so that the
9214     // compiler doesn't need to keep that, and could therefore inline the host
9215     // function if proven worthwhile during optimization.
9216 
9217     // From this point on, we need to have an ID of the target region defined.
9218     assert(OutlinedFnID && "Invalid outlined function ID!");
9219 
9220     // Emit device ID if any.
9221     llvm::Value *DeviceID;
9222     if (Device) {
9223       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9224                                            CGF.Int64Ty, /*isSigned=*/true);
9225     } else {
9226       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9227     }
9228 
9229     // Emit the number of elements in the offloading arrays.
9230     llvm::Value *PointerNum =
9231         CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
9232 
9233     // Return value of the runtime offloading call.
9234     llvm::Value *Return;
9235 
9236     llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D);
9237     llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D);
9238 
9239     // Emit tripcount for the target loop-based directive.
9240     emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter);
9241 
9242     bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
9243     // The target region is an outlined function launched by the runtime
9244     // via calls __tgt_target() or __tgt_target_teams().
9245     //
9246     // __tgt_target() launches a target region with one team and one thread,
9247     // executing a serial region.  This master thread may in turn launch
9248     // more threads within its team upon encountering a parallel region,
9249     // however, no additional teams can be launched on the device.
9250     //
9251     // __tgt_target_teams() launches a target region with one or more teams,
9252     // each with one or more threads.  This call is required for target
9253     // constructs such as:
9254     //  'target teams'
9255     //  'target' / 'teams'
9256     //  'target teams distribute parallel for'
9257     //  'target parallel'
9258     // and so on.
9259     //
9260     // Note that on the host and CPU targets, the runtime implementation of
9261     // these calls simply call the outlined function without forking threads.
9262     // The outlined functions themselves have runtime calls to
9263     // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
9264     // the compiler in emitTeamsCall() and emitParallelCall().
9265     //
9266     // In contrast, on the NVPTX target, the implementation of
9267     // __tgt_target_teams() launches a GPU kernel with the requested number
9268     // of teams and threads so no additional calls to the runtime are required.
9269     if (NumTeams) {
9270       // If we have NumTeams defined this means that we have an enclosed teams
9271       // region. Therefore we also expect to have NumThreads defined. These two
9272       // values should be defined in the presence of a teams directive,
9273       // regardless of having any clauses associated. If the user is using teams
9274       // but no clauses, these two values will be the default that should be
9275       // passed to the runtime library - a 32-bit integer with the value zero.
9276       assert(NumThreads && "Thread limit expression should be available along "
9277                            "with number of teams.");
9278       llvm::Value *OffloadingArgs[] = {DeviceID,
9279                                        OutlinedFnID,
9280                                        PointerNum,
9281                                        InputInfo.BasePointersArray.getPointer(),
9282                                        InputInfo.PointersArray.getPointer(),
9283                                        InputInfo.SizesArray.getPointer(),
9284                                        MapTypesArray,
9285                                        NumTeams,
9286                                        NumThreads};
9287       Return = CGF.EmitRuntimeCall(
9288           createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_teams_nowait
9289                                           : OMPRTL__tgt_target_teams),
9290           OffloadingArgs);
9291     } else {
9292       llvm::Value *OffloadingArgs[] = {DeviceID,
9293                                        OutlinedFnID,
9294                                        PointerNum,
9295                                        InputInfo.BasePointersArray.getPointer(),
9296                                        InputInfo.PointersArray.getPointer(),
9297                                        InputInfo.SizesArray.getPointer(),
9298                                        MapTypesArray};
9299       Return = CGF.EmitRuntimeCall(
9300           createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_nowait
9301                                           : OMPRTL__tgt_target),
9302           OffloadingArgs);
9303     }
9304 
9305     // Check the error code and execute the host version if required.
9306     llvm::BasicBlock *OffloadFailedBlock =
9307         CGF.createBasicBlock("omp_offload.failed");
9308     llvm::BasicBlock *OffloadContBlock =
9309         CGF.createBasicBlock("omp_offload.cont");
9310     llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
9311     CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
9312 
9313     CGF.EmitBlock(OffloadFailedBlock);
9314     if (RequiresOuterTask) {
9315       CapturedVars.clear();
9316       CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9317     }
9318     emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
9319     CGF.EmitBranch(OffloadContBlock);
9320 
9321     CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
9322   };
9323 
9324   // Notify that the host version must be executed.
9325   auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars,
9326                     RequiresOuterTask](CodeGenFunction &CGF,
9327                                        PrePostActionTy &) {
9328     if (RequiresOuterTask) {
9329       CapturedVars.clear();
9330       CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9331     }
9332     emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars);
9333   };
9334 
9335   auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
9336                           &CapturedVars, RequiresOuterTask,
9337                           &CS](CodeGenFunction &CGF, PrePostActionTy &) {
9338     // Fill up the arrays with all the captured variables.
9339     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9340     MappableExprsHandler::MapValuesArrayTy Pointers;
9341     MappableExprsHandler::MapValuesArrayTy Sizes;
9342     MappableExprsHandler::MapFlagsArrayTy MapTypes;
9343 
9344     // Get mappable expression information.
9345     MappableExprsHandler MEHandler(D, CGF);
9346     llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9347 
9348     auto RI = CS.getCapturedRecordDecl()->field_begin();
9349     auto CV = CapturedVars.begin();
9350     for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
9351                                               CE = CS.capture_end();
9352          CI != CE; ++CI, ++RI, ++CV) {
9353       MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers;
9354       MappableExprsHandler::MapValuesArrayTy CurPointers;
9355       MappableExprsHandler::MapValuesArrayTy CurSizes;
9356       MappableExprsHandler::MapFlagsArrayTy CurMapTypes;
9357       MappableExprsHandler::StructRangeInfoTy PartialStruct;
9358 
9359       // VLA sizes are passed to the outlined region by copy and do not have map
9360       // information associated.
9361       if (CI->capturesVariableArrayType()) {
9362         CurBasePointers.push_back(*CV);
9363         CurPointers.push_back(*CV);
9364         CurSizes.push_back(CGF.Builder.CreateIntCast(
9365             CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
9366         // Copy to the device as an argument. No need to retrieve it.
9367         CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
9368                               MappableExprsHandler::OMP_MAP_TARGET_PARAM |
9369                               MappableExprsHandler::OMP_MAP_IMPLICIT);
9370       } else {
9371         // If we have any information in the map clause, we use it, otherwise we
9372         // just do a default mapping.
9373         MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
9374                                          CurSizes, CurMapTypes, PartialStruct);
9375         if (CurBasePointers.empty())
9376           MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
9377                                            CurPointers, CurSizes, CurMapTypes);
9378         // Generate correct mapping for variables captured by reference in
9379         // lambdas.
9380         if (CI->capturesVariable())
9381           MEHandler.generateInfoForLambdaCaptures(
9382               CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes,
9383               CurMapTypes, LambdaPointers);
9384       }
9385       // We expect to have at least an element of information for this capture.
9386       assert(!CurBasePointers.empty() &&
9387              "Non-existing map pointer for capture!");
9388       assert(CurBasePointers.size() == CurPointers.size() &&
9389              CurBasePointers.size() == CurSizes.size() &&
9390              CurBasePointers.size() == CurMapTypes.size() &&
9391              "Inconsistent map information sizes!");
9392 
9393       // If there is an entry in PartialStruct it means we have a struct with
9394       // individual members mapped. Emit an extra combined entry.
9395       if (PartialStruct.Base.isValid())
9396         MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
9397                                     CurMapTypes, PartialStruct);
9398 
9399       // We need to append the results of this capture to what we already have.
9400       BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
9401       Pointers.append(CurPointers.begin(), CurPointers.end());
9402       Sizes.append(CurSizes.begin(), CurSizes.end());
9403       MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
9404     }
9405     // Adjust MEMBER_OF flags for the lambdas captures.
9406     MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers,
9407                                               Pointers, MapTypes);
9408     // Map other list items in the map clause which are not captured variables
9409     // but "declare target link" global variables.
9410     MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
9411                                                MapTypes);
9412 
9413     TargetDataInfo Info;
9414     // Fill up the arrays and create the arguments.
9415     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
9416     emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
9417                                  Info.PointersArray, Info.SizesArray,
9418                                  Info.MapTypesArray, Info);
9419     InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
9420     InputInfo.BasePointersArray =
9421         Address(Info.BasePointersArray, CGM.getPointerAlign());
9422     InputInfo.PointersArray =
9423         Address(Info.PointersArray, CGM.getPointerAlign());
9424     InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign());
9425     MapTypesArray = Info.MapTypesArray;
9426     if (RequiresOuterTask)
9427       CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9428     else
9429       emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
9430   };
9431 
9432   auto &&TargetElseGen = [this, &ElseGen, &D, RequiresOuterTask](
9433                              CodeGenFunction &CGF, PrePostActionTy &) {
9434     if (RequiresOuterTask) {
9435       CodeGenFunction::OMPTargetDataInfo InputInfo;
9436       CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9437     } else {
9438       emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
9439     }
9440   };
9441 
9442   // If we have a target function ID it means that we need to support
9443   // offloading, otherwise, just execute on the host. We need to execute on host
9444   // regardless of the conditional in the if clause if, e.g., the user do not
9445   // specify target triples.
9446   if (OutlinedFnID) {
9447     if (IfCond) {
9448       emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9449     } else {
9450       RegionCodeGenTy ThenRCG(TargetThenGen);
9451       ThenRCG(CGF);
9452     }
9453   } else {
9454     RegionCodeGenTy ElseRCG(TargetElseGen);
9455     ElseRCG(CGF);
9456   }
9457 }
9458 
9459 void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
9460                                                     StringRef ParentName) {
9461   if (!S)
9462     return;
9463 
9464   // Codegen OMP target directives that offload compute to the device.
9465   bool RequiresDeviceCodegen =
9466       isa<OMPExecutableDirective>(S) &&
9467       isOpenMPTargetExecutionDirective(
9468           cast<OMPExecutableDirective>(S)->getDirectiveKind());
9469 
9470   if (RequiresDeviceCodegen) {
9471     const auto &E = *cast<OMPExecutableDirective>(S);
9472     unsigned DeviceID;
9473     unsigned FileID;
9474     unsigned Line;
9475     getTargetEntryUniqueInfo(CGM.getContext(), E.getBeginLoc(), DeviceID,
9476                              FileID, Line);
9477 
9478     // Is this a target region that should not be emitted as an entry point? If
9479     // so just signal we are done with this target region.
9480     if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
9481                                                             ParentName, Line))
9482       return;
9483 
9484     switch (E.getDirectiveKind()) {
9485     case OMPD_target:
9486       CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
9487                                                    cast<OMPTargetDirective>(E));
9488       break;
9489     case OMPD_target_parallel:
9490       CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
9491           CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9492       break;
9493     case OMPD_target_teams:
9494       CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
9495           CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9496       break;
9497     case OMPD_target_teams_distribute:
9498       CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
9499           CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9500       break;
9501     case OMPD_target_teams_distribute_simd:
9502       CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
9503           CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9504       break;
9505     case OMPD_target_parallel_for:
9506       CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
9507           CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9508       break;
9509     case OMPD_target_parallel_for_simd:
9510       CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
9511           CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9512       break;
9513     case OMPD_target_simd:
9514       CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
9515           CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9516       break;
9517     case OMPD_target_teams_distribute_parallel_for:
9518       CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
9519           CGM, ParentName,
9520           cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9521       break;
9522     case OMPD_target_teams_distribute_parallel_for_simd:
9523       CodeGenFunction::
9524           EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
9525               CGM, ParentName,
9526               cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9527       break;
9528     case OMPD_parallel:
9529     case OMPD_for:
9530     case OMPD_parallel_for:
9531     case OMPD_parallel_master:
9532     case OMPD_parallel_sections:
9533     case OMPD_for_simd:
9534     case OMPD_parallel_for_simd:
9535     case OMPD_cancel:
9536     case OMPD_cancellation_point:
9537     case OMPD_ordered:
9538     case OMPD_threadprivate:
9539     case OMPD_allocate:
9540     case OMPD_task:
9541     case OMPD_simd:
9542     case OMPD_sections:
9543     case OMPD_section:
9544     case OMPD_single:
9545     case OMPD_master:
9546     case OMPD_critical:
9547     case OMPD_taskyield:
9548     case OMPD_barrier:
9549     case OMPD_taskwait:
9550     case OMPD_taskgroup:
9551     case OMPD_atomic:
9552     case OMPD_flush:
9553     case OMPD_teams:
9554     case OMPD_target_data:
9555     case OMPD_target_exit_data:
9556     case OMPD_target_enter_data:
9557     case OMPD_distribute:
9558     case OMPD_distribute_simd:
9559     case OMPD_distribute_parallel_for:
9560     case OMPD_distribute_parallel_for_simd:
9561     case OMPD_teams_distribute:
9562     case OMPD_teams_distribute_simd:
9563     case OMPD_teams_distribute_parallel_for:
9564     case OMPD_teams_distribute_parallel_for_simd:
9565     case OMPD_target_update:
9566     case OMPD_declare_simd:
9567     case OMPD_declare_variant:
9568     case OMPD_declare_target:
9569     case OMPD_end_declare_target:
9570     case OMPD_declare_reduction:
9571     case OMPD_declare_mapper:
9572     case OMPD_taskloop:
9573     case OMPD_taskloop_simd:
9574     case OMPD_master_taskloop:
9575     case OMPD_master_taskloop_simd:
9576     case OMPD_parallel_master_taskloop:
9577     case OMPD_parallel_master_taskloop_simd:
9578     case OMPD_requires:
9579     case OMPD_unknown:
9580       llvm_unreachable("Unknown target directive for OpenMP device codegen.");
9581     }
9582     return;
9583   }
9584 
9585   if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9586     if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9587       return;
9588 
9589     scanForTargetRegionsFunctions(
9590         E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
9591     return;
9592   }
9593 
9594   // If this is a lambda function, look into its body.
9595   if (const auto *L = dyn_cast<LambdaExpr>(S))
9596     S = L->getBody();
9597 
9598   // Keep looking for target regions recursively.
9599   for (const Stmt *II : S->children())
9600     scanForTargetRegionsFunctions(II, ParentName);
9601 }
9602 
9603 bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
9604   // If emitting code for the host, we do not process FD here. Instead we do
9605   // the normal code generation.
9606   if (!CGM.getLangOpts().OpenMPIsDevice) {
9607     if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) {
9608       Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9609           OMPDeclareTargetDeclAttr::getDeviceType(FD);
9610       // Do not emit device_type(nohost) functions for the host.
9611       if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9612         return true;
9613     }
9614     return false;
9615   }
9616 
9617   const ValueDecl *VD = cast<ValueDecl>(GD.getDecl());
9618   StringRef Name = CGM.getMangledName(GD);
9619   // Try to detect target regions in the function.
9620   if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9621     scanForTargetRegionsFunctions(FD->getBody(), Name);
9622     Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
9623         OMPDeclareTargetDeclAttr::getDeviceType(FD);
9624     // Do not emit device_type(nohost) functions for the host.
9625     if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9626       return true;
9627   }
9628 
9629   // Do not to emit function if it is not marked as declare target.
9630   return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9631          AlreadyEmittedTargetFunctions.count(Name) == 0;
9632 }
9633 
9634 bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
9635   if (!CGM.getLangOpts().OpenMPIsDevice)
9636     return false;
9637 
9638   // Check if there are Ctors/Dtors in this declaration and look for target
9639   // regions in it. We use the complete variant to produce the kernel name
9640   // mangling.
9641   QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
9642   if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
9643     for (const CXXConstructorDecl *Ctor : RD->ctors()) {
9644       StringRef ParentName =
9645           CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
9646       scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
9647     }
9648     if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
9649       StringRef ParentName =
9650           CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
9651       scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
9652     }
9653   }
9654 
9655   // Do not to emit variable if it is not marked as declare target.
9656   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9657       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9658           cast<VarDecl>(GD.getDecl()));
9659   if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9660       (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9661        HasRequiresUnifiedSharedMemory)) {
9662     DeferredGlobalVariables.insert(cast<VarDecl>(GD.getDecl()));
9663     return true;
9664   }
9665   return false;
9666 }
9667 
9668 llvm::Constant *
9669 CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF,
9670                                                 const VarDecl *VD) {
9671   assert(VD->getType().isConstant(CGM.getContext()) &&
9672          "Expected constant variable.");
9673   StringRef VarName;
9674   llvm::Constant *Addr;
9675   llvm::GlobalValue::LinkageTypes Linkage;
9676   QualType Ty = VD->getType();
9677   SmallString<128> Buffer;
9678   {
9679     unsigned DeviceID;
9680     unsigned FileID;
9681     unsigned Line;
9682     getTargetEntryUniqueInfo(CGM.getContext(), VD->getLocation(), DeviceID,
9683                              FileID, Line);
9684     llvm::raw_svector_ostream OS(Buffer);
9685     OS << "__omp_offloading_firstprivate_" << llvm::format("_%x", DeviceID)
9686        << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
9687     VarName = OS.str();
9688   }
9689   Linkage = llvm::GlobalValue::InternalLinkage;
9690   Addr =
9691       getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(Ty), VarName,
9692                                   getDefaultFirstprivateAddressSpace());
9693   cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage);
9694   CharUnits VarSize = CGM.getContext().getTypeSizeInChars(Ty);
9695   CGM.addCompilerUsedGlobal(cast<llvm::GlobalValue>(Addr));
9696   OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
9697       VarName, Addr, VarSize,
9698       OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo, Linkage);
9699   return Addr;
9700 }
9701 
9702 void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
9703                                                    llvm::Constant *Addr) {
9704   if (CGM.getLangOpts().OMPTargetTriples.empty() &&
9705       !CGM.getLangOpts().OpenMPIsDevice)
9706     return;
9707   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9708       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9709   if (!Res) {
9710     if (CGM.getLangOpts().OpenMPIsDevice) {
9711       // Register non-target variables being emitted in device code (debug info
9712       // may cause this).
9713       StringRef VarName = CGM.getMangledName(VD);
9714       EmittedNonTargetVariables.try_emplace(VarName, Addr);
9715     }
9716     return;
9717   }
9718   // Register declare target variables.
9719   OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags;
9720   StringRef VarName;
9721   CharUnits VarSize;
9722   llvm::GlobalValue::LinkageTypes Linkage;
9723 
9724   if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9725       !HasRequiresUnifiedSharedMemory) {
9726     Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
9727     VarName = CGM.getMangledName(VD);
9728     if (VD->hasDefinition(CGM.getContext()) != VarDecl::DeclarationOnly) {
9729       VarSize = CGM.getContext().getTypeSizeInChars(VD->getType());
9730       assert(!VarSize.isZero() && "Expected non-zero size of the variable");
9731     } else {
9732       VarSize = CharUnits::Zero();
9733     }
9734     Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false);
9735     // Temp solution to prevent optimizations of the internal variables.
9736     if (CGM.getLangOpts().OpenMPIsDevice && !VD->isExternallyVisible()) {
9737       std::string RefName = getName({VarName, "ref"});
9738       if (!CGM.GetGlobalValue(RefName)) {
9739         llvm::Constant *AddrRef =
9740             getOrCreateInternalVariable(Addr->getType(), RefName);
9741         auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef);
9742         GVAddrRef->setConstant(/*Val=*/true);
9743         GVAddrRef->setLinkage(llvm::GlobalValue::InternalLinkage);
9744         GVAddrRef->setInitializer(Addr);
9745         CGM.addCompilerUsedGlobal(GVAddrRef);
9746       }
9747     }
9748   } else {
9749     assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
9750             (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9751              HasRequiresUnifiedSharedMemory)) &&
9752            "Declare target attribute must link or to with unified memory.");
9753     if (*Res == OMPDeclareTargetDeclAttr::MT_Link)
9754       Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink;
9755     else
9756       Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
9757 
9758     if (CGM.getLangOpts().OpenMPIsDevice) {
9759       VarName = Addr->getName();
9760       Addr = nullptr;
9761     } else {
9762       VarName = getAddrOfDeclareTargetVar(VD).getName();
9763       Addr = cast<llvm::Constant>(getAddrOfDeclareTargetVar(VD).getPointer());
9764     }
9765     VarSize = CGM.getPointerSize();
9766     Linkage = llvm::GlobalValue::WeakAnyLinkage;
9767   }
9768 
9769   OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
9770       VarName, Addr, VarSize, Flags, Linkage);
9771 }
9772 
9773 bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
9774   if (isa<FunctionDecl>(GD.getDecl()) ||
9775       isa<OMPDeclareReductionDecl>(GD.getDecl()))
9776     return emitTargetFunctions(GD);
9777 
9778   return emitTargetGlobalVariable(GD);
9779 }
9780 
9781 void CGOpenMPRuntime::emitDeferredTargetDecls() const {
9782   for (const VarDecl *VD : DeferredGlobalVariables) {
9783     llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
9784         OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9785     if (!Res)
9786       continue;
9787     if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9788         !HasRequiresUnifiedSharedMemory) {
9789       CGM.EmitGlobal(VD);
9790     } else {
9791       assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
9792               (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9793                HasRequiresUnifiedSharedMemory)) &&
9794              "Expected link clause or to clause with unified memory.");
9795       (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
9796     }
9797   }
9798 }
9799 
9800 void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
9801     CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
9802   assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
9803          " Expected target-based directive.");
9804 }
9805 
9806 void CGOpenMPRuntime::checkArchForUnifiedAddressing(
9807     const OMPRequiresDecl *D) {
9808   for (const OMPClause *Clause : D->clauselists()) {
9809     if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
9810       HasRequiresUnifiedSharedMemory = true;
9811       break;
9812     }
9813   }
9814 }
9815 
9816 bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
9817                                                        LangAS &AS) {
9818   if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
9819     return false;
9820   const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
9821   switch(A->getAllocatorType()) {
9822   case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
9823   // Not supported, fallback to the default mem space.
9824   case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
9825   case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
9826   case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
9827   case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
9828   case OMPAllocateDeclAttr::OMPThreadMemAlloc:
9829   case OMPAllocateDeclAttr::OMPConstMemAlloc:
9830   case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
9831     AS = LangAS::Default;
9832     return true;
9833   case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
9834     llvm_unreachable("Expected predefined allocator for the variables with the "
9835                      "static storage.");
9836   }
9837   return false;
9838 }
9839 
9840 bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
9841   return HasRequiresUnifiedSharedMemory;
9842 }
9843 
9844 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
9845     CodeGenModule &CGM)
9846     : CGM(CGM) {
9847   if (CGM.getLangOpts().OpenMPIsDevice) {
9848     SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
9849     CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
9850   }
9851 }
9852 
9853 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
9854   if (CGM.getLangOpts().OpenMPIsDevice)
9855     CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
9856 }
9857 
9858 bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
9859   if (!CGM.getLangOpts().OpenMPIsDevice || !ShouldMarkAsGlobal)
9860     return true;
9861 
9862   StringRef Name = CGM.getMangledName(GD);
9863   const auto *D = cast<FunctionDecl>(GD.getDecl());
9864   // Do not to emit function if it is marked as declare target as it was already
9865   // emitted.
9866   if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
9867     if (D->hasBody() && AlreadyEmittedTargetFunctions.count(Name) == 0) {
9868       if (auto *F = dyn_cast_or_null<llvm::Function>(CGM.GetGlobalValue(Name)))
9869         return !F->isDeclaration();
9870       return false;
9871     }
9872     return true;
9873   }
9874 
9875   return !AlreadyEmittedTargetFunctions.insert(Name).second;
9876 }
9877 
9878 llvm::Function *CGOpenMPRuntime::emitRequiresDirectiveRegFun() {
9879   // If we don't have entries or if we are emitting code for the device, we
9880   // don't need to do anything.
9881   if (CGM.getLangOpts().OMPTargetTriples.empty() ||
9882       CGM.getLangOpts().OpenMPSimd || CGM.getLangOpts().OpenMPIsDevice ||
9883       (OffloadEntriesInfoManager.empty() &&
9884        !HasEmittedDeclareTargetRegion &&
9885        !HasEmittedTargetRegion))
9886     return nullptr;
9887 
9888   // Create and register the function that handles the requires directives.
9889   ASTContext &C = CGM.getContext();
9890 
9891   llvm::Function *RequiresRegFn;
9892   {
9893     CodeGenFunction CGF(CGM);
9894     const auto &FI = CGM.getTypes().arrangeNullaryFunction();
9895     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
9896     std::string ReqName = getName({"omp_offloading", "requires_reg"});
9897     RequiresRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, ReqName, FI);
9898     CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {});
9899     OpenMPOffloadingRequiresDirFlags Flags = OMP_REQ_NONE;
9900     // TODO: check for other requires clauses.
9901     // The requires directive takes effect only when a target region is
9902     // present in the compilation unit. Otherwise it is ignored and not
9903     // passed to the runtime. This avoids the runtime from throwing an error
9904     // for mismatching requires clauses across compilation units that don't
9905     // contain at least 1 target region.
9906     assert((HasEmittedTargetRegion ||
9907             HasEmittedDeclareTargetRegion ||
9908             !OffloadEntriesInfoManager.empty()) &&
9909            "Target or declare target region expected.");
9910     if (HasRequiresUnifiedSharedMemory)
9911       Flags = OMP_REQ_UNIFIED_SHARED_MEMORY;
9912     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_requires),
9913         llvm::ConstantInt::get(CGM.Int64Ty, Flags));
9914     CGF.FinishFunction();
9915   }
9916   return RequiresRegFn;
9917 }
9918 
9919 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
9920                                     const OMPExecutableDirective &D,
9921                                     SourceLocation Loc,
9922                                     llvm::Function *OutlinedFn,
9923                                     ArrayRef<llvm::Value *> CapturedVars) {
9924   if (!CGF.HaveInsertPoint())
9925     return;
9926 
9927   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9928   CodeGenFunction::RunCleanupsScope Scope(CGF);
9929 
9930   // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
9931   llvm::Value *Args[] = {
9932       RTLoc,
9933       CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
9934       CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
9935   llvm::SmallVector<llvm::Value *, 16> RealArgs;
9936   RealArgs.append(std::begin(Args), std::end(Args));
9937   RealArgs.append(CapturedVars.begin(), CapturedVars.end());
9938 
9939   llvm::FunctionCallee RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams);
9940   CGF.EmitRuntimeCall(RTLFn, RealArgs);
9941 }
9942 
9943 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
9944                                          const Expr *NumTeams,
9945                                          const Expr *ThreadLimit,
9946                                          SourceLocation Loc) {
9947   if (!CGF.HaveInsertPoint())
9948     return;
9949 
9950   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
9951 
9952   llvm::Value *NumTeamsVal =
9953       NumTeams
9954           ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
9955                                       CGF.CGM.Int32Ty, /* isSigned = */ true)
9956           : CGF.Builder.getInt32(0);
9957 
9958   llvm::Value *ThreadLimitVal =
9959       ThreadLimit
9960           ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
9961                                       CGF.CGM.Int32Ty, /* isSigned = */ true)
9962           : CGF.Builder.getInt32(0);
9963 
9964   // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
9965   llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
9966                                      ThreadLimitVal};
9967   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
9968                       PushNumTeamsArgs);
9969 }
9970 
9971 void CGOpenMPRuntime::emitTargetDataCalls(
9972     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
9973     const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
9974   if (!CGF.HaveInsertPoint())
9975     return;
9976 
9977   // Action used to replace the default codegen action and turn privatization
9978   // off.
9979   PrePostActionTy NoPrivAction;
9980 
9981   // Generate the code for the opening of the data environment. Capture all the
9982   // arguments of the runtime call by reference because they are used in the
9983   // closing of the region.
9984   auto &&BeginThenGen = [this, &D, Device, &Info,
9985                          &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
9986     // Fill up the arrays with all the mapped variables.
9987     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
9988     MappableExprsHandler::MapValuesArrayTy Pointers;
9989     MappableExprsHandler::MapValuesArrayTy Sizes;
9990     MappableExprsHandler::MapFlagsArrayTy MapTypes;
9991 
9992     // Get map clause information.
9993     MappableExprsHandler MCHandler(D, CGF);
9994     MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9995 
9996     // Fill up the arrays and create the arguments.
9997     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
9998 
9999     llvm::Value *BasePointersArrayArg = nullptr;
10000     llvm::Value *PointersArrayArg = nullptr;
10001     llvm::Value *SizesArrayArg = nullptr;
10002     llvm::Value *MapTypesArrayArg = nullptr;
10003     emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
10004                                  SizesArrayArg, MapTypesArrayArg, Info);
10005 
10006     // Emit device ID if any.
10007     llvm::Value *DeviceID = nullptr;
10008     if (Device) {
10009       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10010                                            CGF.Int64Ty, /*isSigned=*/true);
10011     } else {
10012       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10013     }
10014 
10015     // Emit the number of elements in the offloading arrays.
10016     llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
10017 
10018     llvm::Value *OffloadingArgs[] = {
10019         DeviceID,         PointerNum,    BasePointersArrayArg,
10020         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
10021     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin),
10022                         OffloadingArgs);
10023 
10024     // If device pointer privatization is required, emit the body of the region
10025     // here. It will have to be duplicated: with and without privatization.
10026     if (!Info.CaptureDeviceAddrMap.empty())
10027       CodeGen(CGF);
10028   };
10029 
10030   // Generate code for the closing of the data region.
10031   auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF,
10032                                             PrePostActionTy &) {
10033     assert(Info.isValid() && "Invalid data environment closing arguments.");
10034 
10035     llvm::Value *BasePointersArrayArg = nullptr;
10036     llvm::Value *PointersArrayArg = nullptr;
10037     llvm::Value *SizesArrayArg = nullptr;
10038     llvm::Value *MapTypesArrayArg = nullptr;
10039     emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
10040                                  SizesArrayArg, MapTypesArrayArg, Info);
10041 
10042     // Emit device ID if any.
10043     llvm::Value *DeviceID = nullptr;
10044     if (Device) {
10045       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10046                                            CGF.Int64Ty, /*isSigned=*/true);
10047     } else {
10048       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10049     }
10050 
10051     // Emit the number of elements in the offloading arrays.
10052     llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
10053 
10054     llvm::Value *OffloadingArgs[] = {
10055         DeviceID,         PointerNum,    BasePointersArrayArg,
10056         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
10057     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end),
10058                         OffloadingArgs);
10059   };
10060 
10061   // If we need device pointer privatization, we need to emit the body of the
10062   // region with no privatization in the 'else' branch of the conditional.
10063   // Otherwise, we don't have to do anything.
10064   auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF,
10065                                                          PrePostActionTy &) {
10066     if (!Info.CaptureDeviceAddrMap.empty()) {
10067       CodeGen.setAction(NoPrivAction);
10068       CodeGen(CGF);
10069     }
10070   };
10071 
10072   // We don't have to do anything to close the region if the if clause evaluates
10073   // to false.
10074   auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
10075 
10076   if (IfCond) {
10077     emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
10078   } else {
10079     RegionCodeGenTy RCG(BeginThenGen);
10080     RCG(CGF);
10081   }
10082 
10083   // If we don't require privatization of device pointers, we emit the body in
10084   // between the runtime calls. This avoids duplicating the body code.
10085   if (Info.CaptureDeviceAddrMap.empty()) {
10086     CodeGen.setAction(NoPrivAction);
10087     CodeGen(CGF);
10088   }
10089 
10090   if (IfCond) {
10091     emitIfClause(CGF, IfCond, EndThenGen, EndElseGen);
10092   } else {
10093     RegionCodeGenTy RCG(EndThenGen);
10094     RCG(CGF);
10095   }
10096 }
10097 
10098 void CGOpenMPRuntime::emitTargetDataStandAloneCall(
10099     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
10100     const Expr *Device) {
10101   if (!CGF.HaveInsertPoint())
10102     return;
10103 
10104   assert((isa<OMPTargetEnterDataDirective>(D) ||
10105           isa<OMPTargetExitDataDirective>(D) ||
10106           isa<OMPTargetUpdateDirective>(D)) &&
10107          "Expecting either target enter, exit data, or update directives.");
10108 
10109   CodeGenFunction::OMPTargetDataInfo InputInfo;
10110   llvm::Value *MapTypesArray = nullptr;
10111   // Generate the code for the opening of the data environment.
10112   auto &&ThenGen = [this, &D, Device, &InputInfo,
10113                     &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) {
10114     // Emit device ID if any.
10115     llvm::Value *DeviceID = nullptr;
10116     if (Device) {
10117       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10118                                            CGF.Int64Ty, /*isSigned=*/true);
10119     } else {
10120       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10121     }
10122 
10123     // Emit the number of elements in the offloading arrays.
10124     llvm::Constant *PointerNum =
10125         CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
10126 
10127     llvm::Value *OffloadingArgs[] = {DeviceID,
10128                                      PointerNum,
10129                                      InputInfo.BasePointersArray.getPointer(),
10130                                      InputInfo.PointersArray.getPointer(),
10131                                      InputInfo.SizesArray.getPointer(),
10132                                      MapTypesArray};
10133 
10134     // Select the right runtime function call for each expected standalone
10135     // directive.
10136     const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
10137     OpenMPRTLFunction RTLFn;
10138     switch (D.getDirectiveKind()) {
10139     case OMPD_target_enter_data:
10140       RTLFn = HasNowait ? OMPRTL__tgt_target_data_begin_nowait
10141                         : OMPRTL__tgt_target_data_begin;
10142       break;
10143     case OMPD_target_exit_data:
10144       RTLFn = HasNowait ? OMPRTL__tgt_target_data_end_nowait
10145                         : OMPRTL__tgt_target_data_end;
10146       break;
10147     case OMPD_target_update:
10148       RTLFn = HasNowait ? OMPRTL__tgt_target_data_update_nowait
10149                         : OMPRTL__tgt_target_data_update;
10150       break;
10151     case OMPD_parallel:
10152     case OMPD_for:
10153     case OMPD_parallel_for:
10154     case OMPD_parallel_master:
10155     case OMPD_parallel_sections:
10156     case OMPD_for_simd:
10157     case OMPD_parallel_for_simd:
10158     case OMPD_cancel:
10159     case OMPD_cancellation_point:
10160     case OMPD_ordered:
10161     case OMPD_threadprivate:
10162     case OMPD_allocate:
10163     case OMPD_task:
10164     case OMPD_simd:
10165     case OMPD_sections:
10166     case OMPD_section:
10167     case OMPD_single:
10168     case OMPD_master:
10169     case OMPD_critical:
10170     case OMPD_taskyield:
10171     case OMPD_barrier:
10172     case OMPD_taskwait:
10173     case OMPD_taskgroup:
10174     case OMPD_atomic:
10175     case OMPD_flush:
10176     case OMPD_teams:
10177     case OMPD_target_data:
10178     case OMPD_distribute:
10179     case OMPD_distribute_simd:
10180     case OMPD_distribute_parallel_for:
10181     case OMPD_distribute_parallel_for_simd:
10182     case OMPD_teams_distribute:
10183     case OMPD_teams_distribute_simd:
10184     case OMPD_teams_distribute_parallel_for:
10185     case OMPD_teams_distribute_parallel_for_simd:
10186     case OMPD_declare_simd:
10187     case OMPD_declare_variant:
10188     case OMPD_declare_target:
10189     case OMPD_end_declare_target:
10190     case OMPD_declare_reduction:
10191     case OMPD_declare_mapper:
10192     case OMPD_taskloop:
10193     case OMPD_taskloop_simd:
10194     case OMPD_master_taskloop:
10195     case OMPD_master_taskloop_simd:
10196     case OMPD_parallel_master_taskloop:
10197     case OMPD_parallel_master_taskloop_simd:
10198     case OMPD_target:
10199     case OMPD_target_simd:
10200     case OMPD_target_teams_distribute:
10201     case OMPD_target_teams_distribute_simd:
10202     case OMPD_target_teams_distribute_parallel_for:
10203     case OMPD_target_teams_distribute_parallel_for_simd:
10204     case OMPD_target_teams:
10205     case OMPD_target_parallel:
10206     case OMPD_target_parallel_for:
10207     case OMPD_target_parallel_for_simd:
10208     case OMPD_requires:
10209     case OMPD_unknown:
10210       llvm_unreachable("Unexpected standalone target data directive.");
10211       break;
10212     }
10213     CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs);
10214   };
10215 
10216   auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
10217                              CodeGenFunction &CGF, PrePostActionTy &) {
10218     // Fill up the arrays with all the mapped variables.
10219     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
10220     MappableExprsHandler::MapValuesArrayTy Pointers;
10221     MappableExprsHandler::MapValuesArrayTy Sizes;
10222     MappableExprsHandler::MapFlagsArrayTy MapTypes;
10223 
10224     // Get map clause information.
10225     MappableExprsHandler MEHandler(D, CGF);
10226     MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
10227 
10228     TargetDataInfo Info;
10229     // Fill up the arrays and create the arguments.
10230     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
10231     emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
10232                                  Info.PointersArray, Info.SizesArray,
10233                                  Info.MapTypesArray, Info);
10234     InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
10235     InputInfo.BasePointersArray =
10236         Address(Info.BasePointersArray, CGM.getPointerAlign());
10237     InputInfo.PointersArray =
10238         Address(Info.PointersArray, CGM.getPointerAlign());
10239     InputInfo.SizesArray =
10240         Address(Info.SizesArray, CGM.getPointerAlign());
10241     MapTypesArray = Info.MapTypesArray;
10242     if (D.hasClausesOfKind<OMPDependClause>())
10243       CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
10244     else
10245       emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
10246   };
10247 
10248   if (IfCond) {
10249     emitIfClause(CGF, IfCond, TargetThenGen,
10250                  [](CodeGenFunction &CGF, PrePostActionTy &) {});
10251   } else {
10252     RegionCodeGenTy ThenRCG(TargetThenGen);
10253     ThenRCG(CGF);
10254   }
10255 }
10256 
10257 namespace {
10258   /// Kind of parameter in a function with 'declare simd' directive.
10259   enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
10260   /// Attribute set of the parameter.
10261   struct ParamAttrTy {
10262     ParamKindTy Kind = Vector;
10263     llvm::APSInt StrideOrArg;
10264     llvm::APSInt Alignment;
10265   };
10266 } // namespace
10267 
10268 static unsigned evaluateCDTSize(const FunctionDecl *FD,
10269                                 ArrayRef<ParamAttrTy> ParamAttrs) {
10270   // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
10271   // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
10272   // of that clause. The VLEN value must be power of 2.
10273   // In other case the notion of the function`s "characteristic data type" (CDT)
10274   // is used to compute the vector length.
10275   // CDT is defined in the following order:
10276   //   a) For non-void function, the CDT is the return type.
10277   //   b) If the function has any non-uniform, non-linear parameters, then the
10278   //   CDT is the type of the first such parameter.
10279   //   c) If the CDT determined by a) or b) above is struct, union, or class
10280   //   type which is pass-by-value (except for the type that maps to the
10281   //   built-in complex data type), the characteristic data type is int.
10282   //   d) If none of the above three cases is applicable, the CDT is int.
10283   // The VLEN is then determined based on the CDT and the size of vector
10284   // register of that ISA for which current vector version is generated. The
10285   // VLEN is computed using the formula below:
10286   //   VLEN  = sizeof(vector_register) / sizeof(CDT),
10287   // where vector register size specified in section 3.2.1 Registers and the
10288   // Stack Frame of original AMD64 ABI document.
10289   QualType RetType = FD->getReturnType();
10290   if (RetType.isNull())
10291     return 0;
10292   ASTContext &C = FD->getASTContext();
10293   QualType CDT;
10294   if (!RetType.isNull() && !RetType->isVoidType()) {
10295     CDT = RetType;
10296   } else {
10297     unsigned Offset = 0;
10298     if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10299       if (ParamAttrs[Offset].Kind == Vector)
10300         CDT = C.getPointerType(C.getRecordType(MD->getParent()));
10301       ++Offset;
10302     }
10303     if (CDT.isNull()) {
10304       for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
10305         if (ParamAttrs[I + Offset].Kind == Vector) {
10306           CDT = FD->getParamDecl(I)->getType();
10307           break;
10308         }
10309       }
10310     }
10311   }
10312   if (CDT.isNull())
10313     CDT = C.IntTy;
10314   CDT = CDT->getCanonicalTypeUnqualified();
10315   if (CDT->isRecordType() || CDT->isUnionType())
10316     CDT = C.IntTy;
10317   return C.getTypeSize(CDT);
10318 }
10319 
10320 static void
10321 emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
10322                            const llvm::APSInt &VLENVal,
10323                            ArrayRef<ParamAttrTy> ParamAttrs,
10324                            OMPDeclareSimdDeclAttr::BranchStateTy State) {
10325   struct ISADataTy {
10326     char ISA;
10327     unsigned VecRegSize;
10328   };
10329   ISADataTy ISAData[] = {
10330       {
10331           'b', 128
10332       }, // SSE
10333       {
10334           'c', 256
10335       }, // AVX
10336       {
10337           'd', 256
10338       }, // AVX2
10339       {
10340           'e', 512
10341       }, // AVX512
10342   };
10343   llvm::SmallVector<char, 2> Masked;
10344   switch (State) {
10345   case OMPDeclareSimdDeclAttr::BS_Undefined:
10346     Masked.push_back('N');
10347     Masked.push_back('M');
10348     break;
10349   case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10350     Masked.push_back('N');
10351     break;
10352   case OMPDeclareSimdDeclAttr::BS_Inbranch:
10353     Masked.push_back('M');
10354     break;
10355   }
10356   for (char Mask : Masked) {
10357     for (const ISADataTy &Data : ISAData) {
10358       SmallString<256> Buffer;
10359       llvm::raw_svector_ostream Out(Buffer);
10360       Out << "_ZGV" << Data.ISA << Mask;
10361       if (!VLENVal) {
10362         unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
10363         assert(NumElts && "Non-zero simdlen/cdtsize expected");
10364         Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts);
10365       } else {
10366         Out << VLENVal;
10367       }
10368       for (const ParamAttrTy &ParamAttr : ParamAttrs) {
10369         switch (ParamAttr.Kind){
10370         case LinearWithVarStride:
10371           Out << 's' << ParamAttr.StrideOrArg;
10372           break;
10373         case Linear:
10374           Out << 'l';
10375           if (!!ParamAttr.StrideOrArg)
10376             Out << ParamAttr.StrideOrArg;
10377           break;
10378         case Uniform:
10379           Out << 'u';
10380           break;
10381         case Vector:
10382           Out << 'v';
10383           break;
10384         }
10385         if (!!ParamAttr.Alignment)
10386           Out << 'a' << ParamAttr.Alignment;
10387       }
10388       Out << '_' << Fn->getName();
10389       Fn->addFnAttr(Out.str());
10390     }
10391   }
10392 }
10393 
10394 // This are the Functions that are needed to mangle the name of the
10395 // vector functions generated by the compiler, according to the rules
10396 // defined in the "Vector Function ABI specifications for AArch64",
10397 // available at
10398 // https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
10399 
10400 /// Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
10401 ///
10402 /// TODO: Need to implement the behavior for reference marked with a
10403 /// var or no linear modifiers (1.b in the section). For this, we
10404 /// need to extend ParamKindTy to support the linear modifiers.
10405 static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {
10406   QT = QT.getCanonicalType();
10407 
10408   if (QT->isVoidType())
10409     return false;
10410 
10411   if (Kind == ParamKindTy::Uniform)
10412     return false;
10413 
10414   if (Kind == ParamKindTy::Linear)
10415     return false;
10416 
10417   // TODO: Handle linear references with modifiers
10418 
10419   if (Kind == ParamKindTy::LinearWithVarStride)
10420     return false;
10421 
10422   return true;
10423 }
10424 
10425 /// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
10426 static bool getAArch64PBV(QualType QT, ASTContext &C) {
10427   QT = QT.getCanonicalType();
10428   unsigned Size = C.getTypeSize(QT);
10429 
10430   // Only scalars and complex within 16 bytes wide set PVB to true.
10431   if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10432     return false;
10433 
10434   if (QT->isFloatingType())
10435     return true;
10436 
10437   if (QT->isIntegerType())
10438     return true;
10439 
10440   if (QT->isPointerType())
10441     return true;
10442 
10443   // TODO: Add support for complex types (section 3.1.2, item 2).
10444 
10445   return false;
10446 }
10447 
10448 /// Computes the lane size (LS) of a return type or of an input parameter,
10449 /// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
10450 /// TODO: Add support for references, section 3.2.1, item 1.
10451 static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) {
10452   if (getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
10453     QualType PTy = QT.getCanonicalType()->getPointeeType();
10454     if (getAArch64PBV(PTy, C))
10455       return C.getTypeSize(PTy);
10456   }
10457   if (getAArch64PBV(QT, C))
10458     return C.getTypeSize(QT);
10459 
10460   return C.getTypeSize(C.getUIntPtrType());
10461 }
10462 
10463 // Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
10464 // signature of the scalar function, as defined in 3.2.2 of the
10465 // AAVFABI.
10466 static std::tuple<unsigned, unsigned, bool>
10467 getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) {
10468   QualType RetType = FD->getReturnType().getCanonicalType();
10469 
10470   ASTContext &C = FD->getASTContext();
10471 
10472   bool OutputBecomesInput = false;
10473 
10474   llvm::SmallVector<unsigned, 8> Sizes;
10475   if (!RetType->isVoidType()) {
10476     Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C));
10477     if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {}))
10478       OutputBecomesInput = true;
10479   }
10480   for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
10481     QualType QT = FD->getParamDecl(I)->getType().getCanonicalType();
10482     Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C));
10483   }
10484 
10485   assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
10486   // The LS of a function parameter / return value can only be a power
10487   // of 2, starting from 8 bits, up to 128.
10488   assert(std::all_of(Sizes.begin(), Sizes.end(),
10489                      [](unsigned Size) {
10490                        return Size == 8 || Size == 16 || Size == 32 ||
10491                               Size == 64 || Size == 128;
10492                      }) &&
10493          "Invalid size");
10494 
10495   return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10496                          *std::max_element(std::begin(Sizes), std::end(Sizes)),
10497                          OutputBecomesInput);
10498 }
10499 
10500 /// Mangle the parameter part of the vector function name according to
10501 /// their OpenMP classification. The mangling function is defined in
10502 /// section 3.5 of the AAVFABI.
10503 static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {
10504   SmallString<256> Buffer;
10505   llvm::raw_svector_ostream Out(Buffer);
10506   for (const auto &ParamAttr : ParamAttrs) {
10507     switch (ParamAttr.Kind) {
10508     case LinearWithVarStride:
10509       Out << "ls" << ParamAttr.StrideOrArg;
10510       break;
10511     case Linear:
10512       Out << 'l';
10513       // Don't print the step value if it is not present or if it is
10514       // equal to 1.
10515       if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1)
10516         Out << ParamAttr.StrideOrArg;
10517       break;
10518     case Uniform:
10519       Out << 'u';
10520       break;
10521     case Vector:
10522       Out << 'v';
10523       break;
10524     }
10525 
10526     if (!!ParamAttr.Alignment)
10527       Out << 'a' << ParamAttr.Alignment;
10528   }
10529 
10530   return Out.str();
10531 }
10532 
10533 // Function used to add the attribute. The parameter `VLEN` is
10534 // templated to allow the use of "x" when targeting scalable functions
10535 // for SVE.
10536 template <typename T>
10537 static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
10538                                  char ISA, StringRef ParSeq,
10539                                  StringRef MangledName, bool OutputBecomesInput,
10540                                  llvm::Function *Fn) {
10541   SmallString<256> Buffer;
10542   llvm::raw_svector_ostream Out(Buffer);
10543   Out << Prefix << ISA << LMask << VLEN;
10544   if (OutputBecomesInput)
10545     Out << "v";
10546   Out << ParSeq << "_" << MangledName;
10547   Fn->addFnAttr(Out.str());
10548 }
10549 
10550 // Helper function to generate the Advanced SIMD names depending on
10551 // the value of the NDS when simdlen is not present.
10552 static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
10553                                       StringRef Prefix, char ISA,
10554                                       StringRef ParSeq, StringRef MangledName,
10555                                       bool OutputBecomesInput,
10556                                       llvm::Function *Fn) {
10557   switch (NDS) {
10558   case 8:
10559     addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
10560                          OutputBecomesInput, Fn);
10561     addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName,
10562                          OutputBecomesInput, Fn);
10563     break;
10564   case 16:
10565     addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
10566                          OutputBecomesInput, Fn);
10567     addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
10568                          OutputBecomesInput, Fn);
10569     break;
10570   case 32:
10571     addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
10572                          OutputBecomesInput, Fn);
10573     addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
10574                          OutputBecomesInput, Fn);
10575     break;
10576   case 64:
10577   case 128:
10578     addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
10579                          OutputBecomesInput, Fn);
10580     break;
10581   default:
10582     llvm_unreachable("Scalar type is too wide.");
10583   }
10584 }
10585 
10586 /// Emit vector function attributes for AArch64, as defined in the AAVFABI.
10587 static void emitAArch64DeclareSimdFunction(
10588     CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN,
10589     ArrayRef<ParamAttrTy> ParamAttrs,
10590     OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
10591     char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) {
10592 
10593   // Get basic data for building the vector signature.
10594   const auto Data = getNDSWDS(FD, ParamAttrs);
10595   const unsigned NDS = std::get<0>(Data);
10596   const unsigned WDS = std::get<1>(Data);
10597   const bool OutputBecomesInput = std::get<2>(Data);
10598 
10599   // Check the values provided via `simdlen` by the user.
10600   // 1. A `simdlen(1)` doesn't produce vector signatures,
10601   if (UserVLEN == 1) {
10602     unsigned DiagID = CGM.getDiags().getCustomDiagID(
10603         DiagnosticsEngine::Warning,
10604         "The clause simdlen(1) has no effect when targeting aarch64.");
10605     CGM.getDiags().Report(SLoc, DiagID);
10606     return;
10607   }
10608 
10609   // 2. Section 3.3.1, item 1: user input must be a power of 2 for
10610   // Advanced SIMD output.
10611   if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10612     unsigned DiagID = CGM.getDiags().getCustomDiagID(
10613         DiagnosticsEngine::Warning, "The value specified in simdlen must be a "
10614                                     "power of 2 when targeting Advanced SIMD.");
10615     CGM.getDiags().Report(SLoc, DiagID);
10616     return;
10617   }
10618 
10619   // 3. Section 3.4.1. SVE fixed lengh must obey the architectural
10620   // limits.
10621   if (ISA == 's' && UserVLEN != 0) {
10622     if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10623       unsigned DiagID = CGM.getDiags().getCustomDiagID(
10624           DiagnosticsEngine::Warning, "The clause simdlen must fit the %0-bit "
10625                                       "lanes in the architectural constraints "
10626                                       "for SVE (min is 128-bit, max is "
10627                                       "2048-bit, by steps of 128-bit)");
10628       CGM.getDiags().Report(SLoc, DiagID) << WDS;
10629       return;
10630     }
10631   }
10632 
10633   // Sort out parameter sequence.
10634   const std::string ParSeq = mangleVectorParameters(ParamAttrs);
10635   StringRef Prefix = "_ZGV";
10636   // Generate simdlen from user input (if any).
10637   if (UserVLEN) {
10638     if (ISA == 's') {
10639       // SVE generates only a masked function.
10640       addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10641                            OutputBecomesInput, Fn);
10642     } else {
10643       assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");
10644       // Advanced SIMD generates one or two functions, depending on
10645       // the `[not]inbranch` clause.
10646       switch (State) {
10647       case OMPDeclareSimdDeclAttr::BS_Undefined:
10648         addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
10649                              OutputBecomesInput, Fn);
10650         addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10651                              OutputBecomesInput, Fn);
10652         break;
10653       case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10654         addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
10655                              OutputBecomesInput, Fn);
10656         break;
10657       case OMPDeclareSimdDeclAttr::BS_Inbranch:
10658         addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
10659                              OutputBecomesInput, Fn);
10660         break;
10661       }
10662     }
10663   } else {
10664     // If no user simdlen is provided, follow the AAVFABI rules for
10665     // generating the vector length.
10666     if (ISA == 's') {
10667       // SVE, section 3.4.1, item 1.
10668       addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName,
10669                            OutputBecomesInput, Fn);
10670     } else {
10671       assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");
10672       // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or
10673       // two vector names depending on the use of the clause
10674       // `[not]inbranch`.
10675       switch (State) {
10676       case OMPDeclareSimdDeclAttr::BS_Undefined:
10677         addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
10678                                   OutputBecomesInput, Fn);
10679         addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
10680                                   OutputBecomesInput, Fn);
10681         break;
10682       case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10683         addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
10684                                   OutputBecomesInput, Fn);
10685         break;
10686       case OMPDeclareSimdDeclAttr::BS_Inbranch:
10687         addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
10688                                   OutputBecomesInput, Fn);
10689         break;
10690       }
10691     }
10692   }
10693 }
10694 
10695 void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
10696                                               llvm::Function *Fn) {
10697   ASTContext &C = CGM.getContext();
10698   FD = FD->getMostRecentDecl();
10699   // Map params to their positions in function decl.
10700   llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10701   if (isa<CXXMethodDecl>(FD))
10702     ParamPositions.try_emplace(FD, 0);
10703   unsigned ParamPos = ParamPositions.size();
10704   for (const ParmVarDecl *P : FD->parameters()) {
10705     ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
10706     ++ParamPos;
10707   }
10708   while (FD) {
10709     for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
10710       llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
10711       // Mark uniform parameters.
10712       for (const Expr *E : Attr->uniforms()) {
10713         E = E->IgnoreParenImpCasts();
10714         unsigned Pos;
10715         if (isa<CXXThisExpr>(E)) {
10716           Pos = ParamPositions[FD];
10717         } else {
10718           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10719                                 ->getCanonicalDecl();
10720           Pos = ParamPositions[PVD];
10721         }
10722         ParamAttrs[Pos].Kind = Uniform;
10723       }
10724       // Get alignment info.
10725       auto NI = Attr->alignments_begin();
10726       for (const Expr *E : Attr->aligneds()) {
10727         E = E->IgnoreParenImpCasts();
10728         unsigned Pos;
10729         QualType ParmTy;
10730         if (isa<CXXThisExpr>(E)) {
10731           Pos = ParamPositions[FD];
10732           ParmTy = E->getType();
10733         } else {
10734           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10735                                 ->getCanonicalDecl();
10736           Pos = ParamPositions[PVD];
10737           ParmTy = PVD->getType();
10738         }
10739         ParamAttrs[Pos].Alignment =
10740             (*NI)
10741                 ? (*NI)->EvaluateKnownConstInt(C)
10742                 : llvm::APSInt::getUnsigned(
10743                       C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
10744                           .getQuantity());
10745         ++NI;
10746       }
10747       // Mark linear parameters.
10748       auto SI = Attr->steps_begin();
10749       auto MI = Attr->modifiers_begin();
10750       for (const Expr *E : Attr->linears()) {
10751         E = E->IgnoreParenImpCasts();
10752         unsigned Pos;
10753         if (isa<CXXThisExpr>(E)) {
10754           Pos = ParamPositions[FD];
10755         } else {
10756           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10757                                 ->getCanonicalDecl();
10758           Pos = ParamPositions[PVD];
10759         }
10760         ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10761         ParamAttr.Kind = Linear;
10762         if (*SI) {
10763           Expr::EvalResult Result;
10764           if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) {
10765             if (const auto *DRE =
10766                     cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
10767               if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
10768                 ParamAttr.Kind = LinearWithVarStride;
10769                 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
10770                     ParamPositions[StridePVD->getCanonicalDecl()]);
10771               }
10772             }
10773           } else {
10774             ParamAttr.StrideOrArg = Result.Val.getInt();
10775           }
10776         }
10777         ++SI;
10778         ++MI;
10779       }
10780       llvm::APSInt VLENVal;
10781       SourceLocation ExprLoc;
10782       const Expr *VLENExpr = Attr->getSimdlen();
10783       if (VLENExpr) {
10784         VLENVal = VLENExpr->EvaluateKnownConstInt(C);
10785         ExprLoc = VLENExpr->getExprLoc();
10786       }
10787       OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
10788       if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
10789           CGM.getTriple().getArch() == llvm::Triple::x86_64) {
10790         emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
10791       } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
10792         unsigned VLEN = VLENVal.getExtValue();
10793         StringRef MangledName = Fn->getName();
10794         if (CGM.getTarget().hasFeature("sve"))
10795           emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
10796                                          MangledName, 's', 128, Fn, ExprLoc);
10797         if (CGM.getTarget().hasFeature("neon"))
10798           emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
10799                                          MangledName, 'n', 128, Fn, ExprLoc);
10800       }
10801     }
10802     FD = FD->getPreviousDecl();
10803   }
10804 }
10805 
10806 namespace {
10807 /// Cleanup action for doacross support.
10808 class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
10809 public:
10810   static const int DoacrossFinArgs = 2;
10811 
10812 private:
10813   llvm::FunctionCallee RTLFn;
10814   llvm::Value *Args[DoacrossFinArgs];
10815 
10816 public:
10817   DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
10818                     ArrayRef<llvm::Value *> CallArgs)
10819       : RTLFn(RTLFn) {
10820     assert(CallArgs.size() == DoacrossFinArgs);
10821     std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10822   }
10823   void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
10824     if (!CGF.HaveInsertPoint())
10825       return;
10826     CGF.EmitRuntimeCall(RTLFn, Args);
10827   }
10828 };
10829 } // namespace
10830 
10831 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
10832                                        const OMPLoopDirective &D,
10833                                        ArrayRef<Expr *> NumIterations) {
10834   if (!CGF.HaveInsertPoint())
10835     return;
10836 
10837   ASTContext &C = CGM.getContext();
10838   QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
10839   RecordDecl *RD;
10840   if (KmpDimTy.isNull()) {
10841     // Build struct kmp_dim {  // loop bounds info casted to kmp_int64
10842     //  kmp_int64 lo; // lower
10843     //  kmp_int64 up; // upper
10844     //  kmp_int64 st; // stride
10845     // };
10846     RD = C.buildImplicitRecord("kmp_dim");
10847     RD->startDefinition();
10848     addFieldToRecordDecl(C, RD, Int64Ty);
10849     addFieldToRecordDecl(C, RD, Int64Ty);
10850     addFieldToRecordDecl(C, RD, Int64Ty);
10851     RD->completeDefinition();
10852     KmpDimTy = C.getRecordType(RD);
10853   } else {
10854     RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
10855   }
10856   llvm::APInt Size(/*numBits=*/32, NumIterations.size());
10857   QualType ArrayTy =
10858       C.getConstantArrayType(KmpDimTy, Size, nullptr, ArrayType::Normal, 0);
10859 
10860   Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims");
10861   CGF.EmitNullInitialization(DimsAddr, ArrayTy);
10862   enum { LowerFD = 0, UpperFD, StrideFD };
10863   // Fill dims with data.
10864   for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
10865     LValue DimsLVal = CGF.MakeAddrLValue(
10866         CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy);
10867     // dims.upper = num_iterations;
10868     LValue UpperLVal = CGF.EmitLValueForField(
10869         DimsLVal, *std::next(RD->field_begin(), UpperFD));
10870     llvm::Value *NumIterVal =
10871         CGF.EmitScalarConversion(CGF.EmitScalarExpr(NumIterations[I]),
10872                                  D.getNumIterations()->getType(), Int64Ty,
10873                                  D.getNumIterations()->getExprLoc());
10874     CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
10875     // dims.stride = 1;
10876     LValue StrideLVal = CGF.EmitLValueForField(
10877         DimsLVal, *std::next(RD->field_begin(), StrideFD));
10878     CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
10879                           StrideLVal);
10880   }
10881 
10882   // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
10883   // kmp_int32 num_dims, struct kmp_dim * dims);
10884   llvm::Value *Args[] = {
10885       emitUpdateLocation(CGF, D.getBeginLoc()),
10886       getThreadID(CGF, D.getBeginLoc()),
10887       llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
10888       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
10889           CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
10890           CGM.VoidPtrTy)};
10891 
10892   llvm::FunctionCallee RTLFn =
10893       createRuntimeFunction(OMPRTL__kmpc_doacross_init);
10894   CGF.EmitRuntimeCall(RTLFn, Args);
10895   llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
10896       emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())};
10897   llvm::FunctionCallee FiniRTLFn =
10898       createRuntimeFunction(OMPRTL__kmpc_doacross_fini);
10899   CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
10900                                              llvm::makeArrayRef(FiniArgs));
10901 }
10902 
10903 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
10904                                           const OMPDependClause *C) {
10905   QualType Int64Ty =
10906       CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
10907   llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
10908   QualType ArrayTy = CGM.getContext().getConstantArrayType(
10909       Int64Ty, Size, nullptr, ArrayType::Normal, 0);
10910   Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr");
10911   for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
10912     const Expr *CounterVal = C->getLoopData(I);
10913     assert(CounterVal);
10914     llvm::Value *CntVal = CGF.EmitScalarConversion(
10915         CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
10916         CounterVal->getExprLoc());
10917     CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I),
10918                           /*Volatile=*/false, Int64Ty);
10919   }
10920   llvm::Value *Args[] = {
10921       emitUpdateLocation(CGF, C->getBeginLoc()),
10922       getThreadID(CGF, C->getBeginLoc()),
10923       CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
10924   llvm::FunctionCallee RTLFn;
10925   if (C->getDependencyKind() == OMPC_DEPEND_source) {
10926     RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post);
10927   } else {
10928     assert(C->getDependencyKind() == OMPC_DEPEND_sink);
10929     RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait);
10930   }
10931   CGF.EmitRuntimeCall(RTLFn, Args);
10932 }
10933 
10934 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
10935                                llvm::FunctionCallee Callee,
10936                                ArrayRef<llvm::Value *> Args) const {
10937   assert(Loc.isValid() && "Outlined function call location must be valid.");
10938   auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
10939 
10940   if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
10941     if (Fn->doesNotThrow()) {
10942       CGF.EmitNounwindRuntimeCall(Fn, Args);
10943       return;
10944     }
10945   }
10946   CGF.EmitRuntimeCall(Callee, Args);
10947 }
10948 
10949 void CGOpenMPRuntime::emitOutlinedFunctionCall(
10950     CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
10951     ArrayRef<llvm::Value *> Args) const {
10952   emitCall(CGF, Loc, OutlinedFn, Args);
10953 }
10954 
10955 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
10956   if (const auto *FD = dyn_cast<FunctionDecl>(D))
10957     if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
10958       HasEmittedDeclareTargetRegion = true;
10959 }
10960 
10961 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
10962                                              const VarDecl *NativeParam,
10963                                              const VarDecl *TargetParam) const {
10964   return CGF.GetAddrOfLocalVar(NativeParam);
10965 }
10966 
10967 namespace {
10968 /// Cleanup action for allocate support.
10969 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
10970 public:
10971   static const int CleanupArgs = 3;
10972 
10973 private:
10974   llvm::FunctionCallee RTLFn;
10975   llvm::Value *Args[CleanupArgs];
10976 
10977 public:
10978   OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
10979                        ArrayRef<llvm::Value *> CallArgs)
10980       : RTLFn(RTLFn) {
10981     assert(CallArgs.size() == CleanupArgs &&
10982            "Size of arguments does not match.");
10983     std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10984   }
10985   void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
10986     if (!CGF.HaveInsertPoint())
10987       return;
10988     CGF.EmitRuntimeCall(RTLFn, Args);
10989   }
10990 };
10991 } // namespace
10992 
10993 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
10994                                                    const VarDecl *VD) {
10995   if (!VD)
10996     return Address::invalid();
10997   const VarDecl *CVD = VD->getCanonicalDecl();
10998   if (!CVD->hasAttr<OMPAllocateDeclAttr>())
10999     return Address::invalid();
11000   const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
11001   // Use the default allocation.
11002   if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
11003       !AA->getAllocator())
11004     return Address::invalid();
11005   llvm::Value *Size;
11006   CharUnits Align = CGM.getContext().getDeclAlign(CVD);
11007   if (CVD->getType()->isVariablyModifiedType()) {
11008     Size = CGF.getTypeSize(CVD->getType());
11009     // Align the size: ((size + align - 1) / align) * align
11010     Size = CGF.Builder.CreateNUWAdd(
11011         Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
11012     Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
11013     Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
11014   } else {
11015     CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
11016     Size = CGM.getSize(Sz.alignTo(Align));
11017   }
11018   llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc());
11019   assert(AA->getAllocator() &&
11020          "Expected allocator expression for non-default allocator.");
11021   llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
11022   // According to the standard, the original allocator type is a enum (integer).
11023   // Convert to pointer type, if required.
11024   if (Allocator->getType()->isIntegerTy())
11025     Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
11026   else if (Allocator->getType()->isPointerTy())
11027     Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
11028                                                                 CGM.VoidPtrTy);
11029   llvm::Value *Args[] = {ThreadID, Size, Allocator};
11030 
11031   llvm::Value *Addr =
11032       CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_alloc), Args,
11033                           CVD->getName() + ".void.addr");
11034   llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
11035                                                               Allocator};
11036   llvm::FunctionCallee FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_free);
11037 
11038   CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
11039                                                 llvm::makeArrayRef(FiniArgs));
11040   Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
11041       Addr,
11042       CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
11043       CVD->getName() + ".addr");
11044   return Address(Addr, Align);
11045 }
11046 
11047 namespace {
11048 using OMPContextSelectorData =
11049     OpenMPCtxSelectorData<ArrayRef<StringRef>, llvm::APSInt>;
11050 using CompleteOMPContextSelectorData = SmallVector<OMPContextSelectorData, 4>;
11051 } // anonymous namespace
11052 
11053 /// Checks current context and returns true if it matches the context selector.
11054 template <OpenMPContextSelectorSetKind CtxSet, OpenMPContextSelectorKind Ctx,
11055           typename... Arguments>
11056 static bool checkContext(const OMPContextSelectorData &Data,
11057                          Arguments... Params) {
11058   assert(Data.CtxSet != OMP_CTX_SET_unknown && Data.Ctx != OMP_CTX_unknown &&
11059          "Unknown context selector or context selector set.");
11060   return false;
11061 }
11062 
11063 /// Checks for implementation={vendor(<vendor>)} context selector.
11064 /// \returns true iff <vendor>="llvm", false otherwise.
11065 template <>
11066 bool checkContext<OMP_CTX_SET_implementation, OMP_CTX_vendor>(
11067     const OMPContextSelectorData &Data) {
11068   return llvm::all_of(Data.Names,
11069                       [](StringRef S) { return !S.compare_lower("llvm"); });
11070 }
11071 
11072 /// Checks for device={kind(<kind>)} context selector.
11073 /// \returns true if <kind>="host" and compilation is for host.
11074 /// true if <kind>="nohost" and compilation is for device.
11075 /// true if <kind>="cpu" and compilation is for Arm, X86 or PPC CPU.
11076 /// true if <kind>="gpu" and compilation is for NVPTX or AMDGCN.
11077 /// false otherwise.
11078 template <>
11079 bool checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>(
11080     const OMPContextSelectorData &Data, CodeGenModule &CGM) {
11081   for (StringRef Name : Data.Names) {
11082     if (!Name.compare_lower("host")) {
11083       if (CGM.getLangOpts().OpenMPIsDevice)
11084         return false;
11085       continue;
11086     }
11087     if (!Name.compare_lower("nohost")) {
11088       if (!CGM.getLangOpts().OpenMPIsDevice)
11089         return false;
11090       continue;
11091     }
11092     switch (CGM.getTriple().getArch()) {
11093     case llvm::Triple::arm:
11094     case llvm::Triple::armeb:
11095     case llvm::Triple::aarch64:
11096     case llvm::Triple::aarch64_be:
11097     case llvm::Triple::aarch64_32:
11098     case llvm::Triple::ppc:
11099     case llvm::Triple::ppc64:
11100     case llvm::Triple::ppc64le:
11101     case llvm::Triple::x86:
11102     case llvm::Triple::x86_64:
11103       if (Name.compare_lower("cpu"))
11104         return false;
11105       break;
11106     case llvm::Triple::amdgcn:
11107     case llvm::Triple::nvptx:
11108     case llvm::Triple::nvptx64:
11109       if (Name.compare_lower("gpu"))
11110         return false;
11111       break;
11112     case llvm::Triple::UnknownArch:
11113     case llvm::Triple::arc:
11114     case llvm::Triple::avr:
11115     case llvm::Triple::bpfel:
11116     case llvm::Triple::bpfeb:
11117     case llvm::Triple::hexagon:
11118     case llvm::Triple::mips:
11119     case llvm::Triple::mipsel:
11120     case llvm::Triple::mips64:
11121     case llvm::Triple::mips64el:
11122     case llvm::Triple::msp430:
11123     case llvm::Triple::r600:
11124     case llvm::Triple::riscv32:
11125     case llvm::Triple::riscv64:
11126     case llvm::Triple::sparc:
11127     case llvm::Triple::sparcv9:
11128     case llvm::Triple::sparcel:
11129     case llvm::Triple::systemz:
11130     case llvm::Triple::tce:
11131     case llvm::Triple::tcele:
11132     case llvm::Triple::thumb:
11133     case llvm::Triple::thumbeb:
11134     case llvm::Triple::xcore:
11135     case llvm::Triple::le32:
11136     case llvm::Triple::le64:
11137     case llvm::Triple::amdil:
11138     case llvm::Triple::amdil64:
11139     case llvm::Triple::hsail:
11140     case llvm::Triple::hsail64:
11141     case llvm::Triple::spir:
11142     case llvm::Triple::spir64:
11143     case llvm::Triple::kalimba:
11144     case llvm::Triple::shave:
11145     case llvm::Triple::lanai:
11146     case llvm::Triple::wasm32:
11147     case llvm::Triple::wasm64:
11148     case llvm::Triple::renderscript32:
11149     case llvm::Triple::renderscript64:
11150       return false;
11151     }
11152   }
11153   return true;
11154 }
11155 
11156 bool matchesContext(CodeGenModule &CGM,
11157                     const CompleteOMPContextSelectorData &ContextData) {
11158   for (const OMPContextSelectorData &Data : ContextData) {
11159     switch (Data.Ctx) {
11160     case OMP_CTX_vendor:
11161       assert(Data.CtxSet == OMP_CTX_SET_implementation &&
11162              "Expected implementation context selector set.");
11163       if (!checkContext<OMP_CTX_SET_implementation, OMP_CTX_vendor>(Data))
11164         return false;
11165       break;
11166     case OMP_CTX_kind:
11167       assert(Data.CtxSet == OMP_CTX_SET_device &&
11168              "Expected device context selector set.");
11169       if (!checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>(Data,
11170                                                                            CGM))
11171         return false;
11172       break;
11173     case OMP_CTX_unknown:
11174       llvm_unreachable("Unknown context selector kind.");
11175     }
11176   }
11177   return true;
11178 }
11179 
11180 static CompleteOMPContextSelectorData
11181 translateAttrToContextSelectorData(ASTContext &C,
11182                                    const OMPDeclareVariantAttr *A) {
11183   CompleteOMPContextSelectorData Data;
11184   for (unsigned I = 0, E = A->scores_size(); I < E; ++I) {
11185     Data.emplace_back();
11186     auto CtxSet = static_cast<OpenMPContextSelectorSetKind>(
11187         *std::next(A->ctxSelectorSets_begin(), I));
11188     auto Ctx = static_cast<OpenMPContextSelectorKind>(
11189         *std::next(A->ctxSelectors_begin(), I));
11190     Data.back().CtxSet = CtxSet;
11191     Data.back().Ctx = Ctx;
11192     const Expr *Score = *std::next(A->scores_begin(), I);
11193     Data.back().Score = Score->EvaluateKnownConstInt(C);
11194     switch (Ctx) {
11195     case OMP_CTX_vendor:
11196       assert(CtxSet == OMP_CTX_SET_implementation &&
11197              "Expected implementation context selector set.");
11198       Data.back().Names =
11199           llvm::makeArrayRef(A->implVendors_begin(), A->implVendors_end());
11200       break;
11201     case OMP_CTX_kind:
11202       assert(CtxSet == OMP_CTX_SET_device &&
11203              "Expected device context selector set.");
11204       Data.back().Names =
11205           llvm::makeArrayRef(A->deviceKinds_begin(), A->deviceKinds_end());
11206       break;
11207     case OMP_CTX_unknown:
11208       llvm_unreachable("Unknown context selector kind.");
11209     }
11210   }
11211   return Data;
11212 }
11213 
11214 static bool isStrictSubset(const CompleteOMPContextSelectorData &LHS,
11215                            const CompleteOMPContextSelectorData &RHS) {
11216   llvm::SmallDenseMap<std::pair<int, int>, llvm::StringSet<>, 4> RHSData;
11217   for (const OMPContextSelectorData &D : RHS) {
11218     auto &Pair = RHSData.FindAndConstruct(std::make_pair(D.CtxSet, D.Ctx));
11219     Pair.getSecond().insert(D.Names.begin(), D.Names.end());
11220   }
11221   bool AllSetsAreEqual = true;
11222   for (const OMPContextSelectorData &D : LHS) {
11223     auto It = RHSData.find(std::make_pair(D.CtxSet, D.Ctx));
11224     if (It == RHSData.end())
11225       return false;
11226     if (D.Names.size() > It->getSecond().size())
11227       return false;
11228     if (llvm::set_union(It->getSecond(), D.Names))
11229       return false;
11230     AllSetsAreEqual =
11231         AllSetsAreEqual && (D.Names.size() == It->getSecond().size());
11232   }
11233 
11234   return LHS.size() != RHS.size() || !AllSetsAreEqual;
11235 }
11236 
11237 static bool greaterCtxScore(const CompleteOMPContextSelectorData &LHS,
11238                             const CompleteOMPContextSelectorData &RHS) {
11239   // Score is calculated as sum of all scores + 1.
11240   llvm::APSInt LHSScore(llvm::APInt(64, 1), /*isUnsigned=*/false);
11241   bool RHSIsSubsetOfLHS = isStrictSubset(RHS, LHS);
11242   if (RHSIsSubsetOfLHS) {
11243     LHSScore = llvm::APSInt::get(0);
11244   } else {
11245     for (const OMPContextSelectorData &Data : LHS) {
11246       if (Data.Score.getBitWidth() > LHSScore.getBitWidth()) {
11247         LHSScore = LHSScore.extend(Data.Score.getBitWidth()) + Data.Score;
11248       } else if (Data.Score.getBitWidth() < LHSScore.getBitWidth()) {
11249         LHSScore += Data.Score.extend(LHSScore.getBitWidth());
11250       } else {
11251         LHSScore += Data.Score;
11252       }
11253     }
11254   }
11255   llvm::APSInt RHSScore(llvm::APInt(64, 1), /*isUnsigned=*/false);
11256   if (!RHSIsSubsetOfLHS && isStrictSubset(LHS, RHS)) {
11257     RHSScore = llvm::APSInt::get(0);
11258   } else {
11259     for (const OMPContextSelectorData &Data : RHS) {
11260       if (Data.Score.getBitWidth() > RHSScore.getBitWidth()) {
11261         RHSScore = RHSScore.extend(Data.Score.getBitWidth()) + Data.Score;
11262       } else if (Data.Score.getBitWidth() < RHSScore.getBitWidth()) {
11263         RHSScore += Data.Score.extend(RHSScore.getBitWidth());
11264       } else {
11265         RHSScore += Data.Score;
11266       }
11267     }
11268   }
11269   return llvm::APSInt::compareValues(LHSScore, RHSScore) >= 0;
11270 }
11271 
11272 /// Finds the variant function that matches current context with its context
11273 /// selector.
11274 static const FunctionDecl *getDeclareVariantFunction(CodeGenModule &CGM,
11275                                                      const FunctionDecl *FD) {
11276   if (!FD->hasAttrs() || !FD->hasAttr<OMPDeclareVariantAttr>())
11277     return FD;
11278   // Iterate through all DeclareVariant attributes and check context selectors.
11279   const OMPDeclareVariantAttr *TopMostAttr = nullptr;
11280   CompleteOMPContextSelectorData TopMostData;
11281   for (const auto *A : FD->specific_attrs<OMPDeclareVariantAttr>()) {
11282     CompleteOMPContextSelectorData Data =
11283         translateAttrToContextSelectorData(CGM.getContext(), A);
11284     if (!matchesContext(CGM, Data))
11285       continue;
11286     // If the attribute matches the context, find the attribute with the highest
11287     // score.
11288     if (!TopMostAttr || !greaterCtxScore(TopMostData, Data)) {
11289       TopMostAttr = A;
11290       TopMostData.swap(Data);
11291     }
11292   }
11293   if (!TopMostAttr)
11294     return FD;
11295   return cast<FunctionDecl>(
11296       cast<DeclRefExpr>(TopMostAttr->getVariantFuncRef()->IgnoreParenImpCasts())
11297           ->getDecl());
11298 }
11299 
11300 bool CGOpenMPRuntime::emitDeclareVariant(GlobalDecl GD, bool IsForDefinition) {
11301   const auto *D = cast<FunctionDecl>(GD.getDecl());
11302   // If the original function is defined already, use its definition.
11303   StringRef MangledName = CGM.getMangledName(GD);
11304   llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName);
11305   if (Orig && !Orig->isDeclaration())
11306     return false;
11307   const FunctionDecl *NewFD = getDeclareVariantFunction(CGM, D);
11308   // Emit original function if it does not have declare variant attribute or the
11309   // context does not match.
11310   if (NewFD == D)
11311     return false;
11312   GlobalDecl NewGD = GD.getWithDecl(NewFD);
11313   if (tryEmitDeclareVariant(NewGD, GD, Orig, IsForDefinition)) {
11314     DeferredVariantFunction.erase(D);
11315     return true;
11316   }
11317   DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD)));
11318   return true;
11319 }
11320 
11321 llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
11322     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11323     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
11324   llvm_unreachable("Not supported in SIMD-only mode");
11325 }
11326 
11327 llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
11328     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11329     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
11330   llvm_unreachable("Not supported in SIMD-only mode");
11331 }
11332 
11333 llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
11334     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
11335     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
11336     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
11337     bool Tied, unsigned &NumberOfParts) {
11338   llvm_unreachable("Not supported in SIMD-only mode");
11339 }
11340 
11341 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
11342                                            SourceLocation Loc,
11343                                            llvm::Function *OutlinedFn,
11344                                            ArrayRef<llvm::Value *> CapturedVars,
11345                                            const Expr *IfCond) {
11346   llvm_unreachable("Not supported in SIMD-only mode");
11347 }
11348 
11349 void CGOpenMPSIMDRuntime::emitCriticalRegion(
11350     CodeGenFunction &CGF, StringRef CriticalName,
11351     const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
11352     const Expr *Hint) {
11353   llvm_unreachable("Not supported in SIMD-only mode");
11354 }
11355 
11356 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
11357                                            const RegionCodeGenTy &MasterOpGen,
11358                                            SourceLocation Loc) {
11359   llvm_unreachable("Not supported in SIMD-only mode");
11360 }
11361 
11362 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
11363                                             SourceLocation Loc) {
11364   llvm_unreachable("Not supported in SIMD-only mode");
11365 }
11366 
11367 void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
11368     CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
11369     SourceLocation Loc) {
11370   llvm_unreachable("Not supported in SIMD-only mode");
11371 }
11372 
11373 void CGOpenMPSIMDRuntime::emitSingleRegion(
11374     CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
11375     SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
11376     ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
11377     ArrayRef<const Expr *> AssignmentOps) {
11378   llvm_unreachable("Not supported in SIMD-only mode");
11379 }
11380 
11381 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
11382                                             const RegionCodeGenTy &OrderedOpGen,
11383                                             SourceLocation Loc,
11384                                             bool IsThreads) {
11385   llvm_unreachable("Not supported in SIMD-only mode");
11386 }
11387 
11388 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
11389                                           SourceLocation Loc,
11390                                           OpenMPDirectiveKind Kind,
11391                                           bool EmitChecks,
11392                                           bool ForceSimpleCall) {
11393   llvm_unreachable("Not supported in SIMD-only mode");
11394 }
11395 
11396 void CGOpenMPSIMDRuntime::emitForDispatchInit(
11397     CodeGenFunction &CGF, SourceLocation Loc,
11398     const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
11399     bool Ordered, const DispatchRTInput &DispatchValues) {
11400   llvm_unreachable("Not supported in SIMD-only mode");
11401 }
11402 
11403 void CGOpenMPSIMDRuntime::emitForStaticInit(
11404     CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
11405     const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
11406   llvm_unreachable("Not supported in SIMD-only mode");
11407 }
11408 
11409 void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
11410     CodeGenFunction &CGF, SourceLocation Loc,
11411     OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
11412   llvm_unreachable("Not supported in SIMD-only mode");
11413 }
11414 
11415 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
11416                                                      SourceLocation Loc,
11417                                                      unsigned IVSize,
11418                                                      bool IVSigned) {
11419   llvm_unreachable("Not supported in SIMD-only mode");
11420 }
11421 
11422 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
11423                                               SourceLocation Loc,
11424                                               OpenMPDirectiveKind DKind) {
11425   llvm_unreachable("Not supported in SIMD-only mode");
11426 }
11427 
11428 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
11429                                               SourceLocation Loc,
11430                                               unsigned IVSize, bool IVSigned,
11431                                               Address IL, Address LB,
11432                                               Address UB, Address ST) {
11433   llvm_unreachable("Not supported in SIMD-only mode");
11434 }
11435 
11436 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
11437                                                llvm::Value *NumThreads,
11438                                                SourceLocation Loc) {
11439   llvm_unreachable("Not supported in SIMD-only mode");
11440 }
11441 
11442 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
11443                                              OpenMPProcBindClauseKind ProcBind,
11444                                              SourceLocation Loc) {
11445   llvm_unreachable("Not supported in SIMD-only mode");
11446 }
11447 
11448 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
11449                                                     const VarDecl *VD,
11450                                                     Address VDAddr,
11451                                                     SourceLocation Loc) {
11452   llvm_unreachable("Not supported in SIMD-only mode");
11453 }
11454 
11455 llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
11456     const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
11457     CodeGenFunction *CGF) {
11458   llvm_unreachable("Not supported in SIMD-only mode");
11459 }
11460 
11461 Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
11462     CodeGenFunction &CGF, QualType VarType, StringRef Name) {
11463   llvm_unreachable("Not supported in SIMD-only mode");
11464 }
11465 
11466 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
11467                                     ArrayRef<const Expr *> Vars,
11468                                     SourceLocation Loc) {
11469   llvm_unreachable("Not supported in SIMD-only mode");
11470 }
11471 
11472 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
11473                                        const OMPExecutableDirective &D,
11474                                        llvm::Function *TaskFunction,
11475                                        QualType SharedsTy, Address Shareds,
11476                                        const Expr *IfCond,
11477                                        const OMPTaskDataTy &Data) {
11478   llvm_unreachable("Not supported in SIMD-only mode");
11479 }
11480 
11481 void CGOpenMPSIMDRuntime::emitTaskLoopCall(
11482     CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
11483     llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
11484     const Expr *IfCond, const OMPTaskDataTy &Data) {
11485   llvm_unreachable("Not supported in SIMD-only mode");
11486 }
11487 
11488 void CGOpenMPSIMDRuntime::emitReduction(
11489     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
11490     ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
11491     ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
11492   assert(Options.SimpleReduction && "Only simple reduction is expected.");
11493   CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
11494                                  ReductionOps, Options);
11495 }
11496 
11497 llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
11498     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
11499     ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
11500   llvm_unreachable("Not supported in SIMD-only mode");
11501 }
11502 
11503 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
11504                                                   SourceLocation Loc,
11505                                                   ReductionCodeGen &RCG,
11506                                                   unsigned N) {
11507   llvm_unreachable("Not supported in SIMD-only mode");
11508 }
11509 
11510 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
11511                                                   SourceLocation Loc,
11512                                                   llvm::Value *ReductionsPtr,
11513                                                   LValue SharedLVal) {
11514   llvm_unreachable("Not supported in SIMD-only mode");
11515 }
11516 
11517 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
11518                                            SourceLocation Loc) {
11519   llvm_unreachable("Not supported in SIMD-only mode");
11520 }
11521 
11522 void CGOpenMPSIMDRuntime::emitCancellationPointCall(
11523     CodeGenFunction &CGF, SourceLocation Loc,
11524     OpenMPDirectiveKind CancelRegion) {
11525   llvm_unreachable("Not supported in SIMD-only mode");
11526 }
11527 
11528 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
11529                                          SourceLocation Loc, const Expr *IfCond,
11530                                          OpenMPDirectiveKind CancelRegion) {
11531   llvm_unreachable("Not supported in SIMD-only mode");
11532 }
11533 
11534 void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
11535     const OMPExecutableDirective &D, StringRef ParentName,
11536     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
11537     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
11538   llvm_unreachable("Not supported in SIMD-only mode");
11539 }
11540 
11541 void CGOpenMPSIMDRuntime::emitTargetCall(
11542     CodeGenFunction &CGF, const OMPExecutableDirective &D,
11543     llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11544     const Expr *Device,
11545     llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11546                                      const OMPLoopDirective &D)>
11547         SizeEmitter) {
11548   llvm_unreachable("Not supported in SIMD-only mode");
11549 }
11550 
11551 bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
11552   llvm_unreachable("Not supported in SIMD-only mode");
11553 }
11554 
11555 bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11556   llvm_unreachable("Not supported in SIMD-only mode");
11557 }
11558 
11559 bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
11560   return false;
11561 }
11562 
11563 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
11564                                         const OMPExecutableDirective &D,
11565                                         SourceLocation Loc,
11566                                         llvm::Function *OutlinedFn,
11567                                         ArrayRef<llvm::Value *> CapturedVars) {
11568   llvm_unreachable("Not supported in SIMD-only mode");
11569 }
11570 
11571 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11572                                              const Expr *NumTeams,
11573                                              const Expr *ThreadLimit,
11574                                              SourceLocation Loc) {
11575   llvm_unreachable("Not supported in SIMD-only mode");
11576 }
11577 
11578 void CGOpenMPSIMDRuntime::emitTargetDataCalls(
11579     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11580     const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
11581   llvm_unreachable("Not supported in SIMD-only mode");
11582 }
11583 
11584 void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
11585     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11586     const Expr *Device) {
11587   llvm_unreachable("Not supported in SIMD-only mode");
11588 }
11589 
11590 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
11591                                            const OMPLoopDirective &D,
11592                                            ArrayRef<Expr *> NumIterations) {
11593   llvm_unreachable("Not supported in SIMD-only mode");
11594 }
11595 
11596 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
11597                                               const OMPDependClause *C) {
11598   llvm_unreachable("Not supported in SIMD-only mode");
11599 }
11600 
11601 const VarDecl *
11602 CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
11603                                         const VarDecl *NativeParam) const {
11604   llvm_unreachable("Not supported in SIMD-only mode");
11605 }
11606 
11607 Address
11608 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
11609                                          const VarDecl *NativeParam,
11610                                          const VarDecl *TargetParam) const {
11611   llvm_unreachable("Not supported in SIMD-only mode");
11612 }
11613