1 //===------- ItaniumCXXABI.cpp - Emit LLVM Code from ASTs for a Module ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides C++ code generation targeting the Itanium C++ ABI.  The class
11 // in this file generates structures that follow the Itanium C++ ABI, which is
12 // documented at:
13 //  http://www.codesourcery.com/public/cxx-abi/abi.html
14 //  http://www.codesourcery.com/public/cxx-abi/abi-eh.html
15 //
16 // It also supports the closely-related ARM ABI, documented at:
17 // http://infocenter.arm.com/help/topic/com.arm.doc.ihi0041c/IHI0041C_cppabi.pdf
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "CGCXXABI.h"
22 #include "CGRecordLayout.h"
23 #include "CGVTables.h"
24 #include "CodeGenFunction.h"
25 #include "CodeGenModule.h"
26 #include "clang/AST/Mangle.h"
27 #include "clang/AST/Type.h"
28 #include "llvm/IR/CallSite.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/IR/Intrinsics.h"
31 #include "llvm/IR/Value.h"
32 
33 using namespace clang;
34 using namespace CodeGen;
35 
36 namespace {
37 class ItaniumCXXABI : public CodeGen::CGCXXABI {
38   /// VTables - All the vtables which have been defined.
39   llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
40 
41 protected:
42   bool UseARMMethodPtrABI;
43   bool UseARMGuardVarABI;
44 
45   ItaniumMangleContext &getMangleContext() {
46     return cast<ItaniumMangleContext>(CodeGen::CGCXXABI::getMangleContext());
47   }
48 
49 public:
50   ItaniumCXXABI(CodeGen::CodeGenModule &CGM,
51                 bool UseARMMethodPtrABI = false,
52                 bool UseARMGuardVarABI = false) :
53     CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI),
54     UseARMGuardVarABI(UseARMGuardVarABI) { }
55 
56   bool classifyReturnType(CGFunctionInfo &FI) const override;
57 
58   RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override {
59     // Structures with either a non-trivial destructor or a non-trivial
60     // copy constructor are always indirect.
61     // FIXME: Use canCopyArgument() when it is fixed to handle lazily declared
62     // special members.
63     if (RD->hasNonTrivialDestructor() || RD->hasNonTrivialCopyConstructor())
64       return RAA_Indirect;
65     return RAA_Default;
66   }
67 
68   bool isZeroInitializable(const MemberPointerType *MPT) override;
69 
70   llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
71 
72   llvm::Value *
73     EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
74                                     const Expr *E,
75                                     llvm::Value *&This,
76                                     llvm::Value *MemFnPtr,
77                                     const MemberPointerType *MPT) override;
78 
79   llvm::Value *
80     EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
81                                  llvm::Value *Base,
82                                  llvm::Value *MemPtr,
83                                  const MemberPointerType *MPT) override;
84 
85   llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
86                                            const CastExpr *E,
87                                            llvm::Value *Src) override;
88   llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
89                                               llvm::Constant *Src) override;
90 
91   llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
92 
93   llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD) override;
94   llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
95                                         CharUnits offset) override;
96   llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
97   llvm::Constant *BuildMemberPointer(const CXXMethodDecl *MD,
98                                      CharUnits ThisAdjustment);
99 
100   llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
101                                            llvm::Value *L, llvm::Value *R,
102                                            const MemberPointerType *MPT,
103                                            bool Inequality) override;
104 
105   llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
106                                          llvm::Value *Addr,
107                                          const MemberPointerType *MPT) override;
108 
109   void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
110                                llvm::Value *Ptr, QualType ElementType,
111                                const CXXDestructorDecl *Dtor) override;
112 
113   void EmitFundamentalRTTIDescriptor(QualType Type);
114   void EmitFundamentalRTTIDescriptors();
115   llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
116 
117   bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
118   void EmitBadTypeidCall(CodeGenFunction &CGF) override;
119   llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
120                           llvm::Value *ThisPtr,
121                           llvm::Type *StdTypeInfoPtrTy) override;
122 
123   bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
124                                           QualType SrcRecordTy) override;
125 
126   llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
127                                    QualType SrcRecordTy, QualType DestTy,
128                                    QualType DestRecordTy,
129                                    llvm::BasicBlock *CastEnd) override;
130 
131   llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value,
132                                      QualType SrcRecordTy,
133                                      QualType DestTy) override;
134 
135   bool EmitBadCastCall(CodeGenFunction &CGF) override;
136 
137   llvm::Value *
138     GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This,
139                               const CXXRecordDecl *ClassDecl,
140                               const CXXRecordDecl *BaseClassDecl) override;
141 
142   void EmitCXXConstructors(const CXXConstructorDecl *D) override;
143 
144   void buildStructorSignature(const CXXMethodDecl *MD, StructorType T,
145                               SmallVectorImpl<CanQualType> &ArgTys) override;
146 
147   bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
148                               CXXDtorType DT) const override {
149     // Itanium does not emit any destructor variant as an inline thunk.
150     // Delegating may occur as an optimization, but all variants are either
151     // emitted with external linkage or as linkonce if they are inline and used.
152     return false;
153   }
154 
155   void EmitCXXDestructors(const CXXDestructorDecl *D) override;
156 
157   void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
158                                  FunctionArgList &Params) override;
159 
160   void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
161 
162   unsigned addImplicitConstructorArgs(CodeGenFunction &CGF,
163                                       const CXXConstructorDecl *D,
164                                       CXXCtorType Type, bool ForVirtualBase,
165                                       bool Delegating,
166                                       CallArgList &Args) override;
167 
168   void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
169                           CXXDtorType Type, bool ForVirtualBase,
170                           bool Delegating, llvm::Value *This) override;
171 
172   void emitVTableDefinitions(CodeGenVTables &CGVT,
173                              const CXXRecordDecl *RD) override;
174 
175   llvm::Value *getVTableAddressPointInStructor(
176       CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
177       BaseSubobject Base, const CXXRecordDecl *NearestVBase,
178       bool &NeedsVirtualOffset) override;
179 
180   llvm::Constant *
181   getVTableAddressPointForConstExpr(BaseSubobject Base,
182                                     const CXXRecordDecl *VTableClass) override;
183 
184   llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
185                                         CharUnits VPtrOffset) override;
186 
187   llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
188                                          llvm::Value *This,
189                                          llvm::Type *Ty) override;
190 
191   llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
192                                          const CXXDestructorDecl *Dtor,
193                                          CXXDtorType DtorType,
194                                          llvm::Value *This,
195                                          const CXXMemberCallExpr *CE) override;
196 
197   void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
198 
199   void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
200                        bool ReturnAdjustment) override {
201     // Allow inlining of thunks by emitting them with available_externally
202     // linkage together with vtables when needed.
203     if (ForVTable)
204       Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
205   }
206 
207   llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This,
208                                      const ThisAdjustment &TA) override;
209 
210   llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
211                                        const ReturnAdjustment &RA) override;
212 
213   size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
214                               FunctionArgList &Args) const override {
215     assert(!Args.empty() && "expected the arglist to not be empty!");
216     return Args.size() - 1;
217   }
218 
219   StringRef GetPureVirtualCallName() override { return "__cxa_pure_virtual"; }
220   StringRef GetDeletedVirtualCallName() override
221     { return "__cxa_deleted_virtual"; }
222 
223   CharUnits getArrayCookieSizeImpl(QualType elementType) override;
224   llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
225                                      llvm::Value *NewPtr,
226                                      llvm::Value *NumElements,
227                                      const CXXNewExpr *expr,
228                                      QualType ElementType) override;
229   llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
230                                    llvm::Value *allocPtr,
231                                    CharUnits cookieSize) override;
232 
233   void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
234                        llvm::GlobalVariable *DeclPtr,
235                        bool PerformInit) override;
236   void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
237                           llvm::Constant *dtor, llvm::Constant *addr) override;
238 
239   llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD,
240                                                 llvm::Value *Val);
241   void EmitThreadLocalInitFuncs(
242       CodeGenModule &CGM,
243       ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
244           CXXThreadLocals,
245       ArrayRef<llvm::Function *> CXXThreadLocalInits,
246       ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) override;
247 
248   bool usesThreadWrapperFunction() const override { return true; }
249   LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
250                                       QualType LValType) override;
251 
252   bool NeedsVTTParameter(GlobalDecl GD) override;
253 
254   /**************************** RTTI Uniqueness ******************************/
255 
256 protected:
257   /// Returns true if the ABI requires RTTI type_info objects to be unique
258   /// across a program.
259   virtual bool shouldRTTIBeUnique() const { return true; }
260 
261 public:
262   /// What sort of unique-RTTI behavior should we use?
263   enum RTTIUniquenessKind {
264     /// We are guaranteeing, or need to guarantee, that the RTTI string
265     /// is unique.
266     RUK_Unique,
267 
268     /// We are not guaranteeing uniqueness for the RTTI string, so we
269     /// can demote to hidden visibility but must use string comparisons.
270     RUK_NonUniqueHidden,
271 
272     /// We are not guaranteeing uniqueness for the RTTI string, so we
273     /// have to use string comparisons, but we also have to emit it with
274     /// non-hidden visibility.
275     RUK_NonUniqueVisible
276   };
277 
278   /// Return the required visibility status for the given type and linkage in
279   /// the current ABI.
280   RTTIUniquenessKind
281   classifyRTTIUniqueness(QualType CanTy,
282                          llvm::GlobalValue::LinkageTypes Linkage) const;
283   friend class ItaniumRTTIBuilder;
284 
285   void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override;
286 };
287 
288 class ARMCXXABI : public ItaniumCXXABI {
289 public:
290   ARMCXXABI(CodeGen::CodeGenModule &CGM) :
291     ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
292                   /* UseARMGuardVarABI = */ true) {}
293 
294   bool HasThisReturn(GlobalDecl GD) const override {
295     return (isa<CXXConstructorDecl>(GD.getDecl()) || (
296               isa<CXXDestructorDecl>(GD.getDecl()) &&
297               GD.getDtorType() != Dtor_Deleting));
298   }
299 
300   void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV,
301                            QualType ResTy) override;
302 
303   CharUnits getArrayCookieSizeImpl(QualType elementType) override;
304   llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
305                                      llvm::Value *NewPtr,
306                                      llvm::Value *NumElements,
307                                      const CXXNewExpr *expr,
308                                      QualType ElementType) override;
309   llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, llvm::Value *allocPtr,
310                                    CharUnits cookieSize) override;
311 };
312 
313 class iOS64CXXABI : public ARMCXXABI {
314 public:
315   iOS64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {}
316 
317   // ARM64 libraries are prepared for non-unique RTTI.
318   bool shouldRTTIBeUnique() const override { return false; }
319 };
320 }
321 
322 CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
323   switch (CGM.getTarget().getCXXABI().getKind()) {
324   // For IR-generation purposes, there's no significant difference
325   // between the ARM and iOS ABIs.
326   case TargetCXXABI::GenericARM:
327   case TargetCXXABI::iOS:
328     return new ARMCXXABI(CGM);
329 
330   case TargetCXXABI::iOS64:
331     return new iOS64CXXABI(CGM);
332 
333   // Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
334   // include the other 32-bit ARM oddities: constructor/destructor return values
335   // and array cookies.
336   case TargetCXXABI::GenericAArch64:
337     return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
338                              /* UseARMGuardVarABI = */ true);
339 
340   case TargetCXXABI::GenericItanium:
341     if (CGM.getContext().getTargetInfo().getTriple().getArch()
342         == llvm::Triple::le32) {
343       // For PNaCl, use ARM-style method pointers so that PNaCl code
344       // does not assume anything about the alignment of function
345       // pointers.
346       return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true,
347                                /* UseARMGuardVarABI = */ false);
348     }
349     return new ItaniumCXXABI(CGM);
350 
351   case TargetCXXABI::Microsoft:
352     llvm_unreachable("Microsoft ABI is not Itanium-based");
353   }
354   llvm_unreachable("bad ABI kind");
355 }
356 
357 llvm::Type *
358 ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
359   if (MPT->isMemberDataPointer())
360     return CGM.PtrDiffTy;
361   return llvm::StructType::get(CGM.PtrDiffTy, CGM.PtrDiffTy, NULL);
362 }
363 
364 /// In the Itanium and ARM ABIs, method pointers have the form:
365 ///   struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
366 ///
367 /// In the Itanium ABI:
368 ///  - method pointers are virtual if (memptr.ptr & 1) is nonzero
369 ///  - the this-adjustment is (memptr.adj)
370 ///  - the virtual offset is (memptr.ptr - 1)
371 ///
372 /// In the ARM ABI:
373 ///  - method pointers are virtual if (memptr.adj & 1) is nonzero
374 ///  - the this-adjustment is (memptr.adj >> 1)
375 ///  - the virtual offset is (memptr.ptr)
376 /// ARM uses 'adj' for the virtual flag because Thumb functions
377 /// may be only single-byte aligned.
378 ///
379 /// If the member is virtual, the adjusted 'this' pointer points
380 /// to a vtable pointer from which the virtual offset is applied.
381 ///
382 /// If the member is non-virtual, memptr.ptr is the address of
383 /// the function to call.
384 llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
385     CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
386     llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
387   CGBuilderTy &Builder = CGF.Builder;
388 
389   const FunctionProtoType *FPT =
390     MPT->getPointeeType()->getAs<FunctionProtoType>();
391   const CXXRecordDecl *RD =
392     cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
393 
394   llvm::FunctionType *FTy =
395     CGM.getTypes().GetFunctionType(
396       CGM.getTypes().arrangeCXXMethodType(RD, FPT));
397 
398   llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1);
399 
400   llvm::BasicBlock *FnVirtual = CGF.createBasicBlock("memptr.virtual");
401   llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock("memptr.nonvirtual");
402   llvm::BasicBlock *FnEnd = CGF.createBasicBlock("memptr.end");
403 
404   // Extract memptr.adj, which is in the second field.
405   llvm::Value *RawAdj = Builder.CreateExtractValue(MemFnPtr, 1, "memptr.adj");
406 
407   // Compute the true adjustment.
408   llvm::Value *Adj = RawAdj;
409   if (UseARMMethodPtrABI)
410     Adj = Builder.CreateAShr(Adj, ptrdiff_1, "memptr.adj.shifted");
411 
412   // Apply the adjustment and cast back to the original struct type
413   // for consistency.
414   llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy());
415   Ptr = Builder.CreateInBoundsGEP(Ptr, Adj);
416   This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted");
417 
418   // Load the function pointer.
419   llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr");
420 
421   // If the LSB in the function pointer is 1, the function pointer points to
422   // a virtual function.
423   llvm::Value *IsVirtual;
424   if (UseARMMethodPtrABI)
425     IsVirtual = Builder.CreateAnd(RawAdj, ptrdiff_1);
426   else
427     IsVirtual = Builder.CreateAnd(FnAsInt, ptrdiff_1);
428   IsVirtual = Builder.CreateIsNotNull(IsVirtual, "memptr.isvirtual");
429   Builder.CreateCondBr(IsVirtual, FnVirtual, FnNonVirtual);
430 
431   // In the virtual path, the adjustment left 'This' pointing to the
432   // vtable of the correct base subobject.  The "function pointer" is an
433   // offset within the vtable (+1 for the virtual flag on non-ARM).
434   CGF.EmitBlock(FnVirtual);
435 
436   // Cast the adjusted this to a pointer to vtable pointer and load.
437   llvm::Type *VTableTy = Builder.getInt8PtrTy();
438   llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy);
439 
440   // Apply the offset.
441   llvm::Value *VTableOffset = FnAsInt;
442   if (!UseARMMethodPtrABI)
443     VTableOffset = Builder.CreateSub(VTableOffset, ptrdiff_1);
444   VTable = Builder.CreateGEP(VTable, VTableOffset);
445 
446   // Load the virtual function to call.
447   VTable = Builder.CreateBitCast(VTable, FTy->getPointerTo()->getPointerTo());
448   llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "memptr.virtualfn");
449   CGF.EmitBranch(FnEnd);
450 
451   // In the non-virtual path, the function pointer is actually a
452   // function pointer.
453   CGF.EmitBlock(FnNonVirtual);
454   llvm::Value *NonVirtualFn =
455     Builder.CreateIntToPtr(FnAsInt, FTy->getPointerTo(), "memptr.nonvirtualfn");
456 
457   // We're done.
458   CGF.EmitBlock(FnEnd);
459   llvm::PHINode *Callee = Builder.CreatePHI(FTy->getPointerTo(), 2);
460   Callee->addIncoming(VirtualFn, FnVirtual);
461   Callee->addIncoming(NonVirtualFn, FnNonVirtual);
462   return Callee;
463 }
464 
465 /// Compute an l-value by applying the given pointer-to-member to a
466 /// base object.
467 llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
468     CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr,
469     const MemberPointerType *MPT) {
470   assert(MemPtr->getType() == CGM.PtrDiffTy);
471 
472   CGBuilderTy &Builder = CGF.Builder;
473 
474   unsigned AS = Base->getType()->getPointerAddressSpace();
475 
476   // Cast to char*.
477   Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
478 
479   // Apply the offset, which we assume is non-null.
480   llvm::Value *Addr = Builder.CreateInBoundsGEP(Base, MemPtr, "memptr.offset");
481 
482   // Cast the address to the appropriate pointer type, adopting the
483   // address space of the base pointer.
484   llvm::Type *PType
485     = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
486   return Builder.CreateBitCast(Addr, PType);
487 }
488 
489 /// Perform a bitcast, derived-to-base, or base-to-derived member pointer
490 /// conversion.
491 ///
492 /// Bitcast conversions are always a no-op under Itanium.
493 ///
494 /// Obligatory offset/adjustment diagram:
495 ///         <-- offset -->          <-- adjustment -->
496 ///   |--------------------------|----------------------|--------------------|
497 ///   ^Derived address point     ^Base address point    ^Member address point
498 ///
499 /// So when converting a base member pointer to a derived member pointer,
500 /// we add the offset to the adjustment because the address point has
501 /// decreased;  and conversely, when converting a derived MP to a base MP
502 /// we subtract the offset from the adjustment because the address point
503 /// has increased.
504 ///
505 /// The standard forbids (at compile time) conversion to and from
506 /// virtual bases, which is why we don't have to consider them here.
507 ///
508 /// The standard forbids (at run time) casting a derived MP to a base
509 /// MP when the derived MP does not point to a member of the base.
510 /// This is why -1 is a reasonable choice for null data member
511 /// pointers.
512 llvm::Value *
513 ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
514                                            const CastExpr *E,
515                                            llvm::Value *src) {
516   assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
517          E->getCastKind() == CK_BaseToDerivedMemberPointer ||
518          E->getCastKind() == CK_ReinterpretMemberPointer);
519 
520   // Under Itanium, reinterprets don't require any additional processing.
521   if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
522 
523   // Use constant emission if we can.
524   if (isa<llvm::Constant>(src))
525     return EmitMemberPointerConversion(E, cast<llvm::Constant>(src));
526 
527   llvm::Constant *adj = getMemberPointerAdjustment(E);
528   if (!adj) return src;
529 
530   CGBuilderTy &Builder = CGF.Builder;
531   bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
532 
533   const MemberPointerType *destTy =
534     E->getType()->castAs<MemberPointerType>();
535 
536   // For member data pointers, this is just a matter of adding the
537   // offset if the source is non-null.
538   if (destTy->isMemberDataPointer()) {
539     llvm::Value *dst;
540     if (isDerivedToBase)
541       dst = Builder.CreateNSWSub(src, adj, "adj");
542     else
543       dst = Builder.CreateNSWAdd(src, adj, "adj");
544 
545     // Null check.
546     llvm::Value *null = llvm::Constant::getAllOnesValue(src->getType());
547     llvm::Value *isNull = Builder.CreateICmpEQ(src, null, "memptr.isnull");
548     return Builder.CreateSelect(isNull, src, dst);
549   }
550 
551   // The this-adjustment is left-shifted by 1 on ARM.
552   if (UseARMMethodPtrABI) {
553     uint64_t offset = cast<llvm::ConstantInt>(adj)->getZExtValue();
554     offset <<= 1;
555     adj = llvm::ConstantInt::get(adj->getType(), offset);
556   }
557 
558   llvm::Value *srcAdj = Builder.CreateExtractValue(src, 1, "src.adj");
559   llvm::Value *dstAdj;
560   if (isDerivedToBase)
561     dstAdj = Builder.CreateNSWSub(srcAdj, adj, "adj");
562   else
563     dstAdj = Builder.CreateNSWAdd(srcAdj, adj, "adj");
564 
565   return Builder.CreateInsertValue(src, dstAdj, 1);
566 }
567 
568 llvm::Constant *
569 ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
570                                            llvm::Constant *src) {
571   assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
572          E->getCastKind() == CK_BaseToDerivedMemberPointer ||
573          E->getCastKind() == CK_ReinterpretMemberPointer);
574 
575   // Under Itanium, reinterprets don't require any additional processing.
576   if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
577 
578   // If the adjustment is trivial, we don't need to do anything.
579   llvm::Constant *adj = getMemberPointerAdjustment(E);
580   if (!adj) return src;
581 
582   bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
583 
584   const MemberPointerType *destTy =
585     E->getType()->castAs<MemberPointerType>();
586 
587   // For member data pointers, this is just a matter of adding the
588   // offset if the source is non-null.
589   if (destTy->isMemberDataPointer()) {
590     // null maps to null.
591     if (src->isAllOnesValue()) return src;
592 
593     if (isDerivedToBase)
594       return llvm::ConstantExpr::getNSWSub(src, adj);
595     else
596       return llvm::ConstantExpr::getNSWAdd(src, adj);
597   }
598 
599   // The this-adjustment is left-shifted by 1 on ARM.
600   if (UseARMMethodPtrABI) {
601     uint64_t offset = cast<llvm::ConstantInt>(adj)->getZExtValue();
602     offset <<= 1;
603     adj = llvm::ConstantInt::get(adj->getType(), offset);
604   }
605 
606   llvm::Constant *srcAdj = llvm::ConstantExpr::getExtractValue(src, 1);
607   llvm::Constant *dstAdj;
608   if (isDerivedToBase)
609     dstAdj = llvm::ConstantExpr::getNSWSub(srcAdj, adj);
610   else
611     dstAdj = llvm::ConstantExpr::getNSWAdd(srcAdj, adj);
612 
613   return llvm::ConstantExpr::getInsertValue(src, dstAdj, 1);
614 }
615 
616 llvm::Constant *
617 ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
618   // Itanium C++ ABI 2.3:
619   //   A NULL pointer is represented as -1.
620   if (MPT->isMemberDataPointer())
621     return llvm::ConstantInt::get(CGM.PtrDiffTy, -1ULL, /*isSigned=*/true);
622 
623   llvm::Constant *Zero = llvm::ConstantInt::get(CGM.PtrDiffTy, 0);
624   llvm::Constant *Values[2] = { Zero, Zero };
625   return llvm::ConstantStruct::getAnon(Values);
626 }
627 
628 llvm::Constant *
629 ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
630                                      CharUnits offset) {
631   // Itanium C++ ABI 2.3:
632   //   A pointer to data member is an offset from the base address of
633   //   the class object containing it, represented as a ptrdiff_t
634   return llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity());
635 }
636 
637 llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
638   return BuildMemberPointer(MD, CharUnits::Zero());
639 }
640 
641 llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD,
642                                                   CharUnits ThisAdjustment) {
643   assert(MD->isInstance() && "Member function must not be static!");
644   MD = MD->getCanonicalDecl();
645 
646   CodeGenTypes &Types = CGM.getTypes();
647 
648   // Get the function pointer (or index if this is a virtual function).
649   llvm::Constant *MemPtr[2];
650   if (MD->isVirtual()) {
651     uint64_t Index = CGM.getItaniumVTableContext().getMethodVTableIndex(MD);
652 
653     const ASTContext &Context = getContext();
654     CharUnits PointerWidth =
655       Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
656     uint64_t VTableOffset = (Index * PointerWidth.getQuantity());
657 
658     if (UseARMMethodPtrABI) {
659       // ARM C++ ABI 3.2.1:
660       //   This ABI specifies that adj contains twice the this
661       //   adjustment, plus 1 if the member function is virtual. The
662       //   least significant bit of adj then makes exactly the same
663       //   discrimination as the least significant bit of ptr does for
664       //   Itanium.
665       MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset);
666       MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy,
667                                          2 * ThisAdjustment.getQuantity() + 1);
668     } else {
669       // Itanium C++ ABI 2.3:
670       //   For a virtual function, [the pointer field] is 1 plus the
671       //   virtual table offset (in bytes) of the function,
672       //   represented as a ptrdiff_t.
673       MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1);
674       MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy,
675                                          ThisAdjustment.getQuantity());
676     }
677   } else {
678     const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
679     llvm::Type *Ty;
680     // Check whether the function has a computable LLVM signature.
681     if (Types.isFuncTypeConvertible(FPT)) {
682       // The function has a computable LLVM signature; use the correct type.
683       Ty = Types.GetFunctionType(Types.arrangeCXXMethodDeclaration(MD));
684     } else {
685       // Use an arbitrary non-function type to tell GetAddrOfFunction that the
686       // function type is incomplete.
687       Ty = CGM.PtrDiffTy;
688     }
689     llvm::Constant *addr = CGM.GetAddrOfFunction(MD, Ty);
690 
691     MemPtr[0] = llvm::ConstantExpr::getPtrToInt(addr, CGM.PtrDiffTy);
692     MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy,
693                                        (UseARMMethodPtrABI ? 2 : 1) *
694                                        ThisAdjustment.getQuantity());
695   }
696 
697   return llvm::ConstantStruct::getAnon(MemPtr);
698 }
699 
700 llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
701                                                  QualType MPType) {
702   const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
703   const ValueDecl *MPD = MP.getMemberPointerDecl();
704   if (!MPD)
705     return EmitNullMemberPointer(MPT);
706 
707   CharUnits ThisAdjustment = getMemberPointerPathAdjustment(MP);
708 
709   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD))
710     return BuildMemberPointer(MD, ThisAdjustment);
711 
712   CharUnits FieldOffset =
713     getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD));
714   return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset);
715 }
716 
717 /// The comparison algorithm is pretty easy: the member pointers are
718 /// the same if they're either bitwise identical *or* both null.
719 ///
720 /// ARM is different here only because null-ness is more complicated.
721 llvm::Value *
722 ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
723                                            llvm::Value *L,
724                                            llvm::Value *R,
725                                            const MemberPointerType *MPT,
726                                            bool Inequality) {
727   CGBuilderTy &Builder = CGF.Builder;
728 
729   llvm::ICmpInst::Predicate Eq;
730   llvm::Instruction::BinaryOps And, Or;
731   if (Inequality) {
732     Eq = llvm::ICmpInst::ICMP_NE;
733     And = llvm::Instruction::Or;
734     Or = llvm::Instruction::And;
735   } else {
736     Eq = llvm::ICmpInst::ICMP_EQ;
737     And = llvm::Instruction::And;
738     Or = llvm::Instruction::Or;
739   }
740 
741   // Member data pointers are easy because there's a unique null
742   // value, so it just comes down to bitwise equality.
743   if (MPT->isMemberDataPointer())
744     return Builder.CreateICmp(Eq, L, R);
745 
746   // For member function pointers, the tautologies are more complex.
747   // The Itanium tautology is:
748   //   (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
749   // The ARM tautology is:
750   //   (L == R) <==> (L.ptr == R.ptr &&
751   //                  (L.adj == R.adj ||
752   //                   (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
753   // The inequality tautologies have exactly the same structure, except
754   // applying De Morgan's laws.
755 
756   llvm::Value *LPtr = Builder.CreateExtractValue(L, 0, "lhs.memptr.ptr");
757   llvm::Value *RPtr = Builder.CreateExtractValue(R, 0, "rhs.memptr.ptr");
758 
759   // This condition tests whether L.ptr == R.ptr.  This must always be
760   // true for equality to hold.
761   llvm::Value *PtrEq = Builder.CreateICmp(Eq, LPtr, RPtr, "cmp.ptr");
762 
763   // This condition, together with the assumption that L.ptr == R.ptr,
764   // tests whether the pointers are both null.  ARM imposes an extra
765   // condition.
766   llvm::Value *Zero = llvm::Constant::getNullValue(LPtr->getType());
767   llvm::Value *EqZero = Builder.CreateICmp(Eq, LPtr, Zero, "cmp.ptr.null");
768 
769   // This condition tests whether L.adj == R.adj.  If this isn't
770   // true, the pointers are unequal unless they're both null.
771   llvm::Value *LAdj = Builder.CreateExtractValue(L, 1, "lhs.memptr.adj");
772   llvm::Value *RAdj = Builder.CreateExtractValue(R, 1, "rhs.memptr.adj");
773   llvm::Value *AdjEq = Builder.CreateICmp(Eq, LAdj, RAdj, "cmp.adj");
774 
775   // Null member function pointers on ARM clear the low bit of Adj,
776   // so the zero condition has to check that neither low bit is set.
777   if (UseARMMethodPtrABI) {
778     llvm::Value *One = llvm::ConstantInt::get(LPtr->getType(), 1);
779 
780     // Compute (l.adj | r.adj) & 1 and test it against zero.
781     llvm::Value *OrAdj = Builder.CreateOr(LAdj, RAdj, "or.adj");
782     llvm::Value *OrAdjAnd1 = Builder.CreateAnd(OrAdj, One);
783     llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(Eq, OrAdjAnd1, Zero,
784                                                       "cmp.or.adj");
785     EqZero = Builder.CreateBinOp(And, EqZero, OrAdjAnd1EqZero);
786   }
787 
788   // Tie together all our conditions.
789   llvm::Value *Result = Builder.CreateBinOp(Or, EqZero, AdjEq);
790   Result = Builder.CreateBinOp(And, PtrEq, Result,
791                                Inequality ? "memptr.ne" : "memptr.eq");
792   return Result;
793 }
794 
795 llvm::Value *
796 ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
797                                           llvm::Value *MemPtr,
798                                           const MemberPointerType *MPT) {
799   CGBuilderTy &Builder = CGF.Builder;
800 
801   /// For member data pointers, this is just a check against -1.
802   if (MPT->isMemberDataPointer()) {
803     assert(MemPtr->getType() == CGM.PtrDiffTy);
804     llvm::Value *NegativeOne =
805       llvm::Constant::getAllOnesValue(MemPtr->getType());
806     return Builder.CreateICmpNE(MemPtr, NegativeOne, "memptr.tobool");
807   }
808 
809   // In Itanium, a member function pointer is not null if 'ptr' is not null.
810   llvm::Value *Ptr = Builder.CreateExtractValue(MemPtr, 0, "memptr.ptr");
811 
812   llvm::Constant *Zero = llvm::ConstantInt::get(Ptr->getType(), 0);
813   llvm::Value *Result = Builder.CreateICmpNE(Ptr, Zero, "memptr.tobool");
814 
815   // On ARM, a member function pointer is also non-null if the low bit of 'adj'
816   // (the virtual bit) is set.
817   if (UseARMMethodPtrABI) {
818     llvm::Constant *One = llvm::ConstantInt::get(Ptr->getType(), 1);
819     llvm::Value *Adj = Builder.CreateExtractValue(MemPtr, 1, "memptr.adj");
820     llvm::Value *VirtualBit = Builder.CreateAnd(Adj, One, "memptr.virtualbit");
821     llvm::Value *IsVirtual = Builder.CreateICmpNE(VirtualBit, Zero,
822                                                   "memptr.isvirtual");
823     Result = Builder.CreateOr(Result, IsVirtual);
824   }
825 
826   return Result;
827 }
828 
829 bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
830   const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
831   if (!RD)
832     return false;
833 
834   // Return indirectly if we have a non-trivial copy ctor or non-trivial dtor.
835   // FIXME: Use canCopyArgument() when it is fixed to handle lazily declared
836   // special members.
837   if (RD->hasNonTrivialDestructor() || RD->hasNonTrivialCopyConstructor()) {
838     FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false);
839     return true;
840   }
841   return false;
842 }
843 
844 /// The Itanium ABI requires non-zero initialization only for data
845 /// member pointers, for which '0' is a valid offset.
846 bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
847   return MPT->getPointeeType()->isFunctionType();
848 }
849 
850 /// The Itanium ABI always places an offset to the complete object
851 /// at entry -2 in the vtable.
852 void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
853                                             const CXXDeleteExpr *DE,
854                                             llvm::Value *Ptr,
855                                             QualType ElementType,
856                                             const CXXDestructorDecl *Dtor) {
857   bool UseGlobalDelete = DE->isGlobalDelete();
858   if (UseGlobalDelete) {
859     // Derive the complete-object pointer, which is what we need
860     // to pass to the deallocation function.
861 
862     // Grab the vtable pointer as an intptr_t*.
863     llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo());
864 
865     // Track back to entry -2 and pull out the offset there.
866     llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
867         VTable, -2, "complete-offset.ptr");
868     llvm::LoadInst *Offset = CGF.Builder.CreateLoad(OffsetPtr);
869     Offset->setAlignment(CGF.PointerAlignInBytes);
870 
871     // Apply the offset.
872     llvm::Value *CompletePtr = CGF.Builder.CreateBitCast(Ptr, CGF.Int8PtrTy);
873     CompletePtr = CGF.Builder.CreateInBoundsGEP(CompletePtr, Offset);
874 
875     // If we're supposed to call the global delete, make sure we do so
876     // even if the destructor throws.
877     CGF.pushCallObjectDeleteCleanup(DE->getOperatorDelete(), CompletePtr,
878                                     ElementType);
879   }
880 
881   // FIXME: Provide a source location here even though there's no
882   // CXXMemberCallExpr for dtor call.
883   CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
884   EmitVirtualDestructorCall(CGF, Dtor, DtorType, Ptr, /*CE=*/nullptr);
885 
886   if (UseGlobalDelete)
887     CGF.PopCleanupBlock();
888 }
889 
890 static llvm::Constant *getItaniumDynamicCastFn(CodeGenFunction &CGF) {
891   // void *__dynamic_cast(const void *sub,
892   //                      const abi::__class_type_info *src,
893   //                      const abi::__class_type_info *dst,
894   //                      std::ptrdiff_t src2dst_offset);
895 
896   llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
897   llvm::Type *PtrDiffTy =
898     CGF.ConvertType(CGF.getContext().getPointerDiffType());
899 
900   llvm::Type *Args[4] = { Int8PtrTy, Int8PtrTy, Int8PtrTy, PtrDiffTy };
901 
902   llvm::FunctionType *FTy = llvm::FunctionType::get(Int8PtrTy, Args, false);
903 
904   // Mark the function as nounwind readonly.
905   llvm::Attribute::AttrKind FuncAttrs[] = { llvm::Attribute::NoUnwind,
906                                             llvm::Attribute::ReadOnly };
907   llvm::AttributeSet Attrs = llvm::AttributeSet::get(
908       CGF.getLLVMContext(), llvm::AttributeSet::FunctionIndex, FuncAttrs);
909 
910   return CGF.CGM.CreateRuntimeFunction(FTy, "__dynamic_cast", Attrs);
911 }
912 
913 static llvm::Constant *getBadCastFn(CodeGenFunction &CGF) {
914   // void __cxa_bad_cast();
915   llvm::FunctionType *FTy = llvm::FunctionType::get(CGF.VoidTy, false);
916   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_cast");
917 }
918 
919 /// \brief Compute the src2dst_offset hint as described in the
920 /// Itanium C++ ABI [2.9.7]
921 static CharUnits computeOffsetHint(ASTContext &Context,
922                                    const CXXRecordDecl *Src,
923                                    const CXXRecordDecl *Dst) {
924   CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
925                      /*DetectVirtual=*/false);
926 
927   // If Dst is not derived from Src we can skip the whole computation below and
928   // return that Src is not a public base of Dst.  Record all inheritance paths.
929   if (!Dst->isDerivedFrom(Src, Paths))
930     return CharUnits::fromQuantity(-2ULL);
931 
932   unsigned NumPublicPaths = 0;
933   CharUnits Offset;
934 
935   // Now walk all possible inheritance paths.
936   for (CXXBasePaths::paths_iterator I = Paths.begin(), E = Paths.end(); I != E;
937        ++I) {
938     if (I->Access != AS_public) // Ignore non-public inheritance.
939       continue;
940 
941     ++NumPublicPaths;
942 
943     for (CXXBasePath::iterator J = I->begin(), JE = I->end(); J != JE; ++J) {
944       // If the path contains a virtual base class we can't give any hint.
945       // -1: no hint.
946       if (J->Base->isVirtual())
947         return CharUnits::fromQuantity(-1ULL);
948 
949       if (NumPublicPaths > 1) // Won't use offsets, skip computation.
950         continue;
951 
952       // Accumulate the base class offsets.
953       const ASTRecordLayout &L = Context.getASTRecordLayout(J->Class);
954       Offset += L.getBaseClassOffset(J->Base->getType()->getAsCXXRecordDecl());
955     }
956   }
957 
958   // -2: Src is not a public base of Dst.
959   if (NumPublicPaths == 0)
960     return CharUnits::fromQuantity(-2ULL);
961 
962   // -3: Src is a multiple public base type but never a virtual base type.
963   if (NumPublicPaths > 1)
964     return CharUnits::fromQuantity(-3ULL);
965 
966   // Otherwise, the Src type is a unique public nonvirtual base type of Dst.
967   // Return the offset of Src from the origin of Dst.
968   return Offset;
969 }
970 
971 static llvm::Constant *getBadTypeidFn(CodeGenFunction &CGF) {
972   // void __cxa_bad_typeid();
973   llvm::FunctionType *FTy = llvm::FunctionType::get(CGF.VoidTy, false);
974 
975   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid");
976 }
977 
978 bool ItaniumCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
979                                               QualType SrcRecordTy) {
980   return IsDeref;
981 }
982 
983 void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
984   llvm::Value *Fn = getBadTypeidFn(CGF);
985   CGF.EmitRuntimeCallOrInvoke(Fn).setDoesNotReturn();
986   CGF.Builder.CreateUnreachable();
987 }
988 
989 llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
990                                        QualType SrcRecordTy,
991                                        llvm::Value *ThisPtr,
992                                        llvm::Type *StdTypeInfoPtrTy) {
993   llvm::Value *Value =
994       CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo());
995 
996   // Load the type info.
997   Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL);
998   return CGF.Builder.CreateLoad(Value);
999 }
1000 
1001 bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1002                                                        QualType SrcRecordTy) {
1003   return SrcIsPtr;
1004 }
1005 
1006 llvm::Value *ItaniumCXXABI::EmitDynamicCastCall(
1007     CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy,
1008     QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1009   llvm::Type *PtrDiffLTy =
1010       CGF.ConvertType(CGF.getContext().getPointerDiffType());
1011   llvm::Type *DestLTy = CGF.ConvertType(DestTy);
1012 
1013   llvm::Value *SrcRTTI =
1014       CGF.CGM.GetAddrOfRTTIDescriptor(SrcRecordTy.getUnqualifiedType());
1015   llvm::Value *DestRTTI =
1016       CGF.CGM.GetAddrOfRTTIDescriptor(DestRecordTy.getUnqualifiedType());
1017 
1018   // Compute the offset hint.
1019   const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1020   const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1021   llvm::Value *OffsetHint = llvm::ConstantInt::get(
1022       PtrDiffLTy,
1023       computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
1024 
1025   // Emit the call to __dynamic_cast.
1026   Value = CGF.EmitCastToVoidPtr(Value);
1027 
1028   llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
1029   Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), args);
1030   Value = CGF.Builder.CreateBitCast(Value, DestLTy);
1031 
1032   /// C++ [expr.dynamic.cast]p9:
1033   ///   A failed cast to reference type throws std::bad_cast
1034   if (DestTy->isReferenceType()) {
1035     llvm::BasicBlock *BadCastBlock =
1036         CGF.createBasicBlock("dynamic_cast.bad_cast");
1037 
1038     llvm::Value *IsNull = CGF.Builder.CreateIsNull(Value);
1039     CGF.Builder.CreateCondBr(IsNull, BadCastBlock, CastEnd);
1040 
1041     CGF.EmitBlock(BadCastBlock);
1042     EmitBadCastCall(CGF);
1043   }
1044 
1045   return Value;
1046 }
1047 
1048 llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
1049                                                   llvm::Value *Value,
1050                                                   QualType SrcRecordTy,
1051                                                   QualType DestTy) {
1052   llvm::Type *PtrDiffLTy =
1053       CGF.ConvertType(CGF.getContext().getPointerDiffType());
1054   llvm::Type *DestLTy = CGF.ConvertType(DestTy);
1055 
1056   // Get the vtable pointer.
1057   llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo());
1058 
1059   // Get the offset-to-top from the vtable.
1060   llvm::Value *OffsetToTop =
1061       CGF.Builder.CreateConstInBoundsGEP1_64(VTable, -2ULL);
1062   OffsetToTop = CGF.Builder.CreateLoad(OffsetToTop, "offset.to.top");
1063 
1064   // Finally, add the offset to the pointer.
1065   Value = CGF.EmitCastToVoidPtr(Value);
1066   Value = CGF.Builder.CreateInBoundsGEP(Value, OffsetToTop);
1067 
1068   return CGF.Builder.CreateBitCast(Value, DestLTy);
1069 }
1070 
1071 bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1072   llvm::Value *Fn = getBadCastFn(CGF);
1073   CGF.EmitRuntimeCallOrInvoke(Fn).setDoesNotReturn();
1074   CGF.Builder.CreateUnreachable();
1075   return true;
1076 }
1077 
1078 llvm::Value *
1079 ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
1080                                          llvm::Value *This,
1081                                          const CXXRecordDecl *ClassDecl,
1082                                          const CXXRecordDecl *BaseClassDecl) {
1083   llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy);
1084   CharUnits VBaseOffsetOffset =
1085       CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl,
1086                                                                BaseClassDecl);
1087 
1088   llvm::Value *VBaseOffsetPtr =
1089     CGF.Builder.CreateConstGEP1_64(VTablePtr, VBaseOffsetOffset.getQuantity(),
1090                                    "vbase.offset.ptr");
1091   VBaseOffsetPtr = CGF.Builder.CreateBitCast(VBaseOffsetPtr,
1092                                              CGM.PtrDiffTy->getPointerTo());
1093 
1094   llvm::Value *VBaseOffset =
1095     CGF.Builder.CreateLoad(VBaseOffsetPtr, "vbase.offset");
1096 
1097   return VBaseOffset;
1098 }
1099 
1100 void ItaniumCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1101   // Just make sure we're in sync with TargetCXXABI.
1102   assert(CGM.getTarget().getCXXABI().hasConstructorVariants());
1103 
1104   // The constructor used for constructing this as a base class;
1105   // ignores virtual bases.
1106   CGM.EmitGlobal(GlobalDecl(D, Ctor_Base));
1107 
1108   // The constructor used for constructing this as a complete class;
1109   // constucts the virtual bases, then calls the base constructor.
1110   if (!D->getParent()->isAbstract()) {
1111     // We don't need to emit the complete ctor if the class is abstract.
1112     CGM.EmitGlobal(GlobalDecl(D, Ctor_Complete));
1113   }
1114 }
1115 
1116 void
1117 ItaniumCXXABI::buildStructorSignature(const CXXMethodDecl *MD, StructorType T,
1118                                       SmallVectorImpl<CanQualType> &ArgTys) {
1119   ASTContext &Context = getContext();
1120 
1121   // All parameters are already in place except VTT, which goes after 'this'.
1122   // These are Clang types, so we don't need to worry about sret yet.
1123 
1124   // Check if we need to add a VTT parameter (which has type void **).
1125   if (T == StructorType::Base && MD->getParent()->getNumVBases() != 0)
1126     ArgTys.insert(ArgTys.begin() + 1,
1127                   Context.getPointerType(Context.VoidPtrTy));
1128 }
1129 
1130 void ItaniumCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1131   // The destructor used for destructing this as a base class; ignores
1132   // virtual bases.
1133   CGM.EmitGlobal(GlobalDecl(D, Dtor_Base));
1134 
1135   // The destructor used for destructing this as a most-derived class;
1136   // call the base destructor and then destructs any virtual bases.
1137   CGM.EmitGlobal(GlobalDecl(D, Dtor_Complete));
1138 
1139   // The destructor in a virtual table is always a 'deleting'
1140   // destructor, which calls the complete destructor and then uses the
1141   // appropriate operator delete.
1142   if (D->isVirtual())
1143     CGM.EmitGlobal(GlobalDecl(D, Dtor_Deleting));
1144 }
1145 
1146 void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1147                                               QualType &ResTy,
1148                                               FunctionArgList &Params) {
1149   const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
1150   assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1151 
1152   // Check if we need a VTT parameter as well.
1153   if (NeedsVTTParameter(CGF.CurGD)) {
1154     ASTContext &Context = getContext();
1155 
1156     // FIXME: avoid the fake decl
1157     QualType T = Context.getPointerType(Context.VoidPtrTy);
1158     ImplicitParamDecl *VTTDecl
1159       = ImplicitParamDecl::Create(Context, nullptr, MD->getLocation(),
1160                                   &Context.Idents.get("vtt"), T);
1161     Params.insert(Params.begin() + 1, VTTDecl);
1162     getStructorImplicitParamDecl(CGF) = VTTDecl;
1163   }
1164 }
1165 
1166 void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1167   /// Initialize the 'this' slot.
1168   EmitThisParam(CGF);
1169 
1170   /// Initialize the 'vtt' slot if needed.
1171   if (getStructorImplicitParamDecl(CGF)) {
1172     getStructorImplicitParamValue(CGF) = CGF.Builder.CreateLoad(
1173         CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)), "vtt");
1174   }
1175 
1176   /// If this is a function that the ABI specifies returns 'this', initialize
1177   /// the return slot to 'this' at the start of the function.
1178   ///
1179   /// Unlike the setting of return types, this is done within the ABI
1180   /// implementation instead of by clients of CGCXXABI because:
1181   /// 1) getThisValue is currently protected
1182   /// 2) in theory, an ABI could implement 'this' returns some other way;
1183   ///    HasThisReturn only specifies a contract, not the implementation
1184   if (HasThisReturn(CGF.CurGD))
1185     CGF.Builder.CreateStore(getThisValue(CGF), CGF.ReturnValue);
1186 }
1187 
1188 unsigned ItaniumCXXABI::addImplicitConstructorArgs(
1189     CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
1190     bool ForVirtualBase, bool Delegating, CallArgList &Args) {
1191   if (!NeedsVTTParameter(GlobalDecl(D, Type)))
1192     return 0;
1193 
1194   // Insert the implicit 'vtt' argument as the second argument.
1195   llvm::Value *VTT =
1196       CGF.GetVTTParameter(GlobalDecl(D, Type), ForVirtualBase, Delegating);
1197   QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
1198   Args.insert(Args.begin() + 1,
1199               CallArg(RValue::get(VTT), VTTTy, /*needscopy=*/false));
1200   return 1;  // Added one arg.
1201 }
1202 
1203 void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
1204                                        const CXXDestructorDecl *DD,
1205                                        CXXDtorType Type, bool ForVirtualBase,
1206                                        bool Delegating, llvm::Value *This) {
1207   GlobalDecl GD(DD, Type);
1208   llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
1209   QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
1210 
1211   llvm::Value *Callee = nullptr;
1212   if (getContext().getLangOpts().AppleKext)
1213     Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, DD->getParent());
1214 
1215   if (!Callee)
1216     Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type));
1217 
1218   CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), This, VTT,
1219                                   VTTTy, nullptr);
1220 }
1221 
1222 void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
1223                                           const CXXRecordDecl *RD) {
1224   llvm::GlobalVariable *VTable = getAddrOfVTable(RD, CharUnits());
1225   if (VTable->hasInitializer())
1226     return;
1227 
1228   ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
1229   const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
1230   llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
1231   llvm::Constant *RTTI =
1232       CGM.GetAddrOfRTTIDescriptor(CGM.getContext().getTagDeclType(RD));
1233 
1234   // Create and set the initializer.
1235   llvm::Constant *Init = CGVT.CreateVTableInitializer(
1236       RD, VTLayout.vtable_component_begin(), VTLayout.getNumVTableComponents(),
1237       VTLayout.vtable_thunk_begin(), VTLayout.getNumVTableThunks(), RTTI);
1238   VTable->setInitializer(Init);
1239 
1240   // Set the correct linkage.
1241   VTable->setLinkage(Linkage);
1242 
1243   // Set the right visibility.
1244   CGM.setGlobalVisibility(VTable, RD);
1245 
1246   // Use pointer alignment for the vtable. Otherwise we would align them based
1247   // on the size of the initializer which doesn't make sense as only single
1248   // values are read.
1249   unsigned PAlign = CGM.getTarget().getPointerAlign(0);
1250   VTable->setAlignment(getContext().toCharUnitsFromBits(PAlign).getQuantity());
1251 
1252   // If this is the magic class __cxxabiv1::__fundamental_type_info,
1253   // we will emit the typeinfo for the fundamental types. This is the
1254   // same behaviour as GCC.
1255   const DeclContext *DC = RD->getDeclContext();
1256   if (RD->getIdentifier() &&
1257       RD->getIdentifier()->isStr("__fundamental_type_info") &&
1258       isa<NamespaceDecl>(DC) && cast<NamespaceDecl>(DC)->getIdentifier() &&
1259       cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") &&
1260       DC->getParent()->isTranslationUnit())
1261     EmitFundamentalRTTIDescriptors();
1262 }
1263 
1264 llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
1265     CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
1266     const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) {
1267   bool NeedsVTTParam = CGM.getCXXABI().NeedsVTTParameter(CGF.CurGD);
1268   NeedsVirtualOffset = (NeedsVTTParam && NearestVBase);
1269 
1270   llvm::Value *VTableAddressPoint;
1271   if (NeedsVTTParam && (Base.getBase()->getNumVBases() || NearestVBase)) {
1272     // Get the secondary vpointer index.
1273     uint64_t VirtualPointerIndex =
1274         CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base);
1275 
1276     /// Load the VTT.
1277     llvm::Value *VTT = CGF.LoadCXXVTT();
1278     if (VirtualPointerIndex)
1279       VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex);
1280 
1281     // And load the address point from the VTT.
1282     VTableAddressPoint = CGF.Builder.CreateLoad(VTT);
1283   } else {
1284     llvm::Constant *VTable =
1285         CGM.getCXXABI().getAddrOfVTable(VTableClass, CharUnits());
1286     uint64_t AddressPoint = CGM.getItaniumVTableContext()
1287                                 .getVTableLayout(VTableClass)
1288                                 .getAddressPoint(Base);
1289     VTableAddressPoint =
1290         CGF.Builder.CreateConstInBoundsGEP2_64(VTable, 0, AddressPoint);
1291   }
1292 
1293   return VTableAddressPoint;
1294 }
1295 
1296 llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr(
1297     BaseSubobject Base, const CXXRecordDecl *VTableClass) {
1298   llvm::Constant *VTable = getAddrOfVTable(VTableClass, CharUnits());
1299 
1300   // Find the appropriate vtable within the vtable group.
1301   uint64_t AddressPoint = CGM.getItaniumVTableContext()
1302                               .getVTableLayout(VTableClass)
1303                               .getAddressPoint(Base);
1304   llvm::Value *Indices[] = {
1305     llvm::ConstantInt::get(CGM.Int64Ty, 0),
1306     llvm::ConstantInt::get(CGM.Int64Ty, AddressPoint)
1307   };
1308 
1309   return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Indices);
1310 }
1311 
1312 llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
1313                                                      CharUnits VPtrOffset) {
1314   assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
1315 
1316   llvm::GlobalVariable *&VTable = VTables[RD];
1317   if (VTable)
1318     return VTable;
1319 
1320   // Queue up this v-table for possible deferred emission.
1321   CGM.addDeferredVTable(RD);
1322 
1323   SmallString<256> OutName;
1324   llvm::raw_svector_ostream Out(OutName);
1325   getMangleContext().mangleCXXVTable(RD, Out);
1326   Out.flush();
1327   StringRef Name = OutName.str();
1328 
1329   ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
1330   llvm::ArrayType *ArrayType = llvm::ArrayType::get(
1331       CGM.Int8PtrTy, VTContext.getVTableLayout(RD).getNumVTableComponents());
1332 
1333   VTable = CGM.CreateOrReplaceCXXRuntimeVariable(
1334       Name, ArrayType, llvm::GlobalValue::ExternalLinkage);
1335   VTable->setUnnamedAddr(true);
1336 
1337   if (RD->hasAttr<DLLImportAttr>())
1338     VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
1339   else if (RD->hasAttr<DLLExportAttr>())
1340     VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
1341 
1342   return VTable;
1343 }
1344 
1345 llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
1346                                                       GlobalDecl GD,
1347                                                       llvm::Value *This,
1348                                                       llvm::Type *Ty) {
1349   GD = GD.getCanonicalDecl();
1350   Ty = Ty->getPointerTo()->getPointerTo();
1351   llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
1352 
1353   uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
1354   llvm::Value *VFuncPtr =
1355       CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn");
1356   return CGF.Builder.CreateLoad(VFuncPtr);
1357 }
1358 
1359 llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
1360     CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
1361     llvm::Value *This, const CXXMemberCallExpr *CE) {
1362   assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
1363   assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
1364 
1365   const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration(
1366       Dtor, getFromDtorType(DtorType));
1367   llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo);
1368   llvm::Value *Callee =
1369       getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty);
1370 
1371   CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This,
1372                                   /*ImplicitParam=*/nullptr, QualType(), CE);
1373   return nullptr;
1374 }
1375 
1376 void ItaniumCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
1377   CodeGenVTables &VTables = CGM.getVTables();
1378   llvm::GlobalVariable *VTT = VTables.GetAddrOfVTT(RD);
1379   VTables.EmitVTTDefinition(VTT, CGM.getVTableLinkage(RD), RD);
1380 }
1381 
1382 static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
1383                                           llvm::Value *Ptr,
1384                                           int64_t NonVirtualAdjustment,
1385                                           int64_t VirtualAdjustment,
1386                                           bool IsReturnAdjustment) {
1387   if (!NonVirtualAdjustment && !VirtualAdjustment)
1388     return Ptr;
1389 
1390   llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
1391   llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
1392 
1393   if (NonVirtualAdjustment && !IsReturnAdjustment) {
1394     // Perform the non-virtual adjustment for a base-to-derived cast.
1395     V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
1396   }
1397 
1398   if (VirtualAdjustment) {
1399     llvm::Type *PtrDiffTy =
1400         CGF.ConvertType(CGF.getContext().getPointerDiffType());
1401 
1402     // Perform the virtual adjustment.
1403     llvm::Value *VTablePtrPtr =
1404         CGF.Builder.CreateBitCast(V, Int8PtrTy->getPointerTo());
1405 
1406     llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr);
1407 
1408     llvm::Value *OffsetPtr =
1409         CGF.Builder.CreateConstInBoundsGEP1_64(VTablePtr, VirtualAdjustment);
1410 
1411     OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo());
1412 
1413     // Load the adjustment offset from the vtable.
1414     llvm::Value *Offset = CGF.Builder.CreateLoad(OffsetPtr);
1415 
1416     // Adjust our pointer.
1417     V = CGF.Builder.CreateInBoundsGEP(V, Offset);
1418   }
1419 
1420   if (NonVirtualAdjustment && IsReturnAdjustment) {
1421     // Perform the non-virtual adjustment for a derived-to-base cast.
1422     V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
1423   }
1424 
1425   // Cast back to the original type.
1426   return CGF.Builder.CreateBitCast(V, Ptr->getType());
1427 }
1428 
1429 llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF,
1430                                                   llvm::Value *This,
1431                                                   const ThisAdjustment &TA) {
1432   return performTypeAdjustment(CGF, This, TA.NonVirtual,
1433                                TA.Virtual.Itanium.VCallOffsetOffset,
1434                                /*IsReturnAdjustment=*/false);
1435 }
1436 
1437 llvm::Value *
1438 ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
1439                                        const ReturnAdjustment &RA) {
1440   return performTypeAdjustment(CGF, Ret, RA.NonVirtual,
1441                                RA.Virtual.Itanium.VBaseOffsetOffset,
1442                                /*IsReturnAdjustment=*/true);
1443 }
1444 
1445 void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
1446                                     RValue RV, QualType ResultType) {
1447   if (!isa<CXXDestructorDecl>(CGF.CurGD.getDecl()))
1448     return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
1449 
1450   // Destructor thunks in the ARM ABI have indeterminate results.
1451   llvm::Type *T =
1452     cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType();
1453   RValue Undef = RValue::get(llvm::UndefValue::get(T));
1454   return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType);
1455 }
1456 
1457 /************************** Array allocation cookies **************************/
1458 
1459 CharUnits ItaniumCXXABI::getArrayCookieSizeImpl(QualType elementType) {
1460   // The array cookie is a size_t; pad that up to the element alignment.
1461   // The cookie is actually right-justified in that space.
1462   return std::max(CharUnits::fromQuantity(CGM.SizeSizeInBytes),
1463                   CGM.getContext().getTypeAlignInChars(elementType));
1464 }
1465 
1466 llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
1467                                                   llvm::Value *NewPtr,
1468                                                   llvm::Value *NumElements,
1469                                                   const CXXNewExpr *expr,
1470                                                   QualType ElementType) {
1471   assert(requiresArrayCookie(expr));
1472 
1473   unsigned AS = NewPtr->getType()->getPointerAddressSpace();
1474 
1475   ASTContext &Ctx = getContext();
1476   QualType SizeTy = Ctx.getSizeType();
1477   CharUnits SizeSize = Ctx.getTypeSizeInChars(SizeTy);
1478 
1479   // The size of the cookie.
1480   CharUnits CookieSize =
1481     std::max(SizeSize, Ctx.getTypeAlignInChars(ElementType));
1482   assert(CookieSize == getArrayCookieSizeImpl(ElementType));
1483 
1484   // Compute an offset to the cookie.
1485   llvm::Value *CookiePtr = NewPtr;
1486   CharUnits CookieOffset = CookieSize - SizeSize;
1487   if (!CookieOffset.isZero())
1488     CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_64(CookiePtr,
1489                                                  CookieOffset.getQuantity());
1490 
1491   // Write the number of elements into the appropriate slot.
1492   llvm::Type *NumElementsTy = CGF.ConvertType(SizeTy)->getPointerTo(AS);
1493   llvm::Value *NumElementsPtr =
1494       CGF.Builder.CreateBitCast(CookiePtr, NumElementsTy);
1495   llvm::Instruction *SI = CGF.Builder.CreateStore(NumElements, NumElementsPtr);
1496   if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && AS == 0 &&
1497       expr->getOperatorNew()->isReplaceableGlobalAllocationFunction()) {
1498     // The store to the CookiePtr does not need to be instrumented.
1499     CGM.getSanitizerMetadata()->disableSanitizerForInstruction(SI);
1500     llvm::FunctionType *FTy =
1501         llvm::FunctionType::get(CGM.VoidTy, NumElementsTy, false);
1502     llvm::Constant *F =
1503         CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie");
1504     CGF.Builder.CreateCall(F, NumElementsPtr);
1505   }
1506 
1507   // Finally, compute a pointer to the actual data buffer by skipping
1508   // over the cookie completely.
1509   return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr,
1510                                                 CookieSize.getQuantity());
1511 }
1512 
1513 llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
1514                                                 llvm::Value *allocPtr,
1515                                                 CharUnits cookieSize) {
1516   // The element size is right-justified in the cookie.
1517   llvm::Value *numElementsPtr = allocPtr;
1518   CharUnits numElementsOffset =
1519     cookieSize - CharUnits::fromQuantity(CGF.SizeSizeInBytes);
1520   if (!numElementsOffset.isZero())
1521     numElementsPtr =
1522       CGF.Builder.CreateConstInBoundsGEP1_64(numElementsPtr,
1523                                              numElementsOffset.getQuantity());
1524 
1525   unsigned AS = allocPtr->getType()->getPointerAddressSpace();
1526   numElementsPtr =
1527     CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
1528   if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) || AS != 0)
1529     return CGF.Builder.CreateLoad(numElementsPtr);
1530   // In asan mode emit a function call instead of a regular load and let the
1531   // run-time deal with it: if the shadow is properly poisoned return the
1532   // cookie, otherwise return 0 to avoid an infinite loop calling DTORs.
1533   // We can't simply ignore this load using nosanitize metadata because
1534   // the metadata may be lost.
1535   llvm::FunctionType *FTy =
1536       llvm::FunctionType::get(CGF.SizeTy, CGF.SizeTy->getPointerTo(0), false);
1537   llvm::Constant *F =
1538       CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie");
1539   return CGF.Builder.CreateCall(F, numElementsPtr);
1540 }
1541 
1542 CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
1543   // ARM says that the cookie is always:
1544   //   struct array_cookie {
1545   //     std::size_t element_size; // element_size != 0
1546   //     std::size_t element_count;
1547   //   };
1548   // But the base ABI doesn't give anything an alignment greater than
1549   // 8, so we can dismiss this as typical ABI-author blindness to
1550   // actual language complexity and round up to the element alignment.
1551   return std::max(CharUnits::fromQuantity(2 * CGM.SizeSizeInBytes),
1552                   CGM.getContext().getTypeAlignInChars(elementType));
1553 }
1554 
1555 llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
1556                                               llvm::Value *newPtr,
1557                                               llvm::Value *numElements,
1558                                               const CXXNewExpr *expr,
1559                                               QualType elementType) {
1560   assert(requiresArrayCookie(expr));
1561 
1562   // NewPtr is a char*, but we generalize to arbitrary addrspaces.
1563   unsigned AS = newPtr->getType()->getPointerAddressSpace();
1564 
1565   // The cookie is always at the start of the buffer.
1566   llvm::Value *cookie = newPtr;
1567 
1568   // The first element is the element size.
1569   cookie = CGF.Builder.CreateBitCast(cookie, CGF.SizeTy->getPointerTo(AS));
1570   llvm::Value *elementSize = llvm::ConstantInt::get(CGF.SizeTy,
1571                  getContext().getTypeSizeInChars(elementType).getQuantity());
1572   CGF.Builder.CreateStore(elementSize, cookie);
1573 
1574   // The second element is the element count.
1575   cookie = CGF.Builder.CreateConstInBoundsGEP1_32(cookie, 1);
1576   CGF.Builder.CreateStore(numElements, cookie);
1577 
1578   // Finally, compute a pointer to the actual data buffer by skipping
1579   // over the cookie completely.
1580   CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
1581   return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr,
1582                                                 cookieSize.getQuantity());
1583 }
1584 
1585 llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
1586                                             llvm::Value *allocPtr,
1587                                             CharUnits cookieSize) {
1588   // The number of elements is at offset sizeof(size_t) relative to
1589   // the allocated pointer.
1590   llvm::Value *numElementsPtr
1591     = CGF.Builder.CreateConstInBoundsGEP1_64(allocPtr, CGF.SizeSizeInBytes);
1592 
1593   unsigned AS = allocPtr->getType()->getPointerAddressSpace();
1594   numElementsPtr =
1595     CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
1596   return CGF.Builder.CreateLoad(numElementsPtr);
1597 }
1598 
1599 /*********************** Static local initialization **************************/
1600 
1601 static llvm::Constant *getGuardAcquireFn(CodeGenModule &CGM,
1602                                          llvm::PointerType *GuardPtrTy) {
1603   // int __cxa_guard_acquire(__guard *guard_object);
1604   llvm::FunctionType *FTy =
1605     llvm::FunctionType::get(CGM.getTypes().ConvertType(CGM.getContext().IntTy),
1606                             GuardPtrTy, /*isVarArg=*/false);
1607   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire",
1608                                    llvm::AttributeSet::get(CGM.getLLVMContext(),
1609                                               llvm::AttributeSet::FunctionIndex,
1610                                                  llvm::Attribute::NoUnwind));
1611 }
1612 
1613 static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
1614                                          llvm::PointerType *GuardPtrTy) {
1615   // void __cxa_guard_release(__guard *guard_object);
1616   llvm::FunctionType *FTy =
1617     llvm::FunctionType::get(CGM.VoidTy, GuardPtrTy, /*isVarArg=*/false);
1618   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release",
1619                                    llvm::AttributeSet::get(CGM.getLLVMContext(),
1620                                               llvm::AttributeSet::FunctionIndex,
1621                                                  llvm::Attribute::NoUnwind));
1622 }
1623 
1624 static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM,
1625                                        llvm::PointerType *GuardPtrTy) {
1626   // void __cxa_guard_abort(__guard *guard_object);
1627   llvm::FunctionType *FTy =
1628     llvm::FunctionType::get(CGM.VoidTy, GuardPtrTy, /*isVarArg=*/false);
1629   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort",
1630                                    llvm::AttributeSet::get(CGM.getLLVMContext(),
1631                                               llvm::AttributeSet::FunctionIndex,
1632                                                  llvm::Attribute::NoUnwind));
1633 }
1634 
1635 namespace {
1636   struct CallGuardAbort : EHScopeStack::Cleanup {
1637     llvm::GlobalVariable *Guard;
1638     CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
1639 
1640     void Emit(CodeGenFunction &CGF, Flags flags) override {
1641       CGF.EmitNounwindRuntimeCall(getGuardAbortFn(CGF.CGM, Guard->getType()),
1642                                   Guard);
1643     }
1644   };
1645 }
1646 
1647 /// The ARM code here follows the Itanium code closely enough that we
1648 /// just special-case it at particular places.
1649 void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
1650                                     const VarDecl &D,
1651                                     llvm::GlobalVariable *var,
1652                                     bool shouldPerformInit) {
1653   CGBuilderTy &Builder = CGF.Builder;
1654 
1655   // We only need to use thread-safe statics for local non-TLS variables;
1656   // global initialization is always single-threaded.
1657   bool threadsafe = getContext().getLangOpts().ThreadsafeStatics &&
1658                     D.isLocalVarDecl() && !D.getTLSKind();
1659 
1660   // If we have a global variable with internal linkage and thread-safe statics
1661   // are disabled, we can just let the guard variable be of type i8.
1662   bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
1663 
1664   llvm::IntegerType *guardTy;
1665   if (useInt8GuardVariable) {
1666     guardTy = CGF.Int8Ty;
1667   } else {
1668     // Guard variables are 64 bits in the generic ABI and size width on ARM
1669     // (i.e. 32-bit on AArch32, 64-bit on AArch64).
1670     guardTy = (UseARMGuardVarABI ? CGF.SizeTy : CGF.Int64Ty);
1671   }
1672   llvm::PointerType *guardPtrTy = guardTy->getPointerTo();
1673 
1674   // Create the guard variable if we don't already have it (as we
1675   // might if we're double-emitting this function body).
1676   llvm::GlobalVariable *guard = CGM.getStaticLocalDeclGuardAddress(&D);
1677   if (!guard) {
1678     // Mangle the name for the guard.
1679     SmallString<256> guardName;
1680     {
1681       llvm::raw_svector_ostream out(guardName);
1682       getMangleContext().mangleStaticGuardVariable(&D, out);
1683       out.flush();
1684     }
1685 
1686     // Create the guard variable with a zero-initializer.
1687     // Just absorb linkage and visibility from the guarded variable.
1688     guard = new llvm::GlobalVariable(CGM.getModule(), guardTy,
1689                                      false, var->getLinkage(),
1690                                      llvm::ConstantInt::get(guardTy, 0),
1691                                      guardName.str());
1692     guard->setVisibility(var->getVisibility());
1693     // If the variable is thread-local, so is its guard variable.
1694     guard->setThreadLocalMode(var->getThreadLocalMode());
1695 
1696     // The ABI says: It is suggested that it be emitted in the same COMDAT group
1697     // as the associated data object
1698     if (!D.isLocalVarDecl() && var->isWeakForLinker() && CGM.supportsCOMDAT()) {
1699       llvm::Comdat *C = CGM.getModule().getOrInsertComdat(var->getName());
1700       guard->setComdat(C);
1701       var->setComdat(C);
1702       CGF.CurFn->setComdat(C);
1703     }
1704 
1705     CGM.setStaticLocalDeclGuardAddress(&D, guard);
1706   }
1707 
1708   // Test whether the variable has completed initialization.
1709   //
1710   // Itanium C++ ABI 3.3.2:
1711   //   The following is pseudo-code showing how these functions can be used:
1712   //     if (obj_guard.first_byte == 0) {
1713   //       if ( __cxa_guard_acquire (&obj_guard) ) {
1714   //         try {
1715   //           ... initialize the object ...;
1716   //         } catch (...) {
1717   //            __cxa_guard_abort (&obj_guard);
1718   //            throw;
1719   //         }
1720   //         ... queue object destructor with __cxa_atexit() ...;
1721   //         __cxa_guard_release (&obj_guard);
1722   //       }
1723   //     }
1724 
1725   // Load the first byte of the guard variable.
1726   llvm::LoadInst *LI =
1727       Builder.CreateLoad(Builder.CreateBitCast(guard, CGM.Int8PtrTy));
1728   LI->setAlignment(1);
1729 
1730   // Itanium ABI:
1731   //   An implementation supporting thread-safety on multiprocessor
1732   //   systems must also guarantee that references to the initialized
1733   //   object do not occur before the load of the initialization flag.
1734   //
1735   // In LLVM, we do this by marking the load Acquire.
1736   if (threadsafe)
1737     LI->setAtomic(llvm::Acquire);
1738 
1739   // For ARM, we should only check the first bit, rather than the entire byte:
1740   //
1741   // ARM C++ ABI 3.2.3.1:
1742   //   To support the potential use of initialization guard variables
1743   //   as semaphores that are the target of ARM SWP and LDREX/STREX
1744   //   synchronizing instructions we define a static initialization
1745   //   guard variable to be a 4-byte aligned, 4-byte word with the
1746   //   following inline access protocol.
1747   //     #define INITIALIZED 1
1748   //     if ((obj_guard & INITIALIZED) != INITIALIZED) {
1749   //       if (__cxa_guard_acquire(&obj_guard))
1750   //         ...
1751   //     }
1752   //
1753   // and similarly for ARM64:
1754   //
1755   // ARM64 C++ ABI 3.2.2:
1756   //   This ABI instead only specifies the value bit 0 of the static guard
1757   //   variable; all other bits are platform defined. Bit 0 shall be 0 when the
1758   //   variable is not initialized and 1 when it is.
1759   llvm::Value *V =
1760       (UseARMGuardVarABI && !useInt8GuardVariable)
1761           ? Builder.CreateAnd(LI, llvm::ConstantInt::get(CGM.Int8Ty, 1))
1762           : LI;
1763   llvm::Value *isInitialized = Builder.CreateIsNull(V, "guard.uninitialized");
1764 
1765   llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock("init.check");
1766   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end");
1767 
1768   // Check if the first byte of the guard variable is zero.
1769   Builder.CreateCondBr(isInitialized, InitCheckBlock, EndBlock);
1770 
1771   CGF.EmitBlock(InitCheckBlock);
1772 
1773   // Variables used when coping with thread-safe statics and exceptions.
1774   if (threadsafe) {
1775     // Call __cxa_guard_acquire.
1776     llvm::Value *V
1777       = CGF.EmitNounwindRuntimeCall(getGuardAcquireFn(CGM, guardPtrTy), guard);
1778 
1779     llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init");
1780 
1781     Builder.CreateCondBr(Builder.CreateIsNotNull(V, "tobool"),
1782                          InitBlock, EndBlock);
1783 
1784     // Call __cxa_guard_abort along the exceptional edge.
1785     CGF.EHStack.pushCleanup<CallGuardAbort>(EHCleanup, guard);
1786 
1787     CGF.EmitBlock(InitBlock);
1788   }
1789 
1790   // Emit the initializer and add a global destructor if appropriate.
1791   CGF.EmitCXXGlobalVarDeclInit(D, var, shouldPerformInit);
1792 
1793   if (threadsafe) {
1794     // Pop the guard-abort cleanup if we pushed one.
1795     CGF.PopCleanupBlock();
1796 
1797     // Call __cxa_guard_release.  This cannot throw.
1798     CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), guard);
1799   } else {
1800     Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guard);
1801   }
1802 
1803   CGF.EmitBlock(EndBlock);
1804 }
1805 
1806 /// Register a global destructor using __cxa_atexit.
1807 static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF,
1808                                         llvm::Constant *dtor,
1809                                         llvm::Constant *addr,
1810                                         bool TLS) {
1811   const char *Name = "__cxa_atexit";
1812   if (TLS) {
1813     const llvm::Triple &T = CGF.getTarget().getTriple();
1814     Name = T.isMacOSX() ?  "_tlv_atexit" : "__cxa_thread_atexit";
1815   }
1816 
1817   // We're assuming that the destructor function is something we can
1818   // reasonably call with the default CC.  Go ahead and cast it to the
1819   // right prototype.
1820   llvm::Type *dtorTy =
1821     llvm::FunctionType::get(CGF.VoidTy, CGF.Int8PtrTy, false)->getPointerTo();
1822 
1823   // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
1824   llvm::Type *paramTys[] = { dtorTy, CGF.Int8PtrTy, CGF.Int8PtrTy };
1825   llvm::FunctionType *atexitTy =
1826     llvm::FunctionType::get(CGF.IntTy, paramTys, false);
1827 
1828   // Fetch the actual function.
1829   llvm::Constant *atexit = CGF.CGM.CreateRuntimeFunction(atexitTy, Name);
1830   if (llvm::Function *fn = dyn_cast<llvm::Function>(atexit))
1831     fn->setDoesNotThrow();
1832 
1833   // Create a variable that binds the atexit to this shared object.
1834   llvm::Constant *handle =
1835     CGF.CGM.CreateRuntimeVariable(CGF.Int8Ty, "__dso_handle");
1836 
1837   llvm::Value *args[] = {
1838     llvm::ConstantExpr::getBitCast(dtor, dtorTy),
1839     llvm::ConstantExpr::getBitCast(addr, CGF.Int8PtrTy),
1840     handle
1841   };
1842   CGF.EmitNounwindRuntimeCall(atexit, args);
1843 }
1844 
1845 /// Register a global destructor as best as we know how.
1846 void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF,
1847                                        const VarDecl &D,
1848                                        llvm::Constant *dtor,
1849                                        llvm::Constant *addr) {
1850   // Use __cxa_atexit if available.
1851   if (CGM.getCodeGenOpts().CXAAtExit)
1852     return emitGlobalDtorWithCXAAtExit(CGF, dtor, addr, D.getTLSKind());
1853 
1854   if (D.getTLSKind())
1855     CGM.ErrorUnsupported(&D, "non-trivial TLS destruction");
1856 
1857   // In Apple kexts, we want to add a global destructor entry.
1858   // FIXME: shouldn't this be guarded by some variable?
1859   if (CGM.getLangOpts().AppleKext) {
1860     // Generate a global destructor entry.
1861     return CGM.AddCXXDtorEntry(dtor, addr);
1862   }
1863 
1864   CGF.registerGlobalDtorWithAtExit(D, dtor, addr);
1865 }
1866 
1867 static bool isThreadWrapperReplaceable(const VarDecl *VD,
1868                                        CodeGen::CodeGenModule &CGM) {
1869   assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!");
1870   // OS X prefers to have references to thread local variables to go through
1871   // the thread wrapper instead of directly referencing the backing variable.
1872   return VD->getTLSKind() == VarDecl::TLS_Dynamic &&
1873          CGM.getTarget().getTriple().isMacOSX();
1874 }
1875 
1876 /// Get the appropriate linkage for the wrapper function. This is essentially
1877 /// the weak form of the variable's linkage; every translation unit which needs
1878 /// the wrapper emits a copy, and we want the linker to merge them.
1879 static llvm::GlobalValue::LinkageTypes
1880 getThreadLocalWrapperLinkage(const VarDecl *VD, CodeGen::CodeGenModule &CGM) {
1881   llvm::GlobalValue::LinkageTypes VarLinkage =
1882       CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false);
1883 
1884   // For internal linkage variables, we don't need an external or weak wrapper.
1885   if (llvm::GlobalValue::isLocalLinkage(VarLinkage))
1886     return VarLinkage;
1887 
1888   // If the thread wrapper is replaceable, give it appropriate linkage.
1889   if (isThreadWrapperReplaceable(VD, CGM)) {
1890     if (llvm::GlobalVariable::isLinkOnceLinkage(VarLinkage) ||
1891         llvm::GlobalVariable::isWeakODRLinkage(VarLinkage))
1892       return llvm::GlobalVariable::WeakAnyLinkage;
1893     return VarLinkage;
1894   }
1895   return llvm::GlobalValue::WeakODRLinkage;
1896 }
1897 
1898 llvm::Function *
1899 ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
1900                                              llvm::Value *Val) {
1901   // Mangle the name for the thread_local wrapper function.
1902   SmallString<256> WrapperName;
1903   {
1904     llvm::raw_svector_ostream Out(WrapperName);
1905     getMangleContext().mangleItaniumThreadLocalWrapper(VD, Out);
1906     Out.flush();
1907   }
1908 
1909   if (llvm::Value *V = CGM.getModule().getNamedValue(WrapperName))
1910     return cast<llvm::Function>(V);
1911 
1912   llvm::Type *RetTy = Val->getType();
1913   if (VD->getType()->isReferenceType())
1914     RetTy = RetTy->getPointerElementType();
1915 
1916   llvm::FunctionType *FnTy = llvm::FunctionType::get(RetTy, false);
1917   llvm::Function *Wrapper =
1918       llvm::Function::Create(FnTy, getThreadLocalWrapperLinkage(VD, CGM),
1919                              WrapperName.str(), &CGM.getModule());
1920   // Always resolve references to the wrapper at link time.
1921   if (!Wrapper->hasLocalLinkage() && !isThreadWrapperReplaceable(VD, CGM))
1922     Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility);
1923   return Wrapper;
1924 }
1925 
1926 void ItaniumCXXABI::EmitThreadLocalInitFuncs(
1927     CodeGenModule &CGM,
1928     ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
1929         CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits,
1930     ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) {
1931   llvm::Function *InitFunc = nullptr;
1932   if (!CXXThreadLocalInits.empty()) {
1933     // Generate a guarded initialization function.
1934     llvm::FunctionType *FTy =
1935         llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
1936     InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init",
1937                                                       SourceLocation(),
1938                                                       /*TLS=*/true);
1939     llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
1940         CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
1941         llvm::GlobalVariable::InternalLinkage,
1942         llvm::ConstantInt::get(CGM.Int8Ty, 0), "__tls_guard");
1943     Guard->setThreadLocal(true);
1944     CodeGenFunction(CGM)
1945         .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, Guard);
1946   }
1947   for (unsigned I = 0, N = CXXThreadLocals.size(); I != N; ++I) {
1948     const VarDecl *VD = CXXThreadLocals[I].first;
1949     llvm::GlobalVariable *Var = CXXThreadLocals[I].second;
1950 
1951     // Some targets require that all access to thread local variables go through
1952     // the thread wrapper.  This means that we cannot attempt to create a thread
1953     // wrapper or a thread helper.
1954     if (isThreadWrapperReplaceable(VD, CGM) && !VD->hasDefinition())
1955       continue;
1956 
1957     // Mangle the name for the thread_local initialization function.
1958     SmallString<256> InitFnName;
1959     {
1960       llvm::raw_svector_ostream Out(InitFnName);
1961       getMangleContext().mangleItaniumThreadLocalInit(VD, Out);
1962       Out.flush();
1963     }
1964 
1965     // If we have a definition for the variable, emit the initialization
1966     // function as an alias to the global Init function (if any). Otherwise,
1967     // produce a declaration of the initialization function.
1968     llvm::GlobalValue *Init = nullptr;
1969     bool InitIsInitFunc = false;
1970     if (VD->hasDefinition()) {
1971       InitIsInitFunc = true;
1972       if (InitFunc)
1973         Init = llvm::GlobalAlias::create(Var->getLinkage(), InitFnName.str(),
1974                                          InitFunc);
1975     } else {
1976       // Emit a weak global function referring to the initialization function.
1977       // This function will not exist if the TU defining the thread_local
1978       // variable in question does not need any dynamic initialization for
1979       // its thread_local variables.
1980       llvm::FunctionType *FnTy = llvm::FunctionType::get(CGM.VoidTy, false);
1981       Init = llvm::Function::Create(
1982           FnTy, llvm::GlobalVariable::ExternalWeakLinkage, InitFnName.str(),
1983           &CGM.getModule());
1984     }
1985 
1986     if (Init)
1987       Init->setVisibility(Var->getVisibility());
1988 
1989     llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var);
1990     llvm::LLVMContext &Context = CGM.getModule().getContext();
1991     llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper);
1992     CGBuilderTy Builder(Entry);
1993     if (InitIsInitFunc) {
1994       if (Init)
1995         Builder.CreateCall(Init);
1996     } else {
1997       // Don't know whether we have an init function. Call it if it exists.
1998       llvm::Value *Have = Builder.CreateIsNotNull(Init);
1999       llvm::BasicBlock *InitBB = llvm::BasicBlock::Create(Context, "", Wrapper);
2000       llvm::BasicBlock *ExitBB = llvm::BasicBlock::Create(Context, "", Wrapper);
2001       Builder.CreateCondBr(Have, InitBB, ExitBB);
2002 
2003       Builder.SetInsertPoint(InitBB);
2004       Builder.CreateCall(Init);
2005       Builder.CreateBr(ExitBB);
2006 
2007       Builder.SetInsertPoint(ExitBB);
2008     }
2009 
2010     // For a reference, the result of the wrapper function is a pointer to
2011     // the referenced object.
2012     llvm::Value *Val = Var;
2013     if (VD->getType()->isReferenceType()) {
2014       llvm::LoadInst *LI = Builder.CreateLoad(Val);
2015       LI->setAlignment(CGM.getContext().getDeclAlign(VD).getQuantity());
2016       Val = LI;
2017     }
2018     if (Val->getType() != Wrapper->getReturnType())
2019       Val = Builder.CreatePointerBitCastOrAddrSpaceCast(
2020           Val, Wrapper->getReturnType(), "");
2021     Builder.CreateRet(Val);
2022   }
2023 }
2024 
2025 LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
2026                                                    const VarDecl *VD,
2027                                                    QualType LValType) {
2028   QualType T = VD->getType();
2029   llvm::Type *Ty = CGF.getTypes().ConvertTypeForMem(T);
2030   llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD, Ty);
2031   llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val);
2032 
2033   Val = CGF.Builder.CreateCall(Wrapper);
2034 
2035   LValue LV;
2036   if (VD->getType()->isReferenceType())
2037     LV = CGF.MakeNaturalAlignAddrLValue(Val, LValType);
2038   else
2039     LV = CGF.MakeAddrLValue(Val, LValType, CGF.getContext().getDeclAlign(VD));
2040   // FIXME: need setObjCGCLValueClass?
2041   return LV;
2042 }
2043 
2044 /// Return whether the given global decl needs a VTT parameter, which it does
2045 /// if it's a base constructor or destructor with virtual bases.
2046 bool ItaniumCXXABI::NeedsVTTParameter(GlobalDecl GD) {
2047   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
2048 
2049   // We don't have any virtual bases, just return early.
2050   if (!MD->getParent()->getNumVBases())
2051     return false;
2052 
2053   // Check if we have a base constructor.
2054   if (isa<CXXConstructorDecl>(MD) && GD.getCtorType() == Ctor_Base)
2055     return true;
2056 
2057   // Check if we have a base destructor.
2058   if (isa<CXXDestructorDecl>(MD) && GD.getDtorType() == Dtor_Base)
2059     return true;
2060 
2061   return false;
2062 }
2063 
2064 namespace {
2065 class ItaniumRTTIBuilder {
2066   CodeGenModule &CGM;  // Per-module state.
2067   llvm::LLVMContext &VMContext;
2068   const ItaniumCXXABI &CXXABI;  // Per-module state.
2069 
2070   /// Fields - The fields of the RTTI descriptor currently being built.
2071   SmallVector<llvm::Constant *, 16> Fields;
2072 
2073   /// GetAddrOfTypeName - Returns the mangled type name of the given type.
2074   llvm::GlobalVariable *
2075   GetAddrOfTypeName(QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage);
2076 
2077   /// GetAddrOfExternalRTTIDescriptor - Returns the constant for the RTTI
2078   /// descriptor of the given type.
2079   llvm::Constant *GetAddrOfExternalRTTIDescriptor(QualType Ty);
2080 
2081   /// BuildVTablePointer - Build the vtable pointer for the given type.
2082   void BuildVTablePointer(const Type *Ty);
2083 
2084   /// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
2085   /// inheritance, according to the Itanium C++ ABI, 2.9.5p6b.
2086   void BuildSIClassTypeInfo(const CXXRecordDecl *RD);
2087 
2088   /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
2089   /// classes with bases that do not satisfy the abi::__si_class_type_info
2090   /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
2091   void BuildVMIClassTypeInfo(const CXXRecordDecl *RD);
2092 
2093   /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, used
2094   /// for pointer types.
2095   void BuildPointerTypeInfo(QualType PointeeTy);
2096 
2097   /// BuildObjCObjectTypeInfo - Build the appropriate kind of
2098   /// type_info for an object type.
2099   void BuildObjCObjectTypeInfo(const ObjCObjectType *Ty);
2100 
2101   /// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
2102   /// struct, used for member pointer types.
2103   void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty);
2104 
2105 public:
2106   ItaniumRTTIBuilder(const ItaniumCXXABI &ABI)
2107       : CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {}
2108 
2109   // Pointer type info flags.
2110   enum {
2111     /// PTI_Const - Type has const qualifier.
2112     PTI_Const = 0x1,
2113 
2114     /// PTI_Volatile - Type has volatile qualifier.
2115     PTI_Volatile = 0x2,
2116 
2117     /// PTI_Restrict - Type has restrict qualifier.
2118     PTI_Restrict = 0x4,
2119 
2120     /// PTI_Incomplete - Type is incomplete.
2121     PTI_Incomplete = 0x8,
2122 
2123     /// PTI_ContainingClassIncomplete - Containing class is incomplete.
2124     /// (in pointer to member).
2125     PTI_ContainingClassIncomplete = 0x10
2126   };
2127 
2128   // VMI type info flags.
2129   enum {
2130     /// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
2131     VMI_NonDiamondRepeat = 0x1,
2132 
2133     /// VMI_DiamondShaped - Class is diamond shaped.
2134     VMI_DiamondShaped = 0x2
2135   };
2136 
2137   // Base class type info flags.
2138   enum {
2139     /// BCTI_Virtual - Base class is virtual.
2140     BCTI_Virtual = 0x1,
2141 
2142     /// BCTI_Public - Base class is public.
2143     BCTI_Public = 0x2
2144   };
2145 
2146   /// BuildTypeInfo - Build the RTTI type info struct for the given type.
2147   ///
2148   /// \param Force - true to force the creation of this RTTI value
2149   llvm::Constant *BuildTypeInfo(QualType Ty, bool Force = false);
2150 };
2151 }
2152 
2153 llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName(
2154     QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) {
2155   SmallString<256> OutName;
2156   llvm::raw_svector_ostream Out(OutName);
2157   CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
2158   Out.flush();
2159   StringRef Name = OutName.str();
2160 
2161   // We know that the mangled name of the type starts at index 4 of the
2162   // mangled name of the typename, so we can just index into it in order to
2163   // get the mangled name of the type.
2164   llvm::Constant *Init = llvm::ConstantDataArray::getString(VMContext,
2165                                                             Name.substr(4));
2166 
2167   llvm::GlobalVariable *GV =
2168     CGM.CreateOrReplaceCXXRuntimeVariable(Name, Init->getType(), Linkage);
2169 
2170   GV->setInitializer(Init);
2171 
2172   return GV;
2173 }
2174 
2175 llvm::Constant *
2176 ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
2177   // Mangle the RTTI name.
2178   SmallString<256> OutName;
2179   llvm::raw_svector_ostream Out(OutName);
2180   CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
2181   Out.flush();
2182   StringRef Name = OutName.str();
2183 
2184   // Look for an existing global.
2185   llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
2186 
2187   if (!GV) {
2188     // Create a new global variable.
2189     GV = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8PtrTy,
2190                                   /*Constant=*/true,
2191                                   llvm::GlobalValue::ExternalLinkage, nullptr,
2192                                   Name);
2193     if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
2194       const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
2195       if (RD->hasAttr<DLLImportAttr>())
2196         GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
2197     }
2198   }
2199 
2200   return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy);
2201 }
2202 
2203 /// TypeInfoIsInStandardLibrary - Given a builtin type, returns whether the type
2204 /// info for that type is defined in the standard library.
2205 static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
2206   // Itanium C++ ABI 2.9.2:
2207   //   Basic type information (e.g. for "int", "bool", etc.) will be kept in
2208   //   the run-time support library. Specifically, the run-time support
2209   //   library should contain type_info objects for the types X, X* and
2210   //   X const*, for every X in: void, std::nullptr_t, bool, wchar_t, char,
2211   //   unsigned char, signed char, short, unsigned short, int, unsigned int,
2212   //   long, unsigned long, long long, unsigned long long, float, double,
2213   //   long double, char16_t, char32_t, and the IEEE 754r decimal and
2214   //   half-precision floating point types.
2215   switch (Ty->getKind()) {
2216     case BuiltinType::Void:
2217     case BuiltinType::NullPtr:
2218     case BuiltinType::Bool:
2219     case BuiltinType::WChar_S:
2220     case BuiltinType::WChar_U:
2221     case BuiltinType::Char_U:
2222     case BuiltinType::Char_S:
2223     case BuiltinType::UChar:
2224     case BuiltinType::SChar:
2225     case BuiltinType::Short:
2226     case BuiltinType::UShort:
2227     case BuiltinType::Int:
2228     case BuiltinType::UInt:
2229     case BuiltinType::Long:
2230     case BuiltinType::ULong:
2231     case BuiltinType::LongLong:
2232     case BuiltinType::ULongLong:
2233     case BuiltinType::Half:
2234     case BuiltinType::Float:
2235     case BuiltinType::Double:
2236     case BuiltinType::LongDouble:
2237     case BuiltinType::Char16:
2238     case BuiltinType::Char32:
2239     case BuiltinType::Int128:
2240     case BuiltinType::UInt128:
2241     case BuiltinType::OCLImage1d:
2242     case BuiltinType::OCLImage1dArray:
2243     case BuiltinType::OCLImage1dBuffer:
2244     case BuiltinType::OCLImage2d:
2245     case BuiltinType::OCLImage2dArray:
2246     case BuiltinType::OCLImage3d:
2247     case BuiltinType::OCLSampler:
2248     case BuiltinType::OCLEvent:
2249       return true;
2250 
2251     case BuiltinType::Dependent:
2252 #define BUILTIN_TYPE(Id, SingletonId)
2253 #define PLACEHOLDER_TYPE(Id, SingletonId) \
2254     case BuiltinType::Id:
2255 #include "clang/AST/BuiltinTypes.def"
2256       llvm_unreachable("asking for RRTI for a placeholder type!");
2257 
2258     case BuiltinType::ObjCId:
2259     case BuiltinType::ObjCClass:
2260     case BuiltinType::ObjCSel:
2261       llvm_unreachable("FIXME: Objective-C types are unsupported!");
2262   }
2263 
2264   llvm_unreachable("Invalid BuiltinType Kind!");
2265 }
2266 
2267 static bool TypeInfoIsInStandardLibrary(const PointerType *PointerTy) {
2268   QualType PointeeTy = PointerTy->getPointeeType();
2269   const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(PointeeTy);
2270   if (!BuiltinTy)
2271     return false;
2272 
2273   // Check the qualifiers.
2274   Qualifiers Quals = PointeeTy.getQualifiers();
2275   Quals.removeConst();
2276 
2277   if (!Quals.empty())
2278     return false;
2279 
2280   return TypeInfoIsInStandardLibrary(BuiltinTy);
2281 }
2282 
2283 /// IsStandardLibraryRTTIDescriptor - Returns whether the type
2284 /// information for the given type exists in the standard library.
2285 static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
2286   // Type info for builtin types is defined in the standard library.
2287   if (const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Ty))
2288     return TypeInfoIsInStandardLibrary(BuiltinTy);
2289 
2290   // Type info for some pointer types to builtin types is defined in the
2291   // standard library.
2292   if (const PointerType *PointerTy = dyn_cast<PointerType>(Ty))
2293     return TypeInfoIsInStandardLibrary(PointerTy);
2294 
2295   return false;
2296 }
2297 
2298 /// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
2299 /// the given type exists somewhere else, and that we should not emit the type
2300 /// information in this translation unit.  Assumes that it is not a
2301 /// standard-library type.
2302 static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
2303                                             QualType Ty) {
2304   ASTContext &Context = CGM.getContext();
2305 
2306   // If RTTI is disabled, assume it might be disabled in the
2307   // translation unit that defines any potential key function, too.
2308   if (!Context.getLangOpts().RTTI) return false;
2309 
2310   if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
2311     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
2312     if (!RD->hasDefinition())
2313       return false;
2314 
2315     if (!RD->isDynamicClass())
2316       return false;
2317 
2318     // FIXME: this may need to be reconsidered if the key function
2319     // changes.
2320     if (CGM.getVTables().isVTableExternal(RD))
2321       return true;
2322 
2323     if (RD->hasAttr<DLLImportAttr>())
2324       return true;
2325   }
2326 
2327   return false;
2328 }
2329 
2330 /// IsIncompleteClassType - Returns whether the given record type is incomplete.
2331 static bool IsIncompleteClassType(const RecordType *RecordTy) {
2332   return !RecordTy->getDecl()->isCompleteDefinition();
2333 }
2334 
2335 /// ContainsIncompleteClassType - Returns whether the given type contains an
2336 /// incomplete class type. This is true if
2337 ///
2338 ///   * The given type is an incomplete class type.
2339 ///   * The given type is a pointer type whose pointee type contains an
2340 ///     incomplete class type.
2341 ///   * The given type is a member pointer type whose class is an incomplete
2342 ///     class type.
2343 ///   * The given type is a member pointer type whoise pointee type contains an
2344 ///     incomplete class type.
2345 /// is an indirect or direct pointer to an incomplete class type.
2346 static bool ContainsIncompleteClassType(QualType Ty) {
2347   if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
2348     if (IsIncompleteClassType(RecordTy))
2349       return true;
2350   }
2351 
2352   if (const PointerType *PointerTy = dyn_cast<PointerType>(Ty))
2353     return ContainsIncompleteClassType(PointerTy->getPointeeType());
2354 
2355   if (const MemberPointerType *MemberPointerTy =
2356       dyn_cast<MemberPointerType>(Ty)) {
2357     // Check if the class type is incomplete.
2358     const RecordType *ClassType = cast<RecordType>(MemberPointerTy->getClass());
2359     if (IsIncompleteClassType(ClassType))
2360       return true;
2361 
2362     return ContainsIncompleteClassType(MemberPointerTy->getPointeeType());
2363   }
2364 
2365   return false;
2366 }
2367 
2368 // CanUseSingleInheritance - Return whether the given record decl has a "single,
2369 // public, non-virtual base at offset zero (i.e. the derived class is dynamic
2370 // iff the base is)", according to Itanium C++ ABI, 2.95p6b.
2371 static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
2372   // Check the number of bases.
2373   if (RD->getNumBases() != 1)
2374     return false;
2375 
2376   // Get the base.
2377   CXXRecordDecl::base_class_const_iterator Base = RD->bases_begin();
2378 
2379   // Check that the base is not virtual.
2380   if (Base->isVirtual())
2381     return false;
2382 
2383   // Check that the base is public.
2384   if (Base->getAccessSpecifier() != AS_public)
2385     return false;
2386 
2387   // Check that the class is dynamic iff the base is.
2388   const CXXRecordDecl *BaseDecl =
2389     cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
2390   if (!BaseDecl->isEmpty() &&
2391       BaseDecl->isDynamicClass() != RD->isDynamicClass())
2392     return false;
2393 
2394   return true;
2395 }
2396 
2397 void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) {
2398   // abi::__class_type_info.
2399   static const char * const ClassTypeInfo =
2400     "_ZTVN10__cxxabiv117__class_type_infoE";
2401   // abi::__si_class_type_info.
2402   static const char * const SIClassTypeInfo =
2403     "_ZTVN10__cxxabiv120__si_class_type_infoE";
2404   // abi::__vmi_class_type_info.
2405   static const char * const VMIClassTypeInfo =
2406     "_ZTVN10__cxxabiv121__vmi_class_type_infoE";
2407 
2408   const char *VTableName = nullptr;
2409 
2410   switch (Ty->getTypeClass()) {
2411 #define TYPE(Class, Base)
2412 #define ABSTRACT_TYPE(Class, Base)
2413 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
2414 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
2415 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2416 #include "clang/AST/TypeNodes.def"
2417     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
2418 
2419   case Type::LValueReference:
2420   case Type::RValueReference:
2421     llvm_unreachable("References shouldn't get here");
2422 
2423   case Type::Auto:
2424     llvm_unreachable("Undeduced auto type shouldn't get here");
2425 
2426   case Type::Builtin:
2427   // GCC treats vector and complex types as fundamental types.
2428   case Type::Vector:
2429   case Type::ExtVector:
2430   case Type::Complex:
2431   case Type::Atomic:
2432   // FIXME: GCC treats block pointers as fundamental types?!
2433   case Type::BlockPointer:
2434     // abi::__fundamental_type_info.
2435     VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE";
2436     break;
2437 
2438   case Type::ConstantArray:
2439   case Type::IncompleteArray:
2440   case Type::VariableArray:
2441     // abi::__array_type_info.
2442     VTableName = "_ZTVN10__cxxabiv117__array_type_infoE";
2443     break;
2444 
2445   case Type::FunctionNoProto:
2446   case Type::FunctionProto:
2447     // abi::__function_type_info.
2448     VTableName = "_ZTVN10__cxxabiv120__function_type_infoE";
2449     break;
2450 
2451   case Type::Enum:
2452     // abi::__enum_type_info.
2453     VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE";
2454     break;
2455 
2456   case Type::Record: {
2457     const CXXRecordDecl *RD =
2458       cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
2459 
2460     if (!RD->hasDefinition() || !RD->getNumBases()) {
2461       VTableName = ClassTypeInfo;
2462     } else if (CanUseSingleInheritance(RD)) {
2463       VTableName = SIClassTypeInfo;
2464     } else {
2465       VTableName = VMIClassTypeInfo;
2466     }
2467 
2468     break;
2469   }
2470 
2471   case Type::ObjCObject:
2472     // Ignore protocol qualifiers.
2473     Ty = cast<ObjCObjectType>(Ty)->getBaseType().getTypePtr();
2474 
2475     // Handle id and Class.
2476     if (isa<BuiltinType>(Ty)) {
2477       VTableName = ClassTypeInfo;
2478       break;
2479     }
2480 
2481     assert(isa<ObjCInterfaceType>(Ty));
2482     // Fall through.
2483 
2484   case Type::ObjCInterface:
2485     if (cast<ObjCInterfaceType>(Ty)->getDecl()->getSuperClass()) {
2486       VTableName = SIClassTypeInfo;
2487     } else {
2488       VTableName = ClassTypeInfo;
2489     }
2490     break;
2491 
2492   case Type::ObjCObjectPointer:
2493   case Type::Pointer:
2494     // abi::__pointer_type_info.
2495     VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE";
2496     break;
2497 
2498   case Type::MemberPointer:
2499     // abi::__pointer_to_member_type_info.
2500     VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE";
2501     break;
2502   }
2503 
2504   llvm::Constant *VTable =
2505     CGM.getModule().getOrInsertGlobal(VTableName, CGM.Int8PtrTy);
2506 
2507   llvm::Type *PtrDiffTy =
2508     CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
2509 
2510   // The vtable address point is 2.
2511   llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2);
2512   VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Two);
2513   VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy);
2514 
2515   Fields.push_back(VTable);
2516 }
2517 
2518 /// \brief Return the linkage that the type info and type info name constants
2519 /// should have for the given type.
2520 static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM,
2521                                                              QualType Ty) {
2522   // Itanium C++ ABI 2.9.5p7:
2523   //   In addition, it and all of the intermediate abi::__pointer_type_info
2524   //   structs in the chain down to the abi::__class_type_info for the
2525   //   incomplete class type must be prevented from resolving to the
2526   //   corresponding type_info structs for the complete class type, possibly
2527   //   by making them local static objects. Finally, a dummy class RTTI is
2528   //   generated for the incomplete type that will not resolve to the final
2529   //   complete class RTTI (because the latter need not exist), possibly by
2530   //   making it a local static object.
2531   if (ContainsIncompleteClassType(Ty))
2532     return llvm::GlobalValue::InternalLinkage;
2533 
2534   switch (Ty->getLinkage()) {
2535   case NoLinkage:
2536   case InternalLinkage:
2537   case UniqueExternalLinkage:
2538     return llvm::GlobalValue::InternalLinkage;
2539 
2540   case VisibleNoLinkage:
2541   case ExternalLinkage:
2542     if (!CGM.getLangOpts().RTTI) {
2543       // RTTI is not enabled, which means that this type info struct is going
2544       // to be used for exception handling. Give it linkonce_odr linkage.
2545       return llvm::GlobalValue::LinkOnceODRLinkage;
2546     }
2547 
2548     if (const RecordType *Record = dyn_cast<RecordType>(Ty)) {
2549       const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
2550       if (RD->hasAttr<WeakAttr>())
2551         return llvm::GlobalValue::WeakODRLinkage;
2552       if (RD->isDynamicClass())
2553         return CGM.getVTableLinkage(RD);
2554     }
2555 
2556     return llvm::GlobalValue::LinkOnceODRLinkage;
2557   }
2558 
2559   llvm_unreachable("Invalid linkage!");
2560 }
2561 
2562 llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
2563   // We want to operate on the canonical type.
2564   Ty = CGM.getContext().getCanonicalType(Ty);
2565 
2566   // Check if we've already emitted an RTTI descriptor for this type.
2567   SmallString<256> OutName;
2568   llvm::raw_svector_ostream Out(OutName);
2569   CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
2570   Out.flush();
2571   StringRef Name = OutName.str();
2572 
2573   llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
2574   if (OldGV && !OldGV->isDeclaration()) {
2575     assert(!OldGV->hasAvailableExternallyLinkage() &&
2576            "available_externally typeinfos not yet implemented");
2577 
2578     return llvm::ConstantExpr::getBitCast(OldGV, CGM.Int8PtrTy);
2579   }
2580 
2581   // Check if there is already an external RTTI descriptor for this type.
2582   bool IsStdLib = IsStandardLibraryRTTIDescriptor(Ty);
2583   if (!Force && (IsStdLib || ShouldUseExternalRTTIDescriptor(CGM, Ty)))
2584     return GetAddrOfExternalRTTIDescriptor(Ty);
2585 
2586   // Emit the standard library with external linkage.
2587   llvm::GlobalVariable::LinkageTypes Linkage;
2588   if (IsStdLib)
2589     Linkage = llvm::GlobalValue::ExternalLinkage;
2590   else
2591     Linkage = getTypeInfoLinkage(CGM, Ty);
2592 
2593   // Add the vtable pointer.
2594   BuildVTablePointer(cast<Type>(Ty));
2595 
2596   // And the name.
2597   llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
2598   llvm::Constant *TypeNameField;
2599 
2600   // If we're supposed to demote the visibility, be sure to set a flag
2601   // to use a string comparison for type_info comparisons.
2602   ItaniumCXXABI::RTTIUniquenessKind RTTIUniqueness =
2603       CXXABI.classifyRTTIUniqueness(Ty, Linkage);
2604   if (RTTIUniqueness != ItaniumCXXABI::RUK_Unique) {
2605     // The flag is the sign bit, which on ARM64 is defined to be clear
2606     // for global pointers.  This is very ARM64-specific.
2607     TypeNameField = llvm::ConstantExpr::getPtrToInt(TypeName, CGM.Int64Ty);
2608     llvm::Constant *flag =
2609         llvm::ConstantInt::get(CGM.Int64Ty, ((uint64_t)1) << 63);
2610     TypeNameField = llvm::ConstantExpr::getAdd(TypeNameField, flag);
2611     TypeNameField =
2612         llvm::ConstantExpr::getIntToPtr(TypeNameField, CGM.Int8PtrTy);
2613   } else {
2614     TypeNameField = llvm::ConstantExpr::getBitCast(TypeName, CGM.Int8PtrTy);
2615   }
2616   Fields.push_back(TypeNameField);
2617 
2618   switch (Ty->getTypeClass()) {
2619 #define TYPE(Class, Base)
2620 #define ABSTRACT_TYPE(Class, Base)
2621 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
2622 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
2623 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2624 #include "clang/AST/TypeNodes.def"
2625     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
2626 
2627   // GCC treats vector types as fundamental types.
2628   case Type::Builtin:
2629   case Type::Vector:
2630   case Type::ExtVector:
2631   case Type::Complex:
2632   case Type::BlockPointer:
2633     // Itanium C++ ABI 2.9.5p4:
2634     // abi::__fundamental_type_info adds no data members to std::type_info.
2635     break;
2636 
2637   case Type::LValueReference:
2638   case Type::RValueReference:
2639     llvm_unreachable("References shouldn't get here");
2640 
2641   case Type::Auto:
2642     llvm_unreachable("Undeduced auto type shouldn't get here");
2643 
2644   case Type::ConstantArray:
2645   case Type::IncompleteArray:
2646   case Type::VariableArray:
2647     // Itanium C++ ABI 2.9.5p5:
2648     // abi::__array_type_info adds no data members to std::type_info.
2649     break;
2650 
2651   case Type::FunctionNoProto:
2652   case Type::FunctionProto:
2653     // Itanium C++ ABI 2.9.5p5:
2654     // abi::__function_type_info adds no data members to std::type_info.
2655     break;
2656 
2657   case Type::Enum:
2658     // Itanium C++ ABI 2.9.5p5:
2659     // abi::__enum_type_info adds no data members to std::type_info.
2660     break;
2661 
2662   case Type::Record: {
2663     const CXXRecordDecl *RD =
2664       cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
2665     if (!RD->hasDefinition() || !RD->getNumBases()) {
2666       // We don't need to emit any fields.
2667       break;
2668     }
2669 
2670     if (CanUseSingleInheritance(RD))
2671       BuildSIClassTypeInfo(RD);
2672     else
2673       BuildVMIClassTypeInfo(RD);
2674 
2675     break;
2676   }
2677 
2678   case Type::ObjCObject:
2679   case Type::ObjCInterface:
2680     BuildObjCObjectTypeInfo(cast<ObjCObjectType>(Ty));
2681     break;
2682 
2683   case Type::ObjCObjectPointer:
2684     BuildPointerTypeInfo(cast<ObjCObjectPointerType>(Ty)->getPointeeType());
2685     break;
2686 
2687   case Type::Pointer:
2688     BuildPointerTypeInfo(cast<PointerType>(Ty)->getPointeeType());
2689     break;
2690 
2691   case Type::MemberPointer:
2692     BuildPointerToMemberTypeInfo(cast<MemberPointerType>(Ty));
2693     break;
2694 
2695   case Type::Atomic:
2696     // No fields, at least for the moment.
2697     break;
2698   }
2699 
2700   llvm::Constant *Init = llvm::ConstantStruct::getAnon(Fields);
2701 
2702   llvm::GlobalVariable *GV =
2703     new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
2704                              /*Constant=*/true, Linkage, Init, Name);
2705 
2706   // If there's already an old global variable, replace it with the new one.
2707   if (OldGV) {
2708     GV->takeName(OldGV);
2709     llvm::Constant *NewPtr =
2710       llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
2711     OldGV->replaceAllUsesWith(NewPtr);
2712     OldGV->eraseFromParent();
2713   }
2714 
2715   // The Itanium ABI specifies that type_info objects must be globally
2716   // unique, with one exception: if the type is an incomplete class
2717   // type or a (possibly indirect) pointer to one.  That exception
2718   // affects the general case of comparing type_info objects produced
2719   // by the typeid operator, which is why the comparison operators on
2720   // std::type_info generally use the type_info name pointers instead
2721   // of the object addresses.  However, the language's built-in uses
2722   // of RTTI generally require class types to be complete, even when
2723   // manipulating pointers to those class types.  This allows the
2724   // implementation of dynamic_cast to rely on address equality tests,
2725   // which is much faster.
2726 
2727   // All of this is to say that it's important that both the type_info
2728   // object and the type_info name be uniqued when weakly emitted.
2729 
2730   // Give the type_info object and name the formal visibility of the
2731   // type itself.
2732   llvm::GlobalValue::VisibilityTypes llvmVisibility;
2733   if (llvm::GlobalValue::isLocalLinkage(Linkage))
2734     // If the linkage is local, only default visibility makes sense.
2735     llvmVisibility = llvm::GlobalValue::DefaultVisibility;
2736   else if (RTTIUniqueness == ItaniumCXXABI::RUK_NonUniqueHidden)
2737     llvmVisibility = llvm::GlobalValue::HiddenVisibility;
2738   else
2739     llvmVisibility = CodeGenModule::GetLLVMVisibility(Ty->getVisibility());
2740   TypeName->setVisibility(llvmVisibility);
2741   GV->setVisibility(llvmVisibility);
2742 
2743   return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy);
2744 }
2745 
2746 /// ComputeQualifierFlags - Compute the pointer type info flags from the
2747 /// given qualifier.
2748 static unsigned ComputeQualifierFlags(Qualifiers Quals) {
2749   unsigned Flags = 0;
2750 
2751   if (Quals.hasConst())
2752     Flags |= ItaniumRTTIBuilder::PTI_Const;
2753   if (Quals.hasVolatile())
2754     Flags |= ItaniumRTTIBuilder::PTI_Volatile;
2755   if (Quals.hasRestrict())
2756     Flags |= ItaniumRTTIBuilder::PTI_Restrict;
2757 
2758   return Flags;
2759 }
2760 
2761 /// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info
2762 /// for the given Objective-C object type.
2763 void ItaniumRTTIBuilder::BuildObjCObjectTypeInfo(const ObjCObjectType *OT) {
2764   // Drop qualifiers.
2765   const Type *T = OT->getBaseType().getTypePtr();
2766   assert(isa<BuiltinType>(T) || isa<ObjCInterfaceType>(T));
2767 
2768   // The builtin types are abi::__class_type_infos and don't require
2769   // extra fields.
2770   if (isa<BuiltinType>(T)) return;
2771 
2772   ObjCInterfaceDecl *Class = cast<ObjCInterfaceType>(T)->getDecl();
2773   ObjCInterfaceDecl *Super = Class->getSuperClass();
2774 
2775   // Root classes are also __class_type_info.
2776   if (!Super) return;
2777 
2778   QualType SuperTy = CGM.getContext().getObjCInterfaceType(Super);
2779 
2780   // Everything else is single inheritance.
2781   llvm::Constant *BaseTypeInfo =
2782       ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(SuperTy);
2783   Fields.push_back(BaseTypeInfo);
2784 }
2785 
2786 /// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
2787 /// inheritance, according to the Itanium C++ ABI, 2.95p6b.
2788 void ItaniumRTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
2789   // Itanium C++ ABI 2.9.5p6b:
2790   // It adds to abi::__class_type_info a single member pointing to the
2791   // type_info structure for the base type,
2792   llvm::Constant *BaseTypeInfo =
2793     ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(RD->bases_begin()->getType());
2794   Fields.push_back(BaseTypeInfo);
2795 }
2796 
2797 namespace {
2798   /// SeenBases - Contains virtual and non-virtual bases seen when traversing
2799   /// a class hierarchy.
2800   struct SeenBases {
2801     llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
2802     llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
2803   };
2804 }
2805 
2806 /// ComputeVMIClassTypeInfoFlags - Compute the value of the flags member in
2807 /// abi::__vmi_class_type_info.
2808 ///
2809 static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
2810                                              SeenBases &Bases) {
2811 
2812   unsigned Flags = 0;
2813 
2814   const CXXRecordDecl *BaseDecl =
2815     cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
2816 
2817   if (Base->isVirtual()) {
2818     // Mark the virtual base as seen.
2819     if (!Bases.VirtualBases.insert(BaseDecl).second) {
2820       // If this virtual base has been seen before, then the class is diamond
2821       // shaped.
2822       Flags |= ItaniumRTTIBuilder::VMI_DiamondShaped;
2823     } else {
2824       if (Bases.NonVirtualBases.count(BaseDecl))
2825         Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
2826     }
2827   } else {
2828     // Mark the non-virtual base as seen.
2829     if (!Bases.NonVirtualBases.insert(BaseDecl).second) {
2830       // If this non-virtual base has been seen before, then the class has non-
2831       // diamond shaped repeated inheritance.
2832       Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
2833     } else {
2834       if (Bases.VirtualBases.count(BaseDecl))
2835         Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
2836     }
2837   }
2838 
2839   // Walk all bases.
2840   for (const auto &I : BaseDecl->bases())
2841     Flags |= ComputeVMIClassTypeInfoFlags(&I, Bases);
2842 
2843   return Flags;
2844 }
2845 
2846 static unsigned ComputeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
2847   unsigned Flags = 0;
2848   SeenBases Bases;
2849 
2850   // Walk all bases.
2851   for (const auto &I : RD->bases())
2852     Flags |= ComputeVMIClassTypeInfoFlags(&I, Bases);
2853 
2854   return Flags;
2855 }
2856 
2857 /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
2858 /// classes with bases that do not satisfy the abi::__si_class_type_info
2859 /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
2860 void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
2861   llvm::Type *UnsignedIntLTy =
2862     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
2863 
2864   // Itanium C++ ABI 2.9.5p6c:
2865   //   __flags is a word with flags describing details about the class
2866   //   structure, which may be referenced by using the __flags_masks
2867   //   enumeration. These flags refer to both direct and indirect bases.
2868   unsigned Flags = ComputeVMIClassTypeInfoFlags(RD);
2869   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
2870 
2871   // Itanium C++ ABI 2.9.5p6c:
2872   //   __base_count is a word with the number of direct proper base class
2873   //   descriptions that follow.
2874   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, RD->getNumBases()));
2875 
2876   if (!RD->getNumBases())
2877     return;
2878 
2879   llvm::Type *LongLTy =
2880     CGM.getTypes().ConvertType(CGM.getContext().LongTy);
2881 
2882   // Now add the base class descriptions.
2883 
2884   // Itanium C++ ABI 2.9.5p6c:
2885   //   __base_info[] is an array of base class descriptions -- one for every
2886   //   direct proper base. Each description is of the type:
2887   //
2888   //   struct abi::__base_class_type_info {
2889   //   public:
2890   //     const __class_type_info *__base_type;
2891   //     long __offset_flags;
2892   //
2893   //     enum __offset_flags_masks {
2894   //       __virtual_mask = 0x1,
2895   //       __public_mask = 0x2,
2896   //       __offset_shift = 8
2897   //     };
2898   //   };
2899   for (const auto &Base : RD->bases()) {
2900     // The __base_type member points to the RTTI for the base type.
2901     Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Base.getType()));
2902 
2903     const CXXRecordDecl *BaseDecl =
2904       cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
2905 
2906     int64_t OffsetFlags = 0;
2907 
2908     // All but the lower 8 bits of __offset_flags are a signed offset.
2909     // For a non-virtual base, this is the offset in the object of the base
2910     // subobject. For a virtual base, this is the offset in the virtual table of
2911     // the virtual base offset for the virtual base referenced (negative).
2912     CharUnits Offset;
2913     if (Base.isVirtual())
2914       Offset =
2915         CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD, BaseDecl);
2916     else {
2917       const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
2918       Offset = Layout.getBaseClassOffset(BaseDecl);
2919     };
2920 
2921     OffsetFlags = uint64_t(Offset.getQuantity()) << 8;
2922 
2923     // The low-order byte of __offset_flags contains flags, as given by the
2924     // masks from the enumeration __offset_flags_masks.
2925     if (Base.isVirtual())
2926       OffsetFlags |= BCTI_Virtual;
2927     if (Base.getAccessSpecifier() == AS_public)
2928       OffsetFlags |= BCTI_Public;
2929 
2930     Fields.push_back(llvm::ConstantInt::get(LongLTy, OffsetFlags));
2931   }
2932 }
2933 
2934 /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct,
2935 /// used for pointer types.
2936 void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {
2937   Qualifiers Quals;
2938   QualType UnqualifiedPointeeTy =
2939     CGM.getContext().getUnqualifiedArrayType(PointeeTy, Quals);
2940 
2941   // Itanium C++ ABI 2.9.5p7:
2942   //   __flags is a flag word describing the cv-qualification and other
2943   //   attributes of the type pointed to
2944   unsigned Flags = ComputeQualifierFlags(Quals);
2945 
2946   // Itanium C++ ABI 2.9.5p7:
2947   //   When the abi::__pbase_type_info is for a direct or indirect pointer to an
2948   //   incomplete class type, the incomplete target type flag is set.
2949   if (ContainsIncompleteClassType(UnqualifiedPointeeTy))
2950     Flags |= PTI_Incomplete;
2951 
2952   llvm::Type *UnsignedIntLTy =
2953     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
2954   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
2955 
2956   // Itanium C++ ABI 2.9.5p7:
2957   //  __pointee is a pointer to the std::type_info derivation for the
2958   //  unqualified type being pointed to.
2959   llvm::Constant *PointeeTypeInfo =
2960     ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(UnqualifiedPointeeTy);
2961   Fields.push_back(PointeeTypeInfo);
2962 }
2963 
2964 /// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
2965 /// struct, used for member pointer types.
2966 void
2967 ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
2968   QualType PointeeTy = Ty->getPointeeType();
2969 
2970   Qualifiers Quals;
2971   QualType UnqualifiedPointeeTy =
2972     CGM.getContext().getUnqualifiedArrayType(PointeeTy, Quals);
2973 
2974   // Itanium C++ ABI 2.9.5p7:
2975   //   __flags is a flag word describing the cv-qualification and other
2976   //   attributes of the type pointed to.
2977   unsigned Flags = ComputeQualifierFlags(Quals);
2978 
2979   const RecordType *ClassType = cast<RecordType>(Ty->getClass());
2980 
2981   // Itanium C++ ABI 2.9.5p7:
2982   //   When the abi::__pbase_type_info is for a direct or indirect pointer to an
2983   //   incomplete class type, the incomplete target type flag is set.
2984   if (ContainsIncompleteClassType(UnqualifiedPointeeTy))
2985     Flags |= PTI_Incomplete;
2986 
2987   if (IsIncompleteClassType(ClassType))
2988     Flags |= PTI_ContainingClassIncomplete;
2989 
2990   llvm::Type *UnsignedIntLTy =
2991     CGM.getTypes().ConvertType(CGM.getContext().UnsignedIntTy);
2992   Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
2993 
2994   // Itanium C++ ABI 2.9.5p7:
2995   //   __pointee is a pointer to the std::type_info derivation for the
2996   //   unqualified type being pointed to.
2997   llvm::Constant *PointeeTypeInfo =
2998     ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(UnqualifiedPointeeTy);
2999   Fields.push_back(PointeeTypeInfo);
3000 
3001   // Itanium C++ ABI 2.9.5p9:
3002   //   __context is a pointer to an abi::__class_type_info corresponding to the
3003   //   class type containing the member pointed to
3004   //   (e.g., the "A" in "int A::*").
3005   Fields.push_back(
3006       ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(QualType(ClassType, 0)));
3007 }
3008 
3009 llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) {
3010   return ItaniumRTTIBuilder(*this).BuildTypeInfo(Ty);
3011 }
3012 
3013 void ItaniumCXXABI::EmitFundamentalRTTIDescriptor(QualType Type) {
3014   QualType PointerType = getContext().getPointerType(Type);
3015   QualType PointerTypeConst = getContext().getPointerType(Type.withConst());
3016   ItaniumRTTIBuilder(*this).BuildTypeInfo(Type, true);
3017   ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerType, true);
3018   ItaniumRTTIBuilder(*this).BuildTypeInfo(PointerTypeConst, true);
3019 }
3020 
3021 void ItaniumCXXABI::EmitFundamentalRTTIDescriptors() {
3022   QualType FundamentalTypes[] = {
3023       getContext().VoidTy,             getContext().NullPtrTy,
3024       getContext().BoolTy,             getContext().WCharTy,
3025       getContext().CharTy,             getContext().UnsignedCharTy,
3026       getContext().SignedCharTy,       getContext().ShortTy,
3027       getContext().UnsignedShortTy,    getContext().IntTy,
3028       getContext().UnsignedIntTy,      getContext().LongTy,
3029       getContext().UnsignedLongTy,     getContext().LongLongTy,
3030       getContext().UnsignedLongLongTy, getContext().HalfTy,
3031       getContext().FloatTy,            getContext().DoubleTy,
3032       getContext().LongDoubleTy,       getContext().Char16Ty,
3033       getContext().Char32Ty,
3034   };
3035   for (const QualType &FundamentalType : FundamentalTypes)
3036     EmitFundamentalRTTIDescriptor(FundamentalType);
3037 }
3038 
3039 /// What sort of uniqueness rules should we use for the RTTI for the
3040 /// given type?
3041 ItaniumCXXABI::RTTIUniquenessKind ItaniumCXXABI::classifyRTTIUniqueness(
3042     QualType CanTy, llvm::GlobalValue::LinkageTypes Linkage) const {
3043   if (shouldRTTIBeUnique())
3044     return RUK_Unique;
3045 
3046   // It's only necessary for linkonce_odr or weak_odr linkage.
3047   if (Linkage != llvm::GlobalValue::LinkOnceODRLinkage &&
3048       Linkage != llvm::GlobalValue::WeakODRLinkage)
3049     return RUK_Unique;
3050 
3051   // It's only necessary with default visibility.
3052   if (CanTy->getVisibility() != DefaultVisibility)
3053     return RUK_Unique;
3054 
3055   // If we're not required to publish this symbol, hide it.
3056   if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
3057     return RUK_NonUniqueHidden;
3058 
3059   // If we're required to publish this symbol, as we might be under an
3060   // explicit instantiation, leave it with default visibility but
3061   // enable string-comparisons.
3062   assert(Linkage == llvm::GlobalValue::WeakODRLinkage);
3063   return RUK_NonUniqueVisible;
3064 }
3065 
3066 // Find out how to codegen the complete destructor and constructor
3067 namespace {
3068 enum class StructorCodegen { Emit, RAUW, Alias, COMDAT };
3069 }
3070 static StructorCodegen getCodegenToUse(CodeGenModule &CGM,
3071                                        const CXXMethodDecl *MD) {
3072   if (!CGM.getCodeGenOpts().CXXCtorDtorAliases)
3073     return StructorCodegen::Emit;
3074 
3075   // The complete and base structors are not equivalent if there are any virtual
3076   // bases, so emit separate functions.
3077   if (MD->getParent()->getNumVBases())
3078     return StructorCodegen::Emit;
3079 
3080   GlobalDecl AliasDecl;
3081   if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) {
3082     AliasDecl = GlobalDecl(DD, Dtor_Complete);
3083   } else {
3084     const auto *CD = cast<CXXConstructorDecl>(MD);
3085     AliasDecl = GlobalDecl(CD, Ctor_Complete);
3086   }
3087   llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(AliasDecl);
3088 
3089   if (llvm::GlobalValue::isDiscardableIfUnused(Linkage))
3090     return StructorCodegen::RAUW;
3091 
3092   // FIXME: Should we allow available_externally aliases?
3093   if (!llvm::GlobalAlias::isValidLinkage(Linkage))
3094     return StructorCodegen::RAUW;
3095 
3096   if (llvm::GlobalValue::isWeakForLinker(Linkage)) {
3097     // Only ELF supports COMDATs with arbitrary names (C5/D5).
3098     if (CGM.getTarget().getTriple().isOSBinFormatELF())
3099       return StructorCodegen::COMDAT;
3100     return StructorCodegen::Emit;
3101   }
3102 
3103   return StructorCodegen::Alias;
3104 }
3105 
3106 static void emitConstructorDestructorAlias(CodeGenModule &CGM,
3107                                            GlobalDecl AliasDecl,
3108                                            GlobalDecl TargetDecl) {
3109   llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(AliasDecl);
3110 
3111   StringRef MangledName = CGM.getMangledName(AliasDecl);
3112   llvm::GlobalValue *Entry = CGM.GetGlobalValue(MangledName);
3113   if (Entry && !Entry->isDeclaration())
3114     return;
3115 
3116   auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(TargetDecl));
3117   llvm::PointerType *AliasType = Aliasee->getType();
3118 
3119   // Create the alias with no name.
3120   auto *Alias = llvm::GlobalAlias::create(
3121       AliasType->getElementType(), 0, Linkage, "", Aliasee, &CGM.getModule());
3122 
3123   // Switch any previous uses to the alias.
3124   if (Entry) {
3125     assert(Entry->getType() == AliasType &&
3126            "declaration exists with different type");
3127     Alias->takeName(Entry);
3128     Entry->replaceAllUsesWith(Alias);
3129     Entry->eraseFromParent();
3130   } else {
3131     Alias->setName(MangledName);
3132   }
3133 
3134   // Finally, set up the alias with its proper name and attributes.
3135   CGM.setAliasAttributes(cast<NamedDecl>(AliasDecl.getDecl()), Alias);
3136 }
3137 
3138 void ItaniumCXXABI::emitCXXStructor(const CXXMethodDecl *MD,
3139                                     StructorType Type) {
3140   auto *CD = dyn_cast<CXXConstructorDecl>(MD);
3141   const CXXDestructorDecl *DD = CD ? nullptr : cast<CXXDestructorDecl>(MD);
3142 
3143   StructorCodegen CGType = getCodegenToUse(CGM, MD);
3144 
3145   if (Type == StructorType::Complete) {
3146     GlobalDecl CompleteDecl;
3147     GlobalDecl BaseDecl;
3148     if (CD) {
3149       CompleteDecl = GlobalDecl(CD, Ctor_Complete);
3150       BaseDecl = GlobalDecl(CD, Ctor_Base);
3151     } else {
3152       CompleteDecl = GlobalDecl(DD, Dtor_Complete);
3153       BaseDecl = GlobalDecl(DD, Dtor_Base);
3154     }
3155 
3156     if (CGType == StructorCodegen::Alias || CGType == StructorCodegen::COMDAT) {
3157       emitConstructorDestructorAlias(CGM, CompleteDecl, BaseDecl);
3158       return;
3159     }
3160 
3161     if (CGType == StructorCodegen::RAUW) {
3162       StringRef MangledName = CGM.getMangledName(CompleteDecl);
3163       auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(BaseDecl));
3164       CGM.addReplacement(MangledName, Aliasee);
3165       return;
3166     }
3167   }
3168 
3169   // The base destructor is equivalent to the base destructor of its
3170   // base class if there is exactly one non-virtual base class with a
3171   // non-trivial destructor, there are no fields with a non-trivial
3172   // destructor, and the body of the destructor is trivial.
3173   if (DD && Type == StructorType::Base && CGType != StructorCodegen::COMDAT &&
3174       !CGM.TryEmitBaseDestructorAsAlias(DD))
3175     return;
3176 
3177   llvm::Function *Fn = CGM.codegenCXXStructor(MD, Type);
3178 
3179   if (CGType == StructorCodegen::COMDAT) {
3180     SmallString<256> Buffer;
3181     llvm::raw_svector_ostream Out(Buffer);
3182     if (DD)
3183       getMangleContext().mangleCXXDtorComdat(DD, Out);
3184     else
3185       getMangleContext().mangleCXXCtorComdat(CD, Out);
3186     llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Out.str());
3187     Fn->setComdat(C);
3188   }
3189 }
3190