1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines functions to generate various special functions for C
11 // structs.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "llvm/Support/ScopedPrinter.h"
18 #include <array>
19 
20 using namespace clang;
21 using namespace CodeGen;
22 
23 // Return the size of a field in number of bits.
24 static uint64_t getFieldSize(const FieldDecl *FD, QualType FT,
25                              ASTContext &Ctx) {
26   if (FD && FD->isBitField())
27     return FD->getBitWidthValue(Ctx);
28   return Ctx.getTypeSize(FT);
29 }
30 
31 namespace {
32 enum { DstIdx = 0, SrcIdx = 1 };
33 const char *ValNameStr[2] = {"dst", "src"};
34 
35 template <class Derived, class RetTy = void> struct DestructedTypeVisitor {
36   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
37     return asDerived().visit(FT.isDestructedType(), FT,
38                              std::forward<Ts>(Args)...);
39   }
40 
41   template <class... Ts>
42   RetTy visit(QualType::DestructionKind DK, QualType FT, Ts &&... Args) {
43     if (asDerived().getContext().getAsArrayType(FT))
44       return asDerived().visitArray(DK, FT, std::forward<Ts>(Args)...);
45 
46     switch (DK) {
47     case QualType::DK_objc_strong_lifetime:
48       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
49     case QualType::DK_nontrivial_c_struct:
50       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
51     case QualType::DK_none:
52       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
53     case QualType::DK_cxx_destructor:
54       return asDerived().visitCXXDestructor(FT, std::forward<Ts>(Args)...);
55     case QualType::DK_objc_weak_lifetime:
56       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
57     }
58 
59     llvm_unreachable("unknown destruction kind");
60   }
61 
62   Derived &asDerived() { return static_cast<Derived &>(*this); }
63 };
64 
65 template <class Derived, class RetTy = void>
66 struct DefaultInitializedTypeVisitor {
67   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
68     return asDerived().visit(FT.isNonTrivialToPrimitiveDefaultInitialize(), FT,
69                              std::forward<Ts>(Args)...);
70   }
71 
72   template <class... Ts>
73   RetTy visit(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
74               Ts &&... Args) {
75     if (asDerived().getContext().getAsArrayType(FT))
76       return asDerived().visitArray(PDIK, FT, std::forward<Ts>(Args)...);
77 
78     switch (PDIK) {
79     case QualType::PDIK_ARCStrong:
80       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
81     case QualType::PDIK_ARCWeak:
82       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
83     case QualType::PDIK_Struct:
84       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
85     case QualType::PDIK_Trivial:
86       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
87     }
88 
89     llvm_unreachable("unknown default-initialize kind");
90   }
91 
92   Derived &asDerived() { return static_cast<Derived &>(*this); }
93 };
94 
95 template <class Derived, bool IsMove, class RetTy = void>
96 struct CopiedTypeVisitor {
97   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
98     QualType::PrimitiveCopyKind PCK =
99         IsMove ? FT.isNonTrivialToPrimitiveDestructiveMove()
100                : FT.isNonTrivialToPrimitiveCopy();
101     return asDerived().visit(PCK, FT, std::forward<Ts>(Args)...);
102   }
103 
104   template <class... Ts>
105   RetTy visit(QualType::PrimitiveCopyKind PCK, QualType FT, Ts &&... Args) {
106     asDerived().preVisit(PCK, FT, std::forward<Ts>(Args)...);
107 
108     if (asDerived().getContext().getAsArrayType(FT))
109       return asDerived().visitArray(PCK, FT, std::forward<Ts>(Args)...);
110 
111     switch (PCK) {
112     case QualType::PCK_ARCStrong:
113       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
114     case QualType::PCK_ARCWeak:
115       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
116     case QualType::PCK_Struct:
117       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
118     case QualType::PCK_Trivial:
119       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
120     case QualType::PCK_VolatileTrivial:
121       return asDerived().visitVolatileTrivial(FT, std::forward<Ts>(Args)...);
122     }
123 
124     llvm_unreachable("unknown primitive copy kind");
125   }
126 
127   Derived &asDerived() { return static_cast<Derived &>(*this); }
128 };
129 
130 template <class Derived> struct StructVisitor {
131   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
132 
133   template <class... Ts>
134   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
135     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
136 
137     // Iterate over the fields of the struct.
138     for (const FieldDecl *FD : RD->fields()) {
139       QualType FT = FD->getType();
140       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
141       asDerived().visit(FT, FD, CurStructOffset, Args...);
142     }
143 
144     asDerived().flushTrivialFields(Args...);
145   }
146 
147   template <class... Ts> void visitTrivial(Ts... Args) {}
148 
149   template <class... Ts> void visitCXXDestructor(Ts... Args) {
150     llvm_unreachable("field of a C++ struct type is not expected");
151   }
152 
153   template <class... Ts> void flushTrivialFields(Ts... Args) {}
154 
155   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
156     return FD ? Ctx.getASTRecordLayout(FD->getParent())
157                     .getFieldOffset(FD->getFieldIndex())
158               : 0;
159   }
160 
161   CharUnits getFieldOffset(const FieldDecl *FD) {
162     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
163   }
164 
165   Derived &asDerived() { return static_cast<Derived &>(*this); }
166 
167   ASTContext &getContext() { return Ctx; }
168   ASTContext &Ctx;
169 };
170 
171 template <class Derived, bool IsMove>
172 struct CopyStructVisitor : StructVisitor<Derived>,
173                            CopiedTypeVisitor<Derived, IsMove> {
174   using StructVisitor<Derived>::asDerived;
175 
176   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
177 
178   template <class... Ts>
179   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
180                 const FieldDecl *FD, CharUnits CurStructOffsset,
181                 Ts &&... Args) {
182     if (PCK)
183       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
184   }
185 
186   template <class... Ts>
187   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
188                     Ts... Args) {
189     assert(!FT.isVolatileQualified() && "volatile field not expected");
190     ASTContext &Ctx = asDerived().getContext();
191     uint64_t FieldSize = getFieldSize(FD, FT, Ctx);
192 
193     // Ignore zero-sized fields.
194     if (FieldSize == 0)
195       return;
196 
197     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
198     uint64_t FEndInBits = FStartInBits + FieldSize;
199     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
200 
201     // Set Start if this is the first field of a sequence of trivial fields.
202     if (Start == End)
203       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
204     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
205   }
206 
207   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
208 };
209 
210 // This function creates the mangled name of a special function of a non-trivial
211 // C struct. Since there is no ODR in C, the function is mangled based on the
212 // struct contents and not the name. The mangled name has the following
213 // structure:
214 //
215 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
216 // <prefix> ::= "__destructor_" | "__default_constructor_" |
217 //              "__copy_constructor_" | "__move_constructor_" |
218 //              "__copy_assignment_" | "__move_assignment_"
219 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
220 // <struct-field-info> ::= <field-info>+
221 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
222 // <struct-or-scalar-field-info> ::= <struct-field-info> | <strong-field-info> |
223 //                                   <trivial-field-info>
224 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
225 //                        <num-elements> <innermost-element-info> "_AE"
226 // <innermost-element-info> ::= <struct-or-scalar-field-info>
227 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
228 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
229 
230 template <class Derived> struct GenFuncNameBase {
231   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
232     std::string S;
233     if (IsVolatile)
234       S = "v";
235     S += llvm::to_string(Offset.getQuantity());
236     return S;
237   }
238 
239   void visitARCStrong(QualType FT, const FieldDecl *FD,
240                       CharUnits CurStructOffset) {
241     appendStr("_s");
242     if (FT->isBlockPointerType())
243       appendStr("b");
244     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
245     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
246   }
247 
248   void visitARCWeak(QualType FT, const FieldDecl *FD,
249                     CharUnits CurStructOffset) {
250     appendStr("_w");
251     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
252     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
253   }
254 
255   void visitStruct(QualType QT, const FieldDecl *FD,
256                    CharUnits CurStructOffset) {
257     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
258     asDerived().visitStructFields(QT, FieldOffset);
259   }
260 
261   template <class FieldKind>
262   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
263                   CharUnits CurStructOffset) {
264     // String for non-volatile trivial fields is emitted when
265     // flushTrivialFields is called.
266     if (!FK)
267       return asDerived().visitTrivial(QT, FD, CurStructOffset);
268 
269     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
270     ASTContext &Ctx = asDerived().getContext();
271     const auto *AT = Ctx.getAsConstantArrayType(QT);
272     unsigned NumElts = Ctx.getConstantArrayElementCount(AT);
273     QualType EltTy = Ctx.getBaseElementType(AT);
274     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
275     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
276               llvm::to_string(EltSize.getQuantity()) + "n" +
277               llvm::to_string(NumElts));
278     EltTy = QT.isVolatileQualified() ? EltTy.withVolatile() : EltTy;
279     asDerived().visit(FK, EltTy, nullptr, FieldOffset);
280     appendStr("_AE");
281   }
282 
283   void appendStr(StringRef Str) { Name += Str; }
284 
285   std::string getName(QualType QT, bool IsVolatile) {
286     QT = IsVolatile ? QT.withVolatile() : QT;
287     asDerived().visitStructFields(QT, CharUnits::Zero());
288     return Name;
289   }
290 
291   Derived &asDerived() { return static_cast<Derived &>(*this); }
292 
293   std::string Name;
294 };
295 
296 template <class Derived>
297 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
298   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
299       : StructVisitor<Derived>(Ctx) {
300     this->appendStr(Prefix);
301     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
302   }
303 };
304 
305 // Helper function to create a null constant.
306 static llvm::Constant *getNullForVariable(Address Addr) {
307   llvm::Type *Ty = Addr.getElementType();
308   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
309 }
310 
311 template <bool IsMove>
312 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
313                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
314 
315   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
316                     CharUnits SrcAlignment, ASTContext &Ctx)
317       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
318     this->appendStr(Prefix);
319     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
320     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
321   }
322 
323   void flushTrivialFields() {
324     if (this->Start == this->End)
325       return;
326 
327     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
328                     llvm::to_string((this->End - this->Start).getQuantity()));
329 
330     this->Start = this->End = CharUnits::Zero();
331   }
332 
333   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
334                             CharUnits CurStackOffset) {
335     // Because volatile fields can be bit-fields and are individually copied,
336     // their offset and width are in bits.
337     uint64_t OffsetInBits =
338         this->Ctx.toBits(CurStackOffset) + this->getFieldOffsetInBits(FD);
339     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
340                     llvm::to_string(getFieldSize(FD, FT, this->Ctx)));
341   }
342 };
343 
344 struct GenDefaultInitializeFuncName
345     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
346       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
347   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
348       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
349                                                        DstAlignment, Ctx) {}
350 };
351 
352 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
353                                DestructedTypeVisitor<GenDestructorFuncName> {
354   GenDestructorFuncName(CharUnits DstAlignment, ASTContext &Ctx)
355       : GenUnaryFuncName<GenDestructorFuncName>("__destructor_", DstAlignment,
356                                                 Ctx) {}
357 };
358 
359 // Helper function that creates CGFunctionInfo for an N-ary special function.
360 template <size_t N>
361 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
362                                              FunctionArgList &Args) {
363   ASTContext &Ctx = CGM.getContext();
364   llvm::SmallVector<ImplicitParamDecl *, N> Params;
365   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
366 
367   for (unsigned I = 0; I < N; ++I)
368     Params.push_back(ImplicitParamDecl::Create(
369         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
370         ImplicitParamDecl::Other));
371 
372   for (auto &P : Params)
373     Args.push_back(P);
374 
375   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
376 }
377 
378 // Template classes that are used as bases for classes that emit special
379 // functions.
380 template <class Derived> struct GenFuncBase {
381   template <size_t N>
382   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStackOffset,
383                    std::array<Address, N> Addrs) {
384     this->asDerived().callSpecialFunction(
385         FT, CurStackOffset + asDerived().getFieldOffset(FD), Addrs);
386   }
387 
388   template <class FieldKind, size_t N>
389   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
390                   CharUnits CurStackOffset, std::array<Address, N> Addrs) {
391     // Non-volatile trivial fields are copied when flushTrivialFields is called.
392     if (!FK)
393       return asDerived().visitTrivial(QT, FD, CurStackOffset, Addrs);
394 
395     CodeGenFunction &CGF = *this->CGF;
396     ASTContext &Ctx = CGF.getContext();
397 
398     // Compute the end address.
399     QualType BaseEltQT;
400     std::array<Address, N> StartAddrs = Addrs;
401     for (unsigned I = 0; I < N; ++I)
402       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStackOffset, FD);
403     Address DstAddr = StartAddrs[DstIdx];
404     llvm::Value *NumElts =
405         CGF.emitArrayLength(Ctx.getAsArrayType(QT), BaseEltQT, DstAddr);
406     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
407     llvm::Value *BaseEltSizeVal =
408         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
409     llvm::Value *SizeInBytes =
410         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
411     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
412     llvm::Value *DstArrayEnd =
413         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
414     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
415                                             "dstarray.end");
416     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
417 
418     // Create the header block and insert the phi instructions.
419     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
420     CGF.EmitBlock(HeaderBB);
421     llvm::PHINode *PHIs[N];
422 
423     for (unsigned I = 0; I < N; ++I) {
424       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
425       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
426     }
427 
428     // Create the exit and loop body blocks.
429     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
430     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
431 
432     // Emit the comparison and conditional branch instruction that jumps to
433     // either the exit or the loop body.
434     llvm::Value *Done =
435         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
436     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
437 
438     // Visit the element of the array in the loop body.
439     CGF.EmitBlock(LoopBB);
440     QualType EltQT = Ctx.getAsArrayType(QT)->getElementType();
441     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
442     std::array<Address, N> NewAddrs = Addrs;
443 
444     for (unsigned I = 0; I < N; ++I)
445       NewAddrs[I] = Address(
446           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
447 
448     EltQT = QT.isVolatileQualified() ? EltQT.withVolatile() : EltQT;
449     this->asDerived().visit(EltQT, nullptr, CharUnits::Zero(), NewAddrs);
450 
451     LoopBB = CGF.Builder.GetInsertBlock();
452 
453     for (unsigned I = 0; I < N; ++I) {
454       // Instrs to update the destination and source addresses.
455       // Update phi instructions.
456       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
457       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
458     }
459 
460     // Insert an unconditional branch to the header block.
461     CGF.Builder.CreateBr(HeaderBB);
462     CGF.EmitBlock(ExitBB);
463   }
464 
465   /// Return an address with the specified offset from the passed address.
466   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
467     assert(Addr.isValid() && "invalid address");
468     if (Offset.getQuantity() == 0)
469       return Addr;
470     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
471     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity(),
472                                                CharUnits::One());
473     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
474   }
475 
476   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
477                             const FieldDecl *FD) {
478     return getAddrWithOffset(Addr, StructFieldOffset +
479                                        asDerived().getFieldOffset(FD));
480   }
481 
482   template <size_t N>
483   llvm::Function *
484   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
485               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
486     // If the special function already exists in the module, return it.
487     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
488       bool WrongType = false;
489       if (!F->getReturnType()->isVoidTy())
490         WrongType = true;
491       else {
492         for (const llvm::Argument &Arg : F->args())
493           if (Arg.getType() != CGM.Int8PtrPtrTy)
494             WrongType = true;
495       }
496 
497       if (WrongType) {
498         std::string FuncName = F->getName();
499         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
500         CGM.Error(Loc, "special function " + FuncName +
501                            " for non-trivial C struct has incorrect type");
502         return nullptr;
503       }
504       return F;
505     }
506 
507     ASTContext &Ctx = CGM.getContext();
508     FunctionArgList Args;
509     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
510     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
511     llvm::Function *F =
512         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
513                                FuncName, &CGM.getModule());
514     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
515     CGM.SetLLVMFunctionAttributes(nullptr, FI, F);
516     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
517     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
518     FunctionDecl *FD = FunctionDecl::Create(
519         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
520         II, Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
521     CodeGenFunction NewCGF(CGM);
522     setCGF(&NewCGF);
523     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
524 
525     for (unsigned I = 0; I < N; ++I) {
526       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
527       Addrs[I] = Address(V, Alignments[I]);
528     }
529 
530     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
531     CGF->FinishFunction();
532     return F;
533   }
534 
535   template <size_t N>
536   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
537                 CodeGenFunction &CallerCGF) {
538     std::array<CharUnits, N> Alignments;
539     llvm::Value *Ptrs[N];
540 
541     for (unsigned I = 0; I < N; ++I) {
542       Alignments[I] = Addrs[I].getAlignment();
543       Ptrs[I] =
544           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
545               .getPointer();
546     }
547 
548     if (llvm::Function *F =
549             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
550       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
551   }
552 
553   Derived &asDerived() { return static_cast<Derived &>(*this); }
554 
555   void setCGF(CodeGenFunction *F) { CGF = F; }
556 
557   CodeGenFunction *CGF = nullptr;
558 };
559 
560 template <class Derived, bool IsMove>
561 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
562                        GenFuncBase<Derived> {
563   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
564 
565   void flushTrivialFields(std::array<Address, 2> Addrs) {
566     CharUnits Size = this->End - this->Start;
567 
568     if (Size.getQuantity() == 0)
569       return;
570 
571     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
572     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
573 
574     // Emit memcpy.
575     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
576       llvm::Value *SizeVal =
577           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
578       DstAddr =
579           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
580       SrcAddr =
581           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
582       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
583     } else {
584       llvm::Type *Ty = llvm::Type::getIntNTy(
585           this->CGF->getLLVMContext(),
586           Size.getQuantity() * this->CGF->getContext().getCharWidth());
587       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
588       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
589       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
590       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
591     }
592 
593     this->Start = this->End = CharUnits::Zero();
594   }
595 
596   template <class... Ts>
597   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
598                             std::array<Address, 2> Addrs) {
599     LValue DstLV, SrcLV;
600     if (FD) {
601       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
602       llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
603       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
604       LValue DstBase = this->CGF->MakeAddrLValue(
605           this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
606       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
607       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
608       LValue SrcBase = this->CGF->MakeAddrLValue(
609           this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
610       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
611     } else {
612       llvm::PointerType *Ty = this->CGF->ConvertType(FT)->getPointerTo();
613       Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
614       Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
615       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
616       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
617     }
618     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
619     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
620   }
621 };
622 
623 // These classes that emit the special functions for a non-trivial struct.
624 struct GenDestructor : StructVisitor<GenDestructor>,
625                        GenFuncBase<GenDestructor>,
626                        DestructedTypeVisitor<GenDestructor> {
627   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
628   void visitARCStrong(QualType QT, const FieldDecl *FD,
629                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
630     CGF->destroyARCStrongImprecise(
631         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
632   }
633 
634   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
635                     std::array<Address, 1> Addrs) {
636     CGF->destroyARCWeak(
637         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
638   }
639 
640   void callSpecialFunction(QualType FT, CharUnits Offset,
641                            std::array<Address, 1> Addrs) {
642     CGF->callCStructDestructor(
643         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
644   }
645 };
646 
647 struct GenDefaultInitialize
648     : StructVisitor<GenDefaultInitialize>,
649       GenFuncBase<GenDefaultInitialize>,
650       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
651   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
652   GenDefaultInitialize(ASTContext &Ctx)
653       : StructVisitor<GenDefaultInitialize>(Ctx) {}
654 
655   void visitARCStrong(QualType QT, const FieldDecl *FD,
656                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
657     CGF->EmitNullInitialization(
658         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
659   }
660 
661   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
662                     std::array<Address, 1> Addrs) {
663     CGF->EmitNullInitialization(
664         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
665   }
666 
667   template <class FieldKind, size_t... Is>
668   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
669                   CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
670     if (!FK)
671       return visitTrivial(QT, FD, CurStackOffset, Addrs);
672 
673     ASTContext &Ctx = getContext();
674     CharUnits Size = Ctx.getTypeSizeInChars(QT);
675     QualType EltTy = Ctx.getBaseElementType(QT);
676 
677     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
678       GenFuncBaseTy::visitArray(FK, QT, FD, CurStackOffset, Addrs);
679       return;
680     }
681 
682     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
683     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
684     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
685     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
686                               QT.isVolatileQualified());
687   }
688 
689   void callSpecialFunction(QualType FT, CharUnits Offset,
690                            std::array<Address, 1> Addrs) {
691     CGF->callCStructDefaultConstructor(
692         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
693   }
694 };
695 
696 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
697   GenCopyConstructor(ASTContext &Ctx)
698       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
699 
700   void visitARCStrong(QualType QT, const FieldDecl *FD,
701                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
702     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
703     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
704     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
705         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
706     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
707     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
708   }
709 
710   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
711                     std::array<Address, 2> Addrs) {
712     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
713     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
714     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
715   }
716 
717   void callSpecialFunction(QualType FT, CharUnits Offset,
718                            std::array<Address, 2> Addrs) {
719     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
720                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
721   }
722 };
723 
724 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
725   GenMoveConstructor(ASTContext &Ctx)
726       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
727 
728   void visitARCStrong(QualType QT, const FieldDecl *FD,
729                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
730     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
731     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
732     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
733     llvm::Value *SrcVal =
734         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
735     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
736     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
737                            /* isInitialization */ true);
738   }
739 
740   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
741                     std::array<Address, 2> Addrs) {
742     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
743     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
744     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
745   }
746 
747   void callSpecialFunction(QualType FT, CharUnits Offset,
748                            std::array<Address, 2> Addrs) {
749     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
750                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
751   }
752 };
753 
754 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
755   GenCopyAssignment(ASTContext &Ctx)
756       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
757 
758   void visitARCStrong(QualType QT, const FieldDecl *FD,
759                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
760     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
761     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
762     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
763         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
764     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
765                             false);
766   }
767 
768   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
769                     std::array<Address, 2> Addrs) {
770     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
771     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
772     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
773   }
774 
775   void callSpecialFunction(QualType FT, CharUnits Offset,
776                            std::array<Address, 2> Addrs) {
777     CGF->callCStructCopyAssignmentOperator(
778         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
779         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
780   }
781 };
782 
783 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
784   GenMoveAssignment(ASTContext &Ctx)
785       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
786 
787   void visitARCStrong(QualType QT, const FieldDecl *FD,
788                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
789     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
790     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
791     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
792     llvm::Value *SrcVal =
793         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
794     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
795     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
796     llvm::Value *DstVal =
797         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
798     CGF->EmitStoreOfScalar(SrcVal, DstLV);
799     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
800   }
801 
802   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
803                     std::array<Address, 2> Addrs) {
804     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
805     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
806     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
807   }
808 
809   void callSpecialFunction(QualType FT, CharUnits Offset,
810                            std::array<Address, 2> Addrs) {
811     CGF->callCStructMoveAssignmentOperator(
812         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
813         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
814   }
815 };
816 
817 } // namespace
818 
819 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
820                                                Address Addr, QualType Type) {
821   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
822 }
823 
824 // Default-initialize a variable that is a non-trivial struct or an array of
825 // such structure.
826 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
827   GenDefaultInitialize Gen(getContext());
828   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
829   Gen.setCGF(this);
830   QualType QT = Dst.getType();
831   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
832   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
833 }
834 
835 template <class G, size_t N>
836 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
837                                 bool IsVolatile, CodeGenFunction &CGF,
838                                 std::array<Address, N> Addrs) {
839   for (unsigned I = 0; I < N; ++I)
840     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
841   QT = IsVolatile ? QT.withVolatile() : QT;
842   Gen.callFunc(FuncName, QT, Addrs, CGF);
843 }
844 
845 // Functions to emit calls to the special functions of a non-trivial C struct.
846 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
847   bool IsVolatile = Dst.isVolatile();
848   Address DstPtr = Dst.getAddress();
849   QualType QT = Dst.getType();
850   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
851   std::string FuncName = GenName.getName(QT, IsVolatile);
852   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
853                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
854 }
855 
856 void CodeGenFunction::callCStructDestructor(LValue Dst) {
857   bool IsVolatile = Dst.isVolatile();
858   Address DstPtr = Dst.getAddress();
859   QualType QT = Dst.getType();
860   GenDestructorFuncName GenName(DstPtr.getAlignment(), getContext());
861   std::string FuncName = GenName.getName(QT, IsVolatile);
862   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
863                       *this, std::array<Address, 1>({{DstPtr}}));
864 }
865 
866 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
867   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
868   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
869   QualType QT = Dst.getType();
870   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
871                                    SrcPtr.getAlignment(), getContext());
872   std::string FuncName = GenName.getName(QT, IsVolatile);
873   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
874                       IsVolatile, *this,
875                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
876 }
877 
878 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
879 
880 ) {
881   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
882   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
883   QualType QT = Dst.getType();
884   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
885                                    SrcPtr.getAlignment(), getContext());
886   std::string FuncName = GenName.getName(QT, IsVolatile);
887   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
888                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
889 }
890 
891 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
892   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
893   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
894   QualType QT = Dst.getType();
895   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
896                                   SrcPtr.getAlignment(), getContext());
897   std::string FuncName = GenName.getName(QT, IsVolatile);
898   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
899                       IsVolatile, *this,
900                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
901 }
902 
903 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
904 
905 ) {
906   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
907   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
908   QualType QT = Dst.getType();
909   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
910                                   SrcPtr.getAlignment(), getContext());
911   std::string FuncName = GenName.getName(QT, IsVolatile);
912   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
913                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
914 }
915