1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines functions to generate various special functions for C
11 // structs.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "llvm/Support/ScopedPrinter.h"
18 #include <array>
19 
20 using namespace clang;
21 using namespace CodeGen;
22 
23 // Return the size of a field in number of bits.
24 static uint64_t getFieldSize(const FieldDecl *FD, ASTContext &Ctx) {
25   if (FD->isBitField())
26     return FD->getBitWidthValue(Ctx);
27   return Ctx.getTypeSize(FD->getType());
28 }
29 
30 namespace {
31 enum { DstIdx = 0, SrcIdx = 1 };
32 const char *ValNameStr[2] = {"dst", "src"};
33 
34 template <class Derived, class RetTy = void> struct DestructedTypeVisitor {
35   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
36     return asDerived().visit(FT.isDestructedType(), FT,
37                              std::forward<Ts>(Args)...);
38   }
39 
40   template <class... Ts>
41   RetTy visit(QualType::DestructionKind DK, QualType FT, Ts &&... Args) {
42     if (asDerived().getContext().getAsArrayType(FT))
43       return asDerived().visitArray(DK, FT, std::forward<Ts>(Args)...);
44 
45     switch (DK) {
46     case QualType::DK_objc_strong_lifetime:
47       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
48     case QualType::DK_nontrivial_c_struct:
49       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
50     case QualType::DK_none:
51       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
52     case QualType::DK_cxx_destructor:
53       return asDerived().visitCXXDestructor(FT, std::forward<Ts>(Args)...);
54     case QualType::DK_objc_weak_lifetime:
55       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
56     }
57 
58     llvm_unreachable("unknown destruction kind");
59   }
60 
61   Derived &asDerived() { return static_cast<Derived &>(*this); }
62 };
63 
64 template <class Derived, class RetTy = void>
65 struct DefaultInitializedTypeVisitor {
66   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
67     return asDerived().visit(FT.isNonTrivialToPrimitiveDefaultInitialize(), FT,
68                              std::forward<Ts>(Args)...);
69   }
70 
71   template <class... Ts>
72   RetTy visit(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
73               Ts &&... Args) {
74     if (asDerived().getContext().getAsArrayType(FT))
75       return asDerived().visitArray(PDIK, FT, std::forward<Ts>(Args)...);
76 
77     switch (PDIK) {
78     case QualType::PDIK_ARCStrong:
79       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
80     case QualType::PDIK_ARCWeak:
81       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
82     case QualType::PDIK_Struct:
83       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
84     case QualType::PDIK_Trivial:
85       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
86     }
87 
88     llvm_unreachable("unknown default-initialize kind");
89   }
90 
91   Derived &asDerived() { return static_cast<Derived &>(*this); }
92 };
93 
94 template <class Derived, bool IsMove, class RetTy = void>
95 struct CopiedTypeVisitor {
96   template <class... Ts> RetTy visit(QualType FT, Ts &&... Args) {
97     QualType::PrimitiveCopyKind PCK =
98         IsMove ? FT.isNonTrivialToPrimitiveDestructiveMove()
99                : FT.isNonTrivialToPrimitiveCopy();
100     return asDerived().visit(PCK, FT, std::forward<Ts>(Args)...);
101   }
102 
103   template <class... Ts>
104   RetTy visit(QualType::PrimitiveCopyKind PCK, QualType FT, Ts &&... Args) {
105     asDerived().preVisit(PCK, FT, std::forward<Ts>(Args)...);
106 
107     if (asDerived().getContext().getAsArrayType(FT))
108       return asDerived().visitArray(PCK, FT, std::forward<Ts>(Args)...);
109 
110     switch (PCK) {
111     case QualType::PCK_ARCStrong:
112       return asDerived().visitARCStrong(FT, std::forward<Ts>(Args)...);
113     case QualType::PCK_ARCWeak:
114       return asDerived().visitARCWeak(FT, std::forward<Ts>(Args)...);
115     case QualType::PCK_Struct:
116       return asDerived().visitStruct(FT, std::forward<Ts>(Args)...);
117     case QualType::PCK_Trivial:
118       return asDerived().visitTrivial(FT, std::forward<Ts>(Args)...);
119     case QualType::PCK_VolatileTrivial:
120       return asDerived().visitVolatileTrivial(FT, std::forward<Ts>(Args)...);
121     }
122 
123     llvm_unreachable("unknown primitive copy kind");
124   }
125 
126   Derived &asDerived() { return static_cast<Derived &>(*this); }
127 };
128 
129 template <class Derived> struct StructVisitor {
130   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
131 
132   template <class... Ts>
133   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
134     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
135 
136     // Iterate over the fields of the struct.
137     for (const FieldDecl *FD : RD->fields()) {
138       QualType FT = FD->getType();
139       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
140       asDerived().visit(FT, FD, CurStructOffset, Args...);
141     }
142 
143     asDerived().flushTrivialFields(Args...);
144   }
145 
146   template <class... Ts> void visitTrivial(Ts... Args) {}
147 
148   template <class... Ts> void visitCXXDestructor(Ts... Args) {
149     llvm_unreachable("field of a C++ struct type is not expected");
150   }
151 
152   template <class... Ts> void flushTrivialFields(Ts... Args) {}
153 
154   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
155     return FD ? Ctx.getASTRecordLayout(FD->getParent())
156                     .getFieldOffset(FD->getFieldIndex())
157               : 0;
158   }
159 
160   CharUnits getFieldOffset(const FieldDecl *FD) {
161     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
162   }
163 
164   Derived &asDerived() { return static_cast<Derived &>(*this); }
165 
166   ASTContext &getContext() { return Ctx; }
167   ASTContext &Ctx;
168 };
169 
170 template <class Derived, bool IsMove>
171 struct CopyStructVisitor : StructVisitor<Derived>,
172                            CopiedTypeVisitor<Derived, IsMove> {
173   using StructVisitor<Derived>::asDerived;
174 
175   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
176 
177   template <class... Ts>
178   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
179                 const FieldDecl *FD, CharUnits CurStructOffsset,
180                 Ts &&... Args) {
181     if (PCK)
182       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
183   }
184 
185   template <class... Ts>
186   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
187                     Ts... Args) {
188     assert(!FT.isVolatileQualified() && "volatile field not expected");
189     ASTContext &Ctx = asDerived().getContext();
190     uint64_t FieldSize = getFieldSize(FD, Ctx);
191 
192     // Ignore zero-sized fields.
193     if (FieldSize == 0)
194       return;
195 
196     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
197     uint64_t FEndInBits = FStartInBits + FieldSize;
198     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
199 
200     // Set Start if this is the first field of a sequence of trivial fields.
201     if (Start == End)
202       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
203     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
204   }
205 
206   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
207 };
208 
209 // This function creates the mangled name of a special function of a non-trivial
210 // C struct. Since there is no ODR in C, the function is mangled based on the
211 // struct contents and not the name. The mangled name has the following
212 // structure:
213 //
214 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
215 // <prefix> ::= "__destructor_" | "__default_constructor_" |
216 //              "__copy_constructor_" | "__move_constructor_" |
217 //              "__copy_assignment_" | "__move_assignment_"
218 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
219 // <struct-field-info> ::= <field-info>+
220 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
221 // <struct-or-scalar-field-info> ::= <struct-field-info> | <strong-field-info> |
222 //                                   <trivial-field-info>
223 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
224 //                        <num-elements> <innermost-element-info> "_AE"
225 // <innermost-element-info> ::= <struct-or-scalar-field-info>
226 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
227 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
228 
229 template <class Derived> struct GenFuncNameBase {
230   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
231     std::string S;
232     if (IsVolatile)
233       S = "v";
234     S += llvm::to_string(Offset.getQuantity());
235     return S;
236   }
237 
238   void visitARCStrong(QualType FT, const FieldDecl *FD,
239                       CharUnits CurStructOffset) {
240     appendStr("_s");
241     if (FT->isBlockPointerType())
242       appendStr("b");
243     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
244     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
245   }
246 
247   void visitARCWeak(QualType FT, const FieldDecl *FD,
248                     CharUnits CurStructOffset) {
249     appendStr("_w");
250     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
251     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
252   }
253 
254   void visitStruct(QualType QT, const FieldDecl *FD,
255                    CharUnits CurStructOffset) {
256     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
257     asDerived().visitStructFields(QT, FieldOffset);
258   }
259 
260   template <class FieldKind>
261   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
262                   CharUnits CurStructOffset) {
263     // String for non-volatile trivial fields is emitted when
264     // flushTrivialFields is called.
265     if (!FK)
266       return asDerived().visitTrivial(QT, FD, CurStructOffset);
267 
268     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
269     ASTContext &Ctx = asDerived().getContext();
270     const auto *AT = Ctx.getAsConstantArrayType(QT);
271     unsigned NumElts = Ctx.getConstantArrayElementCount(AT);
272     QualType EltTy = Ctx.getBaseElementType(AT);
273     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
274     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
275               llvm::to_string(EltSize.getQuantity()) + "n" +
276               llvm::to_string(NumElts));
277     EltTy = QT.isVolatileQualified() ? EltTy.withVolatile() : EltTy;
278     asDerived().visit(FK, EltTy, nullptr, FieldOffset);
279     appendStr("_AE");
280   }
281 
282   void appendStr(StringRef Str) { Name += Str; }
283 
284   std::string getName(QualType QT, bool IsVolatile) {
285     QT = IsVolatile ? QT.withVolatile() : QT;
286     asDerived().visitStructFields(QT, CharUnits::Zero());
287     return Name;
288   }
289 
290   Derived &asDerived() { return static_cast<Derived &>(*this); }
291 
292   std::string Name;
293 };
294 
295 template <class Derived>
296 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
297   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
298       : StructVisitor<Derived>(Ctx) {
299     this->appendStr(Prefix);
300     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
301   }
302 };
303 
304 // Helper function to create a null constant.
305 static llvm::Constant *getNullForVariable(Address Addr) {
306   llvm::Type *Ty = Addr.getElementType();
307   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
308 }
309 
310 template <bool IsMove>
311 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
312                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
313 
314   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
315                     CharUnits SrcAlignment, ASTContext &Ctx)
316       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
317     this->appendStr(Prefix);
318     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
319     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
320   }
321 
322   void flushTrivialFields() {
323     if (this->Start == this->End)
324       return;
325 
326     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
327                     llvm::to_string((this->End - this->Start).getQuantity()));
328 
329     this->Start = this->End = CharUnits::Zero();
330   }
331 
332   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
333                             CharUnits CurStackOffset) {
334     // Because volatile fields can be bit-fields and are individually copied,
335     // their offset and width are in bits.
336     uint64_t OffsetInBits =
337         this->Ctx.toBits(CurStackOffset) + this->getFieldOffsetInBits(FD);
338     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
339                     llvm::to_string(getFieldSize(FD, this->Ctx)));
340   }
341 };
342 
343 struct GenDefaultInitializeFuncName
344     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
345       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
346   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
347       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
348                                                        DstAlignment, Ctx) {}
349 };
350 
351 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
352                                DestructedTypeVisitor<GenDestructorFuncName> {
353   GenDestructorFuncName(CharUnits DstAlignment, ASTContext &Ctx)
354       : GenUnaryFuncName<GenDestructorFuncName>("__destructor_", DstAlignment,
355                                                 Ctx) {}
356 };
357 
358 // Helper function that creates CGFunctionInfo for an N-ary special function.
359 template <size_t N>
360 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
361                                              FunctionArgList &Args) {
362   ASTContext &Ctx = CGM.getContext();
363   llvm::SmallVector<ImplicitParamDecl *, N> Params;
364   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
365 
366   for (unsigned I = 0; I < N; ++I)
367     Params.push_back(ImplicitParamDecl::Create(
368         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
369         ImplicitParamDecl::Other));
370 
371   for (auto &P : Params)
372     Args.push_back(P);
373 
374   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
375 }
376 
377 // Template classes that are used as bases for classes that emit special
378 // functions.
379 template <class Derived> struct GenFuncBase {
380   template <size_t N>
381   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStackOffset,
382                    std::array<Address, N> Addrs) {
383     this->asDerived().callSpecialFunction(
384         FT, CurStackOffset + asDerived().getFieldOffset(FD), Addrs);
385   }
386 
387   template <class FieldKind, size_t N>
388   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
389                   CharUnits CurStackOffset, std::array<Address, N> Addrs) {
390     // Non-volatile trivial fields are copied when flushTrivialFields is called.
391     if (!FK)
392       return asDerived().visitTrivial(QT, FD, CurStackOffset, Addrs);
393 
394     CodeGenFunction &CGF = *this->CGF;
395     ASTContext &Ctx = CGF.getContext();
396 
397     // Compute the end address.
398     QualType BaseEltQT;
399     std::array<Address, N> StartAddrs = Addrs;
400     for (unsigned I = 0; I < N; ++I)
401       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStackOffset, FD);
402     Address DstAddr = StartAddrs[DstIdx];
403     llvm::Value *NumElts =
404         CGF.emitArrayLength(Ctx.getAsArrayType(QT), BaseEltQT, DstAddr);
405     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
406     llvm::Value *BaseEltSizeVal =
407         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
408     llvm::Value *SizeInBytes =
409         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
410     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
411     llvm::Value *DstArrayEnd =
412         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
413     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
414                                             "dstarray.end");
415     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
416 
417     // Create the header block and insert the phi instructions.
418     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
419     CGF.EmitBlock(HeaderBB);
420     llvm::PHINode *PHIs[N];
421 
422     for (unsigned I = 0; I < N; ++I) {
423       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
424       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
425     }
426 
427     // Create the exit and loop body blocks.
428     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
429     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
430 
431     // Emit the comparison and conditional branch instruction that jumps to
432     // either the exit or the loop body.
433     llvm::Value *Done =
434         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
435     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
436 
437     // Visit the element of the array in the loop body.
438     CGF.EmitBlock(LoopBB);
439     QualType EltQT = Ctx.getAsArrayType(QT)->getElementType();
440     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
441     std::array<Address, N> NewAddrs = Addrs;
442 
443     for (unsigned I = 0; I < N; ++I)
444       NewAddrs[I] = Address(
445           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
446 
447     EltQT = QT.isVolatileQualified() ? EltQT.withVolatile() : EltQT;
448     this->asDerived().visit(EltQT, nullptr, CharUnits::Zero(), NewAddrs);
449 
450     LoopBB = CGF.Builder.GetInsertBlock();
451 
452     for (unsigned I = 0; I < N; ++I) {
453       // Instrs to update the destination and source addresses.
454       // Update phi instructions.
455       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
456       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
457     }
458 
459     // Insert an unconditional branch to the header block.
460     CGF.Builder.CreateBr(HeaderBB);
461     CGF.EmitBlock(ExitBB);
462   }
463 
464   /// Return an address with the specified offset from the passed address.
465   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
466     assert(Addr.isValid() && "invalid address");
467     if (Offset.getQuantity() == 0)
468       return Addr;
469     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
470     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity(),
471                                                CharUnits::One());
472     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
473   }
474 
475   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
476                             const FieldDecl *FD) {
477     return getAddrWithOffset(Addr, StructFieldOffset +
478                                        asDerived().getFieldOffset(FD));
479   }
480 
481   template <size_t N>
482   llvm::Function *
483   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
484               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
485     // If the special function already exists in the module, return it.
486     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
487       bool WrongType = false;
488       if (!F->getReturnType()->isVoidTy())
489         WrongType = true;
490       else {
491         for (const llvm::Argument &Arg : F->args())
492           if (Arg.getType() != CGM.Int8PtrPtrTy)
493             WrongType = true;
494       }
495 
496       if (WrongType) {
497         std::string FuncName = F->getName();
498         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
499         CGM.Error(Loc, "special function " + FuncName +
500                            " for non-trivial C struct has incorrect type");
501         return nullptr;
502       }
503       return F;
504     }
505 
506     ASTContext &Ctx = CGM.getContext();
507     FunctionArgList Args;
508     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
509     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
510     llvm::Function *F =
511         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
512                                FuncName, &CGM.getModule());
513     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
514     CGM.SetLLVMFunctionAttributes(nullptr, FI, F);
515     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
516     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
517     FunctionDecl *FD = FunctionDecl::Create(
518         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
519         II, Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
520     CodeGenFunction NewCGF(CGM);
521     setCGF(&NewCGF);
522     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
523 
524     for (unsigned I = 0; I < N; ++I) {
525       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
526       Addrs[I] = Address(V, Alignments[I]);
527     }
528 
529     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
530     CGF->FinishFunction();
531     return F;
532   }
533 
534   template <size_t N>
535   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
536                 CodeGenFunction &CallerCGF) {
537     std::array<CharUnits, N> Alignments;
538     llvm::Value *Ptrs[N];
539 
540     for (unsigned I = 0; I < N; ++I) {
541       Alignments[I] = Addrs[I].getAlignment();
542       Ptrs[I] =
543           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
544               .getPointer();
545     }
546 
547     if (llvm::Function *F =
548             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
549       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
550   }
551 
552   Derived &asDerived() { return static_cast<Derived &>(*this); }
553 
554   void setCGF(CodeGenFunction *F) { CGF = F; }
555 
556   CodeGenFunction *CGF = nullptr;
557 };
558 
559 template <class Derived, bool IsMove>
560 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
561                        GenFuncBase<Derived> {
562   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
563 
564   void flushTrivialFields(std::array<Address, 2> Addrs) {
565     CharUnits Size = this->End - this->Start;
566 
567     if (Size.getQuantity() == 0)
568       return;
569 
570     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
571     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
572 
573     // Emit memcpy.
574     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
575       llvm::Value *SizeVal =
576           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
577       DstAddr =
578           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
579       SrcAddr =
580           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
581       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
582     } else {
583       llvm::Type *Ty = llvm::Type::getIntNTy(
584           this->CGF->getLLVMContext(),
585           Size.getQuantity() * this->CGF->getContext().getCharWidth());
586       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
587       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
588       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
589       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
590     }
591 
592     this->Start = this->End = CharUnits::Zero();
593   }
594 
595   template <class... Ts>
596   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
597                             std::array<Address, 2> Addrs) {
598     QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
599     llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
600     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
601     LValue DstBase = this->CGF->MakeAddrLValue(
602         this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
603     LValue DstLV = this->CGF->EmitLValueForField(DstBase, FD);
604     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
605     LValue SrcBase = this->CGF->MakeAddrLValue(
606         this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
607     LValue SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
608     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
609     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
610   }
611 };
612 
613 // These classes that emit the special functions for a non-trivial struct.
614 struct GenDestructor : StructVisitor<GenDestructor>,
615                        GenFuncBase<GenDestructor>,
616                        DestructedTypeVisitor<GenDestructor> {
617   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
618   void visitARCStrong(QualType QT, const FieldDecl *FD,
619                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
620     CGF->destroyARCStrongImprecise(
621         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
622   }
623 
624   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
625                     std::array<Address, 1> Addrs) {
626     CGF->destroyARCWeak(
627         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
628   }
629 
630   void callSpecialFunction(QualType FT, CharUnits Offset,
631                            std::array<Address, 1> Addrs) {
632     CGF->callCStructDestructor(
633         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
634   }
635 };
636 
637 struct GenDefaultInitialize
638     : StructVisitor<GenDefaultInitialize>,
639       GenFuncBase<GenDefaultInitialize>,
640       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
641   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
642   GenDefaultInitialize(ASTContext &Ctx)
643       : StructVisitor<GenDefaultInitialize>(Ctx) {}
644 
645   void visitARCStrong(QualType QT, const FieldDecl *FD,
646                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
647     CGF->EmitNullInitialization(
648         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
649   }
650 
651   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
652                     std::array<Address, 1> Addrs) {
653     CGF->EmitNullInitialization(
654         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
655   }
656 
657   template <class FieldKind, size_t... Is>
658   void visitArray(FieldKind FK, QualType QT, const FieldDecl *FD,
659                   CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
660     if (!FK)
661       return visitTrivial(QT, FD, CurStackOffset, Addrs);
662 
663     ASTContext &Ctx = getContext();
664     CharUnits Size = Ctx.getTypeSizeInChars(QT);
665     QualType EltTy = Ctx.getBaseElementType(QT);
666 
667     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
668       GenFuncBaseTy::visitArray(FK, QT, FD, CurStackOffset, Addrs);
669       return;
670     }
671 
672     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
673     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
674     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
675     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
676                               QT.isVolatileQualified());
677   }
678 
679   void callSpecialFunction(QualType FT, CharUnits Offset,
680                            std::array<Address, 1> Addrs) {
681     CGF->callCStructDefaultConstructor(
682         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
683   }
684 };
685 
686 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
687   GenCopyConstructor(ASTContext &Ctx)
688       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
689 
690   void visitARCStrong(QualType QT, const FieldDecl *FD,
691                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
692     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
693     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
694     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
695         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
696     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
697     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
698   }
699 
700   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
701                     std::array<Address, 2> Addrs) {
702     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
703     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
704     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
705   }
706 
707   void callSpecialFunction(QualType FT, CharUnits Offset,
708                            std::array<Address, 2> Addrs) {
709     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
710                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
711   }
712 };
713 
714 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
715   GenMoveConstructor(ASTContext &Ctx)
716       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
717 
718   void visitARCStrong(QualType QT, const FieldDecl *FD,
719                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
720     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
721     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
722     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
723     llvm::Value *SrcVal =
724         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
725     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
726     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
727                            /* isInitialization */ true);
728   }
729 
730   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
731                     std::array<Address, 2> Addrs) {
732     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
733     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
734     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
735   }
736 
737   void callSpecialFunction(QualType FT, CharUnits Offset,
738                            std::array<Address, 2> Addrs) {
739     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
740                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
741   }
742 };
743 
744 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
745   GenCopyAssignment(ASTContext &Ctx)
746       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
747 
748   void visitARCStrong(QualType QT, const FieldDecl *FD,
749                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
750     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
751     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
752     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
753         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
754     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
755                             false);
756   }
757 
758   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
759                     std::array<Address, 2> Addrs) {
760     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
761     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
762     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
763   }
764 
765   void callSpecialFunction(QualType FT, CharUnits Offset,
766                            std::array<Address, 2> Addrs) {
767     CGF->callCStructCopyAssignmentOperator(
768         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
769         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
770   }
771 };
772 
773 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
774   GenMoveAssignment(ASTContext &Ctx)
775       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
776 
777   void visitARCStrong(QualType QT, const FieldDecl *FD,
778                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
779     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
780     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
781     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
782     llvm::Value *SrcVal =
783         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
784     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
785     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
786     llvm::Value *DstVal =
787         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
788     CGF->EmitStoreOfScalar(SrcVal, DstLV);
789     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
790   }
791 
792   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
793                     std::array<Address, 2> Addrs) {
794     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
795     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
796     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
797   }
798 
799   void callSpecialFunction(QualType FT, CharUnits Offset,
800                            std::array<Address, 2> Addrs) {
801     CGF->callCStructMoveAssignmentOperator(
802         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
803         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
804   }
805 };
806 
807 } // namespace
808 
809 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
810                                                Address Addr, QualType Type) {
811   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
812 }
813 
814 // Default-initialize a variable that is a non-trivial struct or an array of
815 // such structure.
816 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
817   GenDefaultInitialize Gen(getContext());
818   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
819   Gen.setCGF(this);
820   QualType QT = Dst.getType();
821   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
822   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
823 }
824 
825 template <class G, size_t N>
826 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
827                                 bool IsVolatile, CodeGenFunction &CGF,
828                                 std::array<Address, N> Addrs) {
829   for (unsigned I = 0; I < N; ++I)
830     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
831   QT = IsVolatile ? QT.withVolatile() : QT;
832   Gen.callFunc(FuncName, QT, Addrs, CGF);
833 }
834 
835 // Functions to emit calls to the special functions of a non-trivial C struct.
836 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
837   bool IsVolatile = Dst.isVolatile();
838   Address DstPtr = Dst.getAddress();
839   QualType QT = Dst.getType();
840   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
841   std::string FuncName = GenName.getName(QT, IsVolatile);
842   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
843                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
844 }
845 
846 void CodeGenFunction::callCStructDestructor(LValue Dst) {
847   bool IsVolatile = Dst.isVolatile();
848   Address DstPtr = Dst.getAddress();
849   QualType QT = Dst.getType();
850   GenDestructorFuncName GenName(DstPtr.getAlignment(), getContext());
851   std::string FuncName = GenName.getName(QT, IsVolatile);
852   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
853                       *this, std::array<Address, 1>({{DstPtr}}));
854 }
855 
856 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
857   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
858   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
859   QualType QT = Dst.getType();
860   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
861                                    SrcPtr.getAlignment(), getContext());
862   std::string FuncName = GenName.getName(QT, IsVolatile);
863   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
864                       IsVolatile, *this,
865                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
866 }
867 
868 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
869 
870 ) {
871   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
872   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
873   QualType QT = Dst.getType();
874   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
875                                    SrcPtr.getAlignment(), getContext());
876   std::string FuncName = GenName.getName(QT, IsVolatile);
877   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
878                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
879 }
880 
881 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
882   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
883   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
884   QualType QT = Dst.getType();
885   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
886                                   SrcPtr.getAlignment(), getContext());
887   std::string FuncName = GenName.getName(QT, IsVolatile);
888   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
889                       IsVolatile, *this,
890                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
891 }
892 
893 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
894 
895 ) {
896   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
897   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
898   QualType QT = Dst.getType();
899   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
900                                   SrcPtr.getAlignment(), getContext());
901   std::string FuncName = GenName.getName(QT, IsVolatile);
902   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
903                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
904 }
905