1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines functions to generate various special functions for C
11 // structs.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "clang/AST/NonTrivialTypeVisitor.h"
18 #include "llvm/Support/ScopedPrinter.h"
19 #include <array>
20 
21 using namespace clang;
22 using namespace CodeGen;
23 
24 // Return the size of a field in number of bits.
25 static uint64_t getFieldSize(const FieldDecl *FD, QualType FT,
26                              ASTContext &Ctx) {
27   if (FD && FD->isBitField())
28     return FD->getBitWidthValue(Ctx);
29   return Ctx.getTypeSize(FT);
30 }
31 
32 namespace {
33 enum { DstIdx = 0, SrcIdx = 1 };
34 const char *ValNameStr[2] = {"dst", "src"};
35 
36 template <class Derived> struct StructVisitor {
37   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
38 
39   template <class... Ts>
40   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
41     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
42 
43     // Iterate over the fields of the struct.
44     for (const FieldDecl *FD : RD->fields()) {
45       QualType FT = FD->getType();
46       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
47       asDerived().visit(FT, FD, CurStructOffset, Args...);
48     }
49 
50     asDerived().flushTrivialFields(Args...);
51   }
52 
53   template <class... Ts> void visitTrivial(Ts... Args) {}
54 
55   template <class... Ts> void visitCXXDestructor(Ts... Args) {
56     llvm_unreachable("field of a C++ struct type is not expected");
57   }
58 
59   template <class... Ts> void flushTrivialFields(Ts... Args) {}
60 
61   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
62     return FD ? Ctx.getASTRecordLayout(FD->getParent())
63                     .getFieldOffset(FD->getFieldIndex())
64               : 0;
65   }
66 
67   CharUnits getFieldOffset(const FieldDecl *FD) {
68     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
69   }
70 
71   Derived &asDerived() { return static_cast<Derived &>(*this); }
72 
73   ASTContext &getContext() { return Ctx; }
74   ASTContext &Ctx;
75 };
76 
77 template <class Derived, bool IsMove>
78 struct CopyStructVisitor : StructVisitor<Derived>,
79                            CopiedTypeVisitor<Derived, IsMove> {
80   using StructVisitor<Derived>::asDerived;
81   using Super = CopiedTypeVisitor<Derived, IsMove>;
82 
83   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
84 
85   template <class... Ts>
86   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
87                 const FieldDecl *FD, CharUnits CurStructOffsset,
88                 Ts &&... Args) {
89     if (PCK)
90       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
91   }
92 
93   template <class... Ts>
94   void visitWithKind(QualType::PrimitiveCopyKind PCK, QualType FT,
95                      const FieldDecl *FD, CharUnits CurStructOffsset,
96                      Ts &&... Args) {
97     if (const auto *AT = asDerived().getContext().getAsArrayType(FT)) {
98       asDerived().visitArray(PCK, AT, FT.isVolatileQualified(), FD,
99                              CurStructOffsset, std::forward<Ts>(Args)...);
100       return;
101     }
102 
103     Super::visitWithKind(PCK, FT, FD, CurStructOffsset,
104                          std::forward<Ts>(Args)...);
105   }
106 
107   template <class... Ts>
108   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
109                     Ts... Args) {
110     assert(!FT.isVolatileQualified() && "volatile field not expected");
111     ASTContext &Ctx = asDerived().getContext();
112     uint64_t FieldSize = getFieldSize(FD, FT, Ctx);
113 
114     // Ignore zero-sized fields.
115     if (FieldSize == 0)
116       return;
117 
118     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
119     uint64_t FEndInBits = FStartInBits + FieldSize;
120     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
121 
122     // Set Start if this is the first field of a sequence of trivial fields.
123     if (Start == End)
124       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
125     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
126   }
127 
128   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
129 };
130 
131 // This function creates the mangled name of a special function of a non-trivial
132 // C struct. Since there is no ODR in C, the function is mangled based on the
133 // struct contents and not the name. The mangled name has the following
134 // structure:
135 //
136 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
137 // <prefix> ::= "__destructor_" | "__default_constructor_" |
138 //              "__copy_constructor_" | "__move_constructor_" |
139 //              "__copy_assignment_" | "__move_assignment_"
140 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
141 // <struct-field-info> ::= <field-info>+
142 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
143 // <struct-or-scalar-field-info> ::= <struct-field-info> | <strong-field-info> |
144 //                                   <trivial-field-info>
145 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
146 //                        <num-elements> <innermost-element-info> "_AE"
147 // <innermost-element-info> ::= <struct-or-scalar-field-info>
148 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
149 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
150 
151 template <class Derived> struct GenFuncNameBase {
152   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
153     std::string S;
154     if (IsVolatile)
155       S = "v";
156     S += llvm::to_string(Offset.getQuantity());
157     return S;
158   }
159 
160   void visitARCStrong(QualType FT, const FieldDecl *FD,
161                       CharUnits CurStructOffset) {
162     appendStr("_s");
163     if (FT->isBlockPointerType())
164       appendStr("b");
165     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
166     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
167   }
168 
169   void visitARCWeak(QualType FT, const FieldDecl *FD,
170                     CharUnits CurStructOffset) {
171     appendStr("_w");
172     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
173     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
174   }
175 
176   void visitStruct(QualType QT, const FieldDecl *FD,
177                    CharUnits CurStructOffset) {
178     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
179     asDerived().visitStructFields(QT, FieldOffset);
180   }
181 
182   template <class FieldKind>
183   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
184                   const FieldDecl *FD, CharUnits CurStructOffset) {
185     // String for non-volatile trivial fields is emitted when
186     // flushTrivialFields is called.
187     if (!FK)
188       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStructOffset);
189 
190     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
191     ASTContext &Ctx = asDerived().getContext();
192     const ConstantArrayType *CAT = cast<ConstantArrayType>(AT);
193     unsigned NumElts = Ctx.getConstantArrayElementCount(CAT);
194     QualType EltTy = Ctx.getBaseElementType(CAT);
195     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
196     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
197               llvm::to_string(EltSize.getQuantity()) + "n" +
198               llvm::to_string(NumElts));
199     EltTy = IsVolatile ? EltTy.withVolatile() : EltTy;
200     asDerived().visitWithKind(FK, EltTy, nullptr, FieldOffset);
201     appendStr("_AE");
202   }
203 
204   void appendStr(StringRef Str) { Name += Str; }
205 
206   std::string getName(QualType QT, bool IsVolatile) {
207     QT = IsVolatile ? QT.withVolatile() : QT;
208     asDerived().visitStructFields(QT, CharUnits::Zero());
209     return Name;
210   }
211 
212   Derived &asDerived() { return static_cast<Derived &>(*this); }
213 
214   std::string Name;
215 };
216 
217 template <class Derived>
218 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
219   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
220       : StructVisitor<Derived>(Ctx) {
221     this->appendStr(Prefix);
222     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
223   }
224 };
225 
226 // Helper function to create a null constant.
227 static llvm::Constant *getNullForVariable(Address Addr) {
228   llvm::Type *Ty = Addr.getElementType();
229   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
230 }
231 
232 template <bool IsMove>
233 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
234                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
235 
236   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
237                     CharUnits SrcAlignment, ASTContext &Ctx)
238       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
239     this->appendStr(Prefix);
240     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
241     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
242   }
243 
244   void flushTrivialFields() {
245     if (this->Start == this->End)
246       return;
247 
248     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
249                     llvm::to_string((this->End - this->Start).getQuantity()));
250 
251     this->Start = this->End = CharUnits::Zero();
252   }
253 
254   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
255                             CharUnits CurStackOffset) {
256     // Because volatile fields can be bit-fields and are individually copied,
257     // their offset and width are in bits.
258     uint64_t OffsetInBits =
259         this->Ctx.toBits(CurStackOffset) + this->getFieldOffsetInBits(FD);
260     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
261                     llvm::to_string(getFieldSize(FD, FT, this->Ctx)));
262   }
263 };
264 
265 struct GenDefaultInitializeFuncName
266     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
267       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
268   using Super = DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName>;
269   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
270       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
271                                                        DstAlignment, Ctx) {}
272   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
273                      const FieldDecl *FD, CharUnits CurStructOffset) {
274     if (const auto *AT = getContext().getAsArrayType(FT)) {
275       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
276       return;
277     }
278 
279     Super::visitWithKind(PDIK, FT, FD, CurStructOffset);
280   }
281 };
282 
283 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
284                                DestructedTypeVisitor<GenDestructorFuncName> {
285   using Super = DestructedTypeVisitor<GenDestructorFuncName>;
286   GenDestructorFuncName(CharUnits DstAlignment, ASTContext &Ctx)
287       : GenUnaryFuncName<GenDestructorFuncName>("__destructor_", DstAlignment,
288                                                 Ctx) {}
289   void visitWithKind(QualType::DestructionKind DK, QualType FT,
290                      const FieldDecl *FD, CharUnits CurStructOffset) {
291     if (const auto *AT = getContext().getAsArrayType(FT)) {
292       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
293       return;
294     }
295 
296     Super::visitWithKind(DK, FT, FD, CurStructOffset);
297   }
298 };
299 
300 // Helper function that creates CGFunctionInfo for an N-ary special function.
301 template <size_t N>
302 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
303                                              FunctionArgList &Args) {
304   ASTContext &Ctx = CGM.getContext();
305   llvm::SmallVector<ImplicitParamDecl *, N> Params;
306   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
307 
308   for (unsigned I = 0; I < N; ++I)
309     Params.push_back(ImplicitParamDecl::Create(
310         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
311         ImplicitParamDecl::Other));
312 
313   for (auto &P : Params)
314     Args.push_back(P);
315 
316   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
317 }
318 
319 // Template classes that are used as bases for classes that emit special
320 // functions.
321 template <class Derived> struct GenFuncBase {
322   template <size_t N>
323   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStackOffset,
324                    std::array<Address, N> Addrs) {
325     this->asDerived().callSpecialFunction(
326         FT, CurStackOffset + asDerived().getFieldOffset(FD), Addrs);
327   }
328 
329   template <class FieldKind, size_t N>
330   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
331                   const FieldDecl *FD, CharUnits CurStackOffset,
332                   std::array<Address, N> Addrs) {
333     // Non-volatile trivial fields are copied when flushTrivialFields is called.
334     if (!FK)
335       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStackOffset,
336                                       Addrs);
337 
338     CodeGenFunction &CGF = *this->CGF;
339     ASTContext &Ctx = CGF.getContext();
340 
341     // Compute the end address.
342     QualType BaseEltQT;
343     std::array<Address, N> StartAddrs = Addrs;
344     for (unsigned I = 0; I < N; ++I)
345       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStackOffset, FD);
346     Address DstAddr = StartAddrs[DstIdx];
347     llvm::Value *NumElts = CGF.emitArrayLength(AT, BaseEltQT, DstAddr);
348     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
349     llvm::Value *BaseEltSizeVal =
350         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
351     llvm::Value *SizeInBytes =
352         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
353     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
354     llvm::Value *DstArrayEnd =
355         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
356     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
357                                             "dstarray.end");
358     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
359 
360     // Create the header block and insert the phi instructions.
361     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
362     CGF.EmitBlock(HeaderBB);
363     llvm::PHINode *PHIs[N];
364 
365     for (unsigned I = 0; I < N; ++I) {
366       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
367       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
368     }
369 
370     // Create the exit and loop body blocks.
371     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
372     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
373 
374     // Emit the comparison and conditional branch instruction that jumps to
375     // either the exit or the loop body.
376     llvm::Value *Done =
377         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
378     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
379 
380     // Visit the element of the array in the loop body.
381     CGF.EmitBlock(LoopBB);
382     QualType EltQT = AT->getElementType();
383     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
384     std::array<Address, N> NewAddrs = Addrs;
385 
386     for (unsigned I = 0; I < N; ++I)
387       NewAddrs[I] = Address(
388           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
389 
390     EltQT = IsVolatile ? EltQT.withVolatile() : EltQT;
391     this->asDerived().visitWithKind(FK, EltQT, nullptr, CharUnits::Zero(),
392                                     NewAddrs);
393 
394     LoopBB = CGF.Builder.GetInsertBlock();
395 
396     for (unsigned I = 0; I < N; ++I) {
397       // Instrs to update the destination and source addresses.
398       // Update phi instructions.
399       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
400       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
401     }
402 
403     // Insert an unconditional branch to the header block.
404     CGF.Builder.CreateBr(HeaderBB);
405     CGF.EmitBlock(ExitBB);
406   }
407 
408   /// Return an address with the specified offset from the passed address.
409   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
410     assert(Addr.isValid() && "invalid address");
411     if (Offset.getQuantity() == 0)
412       return Addr;
413     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
414     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity(),
415                                                CharUnits::One());
416     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
417   }
418 
419   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
420                             const FieldDecl *FD) {
421     return getAddrWithOffset(Addr, StructFieldOffset +
422                                        asDerived().getFieldOffset(FD));
423   }
424 
425   template <size_t N>
426   llvm::Function *
427   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
428               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
429     // If the special function already exists in the module, return it.
430     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
431       bool WrongType = false;
432       if (!F->getReturnType()->isVoidTy())
433         WrongType = true;
434       else {
435         for (const llvm::Argument &Arg : F->args())
436           if (Arg.getType() != CGM.Int8PtrPtrTy)
437             WrongType = true;
438       }
439 
440       if (WrongType) {
441         std::string FuncName = F->getName();
442         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
443         CGM.Error(Loc, "special function " + FuncName +
444                            " for non-trivial C struct has incorrect type");
445         return nullptr;
446       }
447       return F;
448     }
449 
450     ASTContext &Ctx = CGM.getContext();
451     FunctionArgList Args;
452     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
453     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
454     llvm::Function *F =
455         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
456                                FuncName, &CGM.getModule());
457     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
458     CGM.SetLLVMFunctionAttributes(nullptr, FI, F);
459     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
460     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
461     FunctionDecl *FD = FunctionDecl::Create(
462         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
463         II, Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
464     CodeGenFunction NewCGF(CGM);
465     setCGF(&NewCGF);
466     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
467 
468     for (unsigned I = 0; I < N; ++I) {
469       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
470       Addrs[I] = Address(V, Alignments[I]);
471     }
472 
473     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
474     CGF->FinishFunction();
475     return F;
476   }
477 
478   template <size_t N>
479   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
480                 CodeGenFunction &CallerCGF) {
481     std::array<CharUnits, N> Alignments;
482     llvm::Value *Ptrs[N];
483 
484     for (unsigned I = 0; I < N; ++I) {
485       Alignments[I] = Addrs[I].getAlignment();
486       Ptrs[I] =
487           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
488               .getPointer();
489     }
490 
491     if (llvm::Function *F =
492             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
493       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
494   }
495 
496   Derived &asDerived() { return static_cast<Derived &>(*this); }
497 
498   void setCGF(CodeGenFunction *F) { CGF = F; }
499 
500   CodeGenFunction *CGF = nullptr;
501 };
502 
503 template <class Derived, bool IsMove>
504 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
505                        GenFuncBase<Derived> {
506   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
507 
508   void flushTrivialFields(std::array<Address, 2> Addrs) {
509     CharUnits Size = this->End - this->Start;
510 
511     if (Size.getQuantity() == 0)
512       return;
513 
514     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
515     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
516 
517     // Emit memcpy.
518     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
519       llvm::Value *SizeVal =
520           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
521       DstAddr =
522           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
523       SrcAddr =
524           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
525       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
526     } else {
527       llvm::Type *Ty = llvm::Type::getIntNTy(
528           this->CGF->getLLVMContext(),
529           Size.getQuantity() * this->CGF->getContext().getCharWidth());
530       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
531       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
532       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
533       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
534     }
535 
536     this->Start = this->End = CharUnits::Zero();
537   }
538 
539   template <class... Ts>
540   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
541                             std::array<Address, 2> Addrs) {
542     LValue DstLV, SrcLV;
543     if (FD) {
544       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
545       llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
546       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
547       LValue DstBase = this->CGF->MakeAddrLValue(
548           this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
549       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
550       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
551       LValue SrcBase = this->CGF->MakeAddrLValue(
552           this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
553       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
554     } else {
555       llvm::PointerType *Ty = this->CGF->ConvertType(FT)->getPointerTo();
556       Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
557       Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
558       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
559       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
560     }
561     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
562     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
563   }
564 };
565 
566 // These classes that emit the special functions for a non-trivial struct.
567 struct GenDestructor : StructVisitor<GenDestructor>,
568                        GenFuncBase<GenDestructor>,
569                        DestructedTypeVisitor<GenDestructor> {
570   using Super = DestructedTypeVisitor<GenDestructor>;
571   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
572 
573   void visitWithKind(QualType::DestructionKind DK, QualType FT,
574                      const FieldDecl *FD, CharUnits CurStructOffset,
575                      std::array<Address, 1> Addrs) {
576     if (const auto *AT = getContext().getAsArrayType(FT)) {
577       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset, Addrs);
578       return;
579     }
580 
581     Super::visitWithKind(DK, FT, FD, CurStructOffset, Addrs);
582   }
583 
584   void visitARCStrong(QualType QT, const FieldDecl *FD,
585                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
586     CGF->destroyARCStrongImprecise(
587         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
588   }
589 
590   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
591                     std::array<Address, 1> Addrs) {
592     CGF->destroyARCWeak(
593         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
594   }
595 
596   void callSpecialFunction(QualType FT, CharUnits Offset,
597                            std::array<Address, 1> Addrs) {
598     CGF->callCStructDestructor(
599         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
600   }
601 };
602 
603 struct GenDefaultInitialize
604     : StructVisitor<GenDefaultInitialize>,
605       GenFuncBase<GenDefaultInitialize>,
606       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
607   using Super = DefaultInitializedTypeVisitor<GenDefaultInitialize>;
608   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
609 
610   GenDefaultInitialize(ASTContext &Ctx)
611       : StructVisitor<GenDefaultInitialize>(Ctx) {}
612 
613   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
614                      const FieldDecl *FD, CharUnits CurStructOffset,
615                      std::array<Address, 1> Addrs) {
616     if (const auto *AT = getContext().getAsArrayType(FT)) {
617       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset,
618                  Addrs);
619       return;
620     }
621 
622     Super::visitWithKind(PDIK, FT, FD, CurStructOffset, Addrs);
623   }
624 
625   void visitARCStrong(QualType QT, const FieldDecl *FD,
626                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
627     CGF->EmitNullInitialization(
628         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
629   }
630 
631   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
632                     std::array<Address, 1> Addrs) {
633     CGF->EmitNullInitialization(
634         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
635   }
636 
637   template <class FieldKind, size_t... Is>
638   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
639                   const FieldDecl *FD, CharUnits CurStackOffset,
640                   std::array<Address, 1> Addrs) {
641     if (!FK)
642       return visitTrivial(QualType(AT, 0), FD, CurStackOffset, Addrs);
643 
644     ASTContext &Ctx = getContext();
645     CharUnits Size = Ctx.getTypeSizeInChars(QualType(AT, 0));
646     QualType EltTy = Ctx.getBaseElementType(QualType(AT, 0));
647 
648     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
649       GenFuncBaseTy::visitArray(FK, AT, IsVolatile, FD, CurStackOffset, Addrs);
650       return;
651     }
652 
653     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
654     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
655     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
656     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
657                               IsVolatile);
658   }
659 
660   void callSpecialFunction(QualType FT, CharUnits Offset,
661                            std::array<Address, 1> Addrs) {
662     CGF->callCStructDefaultConstructor(
663         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
664   }
665 };
666 
667 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
668   GenCopyConstructor(ASTContext &Ctx)
669       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
670 
671   void visitARCStrong(QualType QT, const FieldDecl *FD,
672                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
673     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
674     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
675     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
676         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
677     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
678     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
679   }
680 
681   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
682                     std::array<Address, 2> Addrs) {
683     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
684     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
685     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
686   }
687 
688   void callSpecialFunction(QualType FT, CharUnits Offset,
689                            std::array<Address, 2> Addrs) {
690     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
691                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
692   }
693 };
694 
695 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
696   GenMoveConstructor(ASTContext &Ctx)
697       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
698 
699   void visitARCStrong(QualType QT, const FieldDecl *FD,
700                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
701     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
702     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
703     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
704     llvm::Value *SrcVal =
705         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
706     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
707     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
708                            /* isInitialization */ true);
709   }
710 
711   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
712                     std::array<Address, 2> Addrs) {
713     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
714     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
715     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
716   }
717 
718   void callSpecialFunction(QualType FT, CharUnits Offset,
719                            std::array<Address, 2> Addrs) {
720     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
721                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
722   }
723 };
724 
725 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
726   GenCopyAssignment(ASTContext &Ctx)
727       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
728 
729   void visitARCStrong(QualType QT, const FieldDecl *FD,
730                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
731     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
732     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
733     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
734         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
735     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
736                             false);
737   }
738 
739   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
740                     std::array<Address, 2> Addrs) {
741     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
742     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
743     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
744   }
745 
746   void callSpecialFunction(QualType FT, CharUnits Offset,
747                            std::array<Address, 2> Addrs) {
748     CGF->callCStructCopyAssignmentOperator(
749         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
750         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
751   }
752 };
753 
754 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
755   GenMoveAssignment(ASTContext &Ctx)
756       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
757 
758   void visitARCStrong(QualType QT, const FieldDecl *FD,
759                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
760     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
761     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
762     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
763     llvm::Value *SrcVal =
764         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
765     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
766     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
767     llvm::Value *DstVal =
768         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
769     CGF->EmitStoreOfScalar(SrcVal, DstLV);
770     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
771   }
772 
773   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
774                     std::array<Address, 2> Addrs) {
775     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
776     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
777     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
778   }
779 
780   void callSpecialFunction(QualType FT, CharUnits Offset,
781                            std::array<Address, 2> Addrs) {
782     CGF->callCStructMoveAssignmentOperator(
783         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
784         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
785   }
786 };
787 
788 } // namespace
789 
790 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
791                                                Address Addr, QualType Type) {
792   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
793 }
794 
795 // Default-initialize a variable that is a non-trivial struct or an array of
796 // such structure.
797 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
798   GenDefaultInitialize Gen(getContext());
799   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
800   Gen.setCGF(this);
801   QualType QT = Dst.getType();
802   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
803   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
804 }
805 
806 template <class G, size_t N>
807 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
808                                 bool IsVolatile, CodeGenFunction &CGF,
809                                 std::array<Address, N> Addrs) {
810   for (unsigned I = 0; I < N; ++I)
811     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
812   QT = IsVolatile ? QT.withVolatile() : QT;
813   Gen.callFunc(FuncName, QT, Addrs, CGF);
814 }
815 
816 // Functions to emit calls to the special functions of a non-trivial C struct.
817 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
818   bool IsVolatile = Dst.isVolatile();
819   Address DstPtr = Dst.getAddress();
820   QualType QT = Dst.getType();
821   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
822   std::string FuncName = GenName.getName(QT, IsVolatile);
823   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
824                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
825 }
826 
827 void CodeGenFunction::callCStructDestructor(LValue Dst) {
828   bool IsVolatile = Dst.isVolatile();
829   Address DstPtr = Dst.getAddress();
830   QualType QT = Dst.getType();
831   GenDestructorFuncName GenName(DstPtr.getAlignment(), getContext());
832   std::string FuncName = GenName.getName(QT, IsVolatile);
833   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
834                       *this, std::array<Address, 1>({{DstPtr}}));
835 }
836 
837 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
838   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
839   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
840   QualType QT = Dst.getType();
841   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
842                                    SrcPtr.getAlignment(), getContext());
843   std::string FuncName = GenName.getName(QT, IsVolatile);
844   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
845                       IsVolatile, *this,
846                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
847 }
848 
849 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
850 
851 ) {
852   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
853   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
854   QualType QT = Dst.getType();
855   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
856                                    SrcPtr.getAlignment(), getContext());
857   std::string FuncName = GenName.getName(QT, IsVolatile);
858   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
859                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
860 }
861 
862 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
863   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
864   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
865   QualType QT = Dst.getType();
866   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
867                                   SrcPtr.getAlignment(), getContext());
868   std::string FuncName = GenName.getName(QT, IsVolatile);
869   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
870                       IsVolatile, *this,
871                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
872 }
873 
874 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
875 
876 ) {
877   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
878   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
879   QualType QT = Dst.getType();
880   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
881                                   SrcPtr.getAlignment(), getContext());
882   std::string FuncName = GenName.getName(QT, IsVolatile);
883   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
884                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
885 }
886