1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines functions to generate various special functions for C
11 // structs.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "clang/AST/NonTrivialTypeVisitor.h"
18 #include "llvm/Support/ScopedPrinter.h"
19 #include <array>
20 
21 using namespace clang;
22 using namespace CodeGen;
23 
24 // Return the size of a field in number of bits.
25 static uint64_t getFieldSize(const FieldDecl *FD, QualType FT,
26                              ASTContext &Ctx) {
27   if (FD && FD->isBitField())
28     return FD->getBitWidthValue(Ctx);
29   return Ctx.getTypeSize(FT);
30 }
31 
32 namespace {
33 enum { DstIdx = 0, SrcIdx = 1 };
34 const char *ValNameStr[2] = {"dst", "src"};
35 
36 template <class Derived> struct StructVisitor {
37   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
38 
39   template <class... Ts>
40   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
41     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
42 
43     // Iterate over the fields of the struct.
44     for (const FieldDecl *FD : RD->fields()) {
45       QualType FT = FD->getType();
46       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
47       asDerived().visit(FT, FD, CurStructOffset, Args...);
48     }
49 
50     asDerived().flushTrivialFields(Args...);
51   }
52 
53   template <class... Ts> void visitTrivial(Ts... Args) {}
54 
55   template <class... Ts> void visitCXXDestructor(Ts... Args) {
56     llvm_unreachable("field of a C++ struct type is not expected");
57   }
58 
59   template <class... Ts> void flushTrivialFields(Ts... Args) {}
60 
61   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
62     return FD ? Ctx.getASTRecordLayout(FD->getParent())
63                     .getFieldOffset(FD->getFieldIndex())
64               : 0;
65   }
66 
67   CharUnits getFieldOffset(const FieldDecl *FD) {
68     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
69   }
70 
71   Derived &asDerived() { return static_cast<Derived &>(*this); }
72 
73   ASTContext &getContext() { return Ctx; }
74   ASTContext &Ctx;
75 };
76 
77 template <class Derived, bool IsMove>
78 struct CopyStructVisitor : StructVisitor<Derived>,
79                            CopiedTypeVisitor<Derived, IsMove> {
80   using StructVisitor<Derived>::asDerived;
81   using Super = CopiedTypeVisitor<Derived, IsMove>;
82 
83   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
84 
85   template <class... Ts>
86   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
87                 const FieldDecl *FD, CharUnits CurStructOffsset,
88                 Ts &&... Args) {
89     if (PCK)
90       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
91   }
92 
93   template <class... Ts>
94   void visitWithKind(QualType::PrimitiveCopyKind PCK, QualType FT,
95                      const FieldDecl *FD, CharUnits CurStructOffsset,
96                      Ts &&... Args) {
97     if (const auto *AT = asDerived().getContext().getAsArrayType(FT)) {
98       asDerived().visitArray(PCK, AT, FT.isVolatileQualified(), FD,
99                              CurStructOffsset, std::forward<Ts>(Args)...);
100       return;
101     }
102 
103     Super::visitWithKind(PCK, FT, FD, CurStructOffsset,
104                          std::forward<Ts>(Args)...);
105   }
106 
107   template <class... Ts>
108   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
109                     Ts... Args) {
110     assert(!FT.isVolatileQualified() && "volatile field not expected");
111     ASTContext &Ctx = asDerived().getContext();
112     uint64_t FieldSize = getFieldSize(FD, FT, Ctx);
113 
114     // Ignore zero-sized fields.
115     if (FieldSize == 0)
116       return;
117 
118     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
119     uint64_t FEndInBits = FStartInBits + FieldSize;
120     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
121 
122     // Set Start if this is the first field of a sequence of trivial fields.
123     if (Start == End)
124       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
125     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
126   }
127 
128   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
129 };
130 
131 // This function creates the mangled name of a special function of a non-trivial
132 // C struct. Since there is no ODR in C, the function is mangled based on the
133 // struct contents and not the name. The mangled name has the following
134 // structure:
135 //
136 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
137 // <prefix> ::= "__destructor_" | "__default_constructor_" |
138 //              "__copy_constructor_" | "__move_constructor_" |
139 //              "__copy_assignment_" | "__move_assignment_"
140 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
141 // <struct-field-info> ::= <field-info>+
142 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
143 // <struct-or-scalar-field-info> ::= <struct-field-info> | <strong-field-info> |
144 //                                   <trivial-field-info>
145 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
146 //                        <num-elements> <innermost-element-info> "_AE"
147 // <innermost-element-info> ::= <struct-or-scalar-field-info>
148 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
149 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
150 
151 template <class Derived> struct GenFuncNameBase {
152   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
153     std::string S;
154     if (IsVolatile)
155       S = "v";
156     S += llvm::to_string(Offset.getQuantity());
157     return S;
158   }
159 
160   void visitARCStrong(QualType FT, const FieldDecl *FD,
161                       CharUnits CurStructOffset) {
162     appendStr("_s");
163     if (FT->isBlockPointerType())
164       appendStr("b");
165     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
166     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
167   }
168 
169   void visitARCWeak(QualType FT, const FieldDecl *FD,
170                     CharUnits CurStructOffset) {
171     appendStr("_w");
172     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
173     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
174   }
175 
176   void visitStruct(QualType QT, const FieldDecl *FD,
177                    CharUnits CurStructOffset) {
178     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
179     asDerived().visitStructFields(QT, FieldOffset);
180   }
181 
182   template <class FieldKind>
183   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
184                   const FieldDecl *FD, CharUnits CurStructOffset) {
185     // String for non-volatile trivial fields is emitted when
186     // flushTrivialFields is called.
187     if (!FK)
188       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStructOffset);
189 
190     asDerived().flushTrivialFields();
191     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
192     ASTContext &Ctx = asDerived().getContext();
193     const ConstantArrayType *CAT = cast<ConstantArrayType>(AT);
194     unsigned NumElts = Ctx.getConstantArrayElementCount(CAT);
195     QualType EltTy = Ctx.getBaseElementType(CAT);
196     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
197     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
198               llvm::to_string(EltSize.getQuantity()) + "n" +
199               llvm::to_string(NumElts));
200     EltTy = IsVolatile ? EltTy.withVolatile() : EltTy;
201     asDerived().visitWithKind(FK, EltTy, nullptr, FieldOffset);
202     appendStr("_AE");
203   }
204 
205   void appendStr(StringRef Str) { Name += Str; }
206 
207   std::string getName(QualType QT, bool IsVolatile) {
208     QT = IsVolatile ? QT.withVolatile() : QT;
209     asDerived().visitStructFields(QT, CharUnits::Zero());
210     return Name;
211   }
212 
213   Derived &asDerived() { return static_cast<Derived &>(*this); }
214 
215   std::string Name;
216 };
217 
218 template <class Derived>
219 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
220   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
221       : StructVisitor<Derived>(Ctx) {
222     this->appendStr(Prefix);
223     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
224   }
225 };
226 
227 // Helper function to create a null constant.
228 static llvm::Constant *getNullForVariable(Address Addr) {
229   llvm::Type *Ty = Addr.getElementType();
230   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
231 }
232 
233 template <bool IsMove>
234 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
235                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
236 
237   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
238                     CharUnits SrcAlignment, ASTContext &Ctx)
239       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
240     this->appendStr(Prefix);
241     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
242     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
243   }
244 
245   void flushTrivialFields() {
246     if (this->Start == this->End)
247       return;
248 
249     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
250                     llvm::to_string((this->End - this->Start).getQuantity()));
251 
252     this->Start = this->End = CharUnits::Zero();
253   }
254 
255   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
256                             CharUnits CurStackOffset) {
257     // Because volatile fields can be bit-fields and are individually copied,
258     // their offset and width are in bits.
259     uint64_t OffsetInBits =
260         this->Ctx.toBits(CurStackOffset) + this->getFieldOffsetInBits(FD);
261     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
262                     llvm::to_string(getFieldSize(FD, FT, this->Ctx)));
263   }
264 };
265 
266 struct GenDefaultInitializeFuncName
267     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
268       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
269   using Super = DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName>;
270   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
271       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
272                                                        DstAlignment, Ctx) {}
273   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
274                      const FieldDecl *FD, CharUnits CurStructOffset) {
275     if (const auto *AT = getContext().getAsArrayType(FT)) {
276       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
277       return;
278     }
279 
280     Super::visitWithKind(PDIK, FT, FD, CurStructOffset);
281   }
282 };
283 
284 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
285                                DestructedTypeVisitor<GenDestructorFuncName> {
286   using Super = DestructedTypeVisitor<GenDestructorFuncName>;
287   GenDestructorFuncName(const char *Prefix, CharUnits DstAlignment,
288                         ASTContext &Ctx)
289       : GenUnaryFuncName<GenDestructorFuncName>(Prefix, DstAlignment,
290                                                 Ctx) {}
291   void visitWithKind(QualType::DestructionKind DK, QualType FT,
292                      const FieldDecl *FD, CharUnits CurStructOffset) {
293     if (const auto *AT = getContext().getAsArrayType(FT)) {
294       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
295       return;
296     }
297 
298     Super::visitWithKind(DK, FT, FD, CurStructOffset);
299   }
300 };
301 
302 // Helper function that creates CGFunctionInfo for an N-ary special function.
303 template <size_t N>
304 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
305                                              FunctionArgList &Args) {
306   ASTContext &Ctx = CGM.getContext();
307   llvm::SmallVector<ImplicitParamDecl *, N> Params;
308   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
309 
310   for (unsigned I = 0; I < N; ++I)
311     Params.push_back(ImplicitParamDecl::Create(
312         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
313         ImplicitParamDecl::Other));
314 
315   for (auto &P : Params)
316     Args.push_back(P);
317 
318   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
319 }
320 
321 // Template classes that are used as bases for classes that emit special
322 // functions.
323 template <class Derived> struct GenFuncBase {
324   template <size_t N>
325   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStackOffset,
326                    std::array<Address, N> Addrs) {
327     this->asDerived().callSpecialFunction(
328         FT, CurStackOffset + asDerived().getFieldOffset(FD), Addrs);
329   }
330 
331   template <class FieldKind, size_t N>
332   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
333                   const FieldDecl *FD, CharUnits CurStackOffset,
334                   std::array<Address, N> Addrs) {
335     // Non-volatile trivial fields are copied when flushTrivialFields is called.
336     if (!FK)
337       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStackOffset,
338                                       Addrs);
339 
340     asDerived().flushTrivialFields(Addrs);
341     CodeGenFunction &CGF = *this->CGF;
342     ASTContext &Ctx = CGF.getContext();
343 
344     // Compute the end address.
345     QualType BaseEltQT;
346     std::array<Address, N> StartAddrs = Addrs;
347     for (unsigned I = 0; I < N; ++I)
348       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStackOffset, FD);
349     Address DstAddr = StartAddrs[DstIdx];
350     llvm::Value *NumElts = CGF.emitArrayLength(AT, BaseEltQT, DstAddr);
351     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
352     llvm::Value *BaseEltSizeVal =
353         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
354     llvm::Value *SizeInBytes =
355         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
356     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
357     llvm::Value *DstArrayEnd =
358         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
359     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
360                                             "dstarray.end");
361     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
362 
363     // Create the header block and insert the phi instructions.
364     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
365     CGF.EmitBlock(HeaderBB);
366     llvm::PHINode *PHIs[N];
367 
368     for (unsigned I = 0; I < N; ++I) {
369       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
370       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
371     }
372 
373     // Create the exit and loop body blocks.
374     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
375     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
376 
377     // Emit the comparison and conditional branch instruction that jumps to
378     // either the exit or the loop body.
379     llvm::Value *Done =
380         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
381     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
382 
383     // Visit the element of the array in the loop body.
384     CGF.EmitBlock(LoopBB);
385     QualType EltQT = AT->getElementType();
386     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
387     std::array<Address, N> NewAddrs = Addrs;
388 
389     for (unsigned I = 0; I < N; ++I)
390       NewAddrs[I] = Address(
391           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
392 
393     EltQT = IsVolatile ? EltQT.withVolatile() : EltQT;
394     this->asDerived().visitWithKind(FK, EltQT, nullptr, CharUnits::Zero(),
395                                     NewAddrs);
396 
397     LoopBB = CGF.Builder.GetInsertBlock();
398 
399     for (unsigned I = 0; I < N; ++I) {
400       // Instrs to update the destination and source addresses.
401       // Update phi instructions.
402       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
403       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
404     }
405 
406     // Insert an unconditional branch to the header block.
407     CGF.Builder.CreateBr(HeaderBB);
408     CGF.EmitBlock(ExitBB);
409   }
410 
411   /// Return an address with the specified offset from the passed address.
412   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
413     assert(Addr.isValid() && "invalid address");
414     if (Offset.getQuantity() == 0)
415       return Addr;
416     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
417     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity(),
418                                                CharUnits::One());
419     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
420   }
421 
422   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
423                             const FieldDecl *FD) {
424     return getAddrWithOffset(Addr, StructFieldOffset +
425                                        asDerived().getFieldOffset(FD));
426   }
427 
428   template <size_t N>
429   llvm::Function *
430   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
431               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
432     // If the special function already exists in the module, return it.
433     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
434       bool WrongType = false;
435       if (!F->getReturnType()->isVoidTy())
436         WrongType = true;
437       else {
438         for (const llvm::Argument &Arg : F->args())
439           if (Arg.getType() != CGM.Int8PtrPtrTy)
440             WrongType = true;
441       }
442 
443       if (WrongType) {
444         std::string FuncName = F->getName();
445         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
446         CGM.Error(Loc, "special function " + FuncName +
447                            " for non-trivial C struct has incorrect type");
448         return nullptr;
449       }
450       return F;
451     }
452 
453     ASTContext &Ctx = CGM.getContext();
454     FunctionArgList Args;
455     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
456     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
457     llvm::Function *F =
458         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
459                                FuncName, &CGM.getModule());
460     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
461     CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F);
462     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
463     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
464     FunctionDecl *FD = FunctionDecl::Create(
465         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
466         II, Ctx.getFunctionType(Ctx.VoidTy, llvm::None, {}), nullptr,
467         SC_PrivateExtern, false, false);
468     CodeGenFunction NewCGF(CGM);
469     setCGF(&NewCGF);
470     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
471 
472     for (unsigned I = 0; I < N; ++I) {
473       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
474       Addrs[I] = Address(V, Alignments[I]);
475     }
476 
477     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
478     CGF->FinishFunction();
479     return F;
480   }
481 
482   template <size_t N>
483   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
484                 CodeGenFunction &CallerCGF) {
485     std::array<CharUnits, N> Alignments;
486     llvm::Value *Ptrs[N];
487 
488     for (unsigned I = 0; I < N; ++I) {
489       Alignments[I] = Addrs[I].getAlignment();
490       Ptrs[I] =
491           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
492               .getPointer();
493     }
494 
495     if (llvm::Function *F =
496             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
497       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
498   }
499 
500   Derived &asDerived() { return static_cast<Derived &>(*this); }
501 
502   void setCGF(CodeGenFunction *F) { CGF = F; }
503 
504   CodeGenFunction *CGF = nullptr;
505 };
506 
507 template <class Derived, bool IsMove>
508 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
509                        GenFuncBase<Derived> {
510   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
511 
512   void flushTrivialFields(std::array<Address, 2> Addrs) {
513     CharUnits Size = this->End - this->Start;
514 
515     if (Size.getQuantity() == 0)
516       return;
517 
518     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
519     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
520 
521     // Emit memcpy.
522     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
523       llvm::Value *SizeVal =
524           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
525       DstAddr =
526           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
527       SrcAddr =
528           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
529       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
530     } else {
531       llvm::Type *Ty = llvm::Type::getIntNTy(
532           this->CGF->getLLVMContext(),
533           Size.getQuantity() * this->CGF->getContext().getCharWidth());
534       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
535       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
536       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
537       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
538     }
539 
540     this->Start = this->End = CharUnits::Zero();
541   }
542 
543   template <class... Ts>
544   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
545                             std::array<Address, 2> Addrs) {
546     LValue DstLV, SrcLV;
547     if (FD) {
548       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
549       llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
550       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
551       LValue DstBase = this->CGF->MakeAddrLValue(
552           this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
553       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
554       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
555       LValue SrcBase = this->CGF->MakeAddrLValue(
556           this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
557       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
558     } else {
559       llvm::PointerType *Ty = this->CGF->ConvertType(FT)->getPointerTo();
560       Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
561       Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
562       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
563       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
564     }
565     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
566     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
567   }
568 };
569 
570 // These classes that emit the special functions for a non-trivial struct.
571 struct GenDestructor : StructVisitor<GenDestructor>,
572                        GenFuncBase<GenDestructor>,
573                        DestructedTypeVisitor<GenDestructor> {
574   using Super = DestructedTypeVisitor<GenDestructor>;
575   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
576 
577   void visitWithKind(QualType::DestructionKind DK, QualType FT,
578                      const FieldDecl *FD, CharUnits CurStructOffset,
579                      std::array<Address, 1> Addrs) {
580     if (const auto *AT = getContext().getAsArrayType(FT)) {
581       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset, Addrs);
582       return;
583     }
584 
585     Super::visitWithKind(DK, FT, FD, CurStructOffset, Addrs);
586   }
587 
588   void visitARCStrong(QualType QT, const FieldDecl *FD,
589                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
590     CGF->destroyARCStrongImprecise(
591         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
592   }
593 
594   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
595                     std::array<Address, 1> Addrs) {
596     CGF->destroyARCWeak(
597         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
598   }
599 
600   void callSpecialFunction(QualType FT, CharUnits Offset,
601                            std::array<Address, 1> Addrs) {
602     CGF->callCStructDestructor(
603         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
604   }
605 };
606 
607 struct GenDefaultInitialize
608     : StructVisitor<GenDefaultInitialize>,
609       GenFuncBase<GenDefaultInitialize>,
610       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
611   using Super = DefaultInitializedTypeVisitor<GenDefaultInitialize>;
612   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
613 
614   GenDefaultInitialize(ASTContext &Ctx)
615       : StructVisitor<GenDefaultInitialize>(Ctx) {}
616 
617   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
618                      const FieldDecl *FD, CharUnits CurStructOffset,
619                      std::array<Address, 1> Addrs) {
620     if (const auto *AT = getContext().getAsArrayType(FT)) {
621       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset,
622                  Addrs);
623       return;
624     }
625 
626     Super::visitWithKind(PDIK, FT, FD, CurStructOffset, Addrs);
627   }
628 
629   void visitARCStrong(QualType QT, const FieldDecl *FD,
630                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
631     CGF->EmitNullInitialization(
632         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
633   }
634 
635   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
636                     std::array<Address, 1> Addrs) {
637     CGF->EmitNullInitialization(
638         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
639   }
640 
641   template <class FieldKind, size_t... Is>
642   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
643                   const FieldDecl *FD, CharUnits CurStackOffset,
644                   std::array<Address, 1> Addrs) {
645     if (!FK)
646       return visitTrivial(QualType(AT, 0), FD, CurStackOffset, Addrs);
647 
648     ASTContext &Ctx = getContext();
649     CharUnits Size = Ctx.getTypeSizeInChars(QualType(AT, 0));
650     QualType EltTy = Ctx.getBaseElementType(QualType(AT, 0));
651 
652     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
653       GenFuncBaseTy::visitArray(FK, AT, IsVolatile, FD, CurStackOffset, Addrs);
654       return;
655     }
656 
657     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
658     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
659     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
660     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
661                               IsVolatile);
662   }
663 
664   void callSpecialFunction(QualType FT, CharUnits Offset,
665                            std::array<Address, 1> Addrs) {
666     CGF->callCStructDefaultConstructor(
667         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
668   }
669 };
670 
671 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
672   GenCopyConstructor(ASTContext &Ctx)
673       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
674 
675   void visitARCStrong(QualType QT, const FieldDecl *FD,
676                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
677     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
678     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
679     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
680         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
681     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
682     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
683   }
684 
685   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
686                     std::array<Address, 2> Addrs) {
687     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
688     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
689     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
690   }
691 
692   void callSpecialFunction(QualType FT, CharUnits Offset,
693                            std::array<Address, 2> Addrs) {
694     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
695                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
696   }
697 };
698 
699 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
700   GenMoveConstructor(ASTContext &Ctx)
701       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
702 
703   void visitARCStrong(QualType QT, const FieldDecl *FD,
704                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
705     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
706     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
707     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
708     llvm::Value *SrcVal =
709         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
710     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
711     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
712                            /* isInitialization */ true);
713   }
714 
715   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
716                     std::array<Address, 2> Addrs) {
717     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
718     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
719     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
720   }
721 
722   void callSpecialFunction(QualType FT, CharUnits Offset,
723                            std::array<Address, 2> Addrs) {
724     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
725                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
726   }
727 };
728 
729 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
730   GenCopyAssignment(ASTContext &Ctx)
731       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
732 
733   void visitARCStrong(QualType QT, const FieldDecl *FD,
734                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
735     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
736     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
737     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
738         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
739     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
740                             false);
741   }
742 
743   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
744                     std::array<Address, 2> Addrs) {
745     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
746     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
747     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
748   }
749 
750   void callSpecialFunction(QualType FT, CharUnits Offset,
751                            std::array<Address, 2> Addrs) {
752     CGF->callCStructCopyAssignmentOperator(
753         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
754         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
755   }
756 };
757 
758 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
759   GenMoveAssignment(ASTContext &Ctx)
760       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
761 
762   void visitARCStrong(QualType QT, const FieldDecl *FD,
763                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
764     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
765     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
766     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
767     llvm::Value *SrcVal =
768         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
769     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
770     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
771     llvm::Value *DstVal =
772         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
773     CGF->EmitStoreOfScalar(SrcVal, DstLV);
774     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
775   }
776 
777   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
778                     std::array<Address, 2> Addrs) {
779     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
780     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
781     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
782   }
783 
784   void callSpecialFunction(QualType FT, CharUnits Offset,
785                            std::array<Address, 2> Addrs) {
786     CGF->callCStructMoveAssignmentOperator(
787         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
788         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
789   }
790 };
791 
792 } // namespace
793 
794 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
795                                                Address Addr, QualType Type) {
796   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
797 }
798 
799 // Default-initialize a variable that is a non-trivial struct or an array of
800 // such structure.
801 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
802   GenDefaultInitialize Gen(getContext());
803   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
804   Gen.setCGF(this);
805   QualType QT = Dst.getType();
806   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
807   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
808 }
809 
810 template <class G, size_t N>
811 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
812                                 bool IsVolatile, CodeGenFunction &CGF,
813                                 std::array<Address, N> Addrs) {
814   for (unsigned I = 0; I < N; ++I)
815     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
816   QT = IsVolatile ? QT.withVolatile() : QT;
817   Gen.callFunc(FuncName, QT, Addrs, CGF);
818 }
819 
820 // Functions to emit calls to the special functions of a non-trivial C struct.
821 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
822   bool IsVolatile = Dst.isVolatile();
823   Address DstPtr = Dst.getAddress();
824   QualType QT = Dst.getType();
825   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
826   std::string FuncName = GenName.getName(QT, IsVolatile);
827   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
828                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
829 }
830 
831 std::string
832 CodeGenFunction::getNonTrivialCopyConstructorStr(QualType QT,
833                                                  CharUnits Alignment,
834                                                  bool IsVolatile,
835                                                  ASTContext &Ctx) {
836   GenBinaryFuncName<false> GenName("", Alignment, Alignment, Ctx);
837   return GenName.getName(QT, IsVolatile);
838 }
839 
840 std::string
841 CodeGenFunction::getNonTrivialDestructorStr(QualType QT, CharUnits Alignment,
842                                             bool IsVolatile, ASTContext &Ctx) {
843   GenDestructorFuncName GenName("", Alignment, Ctx);
844   return GenName.getName(QT, IsVolatile);
845 }
846 
847 void CodeGenFunction::callCStructDestructor(LValue Dst) {
848   bool IsVolatile = Dst.isVolatile();
849   Address DstPtr = Dst.getAddress();
850   QualType QT = Dst.getType();
851   GenDestructorFuncName GenName("__destructor_", DstPtr.getAlignment(),
852                                 getContext());
853   std::string FuncName = GenName.getName(QT, IsVolatile);
854   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
855                       *this, std::array<Address, 1>({{DstPtr}}));
856 }
857 
858 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
859   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
860   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
861   QualType QT = Dst.getType();
862   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
863                                    SrcPtr.getAlignment(), getContext());
864   std::string FuncName = GenName.getName(QT, IsVolatile);
865   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
866                       IsVolatile, *this,
867                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
868 }
869 
870 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
871 
872 ) {
873   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
874   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
875   QualType QT = Dst.getType();
876   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
877                                    SrcPtr.getAlignment(), getContext());
878   std::string FuncName = GenName.getName(QT, IsVolatile);
879   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
880                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
881 }
882 
883 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
884   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
885   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
886   QualType QT = Dst.getType();
887   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
888                                   SrcPtr.getAlignment(), getContext());
889   std::string FuncName = GenName.getName(QT, IsVolatile);
890   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
891                       IsVolatile, *this,
892                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
893 }
894 
895 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
896 
897 ) {
898   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
899   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
900   QualType QT = Dst.getType();
901   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
902                                   SrcPtr.getAlignment(), getContext());
903   std::string FuncName = GenName.getName(QT, IsVolatile);
904   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
905                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
906 }
907