1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines functions to generate various special functions for C
10 // structs.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CodeGenModule.h"
16 #include "clang/AST/NonTrivialTypeVisitor.h"
17 #include "llvm/Support/ScopedPrinter.h"
18 #include <array>
19 
20 using namespace clang;
21 using namespace CodeGen;
22 
23 // Return the size of a field in number of bits.
24 static uint64_t getFieldSize(const FieldDecl *FD, QualType FT,
25                              ASTContext &Ctx) {
26   if (FD && FD->isBitField())
27     return FD->getBitWidthValue(Ctx);
28   return Ctx.getTypeSize(FT);
29 }
30 
31 namespace {
32 enum { DstIdx = 0, SrcIdx = 1 };
33 const char *ValNameStr[2] = {"dst", "src"};
34 
35 template <class Derived> struct StructVisitor {
36   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
37 
38   template <class... Ts>
39   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
40     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
41 
42     // Iterate over the fields of the struct.
43     for (const FieldDecl *FD : RD->fields()) {
44       QualType FT = FD->getType();
45       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
46       asDerived().visit(FT, FD, CurStructOffset, Args...);
47     }
48 
49     asDerived().flushTrivialFields(Args...);
50   }
51 
52   template <class... Ts> void visitTrivial(Ts... Args) {}
53 
54   template <class... Ts> void visitCXXDestructor(Ts... Args) {
55     llvm_unreachable("field of a C++ struct type is not expected");
56   }
57 
58   template <class... Ts> void flushTrivialFields(Ts... Args) {}
59 
60   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
61     return FD ? Ctx.getASTRecordLayout(FD->getParent())
62                     .getFieldOffset(FD->getFieldIndex())
63               : 0;
64   }
65 
66   CharUnits getFieldOffset(const FieldDecl *FD) {
67     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
68   }
69 
70   Derived &asDerived() { return static_cast<Derived &>(*this); }
71 
72   ASTContext &getContext() { return Ctx; }
73   ASTContext &Ctx;
74 };
75 
76 template <class Derived, bool IsMove>
77 struct CopyStructVisitor : StructVisitor<Derived>,
78                            CopiedTypeVisitor<Derived, IsMove> {
79   using StructVisitor<Derived>::asDerived;
80   using Super = CopiedTypeVisitor<Derived, IsMove>;
81 
82   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
83 
84   template <class... Ts>
85   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
86                 const FieldDecl *FD, CharUnits CurStructOffsset,
87                 Ts &&... Args) {
88     if (PCK)
89       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
90   }
91 
92   template <class... Ts>
93   void visitWithKind(QualType::PrimitiveCopyKind PCK, QualType FT,
94                      const FieldDecl *FD, CharUnits CurStructOffsset,
95                      Ts &&... Args) {
96     if (const auto *AT = asDerived().getContext().getAsArrayType(FT)) {
97       asDerived().visitArray(PCK, AT, FT.isVolatileQualified(), FD,
98                              CurStructOffsset, std::forward<Ts>(Args)...);
99       return;
100     }
101 
102     Super::visitWithKind(PCK, FT, FD, CurStructOffsset,
103                          std::forward<Ts>(Args)...);
104   }
105 
106   template <class... Ts>
107   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
108                     Ts... Args) {
109     assert(!FT.isVolatileQualified() && "volatile field not expected");
110     ASTContext &Ctx = asDerived().getContext();
111     uint64_t FieldSize = getFieldSize(FD, FT, Ctx);
112 
113     // Ignore zero-sized fields.
114     if (FieldSize == 0)
115       return;
116 
117     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
118     uint64_t FEndInBits = FStartInBits + FieldSize;
119     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
120 
121     // Set Start if this is the first field of a sequence of trivial fields.
122     if (Start == End)
123       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
124     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
125   }
126 
127   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
128 };
129 
130 // This function creates the mangled name of a special function of a non-trivial
131 // C struct. Since there is no ODR in C, the function is mangled based on the
132 // struct contents and not the name. The mangled name has the following
133 // structure:
134 //
135 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
136 // <prefix> ::= "__destructor_" | "__default_constructor_" |
137 //              "__copy_constructor_" | "__move_constructor_" |
138 //              "__copy_assignment_" | "__move_assignment_"
139 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
140 // <struct-field-info> ::= <field-info>+
141 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
142 // <struct-or-scalar-field-info> ::= <struct-field-info> | <strong-field-info> |
143 //                                   <trivial-field-info>
144 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
145 //                        <num-elements> <innermost-element-info> "_AE"
146 // <innermost-element-info> ::= <struct-or-scalar-field-info>
147 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
148 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
149 
150 template <class Derived> struct GenFuncNameBase {
151   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
152     std::string S;
153     if (IsVolatile)
154       S = "v";
155     S += llvm::to_string(Offset.getQuantity());
156     return S;
157   }
158 
159   void visitARCStrong(QualType FT, const FieldDecl *FD,
160                       CharUnits CurStructOffset) {
161     appendStr("_s");
162     if (FT->isBlockPointerType())
163       appendStr("b");
164     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
165     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
166   }
167 
168   void visitARCWeak(QualType FT, const FieldDecl *FD,
169                     CharUnits CurStructOffset) {
170     appendStr("_w");
171     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
172     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
173   }
174 
175   void visitStruct(QualType QT, const FieldDecl *FD,
176                    CharUnits CurStructOffset) {
177     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
178     asDerived().visitStructFields(QT, FieldOffset);
179   }
180 
181   template <class FieldKind>
182   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
183                   const FieldDecl *FD, CharUnits CurStructOffset) {
184     // String for non-volatile trivial fields is emitted when
185     // flushTrivialFields is called.
186     if (!FK)
187       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStructOffset);
188 
189     asDerived().flushTrivialFields();
190     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
191     ASTContext &Ctx = asDerived().getContext();
192     const ConstantArrayType *CAT = cast<ConstantArrayType>(AT);
193     unsigned NumElts = Ctx.getConstantArrayElementCount(CAT);
194     QualType EltTy = Ctx.getBaseElementType(CAT);
195     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
196     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
197               llvm::to_string(EltSize.getQuantity()) + "n" +
198               llvm::to_string(NumElts));
199     EltTy = IsVolatile ? EltTy.withVolatile() : EltTy;
200     asDerived().visitWithKind(FK, EltTy, nullptr, FieldOffset);
201     appendStr("_AE");
202   }
203 
204   void appendStr(StringRef Str) { Name += Str; }
205 
206   std::string getName(QualType QT, bool IsVolatile) {
207     QT = IsVolatile ? QT.withVolatile() : QT;
208     asDerived().visitStructFields(QT, CharUnits::Zero());
209     return Name;
210   }
211 
212   Derived &asDerived() { return static_cast<Derived &>(*this); }
213 
214   std::string Name;
215 };
216 
217 template <class Derived>
218 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
219   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
220       : StructVisitor<Derived>(Ctx) {
221     this->appendStr(Prefix);
222     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
223   }
224 };
225 
226 // Helper function to create a null constant.
227 static llvm::Constant *getNullForVariable(Address Addr) {
228   llvm::Type *Ty = Addr.getElementType();
229   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
230 }
231 
232 template <bool IsMove>
233 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
234                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
235 
236   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
237                     CharUnits SrcAlignment, ASTContext &Ctx)
238       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
239     this->appendStr(Prefix);
240     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
241     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
242   }
243 
244   void flushTrivialFields() {
245     if (this->Start == this->End)
246       return;
247 
248     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
249                     llvm::to_string((this->End - this->Start).getQuantity()));
250 
251     this->Start = this->End = CharUnits::Zero();
252   }
253 
254   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
255                             CharUnits CurStackOffset) {
256     // Because volatile fields can be bit-fields and are individually copied,
257     // their offset and width are in bits.
258     uint64_t OffsetInBits =
259         this->Ctx.toBits(CurStackOffset) + this->getFieldOffsetInBits(FD);
260     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
261                     llvm::to_string(getFieldSize(FD, FT, this->Ctx)));
262   }
263 };
264 
265 struct GenDefaultInitializeFuncName
266     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
267       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
268   using Super = DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName>;
269   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
270       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
271                                                        DstAlignment, Ctx) {}
272   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
273                      const FieldDecl *FD, CharUnits CurStructOffset) {
274     if (const auto *AT = getContext().getAsArrayType(FT)) {
275       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
276       return;
277     }
278 
279     Super::visitWithKind(PDIK, FT, FD, CurStructOffset);
280   }
281 };
282 
283 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
284                                DestructedTypeVisitor<GenDestructorFuncName> {
285   using Super = DestructedTypeVisitor<GenDestructorFuncName>;
286   GenDestructorFuncName(const char *Prefix, CharUnits DstAlignment,
287                         ASTContext &Ctx)
288       : GenUnaryFuncName<GenDestructorFuncName>(Prefix, DstAlignment,
289                                                 Ctx) {}
290   void visitWithKind(QualType::DestructionKind DK, QualType FT,
291                      const FieldDecl *FD, CharUnits CurStructOffset) {
292     if (const auto *AT = getContext().getAsArrayType(FT)) {
293       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
294       return;
295     }
296 
297     Super::visitWithKind(DK, FT, FD, CurStructOffset);
298   }
299 };
300 
301 // Helper function that creates CGFunctionInfo for an N-ary special function.
302 template <size_t N>
303 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
304                                              FunctionArgList &Args) {
305   ASTContext &Ctx = CGM.getContext();
306   llvm::SmallVector<ImplicitParamDecl *, N> Params;
307   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
308 
309   for (unsigned I = 0; I < N; ++I)
310     Params.push_back(ImplicitParamDecl::Create(
311         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
312         ImplicitParamDecl::Other));
313 
314   for (auto &P : Params)
315     Args.push_back(P);
316 
317   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
318 }
319 
320 // Template classes that are used as bases for classes that emit special
321 // functions.
322 template <class Derived> struct GenFuncBase {
323   template <size_t N>
324   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStackOffset,
325                    std::array<Address, N> Addrs) {
326     this->asDerived().callSpecialFunction(
327         FT, CurStackOffset + asDerived().getFieldOffset(FD), Addrs);
328   }
329 
330   template <class FieldKind, size_t N>
331   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
332                   const FieldDecl *FD, CharUnits CurStackOffset,
333                   std::array<Address, N> Addrs) {
334     // Non-volatile trivial fields are copied when flushTrivialFields is called.
335     if (!FK)
336       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStackOffset,
337                                       Addrs);
338 
339     asDerived().flushTrivialFields(Addrs);
340     CodeGenFunction &CGF = *this->CGF;
341     ASTContext &Ctx = CGF.getContext();
342 
343     // Compute the end address.
344     QualType BaseEltQT;
345     std::array<Address, N> StartAddrs = Addrs;
346     for (unsigned I = 0; I < N; ++I)
347       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStackOffset, FD);
348     Address DstAddr = StartAddrs[DstIdx];
349     llvm::Value *NumElts = CGF.emitArrayLength(AT, BaseEltQT, DstAddr);
350     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
351     llvm::Value *BaseEltSizeVal =
352         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
353     llvm::Value *SizeInBytes =
354         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
355     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
356     llvm::Value *DstArrayEnd =
357         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
358     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
359                                             "dstarray.end");
360     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
361 
362     // Create the header block and insert the phi instructions.
363     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
364     CGF.EmitBlock(HeaderBB);
365     llvm::PHINode *PHIs[N];
366 
367     for (unsigned I = 0; I < N; ++I) {
368       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
369       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
370     }
371 
372     // Create the exit and loop body blocks.
373     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
374     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
375 
376     // Emit the comparison and conditional branch instruction that jumps to
377     // either the exit or the loop body.
378     llvm::Value *Done =
379         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
380     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
381 
382     // Visit the element of the array in the loop body.
383     CGF.EmitBlock(LoopBB);
384     QualType EltQT = AT->getElementType();
385     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
386     std::array<Address, N> NewAddrs = Addrs;
387 
388     for (unsigned I = 0; I < N; ++I)
389       NewAddrs[I] = Address(
390           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
391 
392     EltQT = IsVolatile ? EltQT.withVolatile() : EltQT;
393     this->asDerived().visitWithKind(FK, EltQT, nullptr, CharUnits::Zero(),
394                                     NewAddrs);
395 
396     LoopBB = CGF.Builder.GetInsertBlock();
397 
398     for (unsigned I = 0; I < N; ++I) {
399       // Instrs to update the destination and source addresses.
400       // Update phi instructions.
401       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
402       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
403     }
404 
405     // Insert an unconditional branch to the header block.
406     CGF.Builder.CreateBr(HeaderBB);
407     CGF.EmitBlock(ExitBB);
408   }
409 
410   /// Return an address with the specified offset from the passed address.
411   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
412     assert(Addr.isValid() && "invalid address");
413     if (Offset.getQuantity() == 0)
414       return Addr;
415     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
416     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity(),
417                                                CharUnits::One());
418     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
419   }
420 
421   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
422                             const FieldDecl *FD) {
423     return getAddrWithOffset(Addr, StructFieldOffset +
424                                        asDerived().getFieldOffset(FD));
425   }
426 
427   template <size_t N>
428   llvm::Function *
429   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
430               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
431     // If the special function already exists in the module, return it.
432     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
433       bool WrongType = false;
434       if (!F->getReturnType()->isVoidTy())
435         WrongType = true;
436       else {
437         for (const llvm::Argument &Arg : F->args())
438           if (Arg.getType() != CGM.Int8PtrPtrTy)
439             WrongType = true;
440       }
441 
442       if (WrongType) {
443         std::string FuncName = F->getName();
444         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
445         CGM.Error(Loc, "special function " + FuncName +
446                            " for non-trivial C struct has incorrect type");
447         return nullptr;
448       }
449       return F;
450     }
451 
452     ASTContext &Ctx = CGM.getContext();
453     FunctionArgList Args;
454     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
455     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
456     llvm::Function *F =
457         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
458                                FuncName, &CGM.getModule());
459     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
460     CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F);
461     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
462     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
463     FunctionDecl *FD = FunctionDecl::Create(
464         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
465         II, Ctx.getFunctionType(Ctx.VoidTy, llvm::None, {}), nullptr,
466         SC_PrivateExtern, false, false);
467     CodeGenFunction NewCGF(CGM);
468     setCGF(&NewCGF);
469     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
470 
471     for (unsigned I = 0; I < N; ++I) {
472       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
473       Addrs[I] = Address(V, Alignments[I]);
474     }
475 
476     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
477     CGF->FinishFunction();
478     return F;
479   }
480 
481   template <size_t N>
482   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
483                 CodeGenFunction &CallerCGF) {
484     std::array<CharUnits, N> Alignments;
485     llvm::Value *Ptrs[N];
486 
487     for (unsigned I = 0; I < N; ++I) {
488       Alignments[I] = Addrs[I].getAlignment();
489       Ptrs[I] =
490           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
491               .getPointer();
492     }
493 
494     if (llvm::Function *F =
495             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
496       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
497   }
498 
499   Derived &asDerived() { return static_cast<Derived &>(*this); }
500 
501   void setCGF(CodeGenFunction *F) { CGF = F; }
502 
503   CodeGenFunction *CGF = nullptr;
504 };
505 
506 template <class Derived, bool IsMove>
507 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
508                        GenFuncBase<Derived> {
509   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
510 
511   void flushTrivialFields(std::array<Address, 2> Addrs) {
512     CharUnits Size = this->End - this->Start;
513 
514     if (Size.getQuantity() == 0)
515       return;
516 
517     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
518     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
519 
520     // Emit memcpy.
521     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
522       llvm::Value *SizeVal =
523           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
524       DstAddr =
525           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
526       SrcAddr =
527           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
528       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
529     } else {
530       llvm::Type *Ty = llvm::Type::getIntNTy(
531           this->CGF->getLLVMContext(),
532           Size.getQuantity() * this->CGF->getContext().getCharWidth());
533       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
534       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
535       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
536       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
537     }
538 
539     this->Start = this->End = CharUnits::Zero();
540   }
541 
542   template <class... Ts>
543   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
544                             std::array<Address, 2> Addrs) {
545     LValue DstLV, SrcLV;
546     if (FD) {
547       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
548       llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
549       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
550       LValue DstBase = this->CGF->MakeAddrLValue(
551           this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
552       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
553       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
554       LValue SrcBase = this->CGF->MakeAddrLValue(
555           this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
556       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
557     } else {
558       llvm::PointerType *Ty = this->CGF->ConvertType(FT)->getPointerTo();
559       Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
560       Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
561       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
562       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
563     }
564     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
565     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
566   }
567 };
568 
569 // These classes that emit the special functions for a non-trivial struct.
570 struct GenDestructor : StructVisitor<GenDestructor>,
571                        GenFuncBase<GenDestructor>,
572                        DestructedTypeVisitor<GenDestructor> {
573   using Super = DestructedTypeVisitor<GenDestructor>;
574   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
575 
576   void visitWithKind(QualType::DestructionKind DK, QualType FT,
577                      const FieldDecl *FD, CharUnits CurStructOffset,
578                      std::array<Address, 1> Addrs) {
579     if (const auto *AT = getContext().getAsArrayType(FT)) {
580       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset, Addrs);
581       return;
582     }
583 
584     Super::visitWithKind(DK, FT, FD, CurStructOffset, Addrs);
585   }
586 
587   void visitARCStrong(QualType QT, const FieldDecl *FD,
588                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
589     CGF->destroyARCStrongImprecise(
590         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
591   }
592 
593   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
594                     std::array<Address, 1> Addrs) {
595     CGF->destroyARCWeak(
596         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
597   }
598 
599   void callSpecialFunction(QualType FT, CharUnits Offset,
600                            std::array<Address, 1> Addrs) {
601     CGF->callCStructDestructor(
602         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
603   }
604 };
605 
606 struct GenDefaultInitialize
607     : StructVisitor<GenDefaultInitialize>,
608       GenFuncBase<GenDefaultInitialize>,
609       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
610   using Super = DefaultInitializedTypeVisitor<GenDefaultInitialize>;
611   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
612 
613   GenDefaultInitialize(ASTContext &Ctx)
614       : StructVisitor<GenDefaultInitialize>(Ctx) {}
615 
616   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
617                      const FieldDecl *FD, CharUnits CurStructOffset,
618                      std::array<Address, 1> Addrs) {
619     if (const auto *AT = getContext().getAsArrayType(FT)) {
620       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset,
621                  Addrs);
622       return;
623     }
624 
625     Super::visitWithKind(PDIK, FT, FD, CurStructOffset, Addrs);
626   }
627 
628   void visitARCStrong(QualType QT, const FieldDecl *FD,
629                       CharUnits CurStackOffset, std::array<Address, 1> Addrs) {
630     CGF->EmitNullInitialization(
631         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
632   }
633 
634   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
635                     std::array<Address, 1> Addrs) {
636     CGF->EmitNullInitialization(
637         getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD), QT);
638   }
639 
640   template <class FieldKind, size_t... Is>
641   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
642                   const FieldDecl *FD, CharUnits CurStackOffset,
643                   std::array<Address, 1> Addrs) {
644     if (!FK)
645       return visitTrivial(QualType(AT, 0), FD, CurStackOffset, Addrs);
646 
647     ASTContext &Ctx = getContext();
648     CharUnits Size = Ctx.getTypeSizeInChars(QualType(AT, 0));
649     QualType EltTy = Ctx.getBaseElementType(QualType(AT, 0));
650 
651     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
652       GenFuncBaseTy::visitArray(FK, AT, IsVolatile, FD, CurStackOffset, Addrs);
653       return;
654     }
655 
656     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
657     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
658     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
659     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
660                               IsVolatile);
661   }
662 
663   void callSpecialFunction(QualType FT, CharUnits Offset,
664                            std::array<Address, 1> Addrs) {
665     CGF->callCStructDefaultConstructor(
666         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
667   }
668 };
669 
670 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
671   GenCopyConstructor(ASTContext &Ctx)
672       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
673 
674   void visitARCStrong(QualType QT, const FieldDecl *FD,
675                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
676     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
677     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
678     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
679         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
680     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
681     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
682   }
683 
684   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
685                     std::array<Address, 2> Addrs) {
686     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
687     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
688     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
689   }
690 
691   void callSpecialFunction(QualType FT, CharUnits Offset,
692                            std::array<Address, 2> Addrs) {
693     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
694                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
695   }
696 };
697 
698 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
699   GenMoveConstructor(ASTContext &Ctx)
700       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
701 
702   void visitARCStrong(QualType QT, const FieldDecl *FD,
703                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
704     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
705     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
706     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
707     llvm::Value *SrcVal =
708         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
709     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
710     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
711                            /* isInitialization */ true);
712   }
713 
714   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
715                     std::array<Address, 2> Addrs) {
716     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
717     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
718     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
719   }
720 
721   void callSpecialFunction(QualType FT, CharUnits Offset,
722                            std::array<Address, 2> Addrs) {
723     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
724                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
725   }
726 };
727 
728 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
729   GenCopyAssignment(ASTContext &Ctx)
730       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
731 
732   void visitARCStrong(QualType QT, const FieldDecl *FD,
733                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
734     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
735     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
736     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
737         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
738     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
739                             false);
740   }
741 
742   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
743                     std::array<Address, 2> Addrs) {
744     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
745     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
746     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
747   }
748 
749   void callSpecialFunction(QualType FT, CharUnits Offset,
750                            std::array<Address, 2> Addrs) {
751     CGF->callCStructCopyAssignmentOperator(
752         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
753         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
754   }
755 };
756 
757 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
758   GenMoveAssignment(ASTContext &Ctx)
759       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
760 
761   void visitARCStrong(QualType QT, const FieldDecl *FD,
762                       CharUnits CurStackOffset, std::array<Address, 2> Addrs) {
763     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
764     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
765     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
766     llvm::Value *SrcVal =
767         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
768     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
769     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
770     llvm::Value *DstVal =
771         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
772     CGF->EmitStoreOfScalar(SrcVal, DstLV);
773     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
774   }
775 
776   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStackOffset,
777                     std::array<Address, 2> Addrs) {
778     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStackOffset, FD);
779     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStackOffset, FD);
780     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
781   }
782 
783   void callSpecialFunction(QualType FT, CharUnits Offset,
784                            std::array<Address, 2> Addrs) {
785     CGF->callCStructMoveAssignmentOperator(
786         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
787         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
788   }
789 };
790 
791 } // namespace
792 
793 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
794                                                Address Addr, QualType Type) {
795   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
796 }
797 
798 // Default-initialize a variable that is a non-trivial struct or an array of
799 // such structure.
800 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
801   GenDefaultInitialize Gen(getContext());
802   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
803   Gen.setCGF(this);
804   QualType QT = Dst.getType();
805   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
806   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
807 }
808 
809 template <class G, size_t N>
810 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
811                                 bool IsVolatile, CodeGenFunction &CGF,
812                                 std::array<Address, N> Addrs) {
813   for (unsigned I = 0; I < N; ++I)
814     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
815   QT = IsVolatile ? QT.withVolatile() : QT;
816   Gen.callFunc(FuncName, QT, Addrs, CGF);
817 }
818 
819 // Functions to emit calls to the special functions of a non-trivial C struct.
820 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
821   bool IsVolatile = Dst.isVolatile();
822   Address DstPtr = Dst.getAddress();
823   QualType QT = Dst.getType();
824   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
825   std::string FuncName = GenName.getName(QT, IsVolatile);
826   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
827                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
828 }
829 
830 std::string
831 CodeGenFunction::getNonTrivialCopyConstructorStr(QualType QT,
832                                                  CharUnits Alignment,
833                                                  bool IsVolatile,
834                                                  ASTContext &Ctx) {
835   GenBinaryFuncName<false> GenName("", Alignment, Alignment, Ctx);
836   return GenName.getName(QT, IsVolatile);
837 }
838 
839 std::string
840 CodeGenFunction::getNonTrivialDestructorStr(QualType QT, CharUnits Alignment,
841                                             bool IsVolatile, ASTContext &Ctx) {
842   GenDestructorFuncName GenName("", Alignment, Ctx);
843   return GenName.getName(QT, IsVolatile);
844 }
845 
846 void CodeGenFunction::callCStructDestructor(LValue Dst) {
847   bool IsVolatile = Dst.isVolatile();
848   Address DstPtr = Dst.getAddress();
849   QualType QT = Dst.getType();
850   GenDestructorFuncName GenName("__destructor_", DstPtr.getAlignment(),
851                                 getContext());
852   std::string FuncName = GenName.getName(QT, IsVolatile);
853   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
854                       *this, std::array<Address, 1>({{DstPtr}}));
855 }
856 
857 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
858   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
859   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
860   QualType QT = Dst.getType();
861   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
862                                    SrcPtr.getAlignment(), getContext());
863   std::string FuncName = GenName.getName(QT, IsVolatile);
864   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
865                       IsVolatile, *this,
866                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
867 }
868 
869 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
870 
871 ) {
872   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
873   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
874   QualType QT = Dst.getType();
875   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
876                                    SrcPtr.getAlignment(), getContext());
877   std::string FuncName = GenName.getName(QT, IsVolatile);
878   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
879                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
880 }
881 
882 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
883   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
884   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
885   QualType QT = Dst.getType();
886   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
887                                   SrcPtr.getAlignment(), getContext());
888   std::string FuncName = GenName.getName(QT, IsVolatile);
889   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
890                       IsVolatile, *this,
891                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
892 }
893 
894 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
895 
896 ) {
897   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
898   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
899   QualType QT = Dst.getType();
900   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
901                                   SrcPtr.getAlignment(), getContext());
902   std::string FuncName = GenName.getName(QT, IsVolatile);
903   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
904                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
905 }
906