1 //===--- CGNonTrivialStruct.cpp - Emit Special Functions for C Structs ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines functions to generate various special functions for C
10 // structs.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CodeGenModule.h"
16 #include "clang/AST/NonTrivialTypeVisitor.h"
17 #include "llvm/Support/ScopedPrinter.h"
18 #include <array>
19 
20 using namespace clang;
21 using namespace CodeGen;
22 
23 // Return the size of a field in number of bits.
24 static uint64_t getFieldSize(const FieldDecl *FD, QualType FT,
25                              ASTContext &Ctx) {
26   if (FD && FD->isBitField())
27     return FD->getBitWidthValue(Ctx);
28   return Ctx.getTypeSize(FT);
29 }
30 
31 namespace {
32 enum { DstIdx = 0, SrcIdx = 1 };
33 const char *ValNameStr[2] = {"dst", "src"};
34 
35 template <class Derived> struct StructVisitor {
36   StructVisitor(ASTContext &Ctx) : Ctx(Ctx) {}
37 
38   template <class... Ts>
39   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
40     const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
41 
42     // Iterate over the fields of the struct.
43     for (const FieldDecl *FD : RD->fields()) {
44       QualType FT = FD->getType();
45       FT = QT.isVolatileQualified() ? FT.withVolatile() : FT;
46       asDerived().visit(FT, FD, CurStructOffset, Args...);
47     }
48 
49     asDerived().flushTrivialFields(Args...);
50   }
51 
52   template <class... Ts> void visitTrivial(Ts... Args) {}
53 
54   template <class... Ts> void visitCXXDestructor(Ts... Args) {
55     llvm_unreachable("field of a C++ struct type is not expected");
56   }
57 
58   template <class... Ts> void flushTrivialFields(Ts... Args) {}
59 
60   uint64_t getFieldOffsetInBits(const FieldDecl *FD) {
61     return FD ? Ctx.getASTRecordLayout(FD->getParent())
62                     .getFieldOffset(FD->getFieldIndex())
63               : 0;
64   }
65 
66   CharUnits getFieldOffset(const FieldDecl *FD) {
67     return Ctx.toCharUnitsFromBits(getFieldOffsetInBits(FD));
68   }
69 
70   Derived &asDerived() { return static_cast<Derived &>(*this); }
71 
72   ASTContext &getContext() { return Ctx; }
73   ASTContext &Ctx;
74 };
75 
76 template <class Derived, bool IsMove>
77 struct CopyStructVisitor : StructVisitor<Derived>,
78                            CopiedTypeVisitor<Derived, IsMove> {
79   using StructVisitor<Derived>::asDerived;
80   using Super = CopiedTypeVisitor<Derived, IsMove>;
81 
82   CopyStructVisitor(ASTContext &Ctx) : StructVisitor<Derived>(Ctx) {}
83 
84   template <class... Ts>
85   void preVisit(QualType::PrimitiveCopyKind PCK, QualType FT,
86                 const FieldDecl *FD, CharUnits CurStructOffset, Ts &&... Args) {
87     if (PCK)
88       asDerived().flushTrivialFields(std::forward<Ts>(Args)...);
89   }
90 
91   template <class... Ts>
92   void visitWithKind(QualType::PrimitiveCopyKind PCK, QualType FT,
93                      const FieldDecl *FD, CharUnits CurStructOffset,
94                      Ts &&... Args) {
95     if (const auto *AT = asDerived().getContext().getAsArrayType(FT)) {
96       asDerived().visitArray(PCK, AT, FT.isVolatileQualified(), FD,
97                              CurStructOffset, std::forward<Ts>(Args)...);
98       return;
99     }
100 
101     Super::visitWithKind(PCK, FT, FD, CurStructOffset,
102                          std::forward<Ts>(Args)...);
103   }
104 
105   template <class... Ts>
106   void visitTrivial(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
107                     Ts... Args) {
108     assert(!FT.isVolatileQualified() && "volatile field not expected");
109     ASTContext &Ctx = asDerived().getContext();
110     uint64_t FieldSize = getFieldSize(FD, FT, Ctx);
111 
112     // Ignore zero-sized fields.
113     if (FieldSize == 0)
114       return;
115 
116     uint64_t FStartInBits = asDerived().getFieldOffsetInBits(FD);
117     uint64_t FEndInBits = FStartInBits + FieldSize;
118     uint64_t RoundedFEnd = llvm::alignTo(FEndInBits, Ctx.getCharWidth());
119 
120     // Set Start if this is the first field of a sequence of trivial fields.
121     if (Start == End)
122       Start = CurStructOffset + Ctx.toCharUnitsFromBits(FStartInBits);
123     End = CurStructOffset + Ctx.toCharUnitsFromBits(RoundedFEnd);
124   }
125 
126   CharUnits Start = CharUnits::Zero(), End = CharUnits::Zero();
127 };
128 
129 // This function creates the mangled name of a special function of a non-trivial
130 // C struct. Since there is no ODR in C, the function is mangled based on the
131 // struct contents and not the name. The mangled name has the following
132 // structure:
133 //
134 // <function-name> ::= <prefix> <alignment-info> "_" <struct-field-info>
135 // <prefix> ::= "__destructor_" | "__default_constructor_" |
136 //              "__copy_constructor_" | "__move_constructor_" |
137 //              "__copy_assignment_" | "__move_assignment_"
138 // <alignment-info> ::= <dst-alignment> ["_" <src-alignment>]
139 // <struct-field-info> ::= <field-info>+
140 // <field-info> ::= <struct-or-scalar-field-info> | <array-field-info>
141 // <struct-or-scalar-field-info> ::= "_S" <struct-field-info> |
142 //                                   <strong-field-info> | <trivial-field-info>
143 // <array-field-info> ::= "_AB" <array-offset> "s" <element-size> "n"
144 //                        <num-elements> <innermost-element-info> "_AE"
145 // <innermost-element-info> ::= <struct-or-scalar-field-info>
146 // <strong-field-info> ::= "_s" ["b"] ["v"] <field-offset>
147 // <trivial-field-info> ::= "_t" ["v"] <field-offset> "_" <field-size>
148 
149 template <class Derived> struct GenFuncNameBase {
150   std::string getVolatileOffsetStr(bool IsVolatile, CharUnits Offset) {
151     std::string S;
152     if (IsVolatile)
153       S = "v";
154     S += llvm::to_string(Offset.getQuantity());
155     return S;
156   }
157 
158   void visitARCStrong(QualType FT, const FieldDecl *FD,
159                       CharUnits CurStructOffset) {
160     appendStr("_s");
161     if (FT->isBlockPointerType())
162       appendStr("b");
163     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
164     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
165   }
166 
167   void visitARCWeak(QualType FT, const FieldDecl *FD,
168                     CharUnits CurStructOffset) {
169     appendStr("_w");
170     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
171     appendStr(getVolatileOffsetStr(FT.isVolatileQualified(), FieldOffset));
172   }
173 
174   void visitStruct(QualType QT, const FieldDecl *FD,
175                    CharUnits CurStructOffset) {
176     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
177     appendStr("_S");
178     asDerived().visitStructFields(QT, FieldOffset);
179   }
180 
181   template <class FieldKind>
182   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
183                   const FieldDecl *FD, CharUnits CurStructOffset) {
184     // String for non-volatile trivial fields is emitted when
185     // flushTrivialFields is called.
186     if (!FK)
187       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStructOffset);
188 
189     asDerived().flushTrivialFields();
190     CharUnits FieldOffset = CurStructOffset + asDerived().getFieldOffset(FD);
191     ASTContext &Ctx = asDerived().getContext();
192     const ConstantArrayType *CAT = cast<ConstantArrayType>(AT);
193     unsigned NumElts = Ctx.getConstantArrayElementCount(CAT);
194     QualType EltTy = Ctx.getBaseElementType(CAT);
195     CharUnits EltSize = Ctx.getTypeSizeInChars(EltTy);
196     appendStr("_AB" + llvm::to_string(FieldOffset.getQuantity()) + "s" +
197               llvm::to_string(EltSize.getQuantity()) + "n" +
198               llvm::to_string(NumElts));
199     EltTy = IsVolatile ? EltTy.withVolatile() : EltTy;
200     asDerived().visitWithKind(FK, EltTy, nullptr, FieldOffset);
201     appendStr("_AE");
202   }
203 
204   void appendStr(StringRef Str) { Name += Str; }
205 
206   std::string getName(QualType QT, bool IsVolatile) {
207     QT = IsVolatile ? QT.withVolatile() : QT;
208     asDerived().visitStructFields(QT, CharUnits::Zero());
209     return Name;
210   }
211 
212   Derived &asDerived() { return static_cast<Derived &>(*this); }
213 
214   std::string Name;
215 };
216 
217 template <class Derived>
218 struct GenUnaryFuncName : StructVisitor<Derived>, GenFuncNameBase<Derived> {
219   GenUnaryFuncName(StringRef Prefix, CharUnits DstAlignment, ASTContext &Ctx)
220       : StructVisitor<Derived>(Ctx) {
221     this->appendStr(Prefix);
222     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
223   }
224 };
225 
226 // Helper function to create a null constant.
227 static llvm::Constant *getNullForVariable(Address Addr) {
228   llvm::Type *Ty = Addr.getElementType();
229   return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty));
230 }
231 
232 template <bool IsMove>
233 struct GenBinaryFuncName : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>,
234                            GenFuncNameBase<GenBinaryFuncName<IsMove>> {
235 
236   GenBinaryFuncName(StringRef Prefix, CharUnits DstAlignment,
237                     CharUnits SrcAlignment, ASTContext &Ctx)
238       : CopyStructVisitor<GenBinaryFuncName<IsMove>, IsMove>(Ctx) {
239     this->appendStr(Prefix);
240     this->appendStr(llvm::to_string(DstAlignment.getQuantity()));
241     this->appendStr("_" + llvm::to_string(SrcAlignment.getQuantity()));
242   }
243 
244   void flushTrivialFields() {
245     if (this->Start == this->End)
246       return;
247 
248     this->appendStr("_t" + llvm::to_string(this->Start.getQuantity()) + "w" +
249                     llvm::to_string((this->End - this->Start).getQuantity()));
250 
251     this->Start = this->End = CharUnits::Zero();
252   }
253 
254   void visitVolatileTrivial(QualType FT, const FieldDecl *FD,
255                             CharUnits CurStructOffset) {
256     // Because volatile fields can be bit-fields and are individually copied,
257     // their offset and width are in bits.
258     uint64_t OffsetInBits =
259         this->Ctx.toBits(CurStructOffset) + this->getFieldOffsetInBits(FD);
260     this->appendStr("_tv" + llvm::to_string(OffsetInBits) + "w" +
261                     llvm::to_string(getFieldSize(FD, FT, this->Ctx)));
262   }
263 };
264 
265 struct GenDefaultInitializeFuncName
266     : GenUnaryFuncName<GenDefaultInitializeFuncName>,
267       DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName> {
268   using Super = DefaultInitializedTypeVisitor<GenDefaultInitializeFuncName>;
269   GenDefaultInitializeFuncName(CharUnits DstAlignment, ASTContext &Ctx)
270       : GenUnaryFuncName<GenDefaultInitializeFuncName>("__default_constructor_",
271                                                        DstAlignment, Ctx) {}
272   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
273                      const FieldDecl *FD, CharUnits CurStructOffset) {
274     if (const auto *AT = getContext().getAsArrayType(FT)) {
275       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
276       return;
277     }
278 
279     Super::visitWithKind(PDIK, FT, FD, CurStructOffset);
280   }
281 };
282 
283 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
284                                DestructedTypeVisitor<GenDestructorFuncName> {
285   using Super = DestructedTypeVisitor<GenDestructorFuncName>;
286   GenDestructorFuncName(const char *Prefix, CharUnits DstAlignment,
287                         ASTContext &Ctx)
288       : GenUnaryFuncName<GenDestructorFuncName>(Prefix, DstAlignment, Ctx) {}
289   void visitWithKind(QualType::DestructionKind DK, QualType FT,
290                      const FieldDecl *FD, CharUnits CurStructOffset) {
291     if (const auto *AT = getContext().getAsArrayType(FT)) {
292       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset);
293       return;
294     }
295 
296     Super::visitWithKind(DK, FT, FD, CurStructOffset);
297   }
298 };
299 
300 // Helper function that creates CGFunctionInfo for an N-ary special function.
301 template <size_t N>
302 static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
303                                              FunctionArgList &Args) {
304   ASTContext &Ctx = CGM.getContext();
305   llvm::SmallVector<ImplicitParamDecl *, N> Params;
306   QualType ParamTy = Ctx.getPointerType(Ctx.VoidPtrTy);
307 
308   for (unsigned I = 0; I < N; ++I)
309     Params.push_back(ImplicitParamDecl::Create(
310         Ctx, nullptr, SourceLocation(), &Ctx.Idents.get(ValNameStr[I]), ParamTy,
311         ImplicitParamDecl::Other));
312 
313   for (auto &P : Params)
314     Args.push_back(P);
315 
316   return CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
317 }
318 
319 // Template classes that are used as bases for classes that emit special
320 // functions.
321 template <class Derived> struct GenFuncBase {
322   template <size_t N>
323   void visitStruct(QualType FT, const FieldDecl *FD, CharUnits CurStructOffset,
324                    std::array<Address, N> Addrs) {
325     this->asDerived().callSpecialFunction(
326         FT, CurStructOffset + asDerived().getFieldOffset(FD), Addrs);
327   }
328 
329   template <class FieldKind, size_t N>
330   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
331                   const FieldDecl *FD, CharUnits CurStructOffset,
332                   std::array<Address, N> Addrs) {
333     // Non-volatile trivial fields are copied when flushTrivialFields is called.
334     if (!FK)
335       return asDerived().visitTrivial(QualType(AT, 0), FD, CurStructOffset,
336                                       Addrs);
337 
338     asDerived().flushTrivialFields(Addrs);
339     CodeGenFunction &CGF = *this->CGF;
340     ASTContext &Ctx = CGF.getContext();
341 
342     // Compute the end address.
343     QualType BaseEltQT;
344     std::array<Address, N> StartAddrs = Addrs;
345     for (unsigned I = 0; I < N; ++I)
346       StartAddrs[I] = getAddrWithOffset(Addrs[I], CurStructOffset, FD);
347     Address DstAddr = StartAddrs[DstIdx];
348     llvm::Value *NumElts = CGF.emitArrayLength(AT, BaseEltQT, DstAddr);
349     unsigned BaseEltSize = Ctx.getTypeSizeInChars(BaseEltQT).getQuantity();
350     llvm::Value *BaseEltSizeVal =
351         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
352     llvm::Value *SizeInBytes =
353         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
354     Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
355     llvm::Value *DstArrayEnd =
356         CGF.Builder.CreateInBoundsGEP(BC.getPointer(), SizeInBytes);
357     DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
358                                             "dstarray.end");
359     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
360 
361     // Create the header block and insert the phi instructions.
362     llvm::BasicBlock *HeaderBB = CGF.createBasicBlock("loop.header");
363     CGF.EmitBlock(HeaderBB);
364     llvm::PHINode *PHIs[N];
365 
366     for (unsigned I = 0; I < N; ++I) {
367       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
368       PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
369     }
370 
371     // Create the exit and loop body blocks.
372     llvm::BasicBlock *ExitBB = CGF.createBasicBlock("loop.exit");
373     llvm::BasicBlock *LoopBB = CGF.createBasicBlock("loop.body");
374 
375     // Emit the comparison and conditional branch instruction that jumps to
376     // either the exit or the loop body.
377     llvm::Value *Done =
378         CGF.Builder.CreateICmpEQ(PHIs[DstIdx], DstArrayEnd, "done");
379     CGF.Builder.CreateCondBr(Done, ExitBB, LoopBB);
380 
381     // Visit the element of the array in the loop body.
382     CGF.EmitBlock(LoopBB);
383     QualType EltQT = AT->getElementType();
384     CharUnits EltSize = Ctx.getTypeSizeInChars(EltQT);
385     std::array<Address, N> NewAddrs = Addrs;
386 
387     for (unsigned I = 0; I < N; ++I)
388       NewAddrs[I] = Address(
389           PHIs[I], StartAddrs[I].getAlignment().alignmentAtOffset(EltSize));
390 
391     EltQT = IsVolatile ? EltQT.withVolatile() : EltQT;
392     this->asDerived().visitWithKind(FK, EltQT, nullptr, CharUnits::Zero(),
393                                     NewAddrs);
394 
395     LoopBB = CGF.Builder.GetInsertBlock();
396 
397     for (unsigned I = 0; I < N; ++I) {
398       // Instrs to update the destination and source addresses.
399       // Update phi instructions.
400       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
401       PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
402     }
403 
404     // Insert an unconditional branch to the header block.
405     CGF.Builder.CreateBr(HeaderBB);
406     CGF.EmitBlock(ExitBB);
407   }
408 
409   /// Return an address with the specified offset from the passed address.
410   Address getAddrWithOffset(Address Addr, CharUnits Offset) {
411     assert(Addr.isValid() && "invalid address");
412     if (Offset.getQuantity() == 0)
413       return Addr;
414     Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
415     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity());
416     return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
417   }
418 
419   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
420                             const FieldDecl *FD) {
421     return getAddrWithOffset(Addr, StructFieldOffset +
422                                        asDerived().getFieldOffset(FD));
423   }
424 
425   template <size_t N>
426   llvm::Function *
427   getFunction(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
428               std::array<CharUnits, N> Alignments, CodeGenModule &CGM) {
429     // If the special function already exists in the module, return it.
430     if (llvm::Function *F = CGM.getModule().getFunction(FuncName)) {
431       bool WrongType = false;
432       if (!F->getReturnType()->isVoidTy())
433         WrongType = true;
434       else {
435         for (const llvm::Argument &Arg : F->args())
436           if (Arg.getType() != CGM.Int8PtrPtrTy)
437             WrongType = true;
438       }
439 
440       if (WrongType) {
441         std::string FuncName = F->getName();
442         SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
443         CGM.Error(Loc, "special function " + FuncName +
444                            " for non-trivial C struct has incorrect type");
445         return nullptr;
446       }
447       return F;
448     }
449 
450     ASTContext &Ctx = CGM.getContext();
451     FunctionArgList Args;
452     const CGFunctionInfo &FI = getFunctionInfo<N>(CGM, Args);
453     llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
454     llvm::Function *F =
455         llvm::Function::Create(FuncTy, llvm::GlobalValue::LinkOnceODRLinkage,
456                                FuncName, &CGM.getModule());
457     F->setVisibility(llvm::GlobalValue::HiddenVisibility);
458     CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F);
459     CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
460     IdentifierInfo *II = &Ctx.Idents.get(FuncName);
461     FunctionDecl *FD = FunctionDecl::Create(
462         Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
463         II, Ctx.getFunctionType(Ctx.VoidTy, llvm::None, {}), nullptr,
464         SC_PrivateExtern, false, false);
465     CodeGenFunction NewCGF(CGM);
466     setCGF(&NewCGF);
467     CGF->StartFunction(FD, Ctx.VoidTy, F, FI, Args);
468 
469     for (unsigned I = 0; I < N; ++I) {
470       llvm::Value *V = CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[I]));
471       Addrs[I] = Address(V, Alignments[I]);
472     }
473 
474     asDerived().visitStructFields(QT, CharUnits::Zero(), Addrs);
475     CGF->FinishFunction();
476     return F;
477   }
478 
479   template <size_t N>
480   void callFunc(StringRef FuncName, QualType QT, std::array<Address, N> Addrs,
481                 CodeGenFunction &CallerCGF) {
482     std::array<CharUnits, N> Alignments;
483     llvm::Value *Ptrs[N];
484 
485     for (unsigned I = 0; I < N; ++I) {
486       Alignments[I] = Addrs[I].getAlignment();
487       Ptrs[I] =
488           CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
489               .getPointer();
490     }
491 
492     if (llvm::Function *F =
493             getFunction(FuncName, QT, Addrs, Alignments, CallerCGF.CGM))
494       CallerCGF.EmitNounwindRuntimeCall(F, Ptrs);
495   }
496 
497   Derived &asDerived() { return static_cast<Derived &>(*this); }
498 
499   void setCGF(CodeGenFunction *F) { CGF = F; }
500 
501   CodeGenFunction *CGF = nullptr;
502 };
503 
504 template <class Derived, bool IsMove>
505 struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
506                        GenFuncBase<Derived> {
507   GenBinaryFunc(ASTContext &Ctx) : CopyStructVisitor<Derived, IsMove>(Ctx) {}
508 
509   void flushTrivialFields(std::array<Address, 2> Addrs) {
510     CharUnits Size = this->End - this->Start;
511 
512     if (Size.getQuantity() == 0)
513       return;
514 
515     Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], this->Start);
516     Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], this->Start);
517 
518     // Emit memcpy.
519     if (Size.getQuantity() >= 16 || !llvm::isPowerOf2_32(Size.getQuantity())) {
520       llvm::Value *SizeVal =
521           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
522       DstAddr =
523           this->CGF->Builder.CreateElementBitCast(DstAddr, this->CGF->Int8Ty);
524       SrcAddr =
525           this->CGF->Builder.CreateElementBitCast(SrcAddr, this->CGF->Int8Ty);
526       this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
527     } else {
528       llvm::Type *Ty = llvm::Type::getIntNTy(
529           this->CGF->getLLVMContext(),
530           Size.getQuantity() * this->CGF->getContext().getCharWidth());
531       DstAddr = this->CGF->Builder.CreateElementBitCast(DstAddr, Ty);
532       SrcAddr = this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty);
533       llvm::Value *SrcVal = this->CGF->Builder.CreateLoad(SrcAddr, false);
534       this->CGF->Builder.CreateStore(SrcVal, DstAddr, false);
535     }
536 
537     this->Start = this->End = CharUnits::Zero();
538   }
539 
540   template <class... Ts>
541   void visitVolatileTrivial(QualType FT, const FieldDecl *FD, CharUnits Offset,
542                             std::array<Address, 2> Addrs) {
543     LValue DstLV, SrcLV;
544     if (FD) {
545       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
546       llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
547       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
548       LValue DstBase = this->CGF->MakeAddrLValue(
549           this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
550       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
551       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
552       LValue SrcBase = this->CGF->MakeAddrLValue(
553           this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
554       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
555     } else {
556       llvm::PointerType *Ty = this->CGF->ConvertType(FT)->getPointerTo();
557       Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
558       Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
559       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
560       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
561     }
562     RValue SrcVal = this->CGF->EmitLoadOfLValue(SrcLV, SourceLocation());
563     this->CGF->EmitStoreThroughLValue(SrcVal, DstLV);
564   }
565 };
566 
567 // These classes that emit the special functions for a non-trivial struct.
568 struct GenDestructor : StructVisitor<GenDestructor>,
569                        GenFuncBase<GenDestructor>,
570                        DestructedTypeVisitor<GenDestructor> {
571   using Super = DestructedTypeVisitor<GenDestructor>;
572   GenDestructor(ASTContext &Ctx) : StructVisitor<GenDestructor>(Ctx) {}
573 
574   void visitWithKind(QualType::DestructionKind DK, QualType FT,
575                      const FieldDecl *FD, CharUnits CurStructOffset,
576                      std::array<Address, 1> Addrs) {
577     if (const auto *AT = getContext().getAsArrayType(FT)) {
578       visitArray(DK, AT, FT.isVolatileQualified(), FD, CurStructOffset, Addrs);
579       return;
580     }
581 
582     Super::visitWithKind(DK, FT, FD, CurStructOffset, Addrs);
583   }
584 
585   void visitARCStrong(QualType QT, const FieldDecl *FD,
586                       CharUnits CurStructOffset, std::array<Address, 1> Addrs) {
587     CGF->destroyARCStrongImprecise(
588         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD), QT);
589   }
590 
591   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
592                     std::array<Address, 1> Addrs) {
593     CGF->destroyARCWeak(
594         *CGF, getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD), QT);
595   }
596 
597   void callSpecialFunction(QualType FT, CharUnits Offset,
598                            std::array<Address, 1> Addrs) {
599     CGF->callCStructDestructor(
600         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
601   }
602 };
603 
604 struct GenDefaultInitialize
605     : StructVisitor<GenDefaultInitialize>,
606       GenFuncBase<GenDefaultInitialize>,
607       DefaultInitializedTypeVisitor<GenDefaultInitialize> {
608   using Super = DefaultInitializedTypeVisitor<GenDefaultInitialize>;
609   typedef GenFuncBase<GenDefaultInitialize> GenFuncBaseTy;
610 
611   GenDefaultInitialize(ASTContext &Ctx)
612       : StructVisitor<GenDefaultInitialize>(Ctx) {}
613 
614   void visitWithKind(QualType::PrimitiveDefaultInitializeKind PDIK, QualType FT,
615                      const FieldDecl *FD, CharUnits CurStructOffset,
616                      std::array<Address, 1> Addrs) {
617     if (const auto *AT = getContext().getAsArrayType(FT)) {
618       visitArray(PDIK, AT, FT.isVolatileQualified(), FD, CurStructOffset,
619                  Addrs);
620       return;
621     }
622 
623     Super::visitWithKind(PDIK, FT, FD, CurStructOffset, Addrs);
624   }
625 
626   void visitARCStrong(QualType QT, const FieldDecl *FD,
627                       CharUnits CurStructOffset, std::array<Address, 1> Addrs) {
628     CGF->EmitNullInitialization(
629         getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD), QT);
630   }
631 
632   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
633                     std::array<Address, 1> Addrs) {
634     CGF->EmitNullInitialization(
635         getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD), QT);
636   }
637 
638   template <class FieldKind, size_t... Is>
639   void visitArray(FieldKind FK, const ArrayType *AT, bool IsVolatile,
640                   const FieldDecl *FD, CharUnits CurStructOffset,
641                   std::array<Address, 1> Addrs) {
642     if (!FK)
643       return visitTrivial(QualType(AT, 0), FD, CurStructOffset, Addrs);
644 
645     ASTContext &Ctx = getContext();
646     CharUnits Size = Ctx.getTypeSizeInChars(QualType(AT, 0));
647     QualType EltTy = Ctx.getBaseElementType(QualType(AT, 0));
648 
649     if (Size < CharUnits::fromQuantity(16) || EltTy->getAs<RecordType>()) {
650       GenFuncBaseTy::visitArray(FK, AT, IsVolatile, FD, CurStructOffset, Addrs);
651       return;
652     }
653 
654     llvm::Constant *SizeVal = CGF->Builder.getInt64(Size.getQuantity());
655     Address DstAddr = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
656     Address Loc = CGF->Builder.CreateElementBitCast(DstAddr, CGF->Int8Ty);
657     CGF->Builder.CreateMemSet(Loc, CGF->Builder.getInt8(0), SizeVal,
658                               IsVolatile);
659   }
660 
661   void callSpecialFunction(QualType FT, CharUnits Offset,
662                            std::array<Address, 1> Addrs) {
663     CGF->callCStructDefaultConstructor(
664         CGF->MakeAddrLValue(getAddrWithOffset(Addrs[DstIdx], Offset), FT));
665   }
666 };
667 
668 struct GenCopyConstructor : GenBinaryFunc<GenCopyConstructor, false> {
669   GenCopyConstructor(ASTContext &Ctx)
670       : GenBinaryFunc<GenCopyConstructor, false>(Ctx) {}
671 
672   void visitARCStrong(QualType QT, const FieldDecl *FD,
673                       CharUnits CurStructOffset, std::array<Address, 2> Addrs) {
674     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
675     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
676     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
677         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
678     llvm::Value *Val = CGF->EmitARCRetain(QT, SrcVal);
679     CGF->EmitStoreOfScalar(Val, CGF->MakeAddrLValue(Addrs[DstIdx], QT), true);
680   }
681 
682   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
683                     std::array<Address, 2> Addrs) {
684     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
685     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
686     CGF->EmitARCCopyWeak(Addrs[DstIdx], Addrs[SrcIdx]);
687   }
688 
689   void callSpecialFunction(QualType FT, CharUnits Offset,
690                            std::array<Address, 2> Addrs) {
691     CGF->callCStructCopyConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
692                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
693   }
694 };
695 
696 struct GenMoveConstructor : GenBinaryFunc<GenMoveConstructor, true> {
697   GenMoveConstructor(ASTContext &Ctx)
698       : GenBinaryFunc<GenMoveConstructor, true>(Ctx) {}
699 
700   void visitARCStrong(QualType QT, const FieldDecl *FD,
701                       CharUnits CurStructOffset, std::array<Address, 2> Addrs) {
702     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
703     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
704     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
705     llvm::Value *SrcVal =
706         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
707     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
708     CGF->EmitStoreOfScalar(SrcVal, CGF->MakeAddrLValue(Addrs[DstIdx], QT),
709                            /* isInitialization */ true);
710   }
711 
712   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
713                     std::array<Address, 2> Addrs) {
714     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
715     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
716     CGF->EmitARCMoveWeak(Addrs[DstIdx], Addrs[SrcIdx]);
717   }
718 
719   void callSpecialFunction(QualType FT, CharUnits Offset,
720                            std::array<Address, 2> Addrs) {
721     CGF->callCStructMoveConstructor(CGF->MakeAddrLValue(Addrs[DstIdx], FT),
722                                     CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
723   }
724 };
725 
726 struct GenCopyAssignment : GenBinaryFunc<GenCopyAssignment, false> {
727   GenCopyAssignment(ASTContext &Ctx)
728       : GenBinaryFunc<GenCopyAssignment, false>(Ctx) {}
729 
730   void visitARCStrong(QualType QT, const FieldDecl *FD,
731                       CharUnits CurStructOffset, std::array<Address, 2> Addrs) {
732     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
733     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
734     llvm::Value *SrcVal = CGF->EmitLoadOfScalar(
735         Addrs[SrcIdx], QT.isVolatileQualified(), QT, SourceLocation());
736     CGF->EmitARCStoreStrong(CGF->MakeAddrLValue(Addrs[DstIdx], QT), SrcVal,
737                             false);
738   }
739 
740   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
741                     std::array<Address, 2> Addrs) {
742     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
743     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
744     CGF->emitARCCopyAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
745   }
746 
747   void callSpecialFunction(QualType FT, CharUnits Offset,
748                            std::array<Address, 2> Addrs) {
749     CGF->callCStructCopyAssignmentOperator(
750         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
751         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
752   }
753 };
754 
755 struct GenMoveAssignment : GenBinaryFunc<GenMoveAssignment, true> {
756   GenMoveAssignment(ASTContext &Ctx)
757       : GenBinaryFunc<GenMoveAssignment, true>(Ctx) {}
758 
759   void visitARCStrong(QualType QT, const FieldDecl *FD,
760                       CharUnits CurStructOffset, std::array<Address, 2> Addrs) {
761     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
762     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
763     LValue SrcLV = CGF->MakeAddrLValue(Addrs[SrcIdx], QT);
764     llvm::Value *SrcVal =
765         CGF->EmitLoadOfLValue(SrcLV, SourceLocation()).getScalarVal();
766     CGF->EmitStoreOfScalar(getNullForVariable(SrcLV.getAddress()), SrcLV);
767     LValue DstLV = CGF->MakeAddrLValue(Addrs[DstIdx], QT);
768     llvm::Value *DstVal =
769         CGF->EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
770     CGF->EmitStoreOfScalar(SrcVal, DstLV);
771     CGF->EmitARCRelease(DstVal, ARCImpreciseLifetime);
772   }
773 
774   void visitARCWeak(QualType QT, const FieldDecl *FD, CharUnits CurStructOffset,
775                     std::array<Address, 2> Addrs) {
776     Addrs[DstIdx] = getAddrWithOffset(Addrs[DstIdx], CurStructOffset, FD);
777     Addrs[SrcIdx] = getAddrWithOffset(Addrs[SrcIdx], CurStructOffset, FD);
778     CGF->emitARCMoveAssignWeak(QT, Addrs[DstIdx], Addrs[SrcIdx]);
779   }
780 
781   void callSpecialFunction(QualType FT, CharUnits Offset,
782                            std::array<Address, 2> Addrs) {
783     CGF->callCStructMoveAssignmentOperator(
784         CGF->MakeAddrLValue(Addrs[DstIdx], FT),
785         CGF->MakeAddrLValue(Addrs[SrcIdx], FT));
786   }
787 };
788 
789 } // namespace
790 
791 void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
792                                                Address Addr, QualType Type) {
793   CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Type));
794 }
795 
796 // Default-initialize a variable that is a non-trivial struct or an array of
797 // such structure.
798 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
799   GenDefaultInitialize Gen(getContext());
800   Address DstPtr = Builder.CreateBitCast(Dst.getAddress(), CGM.Int8PtrPtrTy);
801   Gen.setCGF(this);
802   QualType QT = Dst.getType();
803   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
804   Gen.visit(QT, nullptr, CharUnits::Zero(), std::array<Address, 1>({{DstPtr}}));
805 }
806 
807 template <class G, size_t N>
808 static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
809                                 bool IsVolatile, CodeGenFunction &CGF,
810                                 std::array<Address, N> Addrs) {
811   for (unsigned I = 0; I < N; ++I)
812     Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
813   QT = IsVolatile ? QT.withVolatile() : QT;
814   Gen.callFunc(FuncName, QT, Addrs, CGF);
815 }
816 
817 // Functions to emit calls to the special functions of a non-trivial C struct.
818 void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
819   bool IsVolatile = Dst.isVolatile();
820   Address DstPtr = Dst.getAddress();
821   QualType QT = Dst.getType();
822   GenDefaultInitializeFuncName GenName(DstPtr.getAlignment(), getContext());
823   std::string FuncName = GenName.getName(QT, IsVolatile);
824   callSpecialFunction(GenDefaultInitialize(getContext()), FuncName, QT,
825                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
826 }
827 
828 std::string CodeGenFunction::getNonTrivialCopyConstructorStr(
829     QualType QT, CharUnits Alignment, bool IsVolatile, ASTContext &Ctx) {
830   GenBinaryFuncName<false> GenName("", Alignment, Alignment, Ctx);
831   return GenName.getName(QT, IsVolatile);
832 }
833 
834 std::string CodeGenFunction::getNonTrivialDestructorStr(QualType QT,
835                                                         CharUnits Alignment,
836                                                         bool IsVolatile,
837                                                         ASTContext &Ctx) {
838   GenDestructorFuncName GenName("", Alignment, Ctx);
839   return GenName.getName(QT, IsVolatile);
840 }
841 
842 void CodeGenFunction::callCStructDestructor(LValue Dst) {
843   bool IsVolatile = Dst.isVolatile();
844   Address DstPtr = Dst.getAddress();
845   QualType QT = Dst.getType();
846   GenDestructorFuncName GenName("__destructor_", DstPtr.getAlignment(),
847                                 getContext());
848   std::string FuncName = GenName.getName(QT, IsVolatile);
849   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
850                       *this, std::array<Address, 1>({{DstPtr}}));
851 }
852 
853 void CodeGenFunction::callCStructCopyConstructor(LValue Dst, LValue Src) {
854   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
855   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
856   QualType QT = Dst.getType();
857   GenBinaryFuncName<false> GenName("__copy_constructor_", DstPtr.getAlignment(),
858                                    SrcPtr.getAlignment(), getContext());
859   std::string FuncName = GenName.getName(QT, IsVolatile);
860   callSpecialFunction(GenCopyConstructor(getContext()), FuncName, QT,
861                       IsVolatile, *this,
862                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
863 }
864 
865 void CodeGenFunction::callCStructCopyAssignmentOperator(LValue Dst, LValue Src
866 
867 ) {
868   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
869   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
870   QualType QT = Dst.getType();
871   GenBinaryFuncName<false> GenName("__copy_assignment_", DstPtr.getAlignment(),
872                                    SrcPtr.getAlignment(), getContext());
873   std::string FuncName = GenName.getName(QT, IsVolatile);
874   callSpecialFunction(GenCopyAssignment(getContext()), FuncName, QT, IsVolatile,
875                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
876 }
877 
878 void CodeGenFunction::callCStructMoveConstructor(LValue Dst, LValue Src) {
879   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
880   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
881   QualType QT = Dst.getType();
882   GenBinaryFuncName<true> GenName("__move_constructor_", DstPtr.getAlignment(),
883                                   SrcPtr.getAlignment(), getContext());
884   std::string FuncName = GenName.getName(QT, IsVolatile);
885   callSpecialFunction(GenMoveConstructor(getContext()), FuncName, QT,
886                       IsVolatile, *this,
887                       std::array<Address, 2>({{DstPtr, SrcPtr}}));
888 }
889 
890 void CodeGenFunction::callCStructMoveAssignmentOperator(LValue Dst, LValue Src
891 
892 ) {
893   bool IsVolatile = Dst.isVolatile() || Src.isVolatile();
894   Address DstPtr = Dst.getAddress(), SrcPtr = Src.getAddress();
895   QualType QT = Dst.getType();
896   GenBinaryFuncName<true> GenName("__move_assignment_", DstPtr.getAlignment(),
897                                   SrcPtr.getAlignment(), getContext());
898   std::string FuncName = GenName.getName(QT, IsVolatile);
899   callSpecialFunction(GenMoveAssignment(getContext()), FuncName, QT, IsVolatile,
900                       *this, std::array<Address, 2>({{DstPtr, SrcPtr}}));
901 }
902