1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // These classes wrap the information about a call or function
11 // definition used to handle ABI compliancy.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CGCall.h"
16 #include "ABIInfo.h"
17 #include "CGCXXABI.h"
18 #include "CodeGenFunction.h"
19 #include "CodeGenModule.h"
20 #include "TargetInfo.h"
21 #include "clang/AST/Decl.h"
22 #include "clang/AST/DeclCXX.h"
23 #include "clang/AST/DeclObjC.h"
24 #include "clang/Basic/TargetInfo.h"
25 #include "clang/CodeGen/CGFunctionInfo.h"
26 #include "clang/Frontend/CodeGenOptions.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/IR/Attributes.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/InlineAsm.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/Transforms/Utils/Local.h"
34 using namespace clang;
35 using namespace CodeGen;
36 
37 /***/
38 
39 static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
40   switch (CC) {
41   default: return llvm::CallingConv::C;
42   case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
43   case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
44   case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
45   case CC_X86_64Win64: return llvm::CallingConv::X86_64_Win64;
46   case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
47   case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
48   case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
49   case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
50   // TODO: add support for CC_X86Pascal to llvm
51   }
52 }
53 
54 /// Derives the 'this' type for codegen purposes, i.e. ignoring method
55 /// qualification.
56 /// FIXME: address space qualification?
57 static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
58   QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
59   return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
60 }
61 
62 /// Returns the canonical formal type of the given C++ method.
63 static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
64   return MD->getType()->getCanonicalTypeUnqualified()
65            .getAs<FunctionProtoType>();
66 }
67 
68 /// Returns the "extra-canonicalized" return type, which discards
69 /// qualifiers on the return type.  Codegen doesn't care about them,
70 /// and it makes ABI code a little easier to be able to assume that
71 /// all parameter and return types are top-level unqualified.
72 static CanQualType GetReturnType(QualType RetTy) {
73   return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
74 }
75 
76 /// Arrange the argument and result information for a value of the given
77 /// unprototyped freestanding function type.
78 const CGFunctionInfo &
79 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
80   // When translating an unprototyped function type, always use a
81   // variadic type.
82   return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
83                                  false, None, FTNP->getExtInfo(),
84                                  RequiredArgs(0));
85 }
86 
87 /// Arrange the LLVM function layout for a value of the given function
88 /// type, on top of any implicit parameters already stored.
89 static const CGFunctionInfo &
90 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool IsInstanceMethod,
91                         SmallVectorImpl<CanQualType> &prefix,
92                         CanQual<FunctionProtoType> FTP) {
93   RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
94   // FIXME: Kill copy.
95   for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
96     prefix.push_back(FTP->getParamType(i));
97   CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
98   return CGT.arrangeLLVMFunctionInfo(resultType, IsInstanceMethod, prefix,
99                                      FTP->getExtInfo(), required);
100 }
101 
102 /// Arrange the argument and result information for a value of the
103 /// given freestanding function type.
104 const CGFunctionInfo &
105 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
106   SmallVector<CanQualType, 16> argTypes;
107   return ::arrangeLLVMFunctionInfo(*this, false, argTypes, FTP);
108 }
109 
110 static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
111   // Set the appropriate calling convention for the Function.
112   if (D->hasAttr<StdCallAttr>())
113     return CC_X86StdCall;
114 
115   if (D->hasAttr<FastCallAttr>())
116     return CC_X86FastCall;
117 
118   if (D->hasAttr<ThisCallAttr>())
119     return CC_X86ThisCall;
120 
121   if (D->hasAttr<PascalAttr>())
122     return CC_X86Pascal;
123 
124   if (PcsAttr *PCS = D->getAttr<PcsAttr>())
125     return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
126 
127   if (D->hasAttr<PnaclCallAttr>())
128     return CC_PnaclCall;
129 
130   if (D->hasAttr<IntelOclBiccAttr>())
131     return CC_IntelOclBicc;
132 
133   if (D->hasAttr<MSABIAttr>())
134     return IsWindows ? CC_C : CC_X86_64Win64;
135 
136   if (D->hasAttr<SysVABIAttr>())
137     return IsWindows ? CC_X86_64SysV : CC_C;
138 
139   return CC_C;
140 }
141 
142 /// Arrange the argument and result information for a call to an
143 /// unknown C++ non-static member function of the given abstract type.
144 /// (Zero value of RD means we don't have any meaningful "this" argument type,
145 ///  so fall back to a generic pointer type).
146 /// The member function must be an ordinary function, i.e. not a
147 /// constructor or destructor.
148 const CGFunctionInfo &
149 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
150                                    const FunctionProtoType *FTP) {
151   SmallVector<CanQualType, 16> argTypes;
152 
153   // Add the 'this' pointer.
154   if (RD)
155     argTypes.push_back(GetThisType(Context, RD));
156   else
157     argTypes.push_back(Context.VoidPtrTy);
158 
159   return ::arrangeLLVMFunctionInfo(
160       *this, true, argTypes,
161       FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
162 }
163 
164 /// Arrange the argument and result information for a declaration or
165 /// definition of the given C++ non-static member function.  The
166 /// member function must be an ordinary function, i.e. not a
167 /// constructor or destructor.
168 const CGFunctionInfo &
169 CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
170   assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
171   assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
172 
173   CanQual<FunctionProtoType> prototype = GetFormalType(MD);
174 
175   if (MD->isInstance()) {
176     // The abstract case is perfectly fine.
177     const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
178     return arrangeCXXMethodType(ThisType, prototype.getTypePtr());
179   }
180 
181   return arrangeFreeFunctionType(prototype);
182 }
183 
184 /// Arrange the argument and result information for a declaration
185 /// or definition to the given constructor variant.
186 const CGFunctionInfo &
187 CodeGenTypes::arrangeCXXConstructorDeclaration(const CXXConstructorDecl *D,
188                                                CXXCtorType ctorKind) {
189   SmallVector<CanQualType, 16> argTypes;
190   argTypes.push_back(GetThisType(Context, D->getParent()));
191 
192   GlobalDecl GD(D, ctorKind);
193   CanQualType resultType =
194     TheCXXABI.HasThisReturn(GD) ? argTypes.front() : Context.VoidTy;
195 
196   CanQual<FunctionProtoType> FTP = GetFormalType(D);
197 
198   // Add the formal parameters.
199   for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
200     argTypes.push_back(FTP->getParamType(i));
201 
202   TheCXXABI.BuildConstructorSignature(D, ctorKind, resultType, argTypes);
203 
204   RequiredArgs required =
205       (D->isVariadic() ? RequiredArgs(argTypes.size()) : RequiredArgs::All);
206 
207   FunctionType::ExtInfo extInfo = FTP->getExtInfo();
208   return arrangeLLVMFunctionInfo(resultType, true, argTypes, extInfo, required);
209 }
210 
211 /// Arrange a call to a C++ method, passing the given arguments.
212 const CGFunctionInfo &
213 CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
214                                         const CXXConstructorDecl *D,
215                                         CXXCtorType CtorKind,
216                                         unsigned ExtraArgs) {
217   // FIXME: Kill copy.
218   SmallVector<CanQualType, 16> ArgTypes;
219   for (const auto &Arg : args)
220     ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
221 
222   CanQual<FunctionProtoType> FPT = GetFormalType(D);
223   RequiredArgs Required = RequiredArgs::forPrototypePlus(FPT, 1 + ExtraArgs);
224   GlobalDecl GD(D, CtorKind);
225   CanQualType ResultType =
226       TheCXXABI.HasThisReturn(GD) ? ArgTypes.front() : Context.VoidTy;
227 
228   FunctionType::ExtInfo Info = FPT->getExtInfo();
229   return arrangeLLVMFunctionInfo(ResultType, true, ArgTypes, Info, Required);
230 }
231 
232 /// Arrange the argument and result information for a declaration,
233 /// definition, or call to the given destructor variant.  It so
234 /// happens that all three cases produce the same information.
235 const CGFunctionInfo &
236 CodeGenTypes::arrangeCXXDestructor(const CXXDestructorDecl *D,
237                                    CXXDtorType dtorKind) {
238   SmallVector<CanQualType, 2> argTypes;
239   argTypes.push_back(GetThisType(Context, D->getParent()));
240 
241   GlobalDecl GD(D, dtorKind);
242   CanQualType resultType =
243     TheCXXABI.HasThisReturn(GD) ? argTypes.front() : Context.VoidTy;
244 
245   TheCXXABI.BuildDestructorSignature(D, dtorKind, resultType, argTypes);
246 
247   CanQual<FunctionProtoType> FTP = GetFormalType(D);
248   assert(FTP->getNumParams() == 0 && "dtor with formal parameters");
249   assert(FTP->isVariadic() == 0 && "dtor with formal parameters");
250 
251   FunctionType::ExtInfo extInfo = FTP->getExtInfo();
252   return arrangeLLVMFunctionInfo(resultType, true, argTypes, extInfo,
253                                  RequiredArgs::All);
254 }
255 
256 /// Arrange the argument and result information for the declaration or
257 /// definition of the given function.
258 const CGFunctionInfo &
259 CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
260   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
261     if (MD->isInstance())
262       return arrangeCXXMethodDeclaration(MD);
263 
264   CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
265 
266   assert(isa<FunctionType>(FTy));
267 
268   // When declaring a function without a prototype, always use a
269   // non-variadic type.
270   if (isa<FunctionNoProtoType>(FTy)) {
271     CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>();
272     return arrangeLLVMFunctionInfo(noProto->getReturnType(), false, None,
273                                    noProto->getExtInfo(), RequiredArgs::All);
274   }
275 
276   assert(isa<FunctionProtoType>(FTy));
277   return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>());
278 }
279 
280 /// Arrange the argument and result information for the declaration or
281 /// definition of an Objective-C method.
282 const CGFunctionInfo &
283 CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
284   // It happens that this is the same as a call with no optional
285   // arguments, except also using the formal 'self' type.
286   return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
287 }
288 
289 /// Arrange the argument and result information for the function type
290 /// through which to perform a send to the given Objective-C method,
291 /// using the given receiver type.  The receiver type is not always
292 /// the 'self' type of the method or even an Objective-C pointer type.
293 /// This is *not* the right method for actually performing such a
294 /// message send, due to the possibility of optional arguments.
295 const CGFunctionInfo &
296 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
297                                               QualType receiverType) {
298   SmallVector<CanQualType, 16> argTys;
299   argTys.push_back(Context.getCanonicalParamType(receiverType));
300   argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
301   // FIXME: Kill copy?
302   for (const auto *I : MD->params()) {
303     argTys.push_back(Context.getCanonicalParamType(I->getType()));
304   }
305 
306   FunctionType::ExtInfo einfo;
307   bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
308   einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
309 
310   if (getContext().getLangOpts().ObjCAutoRefCount &&
311       MD->hasAttr<NSReturnsRetainedAttr>())
312     einfo = einfo.withProducesResult(true);
313 
314   RequiredArgs required =
315     (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
316 
317   return arrangeLLVMFunctionInfo(GetReturnType(MD->getReturnType()), false,
318                                  argTys, einfo, required);
319 }
320 
321 const CGFunctionInfo &
322 CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
323   // FIXME: Do we need to handle ObjCMethodDecl?
324   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
325 
326   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
327     return arrangeCXXConstructorDeclaration(CD, GD.getCtorType());
328 
329   if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
330     return arrangeCXXDestructor(DD, GD.getDtorType());
331 
332   return arrangeFunctionDeclaration(FD);
333 }
334 
335 /// Arrange a call as unto a free function, except possibly with an
336 /// additional number of formal parameters considered required.
337 static const CGFunctionInfo &
338 arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
339                             CodeGenModule &CGM,
340                             const CallArgList &args,
341                             const FunctionType *fnType,
342                             unsigned numExtraRequiredArgs) {
343   assert(args.size() >= numExtraRequiredArgs);
344 
345   // In most cases, there are no optional arguments.
346   RequiredArgs required = RequiredArgs::All;
347 
348   // If we have a variadic prototype, the required arguments are the
349   // extra prefix plus the arguments in the prototype.
350   if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
351     if (proto->isVariadic())
352       required = RequiredArgs(proto->getNumParams() + numExtraRequiredArgs);
353 
354   // If we don't have a prototype at all, but we're supposed to
355   // explicitly use the variadic convention for unprototyped calls,
356   // treat all of the arguments as required but preserve the nominal
357   // possibility of variadics.
358   } else if (CGM.getTargetCodeGenInfo()
359                 .isNoProtoCallVariadic(args,
360                                        cast<FunctionNoProtoType>(fnType))) {
361     required = RequiredArgs(args.size());
362   }
363 
364   return CGT.arrangeFreeFunctionCall(fnType->getReturnType(), args,
365                                      fnType->getExtInfo(), required);
366 }
367 
368 /// Figure out the rules for calling a function with the given formal
369 /// type using the given arguments.  The arguments are necessary
370 /// because the function might be unprototyped, in which case it's
371 /// target-dependent in crazy ways.
372 const CGFunctionInfo &
373 CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
374                                       const FunctionType *fnType) {
375   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 0);
376 }
377 
378 /// A block function call is essentially a free-function call with an
379 /// extra implicit argument.
380 const CGFunctionInfo &
381 CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
382                                        const FunctionType *fnType) {
383   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1);
384 }
385 
386 const CGFunctionInfo &
387 CodeGenTypes::arrangeFreeFunctionCall(QualType resultType,
388                                       const CallArgList &args,
389                                       FunctionType::ExtInfo info,
390                                       RequiredArgs required) {
391   // FIXME: Kill copy.
392   SmallVector<CanQualType, 16> argTypes;
393   for (const auto &Arg : args)
394     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
395   return arrangeLLVMFunctionInfo(GetReturnType(resultType), false, argTypes,
396                                  info, required);
397 }
398 
399 /// Arrange a call to a C++ method, passing the given arguments.
400 const CGFunctionInfo &
401 CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
402                                    const FunctionProtoType *FPT,
403                                    RequiredArgs required) {
404   // FIXME: Kill copy.
405   SmallVector<CanQualType, 16> argTypes;
406   for (const auto &Arg : args)
407     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
408 
409   FunctionType::ExtInfo info = FPT->getExtInfo();
410   return arrangeLLVMFunctionInfo(GetReturnType(FPT->getReturnType()), true,
411                                  argTypes, info, required);
412 }
413 
414 const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionDeclaration(
415     QualType resultType, const FunctionArgList &args,
416     const FunctionType::ExtInfo &info, bool isVariadic) {
417   // FIXME: Kill copy.
418   SmallVector<CanQualType, 16> argTypes;
419   for (auto Arg : args)
420     argTypes.push_back(Context.getCanonicalParamType(Arg->getType()));
421 
422   RequiredArgs required =
423     (isVariadic ? RequiredArgs(args.size()) : RequiredArgs::All);
424   return arrangeLLVMFunctionInfo(GetReturnType(resultType), false, argTypes, info,
425                                  required);
426 }
427 
428 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
429   return arrangeLLVMFunctionInfo(getContext().VoidTy, false, None,
430                                  FunctionType::ExtInfo(), RequiredArgs::All);
431 }
432 
433 /// Arrange the argument and result information for an abstract value
434 /// of a given function type.  This is the method which all of the
435 /// above functions ultimately defer to.
436 const CGFunctionInfo &
437 CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
438                                       bool IsInstanceMethod,
439                                       ArrayRef<CanQualType> argTypes,
440                                       FunctionType::ExtInfo info,
441                                       RequiredArgs required) {
442 #ifndef NDEBUG
443   for (ArrayRef<CanQualType>::const_iterator
444          I = argTypes.begin(), E = argTypes.end(); I != E; ++I)
445     assert(I->isCanonicalAsParam());
446 #endif
447 
448   unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
449 
450   // Lookup or create unique function info.
451   llvm::FoldingSetNodeID ID;
452   CGFunctionInfo::Profile(ID, IsInstanceMethod, info, required, resultType,
453                           argTypes);
454 
455   void *insertPos = nullptr;
456   CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
457   if (FI)
458     return *FI;
459 
460   // Construct the function info.  We co-allocate the ArgInfos.
461   FI = CGFunctionInfo::create(CC, IsInstanceMethod, info, resultType, argTypes,
462                               required);
463   FunctionInfos.InsertNode(FI, insertPos);
464 
465   bool inserted = FunctionsBeingProcessed.insert(FI); (void)inserted;
466   assert(inserted && "Recursively being processed?");
467 
468   // Compute ABI information.
469   getABIInfo().computeInfo(*FI);
470 
471   // Loop over all of the computed argument and return value info.  If any of
472   // them are direct or extend without a specified coerce type, specify the
473   // default now.
474   ABIArgInfo &retInfo = FI->getReturnInfo();
475   if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
476     retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
477 
478   for (auto &I : FI->arguments())
479     if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
480       I.info.setCoerceToType(ConvertType(I.type));
481 
482   bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
483   assert(erased && "Not in set?");
484 
485   return *FI;
486 }
487 
488 CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
489                                        bool IsInstanceMethod,
490                                        const FunctionType::ExtInfo &info,
491                                        CanQualType resultType,
492                                        ArrayRef<CanQualType> argTypes,
493                                        RequiredArgs required) {
494   void *buffer = operator new(sizeof(CGFunctionInfo) +
495                               sizeof(ArgInfo) * (argTypes.size() + 1));
496   CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
497   FI->CallingConvention = llvmCC;
498   FI->EffectiveCallingConvention = llvmCC;
499   FI->ASTCallingConvention = info.getCC();
500   FI->InstanceMethod = IsInstanceMethod;
501   FI->NoReturn = info.getNoReturn();
502   FI->ReturnsRetained = info.getProducesResult();
503   FI->Required = required;
504   FI->HasRegParm = info.getHasRegParm();
505   FI->RegParm = info.getRegParm();
506   FI->ArgStruct = nullptr;
507   FI->NumArgs = argTypes.size();
508   FI->getArgsBuffer()[0].type = resultType;
509   for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
510     FI->getArgsBuffer()[i + 1].type = argTypes[i];
511   return FI;
512 }
513 
514 /***/
515 
516 void CodeGenTypes::GetExpandedTypes(QualType type,
517                      SmallVectorImpl<llvm::Type*> &expandedTypes) {
518   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(type)) {
519     uint64_t NumElts = AT->getSize().getZExtValue();
520     for (uint64_t Elt = 0; Elt < NumElts; ++Elt)
521       GetExpandedTypes(AT->getElementType(), expandedTypes);
522   } else if (const RecordType *RT = type->getAs<RecordType>()) {
523     const RecordDecl *RD = RT->getDecl();
524     assert(!RD->hasFlexibleArrayMember() &&
525            "Cannot expand structure with flexible array.");
526     if (RD->isUnion()) {
527       // Unions can be here only in degenerative cases - all the fields are same
528       // after flattening. Thus we have to use the "largest" field.
529       const FieldDecl *LargestFD = nullptr;
530       CharUnits UnionSize = CharUnits::Zero();
531 
532       for (const auto *FD : RD->fields()) {
533         assert(!FD->isBitField() &&
534                "Cannot expand structure with bit-field members.");
535         CharUnits FieldSize = getContext().getTypeSizeInChars(FD->getType());
536         if (UnionSize < FieldSize) {
537           UnionSize = FieldSize;
538           LargestFD = FD;
539         }
540       }
541       if (LargestFD)
542         GetExpandedTypes(LargestFD->getType(), expandedTypes);
543     } else {
544       for (const auto *I : RD->fields()) {
545         assert(!I->isBitField() &&
546                "Cannot expand structure with bit-field members.");
547         GetExpandedTypes(I->getType(), expandedTypes);
548       }
549     }
550   } else if (const ComplexType *CT = type->getAs<ComplexType>()) {
551     llvm::Type *EltTy = ConvertType(CT->getElementType());
552     expandedTypes.push_back(EltTy);
553     expandedTypes.push_back(EltTy);
554   } else
555     expandedTypes.push_back(ConvertType(type));
556 }
557 
558 void CodeGenFunction::ExpandTypeFromArgs(
559     QualType Ty, LValue LV, SmallVectorImpl<llvm::Argument *>::iterator &AI) {
560   assert(LV.isSimple() &&
561          "Unexpected non-simple lvalue during struct expansion.");
562 
563   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
564     unsigned NumElts = AT->getSize().getZExtValue();
565     QualType EltTy = AT->getElementType();
566     for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
567       llvm::Value *EltAddr = Builder.CreateConstGEP2_32(LV.getAddress(), 0, Elt);
568       LValue LV = MakeAddrLValue(EltAddr, EltTy);
569       ExpandTypeFromArgs(EltTy, LV, AI);
570     }
571     return;
572   }
573   if (const RecordType *RT = Ty->getAs<RecordType>()) {
574     RecordDecl *RD = RT->getDecl();
575     if (RD->isUnion()) {
576       // Unions can be here only in degenerative cases - all the fields are same
577       // after flattening. Thus we have to use the "largest" field.
578       const FieldDecl *LargestFD = nullptr;
579       CharUnits UnionSize = CharUnits::Zero();
580 
581       for (const auto *FD : RD->fields()) {
582         assert(!FD->isBitField() &&
583                "Cannot expand structure with bit-field members.");
584         CharUnits FieldSize = getContext().getTypeSizeInChars(FD->getType());
585         if (UnionSize < FieldSize) {
586           UnionSize = FieldSize;
587           LargestFD = FD;
588         }
589       }
590       if (LargestFD) {
591         // FIXME: What are the right qualifiers here?
592         LValue SubLV = EmitLValueForField(LV, LargestFD);
593         ExpandTypeFromArgs(LargestFD->getType(), SubLV, AI);
594       }
595     } else {
596       for (const auto *FD : RD->fields()) {
597         QualType FT = FD->getType();
598         // FIXME: What are the right qualifiers here?
599         LValue SubLV = EmitLValueForField(LV, FD);
600         ExpandTypeFromArgs(FT, SubLV, AI);
601       }
602     }
603     return;
604   }
605   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
606     QualType EltTy = CT->getElementType();
607     llvm::Value *RealAddr = Builder.CreateStructGEP(LV.getAddress(), 0, "real");
608     EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(RealAddr, EltTy));
609     llvm::Value *ImagAddr = Builder.CreateStructGEP(LV.getAddress(), 1, "imag");
610     EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(ImagAddr, EltTy));
611     return;
612   }
613   EmitStoreThroughLValue(RValue::get(*AI++), LV);
614 }
615 
616 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
617 /// accessing some number of bytes out of it, try to gep into the struct to get
618 /// at its inner goodness.  Dive as deep as possible without entering an element
619 /// with an in-memory size smaller than DstSize.
620 static llvm::Value *
621 EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
622                                    llvm::StructType *SrcSTy,
623                                    uint64_t DstSize, CodeGenFunction &CGF) {
624   // We can't dive into a zero-element struct.
625   if (SrcSTy->getNumElements() == 0) return SrcPtr;
626 
627   llvm::Type *FirstElt = SrcSTy->getElementType(0);
628 
629   // If the first elt is at least as large as what we're looking for, or if the
630   // first element is the same size as the whole struct, we can enter it.
631   uint64_t FirstEltSize =
632     CGF.CGM.getDataLayout().getTypeAllocSize(FirstElt);
633   if (FirstEltSize < DstSize &&
634       FirstEltSize < CGF.CGM.getDataLayout().getTypeAllocSize(SrcSTy))
635     return SrcPtr;
636 
637   // GEP into the first element.
638   SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
639 
640   // If the first element is a struct, recurse.
641   llvm::Type *SrcTy =
642     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
643   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
644     return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
645 
646   return SrcPtr;
647 }
648 
649 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
650 /// are either integers or pointers.  This does a truncation of the value if it
651 /// is too large or a zero extension if it is too small.
652 ///
653 /// This behaves as if the value were coerced through memory, so on big-endian
654 /// targets the high bits are preserved in a truncation, while little-endian
655 /// targets preserve the low bits.
656 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
657                                              llvm::Type *Ty,
658                                              CodeGenFunction &CGF) {
659   if (Val->getType() == Ty)
660     return Val;
661 
662   if (isa<llvm::PointerType>(Val->getType())) {
663     // If this is Pointer->Pointer avoid conversion to and from int.
664     if (isa<llvm::PointerType>(Ty))
665       return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
666 
667     // Convert the pointer to an integer so we can play with its width.
668     Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
669   }
670 
671   llvm::Type *DestIntTy = Ty;
672   if (isa<llvm::PointerType>(DestIntTy))
673     DestIntTy = CGF.IntPtrTy;
674 
675   if (Val->getType() != DestIntTy) {
676     const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
677     if (DL.isBigEndian()) {
678       // Preserve the high bits on big-endian targets.
679       // That is what memory coercion does.
680       uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
681       uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
682 
683       if (SrcSize > DstSize) {
684         Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
685         Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
686       } else {
687         Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
688         Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
689       }
690     } else {
691       // Little-endian targets preserve the low bits. No shifts required.
692       Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
693     }
694   }
695 
696   if (isa<llvm::PointerType>(Ty))
697     Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
698   return Val;
699 }
700 
701 
702 
703 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
704 /// a pointer to an object of type \arg Ty.
705 ///
706 /// This safely handles the case when the src type is smaller than the
707 /// destination type; in this situation the values of bits which not
708 /// present in the src are undefined.
709 static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
710                                       llvm::Type *Ty,
711                                       CodeGenFunction &CGF) {
712   llvm::Type *SrcTy =
713     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
714 
715   // If SrcTy and Ty are the same, just do a load.
716   if (SrcTy == Ty)
717     return CGF.Builder.CreateLoad(SrcPtr);
718 
719   uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
720 
721   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
722     SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
723     SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
724   }
725 
726   uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
727 
728   // If the source and destination are integer or pointer types, just do an
729   // extension or truncation to the desired type.
730   if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
731       (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
732     llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
733     return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
734   }
735 
736   // If load is legal, just bitcast the src pointer.
737   if (SrcSize >= DstSize) {
738     // Generally SrcSize is never greater than DstSize, since this means we are
739     // losing bits. However, this can happen in cases where the structure has
740     // additional padding, for example due to a user specified alignment.
741     //
742     // FIXME: Assert that we aren't truncating non-padding bits when have access
743     // to that information.
744     llvm::Value *Casted =
745       CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
746     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
747     // FIXME: Use better alignment / avoid requiring aligned load.
748     Load->setAlignment(1);
749     return Load;
750   }
751 
752   // Otherwise do coercion through memory. This is stupid, but
753   // simple.
754   llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
755   llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
756   llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
757   llvm::Value *SrcCasted = CGF.Builder.CreateBitCast(SrcPtr, I8PtrTy);
758   // FIXME: Use better alignment.
759   CGF.Builder.CreateMemCpy(Casted, SrcCasted,
760       llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
761       1, false);
762   return CGF.Builder.CreateLoad(Tmp);
763 }
764 
765 // Function to store a first-class aggregate into memory.  We prefer to
766 // store the elements rather than the aggregate to be more friendly to
767 // fast-isel.
768 // FIXME: Do we need to recurse here?
769 static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
770                           llvm::Value *DestPtr, bool DestIsVolatile,
771                           bool LowAlignment) {
772   // Prefer scalar stores to first-class aggregate stores.
773   if (llvm::StructType *STy =
774         dyn_cast<llvm::StructType>(Val->getType())) {
775     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
776       llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
777       llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
778       llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
779                                                     DestIsVolatile);
780       if (LowAlignment)
781         SI->setAlignment(1);
782     }
783   } else {
784     llvm::StoreInst *SI = CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
785     if (LowAlignment)
786       SI->setAlignment(1);
787   }
788 }
789 
790 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
791 /// where the source and destination may have different types.
792 ///
793 /// This safely handles the case when the src type is larger than the
794 /// destination type; the upper bits of the src will be lost.
795 static void CreateCoercedStore(llvm::Value *Src,
796                                llvm::Value *DstPtr,
797                                bool DstIsVolatile,
798                                CodeGenFunction &CGF) {
799   llvm::Type *SrcTy = Src->getType();
800   llvm::Type *DstTy =
801     cast<llvm::PointerType>(DstPtr->getType())->getElementType();
802   if (SrcTy == DstTy) {
803     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
804     return;
805   }
806 
807   uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
808 
809   if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
810     DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
811     DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
812   }
813 
814   // If the source and destination are integer or pointer types, just do an
815   // extension or truncation to the desired type.
816   if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
817       (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
818     Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
819     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
820     return;
821   }
822 
823   uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
824 
825   // If store is legal, just bitcast the src pointer.
826   if (SrcSize <= DstSize) {
827     llvm::Value *Casted =
828       CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
829     // FIXME: Use better alignment / avoid requiring aligned store.
830     BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
831   } else {
832     // Otherwise do coercion through memory. This is stupid, but
833     // simple.
834 
835     // Generally SrcSize is never greater than DstSize, since this means we are
836     // losing bits. However, this can happen in cases where the structure has
837     // additional padding, for example due to a user specified alignment.
838     //
839     // FIXME: Assert that we aren't truncating non-padding bits when have access
840     // to that information.
841     llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
842     CGF.Builder.CreateStore(Src, Tmp);
843     llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
844     llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
845     llvm::Value *DstCasted = CGF.Builder.CreateBitCast(DstPtr, I8PtrTy);
846     // FIXME: Use better alignment.
847     CGF.Builder.CreateMemCpy(DstCasted, Casted,
848         llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
849         1, false);
850   }
851 }
852 
853 /***/
854 
855 bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
856   return FI.getReturnInfo().isIndirect();
857 }
858 
859 bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
860   return ReturnTypeUsesSRet(FI) &&
861          getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
862 }
863 
864 bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
865   if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
866     switch (BT->getKind()) {
867     default:
868       return false;
869     case BuiltinType::Float:
870       return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
871     case BuiltinType::Double:
872       return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
873     case BuiltinType::LongDouble:
874       return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
875     }
876   }
877 
878   return false;
879 }
880 
881 bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
882   if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
883     if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
884       if (BT->getKind() == BuiltinType::LongDouble)
885         return getTarget().useObjCFP2RetForComplexLongDouble();
886     }
887   }
888 
889   return false;
890 }
891 
892 llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
893   const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
894   return GetFunctionType(FI);
895 }
896 
897 llvm::FunctionType *
898 CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
899 
900   bool Inserted = FunctionsBeingProcessed.insert(&FI); (void)Inserted;
901   assert(Inserted && "Recursively being processed?");
902 
903   bool SwapThisWithSRet = false;
904   SmallVector<llvm::Type*, 8> argTypes;
905   llvm::Type *resultType = nullptr;
906 
907   const ABIArgInfo &retAI = FI.getReturnInfo();
908   switch (retAI.getKind()) {
909   case ABIArgInfo::Expand:
910     llvm_unreachable("Invalid ABI kind for return argument");
911 
912   case ABIArgInfo::Extend:
913   case ABIArgInfo::Direct:
914     resultType = retAI.getCoerceToType();
915     break;
916 
917   case ABIArgInfo::InAlloca:
918     if (retAI.getInAllocaSRet()) {
919       // sret things on win32 aren't void, they return the sret pointer.
920       QualType ret = FI.getReturnType();
921       llvm::Type *ty = ConvertType(ret);
922       unsigned addressSpace = Context.getTargetAddressSpace(ret);
923       resultType = llvm::PointerType::get(ty, addressSpace);
924     } else {
925       resultType = llvm::Type::getVoidTy(getLLVMContext());
926     }
927     break;
928 
929   case ABIArgInfo::Indirect: {
930     assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
931     resultType = llvm::Type::getVoidTy(getLLVMContext());
932 
933     QualType ret = FI.getReturnType();
934     llvm::Type *ty = ConvertType(ret);
935     unsigned addressSpace = Context.getTargetAddressSpace(ret);
936     argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
937 
938     SwapThisWithSRet = retAI.isSRetAfterThis();
939     break;
940   }
941 
942   case ABIArgInfo::Ignore:
943     resultType = llvm::Type::getVoidTy(getLLVMContext());
944     break;
945   }
946 
947   // Add in all of the required arguments.
948   CGFunctionInfo::const_arg_iterator it = FI.arg_begin(), ie;
949   if (FI.isVariadic()) {
950     ie = it + FI.getRequiredArgs().getNumRequiredArgs();
951   } else {
952     ie = FI.arg_end();
953   }
954   for (; it != ie; ++it) {
955     const ABIArgInfo &argAI = it->info;
956 
957     // Insert a padding type to ensure proper alignment.
958     if (llvm::Type *PaddingType = argAI.getPaddingType())
959       argTypes.push_back(PaddingType);
960 
961     switch (argAI.getKind()) {
962     case ABIArgInfo::Ignore:
963     case ABIArgInfo::InAlloca:
964       break;
965 
966     case ABIArgInfo::Indirect: {
967       // indirect arguments are always on the stack, which is addr space #0.
968       llvm::Type *LTy = ConvertTypeForMem(it->type);
969       argTypes.push_back(LTy->getPointerTo());
970       break;
971     }
972 
973     case ABIArgInfo::Extend:
974     case ABIArgInfo::Direct: {
975       // Fast-isel and the optimizer generally like scalar values better than
976       // FCAs, so we flatten them if this is safe to do for this argument.
977       llvm::Type *argType = argAI.getCoerceToType();
978       llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
979       if (st && argAI.isDirect() && argAI.getCanBeFlattened()) {
980         for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
981           argTypes.push_back(st->getElementType(i));
982       } else {
983         argTypes.push_back(argType);
984       }
985       break;
986     }
987 
988     case ABIArgInfo::Expand:
989       GetExpandedTypes(it->type, argTypes);
990       break;
991     }
992   }
993 
994   // Add the inalloca struct as the last parameter type.
995   if (llvm::StructType *ArgStruct = FI.getArgStruct())
996     argTypes.push_back(ArgStruct->getPointerTo());
997 
998   if (SwapThisWithSRet)
999     std::swap(argTypes[0], argTypes[1]);
1000 
1001   bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1002   assert(Erased && "Not in set?");
1003 
1004   return llvm::FunctionType::get(resultType, argTypes, FI.isVariadic());
1005 }
1006 
1007 llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1008   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1009   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1010 
1011   if (!isFuncTypeConvertible(FPT))
1012     return llvm::StructType::get(getLLVMContext());
1013 
1014   const CGFunctionInfo *Info;
1015   if (isa<CXXDestructorDecl>(MD))
1016     Info = &arrangeCXXDestructor(cast<CXXDestructorDecl>(MD), GD.getDtorType());
1017   else
1018     Info = &arrangeCXXMethodDeclaration(MD);
1019   return GetFunctionType(*Info);
1020 }
1021 
1022 namespace {
1023 
1024 /// Encapsulates information about the way function arguments from
1025 /// CGFunctionInfo should be passed to actual LLVM IR function.
1026 class ClangToLLVMArgMapping {
1027   static const unsigned InvalidIndex = ~0U;
1028   unsigned InallocaArgNo;
1029   unsigned SRetArgNo;
1030   unsigned TotalIRArgs;
1031 
1032   /// Arguments of LLVM IR function corresponding to single Clang argument.
1033   struct IRArgs {
1034     unsigned PaddingArgIndex;
1035     // Argument is expanded to IR arguments at positions
1036     // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1037     unsigned FirstArgIndex;
1038     unsigned NumberOfArgs;
1039 
1040     IRArgs()
1041         : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1042           NumberOfArgs(0) {}
1043   };
1044 
1045   SmallVector<IRArgs, 8> ArgInfo;
1046 
1047 public:
1048   ClangToLLVMArgMapping(CodeGenModule &CGM, const CGFunctionInfo &FI)
1049       : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1050         ArgInfo(FI.arg_size()) {
1051     construct(CGM, FI);
1052   }
1053 
1054   bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1055   unsigned getInallocaArgNo() const {
1056     assert(hasInallocaArg());
1057     return InallocaArgNo;
1058   }
1059 
1060   bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1061   unsigned getSRetArgNo() const {
1062     assert(hasSRetArg());
1063     return SRetArgNo;
1064   }
1065 
1066   unsigned totalIRArgs() const { return TotalIRArgs; }
1067 
1068   bool hasPaddingArg(unsigned ArgNo) const {
1069     assert(ArgNo < ArgInfo.size());
1070     return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1071   }
1072   unsigned getPaddingArgNo(unsigned ArgNo) const {
1073     assert(hasPaddingArg(ArgNo));
1074     return ArgInfo[ArgNo].PaddingArgIndex;
1075   }
1076 
1077   /// Returns index of first IR argument corresponding to ArgNo, and their
1078   /// quantity.
1079   std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1080     assert(ArgNo < ArgInfo.size());
1081     return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1082                           ArgInfo[ArgNo].NumberOfArgs);
1083   }
1084 
1085 private:
1086   void construct(CodeGenModule &CGM, const CGFunctionInfo &FI);
1087 };
1088 
1089 void ClangToLLVMArgMapping::construct(CodeGenModule &CGM,
1090                                       const CGFunctionInfo &FI) {
1091   unsigned IRArgNo = 0;
1092   bool SwapThisWithSRet = false;
1093   const ABIArgInfo &RetAI = FI.getReturnInfo();
1094 
1095   if (RetAI.getKind() == ABIArgInfo::Indirect) {
1096     SwapThisWithSRet = RetAI.isSRetAfterThis();
1097     SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1098   }
1099 
1100   unsigned ArgNo = 0;
1101   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
1102                                           E = FI.arg_end();
1103        I != E; ++I, ++ArgNo) {
1104     QualType ArgType = I->type;
1105     const ABIArgInfo &AI = I->info;
1106     // Collect data about IR arguments corresponding to Clang argument ArgNo.
1107     auto &IRArgs = ArgInfo[ArgNo];
1108 
1109     if (AI.getPaddingType())
1110       IRArgs.PaddingArgIndex = IRArgNo++;
1111 
1112     switch (AI.getKind()) {
1113     case ABIArgInfo::Extend:
1114     case ABIArgInfo::Direct: {
1115       // FIXME: handle sseregparm someday...
1116       llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1117       if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1118         IRArgs.NumberOfArgs = STy->getNumElements();
1119       } else {
1120         IRArgs.NumberOfArgs = 1;
1121       }
1122       break;
1123     }
1124     case ABIArgInfo::Indirect:
1125       IRArgs.NumberOfArgs = 1;
1126       break;
1127     case ABIArgInfo::Ignore:
1128     case ABIArgInfo::InAlloca:
1129       // ignore and inalloca doesn't have matching LLVM parameters.
1130       IRArgs.NumberOfArgs = 0;
1131       break;
1132     case ABIArgInfo::Expand: {
1133       SmallVector<llvm::Type*, 8> Types;
1134       // FIXME: This is rather inefficient. Do we ever actually need to do
1135       // anything here? The result should be just reconstructed on the other
1136       // side, so extension should be a non-issue.
1137       CGM.getTypes().GetExpandedTypes(ArgType, Types);
1138       IRArgs.NumberOfArgs = Types.size();
1139       break;
1140     }
1141     }
1142 
1143     if (IRArgs.NumberOfArgs > 0) {
1144       IRArgs.FirstArgIndex = IRArgNo;
1145       IRArgNo += IRArgs.NumberOfArgs;
1146     }
1147 
1148     // Skip over the sret parameter when it comes second.  We already handled it
1149     // above.
1150     if (IRArgNo == 1 && SwapThisWithSRet)
1151       IRArgNo++;
1152   }
1153   assert(ArgNo == FI.arg_size());
1154 
1155   if (FI.usesInAlloca())
1156     InallocaArgNo = IRArgNo++;
1157 
1158   TotalIRArgs = IRArgNo;
1159 }
1160 }  // namespace
1161 
1162 void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
1163                                            const Decl *TargetDecl,
1164                                            AttributeListType &PAL,
1165                                            unsigned &CallingConv,
1166                                            bool AttrOnCallSite) {
1167   llvm::AttrBuilder FuncAttrs;
1168   llvm::AttrBuilder RetAttrs;
1169 
1170   CallingConv = FI.getEffectiveCallingConvention();
1171 
1172   if (FI.isNoReturn())
1173     FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1174 
1175   // FIXME: handle sseregparm someday...
1176   if (TargetDecl) {
1177     if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1178       FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1179     if (TargetDecl->hasAttr<NoThrowAttr>())
1180       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1181     if (TargetDecl->hasAttr<NoReturnAttr>())
1182       FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1183     if (TargetDecl->hasAttr<NoDuplicateAttr>())
1184       FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1185 
1186     if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1187       const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
1188       if (FPT && FPT->isNothrow(getContext()))
1189         FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1190       // Don't use [[noreturn]] or _Noreturn for a call to a virtual function.
1191       // These attributes are not inherited by overloads.
1192       const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1193       if (Fn->isNoReturn() && !(AttrOnCallSite && MD && MD->isVirtual()))
1194         FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1195     }
1196 
1197     // 'const' and 'pure' attribute functions are also nounwind.
1198     if (TargetDecl->hasAttr<ConstAttr>()) {
1199       FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1200       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1201     } else if (TargetDecl->hasAttr<PureAttr>()) {
1202       FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1203       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1204     }
1205     if (TargetDecl->hasAttr<MallocAttr>())
1206       RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1207     if (TargetDecl->hasAttr<ReturnsNonNullAttr>())
1208       RetAttrs.addAttribute(llvm::Attribute::NonNull);
1209   }
1210 
1211   if (CodeGenOpts.OptimizeSize)
1212     FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1213   if (CodeGenOpts.OptimizeSize == 2)
1214     FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1215   if (CodeGenOpts.DisableRedZone)
1216     FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1217   if (CodeGenOpts.NoImplicitFloat)
1218     FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1219   if (CodeGenOpts.EnableSegmentedStacks &&
1220       !(TargetDecl && TargetDecl->hasAttr<NoSplitStackAttr>()))
1221     FuncAttrs.addAttribute("split-stack");
1222 
1223   if (AttrOnCallSite) {
1224     // Attributes that should go on the call site only.
1225     if (!CodeGenOpts.SimplifyLibCalls)
1226       FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1227   } else {
1228     // Attributes that should go on the function, but not the call site.
1229     if (!CodeGenOpts.DisableFPElim) {
1230       FuncAttrs.addAttribute("no-frame-pointer-elim", "false");
1231     } else if (CodeGenOpts.OmitLeafFramePointer) {
1232       FuncAttrs.addAttribute("no-frame-pointer-elim", "false");
1233       FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf");
1234     } else {
1235       FuncAttrs.addAttribute("no-frame-pointer-elim", "true");
1236       FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf");
1237     }
1238 
1239     FuncAttrs.addAttribute("less-precise-fpmad",
1240                            llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1241     FuncAttrs.addAttribute("no-infs-fp-math",
1242                            llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
1243     FuncAttrs.addAttribute("no-nans-fp-math",
1244                            llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
1245     FuncAttrs.addAttribute("unsafe-fp-math",
1246                            llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
1247     FuncAttrs.addAttribute("use-soft-float",
1248                            llvm::toStringRef(CodeGenOpts.SoftFloat));
1249     FuncAttrs.addAttribute("stack-protector-buffer-size",
1250                            llvm::utostr(CodeGenOpts.SSPBufferSize));
1251 
1252     if (!CodeGenOpts.StackRealignment)
1253       FuncAttrs.addAttribute("no-realign-stack");
1254   }
1255 
1256   ClangToLLVMArgMapping IRFunctionArgs(*this, FI);
1257 
1258   QualType RetTy = FI.getReturnType();
1259   const ABIArgInfo &RetAI = FI.getReturnInfo();
1260   switch (RetAI.getKind()) {
1261   case ABIArgInfo::Extend:
1262     if (RetTy->hasSignedIntegerRepresentation())
1263       RetAttrs.addAttribute(llvm::Attribute::SExt);
1264     else if (RetTy->hasUnsignedIntegerRepresentation())
1265       RetAttrs.addAttribute(llvm::Attribute::ZExt);
1266     // FALL THROUGH
1267   case ABIArgInfo::Direct:
1268     if (RetAI.getInReg())
1269       RetAttrs.addAttribute(llvm::Attribute::InReg);
1270     break;
1271   case ABIArgInfo::Ignore:
1272     break;
1273 
1274   case ABIArgInfo::InAlloca:
1275   case ABIArgInfo::Indirect: {
1276     // inalloca and sret disable readnone and readonly
1277     FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1278       .removeAttribute(llvm::Attribute::ReadNone);
1279     break;
1280   }
1281 
1282   case ABIArgInfo::Expand:
1283     llvm_unreachable("Invalid ABI kind for return argument");
1284   }
1285 
1286   if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
1287     QualType PTy = RefTy->getPointeeType();
1288     if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
1289       RetAttrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
1290                                         .getQuantity());
1291     else if (getContext().getTargetAddressSpace(PTy) == 0)
1292       RetAttrs.addAttribute(llvm::Attribute::NonNull);
1293   }
1294 
1295   // Attach return attributes.
1296   if (RetAttrs.hasAttributes()) {
1297     PAL.push_back(llvm::AttributeSet::get(
1298         getLLVMContext(), llvm::AttributeSet::ReturnIndex, RetAttrs));
1299   }
1300 
1301   // Attach attributes to sret.
1302   if (IRFunctionArgs.hasSRetArg()) {
1303     llvm::AttrBuilder SRETAttrs;
1304     SRETAttrs.addAttribute(llvm::Attribute::StructRet);
1305     if (RetAI.getInReg())
1306       SRETAttrs.addAttribute(llvm::Attribute::InReg);
1307     PAL.push_back(llvm::AttributeSet::get(
1308         getLLVMContext(), IRFunctionArgs.getSRetArgNo() + 1, SRETAttrs));
1309   }
1310 
1311   // Attach attributes to inalloca argument.
1312   if (IRFunctionArgs.hasInallocaArg()) {
1313     llvm::AttrBuilder Attrs;
1314     Attrs.addAttribute(llvm::Attribute::InAlloca);
1315     PAL.push_back(llvm::AttributeSet::get(
1316         getLLVMContext(), IRFunctionArgs.getInallocaArgNo() + 1, Attrs));
1317   }
1318 
1319 
1320   unsigned ArgNo = 0;
1321   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
1322                                           E = FI.arg_end();
1323        I != E; ++I, ++ArgNo) {
1324     QualType ParamType = I->type;
1325     const ABIArgInfo &AI = I->info;
1326     llvm::AttrBuilder Attrs;
1327 
1328     // Add attribute for padding argument, if necessary.
1329     if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
1330       if (AI.getPaddingInReg())
1331         PAL.push_back(llvm::AttributeSet::get(
1332             getLLVMContext(), IRFunctionArgs.getPaddingArgNo(ArgNo) + 1,
1333             llvm::Attribute::InReg));
1334     }
1335 
1336     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
1337     // have the corresponding parameter variable.  It doesn't make
1338     // sense to do it here because parameters are so messed up.
1339     switch (AI.getKind()) {
1340     case ABIArgInfo::Extend:
1341       if (ParamType->isSignedIntegerOrEnumerationType())
1342         Attrs.addAttribute(llvm::Attribute::SExt);
1343       else if (ParamType->isUnsignedIntegerOrEnumerationType())
1344         Attrs.addAttribute(llvm::Attribute::ZExt);
1345       // FALL THROUGH
1346     case ABIArgInfo::Direct:
1347       if (AI.getInReg())
1348         Attrs.addAttribute(llvm::Attribute::InReg);
1349       break;
1350 
1351     case ABIArgInfo::Indirect:
1352       if (AI.getInReg())
1353         Attrs.addAttribute(llvm::Attribute::InReg);
1354 
1355       if (AI.getIndirectByVal())
1356         Attrs.addAttribute(llvm::Attribute::ByVal);
1357 
1358       Attrs.addAlignmentAttr(AI.getIndirectAlign());
1359 
1360       // byval disables readnone and readonly.
1361       FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1362         .removeAttribute(llvm::Attribute::ReadNone);
1363       break;
1364 
1365     case ABIArgInfo::Ignore:
1366     case ABIArgInfo::Expand:
1367       continue;
1368 
1369     case ABIArgInfo::InAlloca:
1370       // inalloca disables readnone and readonly.
1371       FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1372           .removeAttribute(llvm::Attribute::ReadNone);
1373       continue;
1374     }
1375 
1376     if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
1377       QualType PTy = RefTy->getPointeeType();
1378       if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
1379         Attrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
1380                                        .getQuantity());
1381       else if (getContext().getTargetAddressSpace(PTy) == 0)
1382         Attrs.addAttribute(llvm::Attribute::NonNull);
1383     }
1384 
1385     if (Attrs.hasAttributes()) {
1386       unsigned FirstIRArg, NumIRArgs;
1387       std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1388       for (unsigned i = 0; i < NumIRArgs; i++)
1389         PAL.push_back(llvm::AttributeSet::get(getLLVMContext(),
1390                                               FirstIRArg + i + 1, Attrs));
1391     }
1392   }
1393   assert(ArgNo == FI.arg_size());
1394 
1395   if (FuncAttrs.hasAttributes())
1396     PAL.push_back(llvm::
1397                   AttributeSet::get(getLLVMContext(),
1398                                     llvm::AttributeSet::FunctionIndex,
1399                                     FuncAttrs));
1400 }
1401 
1402 /// An argument came in as a promoted argument; demote it back to its
1403 /// declared type.
1404 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
1405                                          const VarDecl *var,
1406                                          llvm::Value *value) {
1407   llvm::Type *varType = CGF.ConvertType(var->getType());
1408 
1409   // This can happen with promotions that actually don't change the
1410   // underlying type, like the enum promotions.
1411   if (value->getType() == varType) return value;
1412 
1413   assert((varType->isIntegerTy() || varType->isFloatingPointTy())
1414          && "unexpected promotion type");
1415 
1416   if (isa<llvm::IntegerType>(varType))
1417     return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
1418 
1419   return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
1420 }
1421 
1422 static bool shouldAddNonNullAttr(const Decl *FD, const ParmVarDecl *PVD) {
1423   // FIXME: __attribute__((nonnull)) can also be applied to:
1424   //   - references to pointers, where the pointee is known to be
1425   //     nonnull (apparently a Clang extension)
1426   //   - transparent unions containing pointers
1427   // In the former case, LLVM IR cannot represent the constraint. In
1428   // the latter case, we have no guarantee that the transparent union
1429   // is in fact passed as a pointer.
1430   if (!PVD->getType()->isAnyPointerType() &&
1431       !PVD->getType()->isBlockPointerType())
1432     return false;
1433   // First, check attribute on parameter itself.
1434   if (PVD->hasAttr<NonNullAttr>())
1435     return true;
1436   // Check function attributes.
1437   if (!FD)
1438     return false;
1439   for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
1440     if (NNAttr->isNonNull(PVD->getFunctionScopeIndex()))
1441       return true;
1442   }
1443   return false;
1444 }
1445 
1446 void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
1447                                          llvm::Function *Fn,
1448                                          const FunctionArgList &Args) {
1449   // If this is an implicit-return-zero function, go ahead and
1450   // initialize the return value.  TODO: it might be nice to have
1451   // a more general mechanism for this that didn't require synthesized
1452   // return statements.
1453   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
1454     if (FD->hasImplicitReturnZero()) {
1455       QualType RetTy = FD->getReturnType().getUnqualifiedType();
1456       llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
1457       llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
1458       Builder.CreateStore(Zero, ReturnValue);
1459     }
1460   }
1461 
1462   // FIXME: We no longer need the types from FunctionArgList; lift up and
1463   // simplify.
1464 
1465   ClangToLLVMArgMapping IRFunctionArgs(CGM, FI);
1466   // Flattened function arguments.
1467   SmallVector<llvm::Argument *, 16> FnArgs;
1468   FnArgs.reserve(IRFunctionArgs.totalIRArgs());
1469   for (auto &Arg : Fn->args()) {
1470     FnArgs.push_back(&Arg);
1471   }
1472   assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
1473 
1474   // If we're using inalloca, all the memory arguments are GEPs off of the last
1475   // parameter, which is a pointer to the complete memory area.
1476   llvm::Value *ArgStruct = nullptr;
1477   if (IRFunctionArgs.hasInallocaArg()) {
1478     ArgStruct = FnArgs[IRFunctionArgs.getInallocaArgNo()];
1479     assert(ArgStruct->getType() == FI.getArgStruct()->getPointerTo());
1480   }
1481 
1482   // Name the struct return parameter.
1483   if (IRFunctionArgs.hasSRetArg()) {
1484     auto AI = FnArgs[IRFunctionArgs.getSRetArgNo()];
1485     AI->setName("agg.result");
1486     AI->addAttr(llvm::AttributeSet::get(getLLVMContext(), AI->getArgNo() + 1,
1487                                         llvm::Attribute::NoAlias));
1488   }
1489 
1490   // Track if we received the parameter as a pointer (indirect, byval, or
1491   // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
1492   // into a local alloca for us.
1493   enum ValOrPointer { HaveValue = 0, HavePointer = 1 };
1494   typedef llvm::PointerIntPair<llvm::Value *, 1> ValueAndIsPtr;
1495   SmallVector<ValueAndIsPtr, 16> ArgVals;
1496   ArgVals.reserve(Args.size());
1497 
1498   // Create a pointer value for every parameter declaration.  This usually
1499   // entails copying one or more LLVM IR arguments into an alloca.  Don't push
1500   // any cleanups or do anything that might unwind.  We do that separately, so
1501   // we can push the cleanups in the correct order for the ABI.
1502   assert(FI.arg_size() == Args.size() &&
1503          "Mismatch between function signature & arguments.");
1504   unsigned ArgNo = 0;
1505   CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
1506   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1507        i != e; ++i, ++info_it, ++ArgNo) {
1508     const VarDecl *Arg = *i;
1509     QualType Ty = info_it->type;
1510     const ABIArgInfo &ArgI = info_it->info;
1511 
1512     bool isPromoted =
1513       isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
1514 
1515     unsigned FirstIRArg, NumIRArgs;
1516     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1517 
1518     switch (ArgI.getKind()) {
1519     case ABIArgInfo::InAlloca: {
1520       assert(NumIRArgs == 0);
1521       llvm::Value *V = Builder.CreateStructGEP(
1522           ArgStruct, ArgI.getInAllocaFieldIndex(), Arg->getName());
1523       ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1524       break;
1525     }
1526 
1527     case ABIArgInfo::Indirect: {
1528       assert(NumIRArgs == 1);
1529       llvm::Value *V = FnArgs[FirstIRArg];
1530 
1531       if (!hasScalarEvaluationKind(Ty)) {
1532         // Aggregates and complex variables are accessed by reference.  All we
1533         // need to do is realign the value, if requested
1534         if (ArgI.getIndirectRealign()) {
1535           llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
1536 
1537           // Copy from the incoming argument pointer to the temporary with the
1538           // appropriate alignment.
1539           //
1540           // FIXME: We should have a common utility for generating an aggregate
1541           // copy.
1542           llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
1543           CharUnits Size = getContext().getTypeSizeInChars(Ty);
1544           llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
1545           llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
1546           Builder.CreateMemCpy(Dst,
1547                                Src,
1548                                llvm::ConstantInt::get(IntPtrTy,
1549                                                       Size.getQuantity()),
1550                                ArgI.getIndirectAlign(),
1551                                false);
1552           V = AlignedTemp;
1553         }
1554         ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1555       } else {
1556         // Load scalar value from indirect argument.
1557         CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
1558         V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty,
1559                              Arg->getLocStart());
1560 
1561         if (isPromoted)
1562           V = emitArgumentDemotion(*this, Arg, V);
1563         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1564       }
1565       break;
1566     }
1567 
1568     case ABIArgInfo::Extend:
1569     case ABIArgInfo::Direct: {
1570 
1571       // If we have the trivial case, handle it with no muss and fuss.
1572       if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
1573           ArgI.getCoerceToType() == ConvertType(Ty) &&
1574           ArgI.getDirectOffset() == 0) {
1575         assert(NumIRArgs == 1);
1576         auto AI = FnArgs[FirstIRArg];
1577         llvm::Value *V = AI;
1578 
1579         if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
1580           if (shouldAddNonNullAttr(CurCodeDecl, PVD))
1581             AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1582                                                 AI->getArgNo() + 1,
1583                                                 llvm::Attribute::NonNull));
1584 
1585           QualType OTy = PVD->getOriginalType();
1586           if (const auto *ArrTy =
1587               getContext().getAsConstantArrayType(OTy)) {
1588             // A C99 array parameter declaration with the static keyword also
1589             // indicates dereferenceability, and if the size is constant we can
1590             // use the dereferenceable attribute (which requires the size in
1591             // bytes).
1592             if (ArrTy->getSizeModifier() == ArrayType::Static) {
1593               QualType ETy = ArrTy->getElementType();
1594               uint64_t ArrSize = ArrTy->getSize().getZExtValue();
1595               if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
1596                   ArrSize) {
1597                 llvm::AttrBuilder Attrs;
1598                 Attrs.addDereferenceableAttr(
1599                   getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
1600                 AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1601                                                     AI->getArgNo() + 1, Attrs));
1602               } else if (getContext().getTargetAddressSpace(ETy) == 0) {
1603                 AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1604                                                     AI->getArgNo() + 1,
1605                                                     llvm::Attribute::NonNull));
1606               }
1607             }
1608           } else if (const auto *ArrTy =
1609                      getContext().getAsVariableArrayType(OTy)) {
1610             // For C99 VLAs with the static keyword, we don't know the size so
1611             // we can't use the dereferenceable attribute, but in addrspace(0)
1612             // we know that it must be nonnull.
1613             if (ArrTy->getSizeModifier() == VariableArrayType::Static &&
1614                 !getContext().getTargetAddressSpace(ArrTy->getElementType()))
1615               AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1616                                                   AI->getArgNo() + 1,
1617                                                   llvm::Attribute::NonNull));
1618           }
1619         }
1620 
1621         if (Arg->getType().isRestrictQualified())
1622           AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1623                                               AI->getArgNo() + 1,
1624                                               llvm::Attribute::NoAlias));
1625 
1626         // Ensure the argument is the correct type.
1627         if (V->getType() != ArgI.getCoerceToType())
1628           V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
1629 
1630         if (isPromoted)
1631           V = emitArgumentDemotion(*this, Arg, V);
1632 
1633         if (const CXXMethodDecl *MD =
1634             dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl)) {
1635           if (MD->isVirtual() && Arg == CXXABIThisDecl)
1636             V = CGM.getCXXABI().
1637                 adjustThisParameterInVirtualFunctionPrologue(*this, CurGD, V);
1638         }
1639 
1640         // Because of merging of function types from multiple decls it is
1641         // possible for the type of an argument to not match the corresponding
1642         // type in the function type. Since we are codegening the callee
1643         // in here, add a cast to the argument type.
1644         llvm::Type *LTy = ConvertType(Arg->getType());
1645         if (V->getType() != LTy)
1646           V = Builder.CreateBitCast(V, LTy);
1647 
1648         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1649         break;
1650       }
1651 
1652       llvm::AllocaInst *Alloca = CreateMemTemp(Ty, Arg->getName());
1653 
1654       // The alignment we need to use is the max of the requested alignment for
1655       // the argument plus the alignment required by our access code below.
1656       unsigned AlignmentToUse =
1657         CGM.getDataLayout().getABITypeAlignment(ArgI.getCoerceToType());
1658       AlignmentToUse = std::max(AlignmentToUse,
1659                         (unsigned)getContext().getDeclAlign(Arg).getQuantity());
1660 
1661       Alloca->setAlignment(AlignmentToUse);
1662       llvm::Value *V = Alloca;
1663       llvm::Value *Ptr = V;    // Pointer to store into.
1664 
1665       // If the value is offset in memory, apply the offset now.
1666       if (unsigned Offs = ArgI.getDirectOffset()) {
1667         Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
1668         Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
1669         Ptr = Builder.CreateBitCast(Ptr,
1670                           llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
1671       }
1672 
1673       // Fast-isel and the optimizer generally like scalar values better than
1674       // FCAs, so we flatten them if this is safe to do for this argument.
1675       llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
1676       if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
1677           STy->getNumElements() > 1) {
1678         uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
1679         llvm::Type *DstTy =
1680           cast<llvm::PointerType>(Ptr->getType())->getElementType();
1681         uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
1682 
1683         if (SrcSize <= DstSize) {
1684           Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
1685 
1686           assert(STy->getNumElements() == NumIRArgs);
1687           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1688             auto AI = FnArgs[FirstIRArg + i];
1689             AI->setName(Arg->getName() + ".coerce" + Twine(i));
1690             llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
1691             Builder.CreateStore(AI, EltPtr);
1692           }
1693         } else {
1694           llvm::AllocaInst *TempAlloca =
1695             CreateTempAlloca(ArgI.getCoerceToType(), "coerce");
1696           TempAlloca->setAlignment(AlignmentToUse);
1697           llvm::Value *TempV = TempAlloca;
1698 
1699           assert(STy->getNumElements() == NumIRArgs);
1700           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1701             auto AI = FnArgs[FirstIRArg + i];
1702             AI->setName(Arg->getName() + ".coerce" + Twine(i));
1703             llvm::Value *EltPtr = Builder.CreateConstGEP2_32(TempV, 0, i);
1704             Builder.CreateStore(AI, EltPtr);
1705           }
1706 
1707           Builder.CreateMemCpy(Ptr, TempV, DstSize, AlignmentToUse);
1708         }
1709       } else {
1710         // Simple case, just do a coerced store of the argument into the alloca.
1711         assert(NumIRArgs == 1);
1712         auto AI = FnArgs[FirstIRArg];
1713         AI->setName(Arg->getName() + ".coerce");
1714         CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, *this);
1715       }
1716 
1717 
1718       // Match to what EmitParmDecl is expecting for this type.
1719       if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
1720         V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty, Arg->getLocStart());
1721         if (isPromoted)
1722           V = emitArgumentDemotion(*this, Arg, V);
1723         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1724       } else {
1725         ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1726       }
1727       break;
1728     }
1729 
1730     case ABIArgInfo::Expand: {
1731       // If this structure was expanded into multiple arguments then
1732       // we need to create a temporary and reconstruct it from the
1733       // arguments.
1734       llvm::AllocaInst *Alloca = CreateMemTemp(Ty);
1735       CharUnits Align = getContext().getDeclAlign(Arg);
1736       Alloca->setAlignment(Align.getQuantity());
1737       LValue LV = MakeAddrLValue(Alloca, Ty, Align);
1738       ArgVals.push_back(ValueAndIsPtr(Alloca, HavePointer));
1739 
1740       auto FnArgIter = FnArgs.begin() + FirstIRArg;
1741       ExpandTypeFromArgs(Ty, LV, FnArgIter);
1742       assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
1743       for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
1744         auto AI = FnArgs[FirstIRArg + i];
1745         AI->setName(Arg->getName() + "." + Twine(i));
1746       }
1747       break;
1748     }
1749 
1750     case ABIArgInfo::Ignore:
1751       assert(NumIRArgs == 0);
1752       // Initialize the local variable appropriately.
1753       if (!hasScalarEvaluationKind(Ty)) {
1754         ArgVals.push_back(ValueAndIsPtr(CreateMemTemp(Ty), HavePointer));
1755       } else {
1756         llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
1757         ArgVals.push_back(ValueAndIsPtr(U, HaveValue));
1758       }
1759       break;
1760     }
1761   }
1762 
1763   if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
1764     for (int I = Args.size() - 1; I >= 0; --I)
1765       EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
1766                    I + 1);
1767   } else {
1768     for (unsigned I = 0, E = Args.size(); I != E; ++I)
1769       EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
1770                    I + 1);
1771   }
1772 }
1773 
1774 static void eraseUnusedBitCasts(llvm::Instruction *insn) {
1775   while (insn->use_empty()) {
1776     llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
1777     if (!bitcast) return;
1778 
1779     // This is "safe" because we would have used a ConstantExpr otherwise.
1780     insn = cast<llvm::Instruction>(bitcast->getOperand(0));
1781     bitcast->eraseFromParent();
1782   }
1783 }
1784 
1785 /// Try to emit a fused autorelease of a return result.
1786 static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1787                                                     llvm::Value *result) {
1788   // We must be immediately followed the cast.
1789   llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1790   if (BB->empty()) return nullptr;
1791   if (&BB->back() != result) return nullptr;
1792 
1793   llvm::Type *resultType = result->getType();
1794 
1795   // result is in a BasicBlock and is therefore an Instruction.
1796   llvm::Instruction *generator = cast<llvm::Instruction>(result);
1797 
1798   SmallVector<llvm::Instruction*,4> insnsToKill;
1799 
1800   // Look for:
1801   //  %generator = bitcast %type1* %generator2 to %type2*
1802   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
1803     // We would have emitted this as a constant if the operand weren't
1804     // an Instruction.
1805     generator = cast<llvm::Instruction>(bitcast->getOperand(0));
1806 
1807     // Require the generator to be immediately followed by the cast.
1808     if (generator->getNextNode() != bitcast)
1809       return nullptr;
1810 
1811     insnsToKill.push_back(bitcast);
1812   }
1813 
1814   // Look for:
1815   //   %generator = call i8* @objc_retain(i8* %originalResult)
1816   // or
1817   //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
1818   llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
1819   if (!call) return nullptr;
1820 
1821   bool doRetainAutorelease;
1822 
1823   if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
1824     doRetainAutorelease = true;
1825   } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
1826                                           .objc_retainAutoreleasedReturnValue) {
1827     doRetainAutorelease = false;
1828 
1829     // If we emitted an assembly marker for this call (and the
1830     // ARCEntrypoints field should have been set if so), go looking
1831     // for that call.  If we can't find it, we can't do this
1832     // optimization.  But it should always be the immediately previous
1833     // instruction, unless we needed bitcasts around the call.
1834     if (CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker) {
1835       llvm::Instruction *prev = call->getPrevNode();
1836       assert(prev);
1837       if (isa<llvm::BitCastInst>(prev)) {
1838         prev = prev->getPrevNode();
1839         assert(prev);
1840       }
1841       assert(isa<llvm::CallInst>(prev));
1842       assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
1843                CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker);
1844       insnsToKill.push_back(prev);
1845     }
1846   } else {
1847     return nullptr;
1848   }
1849 
1850   result = call->getArgOperand(0);
1851   insnsToKill.push_back(call);
1852 
1853   // Keep killing bitcasts, for sanity.  Note that we no longer care
1854   // about precise ordering as long as there's exactly one use.
1855   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
1856     if (!bitcast->hasOneUse()) break;
1857     insnsToKill.push_back(bitcast);
1858     result = bitcast->getOperand(0);
1859   }
1860 
1861   // Delete all the unnecessary instructions, from latest to earliest.
1862   for (SmallVectorImpl<llvm::Instruction*>::iterator
1863          i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
1864     (*i)->eraseFromParent();
1865 
1866   // Do the fused retain/autorelease if we were asked to.
1867   if (doRetainAutorelease)
1868     result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
1869 
1870   // Cast back to the result type.
1871   return CGF.Builder.CreateBitCast(result, resultType);
1872 }
1873 
1874 /// If this is a +1 of the value of an immutable 'self', remove it.
1875 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
1876                                           llvm::Value *result) {
1877   // This is only applicable to a method with an immutable 'self'.
1878   const ObjCMethodDecl *method =
1879     dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
1880   if (!method) return nullptr;
1881   const VarDecl *self = method->getSelfDecl();
1882   if (!self->getType().isConstQualified()) return nullptr;
1883 
1884   // Look for a retain call.
1885   llvm::CallInst *retainCall =
1886     dyn_cast<llvm::CallInst>(result->stripPointerCasts());
1887   if (!retainCall ||
1888       retainCall->getCalledValue() != CGF.CGM.getARCEntrypoints().objc_retain)
1889     return nullptr;
1890 
1891   // Look for an ordinary load of 'self'.
1892   llvm::Value *retainedValue = retainCall->getArgOperand(0);
1893   llvm::LoadInst *load =
1894     dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
1895   if (!load || load->isAtomic() || load->isVolatile() ||
1896       load->getPointerOperand() != CGF.GetAddrOfLocalVar(self))
1897     return nullptr;
1898 
1899   // Okay!  Burn it all down.  This relies for correctness on the
1900   // assumption that the retain is emitted as part of the return and
1901   // that thereafter everything is used "linearly".
1902   llvm::Type *resultType = result->getType();
1903   eraseUnusedBitCasts(cast<llvm::Instruction>(result));
1904   assert(retainCall->use_empty());
1905   retainCall->eraseFromParent();
1906   eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
1907 
1908   return CGF.Builder.CreateBitCast(load, resultType);
1909 }
1910 
1911 /// Emit an ARC autorelease of the result of a function.
1912 ///
1913 /// \return the value to actually return from the function
1914 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
1915                                             llvm::Value *result) {
1916   // If we're returning 'self', kill the initial retain.  This is a
1917   // heuristic attempt to "encourage correctness" in the really unfortunate
1918   // case where we have a return of self during a dealloc and we desperately
1919   // need to avoid the possible autorelease.
1920   if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
1921     return self;
1922 
1923   // At -O0, try to emit a fused retain/autorelease.
1924   if (CGF.shouldUseFusedARCCalls())
1925     if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
1926       return fused;
1927 
1928   return CGF.EmitARCAutoreleaseReturnValue(result);
1929 }
1930 
1931 /// Heuristically search for a dominating store to the return-value slot.
1932 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
1933   // If there are multiple uses of the return-value slot, just check
1934   // for something immediately preceding the IP.  Sometimes this can
1935   // happen with how we generate implicit-returns; it can also happen
1936   // with noreturn cleanups.
1937   if (!CGF.ReturnValue->hasOneUse()) {
1938     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
1939     if (IP->empty()) return nullptr;
1940     llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(&IP->back());
1941     if (!store) return nullptr;
1942     if (store->getPointerOperand() != CGF.ReturnValue) return nullptr;
1943     assert(!store->isAtomic() && !store->isVolatile()); // see below
1944     return store;
1945   }
1946 
1947   llvm::StoreInst *store =
1948     dyn_cast<llvm::StoreInst>(CGF.ReturnValue->user_back());
1949   if (!store) return nullptr;
1950 
1951   // These aren't actually possible for non-coerced returns, and we
1952   // only care about non-coerced returns on this code path.
1953   assert(!store->isAtomic() && !store->isVolatile());
1954 
1955   // Now do a first-and-dirty dominance check: just walk up the
1956   // single-predecessors chain from the current insertion point.
1957   llvm::BasicBlock *StoreBB = store->getParent();
1958   llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
1959   while (IP != StoreBB) {
1960     if (!(IP = IP->getSinglePredecessor()))
1961       return nullptr;
1962   }
1963 
1964   // Okay, the store's basic block dominates the insertion point; we
1965   // can do our thing.
1966   return store;
1967 }
1968 
1969 void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
1970                                          bool EmitRetDbgLoc,
1971                                          SourceLocation EndLoc) {
1972   // Functions with no result always return void.
1973   if (!ReturnValue) {
1974     Builder.CreateRetVoid();
1975     return;
1976   }
1977 
1978   llvm::DebugLoc RetDbgLoc;
1979   llvm::Value *RV = nullptr;
1980   QualType RetTy = FI.getReturnType();
1981   const ABIArgInfo &RetAI = FI.getReturnInfo();
1982 
1983   switch (RetAI.getKind()) {
1984   case ABIArgInfo::InAlloca:
1985     // Aggregrates get evaluated directly into the destination.  Sometimes we
1986     // need to return the sret value in a register, though.
1987     assert(hasAggregateEvaluationKind(RetTy));
1988     if (RetAI.getInAllocaSRet()) {
1989       llvm::Function::arg_iterator EI = CurFn->arg_end();
1990       --EI;
1991       llvm::Value *ArgStruct = EI;
1992       llvm::Value *SRet =
1993           Builder.CreateStructGEP(ArgStruct, RetAI.getInAllocaFieldIndex());
1994       RV = Builder.CreateLoad(SRet, "sret");
1995     }
1996     break;
1997 
1998   case ABIArgInfo::Indirect: {
1999     auto AI = CurFn->arg_begin();
2000     if (RetAI.isSRetAfterThis())
2001       ++AI;
2002     switch (getEvaluationKind(RetTy)) {
2003     case TEK_Complex: {
2004       ComplexPairTy RT =
2005         EmitLoadOfComplex(MakeNaturalAlignAddrLValue(ReturnValue, RetTy),
2006                           EndLoc);
2007       EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(AI, RetTy),
2008                          /*isInit*/ true);
2009       break;
2010     }
2011     case TEK_Aggregate:
2012       // Do nothing; aggregrates get evaluated directly into the destination.
2013       break;
2014     case TEK_Scalar:
2015       EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
2016                         MakeNaturalAlignAddrLValue(AI, RetTy),
2017                         /*isInit*/ true);
2018       break;
2019     }
2020     break;
2021   }
2022 
2023   case ABIArgInfo::Extend:
2024   case ABIArgInfo::Direct:
2025     if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
2026         RetAI.getDirectOffset() == 0) {
2027       // The internal return value temp always will have pointer-to-return-type
2028       // type, just do a load.
2029 
2030       // If there is a dominating store to ReturnValue, we can elide
2031       // the load, zap the store, and usually zap the alloca.
2032       if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) {
2033         // Reuse the debug location from the store unless there is
2034         // cleanup code to be emitted between the store and return
2035         // instruction.
2036         if (EmitRetDbgLoc && !AutoreleaseResult)
2037           RetDbgLoc = SI->getDebugLoc();
2038         // Get the stored value and nuke the now-dead store.
2039         RV = SI->getValueOperand();
2040         SI->eraseFromParent();
2041 
2042         // If that was the only use of the return value, nuke it as well now.
2043         if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
2044           cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
2045           ReturnValue = nullptr;
2046         }
2047 
2048       // Otherwise, we have to do a simple load.
2049       } else {
2050         RV = Builder.CreateLoad(ReturnValue);
2051       }
2052     } else {
2053       llvm::Value *V = ReturnValue;
2054       // If the value is offset in memory, apply the offset now.
2055       if (unsigned Offs = RetAI.getDirectOffset()) {
2056         V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
2057         V = Builder.CreateConstGEP1_32(V, Offs);
2058         V = Builder.CreateBitCast(V,
2059                          llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
2060       }
2061 
2062       RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
2063     }
2064 
2065     // In ARC, end functions that return a retainable type with a call
2066     // to objc_autoreleaseReturnValue.
2067     if (AutoreleaseResult) {
2068       assert(getLangOpts().ObjCAutoRefCount &&
2069              !FI.isReturnsRetained() &&
2070              RetTy->isObjCRetainableType());
2071       RV = emitAutoreleaseOfResult(*this, RV);
2072     }
2073 
2074     break;
2075 
2076   case ABIArgInfo::Ignore:
2077     break;
2078 
2079   case ABIArgInfo::Expand:
2080     llvm_unreachable("Invalid ABI kind for return argument");
2081   }
2082 
2083   llvm::Instruction *Ret;
2084   if (RV) {
2085     if (SanOpts->ReturnsNonnullAttribute &&
2086         CurGD.getDecl()->hasAttr<ReturnsNonNullAttr>()) {
2087       SanitizerScope SanScope(this);
2088       llvm::Value *Cond =
2089           Builder.CreateICmpNE(RV, llvm::Constant::getNullValue(RV->getType()));
2090       llvm::Constant *StaticData[] = {
2091         EmitCheckSourceLocation(EndLoc)
2092       };
2093       EmitCheck(Cond, "nonnull_return", StaticData, None, CRK_Recoverable);
2094     }
2095     Ret = Builder.CreateRet(RV);
2096   } else {
2097     Ret = Builder.CreateRetVoid();
2098   }
2099 
2100   if (!RetDbgLoc.isUnknown())
2101     Ret->setDebugLoc(RetDbgLoc);
2102 }
2103 
2104 static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
2105   const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
2106   return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
2107 }
2108 
2109 static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
2110   // FIXME: Generate IR in one pass, rather than going back and fixing up these
2111   // placeholders.
2112   llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
2113   llvm::Value *Placeholder =
2114       llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo());
2115   Placeholder = CGF.Builder.CreateLoad(Placeholder);
2116   return AggValueSlot::forAddr(Placeholder, CharUnits::Zero(),
2117                                Ty.getQualifiers(),
2118                                AggValueSlot::IsNotDestructed,
2119                                AggValueSlot::DoesNotNeedGCBarriers,
2120                                AggValueSlot::IsNotAliased);
2121 }
2122 
2123 void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
2124                                           const VarDecl *param,
2125                                           SourceLocation loc) {
2126   // StartFunction converted the ABI-lowered parameter(s) into a
2127   // local alloca.  We need to turn that into an r-value suitable
2128   // for EmitCall.
2129   llvm::Value *local = GetAddrOfLocalVar(param);
2130 
2131   QualType type = param->getType();
2132 
2133   // For the most part, we just need to load the alloca, except:
2134   // 1) aggregate r-values are actually pointers to temporaries, and
2135   // 2) references to non-scalars are pointers directly to the aggregate.
2136   // I don't know why references to scalars are different here.
2137   if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
2138     if (!hasScalarEvaluationKind(ref->getPointeeType()))
2139       return args.add(RValue::getAggregate(local), type);
2140 
2141     // Locals which are references to scalars are represented
2142     // with allocas holding the pointer.
2143     return args.add(RValue::get(Builder.CreateLoad(local)), type);
2144   }
2145 
2146   assert(!isInAllocaArgument(CGM.getCXXABI(), type) &&
2147          "cannot emit delegate call arguments for inalloca arguments!");
2148 
2149   args.add(convertTempToRValue(local, type, loc), type);
2150 }
2151 
2152 static bool isProvablyNull(llvm::Value *addr) {
2153   return isa<llvm::ConstantPointerNull>(addr);
2154 }
2155 
2156 static bool isProvablyNonNull(llvm::Value *addr) {
2157   return isa<llvm::AllocaInst>(addr);
2158 }
2159 
2160 /// Emit the actual writing-back of a writeback.
2161 static void emitWriteback(CodeGenFunction &CGF,
2162                           const CallArgList::Writeback &writeback) {
2163   const LValue &srcLV = writeback.Source;
2164   llvm::Value *srcAddr = srcLV.getAddress();
2165   assert(!isProvablyNull(srcAddr) &&
2166          "shouldn't have writeback for provably null argument");
2167 
2168   llvm::BasicBlock *contBB = nullptr;
2169 
2170   // If the argument wasn't provably non-null, we need to null check
2171   // before doing the store.
2172   bool provablyNonNull = isProvablyNonNull(srcAddr);
2173   if (!provablyNonNull) {
2174     llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
2175     contBB = CGF.createBasicBlock("icr.done");
2176 
2177     llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
2178     CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
2179     CGF.EmitBlock(writebackBB);
2180   }
2181 
2182   // Load the value to writeback.
2183   llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
2184 
2185   // Cast it back, in case we're writing an id to a Foo* or something.
2186   value = CGF.Builder.CreateBitCast(value,
2187                cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
2188                             "icr.writeback-cast");
2189 
2190   // Perform the writeback.
2191 
2192   // If we have a "to use" value, it's something we need to emit a use
2193   // of.  This has to be carefully threaded in: if it's done after the
2194   // release it's potentially undefined behavior (and the optimizer
2195   // will ignore it), and if it happens before the retain then the
2196   // optimizer could move the release there.
2197   if (writeback.ToUse) {
2198     assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
2199 
2200     // Retain the new value.  No need to block-copy here:  the block's
2201     // being passed up the stack.
2202     value = CGF.EmitARCRetainNonBlock(value);
2203 
2204     // Emit the intrinsic use here.
2205     CGF.EmitARCIntrinsicUse(writeback.ToUse);
2206 
2207     // Load the old value (primitively).
2208     llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
2209 
2210     // Put the new value in place (primitively).
2211     CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
2212 
2213     // Release the old value.
2214     CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
2215 
2216   // Otherwise, we can just do a normal lvalue store.
2217   } else {
2218     CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
2219   }
2220 
2221   // Jump to the continuation block.
2222   if (!provablyNonNull)
2223     CGF.EmitBlock(contBB);
2224 }
2225 
2226 static void emitWritebacks(CodeGenFunction &CGF,
2227                            const CallArgList &args) {
2228   for (const auto &I : args.writebacks())
2229     emitWriteback(CGF, I);
2230 }
2231 
2232 static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
2233                                             const CallArgList &CallArgs) {
2234   assert(CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee());
2235   ArrayRef<CallArgList::CallArgCleanup> Cleanups =
2236     CallArgs.getCleanupsToDeactivate();
2237   // Iterate in reverse to increase the likelihood of popping the cleanup.
2238   for (ArrayRef<CallArgList::CallArgCleanup>::reverse_iterator
2239          I = Cleanups.rbegin(), E = Cleanups.rend(); I != E; ++I) {
2240     CGF.DeactivateCleanupBlock(I->Cleanup, I->IsActiveIP);
2241     I->IsActiveIP->eraseFromParent();
2242   }
2243 }
2244 
2245 static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
2246   if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
2247     if (uop->getOpcode() == UO_AddrOf)
2248       return uop->getSubExpr();
2249   return nullptr;
2250 }
2251 
2252 /// Emit an argument that's being passed call-by-writeback.  That is,
2253 /// we are passing the address of
2254 static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
2255                              const ObjCIndirectCopyRestoreExpr *CRE) {
2256   LValue srcLV;
2257 
2258   // Make an optimistic effort to emit the address as an l-value.
2259   // This can fail if the the argument expression is more complicated.
2260   if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
2261     srcLV = CGF.EmitLValue(lvExpr);
2262 
2263   // Otherwise, just emit it as a scalar.
2264   } else {
2265     llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
2266 
2267     QualType srcAddrType =
2268       CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
2269     srcLV = CGF.MakeNaturalAlignAddrLValue(srcAddr, srcAddrType);
2270   }
2271   llvm::Value *srcAddr = srcLV.getAddress();
2272 
2273   // The dest and src types don't necessarily match in LLVM terms
2274   // because of the crazy ObjC compatibility rules.
2275 
2276   llvm::PointerType *destType =
2277     cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
2278 
2279   // If the address is a constant null, just pass the appropriate null.
2280   if (isProvablyNull(srcAddr)) {
2281     args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
2282              CRE->getType());
2283     return;
2284   }
2285 
2286   // Create the temporary.
2287   llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
2288                                            "icr.temp");
2289   // Loading an l-value can introduce a cleanup if the l-value is __weak,
2290   // and that cleanup will be conditional if we can't prove that the l-value
2291   // isn't null, so we need to register a dominating point so that the cleanups
2292   // system will make valid IR.
2293   CodeGenFunction::ConditionalEvaluation condEval(CGF);
2294 
2295   // Zero-initialize it if we're not doing a copy-initialization.
2296   bool shouldCopy = CRE->shouldCopy();
2297   if (!shouldCopy) {
2298     llvm::Value *null =
2299       llvm::ConstantPointerNull::get(
2300         cast<llvm::PointerType>(destType->getElementType()));
2301     CGF.Builder.CreateStore(null, temp);
2302   }
2303 
2304   llvm::BasicBlock *contBB = nullptr;
2305   llvm::BasicBlock *originBB = nullptr;
2306 
2307   // If the address is *not* known to be non-null, we need to switch.
2308   llvm::Value *finalArgument;
2309 
2310   bool provablyNonNull = isProvablyNonNull(srcAddr);
2311   if (provablyNonNull) {
2312     finalArgument = temp;
2313   } else {
2314     llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
2315 
2316     finalArgument = CGF.Builder.CreateSelect(isNull,
2317                                    llvm::ConstantPointerNull::get(destType),
2318                                              temp, "icr.argument");
2319 
2320     // If we need to copy, then the load has to be conditional, which
2321     // means we need control flow.
2322     if (shouldCopy) {
2323       originBB = CGF.Builder.GetInsertBlock();
2324       contBB = CGF.createBasicBlock("icr.cont");
2325       llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
2326       CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
2327       CGF.EmitBlock(copyBB);
2328       condEval.begin(CGF);
2329     }
2330   }
2331 
2332   llvm::Value *valueToUse = nullptr;
2333 
2334   // Perform a copy if necessary.
2335   if (shouldCopy) {
2336     RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
2337     assert(srcRV.isScalar());
2338 
2339     llvm::Value *src = srcRV.getScalarVal();
2340     src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
2341                                     "icr.cast");
2342 
2343     // Use an ordinary store, not a store-to-lvalue.
2344     CGF.Builder.CreateStore(src, temp);
2345 
2346     // If optimization is enabled, and the value was held in a
2347     // __strong variable, we need to tell the optimizer that this
2348     // value has to stay alive until we're doing the store back.
2349     // This is because the temporary is effectively unretained,
2350     // and so otherwise we can violate the high-level semantics.
2351     if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
2352         srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
2353       valueToUse = src;
2354     }
2355   }
2356 
2357   // Finish the control flow if we needed it.
2358   if (shouldCopy && !provablyNonNull) {
2359     llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
2360     CGF.EmitBlock(contBB);
2361 
2362     // Make a phi for the value to intrinsically use.
2363     if (valueToUse) {
2364       llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
2365                                                       "icr.to-use");
2366       phiToUse->addIncoming(valueToUse, copyBB);
2367       phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
2368                             originBB);
2369       valueToUse = phiToUse;
2370     }
2371 
2372     condEval.end(CGF);
2373   }
2374 
2375   args.addWriteback(srcLV, temp, valueToUse);
2376   args.add(RValue::get(finalArgument), CRE->getType());
2377 }
2378 
2379 void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
2380   assert(!StackBase && !StackCleanup.isValid());
2381 
2382   // Save the stack.
2383   llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
2384   StackBase = CGF.Builder.CreateCall(F, "inalloca.save");
2385 
2386   // Control gets really tied up in landing pads, so we have to spill the
2387   // stacksave to an alloca to avoid violating SSA form.
2388   // TODO: This is dead if we never emit the cleanup.  We should create the
2389   // alloca and store lazily on the first cleanup emission.
2390   StackBaseMem = CGF.CreateTempAlloca(CGF.Int8PtrTy, "inalloca.spmem");
2391   CGF.Builder.CreateStore(StackBase, StackBaseMem);
2392   CGF.pushStackRestore(EHCleanup, StackBaseMem);
2393   StackCleanup = CGF.EHStack.getInnermostEHScope();
2394   assert(StackCleanup.isValid());
2395 }
2396 
2397 void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
2398   if (StackBase) {
2399     CGF.DeactivateCleanupBlock(StackCleanup, StackBase);
2400     llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
2401     // We could load StackBase from StackBaseMem, but in the non-exceptional
2402     // case we can skip it.
2403     CGF.Builder.CreateCall(F, StackBase);
2404   }
2405 }
2406 
2407 void CodeGenFunction::EmitCallArgs(CallArgList &Args,
2408                                    ArrayRef<QualType> ArgTypes,
2409                                    CallExpr::const_arg_iterator ArgBeg,
2410                                    CallExpr::const_arg_iterator ArgEnd,
2411                                    bool ForceColumnInfo) {
2412   CGDebugInfo *DI = getDebugInfo();
2413   SourceLocation CallLoc;
2414   if (DI) CallLoc = DI->getLocation();
2415 
2416   // We *have* to evaluate arguments from right to left in the MS C++ ABI,
2417   // because arguments are destroyed left to right in the callee.
2418   if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2419     // Insert a stack save if we're going to need any inalloca args.
2420     bool HasInAllocaArgs = false;
2421     for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
2422          I != E && !HasInAllocaArgs; ++I)
2423       HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
2424     if (HasInAllocaArgs) {
2425       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
2426       Args.allocateArgumentMemory(*this);
2427     }
2428 
2429     // Evaluate each argument.
2430     size_t CallArgsStart = Args.size();
2431     for (int I = ArgTypes.size() - 1; I >= 0; --I) {
2432       CallExpr::const_arg_iterator Arg = ArgBeg + I;
2433       EmitCallArg(Args, *Arg, ArgTypes[I]);
2434       // Restore the debug location.
2435       if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
2436     }
2437 
2438     // Un-reverse the arguments we just evaluated so they match up with the LLVM
2439     // IR function.
2440     std::reverse(Args.begin() + CallArgsStart, Args.end());
2441     return;
2442   }
2443 
2444   for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
2445     CallExpr::const_arg_iterator Arg = ArgBeg + I;
2446     assert(Arg != ArgEnd);
2447     EmitCallArg(Args, *Arg, ArgTypes[I]);
2448     // Restore the debug location.
2449     if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
2450   }
2451 }
2452 
2453 namespace {
2454 
2455 struct DestroyUnpassedArg : EHScopeStack::Cleanup {
2456   DestroyUnpassedArg(llvm::Value *Addr, QualType Ty)
2457       : Addr(Addr), Ty(Ty) {}
2458 
2459   llvm::Value *Addr;
2460   QualType Ty;
2461 
2462   void Emit(CodeGenFunction &CGF, Flags flags) override {
2463     const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
2464     assert(!Dtor->isTrivial());
2465     CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
2466                               /*Delegating=*/false, Addr);
2467   }
2468 };
2469 
2470 }
2471 
2472 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
2473                                   QualType type) {
2474   if (const ObjCIndirectCopyRestoreExpr *CRE
2475         = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
2476     assert(getLangOpts().ObjCAutoRefCount);
2477     assert(getContext().hasSameType(E->getType(), type));
2478     return emitWritebackArg(*this, args, CRE);
2479   }
2480 
2481   assert(type->isReferenceType() == E->isGLValue() &&
2482          "reference binding to unmaterialized r-value!");
2483 
2484   if (E->isGLValue()) {
2485     assert(E->getObjectKind() == OK_Ordinary);
2486     return args.add(EmitReferenceBindingToExpr(E), type);
2487   }
2488 
2489   bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
2490 
2491   // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
2492   // However, we still have to push an EH-only cleanup in case we unwind before
2493   // we make it to the call.
2494   if (HasAggregateEvalKind &&
2495       CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2496     // If we're using inalloca, use the argument memory.  Otherwise, use a
2497     // temporary.
2498     AggValueSlot Slot;
2499     if (args.isUsingInAlloca())
2500       Slot = createPlaceholderSlot(*this, type);
2501     else
2502       Slot = CreateAggTemp(type, "agg.tmp");
2503 
2504     const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
2505     bool DestroyedInCallee =
2506         RD && RD->hasNonTrivialDestructor() &&
2507         CGM.getCXXABI().getRecordArgABI(RD) != CGCXXABI::RAA_Default;
2508     if (DestroyedInCallee)
2509       Slot.setExternallyDestructed();
2510 
2511     EmitAggExpr(E, Slot);
2512     RValue RV = Slot.asRValue();
2513     args.add(RV, type);
2514 
2515     if (DestroyedInCallee) {
2516       // Create a no-op GEP between the placeholder and the cleanup so we can
2517       // RAUW it successfully.  It also serves as a marker of the first
2518       // instruction where the cleanup is active.
2519       pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddr(), type);
2520       // This unreachable is a temporary marker which will be removed later.
2521       llvm::Instruction *IsActive = Builder.CreateUnreachable();
2522       args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
2523     }
2524     return;
2525   }
2526 
2527   if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
2528       cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
2529     LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
2530     assert(L.isSimple());
2531     if (L.getAlignment() >= getContext().getTypeAlignInChars(type)) {
2532       args.add(L.asAggregateRValue(), type, /*NeedsCopy*/true);
2533     } else {
2534       // We can't represent a misaligned lvalue in the CallArgList, so copy
2535       // to an aligned temporary now.
2536       llvm::Value *tmp = CreateMemTemp(type);
2537       EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile(),
2538                         L.getAlignment());
2539       args.add(RValue::getAggregate(tmp), type);
2540     }
2541     return;
2542   }
2543 
2544   args.add(EmitAnyExprToTemp(E), type);
2545 }
2546 
2547 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
2548 // optimizer it can aggressively ignore unwind edges.
2549 void
2550 CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
2551   if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
2552       !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
2553     Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
2554                       CGM.getNoObjCARCExceptionsMetadata());
2555 }
2556 
2557 /// Emits a call to the given no-arguments nounwind runtime function.
2558 llvm::CallInst *
2559 CodeGenFunction::EmitNounwindRuntimeCall(llvm::Value *callee,
2560                                          const llvm::Twine &name) {
2561   return EmitNounwindRuntimeCall(callee, None, name);
2562 }
2563 
2564 /// Emits a call to the given nounwind runtime function.
2565 llvm::CallInst *
2566 CodeGenFunction::EmitNounwindRuntimeCall(llvm::Value *callee,
2567                                          ArrayRef<llvm::Value*> args,
2568                                          const llvm::Twine &name) {
2569   llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
2570   call->setDoesNotThrow();
2571   return call;
2572 }
2573 
2574 /// Emits a simple call (never an invoke) to the given no-arguments
2575 /// runtime function.
2576 llvm::CallInst *
2577 CodeGenFunction::EmitRuntimeCall(llvm::Value *callee,
2578                                  const llvm::Twine &name) {
2579   return EmitRuntimeCall(callee, None, name);
2580 }
2581 
2582 /// Emits a simple call (never an invoke) to the given runtime
2583 /// function.
2584 llvm::CallInst *
2585 CodeGenFunction::EmitRuntimeCall(llvm::Value *callee,
2586                                  ArrayRef<llvm::Value*> args,
2587                                  const llvm::Twine &name) {
2588   llvm::CallInst *call = Builder.CreateCall(callee, args, name);
2589   call->setCallingConv(getRuntimeCC());
2590   return call;
2591 }
2592 
2593 /// Emits a call or invoke to the given noreturn runtime function.
2594 void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
2595                                                ArrayRef<llvm::Value*> args) {
2596   if (getInvokeDest()) {
2597     llvm::InvokeInst *invoke =
2598       Builder.CreateInvoke(callee,
2599                            getUnreachableBlock(),
2600                            getInvokeDest(),
2601                            args);
2602     invoke->setDoesNotReturn();
2603     invoke->setCallingConv(getRuntimeCC());
2604   } else {
2605     llvm::CallInst *call = Builder.CreateCall(callee, args);
2606     call->setDoesNotReturn();
2607     call->setCallingConv(getRuntimeCC());
2608     Builder.CreateUnreachable();
2609   }
2610   PGO.setCurrentRegionUnreachable();
2611 }
2612 
2613 /// Emits a call or invoke instruction to the given nullary runtime
2614 /// function.
2615 llvm::CallSite
2616 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::Value *callee,
2617                                          const Twine &name) {
2618   return EmitRuntimeCallOrInvoke(callee, None, name);
2619 }
2620 
2621 /// Emits a call or invoke instruction to the given runtime function.
2622 llvm::CallSite
2623 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::Value *callee,
2624                                          ArrayRef<llvm::Value*> args,
2625                                          const Twine &name) {
2626   llvm::CallSite callSite = EmitCallOrInvoke(callee, args, name);
2627   callSite.setCallingConv(getRuntimeCC());
2628   return callSite;
2629 }
2630 
2631 llvm::CallSite
2632 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
2633                                   const Twine &Name) {
2634   return EmitCallOrInvoke(Callee, None, Name);
2635 }
2636 
2637 /// Emits a call or invoke instruction to the given function, depending
2638 /// on the current state of the EH stack.
2639 llvm::CallSite
2640 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
2641                                   ArrayRef<llvm::Value *> Args,
2642                                   const Twine &Name) {
2643   llvm::BasicBlock *InvokeDest = getInvokeDest();
2644 
2645   llvm::Instruction *Inst;
2646   if (!InvokeDest)
2647     Inst = Builder.CreateCall(Callee, Args, Name);
2648   else {
2649     llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
2650     Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, Name);
2651     EmitBlock(ContBB);
2652   }
2653 
2654   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
2655   // optimizer it can aggressively ignore unwind edges.
2656   if (CGM.getLangOpts().ObjCAutoRefCount)
2657     AddObjCARCExceptionMetadata(Inst);
2658 
2659   return Inst;
2660 }
2661 
2662 void CodeGenFunction::ExpandTypeToArgs(
2663     QualType Ty, RValue RV, llvm::FunctionType *IRFuncTy,
2664     SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
2665   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
2666     unsigned NumElts = AT->getSize().getZExtValue();
2667     QualType EltTy = AT->getElementType();
2668     llvm::Value *Addr = RV.getAggregateAddr();
2669     for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
2670       llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
2671       RValue EltRV = convertTempToRValue(EltAddr, EltTy, SourceLocation());
2672       ExpandTypeToArgs(EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos);
2673     }
2674   } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
2675     RecordDecl *RD = RT->getDecl();
2676     assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
2677     LValue LV = MakeAddrLValue(RV.getAggregateAddr(), Ty);
2678 
2679     if (RD->isUnion()) {
2680       const FieldDecl *LargestFD = nullptr;
2681       CharUnits UnionSize = CharUnits::Zero();
2682 
2683       for (const auto *FD : RD->fields()) {
2684         assert(!FD->isBitField() &&
2685                "Cannot expand structure with bit-field members.");
2686         CharUnits FieldSize = getContext().getTypeSizeInChars(FD->getType());
2687         if (UnionSize < FieldSize) {
2688           UnionSize = FieldSize;
2689           LargestFD = FD;
2690         }
2691       }
2692       if (LargestFD) {
2693         RValue FldRV = EmitRValueForField(LV, LargestFD, SourceLocation());
2694         ExpandTypeToArgs(LargestFD->getType(), FldRV, IRFuncTy, IRCallArgs,
2695                          IRCallArgPos);
2696       }
2697     } else {
2698       for (const auto *FD : RD->fields()) {
2699         RValue FldRV = EmitRValueForField(LV, FD, SourceLocation());
2700         ExpandTypeToArgs(FD->getType(), FldRV, IRFuncTy, IRCallArgs, IRCallArgPos);
2701       }
2702     }
2703   } else if (Ty->isAnyComplexType()) {
2704     ComplexPairTy CV = RV.getComplexVal();
2705     IRCallArgs[IRCallArgPos++] = CV.first;
2706     IRCallArgs[IRCallArgPos++] = CV.second;
2707   } else {
2708     assert(RV.isScalar() &&
2709            "Unexpected non-scalar rvalue during struct expansion.");
2710 
2711     // Insert a bitcast as needed.
2712     llvm::Value *V = RV.getScalarVal();
2713     if (IRCallArgPos < IRFuncTy->getNumParams() &&
2714         V->getType() != IRFuncTy->getParamType(IRCallArgPos))
2715       V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
2716 
2717     IRCallArgs[IRCallArgPos++] = V;
2718   }
2719 }
2720 
2721 /// \brief Store a non-aggregate value to an address to initialize it.  For
2722 /// initialization, a non-atomic store will be used.
2723 static void EmitInitStoreOfNonAggregate(CodeGenFunction &CGF, RValue Src,
2724                                         LValue Dst) {
2725   if (Src.isScalar())
2726     CGF.EmitStoreOfScalar(Src.getScalarVal(), Dst, /*init=*/true);
2727   else
2728     CGF.EmitStoreOfComplex(Src.getComplexVal(), Dst, /*init=*/true);
2729 }
2730 
2731 void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
2732                                                   llvm::Value *New) {
2733   DeferredReplacements.push_back(std::make_pair(Old, New));
2734 }
2735 
2736 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
2737                                  llvm::Value *Callee,
2738                                  ReturnValueSlot ReturnValue,
2739                                  const CallArgList &CallArgs,
2740                                  const Decl *TargetDecl,
2741                                  llvm::Instruction **callOrInvoke) {
2742   // FIXME: We no longer need the types from CallArgs; lift up and simplify.
2743 
2744   // Handle struct-return functions by passing a pointer to the
2745   // location that we would like to return into.
2746   QualType RetTy = CallInfo.getReturnType();
2747   const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
2748 
2749   llvm::FunctionType *IRFuncTy =
2750     cast<llvm::FunctionType>(
2751                   cast<llvm::PointerType>(Callee->getType())->getElementType());
2752 
2753   // If we're using inalloca, insert the allocation after the stack save.
2754   // FIXME: Do this earlier rather than hacking it in here!
2755   llvm::Value *ArgMemory = nullptr;
2756   if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
2757     llvm::Instruction *IP = CallArgs.getStackBase();
2758     llvm::AllocaInst *AI;
2759     if (IP) {
2760       IP = IP->getNextNode();
2761       AI = new llvm::AllocaInst(ArgStruct, "argmem", IP);
2762     } else {
2763       AI = CreateTempAlloca(ArgStruct, "argmem");
2764     }
2765     AI->setUsedWithInAlloca(true);
2766     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
2767     ArgMemory = AI;
2768   }
2769 
2770   ClangToLLVMArgMapping IRFunctionArgs(CGM, CallInfo);
2771   SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
2772 
2773   // If the call returns a temporary with struct return, create a temporary
2774   // alloca to hold the result, unless one is given to us.
2775   llvm::Value *SRetPtr = nullptr;
2776   if (RetAI.isIndirect() || RetAI.isInAlloca()) {
2777     SRetPtr = ReturnValue.getValue();
2778     if (!SRetPtr)
2779       SRetPtr = CreateMemTemp(RetTy);
2780     if (IRFunctionArgs.hasSRetArg()) {
2781       IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr;
2782     } else {
2783       llvm::Value *Addr =
2784           Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
2785       Builder.CreateStore(SRetPtr, Addr);
2786     }
2787   }
2788 
2789   assert(CallInfo.arg_size() == CallArgs.size() &&
2790          "Mismatch between function signature & arguments.");
2791   unsigned ArgNo = 0;
2792   CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
2793   for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
2794        I != E; ++I, ++info_it, ++ArgNo) {
2795     const ABIArgInfo &ArgInfo = info_it->info;
2796     RValue RV = I->RV;
2797 
2798     CharUnits TypeAlign = getContext().getTypeAlignInChars(I->Ty);
2799 
2800     // Insert a padding argument to ensure proper alignment.
2801     if (IRFunctionArgs.hasPaddingArg(ArgNo))
2802       IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2803           llvm::UndefValue::get(ArgInfo.getPaddingType());
2804 
2805     unsigned FirstIRArg, NumIRArgs;
2806     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2807 
2808     switch (ArgInfo.getKind()) {
2809     case ABIArgInfo::InAlloca: {
2810       assert(NumIRArgs == 0);
2811       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
2812       if (RV.isAggregate()) {
2813         // Replace the placeholder with the appropriate argument slot GEP.
2814         llvm::Instruction *Placeholder =
2815             cast<llvm::Instruction>(RV.getAggregateAddr());
2816         CGBuilderTy::InsertPoint IP = Builder.saveIP();
2817         Builder.SetInsertPoint(Placeholder);
2818         llvm::Value *Addr = Builder.CreateStructGEP(
2819             ArgMemory, ArgInfo.getInAllocaFieldIndex());
2820         Builder.restoreIP(IP);
2821         deferPlaceholderReplacement(Placeholder, Addr);
2822       } else {
2823         // Store the RValue into the argument struct.
2824         llvm::Value *Addr =
2825             Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
2826         unsigned AS = Addr->getType()->getPointerAddressSpace();
2827         llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
2828         // There are some cases where a trivial bitcast is not avoidable.  The
2829         // definition of a type later in a translation unit may change it's type
2830         // from {}* to (%struct.foo*)*.
2831         if (Addr->getType() != MemType)
2832           Addr = Builder.CreateBitCast(Addr, MemType);
2833         LValue argLV = MakeAddrLValue(Addr, I->Ty, TypeAlign);
2834         EmitInitStoreOfNonAggregate(*this, RV, argLV);
2835       }
2836       break;
2837     }
2838 
2839     case ABIArgInfo::Indirect: {
2840       assert(NumIRArgs == 1);
2841       if (RV.isScalar() || RV.isComplex()) {
2842         // Make a temporary alloca to pass the argument.
2843         llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
2844         if (ArgInfo.getIndirectAlign() > AI->getAlignment())
2845           AI->setAlignment(ArgInfo.getIndirectAlign());
2846         IRCallArgs[FirstIRArg] = AI;
2847 
2848         LValue argLV = MakeAddrLValue(AI, I->Ty, TypeAlign);
2849         EmitInitStoreOfNonAggregate(*this, RV, argLV);
2850       } else {
2851         // We want to avoid creating an unnecessary temporary+copy here;
2852         // however, we need one in three cases:
2853         // 1. If the argument is not byval, and we are required to copy the
2854         //    source.  (This case doesn't occur on any common architecture.)
2855         // 2. If the argument is byval, RV is not sufficiently aligned, and
2856         //    we cannot force it to be sufficiently aligned.
2857         // 3. If the argument is byval, but RV is located in an address space
2858         //    different than that of the argument (0).
2859         llvm::Value *Addr = RV.getAggregateAddr();
2860         unsigned Align = ArgInfo.getIndirectAlign();
2861         const llvm::DataLayout *TD = &CGM.getDataLayout();
2862         const unsigned RVAddrSpace = Addr->getType()->getPointerAddressSpace();
2863         const unsigned ArgAddrSpace =
2864             (FirstIRArg < IRFuncTy->getNumParams()
2865                  ? IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace()
2866                  : 0);
2867         if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
2868             (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align &&
2869              llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align) ||
2870              (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) {
2871           // Create an aligned temporary, and copy to it.
2872           llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
2873           if (Align > AI->getAlignment())
2874             AI->setAlignment(Align);
2875           IRCallArgs[FirstIRArg] = AI;
2876           EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
2877         } else {
2878           // Skip the extra memcpy call.
2879           IRCallArgs[FirstIRArg] = Addr;
2880         }
2881       }
2882       break;
2883     }
2884 
2885     case ABIArgInfo::Ignore:
2886       assert(NumIRArgs == 0);
2887       break;
2888 
2889     case ABIArgInfo::Extend:
2890     case ABIArgInfo::Direct: {
2891       if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
2892           ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
2893           ArgInfo.getDirectOffset() == 0) {
2894         assert(NumIRArgs == 1);
2895         llvm::Value *V;
2896         if (RV.isScalar())
2897           V = RV.getScalarVal();
2898         else
2899           V = Builder.CreateLoad(RV.getAggregateAddr());
2900 
2901         // If the argument doesn't match, perform a bitcast to coerce it.  This
2902         // can happen due to trivial type mismatches.
2903         if (FirstIRArg < IRFuncTy->getNumParams() &&
2904             V->getType() != IRFuncTy->getParamType(FirstIRArg))
2905           V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
2906         IRCallArgs[FirstIRArg] = V;
2907         break;
2908       }
2909 
2910       // FIXME: Avoid the conversion through memory if possible.
2911       llvm::Value *SrcPtr;
2912       if (RV.isScalar() || RV.isComplex()) {
2913         SrcPtr = CreateMemTemp(I->Ty, "coerce");
2914         LValue SrcLV = MakeAddrLValue(SrcPtr, I->Ty, TypeAlign);
2915         EmitInitStoreOfNonAggregate(*this, RV, SrcLV);
2916       } else
2917         SrcPtr = RV.getAggregateAddr();
2918 
2919       // If the value is offset in memory, apply the offset now.
2920       if (unsigned Offs = ArgInfo.getDirectOffset()) {
2921         SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
2922         SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
2923         SrcPtr = Builder.CreateBitCast(SrcPtr,
2924                        llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
2925 
2926       }
2927 
2928       // Fast-isel and the optimizer generally like scalar values better than
2929       // FCAs, so we flatten them if this is safe to do for this argument.
2930       llvm::StructType *STy =
2931             dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
2932       if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
2933         llvm::Type *SrcTy =
2934           cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
2935         uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
2936         uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
2937 
2938         // If the source type is smaller than the destination type of the
2939         // coerce-to logic, copy the source value into a temp alloca the size
2940         // of the destination type to allow loading all of it. The bits past
2941         // the source value are left undef.
2942         if (SrcSize < DstSize) {
2943           llvm::AllocaInst *TempAlloca
2944             = CreateTempAlloca(STy, SrcPtr->getName() + ".coerce");
2945           Builder.CreateMemCpy(TempAlloca, SrcPtr, SrcSize, 0);
2946           SrcPtr = TempAlloca;
2947         } else {
2948           SrcPtr = Builder.CreateBitCast(SrcPtr,
2949                                          llvm::PointerType::getUnqual(STy));
2950         }
2951 
2952         assert(NumIRArgs == STy->getNumElements());
2953         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
2954           llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
2955           llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
2956           // We don't know what we're loading from.
2957           LI->setAlignment(1);
2958           IRCallArgs[FirstIRArg + i] = LI;
2959         }
2960       } else {
2961         // In the simple case, just pass the coerced loaded value.
2962         assert(NumIRArgs == 1);
2963         IRCallArgs[FirstIRArg] =
2964             CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), *this);
2965       }
2966 
2967       break;
2968     }
2969 
2970     case ABIArgInfo::Expand:
2971       unsigned IRArgPos = FirstIRArg;
2972       ExpandTypeToArgs(I->Ty, RV, IRFuncTy, IRCallArgs, IRArgPos);
2973       assert(IRArgPos == FirstIRArg + NumIRArgs);
2974       break;
2975     }
2976   }
2977 
2978   if (ArgMemory) {
2979     llvm::Value *Arg = ArgMemory;
2980     if (CallInfo.isVariadic()) {
2981       // When passing non-POD arguments by value to variadic functions, we will
2982       // end up with a variadic prototype and an inalloca call site.  In such
2983       // cases, we can't do any parameter mismatch checks.  Give up and bitcast
2984       // the callee.
2985       unsigned CalleeAS =
2986           cast<llvm::PointerType>(Callee->getType())->getAddressSpace();
2987       Callee = Builder.CreateBitCast(
2988           Callee, getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS));
2989     } else {
2990       llvm::Type *LastParamTy =
2991           IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
2992       if (Arg->getType() != LastParamTy) {
2993 #ifndef NDEBUG
2994         // Assert that these structs have equivalent element types.
2995         llvm::StructType *FullTy = CallInfo.getArgStruct();
2996         llvm::StructType *DeclaredTy = cast<llvm::StructType>(
2997             cast<llvm::PointerType>(LastParamTy)->getElementType());
2998         assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
2999         for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
3000                                                 DE = DeclaredTy->element_end(),
3001                                                 FI = FullTy->element_begin();
3002              DI != DE; ++DI, ++FI)
3003           assert(*DI == *FI);
3004 #endif
3005         Arg = Builder.CreateBitCast(Arg, LastParamTy);
3006       }
3007     }
3008     assert(IRFunctionArgs.hasInallocaArg());
3009     IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
3010   }
3011 
3012   if (!CallArgs.getCleanupsToDeactivate().empty())
3013     deactivateArgCleanupsBeforeCall(*this, CallArgs);
3014 
3015   // If the callee is a bitcast of a function to a varargs pointer to function
3016   // type, check to see if we can remove the bitcast.  This handles some cases
3017   // with unprototyped functions.
3018   if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
3019     if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
3020       llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
3021       llvm::FunctionType *CurFT =
3022         cast<llvm::FunctionType>(CurPT->getElementType());
3023       llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
3024 
3025       if (CE->getOpcode() == llvm::Instruction::BitCast &&
3026           ActualFT->getReturnType() == CurFT->getReturnType() &&
3027           ActualFT->getNumParams() == CurFT->getNumParams() &&
3028           ActualFT->getNumParams() == IRCallArgs.size() &&
3029           (CurFT->isVarArg() || !ActualFT->isVarArg())) {
3030         bool ArgsMatch = true;
3031         for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
3032           if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
3033             ArgsMatch = false;
3034             break;
3035           }
3036 
3037         // Strip the cast if we can get away with it.  This is a nice cleanup,
3038         // but also allows us to inline the function at -O0 if it is marked
3039         // always_inline.
3040         if (ArgsMatch)
3041           Callee = CalleeF;
3042       }
3043     }
3044 
3045   assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
3046   for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
3047     // Inalloca argument can have different type.
3048     if (IRFunctionArgs.hasInallocaArg() &&
3049         i == IRFunctionArgs.getInallocaArgNo())
3050       continue;
3051     if (i < IRFuncTy->getNumParams())
3052       assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
3053   }
3054 
3055   unsigned CallingConv;
3056   CodeGen::AttributeListType AttributeList;
3057   CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList,
3058                              CallingConv, true);
3059   llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(),
3060                                                      AttributeList);
3061 
3062   llvm::BasicBlock *InvokeDest = nullptr;
3063   if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
3064                           llvm::Attribute::NoUnwind))
3065     InvokeDest = getInvokeDest();
3066 
3067   llvm::CallSite CS;
3068   if (!InvokeDest) {
3069     CS = Builder.CreateCall(Callee, IRCallArgs);
3070   } else {
3071     llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
3072     CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs);
3073     EmitBlock(Cont);
3074   }
3075   if (callOrInvoke)
3076     *callOrInvoke = CS.getInstruction();
3077 
3078   if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
3079       !CS.hasFnAttr(llvm::Attribute::NoInline))
3080     Attrs =
3081         Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
3082                            llvm::Attribute::AlwaysInline);
3083 
3084   CS.setAttributes(Attrs);
3085   CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
3086 
3087   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3088   // optimizer it can aggressively ignore unwind edges.
3089   if (CGM.getLangOpts().ObjCAutoRefCount)
3090     AddObjCARCExceptionMetadata(CS.getInstruction());
3091 
3092   // If the call doesn't return, finish the basic block and clear the
3093   // insertion point; this allows the rest of IRgen to discard
3094   // unreachable code.
3095   if (CS.doesNotReturn()) {
3096     Builder.CreateUnreachable();
3097     Builder.ClearInsertionPoint();
3098 
3099     // FIXME: For now, emit a dummy basic block because expr emitters in
3100     // generally are not ready to handle emitting expressions at unreachable
3101     // points.
3102     EnsureInsertPoint();
3103 
3104     // Return a reasonable RValue.
3105     return GetUndefRValue(RetTy);
3106   }
3107 
3108   llvm::Instruction *CI = CS.getInstruction();
3109   if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
3110     CI->setName("call");
3111 
3112   // Emit any writebacks immediately.  Arguably this should happen
3113   // after any return-value munging.
3114   if (CallArgs.hasWritebacks())
3115     emitWritebacks(*this, CallArgs);
3116 
3117   // The stack cleanup for inalloca arguments has to run out of the normal
3118   // lexical order, so deactivate it and run it manually here.
3119   CallArgs.freeArgumentMemory(*this);
3120 
3121   switch (RetAI.getKind()) {
3122   case ABIArgInfo::InAlloca:
3123   case ABIArgInfo::Indirect:
3124     return convertTempToRValue(SRetPtr, RetTy, SourceLocation());
3125 
3126   case ABIArgInfo::Ignore:
3127     // If we are ignoring an argument that had a result, make sure to
3128     // construct the appropriate return value for our caller.
3129     return GetUndefRValue(RetTy);
3130 
3131   case ABIArgInfo::Extend:
3132   case ABIArgInfo::Direct: {
3133     llvm::Type *RetIRTy = ConvertType(RetTy);
3134     if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
3135       switch (getEvaluationKind(RetTy)) {
3136       case TEK_Complex: {
3137         llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
3138         llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
3139         return RValue::getComplex(std::make_pair(Real, Imag));
3140       }
3141       case TEK_Aggregate: {
3142         llvm::Value *DestPtr = ReturnValue.getValue();
3143         bool DestIsVolatile = ReturnValue.isVolatile();
3144 
3145         if (!DestPtr) {
3146           DestPtr = CreateMemTemp(RetTy, "agg.tmp");
3147           DestIsVolatile = false;
3148         }
3149         BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
3150         return RValue::getAggregate(DestPtr);
3151       }
3152       case TEK_Scalar: {
3153         // If the argument doesn't match, perform a bitcast to coerce it.  This
3154         // can happen due to trivial type mismatches.
3155         llvm::Value *V = CI;
3156         if (V->getType() != RetIRTy)
3157           V = Builder.CreateBitCast(V, RetIRTy);
3158         return RValue::get(V);
3159       }
3160       }
3161       llvm_unreachable("bad evaluation kind");
3162     }
3163 
3164     llvm::Value *DestPtr = ReturnValue.getValue();
3165     bool DestIsVolatile = ReturnValue.isVolatile();
3166 
3167     if (!DestPtr) {
3168       DestPtr = CreateMemTemp(RetTy, "coerce");
3169       DestIsVolatile = false;
3170     }
3171 
3172     // If the value is offset in memory, apply the offset now.
3173     llvm::Value *StorePtr = DestPtr;
3174     if (unsigned Offs = RetAI.getDirectOffset()) {
3175       StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
3176       StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
3177       StorePtr = Builder.CreateBitCast(StorePtr,
3178                          llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
3179     }
3180     CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
3181 
3182     return convertTempToRValue(DestPtr, RetTy, SourceLocation());
3183   }
3184 
3185   case ABIArgInfo::Expand:
3186     llvm_unreachable("Invalid ABI kind for return argument");
3187   }
3188 
3189   llvm_unreachable("Unhandled ABIArgInfo::Kind");
3190 }
3191 
3192 /* VarArg handling */
3193 
3194 llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
3195   return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
3196 }
3197