1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // These classes wrap the information about a call or function
11 // definition used to handle ABI compliancy.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CGCall.h"
16 #include "ABIInfo.h"
17 #include "CGCXXABI.h"
18 #include "CodeGenFunction.h"
19 #include "CodeGenModule.h"
20 #include "TargetInfo.h"
21 #include "clang/AST/Decl.h"
22 #include "clang/AST/DeclCXX.h"
23 #include "clang/AST/DeclObjC.h"
24 #include "clang/Basic/TargetInfo.h"
25 #include "clang/CodeGen/CGFunctionInfo.h"
26 #include "clang/Frontend/CodeGenOptions.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/IR/Attributes.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/InlineAsm.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/Transforms/Utils/Local.h"
34 using namespace clang;
35 using namespace CodeGen;
36 
37 /***/
38 
39 static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
40   switch (CC) {
41   default: return llvm::CallingConv::C;
42   case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
43   case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
44   case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
45   case CC_X86_64Win64: return llvm::CallingConv::X86_64_Win64;
46   case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
47   case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
48   case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
49   case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
50   // TODO: Add support for __pascal to LLVM.
51   case CC_X86Pascal: return llvm::CallingConv::C;
52   // TODO: Add support for __vectorcall to LLVM.
53   case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
54   }
55 }
56 
57 /// Derives the 'this' type for codegen purposes, i.e. ignoring method
58 /// qualification.
59 /// FIXME: address space qualification?
60 static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
61   QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
62   return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
63 }
64 
65 /// Returns the canonical formal type of the given C++ method.
66 static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
67   return MD->getType()->getCanonicalTypeUnqualified()
68            .getAs<FunctionProtoType>();
69 }
70 
71 /// Returns the "extra-canonicalized" return type, which discards
72 /// qualifiers on the return type.  Codegen doesn't care about them,
73 /// and it makes ABI code a little easier to be able to assume that
74 /// all parameter and return types are top-level unqualified.
75 static CanQualType GetReturnType(QualType RetTy) {
76   return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
77 }
78 
79 /// Arrange the argument and result information for a value of the given
80 /// unprototyped freestanding function type.
81 const CGFunctionInfo &
82 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
83   // When translating an unprototyped function type, always use a
84   // variadic type.
85   return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
86                                  false, None, FTNP->getExtInfo(),
87                                  RequiredArgs(0));
88 }
89 
90 /// Arrange the LLVM function layout for a value of the given function
91 /// type, on top of any implicit parameters already stored.
92 static const CGFunctionInfo &
93 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool IsInstanceMethod,
94                         SmallVectorImpl<CanQualType> &prefix,
95                         CanQual<FunctionProtoType> FTP) {
96   RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
97   // FIXME: Kill copy.
98   for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
99     prefix.push_back(FTP->getParamType(i));
100   CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
101   return CGT.arrangeLLVMFunctionInfo(resultType, IsInstanceMethod, prefix,
102                                      FTP->getExtInfo(), required);
103 }
104 
105 /// Arrange the argument and result information for a value of the
106 /// given freestanding function type.
107 const CGFunctionInfo &
108 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
109   SmallVector<CanQualType, 16> argTypes;
110   return ::arrangeLLVMFunctionInfo(*this, false, argTypes, FTP);
111 }
112 
113 static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
114   // Set the appropriate calling convention for the Function.
115   if (D->hasAttr<StdCallAttr>())
116     return CC_X86StdCall;
117 
118   if (D->hasAttr<FastCallAttr>())
119     return CC_X86FastCall;
120 
121   if (D->hasAttr<ThisCallAttr>())
122     return CC_X86ThisCall;
123 
124   if (D->hasAttr<VectorCallAttr>())
125     return CC_X86VectorCall;
126 
127   if (D->hasAttr<PascalAttr>())
128     return CC_X86Pascal;
129 
130   if (PcsAttr *PCS = D->getAttr<PcsAttr>())
131     return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
132 
133   if (D->hasAttr<PnaclCallAttr>())
134     return CC_PnaclCall;
135 
136   if (D->hasAttr<IntelOclBiccAttr>())
137     return CC_IntelOclBicc;
138 
139   if (D->hasAttr<MSABIAttr>())
140     return IsWindows ? CC_C : CC_X86_64Win64;
141 
142   if (D->hasAttr<SysVABIAttr>())
143     return IsWindows ? CC_X86_64SysV : CC_C;
144 
145   return CC_C;
146 }
147 
148 /// Arrange the argument and result information for a call to an
149 /// unknown C++ non-static member function of the given abstract type.
150 /// (Zero value of RD means we don't have any meaningful "this" argument type,
151 ///  so fall back to a generic pointer type).
152 /// The member function must be an ordinary function, i.e. not a
153 /// constructor or destructor.
154 const CGFunctionInfo &
155 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
156                                    const FunctionProtoType *FTP) {
157   SmallVector<CanQualType, 16> argTypes;
158 
159   // Add the 'this' pointer.
160   if (RD)
161     argTypes.push_back(GetThisType(Context, RD));
162   else
163     argTypes.push_back(Context.VoidPtrTy);
164 
165   return ::arrangeLLVMFunctionInfo(
166       *this, true, argTypes,
167       FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
168 }
169 
170 /// Arrange the argument and result information for a declaration or
171 /// definition of the given C++ non-static member function.  The
172 /// member function must be an ordinary function, i.e. not a
173 /// constructor or destructor.
174 const CGFunctionInfo &
175 CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
176   assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
177   assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
178 
179   CanQual<FunctionProtoType> prototype = GetFormalType(MD);
180 
181   if (MD->isInstance()) {
182     // The abstract case is perfectly fine.
183     const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
184     return arrangeCXXMethodType(ThisType, prototype.getTypePtr());
185   }
186 
187   return arrangeFreeFunctionType(prototype);
188 }
189 
190 const CGFunctionInfo &
191 CodeGenTypes::arrangeCXXStructorDeclaration(const CXXMethodDecl *MD,
192                                             StructorType Type) {
193 
194   SmallVector<CanQualType, 16> argTypes;
195   argTypes.push_back(GetThisType(Context, MD->getParent()));
196 
197   GlobalDecl GD;
198   if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
199     GD = GlobalDecl(CD, toCXXCtorType(Type));
200   } else {
201     auto *DD = dyn_cast<CXXDestructorDecl>(MD);
202     GD = GlobalDecl(DD, toCXXDtorType(Type));
203   }
204 
205   CanQual<FunctionProtoType> FTP = GetFormalType(MD);
206 
207   // Add the formal parameters.
208   for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i)
209     argTypes.push_back(FTP->getParamType(i));
210 
211   TheCXXABI.buildStructorSignature(MD, Type, argTypes);
212 
213   RequiredArgs required =
214       (MD->isVariadic() ? RequiredArgs(argTypes.size()) : RequiredArgs::All);
215 
216   FunctionType::ExtInfo extInfo = FTP->getExtInfo();
217   CanQualType resultType = TheCXXABI.HasThisReturn(GD)
218                                ? argTypes.front()
219                                : TheCXXABI.hasMostDerivedReturn(GD)
220                                      ? CGM.getContext().VoidPtrTy
221                                      : Context.VoidTy;
222   return arrangeLLVMFunctionInfo(resultType, true, argTypes, extInfo, required);
223 }
224 
225 /// Arrange a call to a C++ method, passing the given arguments.
226 const CGFunctionInfo &
227 CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
228                                         const CXXConstructorDecl *D,
229                                         CXXCtorType CtorKind,
230                                         unsigned ExtraArgs) {
231   // FIXME: Kill copy.
232   SmallVector<CanQualType, 16> ArgTypes;
233   for (const auto &Arg : args)
234     ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
235 
236   CanQual<FunctionProtoType> FPT = GetFormalType(D);
237   RequiredArgs Required = RequiredArgs::forPrototypePlus(FPT, 1 + ExtraArgs);
238   GlobalDecl GD(D, CtorKind);
239   CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
240                                ? ArgTypes.front()
241                                : TheCXXABI.hasMostDerivedReturn(GD)
242                                      ? CGM.getContext().VoidPtrTy
243                                      : Context.VoidTy;
244 
245   FunctionType::ExtInfo Info = FPT->getExtInfo();
246   return arrangeLLVMFunctionInfo(ResultType, true, ArgTypes, Info, Required);
247 }
248 
249 /// Arrange the argument and result information for the declaration or
250 /// definition of the given function.
251 const CGFunctionInfo &
252 CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
253   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
254     if (MD->isInstance())
255       return arrangeCXXMethodDeclaration(MD);
256 
257   CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
258 
259   assert(isa<FunctionType>(FTy));
260 
261   // When declaring a function without a prototype, always use a
262   // non-variadic type.
263   if (isa<FunctionNoProtoType>(FTy)) {
264     CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>();
265     return arrangeLLVMFunctionInfo(noProto->getReturnType(), false, None,
266                                    noProto->getExtInfo(), RequiredArgs::All);
267   }
268 
269   assert(isa<FunctionProtoType>(FTy));
270   return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>());
271 }
272 
273 /// Arrange the argument and result information for the declaration or
274 /// definition of an Objective-C method.
275 const CGFunctionInfo &
276 CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
277   // It happens that this is the same as a call with no optional
278   // arguments, except also using the formal 'self' type.
279   return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
280 }
281 
282 /// Arrange the argument and result information for the function type
283 /// through which to perform a send to the given Objective-C method,
284 /// using the given receiver type.  The receiver type is not always
285 /// the 'self' type of the method or even an Objective-C pointer type.
286 /// This is *not* the right method for actually performing such a
287 /// message send, due to the possibility of optional arguments.
288 const CGFunctionInfo &
289 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
290                                               QualType receiverType) {
291   SmallVector<CanQualType, 16> argTys;
292   argTys.push_back(Context.getCanonicalParamType(receiverType));
293   argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
294   // FIXME: Kill copy?
295   for (const auto *I : MD->params()) {
296     argTys.push_back(Context.getCanonicalParamType(I->getType()));
297   }
298 
299   FunctionType::ExtInfo einfo;
300   bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
301   einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
302 
303   if (getContext().getLangOpts().ObjCAutoRefCount &&
304       MD->hasAttr<NSReturnsRetainedAttr>())
305     einfo = einfo.withProducesResult(true);
306 
307   RequiredArgs required =
308     (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
309 
310   return arrangeLLVMFunctionInfo(GetReturnType(MD->getReturnType()), false,
311                                  argTys, einfo, required);
312 }
313 
314 const CGFunctionInfo &
315 CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
316   // FIXME: Do we need to handle ObjCMethodDecl?
317   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
318 
319   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
320     return arrangeCXXStructorDeclaration(CD, getFromCtorType(GD.getCtorType()));
321 
322   if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
323     return arrangeCXXStructorDeclaration(DD, getFromDtorType(GD.getDtorType()));
324 
325   return arrangeFunctionDeclaration(FD);
326 }
327 
328 /// Arrange a thunk that takes 'this' as the first parameter followed by
329 /// varargs.  Return a void pointer, regardless of the actual return type.
330 /// The body of the thunk will end in a musttail call to a function of the
331 /// correct type, and the caller will bitcast the function to the correct
332 /// prototype.
333 const CGFunctionInfo &
334 CodeGenTypes::arrangeMSMemberPointerThunk(const CXXMethodDecl *MD) {
335   assert(MD->isVirtual() && "only virtual memptrs have thunks");
336   CanQual<FunctionProtoType> FTP = GetFormalType(MD);
337   CanQualType ArgTys[] = { GetThisType(Context, MD->getParent()) };
338   return arrangeLLVMFunctionInfo(Context.VoidTy, false, ArgTys,
339                                  FTP->getExtInfo(), RequiredArgs(1));
340 }
341 
342 /// Arrange a call as unto a free function, except possibly with an
343 /// additional number of formal parameters considered required.
344 static const CGFunctionInfo &
345 arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
346                             CodeGenModule &CGM,
347                             const CallArgList &args,
348                             const FunctionType *fnType,
349                             unsigned numExtraRequiredArgs) {
350   assert(args.size() >= numExtraRequiredArgs);
351 
352   // In most cases, there are no optional arguments.
353   RequiredArgs required = RequiredArgs::All;
354 
355   // If we have a variadic prototype, the required arguments are the
356   // extra prefix plus the arguments in the prototype.
357   if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
358     if (proto->isVariadic())
359       required = RequiredArgs(proto->getNumParams() + numExtraRequiredArgs);
360 
361   // If we don't have a prototype at all, but we're supposed to
362   // explicitly use the variadic convention for unprototyped calls,
363   // treat all of the arguments as required but preserve the nominal
364   // possibility of variadics.
365   } else if (CGM.getTargetCodeGenInfo()
366                 .isNoProtoCallVariadic(args,
367                                        cast<FunctionNoProtoType>(fnType))) {
368     required = RequiredArgs(args.size());
369   }
370 
371   return CGT.arrangeFreeFunctionCall(fnType->getReturnType(), args,
372                                      fnType->getExtInfo(), required);
373 }
374 
375 /// Figure out the rules for calling a function with the given formal
376 /// type using the given arguments.  The arguments are necessary
377 /// because the function might be unprototyped, in which case it's
378 /// target-dependent in crazy ways.
379 const CGFunctionInfo &
380 CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
381                                       const FunctionType *fnType) {
382   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 0);
383 }
384 
385 /// A block function call is essentially a free-function call with an
386 /// extra implicit argument.
387 const CGFunctionInfo &
388 CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
389                                        const FunctionType *fnType) {
390   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1);
391 }
392 
393 const CGFunctionInfo &
394 CodeGenTypes::arrangeFreeFunctionCall(QualType resultType,
395                                       const CallArgList &args,
396                                       FunctionType::ExtInfo info,
397                                       RequiredArgs required) {
398   // FIXME: Kill copy.
399   SmallVector<CanQualType, 16> argTypes;
400   for (const auto &Arg : args)
401     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
402   return arrangeLLVMFunctionInfo(GetReturnType(resultType), false, argTypes,
403                                  info, required);
404 }
405 
406 /// Arrange a call to a C++ method, passing the given arguments.
407 const CGFunctionInfo &
408 CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
409                                    const FunctionProtoType *FPT,
410                                    RequiredArgs required) {
411   // FIXME: Kill copy.
412   SmallVector<CanQualType, 16> argTypes;
413   for (const auto &Arg : args)
414     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
415 
416   FunctionType::ExtInfo info = FPT->getExtInfo();
417   return arrangeLLVMFunctionInfo(GetReturnType(FPT->getReturnType()), true,
418                                  argTypes, info, required);
419 }
420 
421 const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionDeclaration(
422     QualType resultType, const FunctionArgList &args,
423     const FunctionType::ExtInfo &info, bool isVariadic) {
424   // FIXME: Kill copy.
425   SmallVector<CanQualType, 16> argTypes;
426   for (auto Arg : args)
427     argTypes.push_back(Context.getCanonicalParamType(Arg->getType()));
428 
429   RequiredArgs required =
430     (isVariadic ? RequiredArgs(args.size()) : RequiredArgs::All);
431   return arrangeLLVMFunctionInfo(GetReturnType(resultType), false, argTypes, info,
432                                  required);
433 }
434 
435 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
436   return arrangeLLVMFunctionInfo(getContext().VoidTy, false, None,
437                                  FunctionType::ExtInfo(), RequiredArgs::All);
438 }
439 
440 /// Arrange the argument and result information for an abstract value
441 /// of a given function type.  This is the method which all of the
442 /// above functions ultimately defer to.
443 const CGFunctionInfo &
444 CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
445                                       bool IsInstanceMethod,
446                                       ArrayRef<CanQualType> argTypes,
447                                       FunctionType::ExtInfo info,
448                                       RequiredArgs required) {
449 #ifndef NDEBUG
450   for (ArrayRef<CanQualType>::const_iterator
451          I = argTypes.begin(), E = argTypes.end(); I != E; ++I)
452     assert(I->isCanonicalAsParam());
453 #endif
454 
455   unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
456 
457   // Lookup or create unique function info.
458   llvm::FoldingSetNodeID ID;
459   CGFunctionInfo::Profile(ID, IsInstanceMethod, info, required, resultType,
460                           argTypes);
461 
462   void *insertPos = nullptr;
463   CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
464   if (FI)
465     return *FI;
466 
467   // Construct the function info.  We co-allocate the ArgInfos.
468   FI = CGFunctionInfo::create(CC, IsInstanceMethod, info, resultType, argTypes,
469                               required);
470   FunctionInfos.InsertNode(FI, insertPos);
471 
472   bool inserted = FunctionsBeingProcessed.insert(FI).second;
473   (void)inserted;
474   assert(inserted && "Recursively being processed?");
475 
476   // Compute ABI information.
477   getABIInfo().computeInfo(*FI);
478 
479   // Loop over all of the computed argument and return value info.  If any of
480   // them are direct or extend without a specified coerce type, specify the
481   // default now.
482   ABIArgInfo &retInfo = FI->getReturnInfo();
483   if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
484     retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
485 
486   for (auto &I : FI->arguments())
487     if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
488       I.info.setCoerceToType(ConvertType(I.type));
489 
490   bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
491   assert(erased && "Not in set?");
492 
493   return *FI;
494 }
495 
496 CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
497                                        bool IsInstanceMethod,
498                                        const FunctionType::ExtInfo &info,
499                                        CanQualType resultType,
500                                        ArrayRef<CanQualType> argTypes,
501                                        RequiredArgs required) {
502   void *buffer = operator new(sizeof(CGFunctionInfo) +
503                               sizeof(ArgInfo) * (argTypes.size() + 1));
504   CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
505   FI->CallingConvention = llvmCC;
506   FI->EffectiveCallingConvention = llvmCC;
507   FI->ASTCallingConvention = info.getCC();
508   FI->InstanceMethod = IsInstanceMethod;
509   FI->NoReturn = info.getNoReturn();
510   FI->ReturnsRetained = info.getProducesResult();
511   FI->Required = required;
512   FI->HasRegParm = info.getHasRegParm();
513   FI->RegParm = info.getRegParm();
514   FI->ArgStruct = nullptr;
515   FI->NumArgs = argTypes.size();
516   FI->getArgsBuffer()[0].type = resultType;
517   for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
518     FI->getArgsBuffer()[i + 1].type = argTypes[i];
519   return FI;
520 }
521 
522 /***/
523 
524 namespace {
525 // ABIArgInfo::Expand implementation.
526 
527 // Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
528 struct TypeExpansion {
529   enum TypeExpansionKind {
530     // Elements of constant arrays are expanded recursively.
531     TEK_ConstantArray,
532     // Record fields are expanded recursively (but if record is a union, only
533     // the field with the largest size is expanded).
534     TEK_Record,
535     // For complex types, real and imaginary parts are expanded recursively.
536     TEK_Complex,
537     // All other types are not expandable.
538     TEK_None
539   };
540 
541   const TypeExpansionKind Kind;
542 
543   TypeExpansion(TypeExpansionKind K) : Kind(K) {}
544   virtual ~TypeExpansion() {}
545 };
546 
547 struct ConstantArrayExpansion : TypeExpansion {
548   QualType EltTy;
549   uint64_t NumElts;
550 
551   ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
552       : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
553   static bool classof(const TypeExpansion *TE) {
554     return TE->Kind == TEK_ConstantArray;
555   }
556 };
557 
558 struct RecordExpansion : TypeExpansion {
559   SmallVector<const CXXBaseSpecifier *, 1> Bases;
560 
561   SmallVector<const FieldDecl *, 1> Fields;
562 
563   RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
564                   SmallVector<const FieldDecl *, 1> &&Fields)
565       : TypeExpansion(TEK_Record), Bases(Bases), Fields(Fields) {}
566   static bool classof(const TypeExpansion *TE) {
567     return TE->Kind == TEK_Record;
568   }
569 };
570 
571 struct ComplexExpansion : TypeExpansion {
572   QualType EltTy;
573 
574   ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
575   static bool classof(const TypeExpansion *TE) {
576     return TE->Kind == TEK_Complex;
577   }
578 };
579 
580 struct NoExpansion : TypeExpansion {
581   NoExpansion() : TypeExpansion(TEK_None) {}
582   static bool classof(const TypeExpansion *TE) {
583     return TE->Kind == TEK_None;
584   }
585 };
586 }  // namespace
587 
588 static std::unique_ptr<TypeExpansion>
589 getTypeExpansion(QualType Ty, const ASTContext &Context) {
590   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
591     return llvm::make_unique<ConstantArrayExpansion>(
592         AT->getElementType(), AT->getSize().getZExtValue());
593   }
594   if (const RecordType *RT = Ty->getAs<RecordType>()) {
595     SmallVector<const CXXBaseSpecifier *, 1> Bases;
596     SmallVector<const FieldDecl *, 1> Fields;
597     const RecordDecl *RD = RT->getDecl();
598     assert(!RD->hasFlexibleArrayMember() &&
599            "Cannot expand structure with flexible array.");
600     if (RD->isUnion()) {
601       // Unions can be here only in degenerative cases - all the fields are same
602       // after flattening. Thus we have to use the "largest" field.
603       const FieldDecl *LargestFD = nullptr;
604       CharUnits UnionSize = CharUnits::Zero();
605 
606       for (const auto *FD : RD->fields()) {
607         // Skip zero length bitfields.
608         if (FD->isBitField() && FD->getBitWidthValue(Context) == 0)
609           continue;
610         assert(!FD->isBitField() &&
611                "Cannot expand structure with bit-field members.");
612         CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
613         if (UnionSize < FieldSize) {
614           UnionSize = FieldSize;
615           LargestFD = FD;
616         }
617       }
618       if (LargestFD)
619         Fields.push_back(LargestFD);
620     } else {
621       if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
622         assert(!CXXRD->isDynamicClass() &&
623                "cannot expand vtable pointers in dynamic classes");
624         for (const CXXBaseSpecifier &BS : CXXRD->bases())
625           Bases.push_back(&BS);
626       }
627 
628       for (const auto *FD : RD->fields()) {
629         // Skip zero length bitfields.
630         if (FD->isBitField() && FD->getBitWidthValue(Context) == 0)
631           continue;
632         assert(!FD->isBitField() &&
633                "Cannot expand structure with bit-field members.");
634         Fields.push_back(FD);
635       }
636     }
637     return llvm::make_unique<RecordExpansion>(std::move(Bases),
638                                               std::move(Fields));
639   }
640   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
641     return llvm::make_unique<ComplexExpansion>(CT->getElementType());
642   }
643   return llvm::make_unique<NoExpansion>();
644 }
645 
646 static int getExpansionSize(QualType Ty, const ASTContext &Context) {
647   auto Exp = getTypeExpansion(Ty, Context);
648   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
649     return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
650   }
651   if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
652     int Res = 0;
653     for (auto BS : RExp->Bases)
654       Res += getExpansionSize(BS->getType(), Context);
655     for (auto FD : RExp->Fields)
656       Res += getExpansionSize(FD->getType(), Context);
657     return Res;
658   }
659   if (isa<ComplexExpansion>(Exp.get()))
660     return 2;
661   assert(isa<NoExpansion>(Exp.get()));
662   return 1;
663 }
664 
665 void
666 CodeGenTypes::getExpandedTypes(QualType Ty,
667                                SmallVectorImpl<llvm::Type *>::iterator &TI) {
668   auto Exp = getTypeExpansion(Ty, Context);
669   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
670     for (int i = 0, n = CAExp->NumElts; i < n; i++) {
671       getExpandedTypes(CAExp->EltTy, TI);
672     }
673   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
674     for (auto BS : RExp->Bases)
675       getExpandedTypes(BS->getType(), TI);
676     for (auto FD : RExp->Fields)
677       getExpandedTypes(FD->getType(), TI);
678   } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
679     llvm::Type *EltTy = ConvertType(CExp->EltTy);
680     *TI++ = EltTy;
681     *TI++ = EltTy;
682   } else {
683     assert(isa<NoExpansion>(Exp.get()));
684     *TI++ = ConvertType(Ty);
685   }
686 }
687 
688 void CodeGenFunction::ExpandTypeFromArgs(
689     QualType Ty, LValue LV, SmallVectorImpl<llvm::Argument *>::iterator &AI) {
690   assert(LV.isSimple() &&
691          "Unexpected non-simple lvalue during struct expansion.");
692 
693   auto Exp = getTypeExpansion(Ty, getContext());
694   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
695     for (int i = 0, n = CAExp->NumElts; i < n; i++) {
696       llvm::Value *EltAddr = Builder.CreateConstGEP2_32(LV.getAddress(), 0, i);
697       LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
698       ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
699     }
700   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
701     llvm::Value *This = LV.getAddress();
702     for (const CXXBaseSpecifier *BS : RExp->Bases) {
703       // Perform a single step derived-to-base conversion.
704       llvm::Value *Base =
705           GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
706                                 /*NullCheckValue=*/false, SourceLocation());
707       LValue SubLV = MakeAddrLValue(Base, BS->getType());
708 
709       // Recurse onto bases.
710       ExpandTypeFromArgs(BS->getType(), SubLV, AI);
711     }
712     for (auto FD : RExp->Fields) {
713       // FIXME: What are the right qualifiers here?
714       LValue SubLV = EmitLValueForField(LV, FD);
715       ExpandTypeFromArgs(FD->getType(), SubLV, AI);
716     }
717   } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
718     llvm::Value *RealAddr = Builder.CreateStructGEP(LV.getAddress(), 0, "real");
719     EmitStoreThroughLValue(RValue::get(*AI++),
720                            MakeAddrLValue(RealAddr, CExp->EltTy));
721     llvm::Value *ImagAddr = Builder.CreateStructGEP(LV.getAddress(), 1, "imag");
722     EmitStoreThroughLValue(RValue::get(*AI++),
723                            MakeAddrLValue(ImagAddr, CExp->EltTy));
724   } else {
725     assert(isa<NoExpansion>(Exp.get()));
726     EmitStoreThroughLValue(RValue::get(*AI++), LV);
727   }
728 }
729 
730 void CodeGenFunction::ExpandTypeToArgs(
731     QualType Ty, RValue RV, llvm::FunctionType *IRFuncTy,
732     SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
733   auto Exp = getTypeExpansion(Ty, getContext());
734   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
735     llvm::Value *Addr = RV.getAggregateAddr();
736     for (int i = 0, n = CAExp->NumElts; i < n; i++) {
737       llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, i);
738       RValue EltRV =
739           convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation());
740       ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos);
741     }
742   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
743     llvm::Value *This = RV.getAggregateAddr();
744     for (const CXXBaseSpecifier *BS : RExp->Bases) {
745       // Perform a single step derived-to-base conversion.
746       llvm::Value *Base =
747           GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
748                                 /*NullCheckValue=*/false, SourceLocation());
749       RValue BaseRV = RValue::getAggregate(Base);
750 
751       // Recurse onto bases.
752       ExpandTypeToArgs(BS->getType(), BaseRV, IRFuncTy, IRCallArgs,
753                        IRCallArgPos);
754     }
755 
756     LValue LV = MakeAddrLValue(This, Ty);
757     for (auto FD : RExp->Fields) {
758       RValue FldRV = EmitRValueForField(LV, FD, SourceLocation());
759       ExpandTypeToArgs(FD->getType(), FldRV, IRFuncTy, IRCallArgs,
760                        IRCallArgPos);
761     }
762   } else if (isa<ComplexExpansion>(Exp.get())) {
763     ComplexPairTy CV = RV.getComplexVal();
764     IRCallArgs[IRCallArgPos++] = CV.first;
765     IRCallArgs[IRCallArgPos++] = CV.second;
766   } else {
767     assert(isa<NoExpansion>(Exp.get()));
768     assert(RV.isScalar() &&
769            "Unexpected non-scalar rvalue during struct expansion.");
770 
771     // Insert a bitcast as needed.
772     llvm::Value *V = RV.getScalarVal();
773     if (IRCallArgPos < IRFuncTy->getNumParams() &&
774         V->getType() != IRFuncTy->getParamType(IRCallArgPos))
775       V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
776 
777     IRCallArgs[IRCallArgPos++] = V;
778   }
779 }
780 
781 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
782 /// accessing some number of bytes out of it, try to gep into the struct to get
783 /// at its inner goodness.  Dive as deep as possible without entering an element
784 /// with an in-memory size smaller than DstSize.
785 static llvm::Value *
786 EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
787                                    llvm::StructType *SrcSTy,
788                                    uint64_t DstSize, CodeGenFunction &CGF) {
789   // We can't dive into a zero-element struct.
790   if (SrcSTy->getNumElements() == 0) return SrcPtr;
791 
792   llvm::Type *FirstElt = SrcSTy->getElementType(0);
793 
794   // If the first elt is at least as large as what we're looking for, or if the
795   // first element is the same size as the whole struct, we can enter it. The
796   // comparison must be made on the store size and not the alloca size. Using
797   // the alloca size may overstate the size of the load.
798   uint64_t FirstEltSize =
799     CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
800   if (FirstEltSize < DstSize &&
801       FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
802     return SrcPtr;
803 
804   // GEP into the first element.
805   SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
806 
807   // If the first element is a struct, recurse.
808   llvm::Type *SrcTy =
809     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
810   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
811     return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
812 
813   return SrcPtr;
814 }
815 
816 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
817 /// are either integers or pointers.  This does a truncation of the value if it
818 /// is too large or a zero extension if it is too small.
819 ///
820 /// This behaves as if the value were coerced through memory, so on big-endian
821 /// targets the high bits are preserved in a truncation, while little-endian
822 /// targets preserve the low bits.
823 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
824                                              llvm::Type *Ty,
825                                              CodeGenFunction &CGF) {
826   if (Val->getType() == Ty)
827     return Val;
828 
829   if (isa<llvm::PointerType>(Val->getType())) {
830     // If this is Pointer->Pointer avoid conversion to and from int.
831     if (isa<llvm::PointerType>(Ty))
832       return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
833 
834     // Convert the pointer to an integer so we can play with its width.
835     Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
836   }
837 
838   llvm::Type *DestIntTy = Ty;
839   if (isa<llvm::PointerType>(DestIntTy))
840     DestIntTy = CGF.IntPtrTy;
841 
842   if (Val->getType() != DestIntTy) {
843     const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
844     if (DL.isBigEndian()) {
845       // Preserve the high bits on big-endian targets.
846       // That is what memory coercion does.
847       uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
848       uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
849 
850       if (SrcSize > DstSize) {
851         Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
852         Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
853       } else {
854         Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
855         Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
856       }
857     } else {
858       // Little-endian targets preserve the low bits. No shifts required.
859       Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
860     }
861   }
862 
863   if (isa<llvm::PointerType>(Ty))
864     Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
865   return Val;
866 }
867 
868 
869 
870 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
871 /// a pointer to an object of type \arg Ty.
872 ///
873 /// This safely handles the case when the src type is smaller than the
874 /// destination type; in this situation the values of bits which not
875 /// present in the src are undefined.
876 static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
877                                       llvm::Type *Ty,
878                                       CodeGenFunction &CGF) {
879   llvm::Type *SrcTy =
880     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
881 
882   // If SrcTy and Ty are the same, just do a load.
883   if (SrcTy == Ty)
884     return CGF.Builder.CreateLoad(SrcPtr);
885 
886   uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
887 
888   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
889     SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
890     SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
891   }
892 
893   uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
894 
895   // If the source and destination are integer or pointer types, just do an
896   // extension or truncation to the desired type.
897   if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
898       (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
899     llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
900     return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
901   }
902 
903   // If load is legal, just bitcast the src pointer.
904   if (SrcSize >= DstSize) {
905     // Generally SrcSize is never greater than DstSize, since this means we are
906     // losing bits. However, this can happen in cases where the structure has
907     // additional padding, for example due to a user specified alignment.
908     //
909     // FIXME: Assert that we aren't truncating non-padding bits when have access
910     // to that information.
911     llvm::Value *Casted =
912       CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
913     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
914     // FIXME: Use better alignment / avoid requiring aligned load.
915     Load->setAlignment(1);
916     return Load;
917   }
918 
919   // Otherwise do coercion through memory. This is stupid, but
920   // simple.
921   llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
922   llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
923   llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
924   llvm::Value *SrcCasted = CGF.Builder.CreateBitCast(SrcPtr, I8PtrTy);
925   // FIXME: Use better alignment.
926   CGF.Builder.CreateMemCpy(Casted, SrcCasted,
927       llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
928       1, false);
929   return CGF.Builder.CreateLoad(Tmp);
930 }
931 
932 // Function to store a first-class aggregate into memory.  We prefer to
933 // store the elements rather than the aggregate to be more friendly to
934 // fast-isel.
935 // FIXME: Do we need to recurse here?
936 static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
937                           llvm::Value *DestPtr, bool DestIsVolatile,
938                           bool LowAlignment) {
939   // Prefer scalar stores to first-class aggregate stores.
940   if (llvm::StructType *STy =
941         dyn_cast<llvm::StructType>(Val->getType())) {
942     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
943       llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
944       llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
945       llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
946                                                     DestIsVolatile);
947       if (LowAlignment)
948         SI->setAlignment(1);
949     }
950   } else {
951     llvm::StoreInst *SI = CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
952     if (LowAlignment)
953       SI->setAlignment(1);
954   }
955 }
956 
957 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
958 /// where the source and destination may have different types.
959 ///
960 /// This safely handles the case when the src type is larger than the
961 /// destination type; the upper bits of the src will be lost.
962 static void CreateCoercedStore(llvm::Value *Src,
963                                llvm::Value *DstPtr,
964                                bool DstIsVolatile,
965                                CodeGenFunction &CGF) {
966   llvm::Type *SrcTy = Src->getType();
967   llvm::Type *DstTy =
968     cast<llvm::PointerType>(DstPtr->getType())->getElementType();
969   if (SrcTy == DstTy) {
970     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
971     return;
972   }
973 
974   uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
975 
976   if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
977     DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
978     DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
979   }
980 
981   // If the source and destination are integer or pointer types, just do an
982   // extension or truncation to the desired type.
983   if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
984       (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
985     Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
986     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
987     return;
988   }
989 
990   uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
991 
992   // If store is legal, just bitcast the src pointer.
993   if (SrcSize <= DstSize) {
994     llvm::Value *Casted =
995       CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
996     // FIXME: Use better alignment / avoid requiring aligned store.
997     BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
998   } else {
999     // Otherwise do coercion through memory. This is stupid, but
1000     // simple.
1001 
1002     // Generally SrcSize is never greater than DstSize, since this means we are
1003     // losing bits. However, this can happen in cases where the structure has
1004     // additional padding, for example due to a user specified alignment.
1005     //
1006     // FIXME: Assert that we aren't truncating non-padding bits when have access
1007     // to that information.
1008     llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
1009     CGF.Builder.CreateStore(Src, Tmp);
1010     llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
1011     llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
1012     llvm::Value *DstCasted = CGF.Builder.CreateBitCast(DstPtr, I8PtrTy);
1013     // FIXME: Use better alignment.
1014     CGF.Builder.CreateMemCpy(DstCasted, Casted,
1015         llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
1016         1, false);
1017   }
1018 }
1019 
1020 namespace {
1021 
1022 /// Encapsulates information about the way function arguments from
1023 /// CGFunctionInfo should be passed to actual LLVM IR function.
1024 class ClangToLLVMArgMapping {
1025   static const unsigned InvalidIndex = ~0U;
1026   unsigned InallocaArgNo;
1027   unsigned SRetArgNo;
1028   unsigned TotalIRArgs;
1029 
1030   /// Arguments of LLVM IR function corresponding to single Clang argument.
1031   struct IRArgs {
1032     unsigned PaddingArgIndex;
1033     // Argument is expanded to IR arguments at positions
1034     // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1035     unsigned FirstArgIndex;
1036     unsigned NumberOfArgs;
1037 
1038     IRArgs()
1039         : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1040           NumberOfArgs(0) {}
1041   };
1042 
1043   SmallVector<IRArgs, 8> ArgInfo;
1044 
1045 public:
1046   ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1047                         bool OnlyRequiredArgs = false)
1048       : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1049         ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1050     construct(Context, FI, OnlyRequiredArgs);
1051   }
1052 
1053   bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1054   unsigned getInallocaArgNo() const {
1055     assert(hasInallocaArg());
1056     return InallocaArgNo;
1057   }
1058 
1059   bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1060   unsigned getSRetArgNo() const {
1061     assert(hasSRetArg());
1062     return SRetArgNo;
1063   }
1064 
1065   unsigned totalIRArgs() const { return TotalIRArgs; }
1066 
1067   bool hasPaddingArg(unsigned ArgNo) const {
1068     assert(ArgNo < ArgInfo.size());
1069     return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1070   }
1071   unsigned getPaddingArgNo(unsigned ArgNo) const {
1072     assert(hasPaddingArg(ArgNo));
1073     return ArgInfo[ArgNo].PaddingArgIndex;
1074   }
1075 
1076   /// Returns index of first IR argument corresponding to ArgNo, and their
1077   /// quantity.
1078   std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1079     assert(ArgNo < ArgInfo.size());
1080     return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1081                           ArgInfo[ArgNo].NumberOfArgs);
1082   }
1083 
1084 private:
1085   void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1086                  bool OnlyRequiredArgs);
1087 };
1088 
1089 void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1090                                       const CGFunctionInfo &FI,
1091                                       bool OnlyRequiredArgs) {
1092   unsigned IRArgNo = 0;
1093   bool SwapThisWithSRet = false;
1094   const ABIArgInfo &RetAI = FI.getReturnInfo();
1095 
1096   if (RetAI.getKind() == ABIArgInfo::Indirect) {
1097     SwapThisWithSRet = RetAI.isSRetAfterThis();
1098     SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1099   }
1100 
1101   unsigned ArgNo = 0;
1102   unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1103   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1104        ++I, ++ArgNo) {
1105     assert(I != FI.arg_end());
1106     QualType ArgType = I->type;
1107     const ABIArgInfo &AI = I->info;
1108     // Collect data about IR arguments corresponding to Clang argument ArgNo.
1109     auto &IRArgs = ArgInfo[ArgNo];
1110 
1111     if (AI.getPaddingType())
1112       IRArgs.PaddingArgIndex = IRArgNo++;
1113 
1114     switch (AI.getKind()) {
1115     case ABIArgInfo::Extend:
1116     case ABIArgInfo::Direct: {
1117       // FIXME: handle sseregparm someday...
1118       llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1119       if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1120         IRArgs.NumberOfArgs = STy->getNumElements();
1121       } else {
1122         IRArgs.NumberOfArgs = 1;
1123       }
1124       break;
1125     }
1126     case ABIArgInfo::Indirect:
1127       IRArgs.NumberOfArgs = 1;
1128       break;
1129     case ABIArgInfo::Ignore:
1130     case ABIArgInfo::InAlloca:
1131       // ignore and inalloca doesn't have matching LLVM parameters.
1132       IRArgs.NumberOfArgs = 0;
1133       break;
1134     case ABIArgInfo::Expand: {
1135       IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1136       break;
1137     }
1138     }
1139 
1140     if (IRArgs.NumberOfArgs > 0) {
1141       IRArgs.FirstArgIndex = IRArgNo;
1142       IRArgNo += IRArgs.NumberOfArgs;
1143     }
1144 
1145     // Skip over the sret parameter when it comes second.  We already handled it
1146     // above.
1147     if (IRArgNo == 1 && SwapThisWithSRet)
1148       IRArgNo++;
1149   }
1150   assert(ArgNo == ArgInfo.size());
1151 
1152   if (FI.usesInAlloca())
1153     InallocaArgNo = IRArgNo++;
1154 
1155   TotalIRArgs = IRArgNo;
1156 }
1157 }  // namespace
1158 
1159 /***/
1160 
1161 bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1162   return FI.getReturnInfo().isIndirect();
1163 }
1164 
1165 bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1166   return ReturnTypeUsesSRet(FI) &&
1167          getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1168 }
1169 
1170 bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1171   if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1172     switch (BT->getKind()) {
1173     default:
1174       return false;
1175     case BuiltinType::Float:
1176       return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1177     case BuiltinType::Double:
1178       return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1179     case BuiltinType::LongDouble:
1180       return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1181     }
1182   }
1183 
1184   return false;
1185 }
1186 
1187 bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1188   if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1189     if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1190       if (BT->getKind() == BuiltinType::LongDouble)
1191         return getTarget().useObjCFP2RetForComplexLongDouble();
1192     }
1193   }
1194 
1195   return false;
1196 }
1197 
1198 llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1199   const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1200   return GetFunctionType(FI);
1201 }
1202 
1203 llvm::FunctionType *
1204 CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1205 
1206   bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1207   (void)Inserted;
1208   assert(Inserted && "Recursively being processed?");
1209 
1210   llvm::Type *resultType = nullptr;
1211   const ABIArgInfo &retAI = FI.getReturnInfo();
1212   switch (retAI.getKind()) {
1213   case ABIArgInfo::Expand:
1214     llvm_unreachable("Invalid ABI kind for return argument");
1215 
1216   case ABIArgInfo::Extend:
1217   case ABIArgInfo::Direct:
1218     resultType = retAI.getCoerceToType();
1219     break;
1220 
1221   case ABIArgInfo::InAlloca:
1222     if (retAI.getInAllocaSRet()) {
1223       // sret things on win32 aren't void, they return the sret pointer.
1224       QualType ret = FI.getReturnType();
1225       llvm::Type *ty = ConvertType(ret);
1226       unsigned addressSpace = Context.getTargetAddressSpace(ret);
1227       resultType = llvm::PointerType::get(ty, addressSpace);
1228     } else {
1229       resultType = llvm::Type::getVoidTy(getLLVMContext());
1230     }
1231     break;
1232 
1233   case ABIArgInfo::Indirect: {
1234     assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
1235     resultType = llvm::Type::getVoidTy(getLLVMContext());
1236     break;
1237   }
1238 
1239   case ABIArgInfo::Ignore:
1240     resultType = llvm::Type::getVoidTy(getLLVMContext());
1241     break;
1242   }
1243 
1244   ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1245   SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1246 
1247   // Add type for sret argument.
1248   if (IRFunctionArgs.hasSRetArg()) {
1249     QualType Ret = FI.getReturnType();
1250     llvm::Type *Ty = ConvertType(Ret);
1251     unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1252     ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1253         llvm::PointerType::get(Ty, AddressSpace);
1254   }
1255 
1256   // Add type for inalloca argument.
1257   if (IRFunctionArgs.hasInallocaArg()) {
1258     auto ArgStruct = FI.getArgStruct();
1259     assert(ArgStruct);
1260     ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1261   }
1262 
1263   // Add in all of the required arguments.
1264   unsigned ArgNo = 0;
1265   CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1266                                      ie = it + FI.getNumRequiredArgs();
1267   for (; it != ie; ++it, ++ArgNo) {
1268     const ABIArgInfo &ArgInfo = it->info;
1269 
1270     // Insert a padding type to ensure proper alignment.
1271     if (IRFunctionArgs.hasPaddingArg(ArgNo))
1272       ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1273           ArgInfo.getPaddingType();
1274 
1275     unsigned FirstIRArg, NumIRArgs;
1276     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1277 
1278     switch (ArgInfo.getKind()) {
1279     case ABIArgInfo::Ignore:
1280     case ABIArgInfo::InAlloca:
1281       assert(NumIRArgs == 0);
1282       break;
1283 
1284     case ABIArgInfo::Indirect: {
1285       assert(NumIRArgs == 1);
1286       // indirect arguments are always on the stack, which is addr space #0.
1287       llvm::Type *LTy = ConvertTypeForMem(it->type);
1288       ArgTypes[FirstIRArg] = LTy->getPointerTo();
1289       break;
1290     }
1291 
1292     case ABIArgInfo::Extend:
1293     case ABIArgInfo::Direct: {
1294       // Fast-isel and the optimizer generally like scalar values better than
1295       // FCAs, so we flatten them if this is safe to do for this argument.
1296       llvm::Type *argType = ArgInfo.getCoerceToType();
1297       llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1298       if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1299         assert(NumIRArgs == st->getNumElements());
1300         for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1301           ArgTypes[FirstIRArg + i] = st->getElementType(i);
1302       } else {
1303         assert(NumIRArgs == 1);
1304         ArgTypes[FirstIRArg] = argType;
1305       }
1306       break;
1307     }
1308 
1309     case ABIArgInfo::Expand:
1310       auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1311       getExpandedTypes(it->type, ArgTypesIter);
1312       assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1313       break;
1314     }
1315   }
1316 
1317   bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1318   assert(Erased && "Not in set?");
1319 
1320   return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1321 }
1322 
1323 llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1324   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1325   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1326 
1327   if (!isFuncTypeConvertible(FPT))
1328     return llvm::StructType::get(getLLVMContext());
1329 
1330   const CGFunctionInfo *Info;
1331   if (isa<CXXDestructorDecl>(MD))
1332     Info =
1333         &arrangeCXXStructorDeclaration(MD, getFromDtorType(GD.getDtorType()));
1334   else
1335     Info = &arrangeCXXMethodDeclaration(MD);
1336   return GetFunctionType(*Info);
1337 }
1338 
1339 void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
1340                                            const Decl *TargetDecl,
1341                                            AttributeListType &PAL,
1342                                            unsigned &CallingConv,
1343                                            bool AttrOnCallSite) {
1344   llvm::AttrBuilder FuncAttrs;
1345   llvm::AttrBuilder RetAttrs;
1346 
1347   CallingConv = FI.getEffectiveCallingConvention();
1348 
1349   if (FI.isNoReturn())
1350     FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1351 
1352   // FIXME: handle sseregparm someday...
1353   if (TargetDecl) {
1354     if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1355       FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1356     if (TargetDecl->hasAttr<NoThrowAttr>())
1357       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1358     if (TargetDecl->hasAttr<NoReturnAttr>())
1359       FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1360     if (TargetDecl->hasAttr<NoDuplicateAttr>())
1361       FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1362 
1363     if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1364       const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
1365       if (FPT && FPT->isNothrow(getContext()))
1366         FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1367       // Don't use [[noreturn]] or _Noreturn for a call to a virtual function.
1368       // These attributes are not inherited by overloads.
1369       const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1370       if (Fn->isNoReturn() && !(AttrOnCallSite && MD && MD->isVirtual()))
1371         FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1372     }
1373 
1374     // 'const' and 'pure' attribute functions are also nounwind.
1375     if (TargetDecl->hasAttr<ConstAttr>()) {
1376       FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1377       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1378     } else if (TargetDecl->hasAttr<PureAttr>()) {
1379       FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1380       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1381     }
1382     if (TargetDecl->hasAttr<MallocAttr>())
1383       RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1384     if (TargetDecl->hasAttr<ReturnsNonNullAttr>())
1385       RetAttrs.addAttribute(llvm::Attribute::NonNull);
1386   }
1387 
1388   if (CodeGenOpts.OptimizeSize)
1389     FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1390   if (CodeGenOpts.OptimizeSize == 2)
1391     FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1392   if (CodeGenOpts.DisableRedZone)
1393     FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1394   if (CodeGenOpts.NoImplicitFloat)
1395     FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1396   if (CodeGenOpts.EnableSegmentedStacks &&
1397       !(TargetDecl && TargetDecl->hasAttr<NoSplitStackAttr>()))
1398     FuncAttrs.addAttribute("split-stack");
1399 
1400   if (AttrOnCallSite) {
1401     // Attributes that should go on the call site only.
1402     if (!CodeGenOpts.SimplifyLibCalls)
1403       FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1404   } else {
1405     // Attributes that should go on the function, but not the call site.
1406     if (!CodeGenOpts.DisableFPElim) {
1407       FuncAttrs.addAttribute("no-frame-pointer-elim", "false");
1408     } else if (CodeGenOpts.OmitLeafFramePointer) {
1409       FuncAttrs.addAttribute("no-frame-pointer-elim", "false");
1410       FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf");
1411     } else {
1412       FuncAttrs.addAttribute("no-frame-pointer-elim", "true");
1413       FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf");
1414     }
1415 
1416     FuncAttrs.addAttribute("less-precise-fpmad",
1417                            llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1418     FuncAttrs.addAttribute("no-infs-fp-math",
1419                            llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
1420     FuncAttrs.addAttribute("no-nans-fp-math",
1421                            llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
1422     FuncAttrs.addAttribute("unsafe-fp-math",
1423                            llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
1424     FuncAttrs.addAttribute("use-soft-float",
1425                            llvm::toStringRef(CodeGenOpts.SoftFloat));
1426     FuncAttrs.addAttribute("stack-protector-buffer-size",
1427                            llvm::utostr(CodeGenOpts.SSPBufferSize));
1428 
1429     if (!CodeGenOpts.StackRealignment)
1430       FuncAttrs.addAttribute("no-realign-stack");
1431   }
1432 
1433   ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
1434 
1435   QualType RetTy = FI.getReturnType();
1436   const ABIArgInfo &RetAI = FI.getReturnInfo();
1437   switch (RetAI.getKind()) {
1438   case ABIArgInfo::Extend:
1439     if (RetTy->hasSignedIntegerRepresentation())
1440       RetAttrs.addAttribute(llvm::Attribute::SExt);
1441     else if (RetTy->hasUnsignedIntegerRepresentation())
1442       RetAttrs.addAttribute(llvm::Attribute::ZExt);
1443     // FALL THROUGH
1444   case ABIArgInfo::Direct:
1445     if (RetAI.getInReg())
1446       RetAttrs.addAttribute(llvm::Attribute::InReg);
1447     break;
1448   case ABIArgInfo::Ignore:
1449     break;
1450 
1451   case ABIArgInfo::InAlloca:
1452   case ABIArgInfo::Indirect: {
1453     // inalloca and sret disable readnone and readonly
1454     FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1455       .removeAttribute(llvm::Attribute::ReadNone);
1456     break;
1457   }
1458 
1459   case ABIArgInfo::Expand:
1460     llvm_unreachable("Invalid ABI kind for return argument");
1461   }
1462 
1463   if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
1464     QualType PTy = RefTy->getPointeeType();
1465     if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
1466       RetAttrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
1467                                         .getQuantity());
1468     else if (getContext().getTargetAddressSpace(PTy) == 0)
1469       RetAttrs.addAttribute(llvm::Attribute::NonNull);
1470   }
1471 
1472   // Attach return attributes.
1473   if (RetAttrs.hasAttributes()) {
1474     PAL.push_back(llvm::AttributeSet::get(
1475         getLLVMContext(), llvm::AttributeSet::ReturnIndex, RetAttrs));
1476   }
1477 
1478   // Attach attributes to sret.
1479   if (IRFunctionArgs.hasSRetArg()) {
1480     llvm::AttrBuilder SRETAttrs;
1481     SRETAttrs.addAttribute(llvm::Attribute::StructRet);
1482     if (RetAI.getInReg())
1483       SRETAttrs.addAttribute(llvm::Attribute::InReg);
1484     PAL.push_back(llvm::AttributeSet::get(
1485         getLLVMContext(), IRFunctionArgs.getSRetArgNo() + 1, SRETAttrs));
1486   }
1487 
1488   // Attach attributes to inalloca argument.
1489   if (IRFunctionArgs.hasInallocaArg()) {
1490     llvm::AttrBuilder Attrs;
1491     Attrs.addAttribute(llvm::Attribute::InAlloca);
1492     PAL.push_back(llvm::AttributeSet::get(
1493         getLLVMContext(), IRFunctionArgs.getInallocaArgNo() + 1, Attrs));
1494   }
1495 
1496 
1497   unsigned ArgNo = 0;
1498   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
1499                                           E = FI.arg_end();
1500        I != E; ++I, ++ArgNo) {
1501     QualType ParamType = I->type;
1502     const ABIArgInfo &AI = I->info;
1503     llvm::AttrBuilder Attrs;
1504 
1505     // Add attribute for padding argument, if necessary.
1506     if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
1507       if (AI.getPaddingInReg())
1508         PAL.push_back(llvm::AttributeSet::get(
1509             getLLVMContext(), IRFunctionArgs.getPaddingArgNo(ArgNo) + 1,
1510             llvm::Attribute::InReg));
1511     }
1512 
1513     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
1514     // have the corresponding parameter variable.  It doesn't make
1515     // sense to do it here because parameters are so messed up.
1516     switch (AI.getKind()) {
1517     case ABIArgInfo::Extend:
1518       if (ParamType->isSignedIntegerOrEnumerationType())
1519         Attrs.addAttribute(llvm::Attribute::SExt);
1520       else if (ParamType->isUnsignedIntegerOrEnumerationType())
1521         Attrs.addAttribute(llvm::Attribute::ZExt);
1522       // FALL THROUGH
1523     case ABIArgInfo::Direct:
1524       if (AI.getInReg())
1525         Attrs.addAttribute(llvm::Attribute::InReg);
1526       break;
1527 
1528     case ABIArgInfo::Indirect:
1529       if (AI.getInReg())
1530         Attrs.addAttribute(llvm::Attribute::InReg);
1531 
1532       if (AI.getIndirectByVal())
1533         Attrs.addAttribute(llvm::Attribute::ByVal);
1534 
1535       Attrs.addAlignmentAttr(AI.getIndirectAlign());
1536 
1537       // byval disables readnone and readonly.
1538       FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1539         .removeAttribute(llvm::Attribute::ReadNone);
1540       break;
1541 
1542     case ABIArgInfo::Ignore:
1543     case ABIArgInfo::Expand:
1544       continue;
1545 
1546     case ABIArgInfo::InAlloca:
1547       // inalloca disables readnone and readonly.
1548       FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1549           .removeAttribute(llvm::Attribute::ReadNone);
1550       continue;
1551     }
1552 
1553     if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
1554       QualType PTy = RefTy->getPointeeType();
1555       if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
1556         Attrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
1557                                        .getQuantity());
1558       else if (getContext().getTargetAddressSpace(PTy) == 0)
1559         Attrs.addAttribute(llvm::Attribute::NonNull);
1560     }
1561 
1562     if (Attrs.hasAttributes()) {
1563       unsigned FirstIRArg, NumIRArgs;
1564       std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1565       for (unsigned i = 0; i < NumIRArgs; i++)
1566         PAL.push_back(llvm::AttributeSet::get(getLLVMContext(),
1567                                               FirstIRArg + i + 1, Attrs));
1568     }
1569   }
1570   assert(ArgNo == FI.arg_size());
1571 
1572   if (FuncAttrs.hasAttributes())
1573     PAL.push_back(llvm::
1574                   AttributeSet::get(getLLVMContext(),
1575                                     llvm::AttributeSet::FunctionIndex,
1576                                     FuncAttrs));
1577 }
1578 
1579 /// An argument came in as a promoted argument; demote it back to its
1580 /// declared type.
1581 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
1582                                          const VarDecl *var,
1583                                          llvm::Value *value) {
1584   llvm::Type *varType = CGF.ConvertType(var->getType());
1585 
1586   // This can happen with promotions that actually don't change the
1587   // underlying type, like the enum promotions.
1588   if (value->getType() == varType) return value;
1589 
1590   assert((varType->isIntegerTy() || varType->isFloatingPointTy())
1591          && "unexpected promotion type");
1592 
1593   if (isa<llvm::IntegerType>(varType))
1594     return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
1595 
1596   return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
1597 }
1598 
1599 /// Returns the attribute (either parameter attribute, or function
1600 /// attribute), which declares argument ArgNo to be non-null.
1601 static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
1602                                          QualType ArgType, unsigned ArgNo) {
1603   // FIXME: __attribute__((nonnull)) can also be applied to:
1604   //   - references to pointers, where the pointee is known to be
1605   //     nonnull (apparently a Clang extension)
1606   //   - transparent unions containing pointers
1607   // In the former case, LLVM IR cannot represent the constraint. In
1608   // the latter case, we have no guarantee that the transparent union
1609   // is in fact passed as a pointer.
1610   if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
1611     return nullptr;
1612   // First, check attribute on parameter itself.
1613   if (PVD) {
1614     if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
1615       return ParmNNAttr;
1616   }
1617   // Check function attributes.
1618   if (!FD)
1619     return nullptr;
1620   for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
1621     if (NNAttr->isNonNull(ArgNo))
1622       return NNAttr;
1623   }
1624   return nullptr;
1625 }
1626 
1627 void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
1628                                          llvm::Function *Fn,
1629                                          const FunctionArgList &Args) {
1630   if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
1631     // Naked functions don't have prologues.
1632     return;
1633 
1634   // If this is an implicit-return-zero function, go ahead and
1635   // initialize the return value.  TODO: it might be nice to have
1636   // a more general mechanism for this that didn't require synthesized
1637   // return statements.
1638   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
1639     if (FD->hasImplicitReturnZero()) {
1640       QualType RetTy = FD->getReturnType().getUnqualifiedType();
1641       llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
1642       llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
1643       Builder.CreateStore(Zero, ReturnValue);
1644     }
1645   }
1646 
1647   // FIXME: We no longer need the types from FunctionArgList; lift up and
1648   // simplify.
1649 
1650   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
1651   // Flattened function arguments.
1652   SmallVector<llvm::Argument *, 16> FnArgs;
1653   FnArgs.reserve(IRFunctionArgs.totalIRArgs());
1654   for (auto &Arg : Fn->args()) {
1655     FnArgs.push_back(&Arg);
1656   }
1657   assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
1658 
1659   // If we're using inalloca, all the memory arguments are GEPs off of the last
1660   // parameter, which is a pointer to the complete memory area.
1661   llvm::Value *ArgStruct = nullptr;
1662   if (IRFunctionArgs.hasInallocaArg()) {
1663     ArgStruct = FnArgs[IRFunctionArgs.getInallocaArgNo()];
1664     assert(ArgStruct->getType() == FI.getArgStruct()->getPointerTo());
1665   }
1666 
1667   // Name the struct return parameter.
1668   if (IRFunctionArgs.hasSRetArg()) {
1669     auto AI = FnArgs[IRFunctionArgs.getSRetArgNo()];
1670     AI->setName("agg.result");
1671     AI->addAttr(llvm::AttributeSet::get(getLLVMContext(), AI->getArgNo() + 1,
1672                                         llvm::Attribute::NoAlias));
1673   }
1674 
1675   // Track if we received the parameter as a pointer (indirect, byval, or
1676   // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
1677   // into a local alloca for us.
1678   enum ValOrPointer { HaveValue = 0, HavePointer = 1 };
1679   typedef llvm::PointerIntPair<llvm::Value *, 1> ValueAndIsPtr;
1680   SmallVector<ValueAndIsPtr, 16> ArgVals;
1681   ArgVals.reserve(Args.size());
1682 
1683   // Create a pointer value for every parameter declaration.  This usually
1684   // entails copying one or more LLVM IR arguments into an alloca.  Don't push
1685   // any cleanups or do anything that might unwind.  We do that separately, so
1686   // we can push the cleanups in the correct order for the ABI.
1687   assert(FI.arg_size() == Args.size() &&
1688          "Mismatch between function signature & arguments.");
1689   unsigned ArgNo = 0;
1690   CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
1691   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1692        i != e; ++i, ++info_it, ++ArgNo) {
1693     const VarDecl *Arg = *i;
1694     QualType Ty = info_it->type;
1695     const ABIArgInfo &ArgI = info_it->info;
1696 
1697     bool isPromoted =
1698       isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
1699 
1700     unsigned FirstIRArg, NumIRArgs;
1701     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1702 
1703     switch (ArgI.getKind()) {
1704     case ABIArgInfo::InAlloca: {
1705       assert(NumIRArgs == 0);
1706       llvm::Value *V = Builder.CreateStructGEP(
1707           ArgStruct, ArgI.getInAllocaFieldIndex(), Arg->getName());
1708       ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1709       break;
1710     }
1711 
1712     case ABIArgInfo::Indirect: {
1713       assert(NumIRArgs == 1);
1714       llvm::Value *V = FnArgs[FirstIRArg];
1715 
1716       if (!hasScalarEvaluationKind(Ty)) {
1717         // Aggregates and complex variables are accessed by reference.  All we
1718         // need to do is realign the value, if requested
1719         if (ArgI.getIndirectRealign()) {
1720           llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
1721 
1722           // Copy from the incoming argument pointer to the temporary with the
1723           // appropriate alignment.
1724           //
1725           // FIXME: We should have a common utility for generating an aggregate
1726           // copy.
1727           llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
1728           CharUnits Size = getContext().getTypeSizeInChars(Ty);
1729           llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
1730           llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
1731           Builder.CreateMemCpy(Dst,
1732                                Src,
1733                                llvm::ConstantInt::get(IntPtrTy,
1734                                                       Size.getQuantity()),
1735                                ArgI.getIndirectAlign(),
1736                                false);
1737           V = AlignedTemp;
1738         }
1739         ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1740       } else {
1741         // Load scalar value from indirect argument.
1742         CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
1743         V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty,
1744                              Arg->getLocStart());
1745 
1746         if (isPromoted)
1747           V = emitArgumentDemotion(*this, Arg, V);
1748         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1749       }
1750       break;
1751     }
1752 
1753     case ABIArgInfo::Extend:
1754     case ABIArgInfo::Direct: {
1755 
1756       // If we have the trivial case, handle it with no muss and fuss.
1757       if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
1758           ArgI.getCoerceToType() == ConvertType(Ty) &&
1759           ArgI.getDirectOffset() == 0) {
1760         assert(NumIRArgs == 1);
1761         auto AI = FnArgs[FirstIRArg];
1762         llvm::Value *V = AI;
1763 
1764         if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
1765           if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
1766                              PVD->getFunctionScopeIndex()))
1767             AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1768                                                 AI->getArgNo() + 1,
1769                                                 llvm::Attribute::NonNull));
1770 
1771           QualType OTy = PVD->getOriginalType();
1772           if (const auto *ArrTy =
1773               getContext().getAsConstantArrayType(OTy)) {
1774             // A C99 array parameter declaration with the static keyword also
1775             // indicates dereferenceability, and if the size is constant we can
1776             // use the dereferenceable attribute (which requires the size in
1777             // bytes).
1778             if (ArrTy->getSizeModifier() == ArrayType::Static) {
1779               QualType ETy = ArrTy->getElementType();
1780               uint64_t ArrSize = ArrTy->getSize().getZExtValue();
1781               if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
1782                   ArrSize) {
1783                 llvm::AttrBuilder Attrs;
1784                 Attrs.addDereferenceableAttr(
1785                   getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
1786                 AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1787                                                     AI->getArgNo() + 1, Attrs));
1788               } else if (getContext().getTargetAddressSpace(ETy) == 0) {
1789                 AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1790                                                     AI->getArgNo() + 1,
1791                                                     llvm::Attribute::NonNull));
1792               }
1793             }
1794           } else if (const auto *ArrTy =
1795                      getContext().getAsVariableArrayType(OTy)) {
1796             // For C99 VLAs with the static keyword, we don't know the size so
1797             // we can't use the dereferenceable attribute, but in addrspace(0)
1798             // we know that it must be nonnull.
1799             if (ArrTy->getSizeModifier() == VariableArrayType::Static &&
1800                 !getContext().getTargetAddressSpace(ArrTy->getElementType()))
1801               AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1802                                                   AI->getArgNo() + 1,
1803                                                   llvm::Attribute::NonNull));
1804           }
1805 
1806           const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
1807           if (!AVAttr)
1808             if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
1809               AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
1810           if (AVAttr) {
1811             llvm::Value *AlignmentValue =
1812               EmitScalarExpr(AVAttr->getAlignment());
1813             llvm::ConstantInt *AlignmentCI =
1814               cast<llvm::ConstantInt>(AlignmentValue);
1815             unsigned Alignment =
1816               std::min((unsigned) AlignmentCI->getZExtValue(),
1817                        +llvm::Value::MaximumAlignment);
1818 
1819             llvm::AttrBuilder Attrs;
1820             Attrs.addAlignmentAttr(Alignment);
1821             AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1822                                                 AI->getArgNo() + 1, Attrs));
1823           }
1824         }
1825 
1826         if (Arg->getType().isRestrictQualified())
1827           AI->addAttr(llvm::AttributeSet::get(getLLVMContext(),
1828                                               AI->getArgNo() + 1,
1829                                               llvm::Attribute::NoAlias));
1830 
1831         // Ensure the argument is the correct type.
1832         if (V->getType() != ArgI.getCoerceToType())
1833           V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
1834 
1835         if (isPromoted)
1836           V = emitArgumentDemotion(*this, Arg, V);
1837 
1838         if (const CXXMethodDecl *MD =
1839             dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl)) {
1840           if (MD->isVirtual() && Arg == CXXABIThisDecl)
1841             V = CGM.getCXXABI().
1842                 adjustThisParameterInVirtualFunctionPrologue(*this, CurGD, V);
1843         }
1844 
1845         // Because of merging of function types from multiple decls it is
1846         // possible for the type of an argument to not match the corresponding
1847         // type in the function type. Since we are codegening the callee
1848         // in here, add a cast to the argument type.
1849         llvm::Type *LTy = ConvertType(Arg->getType());
1850         if (V->getType() != LTy)
1851           V = Builder.CreateBitCast(V, LTy);
1852 
1853         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1854         break;
1855       }
1856 
1857       llvm::AllocaInst *Alloca = CreateMemTemp(Ty, Arg->getName());
1858 
1859       // The alignment we need to use is the max of the requested alignment for
1860       // the argument plus the alignment required by our access code below.
1861       unsigned AlignmentToUse =
1862         CGM.getDataLayout().getABITypeAlignment(ArgI.getCoerceToType());
1863       AlignmentToUse = std::max(AlignmentToUse,
1864                         (unsigned)getContext().getDeclAlign(Arg).getQuantity());
1865 
1866       Alloca->setAlignment(AlignmentToUse);
1867       llvm::Value *V = Alloca;
1868       llvm::Value *Ptr = V;    // Pointer to store into.
1869 
1870       // If the value is offset in memory, apply the offset now.
1871       if (unsigned Offs = ArgI.getDirectOffset()) {
1872         Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
1873         Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
1874         Ptr = Builder.CreateBitCast(Ptr,
1875                           llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
1876       }
1877 
1878       // Fast-isel and the optimizer generally like scalar values better than
1879       // FCAs, so we flatten them if this is safe to do for this argument.
1880       llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
1881       if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
1882           STy->getNumElements() > 1) {
1883         uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
1884         llvm::Type *DstTy =
1885           cast<llvm::PointerType>(Ptr->getType())->getElementType();
1886         uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
1887 
1888         if (SrcSize <= DstSize) {
1889           Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
1890 
1891           assert(STy->getNumElements() == NumIRArgs);
1892           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1893             auto AI = FnArgs[FirstIRArg + i];
1894             AI->setName(Arg->getName() + ".coerce" + Twine(i));
1895             llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
1896             Builder.CreateStore(AI, EltPtr);
1897           }
1898         } else {
1899           llvm::AllocaInst *TempAlloca =
1900             CreateTempAlloca(ArgI.getCoerceToType(), "coerce");
1901           TempAlloca->setAlignment(AlignmentToUse);
1902           llvm::Value *TempV = TempAlloca;
1903 
1904           assert(STy->getNumElements() == NumIRArgs);
1905           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1906             auto AI = FnArgs[FirstIRArg + i];
1907             AI->setName(Arg->getName() + ".coerce" + Twine(i));
1908             llvm::Value *EltPtr = Builder.CreateConstGEP2_32(TempV, 0, i);
1909             Builder.CreateStore(AI, EltPtr);
1910           }
1911 
1912           Builder.CreateMemCpy(Ptr, TempV, DstSize, AlignmentToUse);
1913         }
1914       } else {
1915         // Simple case, just do a coerced store of the argument into the alloca.
1916         assert(NumIRArgs == 1);
1917         auto AI = FnArgs[FirstIRArg];
1918         AI->setName(Arg->getName() + ".coerce");
1919         CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, *this);
1920       }
1921 
1922 
1923       // Match to what EmitParmDecl is expecting for this type.
1924       if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
1925         V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty, Arg->getLocStart());
1926         if (isPromoted)
1927           V = emitArgumentDemotion(*this, Arg, V);
1928         ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
1929       } else {
1930         ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
1931       }
1932       break;
1933     }
1934 
1935     case ABIArgInfo::Expand: {
1936       // If this structure was expanded into multiple arguments then
1937       // we need to create a temporary and reconstruct it from the
1938       // arguments.
1939       llvm::AllocaInst *Alloca = CreateMemTemp(Ty);
1940       CharUnits Align = getContext().getDeclAlign(Arg);
1941       Alloca->setAlignment(Align.getQuantity());
1942       LValue LV = MakeAddrLValue(Alloca, Ty, Align);
1943       ArgVals.push_back(ValueAndIsPtr(Alloca, HavePointer));
1944 
1945       auto FnArgIter = FnArgs.begin() + FirstIRArg;
1946       ExpandTypeFromArgs(Ty, LV, FnArgIter);
1947       assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
1948       for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
1949         auto AI = FnArgs[FirstIRArg + i];
1950         AI->setName(Arg->getName() + "." + Twine(i));
1951       }
1952       break;
1953     }
1954 
1955     case ABIArgInfo::Ignore:
1956       assert(NumIRArgs == 0);
1957       // Initialize the local variable appropriately.
1958       if (!hasScalarEvaluationKind(Ty)) {
1959         ArgVals.push_back(ValueAndIsPtr(CreateMemTemp(Ty), HavePointer));
1960       } else {
1961         llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
1962         ArgVals.push_back(ValueAndIsPtr(U, HaveValue));
1963       }
1964       break;
1965     }
1966   }
1967 
1968   if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
1969     for (int I = Args.size() - 1; I >= 0; --I)
1970       EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
1971                    I + 1);
1972   } else {
1973     for (unsigned I = 0, E = Args.size(); I != E; ++I)
1974       EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
1975                    I + 1);
1976   }
1977 }
1978 
1979 static void eraseUnusedBitCasts(llvm::Instruction *insn) {
1980   while (insn->use_empty()) {
1981     llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
1982     if (!bitcast) return;
1983 
1984     // This is "safe" because we would have used a ConstantExpr otherwise.
1985     insn = cast<llvm::Instruction>(bitcast->getOperand(0));
1986     bitcast->eraseFromParent();
1987   }
1988 }
1989 
1990 /// Try to emit a fused autorelease of a return result.
1991 static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1992                                                     llvm::Value *result) {
1993   // We must be immediately followed the cast.
1994   llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1995   if (BB->empty()) return nullptr;
1996   if (&BB->back() != result) return nullptr;
1997 
1998   llvm::Type *resultType = result->getType();
1999 
2000   // result is in a BasicBlock and is therefore an Instruction.
2001   llvm::Instruction *generator = cast<llvm::Instruction>(result);
2002 
2003   SmallVector<llvm::Instruction*,4> insnsToKill;
2004 
2005   // Look for:
2006   //  %generator = bitcast %type1* %generator2 to %type2*
2007   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2008     // We would have emitted this as a constant if the operand weren't
2009     // an Instruction.
2010     generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2011 
2012     // Require the generator to be immediately followed by the cast.
2013     if (generator->getNextNode() != bitcast)
2014       return nullptr;
2015 
2016     insnsToKill.push_back(bitcast);
2017   }
2018 
2019   // Look for:
2020   //   %generator = call i8* @objc_retain(i8* %originalResult)
2021   // or
2022   //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2023   llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
2024   if (!call) return nullptr;
2025 
2026   bool doRetainAutorelease;
2027 
2028   if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
2029     doRetainAutorelease = true;
2030   } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
2031                                           .objc_retainAutoreleasedReturnValue) {
2032     doRetainAutorelease = false;
2033 
2034     // If we emitted an assembly marker for this call (and the
2035     // ARCEntrypoints field should have been set if so), go looking
2036     // for that call.  If we can't find it, we can't do this
2037     // optimization.  But it should always be the immediately previous
2038     // instruction, unless we needed bitcasts around the call.
2039     if (CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker) {
2040       llvm::Instruction *prev = call->getPrevNode();
2041       assert(prev);
2042       if (isa<llvm::BitCastInst>(prev)) {
2043         prev = prev->getPrevNode();
2044         assert(prev);
2045       }
2046       assert(isa<llvm::CallInst>(prev));
2047       assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
2048                CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker);
2049       insnsToKill.push_back(prev);
2050     }
2051   } else {
2052     return nullptr;
2053   }
2054 
2055   result = call->getArgOperand(0);
2056   insnsToKill.push_back(call);
2057 
2058   // Keep killing bitcasts, for sanity.  Note that we no longer care
2059   // about precise ordering as long as there's exactly one use.
2060   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2061     if (!bitcast->hasOneUse()) break;
2062     insnsToKill.push_back(bitcast);
2063     result = bitcast->getOperand(0);
2064   }
2065 
2066   // Delete all the unnecessary instructions, from latest to earliest.
2067   for (SmallVectorImpl<llvm::Instruction*>::iterator
2068          i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
2069     (*i)->eraseFromParent();
2070 
2071   // Do the fused retain/autorelease if we were asked to.
2072   if (doRetainAutorelease)
2073     result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2074 
2075   // Cast back to the result type.
2076   return CGF.Builder.CreateBitCast(result, resultType);
2077 }
2078 
2079 /// If this is a +1 of the value of an immutable 'self', remove it.
2080 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2081                                           llvm::Value *result) {
2082   // This is only applicable to a method with an immutable 'self'.
2083   const ObjCMethodDecl *method =
2084     dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
2085   if (!method) return nullptr;
2086   const VarDecl *self = method->getSelfDecl();
2087   if (!self->getType().isConstQualified()) return nullptr;
2088 
2089   // Look for a retain call.
2090   llvm::CallInst *retainCall =
2091     dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2092   if (!retainCall ||
2093       retainCall->getCalledValue() != CGF.CGM.getARCEntrypoints().objc_retain)
2094     return nullptr;
2095 
2096   // Look for an ordinary load of 'self'.
2097   llvm::Value *retainedValue = retainCall->getArgOperand(0);
2098   llvm::LoadInst *load =
2099     dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
2100   if (!load || load->isAtomic() || load->isVolatile() ||
2101       load->getPointerOperand() != CGF.GetAddrOfLocalVar(self))
2102     return nullptr;
2103 
2104   // Okay!  Burn it all down.  This relies for correctness on the
2105   // assumption that the retain is emitted as part of the return and
2106   // that thereafter everything is used "linearly".
2107   llvm::Type *resultType = result->getType();
2108   eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2109   assert(retainCall->use_empty());
2110   retainCall->eraseFromParent();
2111   eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2112 
2113   return CGF.Builder.CreateBitCast(load, resultType);
2114 }
2115 
2116 /// Emit an ARC autorelease of the result of a function.
2117 ///
2118 /// \return the value to actually return from the function
2119 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2120                                             llvm::Value *result) {
2121   // If we're returning 'self', kill the initial retain.  This is a
2122   // heuristic attempt to "encourage correctness" in the really unfortunate
2123   // case where we have a return of self during a dealloc and we desperately
2124   // need to avoid the possible autorelease.
2125   if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2126     return self;
2127 
2128   // At -O0, try to emit a fused retain/autorelease.
2129   if (CGF.shouldUseFusedARCCalls())
2130     if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2131       return fused;
2132 
2133   return CGF.EmitARCAutoreleaseReturnValue(result);
2134 }
2135 
2136 /// Heuristically search for a dominating store to the return-value slot.
2137 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
2138   // If there are multiple uses of the return-value slot, just check
2139   // for something immediately preceding the IP.  Sometimes this can
2140   // happen with how we generate implicit-returns; it can also happen
2141   // with noreturn cleanups.
2142   if (!CGF.ReturnValue->hasOneUse()) {
2143     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2144     if (IP->empty()) return nullptr;
2145     llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(&IP->back());
2146     if (!store) return nullptr;
2147     if (store->getPointerOperand() != CGF.ReturnValue) return nullptr;
2148     assert(!store->isAtomic() && !store->isVolatile()); // see below
2149     return store;
2150   }
2151 
2152   llvm::StoreInst *store =
2153     dyn_cast<llvm::StoreInst>(CGF.ReturnValue->user_back());
2154   if (!store) return nullptr;
2155 
2156   // These aren't actually possible for non-coerced returns, and we
2157   // only care about non-coerced returns on this code path.
2158   assert(!store->isAtomic() && !store->isVolatile());
2159 
2160   // Now do a first-and-dirty dominance check: just walk up the
2161   // single-predecessors chain from the current insertion point.
2162   llvm::BasicBlock *StoreBB = store->getParent();
2163   llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2164   while (IP != StoreBB) {
2165     if (!(IP = IP->getSinglePredecessor()))
2166       return nullptr;
2167   }
2168 
2169   // Okay, the store's basic block dominates the insertion point; we
2170   // can do our thing.
2171   return store;
2172 }
2173 
2174 void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
2175                                          bool EmitRetDbgLoc,
2176                                          SourceLocation EndLoc) {
2177   if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
2178     // Naked functions don't have epilogues.
2179     Builder.CreateUnreachable();
2180     return;
2181   }
2182 
2183   // Functions with no result always return void.
2184   if (!ReturnValue) {
2185     Builder.CreateRetVoid();
2186     return;
2187   }
2188 
2189   llvm::DebugLoc RetDbgLoc;
2190   llvm::Value *RV = nullptr;
2191   QualType RetTy = FI.getReturnType();
2192   const ABIArgInfo &RetAI = FI.getReturnInfo();
2193 
2194   switch (RetAI.getKind()) {
2195   case ABIArgInfo::InAlloca:
2196     // Aggregrates get evaluated directly into the destination.  Sometimes we
2197     // need to return the sret value in a register, though.
2198     assert(hasAggregateEvaluationKind(RetTy));
2199     if (RetAI.getInAllocaSRet()) {
2200       llvm::Function::arg_iterator EI = CurFn->arg_end();
2201       --EI;
2202       llvm::Value *ArgStruct = EI;
2203       llvm::Value *SRet =
2204           Builder.CreateStructGEP(ArgStruct, RetAI.getInAllocaFieldIndex());
2205       RV = Builder.CreateLoad(SRet, "sret");
2206     }
2207     break;
2208 
2209   case ABIArgInfo::Indirect: {
2210     auto AI = CurFn->arg_begin();
2211     if (RetAI.isSRetAfterThis())
2212       ++AI;
2213     switch (getEvaluationKind(RetTy)) {
2214     case TEK_Complex: {
2215       ComplexPairTy RT =
2216         EmitLoadOfComplex(MakeNaturalAlignAddrLValue(ReturnValue, RetTy),
2217                           EndLoc);
2218       EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(AI, RetTy),
2219                          /*isInit*/ true);
2220       break;
2221     }
2222     case TEK_Aggregate:
2223       // Do nothing; aggregrates get evaluated directly into the destination.
2224       break;
2225     case TEK_Scalar:
2226       EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
2227                         MakeNaturalAlignAddrLValue(AI, RetTy),
2228                         /*isInit*/ true);
2229       break;
2230     }
2231     break;
2232   }
2233 
2234   case ABIArgInfo::Extend:
2235   case ABIArgInfo::Direct:
2236     if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
2237         RetAI.getDirectOffset() == 0) {
2238       // The internal return value temp always will have pointer-to-return-type
2239       // type, just do a load.
2240 
2241       // If there is a dominating store to ReturnValue, we can elide
2242       // the load, zap the store, and usually zap the alloca.
2243       if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) {
2244         // Reuse the debug location from the store unless there is
2245         // cleanup code to be emitted between the store and return
2246         // instruction.
2247         if (EmitRetDbgLoc && !AutoreleaseResult)
2248           RetDbgLoc = SI->getDebugLoc();
2249         // Get the stored value and nuke the now-dead store.
2250         RV = SI->getValueOperand();
2251         SI->eraseFromParent();
2252 
2253         // If that was the only use of the return value, nuke it as well now.
2254         if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
2255           cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
2256           ReturnValue = nullptr;
2257         }
2258 
2259       // Otherwise, we have to do a simple load.
2260       } else {
2261         RV = Builder.CreateLoad(ReturnValue);
2262       }
2263     } else {
2264       llvm::Value *V = ReturnValue;
2265       // If the value is offset in memory, apply the offset now.
2266       if (unsigned Offs = RetAI.getDirectOffset()) {
2267         V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
2268         V = Builder.CreateConstGEP1_32(V, Offs);
2269         V = Builder.CreateBitCast(V,
2270                          llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
2271       }
2272 
2273       RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
2274     }
2275 
2276     // In ARC, end functions that return a retainable type with a call
2277     // to objc_autoreleaseReturnValue.
2278     if (AutoreleaseResult) {
2279       assert(getLangOpts().ObjCAutoRefCount &&
2280              !FI.isReturnsRetained() &&
2281              RetTy->isObjCRetainableType());
2282       RV = emitAutoreleaseOfResult(*this, RV);
2283     }
2284 
2285     break;
2286 
2287   case ABIArgInfo::Ignore:
2288     break;
2289 
2290   case ABIArgInfo::Expand:
2291     llvm_unreachable("Invalid ABI kind for return argument");
2292   }
2293 
2294   llvm::Instruction *Ret;
2295   if (RV) {
2296     if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) {
2297       if (auto RetNNAttr = CurGD.getDecl()->getAttr<ReturnsNonNullAttr>()) {
2298         SanitizerScope SanScope(this);
2299         llvm::Value *Cond = Builder.CreateICmpNE(
2300             RV, llvm::Constant::getNullValue(RV->getType()));
2301         llvm::Constant *StaticData[] = {
2302             EmitCheckSourceLocation(EndLoc),
2303             EmitCheckSourceLocation(RetNNAttr->getLocation()),
2304         };
2305         EmitCheck(std::make_pair(Cond, SanitizerKind::ReturnsNonnullAttribute),
2306                   "nonnull_return", StaticData, None);
2307       }
2308     }
2309     Ret = Builder.CreateRet(RV);
2310   } else {
2311     Ret = Builder.CreateRetVoid();
2312   }
2313 
2314   if (!RetDbgLoc.isUnknown())
2315     Ret->setDebugLoc(RetDbgLoc);
2316 }
2317 
2318 static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
2319   const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
2320   return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
2321 }
2322 
2323 static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
2324   // FIXME: Generate IR in one pass, rather than going back and fixing up these
2325   // placeholders.
2326   llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
2327   llvm::Value *Placeholder =
2328       llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo());
2329   Placeholder = CGF.Builder.CreateLoad(Placeholder);
2330   return AggValueSlot::forAddr(Placeholder, CharUnits::Zero(),
2331                                Ty.getQualifiers(),
2332                                AggValueSlot::IsNotDestructed,
2333                                AggValueSlot::DoesNotNeedGCBarriers,
2334                                AggValueSlot::IsNotAliased);
2335 }
2336 
2337 void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
2338                                           const VarDecl *param,
2339                                           SourceLocation loc) {
2340   // StartFunction converted the ABI-lowered parameter(s) into a
2341   // local alloca.  We need to turn that into an r-value suitable
2342   // for EmitCall.
2343   llvm::Value *local = GetAddrOfLocalVar(param);
2344 
2345   QualType type = param->getType();
2346 
2347   // For the most part, we just need to load the alloca, except:
2348   // 1) aggregate r-values are actually pointers to temporaries, and
2349   // 2) references to non-scalars are pointers directly to the aggregate.
2350   // I don't know why references to scalars are different here.
2351   if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
2352     if (!hasScalarEvaluationKind(ref->getPointeeType()))
2353       return args.add(RValue::getAggregate(local), type);
2354 
2355     // Locals which are references to scalars are represented
2356     // with allocas holding the pointer.
2357     return args.add(RValue::get(Builder.CreateLoad(local)), type);
2358   }
2359 
2360   assert(!isInAllocaArgument(CGM.getCXXABI(), type) &&
2361          "cannot emit delegate call arguments for inalloca arguments!");
2362 
2363   args.add(convertTempToRValue(local, type, loc), type);
2364 }
2365 
2366 static bool isProvablyNull(llvm::Value *addr) {
2367   return isa<llvm::ConstantPointerNull>(addr);
2368 }
2369 
2370 static bool isProvablyNonNull(llvm::Value *addr) {
2371   return isa<llvm::AllocaInst>(addr);
2372 }
2373 
2374 /// Emit the actual writing-back of a writeback.
2375 static void emitWriteback(CodeGenFunction &CGF,
2376                           const CallArgList::Writeback &writeback) {
2377   const LValue &srcLV = writeback.Source;
2378   llvm::Value *srcAddr = srcLV.getAddress();
2379   assert(!isProvablyNull(srcAddr) &&
2380          "shouldn't have writeback for provably null argument");
2381 
2382   llvm::BasicBlock *contBB = nullptr;
2383 
2384   // If the argument wasn't provably non-null, we need to null check
2385   // before doing the store.
2386   bool provablyNonNull = isProvablyNonNull(srcAddr);
2387   if (!provablyNonNull) {
2388     llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
2389     contBB = CGF.createBasicBlock("icr.done");
2390 
2391     llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
2392     CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
2393     CGF.EmitBlock(writebackBB);
2394   }
2395 
2396   // Load the value to writeback.
2397   llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
2398 
2399   // Cast it back, in case we're writing an id to a Foo* or something.
2400   value = CGF.Builder.CreateBitCast(value,
2401                cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
2402                             "icr.writeback-cast");
2403 
2404   // Perform the writeback.
2405 
2406   // If we have a "to use" value, it's something we need to emit a use
2407   // of.  This has to be carefully threaded in: if it's done after the
2408   // release it's potentially undefined behavior (and the optimizer
2409   // will ignore it), and if it happens before the retain then the
2410   // optimizer could move the release there.
2411   if (writeback.ToUse) {
2412     assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
2413 
2414     // Retain the new value.  No need to block-copy here:  the block's
2415     // being passed up the stack.
2416     value = CGF.EmitARCRetainNonBlock(value);
2417 
2418     // Emit the intrinsic use here.
2419     CGF.EmitARCIntrinsicUse(writeback.ToUse);
2420 
2421     // Load the old value (primitively).
2422     llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
2423 
2424     // Put the new value in place (primitively).
2425     CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
2426 
2427     // Release the old value.
2428     CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
2429 
2430   // Otherwise, we can just do a normal lvalue store.
2431   } else {
2432     CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
2433   }
2434 
2435   // Jump to the continuation block.
2436   if (!provablyNonNull)
2437     CGF.EmitBlock(contBB);
2438 }
2439 
2440 static void emitWritebacks(CodeGenFunction &CGF,
2441                            const CallArgList &args) {
2442   for (const auto &I : args.writebacks())
2443     emitWriteback(CGF, I);
2444 }
2445 
2446 static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
2447                                             const CallArgList &CallArgs) {
2448   assert(CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee());
2449   ArrayRef<CallArgList::CallArgCleanup> Cleanups =
2450     CallArgs.getCleanupsToDeactivate();
2451   // Iterate in reverse to increase the likelihood of popping the cleanup.
2452   for (ArrayRef<CallArgList::CallArgCleanup>::reverse_iterator
2453          I = Cleanups.rbegin(), E = Cleanups.rend(); I != E; ++I) {
2454     CGF.DeactivateCleanupBlock(I->Cleanup, I->IsActiveIP);
2455     I->IsActiveIP->eraseFromParent();
2456   }
2457 }
2458 
2459 static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
2460   if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
2461     if (uop->getOpcode() == UO_AddrOf)
2462       return uop->getSubExpr();
2463   return nullptr;
2464 }
2465 
2466 /// Emit an argument that's being passed call-by-writeback.  That is,
2467 /// we are passing the address of
2468 static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
2469                              const ObjCIndirectCopyRestoreExpr *CRE) {
2470   LValue srcLV;
2471 
2472   // Make an optimistic effort to emit the address as an l-value.
2473   // This can fail if the the argument expression is more complicated.
2474   if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
2475     srcLV = CGF.EmitLValue(lvExpr);
2476 
2477   // Otherwise, just emit it as a scalar.
2478   } else {
2479     llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
2480 
2481     QualType srcAddrType =
2482       CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
2483     srcLV = CGF.MakeNaturalAlignAddrLValue(srcAddr, srcAddrType);
2484   }
2485   llvm::Value *srcAddr = srcLV.getAddress();
2486 
2487   // The dest and src types don't necessarily match in LLVM terms
2488   // because of the crazy ObjC compatibility rules.
2489 
2490   llvm::PointerType *destType =
2491     cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
2492 
2493   // If the address is a constant null, just pass the appropriate null.
2494   if (isProvablyNull(srcAddr)) {
2495     args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
2496              CRE->getType());
2497     return;
2498   }
2499 
2500   // Create the temporary.
2501   llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
2502                                            "icr.temp");
2503   // Loading an l-value can introduce a cleanup if the l-value is __weak,
2504   // and that cleanup will be conditional if we can't prove that the l-value
2505   // isn't null, so we need to register a dominating point so that the cleanups
2506   // system will make valid IR.
2507   CodeGenFunction::ConditionalEvaluation condEval(CGF);
2508 
2509   // Zero-initialize it if we're not doing a copy-initialization.
2510   bool shouldCopy = CRE->shouldCopy();
2511   if (!shouldCopy) {
2512     llvm::Value *null =
2513       llvm::ConstantPointerNull::get(
2514         cast<llvm::PointerType>(destType->getElementType()));
2515     CGF.Builder.CreateStore(null, temp);
2516   }
2517 
2518   llvm::BasicBlock *contBB = nullptr;
2519   llvm::BasicBlock *originBB = nullptr;
2520 
2521   // If the address is *not* known to be non-null, we need to switch.
2522   llvm::Value *finalArgument;
2523 
2524   bool provablyNonNull = isProvablyNonNull(srcAddr);
2525   if (provablyNonNull) {
2526     finalArgument = temp;
2527   } else {
2528     llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
2529 
2530     finalArgument = CGF.Builder.CreateSelect(isNull,
2531                                    llvm::ConstantPointerNull::get(destType),
2532                                              temp, "icr.argument");
2533 
2534     // If we need to copy, then the load has to be conditional, which
2535     // means we need control flow.
2536     if (shouldCopy) {
2537       originBB = CGF.Builder.GetInsertBlock();
2538       contBB = CGF.createBasicBlock("icr.cont");
2539       llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
2540       CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
2541       CGF.EmitBlock(copyBB);
2542       condEval.begin(CGF);
2543     }
2544   }
2545 
2546   llvm::Value *valueToUse = nullptr;
2547 
2548   // Perform a copy if necessary.
2549   if (shouldCopy) {
2550     RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
2551     assert(srcRV.isScalar());
2552 
2553     llvm::Value *src = srcRV.getScalarVal();
2554     src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
2555                                     "icr.cast");
2556 
2557     // Use an ordinary store, not a store-to-lvalue.
2558     CGF.Builder.CreateStore(src, temp);
2559 
2560     // If optimization is enabled, and the value was held in a
2561     // __strong variable, we need to tell the optimizer that this
2562     // value has to stay alive until we're doing the store back.
2563     // This is because the temporary is effectively unretained,
2564     // and so otherwise we can violate the high-level semantics.
2565     if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
2566         srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
2567       valueToUse = src;
2568     }
2569   }
2570 
2571   // Finish the control flow if we needed it.
2572   if (shouldCopy && !provablyNonNull) {
2573     llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
2574     CGF.EmitBlock(contBB);
2575 
2576     // Make a phi for the value to intrinsically use.
2577     if (valueToUse) {
2578       llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
2579                                                       "icr.to-use");
2580       phiToUse->addIncoming(valueToUse, copyBB);
2581       phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
2582                             originBB);
2583       valueToUse = phiToUse;
2584     }
2585 
2586     condEval.end(CGF);
2587   }
2588 
2589   args.addWriteback(srcLV, temp, valueToUse);
2590   args.add(RValue::get(finalArgument), CRE->getType());
2591 }
2592 
2593 void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
2594   assert(!StackBase && !StackCleanup.isValid());
2595 
2596   // Save the stack.
2597   llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
2598   StackBase = CGF.Builder.CreateCall(F, "inalloca.save");
2599 
2600   // Control gets really tied up in landing pads, so we have to spill the
2601   // stacksave to an alloca to avoid violating SSA form.
2602   // TODO: This is dead if we never emit the cleanup.  We should create the
2603   // alloca and store lazily on the first cleanup emission.
2604   StackBaseMem = CGF.CreateTempAlloca(CGF.Int8PtrTy, "inalloca.spmem");
2605   CGF.Builder.CreateStore(StackBase, StackBaseMem);
2606   CGF.pushStackRestore(EHCleanup, StackBaseMem);
2607   StackCleanup = CGF.EHStack.getInnermostEHScope();
2608   assert(StackCleanup.isValid());
2609 }
2610 
2611 void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
2612   if (StackBase) {
2613     CGF.DeactivateCleanupBlock(StackCleanup, StackBase);
2614     llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
2615     // We could load StackBase from StackBaseMem, but in the non-exceptional
2616     // case we can skip it.
2617     CGF.Builder.CreateCall(F, StackBase);
2618   }
2619 }
2620 
2621 static void emitNonNullArgCheck(CodeGenFunction &CGF, RValue RV,
2622                                 QualType ArgType, SourceLocation ArgLoc,
2623                                 const FunctionDecl *FD, unsigned ParmNum) {
2624   if (!CGF.SanOpts.has(SanitizerKind::NonnullAttribute) || !FD)
2625     return;
2626   auto PVD = ParmNum < FD->getNumParams() ? FD->getParamDecl(ParmNum) : nullptr;
2627   unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
2628   auto NNAttr = getNonNullAttr(FD, PVD, ArgType, ArgNo);
2629   if (!NNAttr)
2630     return;
2631   CodeGenFunction::SanitizerScope SanScope(&CGF);
2632   assert(RV.isScalar());
2633   llvm::Value *V = RV.getScalarVal();
2634   llvm::Value *Cond =
2635       CGF.Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
2636   llvm::Constant *StaticData[] = {
2637       CGF.EmitCheckSourceLocation(ArgLoc),
2638       CGF.EmitCheckSourceLocation(NNAttr->getLocation()),
2639       llvm::ConstantInt::get(CGF.Int32Ty, ArgNo + 1),
2640   };
2641   CGF.EmitCheck(std::make_pair(Cond, SanitizerKind::NonnullAttribute),
2642                 "nonnull_arg", StaticData, None);
2643 }
2644 
2645 void CodeGenFunction::EmitCallArgs(CallArgList &Args,
2646                                    ArrayRef<QualType> ArgTypes,
2647                                    CallExpr::const_arg_iterator ArgBeg,
2648                                    CallExpr::const_arg_iterator ArgEnd,
2649                                    const FunctionDecl *CalleeDecl,
2650                                    unsigned ParamsToSkip,
2651                                    bool ForceColumnInfo) {
2652   CGDebugInfo *DI = getDebugInfo();
2653   SourceLocation CallLoc;
2654   if (DI) CallLoc = DI->getLocation();
2655 
2656   // We *have* to evaluate arguments from right to left in the MS C++ ABI,
2657   // because arguments are destroyed left to right in the callee.
2658   if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2659     // Insert a stack save if we're going to need any inalloca args.
2660     bool HasInAllocaArgs = false;
2661     for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
2662          I != E && !HasInAllocaArgs; ++I)
2663       HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
2664     if (HasInAllocaArgs) {
2665       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
2666       Args.allocateArgumentMemory(*this);
2667     }
2668 
2669     // Evaluate each argument.
2670     size_t CallArgsStart = Args.size();
2671     for (int I = ArgTypes.size() - 1; I >= 0; --I) {
2672       CallExpr::const_arg_iterator Arg = ArgBeg + I;
2673       EmitCallArg(Args, *Arg, ArgTypes[I]);
2674       emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
2675                           CalleeDecl, ParamsToSkip + I);
2676       // Restore the debug location.
2677       if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
2678     }
2679 
2680     // Un-reverse the arguments we just evaluated so they match up with the LLVM
2681     // IR function.
2682     std::reverse(Args.begin() + CallArgsStart, Args.end());
2683     return;
2684   }
2685 
2686   for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
2687     CallExpr::const_arg_iterator Arg = ArgBeg + I;
2688     assert(Arg != ArgEnd);
2689     EmitCallArg(Args, *Arg, ArgTypes[I]);
2690     emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
2691                         CalleeDecl, ParamsToSkip + I);
2692     // Restore the debug location.
2693     if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo);
2694   }
2695 }
2696 
2697 namespace {
2698 
2699 struct DestroyUnpassedArg : EHScopeStack::Cleanup {
2700   DestroyUnpassedArg(llvm::Value *Addr, QualType Ty)
2701       : Addr(Addr), Ty(Ty) {}
2702 
2703   llvm::Value *Addr;
2704   QualType Ty;
2705 
2706   void Emit(CodeGenFunction &CGF, Flags flags) override {
2707     const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
2708     assert(!Dtor->isTrivial());
2709     CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
2710                               /*Delegating=*/false, Addr);
2711   }
2712 };
2713 
2714 }
2715 
2716 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
2717                                   QualType type) {
2718   if (const ObjCIndirectCopyRestoreExpr *CRE
2719         = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
2720     assert(getLangOpts().ObjCAutoRefCount);
2721     assert(getContext().hasSameType(E->getType(), type));
2722     return emitWritebackArg(*this, args, CRE);
2723   }
2724 
2725   assert(type->isReferenceType() == E->isGLValue() &&
2726          "reference binding to unmaterialized r-value!");
2727 
2728   if (E->isGLValue()) {
2729     assert(E->getObjectKind() == OK_Ordinary);
2730     return args.add(EmitReferenceBindingToExpr(E), type);
2731   }
2732 
2733   bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
2734 
2735   // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
2736   // However, we still have to push an EH-only cleanup in case we unwind before
2737   // we make it to the call.
2738   if (HasAggregateEvalKind &&
2739       CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2740     // If we're using inalloca, use the argument memory.  Otherwise, use a
2741     // temporary.
2742     AggValueSlot Slot;
2743     if (args.isUsingInAlloca())
2744       Slot = createPlaceholderSlot(*this, type);
2745     else
2746       Slot = CreateAggTemp(type, "agg.tmp");
2747 
2748     const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
2749     bool DestroyedInCallee =
2750         RD && RD->hasNonTrivialDestructor() &&
2751         CGM.getCXXABI().getRecordArgABI(RD) != CGCXXABI::RAA_Default;
2752     if (DestroyedInCallee)
2753       Slot.setExternallyDestructed();
2754 
2755     EmitAggExpr(E, Slot);
2756     RValue RV = Slot.asRValue();
2757     args.add(RV, type);
2758 
2759     if (DestroyedInCallee) {
2760       // Create a no-op GEP between the placeholder and the cleanup so we can
2761       // RAUW it successfully.  It also serves as a marker of the first
2762       // instruction where the cleanup is active.
2763       pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddr(), type);
2764       // This unreachable is a temporary marker which will be removed later.
2765       llvm::Instruction *IsActive = Builder.CreateUnreachable();
2766       args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
2767     }
2768     return;
2769   }
2770 
2771   if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
2772       cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
2773     LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
2774     assert(L.isSimple());
2775     if (L.getAlignment() >= getContext().getTypeAlignInChars(type)) {
2776       args.add(L.asAggregateRValue(), type, /*NeedsCopy*/true);
2777     } else {
2778       // We can't represent a misaligned lvalue in the CallArgList, so copy
2779       // to an aligned temporary now.
2780       llvm::Value *tmp = CreateMemTemp(type);
2781       EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile(),
2782                         L.getAlignment());
2783       args.add(RValue::getAggregate(tmp), type);
2784     }
2785     return;
2786   }
2787 
2788   args.add(EmitAnyExprToTemp(E), type);
2789 }
2790 
2791 QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
2792   // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
2793   // implicitly widens null pointer constants that are arguments to varargs
2794   // functions to pointer-sized ints.
2795   if (!getTarget().getTriple().isOSWindows())
2796     return Arg->getType();
2797 
2798   if (Arg->getType()->isIntegerType() &&
2799       getContext().getTypeSize(Arg->getType()) <
2800           getContext().getTargetInfo().getPointerWidth(0) &&
2801       Arg->isNullPointerConstant(getContext(),
2802                                  Expr::NPC_ValueDependentIsNotNull)) {
2803     return getContext().getIntPtrType();
2804   }
2805 
2806   return Arg->getType();
2807 }
2808 
2809 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
2810 // optimizer it can aggressively ignore unwind edges.
2811 void
2812 CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
2813   if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
2814       !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
2815     Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
2816                       CGM.getNoObjCARCExceptionsMetadata());
2817 }
2818 
2819 /// Emits a call to the given no-arguments nounwind runtime function.
2820 llvm::CallInst *
2821 CodeGenFunction::EmitNounwindRuntimeCall(llvm::Value *callee,
2822                                          const llvm::Twine &name) {
2823   return EmitNounwindRuntimeCall(callee, None, name);
2824 }
2825 
2826 /// Emits a call to the given nounwind runtime function.
2827 llvm::CallInst *
2828 CodeGenFunction::EmitNounwindRuntimeCall(llvm::Value *callee,
2829                                          ArrayRef<llvm::Value*> args,
2830                                          const llvm::Twine &name) {
2831   llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
2832   call->setDoesNotThrow();
2833   return call;
2834 }
2835 
2836 /// Emits a simple call (never an invoke) to the given no-arguments
2837 /// runtime function.
2838 llvm::CallInst *
2839 CodeGenFunction::EmitRuntimeCall(llvm::Value *callee,
2840                                  const llvm::Twine &name) {
2841   return EmitRuntimeCall(callee, None, name);
2842 }
2843 
2844 /// Emits a simple call (never an invoke) to the given runtime
2845 /// function.
2846 llvm::CallInst *
2847 CodeGenFunction::EmitRuntimeCall(llvm::Value *callee,
2848                                  ArrayRef<llvm::Value*> args,
2849                                  const llvm::Twine &name) {
2850   llvm::CallInst *call = Builder.CreateCall(callee, args, name);
2851   call->setCallingConv(getRuntimeCC());
2852   return call;
2853 }
2854 
2855 /// Emits a call or invoke to the given noreturn runtime function.
2856 void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
2857                                                ArrayRef<llvm::Value*> args) {
2858   if (getInvokeDest()) {
2859     llvm::InvokeInst *invoke =
2860       Builder.CreateInvoke(callee,
2861                            getUnreachableBlock(),
2862                            getInvokeDest(),
2863                            args);
2864     invoke->setDoesNotReturn();
2865     invoke->setCallingConv(getRuntimeCC());
2866   } else {
2867     llvm::CallInst *call = Builder.CreateCall(callee, args);
2868     call->setDoesNotReturn();
2869     call->setCallingConv(getRuntimeCC());
2870     Builder.CreateUnreachable();
2871   }
2872   PGO.setCurrentRegionUnreachable();
2873 }
2874 
2875 /// Emits a call or invoke instruction to the given nullary runtime
2876 /// function.
2877 llvm::CallSite
2878 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::Value *callee,
2879                                          const Twine &name) {
2880   return EmitRuntimeCallOrInvoke(callee, None, name);
2881 }
2882 
2883 /// Emits a call or invoke instruction to the given runtime function.
2884 llvm::CallSite
2885 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::Value *callee,
2886                                          ArrayRef<llvm::Value*> args,
2887                                          const Twine &name) {
2888   llvm::CallSite callSite = EmitCallOrInvoke(callee, args, name);
2889   callSite.setCallingConv(getRuntimeCC());
2890   return callSite;
2891 }
2892 
2893 llvm::CallSite
2894 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
2895                                   const Twine &Name) {
2896   return EmitCallOrInvoke(Callee, None, Name);
2897 }
2898 
2899 /// Emits a call or invoke instruction to the given function, depending
2900 /// on the current state of the EH stack.
2901 llvm::CallSite
2902 CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
2903                                   ArrayRef<llvm::Value *> Args,
2904                                   const Twine &Name) {
2905   llvm::BasicBlock *InvokeDest = getInvokeDest();
2906 
2907   llvm::Instruction *Inst;
2908   if (!InvokeDest)
2909     Inst = Builder.CreateCall(Callee, Args, Name);
2910   else {
2911     llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
2912     Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, Name);
2913     EmitBlock(ContBB);
2914   }
2915 
2916   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
2917   // optimizer it can aggressively ignore unwind edges.
2918   if (CGM.getLangOpts().ObjCAutoRefCount)
2919     AddObjCARCExceptionMetadata(Inst);
2920 
2921   return Inst;
2922 }
2923 
2924 /// \brief Store a non-aggregate value to an address to initialize it.  For
2925 /// initialization, a non-atomic store will be used.
2926 static void EmitInitStoreOfNonAggregate(CodeGenFunction &CGF, RValue Src,
2927                                         LValue Dst) {
2928   if (Src.isScalar())
2929     CGF.EmitStoreOfScalar(Src.getScalarVal(), Dst, /*init=*/true);
2930   else
2931     CGF.EmitStoreOfComplex(Src.getComplexVal(), Dst, /*init=*/true);
2932 }
2933 
2934 void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
2935                                                   llvm::Value *New) {
2936   DeferredReplacements.push_back(std::make_pair(Old, New));
2937 }
2938 
2939 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
2940                                  llvm::Value *Callee,
2941                                  ReturnValueSlot ReturnValue,
2942                                  const CallArgList &CallArgs,
2943                                  const Decl *TargetDecl,
2944                                  llvm::Instruction **callOrInvoke) {
2945   // FIXME: We no longer need the types from CallArgs; lift up and simplify.
2946 
2947   // Handle struct-return functions by passing a pointer to the
2948   // location that we would like to return into.
2949   QualType RetTy = CallInfo.getReturnType();
2950   const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
2951 
2952   llvm::FunctionType *IRFuncTy =
2953     cast<llvm::FunctionType>(
2954                   cast<llvm::PointerType>(Callee->getType())->getElementType());
2955 
2956   // If we're using inalloca, insert the allocation after the stack save.
2957   // FIXME: Do this earlier rather than hacking it in here!
2958   llvm::Value *ArgMemory = nullptr;
2959   if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
2960     llvm::Instruction *IP = CallArgs.getStackBase();
2961     llvm::AllocaInst *AI;
2962     if (IP) {
2963       IP = IP->getNextNode();
2964       AI = new llvm::AllocaInst(ArgStruct, "argmem", IP);
2965     } else {
2966       AI = CreateTempAlloca(ArgStruct, "argmem");
2967     }
2968     AI->setUsedWithInAlloca(true);
2969     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
2970     ArgMemory = AI;
2971   }
2972 
2973   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
2974   SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
2975 
2976   // If the call returns a temporary with struct return, create a temporary
2977   // alloca to hold the result, unless one is given to us.
2978   llvm::Value *SRetPtr = nullptr;
2979   if (RetAI.isIndirect() || RetAI.isInAlloca()) {
2980     SRetPtr = ReturnValue.getValue();
2981     if (!SRetPtr)
2982       SRetPtr = CreateMemTemp(RetTy);
2983     if (IRFunctionArgs.hasSRetArg()) {
2984       IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr;
2985     } else {
2986       llvm::Value *Addr =
2987           Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
2988       Builder.CreateStore(SRetPtr, Addr);
2989     }
2990   }
2991 
2992   assert(CallInfo.arg_size() == CallArgs.size() &&
2993          "Mismatch between function signature & arguments.");
2994   unsigned ArgNo = 0;
2995   CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
2996   for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
2997        I != E; ++I, ++info_it, ++ArgNo) {
2998     const ABIArgInfo &ArgInfo = info_it->info;
2999     RValue RV = I->RV;
3000 
3001     CharUnits TypeAlign = getContext().getTypeAlignInChars(I->Ty);
3002 
3003     // Insert a padding argument to ensure proper alignment.
3004     if (IRFunctionArgs.hasPaddingArg(ArgNo))
3005       IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
3006           llvm::UndefValue::get(ArgInfo.getPaddingType());
3007 
3008     unsigned FirstIRArg, NumIRArgs;
3009     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3010 
3011     switch (ArgInfo.getKind()) {
3012     case ABIArgInfo::InAlloca: {
3013       assert(NumIRArgs == 0);
3014       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3015       if (RV.isAggregate()) {
3016         // Replace the placeholder with the appropriate argument slot GEP.
3017         llvm::Instruction *Placeholder =
3018             cast<llvm::Instruction>(RV.getAggregateAddr());
3019         CGBuilderTy::InsertPoint IP = Builder.saveIP();
3020         Builder.SetInsertPoint(Placeholder);
3021         llvm::Value *Addr = Builder.CreateStructGEP(
3022             ArgMemory, ArgInfo.getInAllocaFieldIndex());
3023         Builder.restoreIP(IP);
3024         deferPlaceholderReplacement(Placeholder, Addr);
3025       } else {
3026         // Store the RValue into the argument struct.
3027         llvm::Value *Addr =
3028             Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
3029         unsigned AS = Addr->getType()->getPointerAddressSpace();
3030         llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
3031         // There are some cases where a trivial bitcast is not avoidable.  The
3032         // definition of a type later in a translation unit may change it's type
3033         // from {}* to (%struct.foo*)*.
3034         if (Addr->getType() != MemType)
3035           Addr = Builder.CreateBitCast(Addr, MemType);
3036         LValue argLV = MakeAddrLValue(Addr, I->Ty, TypeAlign);
3037         EmitInitStoreOfNonAggregate(*this, RV, argLV);
3038       }
3039       break;
3040     }
3041 
3042     case ABIArgInfo::Indirect: {
3043       assert(NumIRArgs == 1);
3044       if (RV.isScalar() || RV.isComplex()) {
3045         // Make a temporary alloca to pass the argument.
3046         llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
3047         if (ArgInfo.getIndirectAlign() > AI->getAlignment())
3048           AI->setAlignment(ArgInfo.getIndirectAlign());
3049         IRCallArgs[FirstIRArg] = AI;
3050 
3051         LValue argLV = MakeAddrLValue(AI, I->Ty, TypeAlign);
3052         EmitInitStoreOfNonAggregate(*this, RV, argLV);
3053       } else {
3054         // We want to avoid creating an unnecessary temporary+copy here;
3055         // however, we need one in three cases:
3056         // 1. If the argument is not byval, and we are required to copy the
3057         //    source.  (This case doesn't occur on any common architecture.)
3058         // 2. If the argument is byval, RV is not sufficiently aligned, and
3059         //    we cannot force it to be sufficiently aligned.
3060         // 3. If the argument is byval, but RV is located in an address space
3061         //    different than that of the argument (0).
3062         llvm::Value *Addr = RV.getAggregateAddr();
3063         unsigned Align = ArgInfo.getIndirectAlign();
3064         const llvm::DataLayout *TD = &CGM.getDataLayout();
3065         const unsigned RVAddrSpace = Addr->getType()->getPointerAddressSpace();
3066         const unsigned ArgAddrSpace =
3067             (FirstIRArg < IRFuncTy->getNumParams()
3068                  ? IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace()
3069                  : 0);
3070         if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
3071             (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align &&
3072              llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align) ||
3073              (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) {
3074           // Create an aligned temporary, and copy to it.
3075           llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
3076           if (Align > AI->getAlignment())
3077             AI->setAlignment(Align);
3078           IRCallArgs[FirstIRArg] = AI;
3079           EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
3080         } else {
3081           // Skip the extra memcpy call.
3082           IRCallArgs[FirstIRArg] = Addr;
3083         }
3084       }
3085       break;
3086     }
3087 
3088     case ABIArgInfo::Ignore:
3089       assert(NumIRArgs == 0);
3090       break;
3091 
3092     case ABIArgInfo::Extend:
3093     case ABIArgInfo::Direct: {
3094       if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
3095           ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
3096           ArgInfo.getDirectOffset() == 0) {
3097         assert(NumIRArgs == 1);
3098         llvm::Value *V;
3099         if (RV.isScalar())
3100           V = RV.getScalarVal();
3101         else
3102           V = Builder.CreateLoad(RV.getAggregateAddr());
3103 
3104         // We might have to widen integers, but we should never truncate.
3105         if (ArgInfo.getCoerceToType() != V->getType() &&
3106             V->getType()->isIntegerTy())
3107           V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
3108 
3109         // If the argument doesn't match, perform a bitcast to coerce it.  This
3110         // can happen due to trivial type mismatches.
3111         if (FirstIRArg < IRFuncTy->getNumParams() &&
3112             V->getType() != IRFuncTy->getParamType(FirstIRArg))
3113           V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
3114         IRCallArgs[FirstIRArg] = V;
3115         break;
3116       }
3117 
3118       // FIXME: Avoid the conversion through memory if possible.
3119       llvm::Value *SrcPtr;
3120       if (RV.isScalar() || RV.isComplex()) {
3121         SrcPtr = CreateMemTemp(I->Ty, "coerce");
3122         LValue SrcLV = MakeAddrLValue(SrcPtr, I->Ty, TypeAlign);
3123         EmitInitStoreOfNonAggregate(*this, RV, SrcLV);
3124       } else
3125         SrcPtr = RV.getAggregateAddr();
3126 
3127       // If the value is offset in memory, apply the offset now.
3128       if (unsigned Offs = ArgInfo.getDirectOffset()) {
3129         SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
3130         SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
3131         SrcPtr = Builder.CreateBitCast(SrcPtr,
3132                        llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
3133 
3134       }
3135 
3136       // Fast-isel and the optimizer generally like scalar values better than
3137       // FCAs, so we flatten them if this is safe to do for this argument.
3138       llvm::StructType *STy =
3139             dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
3140       if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
3141         llvm::Type *SrcTy =
3142           cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
3143         uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
3144         uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
3145 
3146         // If the source type is smaller than the destination type of the
3147         // coerce-to logic, copy the source value into a temp alloca the size
3148         // of the destination type to allow loading all of it. The bits past
3149         // the source value are left undef.
3150         if (SrcSize < DstSize) {
3151           llvm::AllocaInst *TempAlloca
3152             = CreateTempAlloca(STy, SrcPtr->getName() + ".coerce");
3153           Builder.CreateMemCpy(TempAlloca, SrcPtr, SrcSize, 0);
3154           SrcPtr = TempAlloca;
3155         } else {
3156           SrcPtr = Builder.CreateBitCast(SrcPtr,
3157                                          llvm::PointerType::getUnqual(STy));
3158         }
3159 
3160         assert(NumIRArgs == STy->getNumElements());
3161         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3162           llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
3163           llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
3164           // We don't know what we're loading from.
3165           LI->setAlignment(1);
3166           IRCallArgs[FirstIRArg + i] = LI;
3167         }
3168       } else {
3169         // In the simple case, just pass the coerced loaded value.
3170         assert(NumIRArgs == 1);
3171         IRCallArgs[FirstIRArg] =
3172             CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), *this);
3173       }
3174 
3175       break;
3176     }
3177 
3178     case ABIArgInfo::Expand:
3179       unsigned IRArgPos = FirstIRArg;
3180       ExpandTypeToArgs(I->Ty, RV, IRFuncTy, IRCallArgs, IRArgPos);
3181       assert(IRArgPos == FirstIRArg + NumIRArgs);
3182       break;
3183     }
3184   }
3185 
3186   if (ArgMemory) {
3187     llvm::Value *Arg = ArgMemory;
3188     if (CallInfo.isVariadic()) {
3189       // When passing non-POD arguments by value to variadic functions, we will
3190       // end up with a variadic prototype and an inalloca call site.  In such
3191       // cases, we can't do any parameter mismatch checks.  Give up and bitcast
3192       // the callee.
3193       unsigned CalleeAS =
3194           cast<llvm::PointerType>(Callee->getType())->getAddressSpace();
3195       Callee = Builder.CreateBitCast(
3196           Callee, getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS));
3197     } else {
3198       llvm::Type *LastParamTy =
3199           IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
3200       if (Arg->getType() != LastParamTy) {
3201 #ifndef NDEBUG
3202         // Assert that these structs have equivalent element types.
3203         llvm::StructType *FullTy = CallInfo.getArgStruct();
3204         llvm::StructType *DeclaredTy = cast<llvm::StructType>(
3205             cast<llvm::PointerType>(LastParamTy)->getElementType());
3206         assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
3207         for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
3208                                                 DE = DeclaredTy->element_end(),
3209                                                 FI = FullTy->element_begin();
3210              DI != DE; ++DI, ++FI)
3211           assert(*DI == *FI);
3212 #endif
3213         Arg = Builder.CreateBitCast(Arg, LastParamTy);
3214       }
3215     }
3216     assert(IRFunctionArgs.hasInallocaArg());
3217     IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
3218   }
3219 
3220   if (!CallArgs.getCleanupsToDeactivate().empty())
3221     deactivateArgCleanupsBeforeCall(*this, CallArgs);
3222 
3223   // If the callee is a bitcast of a function to a varargs pointer to function
3224   // type, check to see if we can remove the bitcast.  This handles some cases
3225   // with unprototyped functions.
3226   if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
3227     if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
3228       llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
3229       llvm::FunctionType *CurFT =
3230         cast<llvm::FunctionType>(CurPT->getElementType());
3231       llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
3232 
3233       if (CE->getOpcode() == llvm::Instruction::BitCast &&
3234           ActualFT->getReturnType() == CurFT->getReturnType() &&
3235           ActualFT->getNumParams() == CurFT->getNumParams() &&
3236           ActualFT->getNumParams() == IRCallArgs.size() &&
3237           (CurFT->isVarArg() || !ActualFT->isVarArg())) {
3238         bool ArgsMatch = true;
3239         for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
3240           if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
3241             ArgsMatch = false;
3242             break;
3243           }
3244 
3245         // Strip the cast if we can get away with it.  This is a nice cleanup,
3246         // but also allows us to inline the function at -O0 if it is marked
3247         // always_inline.
3248         if (ArgsMatch)
3249           Callee = CalleeF;
3250       }
3251     }
3252 
3253   assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
3254   for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
3255     // Inalloca argument can have different type.
3256     if (IRFunctionArgs.hasInallocaArg() &&
3257         i == IRFunctionArgs.getInallocaArgNo())
3258       continue;
3259     if (i < IRFuncTy->getNumParams())
3260       assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
3261   }
3262 
3263   unsigned CallingConv;
3264   CodeGen::AttributeListType AttributeList;
3265   CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList,
3266                              CallingConv, true);
3267   llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(),
3268                                                      AttributeList);
3269 
3270   llvm::BasicBlock *InvokeDest = nullptr;
3271   if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
3272                           llvm::Attribute::NoUnwind))
3273     InvokeDest = getInvokeDest();
3274 
3275   llvm::CallSite CS;
3276   if (!InvokeDest) {
3277     CS = Builder.CreateCall(Callee, IRCallArgs);
3278   } else {
3279     llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
3280     CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs);
3281     EmitBlock(Cont);
3282   }
3283   if (callOrInvoke)
3284     *callOrInvoke = CS.getInstruction();
3285 
3286   if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
3287       !CS.hasFnAttr(llvm::Attribute::NoInline))
3288     Attrs =
3289         Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
3290                            llvm::Attribute::AlwaysInline);
3291 
3292   CS.setAttributes(Attrs);
3293   CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
3294 
3295   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3296   // optimizer it can aggressively ignore unwind edges.
3297   if (CGM.getLangOpts().ObjCAutoRefCount)
3298     AddObjCARCExceptionMetadata(CS.getInstruction());
3299 
3300   // If the call doesn't return, finish the basic block and clear the
3301   // insertion point; this allows the rest of IRgen to discard
3302   // unreachable code.
3303   if (CS.doesNotReturn()) {
3304     Builder.CreateUnreachable();
3305     Builder.ClearInsertionPoint();
3306 
3307     // FIXME: For now, emit a dummy basic block because expr emitters in
3308     // generally are not ready to handle emitting expressions at unreachable
3309     // points.
3310     EnsureInsertPoint();
3311 
3312     // Return a reasonable RValue.
3313     return GetUndefRValue(RetTy);
3314   }
3315 
3316   llvm::Instruction *CI = CS.getInstruction();
3317   if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
3318     CI->setName("call");
3319 
3320   // Emit any writebacks immediately.  Arguably this should happen
3321   // after any return-value munging.
3322   if (CallArgs.hasWritebacks())
3323     emitWritebacks(*this, CallArgs);
3324 
3325   // The stack cleanup for inalloca arguments has to run out of the normal
3326   // lexical order, so deactivate it and run it manually here.
3327   CallArgs.freeArgumentMemory(*this);
3328 
3329   RValue Ret = [&] {
3330     switch (RetAI.getKind()) {
3331     case ABIArgInfo::InAlloca:
3332     case ABIArgInfo::Indirect:
3333       return convertTempToRValue(SRetPtr, RetTy, SourceLocation());
3334 
3335     case ABIArgInfo::Ignore:
3336       // If we are ignoring an argument that had a result, make sure to
3337       // construct the appropriate return value for our caller.
3338       return GetUndefRValue(RetTy);
3339 
3340     case ABIArgInfo::Extend:
3341     case ABIArgInfo::Direct: {
3342       llvm::Type *RetIRTy = ConvertType(RetTy);
3343       if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
3344         switch (getEvaluationKind(RetTy)) {
3345         case TEK_Complex: {
3346           llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
3347           llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
3348           return RValue::getComplex(std::make_pair(Real, Imag));
3349         }
3350         case TEK_Aggregate: {
3351           llvm::Value *DestPtr = ReturnValue.getValue();
3352           bool DestIsVolatile = ReturnValue.isVolatile();
3353 
3354           if (!DestPtr) {
3355             DestPtr = CreateMemTemp(RetTy, "agg.tmp");
3356             DestIsVolatile = false;
3357           }
3358           BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
3359           return RValue::getAggregate(DestPtr);
3360         }
3361         case TEK_Scalar: {
3362           // If the argument doesn't match, perform a bitcast to coerce it.  This
3363           // can happen due to trivial type mismatches.
3364           llvm::Value *V = CI;
3365           if (V->getType() != RetIRTy)
3366             V = Builder.CreateBitCast(V, RetIRTy);
3367           return RValue::get(V);
3368         }
3369         }
3370         llvm_unreachable("bad evaluation kind");
3371       }
3372 
3373       llvm::Value *DestPtr = ReturnValue.getValue();
3374       bool DestIsVolatile = ReturnValue.isVolatile();
3375 
3376       if (!DestPtr) {
3377         DestPtr = CreateMemTemp(RetTy, "coerce");
3378         DestIsVolatile = false;
3379       }
3380 
3381       // If the value is offset in memory, apply the offset now.
3382       llvm::Value *StorePtr = DestPtr;
3383       if (unsigned Offs = RetAI.getDirectOffset()) {
3384         StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
3385         StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
3386         StorePtr = Builder.CreateBitCast(StorePtr,
3387                            llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
3388       }
3389       CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
3390 
3391       return convertTempToRValue(DestPtr, RetTy, SourceLocation());
3392     }
3393 
3394     case ABIArgInfo::Expand:
3395       llvm_unreachable("Invalid ABI kind for return argument");
3396     }
3397 
3398     llvm_unreachable("Unhandled ABIArgInfo::Kind");
3399   } ();
3400 
3401   if (Ret.isScalar() && TargetDecl) {
3402     if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) {
3403       llvm::Value *OffsetValue = nullptr;
3404       if (const auto *Offset = AA->getOffset())
3405         OffsetValue = EmitScalarExpr(Offset);
3406 
3407       llvm::Value *Alignment = EmitScalarExpr(AA->getAlignment());
3408       llvm::ConstantInt *AlignmentCI = cast<llvm::ConstantInt>(Alignment);
3409       EmitAlignmentAssumption(Ret.getScalarVal(), AlignmentCI->getZExtValue(),
3410                               OffsetValue);
3411     }
3412   }
3413 
3414   return Ret;
3415 }
3416 
3417 /* VarArg handling */
3418 
3419 llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
3420   return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
3421 }
3422