1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit Builtin calls as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CGObjCRuntime.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/AST/Decl.h"
20 #include "clang/Basic/TargetBuiltins.h"
21 #include "clang/Basic/TargetInfo.h"
22 #include "clang/CodeGen/CGFunctionInfo.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/Intrinsics.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 using namespace llvm;
29 
30 /// getBuiltinLibFunction - Given a builtin id for a function like
31 /// "__builtin_fabsf", return a Function* for "fabsf".
32 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
33                                                   unsigned BuiltinID) {
34   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
35 
36   // Get the name, skip over the __builtin_ prefix (if necessary).
37   StringRef Name;
38   GlobalDecl D(FD);
39 
40   // If the builtin has been declared explicitly with an assembler label,
41   // use the mangled name. This differs from the plain label on platforms
42   // that prefix labels.
43   if (FD->hasAttr<AsmLabelAttr>())
44     Name = getMangledName(D);
45   else
46     Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
47 
48   llvm::FunctionType *Ty =
49     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
50 
51   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
52 }
53 
54 /// Emit the conversions required to turn the given value into an
55 /// integer of the given size.
56 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
57                         QualType T, llvm::IntegerType *IntType) {
58   V = CGF.EmitToMemory(V, T);
59 
60   if (V->getType()->isPointerTy())
61     return CGF.Builder.CreatePtrToInt(V, IntType);
62 
63   assert(V->getType() == IntType);
64   return V;
65 }
66 
67 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
68                           QualType T, llvm::Type *ResultType) {
69   V = CGF.EmitFromMemory(V, T);
70 
71   if (ResultType->isPointerTy())
72     return CGF.Builder.CreateIntToPtr(V, ResultType);
73 
74   assert(V->getType() == ResultType);
75   return V;
76 }
77 
78 /// Utility to insert an atomic instruction based on Instrinsic::ID
79 /// and the expression node.
80 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
81                                llvm::AtomicRMWInst::BinOp Kind,
82                                const CallExpr *E) {
83   QualType T = E->getType();
84   assert(E->getArg(0)->getType()->isPointerType());
85   assert(CGF.getContext().hasSameUnqualifiedType(T,
86                                   E->getArg(0)->getType()->getPointeeType()));
87   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
88 
89   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
90   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
91 
92   llvm::IntegerType *IntType =
93     llvm::IntegerType::get(CGF.getLLVMContext(),
94                            CGF.getContext().getTypeSize(T));
95   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
96 
97   llvm::Value *Args[2];
98   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
99   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
100   llvm::Type *ValueType = Args[1]->getType();
101   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
102 
103   llvm::Value *Result =
104       CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
105                                   llvm::SequentiallyConsistent);
106   Result = EmitFromInt(CGF, Result, T, ValueType);
107   return RValue::get(Result);
108 }
109 
110 /// Utility to insert an atomic instruction based Instrinsic::ID and
111 /// the expression node, where the return value is the result of the
112 /// operation.
113 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
114                                    llvm::AtomicRMWInst::BinOp Kind,
115                                    const CallExpr *E,
116                                    Instruction::BinaryOps Op) {
117   QualType T = E->getType();
118   assert(E->getArg(0)->getType()->isPointerType());
119   assert(CGF.getContext().hasSameUnqualifiedType(T,
120                                   E->getArg(0)->getType()->getPointeeType()));
121   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
122 
123   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
124   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
125 
126   llvm::IntegerType *IntType =
127     llvm::IntegerType::get(CGF.getLLVMContext(),
128                            CGF.getContext().getTypeSize(T));
129   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
130 
131   llvm::Value *Args[2];
132   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
133   llvm::Type *ValueType = Args[1]->getType();
134   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
135   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
136 
137   llvm::Value *Result =
138       CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
139                                   llvm::SequentiallyConsistent);
140   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
141   Result = EmitFromInt(CGF, Result, T, ValueType);
142   return RValue::get(Result);
143 }
144 
145 /// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy,
146 /// which must be a scalar floating point type.
147 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
148   const BuiltinType *ValTyP = ValTy->getAs<BuiltinType>();
149   assert(ValTyP && "isn't scalar fp type!");
150 
151   StringRef FnName;
152   switch (ValTyP->getKind()) {
153   default: llvm_unreachable("Isn't a scalar fp type!");
154   case BuiltinType::Float:      FnName = "fabsf"; break;
155   case BuiltinType::Double:     FnName = "fabs"; break;
156   case BuiltinType::LongDouble: FnName = "fabsl"; break;
157   }
158 
159   // The prototype is something that takes and returns whatever V's type is.
160   llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(),
161                                                    false);
162   llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction(FT, FnName);
163 
164   return CGF.EmitNounwindRuntimeCall(Fn, V, "abs");
165 }
166 
167 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn,
168                               const CallExpr *E, llvm::Value *calleeValue) {
169   return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E->getLocStart(),
170                       ReturnValueSlot(), E->arg_begin(), E->arg_end(), Fn);
171 }
172 
173 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
174 /// depending on IntrinsicID.
175 ///
176 /// \arg CGF The current codegen function.
177 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
178 /// \arg X The first argument to the llvm.*.with.overflow.*.
179 /// \arg Y The second argument to the llvm.*.with.overflow.*.
180 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
181 /// \returns The result (i.e. sum/product) returned by the intrinsic.
182 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
183                                           const llvm::Intrinsic::ID IntrinsicID,
184                                           llvm::Value *X, llvm::Value *Y,
185                                           llvm::Value *&Carry) {
186   // Make sure we have integers of the same width.
187   assert(X->getType() == Y->getType() &&
188          "Arguments must be the same type. (Did you forget to make sure both "
189          "arguments have the same integer width?)");
190 
191   llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
192   llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y);
193   Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
194   return CGF.Builder.CreateExtractValue(Tmp, 0);
195 }
196 
197 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
198                                         unsigned BuiltinID, const CallExpr *E) {
199   // See if we can constant fold this builtin.  If so, don't emit it at all.
200   Expr::EvalResult Result;
201   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
202       !Result.hasSideEffects()) {
203     if (Result.Val.isInt())
204       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
205                                                 Result.Val.getInt()));
206     if (Result.Val.isFloat())
207       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
208                                                Result.Val.getFloat()));
209   }
210 
211   switch (BuiltinID) {
212   default: break;  // Handle intrinsics and libm functions below.
213   case Builtin::BI__builtin___CFStringMakeConstantString:
214   case Builtin::BI__builtin___NSStringMakeConstantString:
215     return RValue::get(CGM.EmitConstantExpr(E, E->getType(), 0));
216   case Builtin::BI__builtin_stdarg_start:
217   case Builtin::BI__builtin_va_start:
218   case Builtin::BI__builtin_va_end: {
219     Value *ArgValue = EmitVAListRef(E->getArg(0));
220     llvm::Type *DestType = Int8PtrTy;
221     if (ArgValue->getType() != DestType)
222       ArgValue = Builder.CreateBitCast(ArgValue, DestType,
223                                        ArgValue->getName().data());
224 
225     Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ?
226       Intrinsic::vaend : Intrinsic::vastart;
227     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue));
228   }
229   case Builtin::BI__builtin_va_copy: {
230     Value *DstPtr = EmitVAListRef(E->getArg(0));
231     Value *SrcPtr = EmitVAListRef(E->getArg(1));
232 
233     llvm::Type *Type = Int8PtrTy;
234 
235     DstPtr = Builder.CreateBitCast(DstPtr, Type);
236     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
237     return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy),
238                                            DstPtr, SrcPtr));
239   }
240   case Builtin::BI__builtin_abs:
241   case Builtin::BI__builtin_labs:
242   case Builtin::BI__builtin_llabs: {
243     Value *ArgValue = EmitScalarExpr(E->getArg(0));
244 
245     Value *NegOp = Builder.CreateNeg(ArgValue, "neg");
246     Value *CmpResult =
247     Builder.CreateICmpSGE(ArgValue,
248                           llvm::Constant::getNullValue(ArgValue->getType()),
249                                                             "abscond");
250     Value *Result =
251       Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs");
252 
253     return RValue::get(Result);
254   }
255 
256   case Builtin::BI__builtin_conj:
257   case Builtin::BI__builtin_conjf:
258   case Builtin::BI__builtin_conjl: {
259     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
260     Value *Real = ComplexVal.first;
261     Value *Imag = ComplexVal.second;
262     Value *Zero =
263       Imag->getType()->isFPOrFPVectorTy()
264         ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
265         : llvm::Constant::getNullValue(Imag->getType());
266 
267     Imag = Builder.CreateFSub(Zero, Imag, "sub");
268     return RValue::getComplex(std::make_pair(Real, Imag));
269   }
270   case Builtin::BI__builtin_creal:
271   case Builtin::BI__builtin_crealf:
272   case Builtin::BI__builtin_creall:
273   case Builtin::BIcreal:
274   case Builtin::BIcrealf:
275   case Builtin::BIcreall: {
276     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
277     return RValue::get(ComplexVal.first);
278   }
279 
280   case Builtin::BI__builtin_cimag:
281   case Builtin::BI__builtin_cimagf:
282   case Builtin::BI__builtin_cimagl:
283   case Builtin::BIcimag:
284   case Builtin::BIcimagf:
285   case Builtin::BIcimagl: {
286     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
287     return RValue::get(ComplexVal.second);
288   }
289 
290   case Builtin::BI__builtin_ctzs:
291   case Builtin::BI__builtin_ctz:
292   case Builtin::BI__builtin_ctzl:
293   case Builtin::BI__builtin_ctzll: {
294     Value *ArgValue = EmitScalarExpr(E->getArg(0));
295 
296     llvm::Type *ArgType = ArgValue->getType();
297     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
298 
299     llvm::Type *ResultType = ConvertType(E->getType());
300     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
301     Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
302     if (Result->getType() != ResultType)
303       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
304                                      "cast");
305     return RValue::get(Result);
306   }
307   case Builtin::BI__builtin_clzs:
308   case Builtin::BI__builtin_clz:
309   case Builtin::BI__builtin_clzl:
310   case Builtin::BI__builtin_clzll: {
311     Value *ArgValue = EmitScalarExpr(E->getArg(0));
312 
313     llvm::Type *ArgType = ArgValue->getType();
314     Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
315 
316     llvm::Type *ResultType = ConvertType(E->getType());
317     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
318     Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
319     if (Result->getType() != ResultType)
320       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
321                                      "cast");
322     return RValue::get(Result);
323   }
324   case Builtin::BI__builtin_ffs:
325   case Builtin::BI__builtin_ffsl:
326   case Builtin::BI__builtin_ffsll: {
327     // ffs(x) -> x ? cttz(x) + 1 : 0
328     Value *ArgValue = EmitScalarExpr(E->getArg(0));
329 
330     llvm::Type *ArgType = ArgValue->getType();
331     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
332 
333     llvm::Type *ResultType = ConvertType(E->getType());
334     Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue,
335                                                        Builder.getTrue()),
336                                    llvm::ConstantInt::get(ArgType, 1));
337     Value *Zero = llvm::Constant::getNullValue(ArgType);
338     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
339     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
340     if (Result->getType() != ResultType)
341       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
342                                      "cast");
343     return RValue::get(Result);
344   }
345   case Builtin::BI__builtin_parity:
346   case Builtin::BI__builtin_parityl:
347   case Builtin::BI__builtin_parityll: {
348     // parity(x) -> ctpop(x) & 1
349     Value *ArgValue = EmitScalarExpr(E->getArg(0));
350 
351     llvm::Type *ArgType = ArgValue->getType();
352     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
353 
354     llvm::Type *ResultType = ConvertType(E->getType());
355     Value *Tmp = Builder.CreateCall(F, ArgValue);
356     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
357     if (Result->getType() != ResultType)
358       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
359                                      "cast");
360     return RValue::get(Result);
361   }
362   case Builtin::BI__builtin_popcount:
363   case Builtin::BI__builtin_popcountl:
364   case Builtin::BI__builtin_popcountll: {
365     Value *ArgValue = EmitScalarExpr(E->getArg(0));
366 
367     llvm::Type *ArgType = ArgValue->getType();
368     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
369 
370     llvm::Type *ResultType = ConvertType(E->getType());
371     Value *Result = Builder.CreateCall(F, ArgValue);
372     if (Result->getType() != ResultType)
373       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
374                                      "cast");
375     return RValue::get(Result);
376   }
377   case Builtin::BI__builtin_expect: {
378     Value *ArgValue = EmitScalarExpr(E->getArg(0));
379     llvm::Type *ArgType = ArgValue->getType();
380 
381     Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
382     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
383 
384     Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue,
385                                         "expval");
386     return RValue::get(Result);
387   }
388   case Builtin::BI__builtin_bswap16:
389   case Builtin::BI__builtin_bswap32:
390   case Builtin::BI__builtin_bswap64: {
391     Value *ArgValue = EmitScalarExpr(E->getArg(0));
392     llvm::Type *ArgType = ArgValue->getType();
393     Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType);
394     return RValue::get(Builder.CreateCall(F, ArgValue));
395   }
396   case Builtin::BI__builtin_object_size: {
397     // We rely on constant folding to deal with expressions with side effects.
398     assert(!E->getArg(0)->HasSideEffects(getContext()) &&
399            "should have been constant folded");
400 
401     // We pass this builtin onto the optimizer so that it can
402     // figure out the object size in more complex cases.
403     llvm::Type *ResType = ConvertType(E->getType());
404 
405     // LLVM only supports 0 and 2, make sure that we pass along that
406     // as a boolean.
407     Value *Ty = EmitScalarExpr(E->getArg(1));
408     ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
409     assert(CI);
410     uint64_t val = CI->getZExtValue();
411     CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);
412     // FIXME: Get right address space.
413     llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) };
414     Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
415     return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI));
416   }
417   case Builtin::BI__builtin_prefetch: {
418     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
419     // FIXME: Technically these constants should of type 'int', yes?
420     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
421       llvm::ConstantInt::get(Int32Ty, 0);
422     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
423       llvm::ConstantInt::get(Int32Ty, 3);
424     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
425     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
426     return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data));
427   }
428   case Builtin::BI__builtin_readcyclecounter: {
429     Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
430     return RValue::get(Builder.CreateCall(F));
431   }
432   case Builtin::BI__builtin_trap: {
433     Value *F = CGM.getIntrinsic(Intrinsic::trap);
434     return RValue::get(Builder.CreateCall(F));
435   }
436   case Builtin::BI__debugbreak: {
437     Value *F = CGM.getIntrinsic(Intrinsic::debugtrap);
438     return RValue::get(Builder.CreateCall(F));
439   }
440   case Builtin::BI__builtin_unreachable: {
441     if (SanOpts->Unreachable)
442       EmitCheck(Builder.getFalse(), "builtin_unreachable",
443                 EmitCheckSourceLocation(E->getExprLoc()),
444                 ArrayRef<llvm::Value *>(), CRK_Unrecoverable);
445     else
446       Builder.CreateUnreachable();
447 
448     // We do need to preserve an insertion point.
449     EmitBlock(createBasicBlock("unreachable.cont"));
450 
451     return RValue::get(0);
452   }
453 
454   case Builtin::BI__builtin_powi:
455   case Builtin::BI__builtin_powif:
456   case Builtin::BI__builtin_powil: {
457     Value *Base = EmitScalarExpr(E->getArg(0));
458     Value *Exponent = EmitScalarExpr(E->getArg(1));
459     llvm::Type *ArgType = Base->getType();
460     Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
461     return RValue::get(Builder.CreateCall2(F, Base, Exponent));
462   }
463 
464   case Builtin::BI__builtin_isgreater:
465   case Builtin::BI__builtin_isgreaterequal:
466   case Builtin::BI__builtin_isless:
467   case Builtin::BI__builtin_islessequal:
468   case Builtin::BI__builtin_islessgreater:
469   case Builtin::BI__builtin_isunordered: {
470     // Ordered comparisons: we know the arguments to these are matching scalar
471     // floating point values.
472     Value *LHS = EmitScalarExpr(E->getArg(0));
473     Value *RHS = EmitScalarExpr(E->getArg(1));
474 
475     switch (BuiltinID) {
476     default: llvm_unreachable("Unknown ordered comparison");
477     case Builtin::BI__builtin_isgreater:
478       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
479       break;
480     case Builtin::BI__builtin_isgreaterequal:
481       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
482       break;
483     case Builtin::BI__builtin_isless:
484       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
485       break;
486     case Builtin::BI__builtin_islessequal:
487       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
488       break;
489     case Builtin::BI__builtin_islessgreater:
490       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
491       break;
492     case Builtin::BI__builtin_isunordered:
493       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
494       break;
495     }
496     // ZExt bool to int type.
497     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
498   }
499   case Builtin::BI__builtin_isnan: {
500     Value *V = EmitScalarExpr(E->getArg(0));
501     V = Builder.CreateFCmpUNO(V, V, "cmp");
502     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
503   }
504 
505   case Builtin::BI__builtin_isinf: {
506     // isinf(x) --> fabs(x) == infinity
507     Value *V = EmitScalarExpr(E->getArg(0));
508     V = EmitFAbs(*this, V, E->getArg(0)->getType());
509 
510     V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf");
511     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
512   }
513 
514   // TODO: BI__builtin_isinf_sign
515   //   isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0
516 
517   case Builtin::BI__builtin_isnormal: {
518     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
519     Value *V = EmitScalarExpr(E->getArg(0));
520     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
521 
522     Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
523     Value *IsLessThanInf =
524       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
525     APFloat Smallest = APFloat::getSmallestNormalized(
526                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
527     Value *IsNormal =
528       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
529                             "isnormal");
530     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
531     V = Builder.CreateAnd(V, IsNormal, "and");
532     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
533   }
534 
535   case Builtin::BI__builtin_isfinite: {
536     // isfinite(x) --> x == x && fabs(x) != infinity;
537     Value *V = EmitScalarExpr(E->getArg(0));
538     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
539 
540     Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
541     Value *IsNotInf =
542       Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
543 
544     V = Builder.CreateAnd(Eq, IsNotInf, "and");
545     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
546   }
547 
548   case Builtin::BI__builtin_fpclassify: {
549     Value *V = EmitScalarExpr(E->getArg(5));
550     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
551 
552     // Create Result
553     BasicBlock *Begin = Builder.GetInsertBlock();
554     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
555     Builder.SetInsertPoint(End);
556     PHINode *Result =
557       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
558                         "fpclassify_result");
559 
560     // if (V==0) return FP_ZERO
561     Builder.SetInsertPoint(Begin);
562     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
563                                           "iszero");
564     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
565     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
566     Builder.CreateCondBr(IsZero, End, NotZero);
567     Result->addIncoming(ZeroLiteral, Begin);
568 
569     // if (V != V) return FP_NAN
570     Builder.SetInsertPoint(NotZero);
571     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
572     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
573     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
574     Builder.CreateCondBr(IsNan, End, NotNan);
575     Result->addIncoming(NanLiteral, NotZero);
576 
577     // if (fabs(V) == infinity) return FP_INFINITY
578     Builder.SetInsertPoint(NotNan);
579     Value *VAbs = EmitFAbs(*this, V, E->getArg(5)->getType());
580     Value *IsInf =
581       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
582                             "isinf");
583     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
584     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
585     Builder.CreateCondBr(IsInf, End, NotInf);
586     Result->addIncoming(InfLiteral, NotNan);
587 
588     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
589     Builder.SetInsertPoint(NotInf);
590     APFloat Smallest = APFloat::getSmallestNormalized(
591         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
592     Value *IsNormal =
593       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
594                             "isnormal");
595     Value *NormalResult =
596       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
597                            EmitScalarExpr(E->getArg(3)));
598     Builder.CreateBr(End);
599     Result->addIncoming(NormalResult, NotInf);
600 
601     // return Result
602     Builder.SetInsertPoint(End);
603     return RValue::get(Result);
604   }
605 
606   case Builtin::BIalloca:
607   case Builtin::BI_alloca:
608   case Builtin::BI__builtin_alloca: {
609     Value *Size = EmitScalarExpr(E->getArg(0));
610     return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size));
611   }
612   case Builtin::BIbzero:
613   case Builtin::BI__builtin_bzero: {
614     std::pair<llvm::Value*, unsigned> Dest =
615         EmitPointerWithAlignment(E->getArg(0));
616     Value *SizeVal = EmitScalarExpr(E->getArg(1));
617     Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal,
618                          Dest.second, false);
619     return RValue::get(Dest.first);
620   }
621   case Builtin::BImemcpy:
622   case Builtin::BI__builtin_memcpy: {
623     std::pair<llvm::Value*, unsigned> Dest =
624         EmitPointerWithAlignment(E->getArg(0));
625     std::pair<llvm::Value*, unsigned> Src =
626         EmitPointerWithAlignment(E->getArg(1));
627     Value *SizeVal = EmitScalarExpr(E->getArg(2));
628     unsigned Align = std::min(Dest.second, Src.second);
629     Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
630     return RValue::get(Dest.first);
631   }
632 
633   case Builtin::BI__builtin___memcpy_chk: {
634     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
635     llvm::APSInt Size, DstSize;
636     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
637         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
638       break;
639     if (Size.ugt(DstSize))
640       break;
641     std::pair<llvm::Value*, unsigned> Dest =
642         EmitPointerWithAlignment(E->getArg(0));
643     std::pair<llvm::Value*, unsigned> Src =
644         EmitPointerWithAlignment(E->getArg(1));
645     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
646     unsigned Align = std::min(Dest.second, Src.second);
647     Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
648     return RValue::get(Dest.first);
649   }
650 
651   case Builtin::BI__builtin_objc_memmove_collectable: {
652     Value *Address = EmitScalarExpr(E->getArg(0));
653     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
654     Value *SizeVal = EmitScalarExpr(E->getArg(2));
655     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
656                                                   Address, SrcAddr, SizeVal);
657     return RValue::get(Address);
658   }
659 
660   case Builtin::BI__builtin___memmove_chk: {
661     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
662     llvm::APSInt Size, DstSize;
663     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
664         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
665       break;
666     if (Size.ugt(DstSize))
667       break;
668     std::pair<llvm::Value*, unsigned> Dest =
669         EmitPointerWithAlignment(E->getArg(0));
670     std::pair<llvm::Value*, unsigned> Src =
671         EmitPointerWithAlignment(E->getArg(1));
672     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
673     unsigned Align = std::min(Dest.second, Src.second);
674     Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
675     return RValue::get(Dest.first);
676   }
677 
678   case Builtin::BImemmove:
679   case Builtin::BI__builtin_memmove: {
680     std::pair<llvm::Value*, unsigned> Dest =
681         EmitPointerWithAlignment(E->getArg(0));
682     std::pair<llvm::Value*, unsigned> Src =
683         EmitPointerWithAlignment(E->getArg(1));
684     Value *SizeVal = EmitScalarExpr(E->getArg(2));
685     unsigned Align = std::min(Dest.second, Src.second);
686     Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
687     return RValue::get(Dest.first);
688   }
689   case Builtin::BImemset:
690   case Builtin::BI__builtin_memset: {
691     std::pair<llvm::Value*, unsigned> Dest =
692         EmitPointerWithAlignment(E->getArg(0));
693     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
694                                          Builder.getInt8Ty());
695     Value *SizeVal = EmitScalarExpr(E->getArg(2));
696     Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
697     return RValue::get(Dest.first);
698   }
699   case Builtin::BI__builtin___memset_chk: {
700     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
701     llvm::APSInt Size, DstSize;
702     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
703         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
704       break;
705     if (Size.ugt(DstSize))
706       break;
707     std::pair<llvm::Value*, unsigned> Dest =
708         EmitPointerWithAlignment(E->getArg(0));
709     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
710                                          Builder.getInt8Ty());
711     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
712     Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
713     return RValue::get(Dest.first);
714   }
715   case Builtin::BI__builtin_dwarf_cfa: {
716     // The offset in bytes from the first argument to the CFA.
717     //
718     // Why on earth is this in the frontend?  Is there any reason at
719     // all that the backend can't reasonably determine this while
720     // lowering llvm.eh.dwarf.cfa()?
721     //
722     // TODO: If there's a satisfactory reason, add a target hook for
723     // this instead of hard-coding 0, which is correct for most targets.
724     int32_t Offset = 0;
725 
726     Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
727     return RValue::get(Builder.CreateCall(F,
728                                       llvm::ConstantInt::get(Int32Ty, Offset)));
729   }
730   case Builtin::BI__builtin_return_address: {
731     Value *Depth = EmitScalarExpr(E->getArg(0));
732     Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
733     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
734     return RValue::get(Builder.CreateCall(F, Depth));
735   }
736   case Builtin::BI__builtin_frame_address: {
737     Value *Depth = EmitScalarExpr(E->getArg(0));
738     Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
739     Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
740     return RValue::get(Builder.CreateCall(F, Depth));
741   }
742   case Builtin::BI__builtin_extract_return_addr: {
743     Value *Address = EmitScalarExpr(E->getArg(0));
744     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
745     return RValue::get(Result);
746   }
747   case Builtin::BI__builtin_frob_return_addr: {
748     Value *Address = EmitScalarExpr(E->getArg(0));
749     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
750     return RValue::get(Result);
751   }
752   case Builtin::BI__builtin_dwarf_sp_column: {
753     llvm::IntegerType *Ty
754       = cast<llvm::IntegerType>(ConvertType(E->getType()));
755     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
756     if (Column == -1) {
757       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
758       return RValue::get(llvm::UndefValue::get(Ty));
759     }
760     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
761   }
762   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
763     Value *Address = EmitScalarExpr(E->getArg(0));
764     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
765       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
766     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
767   }
768   case Builtin::BI__builtin_eh_return: {
769     Value *Int = EmitScalarExpr(E->getArg(0));
770     Value *Ptr = EmitScalarExpr(E->getArg(1));
771 
772     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
773     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
774            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
775     Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
776                                   ? Intrinsic::eh_return_i32
777                                   : Intrinsic::eh_return_i64);
778     Builder.CreateCall2(F, Int, Ptr);
779     Builder.CreateUnreachable();
780 
781     // We do need to preserve an insertion point.
782     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
783 
784     return RValue::get(0);
785   }
786   case Builtin::BI__builtin_unwind_init: {
787     Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
788     return RValue::get(Builder.CreateCall(F));
789   }
790   case Builtin::BI__builtin_extend_pointer: {
791     // Extends a pointer to the size of an _Unwind_Word, which is
792     // uint64_t on all platforms.  Generally this gets poked into a
793     // register and eventually used as an address, so if the
794     // addressing registers are wider than pointers and the platform
795     // doesn't implicitly ignore high-order bits when doing
796     // addressing, we need to make sure we zext / sext based on
797     // the platform's expectations.
798     //
799     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
800 
801     // Cast the pointer to intptr_t.
802     Value *Ptr = EmitScalarExpr(E->getArg(0));
803     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
804 
805     // If that's 64 bits, we're done.
806     if (IntPtrTy->getBitWidth() == 64)
807       return RValue::get(Result);
808 
809     // Otherwise, ask the codegen data what to do.
810     if (getTargetHooks().extendPointerWithSExt())
811       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
812     else
813       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
814   }
815   case Builtin::BI__builtin_setjmp: {
816     // Buffer is a void**.
817     Value *Buf = EmitScalarExpr(E->getArg(0));
818 
819     // Store the frame pointer to the setjmp buffer.
820     Value *FrameAddr =
821       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
822                          ConstantInt::get(Int32Ty, 0));
823     Builder.CreateStore(FrameAddr, Buf);
824 
825     // Store the stack pointer to the setjmp buffer.
826     Value *StackAddr =
827       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
828     Value *StackSaveSlot =
829       Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2));
830     Builder.CreateStore(StackAddr, StackSaveSlot);
831 
832     // Call LLVM's EH setjmp, which is lightweight.
833     Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
834     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
835     return RValue::get(Builder.CreateCall(F, Buf));
836   }
837   case Builtin::BI__builtin_longjmp: {
838     Value *Buf = EmitScalarExpr(E->getArg(0));
839     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
840 
841     // Call LLVM's EH longjmp, which is lightweight.
842     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
843 
844     // longjmp doesn't return; mark this as unreachable.
845     Builder.CreateUnreachable();
846 
847     // We do need to preserve an insertion point.
848     EmitBlock(createBasicBlock("longjmp.cont"));
849 
850     return RValue::get(0);
851   }
852   case Builtin::BI__sync_fetch_and_add:
853   case Builtin::BI__sync_fetch_and_sub:
854   case Builtin::BI__sync_fetch_and_or:
855   case Builtin::BI__sync_fetch_and_and:
856   case Builtin::BI__sync_fetch_and_xor:
857   case Builtin::BI__sync_add_and_fetch:
858   case Builtin::BI__sync_sub_and_fetch:
859   case Builtin::BI__sync_and_and_fetch:
860   case Builtin::BI__sync_or_and_fetch:
861   case Builtin::BI__sync_xor_and_fetch:
862   case Builtin::BI__sync_val_compare_and_swap:
863   case Builtin::BI__sync_bool_compare_and_swap:
864   case Builtin::BI__sync_lock_test_and_set:
865   case Builtin::BI__sync_lock_release:
866   case Builtin::BI__sync_swap:
867     llvm_unreachable("Shouldn't make it through sema");
868   case Builtin::BI__sync_fetch_and_add_1:
869   case Builtin::BI__sync_fetch_and_add_2:
870   case Builtin::BI__sync_fetch_and_add_4:
871   case Builtin::BI__sync_fetch_and_add_8:
872   case Builtin::BI__sync_fetch_and_add_16:
873     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
874   case Builtin::BI__sync_fetch_and_sub_1:
875   case Builtin::BI__sync_fetch_and_sub_2:
876   case Builtin::BI__sync_fetch_and_sub_4:
877   case Builtin::BI__sync_fetch_and_sub_8:
878   case Builtin::BI__sync_fetch_and_sub_16:
879     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
880   case Builtin::BI__sync_fetch_and_or_1:
881   case Builtin::BI__sync_fetch_and_or_2:
882   case Builtin::BI__sync_fetch_and_or_4:
883   case Builtin::BI__sync_fetch_and_or_8:
884   case Builtin::BI__sync_fetch_and_or_16:
885     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
886   case Builtin::BI__sync_fetch_and_and_1:
887   case Builtin::BI__sync_fetch_and_and_2:
888   case Builtin::BI__sync_fetch_and_and_4:
889   case Builtin::BI__sync_fetch_and_and_8:
890   case Builtin::BI__sync_fetch_and_and_16:
891     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
892   case Builtin::BI__sync_fetch_and_xor_1:
893   case Builtin::BI__sync_fetch_and_xor_2:
894   case Builtin::BI__sync_fetch_and_xor_4:
895   case Builtin::BI__sync_fetch_and_xor_8:
896   case Builtin::BI__sync_fetch_and_xor_16:
897     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
898 
899   // Clang extensions: not overloaded yet.
900   case Builtin::BI__sync_fetch_and_min:
901     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
902   case Builtin::BI__sync_fetch_and_max:
903     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
904   case Builtin::BI__sync_fetch_and_umin:
905     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
906   case Builtin::BI__sync_fetch_and_umax:
907     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
908 
909   case Builtin::BI__sync_add_and_fetch_1:
910   case Builtin::BI__sync_add_and_fetch_2:
911   case Builtin::BI__sync_add_and_fetch_4:
912   case Builtin::BI__sync_add_and_fetch_8:
913   case Builtin::BI__sync_add_and_fetch_16:
914     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
915                                 llvm::Instruction::Add);
916   case Builtin::BI__sync_sub_and_fetch_1:
917   case Builtin::BI__sync_sub_and_fetch_2:
918   case Builtin::BI__sync_sub_and_fetch_4:
919   case Builtin::BI__sync_sub_and_fetch_8:
920   case Builtin::BI__sync_sub_and_fetch_16:
921     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
922                                 llvm::Instruction::Sub);
923   case Builtin::BI__sync_and_and_fetch_1:
924   case Builtin::BI__sync_and_and_fetch_2:
925   case Builtin::BI__sync_and_and_fetch_4:
926   case Builtin::BI__sync_and_and_fetch_8:
927   case Builtin::BI__sync_and_and_fetch_16:
928     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
929                                 llvm::Instruction::And);
930   case Builtin::BI__sync_or_and_fetch_1:
931   case Builtin::BI__sync_or_and_fetch_2:
932   case Builtin::BI__sync_or_and_fetch_4:
933   case Builtin::BI__sync_or_and_fetch_8:
934   case Builtin::BI__sync_or_and_fetch_16:
935     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
936                                 llvm::Instruction::Or);
937   case Builtin::BI__sync_xor_and_fetch_1:
938   case Builtin::BI__sync_xor_and_fetch_2:
939   case Builtin::BI__sync_xor_and_fetch_4:
940   case Builtin::BI__sync_xor_and_fetch_8:
941   case Builtin::BI__sync_xor_and_fetch_16:
942     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
943                                 llvm::Instruction::Xor);
944 
945   case Builtin::BI__sync_val_compare_and_swap_1:
946   case Builtin::BI__sync_val_compare_and_swap_2:
947   case Builtin::BI__sync_val_compare_and_swap_4:
948   case Builtin::BI__sync_val_compare_and_swap_8:
949   case Builtin::BI__sync_val_compare_and_swap_16: {
950     QualType T = E->getType();
951     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
952     unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
953 
954     llvm::IntegerType *IntType =
955       llvm::IntegerType::get(getLLVMContext(),
956                              getContext().getTypeSize(T));
957     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
958 
959     Value *Args[3];
960     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
961     Args[1] = EmitScalarExpr(E->getArg(1));
962     llvm::Type *ValueType = Args[1]->getType();
963     Args[1] = EmitToInt(*this, Args[1], T, IntType);
964     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
965 
966     Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
967                                                 llvm::SequentiallyConsistent);
968     Result = EmitFromInt(*this, Result, T, ValueType);
969     return RValue::get(Result);
970   }
971 
972   case Builtin::BI__sync_bool_compare_and_swap_1:
973   case Builtin::BI__sync_bool_compare_and_swap_2:
974   case Builtin::BI__sync_bool_compare_and_swap_4:
975   case Builtin::BI__sync_bool_compare_and_swap_8:
976   case Builtin::BI__sync_bool_compare_and_swap_16: {
977     QualType T = E->getArg(1)->getType();
978     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
979     unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
980 
981     llvm::IntegerType *IntType =
982       llvm::IntegerType::get(getLLVMContext(),
983                              getContext().getTypeSize(T));
984     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
985 
986     Value *Args[3];
987     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
988     Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType);
989     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
990 
991     Value *OldVal = Args[1];
992     Value *PrevVal = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
993                                                  llvm::SequentiallyConsistent);
994     Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal);
995     // zext bool to int.
996     Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
997     return RValue::get(Result);
998   }
999 
1000   case Builtin::BI__sync_swap_1:
1001   case Builtin::BI__sync_swap_2:
1002   case Builtin::BI__sync_swap_4:
1003   case Builtin::BI__sync_swap_8:
1004   case Builtin::BI__sync_swap_16:
1005     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1006 
1007   case Builtin::BI__sync_lock_test_and_set_1:
1008   case Builtin::BI__sync_lock_test_and_set_2:
1009   case Builtin::BI__sync_lock_test_and_set_4:
1010   case Builtin::BI__sync_lock_test_and_set_8:
1011   case Builtin::BI__sync_lock_test_and_set_16:
1012     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1013 
1014   case Builtin::BI__sync_lock_release_1:
1015   case Builtin::BI__sync_lock_release_2:
1016   case Builtin::BI__sync_lock_release_4:
1017   case Builtin::BI__sync_lock_release_8:
1018   case Builtin::BI__sync_lock_release_16: {
1019     Value *Ptr = EmitScalarExpr(E->getArg(0));
1020     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
1021     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1022     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1023                                              StoreSize.getQuantity() * 8);
1024     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1025     llvm::StoreInst *Store =
1026       Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
1027     Store->setAlignment(StoreSize.getQuantity());
1028     Store->setAtomic(llvm::Release);
1029     return RValue::get(0);
1030   }
1031 
1032   case Builtin::BI__sync_synchronize: {
1033     // We assume this is supposed to correspond to a C++0x-style
1034     // sequentially-consistent fence (i.e. this is only usable for
1035     // synchonization, not device I/O or anything like that). This intrinsic
1036     // is really badly designed in the sense that in theory, there isn't
1037     // any way to safely use it... but in practice, it mostly works
1038     // to use it with non-atomic loads and stores to get acquire/release
1039     // semantics.
1040     Builder.CreateFence(llvm::SequentiallyConsistent);
1041     return RValue::get(0);
1042   }
1043 
1044   case Builtin::BI__c11_atomic_is_lock_free:
1045   case Builtin::BI__atomic_is_lock_free: {
1046     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
1047     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
1048     // _Atomic(T) is always properly-aligned.
1049     const char *LibCallName = "__atomic_is_lock_free";
1050     CallArgList Args;
1051     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
1052              getContext().getSizeType());
1053     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1054       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
1055                getContext().VoidPtrTy);
1056     else
1057       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1058                getContext().VoidPtrTy);
1059     const CGFunctionInfo &FuncInfo =
1060         CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args,
1061                                                FunctionType::ExtInfo(),
1062                                                RequiredArgs::All);
1063     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1064     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1065     return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
1066   }
1067 
1068   case Builtin::BI__atomic_test_and_set: {
1069     // Look at the argument type to determine whether this is a volatile
1070     // operation. The parameter type is always volatile.
1071     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1072     bool Volatile =
1073         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1074 
1075     Value *Ptr = EmitScalarExpr(E->getArg(0));
1076     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1077     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1078     Value *NewVal = Builder.getInt8(1);
1079     Value *Order = EmitScalarExpr(E->getArg(1));
1080     if (isa<llvm::ConstantInt>(Order)) {
1081       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1082       AtomicRMWInst *Result = 0;
1083       switch (ord) {
1084       case 0:  // memory_order_relaxed
1085       default: // invalid order
1086         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1087                                          Ptr, NewVal,
1088                                          llvm::Monotonic);
1089         break;
1090       case 1:  // memory_order_consume
1091       case 2:  // memory_order_acquire
1092         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1093                                          Ptr, NewVal,
1094                                          llvm::Acquire);
1095         break;
1096       case 3:  // memory_order_release
1097         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1098                                          Ptr, NewVal,
1099                                          llvm::Release);
1100         break;
1101       case 4:  // memory_order_acq_rel
1102         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1103                                          Ptr, NewVal,
1104                                          llvm::AcquireRelease);
1105         break;
1106       case 5:  // memory_order_seq_cst
1107         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1108                                          Ptr, NewVal,
1109                                          llvm::SequentiallyConsistent);
1110         break;
1111       }
1112       Result->setVolatile(Volatile);
1113       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1114     }
1115 
1116     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1117 
1118     llvm::BasicBlock *BBs[5] = {
1119       createBasicBlock("monotonic", CurFn),
1120       createBasicBlock("acquire", CurFn),
1121       createBasicBlock("release", CurFn),
1122       createBasicBlock("acqrel", CurFn),
1123       createBasicBlock("seqcst", CurFn)
1124     };
1125     llvm::AtomicOrdering Orders[5] = {
1126       llvm::Monotonic, llvm::Acquire, llvm::Release,
1127       llvm::AcquireRelease, llvm::SequentiallyConsistent
1128     };
1129 
1130     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1131     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1132 
1133     Builder.SetInsertPoint(ContBB);
1134     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
1135 
1136     for (unsigned i = 0; i < 5; ++i) {
1137       Builder.SetInsertPoint(BBs[i]);
1138       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1139                                                    Ptr, NewVal, Orders[i]);
1140       RMW->setVolatile(Volatile);
1141       Result->addIncoming(RMW, BBs[i]);
1142       Builder.CreateBr(ContBB);
1143     }
1144 
1145     SI->addCase(Builder.getInt32(0), BBs[0]);
1146     SI->addCase(Builder.getInt32(1), BBs[1]);
1147     SI->addCase(Builder.getInt32(2), BBs[1]);
1148     SI->addCase(Builder.getInt32(3), BBs[2]);
1149     SI->addCase(Builder.getInt32(4), BBs[3]);
1150     SI->addCase(Builder.getInt32(5), BBs[4]);
1151 
1152     Builder.SetInsertPoint(ContBB);
1153     return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1154   }
1155 
1156   case Builtin::BI__atomic_clear: {
1157     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1158     bool Volatile =
1159         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1160 
1161     Value *Ptr = EmitScalarExpr(E->getArg(0));
1162     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1163     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1164     Value *NewVal = Builder.getInt8(0);
1165     Value *Order = EmitScalarExpr(E->getArg(1));
1166     if (isa<llvm::ConstantInt>(Order)) {
1167       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1168       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1169       Store->setAlignment(1);
1170       switch (ord) {
1171       case 0:  // memory_order_relaxed
1172       default: // invalid order
1173         Store->setOrdering(llvm::Monotonic);
1174         break;
1175       case 3:  // memory_order_release
1176         Store->setOrdering(llvm::Release);
1177         break;
1178       case 5:  // memory_order_seq_cst
1179         Store->setOrdering(llvm::SequentiallyConsistent);
1180         break;
1181       }
1182       return RValue::get(0);
1183     }
1184 
1185     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1186 
1187     llvm::BasicBlock *BBs[3] = {
1188       createBasicBlock("monotonic", CurFn),
1189       createBasicBlock("release", CurFn),
1190       createBasicBlock("seqcst", CurFn)
1191     };
1192     llvm::AtomicOrdering Orders[3] = {
1193       llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent
1194     };
1195 
1196     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1197     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1198 
1199     for (unsigned i = 0; i < 3; ++i) {
1200       Builder.SetInsertPoint(BBs[i]);
1201       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1202       Store->setAlignment(1);
1203       Store->setOrdering(Orders[i]);
1204       Builder.CreateBr(ContBB);
1205     }
1206 
1207     SI->addCase(Builder.getInt32(0), BBs[0]);
1208     SI->addCase(Builder.getInt32(3), BBs[1]);
1209     SI->addCase(Builder.getInt32(5), BBs[2]);
1210 
1211     Builder.SetInsertPoint(ContBB);
1212     return RValue::get(0);
1213   }
1214 
1215   case Builtin::BI__atomic_thread_fence:
1216   case Builtin::BI__atomic_signal_fence:
1217   case Builtin::BI__c11_atomic_thread_fence:
1218   case Builtin::BI__c11_atomic_signal_fence: {
1219     llvm::SynchronizationScope Scope;
1220     if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1221         BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1222       Scope = llvm::SingleThread;
1223     else
1224       Scope = llvm::CrossThread;
1225     Value *Order = EmitScalarExpr(E->getArg(0));
1226     if (isa<llvm::ConstantInt>(Order)) {
1227       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1228       switch (ord) {
1229       case 0:  // memory_order_relaxed
1230       default: // invalid order
1231         break;
1232       case 1:  // memory_order_consume
1233       case 2:  // memory_order_acquire
1234         Builder.CreateFence(llvm::Acquire, Scope);
1235         break;
1236       case 3:  // memory_order_release
1237         Builder.CreateFence(llvm::Release, Scope);
1238         break;
1239       case 4:  // memory_order_acq_rel
1240         Builder.CreateFence(llvm::AcquireRelease, Scope);
1241         break;
1242       case 5:  // memory_order_seq_cst
1243         Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1244         break;
1245       }
1246       return RValue::get(0);
1247     }
1248 
1249     llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1250     AcquireBB = createBasicBlock("acquire", CurFn);
1251     ReleaseBB = createBasicBlock("release", CurFn);
1252     AcqRelBB = createBasicBlock("acqrel", CurFn);
1253     SeqCstBB = createBasicBlock("seqcst", CurFn);
1254     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1255 
1256     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1257     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
1258 
1259     Builder.SetInsertPoint(AcquireBB);
1260     Builder.CreateFence(llvm::Acquire, Scope);
1261     Builder.CreateBr(ContBB);
1262     SI->addCase(Builder.getInt32(1), AcquireBB);
1263     SI->addCase(Builder.getInt32(2), AcquireBB);
1264 
1265     Builder.SetInsertPoint(ReleaseBB);
1266     Builder.CreateFence(llvm::Release, Scope);
1267     Builder.CreateBr(ContBB);
1268     SI->addCase(Builder.getInt32(3), ReleaseBB);
1269 
1270     Builder.SetInsertPoint(AcqRelBB);
1271     Builder.CreateFence(llvm::AcquireRelease, Scope);
1272     Builder.CreateBr(ContBB);
1273     SI->addCase(Builder.getInt32(4), AcqRelBB);
1274 
1275     Builder.SetInsertPoint(SeqCstBB);
1276     Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1277     Builder.CreateBr(ContBB);
1278     SI->addCase(Builder.getInt32(5), SeqCstBB);
1279 
1280     Builder.SetInsertPoint(ContBB);
1281     return RValue::get(0);
1282   }
1283 
1284     // Library functions with special handling.
1285   case Builtin::BIsqrt:
1286   case Builtin::BIsqrtf:
1287   case Builtin::BIsqrtl: {
1288     // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only
1289     // in finite- or unsafe-math mode (the intrinsic has different semantics
1290     // for handling negative numbers compared to the library function, so
1291     // -fmath-errno=0 is not enough).
1292     if (!FD->hasAttr<ConstAttr>())
1293       break;
1294     if (!(CGM.getCodeGenOpts().UnsafeFPMath ||
1295           CGM.getCodeGenOpts().NoNaNsFPMath))
1296       break;
1297     Value *Arg0 = EmitScalarExpr(E->getArg(0));
1298     llvm::Type *ArgType = Arg0->getType();
1299     Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType);
1300     return RValue::get(Builder.CreateCall(F, Arg0));
1301   }
1302 
1303   case Builtin::BIpow:
1304   case Builtin::BIpowf:
1305   case Builtin::BIpowl: {
1306     // Transform a call to pow* into a @llvm.pow.* intrinsic call.
1307     if (!FD->hasAttr<ConstAttr>())
1308       break;
1309     Value *Base = EmitScalarExpr(E->getArg(0));
1310     Value *Exponent = EmitScalarExpr(E->getArg(1));
1311     llvm::Type *ArgType = Base->getType();
1312     Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
1313     return RValue::get(Builder.CreateCall2(F, Base, Exponent));
1314     break;
1315   }
1316 
1317   case Builtin::BIfma:
1318   case Builtin::BIfmaf:
1319   case Builtin::BIfmal:
1320   case Builtin::BI__builtin_fma:
1321   case Builtin::BI__builtin_fmaf:
1322   case Builtin::BI__builtin_fmal: {
1323     // Rewrite fma to intrinsic.
1324     Value *FirstArg = EmitScalarExpr(E->getArg(0));
1325     llvm::Type *ArgType = FirstArg->getType();
1326     Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
1327     return RValue::get(Builder.CreateCall3(F, FirstArg,
1328                                               EmitScalarExpr(E->getArg(1)),
1329                                               EmitScalarExpr(E->getArg(2))));
1330   }
1331 
1332   case Builtin::BI__builtin_signbit:
1333   case Builtin::BI__builtin_signbitf:
1334   case Builtin::BI__builtin_signbitl: {
1335     LLVMContext &C = CGM.getLLVMContext();
1336 
1337     Value *Arg = EmitScalarExpr(E->getArg(0));
1338     llvm::Type *ArgTy = Arg->getType();
1339     if (ArgTy->isPPC_FP128Ty())
1340       break; // FIXME: I'm not sure what the right implementation is here.
1341     int ArgWidth = ArgTy->getPrimitiveSizeInBits();
1342     llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
1343     Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy);
1344     Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy);
1345     Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp);
1346     return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
1347   }
1348   case Builtin::BI__builtin_annotation: {
1349     llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1350     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1351                                       AnnVal->getType());
1352 
1353     // Get the annotation string, go through casts. Sema requires this to be a
1354     // non-wide string literal, potentially casted, so the cast<> is safe.
1355     const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
1356     StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1357     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
1358   }
1359   case Builtin::BI__builtin_addcb:
1360   case Builtin::BI__builtin_addcs:
1361   case Builtin::BI__builtin_addc:
1362   case Builtin::BI__builtin_addcl:
1363   case Builtin::BI__builtin_addcll:
1364   case Builtin::BI__builtin_subcb:
1365   case Builtin::BI__builtin_subcs:
1366   case Builtin::BI__builtin_subc:
1367   case Builtin::BI__builtin_subcl:
1368   case Builtin::BI__builtin_subcll: {
1369 
1370     // We translate all of these builtins from expressions of the form:
1371     //   int x = ..., y = ..., carryin = ..., carryout, result;
1372     //   result = __builtin_addc(x, y, carryin, &carryout);
1373     //
1374     // to LLVM IR of the form:
1375     //
1376     //   %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
1377     //   %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
1378     //   %carry1 = extractvalue {i32, i1} %tmp1, 1
1379     //   %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
1380     //                                                       i32 %carryin)
1381     //   %result = extractvalue {i32, i1} %tmp2, 0
1382     //   %carry2 = extractvalue {i32, i1} %tmp2, 1
1383     //   %tmp3 = or i1 %carry1, %carry2
1384     //   %tmp4 = zext i1 %tmp3 to i32
1385     //   store i32 %tmp4, i32* %carryout
1386 
1387     // Scalarize our inputs.
1388     llvm::Value *X = EmitScalarExpr(E->getArg(0));
1389     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
1390     llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
1391     std::pair<llvm::Value*, unsigned> CarryOutPtr =
1392       EmitPointerWithAlignment(E->getArg(3));
1393 
1394     // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
1395     llvm::Intrinsic::ID IntrinsicId;
1396     switch (BuiltinID) {
1397     default: llvm_unreachable("Unknown multiprecision builtin id.");
1398     case Builtin::BI__builtin_addcb:
1399     case Builtin::BI__builtin_addcs:
1400     case Builtin::BI__builtin_addc:
1401     case Builtin::BI__builtin_addcl:
1402     case Builtin::BI__builtin_addcll:
1403       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1404       break;
1405     case Builtin::BI__builtin_subcb:
1406     case Builtin::BI__builtin_subcs:
1407     case Builtin::BI__builtin_subc:
1408     case Builtin::BI__builtin_subcl:
1409     case Builtin::BI__builtin_subcll:
1410       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1411       break;
1412     }
1413 
1414     // Construct our resulting LLVM IR expression.
1415     llvm::Value *Carry1;
1416     llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
1417                                               X, Y, Carry1);
1418     llvm::Value *Carry2;
1419     llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
1420                                               Sum1, Carryin, Carry2);
1421     llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
1422                                                X->getType());
1423     llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut,
1424                                                          CarryOutPtr.first);
1425     CarryOutStore->setAlignment(CarryOutPtr.second);
1426     return RValue::get(Sum2);
1427   }
1428   case Builtin::BI__builtin_uadd_overflow:
1429   case Builtin::BI__builtin_uaddl_overflow:
1430   case Builtin::BI__builtin_uaddll_overflow:
1431   case Builtin::BI__builtin_usub_overflow:
1432   case Builtin::BI__builtin_usubl_overflow:
1433   case Builtin::BI__builtin_usubll_overflow:
1434   case Builtin::BI__builtin_umul_overflow:
1435   case Builtin::BI__builtin_umull_overflow:
1436   case Builtin::BI__builtin_umulll_overflow:
1437   case Builtin::BI__builtin_sadd_overflow:
1438   case Builtin::BI__builtin_saddl_overflow:
1439   case Builtin::BI__builtin_saddll_overflow:
1440   case Builtin::BI__builtin_ssub_overflow:
1441   case Builtin::BI__builtin_ssubl_overflow:
1442   case Builtin::BI__builtin_ssubll_overflow:
1443   case Builtin::BI__builtin_smul_overflow:
1444   case Builtin::BI__builtin_smull_overflow:
1445   case Builtin::BI__builtin_smulll_overflow: {
1446 
1447     // We translate all of these builtins directly to the relevant llvm IR node.
1448 
1449     // Scalarize our inputs.
1450     llvm::Value *X = EmitScalarExpr(E->getArg(0));
1451     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
1452     std::pair<llvm::Value *, unsigned> SumOutPtr =
1453       EmitPointerWithAlignment(E->getArg(2));
1454 
1455     // Decide which of the overflow intrinsics we are lowering to:
1456     llvm::Intrinsic::ID IntrinsicId;
1457     switch (BuiltinID) {
1458     default: llvm_unreachable("Unknown security overflow builtin id.");
1459     case Builtin::BI__builtin_uadd_overflow:
1460     case Builtin::BI__builtin_uaddl_overflow:
1461     case Builtin::BI__builtin_uaddll_overflow:
1462       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1463       break;
1464     case Builtin::BI__builtin_usub_overflow:
1465     case Builtin::BI__builtin_usubl_overflow:
1466     case Builtin::BI__builtin_usubll_overflow:
1467       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1468       break;
1469     case Builtin::BI__builtin_umul_overflow:
1470     case Builtin::BI__builtin_umull_overflow:
1471     case Builtin::BI__builtin_umulll_overflow:
1472       IntrinsicId = llvm::Intrinsic::umul_with_overflow;
1473       break;
1474     case Builtin::BI__builtin_sadd_overflow:
1475     case Builtin::BI__builtin_saddl_overflow:
1476     case Builtin::BI__builtin_saddll_overflow:
1477       IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
1478       break;
1479     case Builtin::BI__builtin_ssub_overflow:
1480     case Builtin::BI__builtin_ssubl_overflow:
1481     case Builtin::BI__builtin_ssubll_overflow:
1482       IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
1483       break;
1484     case Builtin::BI__builtin_smul_overflow:
1485     case Builtin::BI__builtin_smull_overflow:
1486     case Builtin::BI__builtin_smulll_overflow:
1487       IntrinsicId = llvm::Intrinsic::smul_with_overflow;
1488       break;
1489     }
1490 
1491 
1492     llvm::Value *Carry;
1493     llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
1494     llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first);
1495     SumOutStore->setAlignment(SumOutPtr.second);
1496 
1497     return RValue::get(Carry);
1498   }
1499   case Builtin::BI__builtin_addressof:
1500     return RValue::get(EmitLValue(E->getArg(0)).getAddress());
1501   case Builtin::BI__noop:
1502     return RValue::get(0);
1503   }
1504 
1505   // If this is an alias for a lib function (e.g. __builtin_sin), emit
1506   // the call using the normal call path, but using the unmangled
1507   // version of the function name.
1508   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
1509     return emitLibraryCall(*this, FD, E,
1510                            CGM.getBuiltinLibFunction(FD, BuiltinID));
1511 
1512   // If this is a predefined lib function (e.g. malloc), emit the call
1513   // using exactly the normal call path.
1514   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
1515     return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee()));
1516 
1517   // See if we have a target specific intrinsic.
1518   const char *Name = getContext().BuiltinInfo.GetName(BuiltinID);
1519   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
1520   if (const char *Prefix =
1521       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch()))
1522     IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name);
1523 
1524   if (IntrinsicID != Intrinsic::not_intrinsic) {
1525     SmallVector<Value*, 16> Args;
1526 
1527     // Find out if any arguments are required to be integer constant
1528     // expressions.
1529     unsigned ICEArguments = 0;
1530     ASTContext::GetBuiltinTypeError Error;
1531     getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
1532     assert(Error == ASTContext::GE_None && "Should not codegen an error");
1533 
1534     Function *F = CGM.getIntrinsic(IntrinsicID);
1535     llvm::FunctionType *FTy = F->getFunctionType();
1536 
1537     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
1538       Value *ArgValue;
1539       // If this is a normal argument, just emit it as a scalar.
1540       if ((ICEArguments & (1 << i)) == 0) {
1541         ArgValue = EmitScalarExpr(E->getArg(i));
1542       } else {
1543         // If this is required to be a constant, constant fold it so that we
1544         // know that the generated intrinsic gets a ConstantInt.
1545         llvm::APSInt Result;
1546         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
1547         assert(IsConst && "Constant arg isn't actually constant?");
1548         (void)IsConst;
1549         ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
1550       }
1551 
1552       // If the intrinsic arg type is different from the builtin arg type
1553       // we need to do a bit cast.
1554       llvm::Type *PTy = FTy->getParamType(i);
1555       if (PTy != ArgValue->getType()) {
1556         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
1557                "Must be able to losslessly bit cast to param");
1558         ArgValue = Builder.CreateBitCast(ArgValue, PTy);
1559       }
1560 
1561       Args.push_back(ArgValue);
1562     }
1563 
1564     Value *V = Builder.CreateCall(F, Args);
1565     QualType BuiltinRetType = E->getType();
1566 
1567     llvm::Type *RetTy = VoidTy;
1568     if (!BuiltinRetType->isVoidType())
1569       RetTy = ConvertType(BuiltinRetType);
1570 
1571     if (RetTy != V->getType()) {
1572       assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
1573              "Must be able to losslessly bit cast result type");
1574       V = Builder.CreateBitCast(V, RetTy);
1575     }
1576 
1577     return RValue::get(V);
1578   }
1579 
1580   // See if we have a target specific builtin that needs to be lowered.
1581   if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
1582     return RValue::get(V);
1583 
1584   ErrorUnsupported(E, "builtin function");
1585 
1586   // Unknown builtin, for now just dump it out and return undef.
1587   return GetUndefRValue(E->getType());
1588 }
1589 
1590 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
1591                                               const CallExpr *E) {
1592   switch (getTarget().getTriple().getArch()) {
1593   case llvm::Triple::aarch64:
1594     return EmitAArch64BuiltinExpr(BuiltinID, E);
1595   case llvm::Triple::arm:
1596   case llvm::Triple::thumb:
1597     return EmitARMBuiltinExpr(BuiltinID, E);
1598   case llvm::Triple::x86:
1599   case llvm::Triple::x86_64:
1600     return EmitX86BuiltinExpr(BuiltinID, E);
1601   case llvm::Triple::ppc:
1602   case llvm::Triple::ppc64:
1603   case llvm::Triple::ppc64le:
1604     return EmitPPCBuiltinExpr(BuiltinID, E);
1605   default:
1606     return 0;
1607   }
1608 }
1609 
1610 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
1611                                      NeonTypeFlags TypeFlags,
1612                                      bool V1Ty=false) {
1613   int IsQuad = TypeFlags.isQuad();
1614   switch (TypeFlags.getEltType()) {
1615   case NeonTypeFlags::Int8:
1616   case NeonTypeFlags::Poly8:
1617     return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
1618   case NeonTypeFlags::Int16:
1619   case NeonTypeFlags::Poly16:
1620   case NeonTypeFlags::Float16:
1621     return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
1622   case NeonTypeFlags::Int32:
1623     return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
1624   case NeonTypeFlags::Int64:
1625   case NeonTypeFlags::Poly64:
1626     return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
1627   case NeonTypeFlags::Poly128:
1628     // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
1629     // There is a lot of i128 and f128 API missing.
1630     // so we use v16i8 to represent poly128 and get pattern matched.
1631     return llvm::VectorType::get(CGF->Int8Ty, 16);
1632   case NeonTypeFlags::Float32:
1633     return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
1634   case NeonTypeFlags::Float64:
1635     return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
1636   }
1637   llvm_unreachable("Unknown vector element type!");
1638 }
1639 
1640 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
1641   unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
1642   Value* SV = llvm::ConstantVector::getSplat(nElts, C);
1643   return Builder.CreateShuffleVector(V, V, SV, "lane");
1644 }
1645 
1646 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
1647                                      const char *name,
1648                                      unsigned shift, bool rightshift) {
1649   unsigned j = 0;
1650   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1651        ai != ae; ++ai, ++j)
1652     if (shift > 0 && shift == j)
1653       Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
1654     else
1655       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
1656 
1657   return Builder.CreateCall(F, Ops, name);
1658 }
1659 
1660 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
1661                                             bool neg) {
1662   int SV = cast<ConstantInt>(V)->getSExtValue();
1663 
1664   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
1665   llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
1666   return llvm::ConstantVector::getSplat(VTy->getNumElements(), C);
1667 }
1668 
1669 // \brief Right-shift a vector by a constant.
1670 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
1671                                           llvm::Type *Ty, bool usgn,
1672                                           const char *name) {
1673   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
1674 
1675   int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
1676   int EltSize = VTy->getScalarSizeInBits();
1677 
1678   Vec = Builder.CreateBitCast(Vec, Ty);
1679 
1680   // lshr/ashr are undefined when the shift amount is equal to the vector
1681   // element size.
1682   if (ShiftAmt == EltSize) {
1683     if (usgn) {
1684       // Right-shifting an unsigned value by its size yields 0.
1685       llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0);
1686       return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero);
1687     } else {
1688       // Right-shifting a signed value by its size is equivalent
1689       // to a shift of size-1.
1690       --ShiftAmt;
1691       Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
1692     }
1693   }
1694 
1695   Shift = EmitNeonShiftVector(Shift, Ty, false);
1696   if (usgn)
1697     return Builder.CreateLShr(Vec, Shift, name);
1698   else
1699     return Builder.CreateAShr(Vec, Shift, name);
1700 }
1701 
1702 /// GetPointeeAlignment - Given an expression with a pointer type, find the
1703 /// alignment of the type referenced by the pointer.  Skip over implicit
1704 /// casts.
1705 std::pair<llvm::Value*, unsigned>
1706 CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) {
1707   assert(Addr->getType()->isPointerType());
1708   Addr = Addr->IgnoreParens();
1709   if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) {
1710     if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) &&
1711         ICE->getSubExpr()->getType()->isPointerType()) {
1712       std::pair<llvm::Value*, unsigned> Ptr =
1713           EmitPointerWithAlignment(ICE->getSubExpr());
1714       Ptr.first = Builder.CreateBitCast(Ptr.first,
1715                                         ConvertType(Addr->getType()));
1716       return Ptr;
1717     } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
1718       LValue LV = EmitLValue(ICE->getSubExpr());
1719       unsigned Align = LV.getAlignment().getQuantity();
1720       if (!Align) {
1721         // FIXME: Once LValues are fixed to always set alignment,
1722         // zap this code.
1723         QualType PtTy = ICE->getSubExpr()->getType();
1724         if (!PtTy->isIncompleteType())
1725           Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1726         else
1727           Align = 1;
1728       }
1729       return std::make_pair(LV.getAddress(), Align);
1730     }
1731   }
1732   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) {
1733     if (UO->getOpcode() == UO_AddrOf) {
1734       LValue LV = EmitLValue(UO->getSubExpr());
1735       unsigned Align = LV.getAlignment().getQuantity();
1736       if (!Align) {
1737         // FIXME: Once LValues are fixed to always set alignment,
1738         // zap this code.
1739         QualType PtTy = UO->getSubExpr()->getType();
1740         if (!PtTy->isIncompleteType())
1741           Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1742         else
1743           Align = 1;
1744       }
1745       return std::make_pair(LV.getAddress(), Align);
1746     }
1747   }
1748 
1749   unsigned Align = 1;
1750   QualType PtTy = Addr->getType()->getPointeeType();
1751   if (!PtTy->isIncompleteType())
1752     Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1753 
1754   return std::make_pair(EmitScalarExpr(Addr), Align);
1755 }
1756 
1757 static Value *EmitAArch64ScalarBuiltinExpr(CodeGenFunction &CGF,
1758                                            unsigned BuiltinID,
1759                                            const CallExpr *E) {
1760   unsigned int Int = 0;
1761   unsigned IntTypes = 0;
1762   enum {
1763     ScalarRet = (1 << 0),
1764     VectorRet = (1 << 1),
1765     ScalarArg0 = (1 << 2),
1766     VectorGetArg0 = (1 << 3),
1767     VectorCastArg0 = (1 << 4),
1768     ScalarArg1 = (1 << 5),
1769     VectorGetArg1 = (1 << 6),
1770     VectorCastArg1 = (1 << 7),
1771     ScalarFpCmpzArg1 = (1 << 8)
1772   };
1773   const char *s = NULL;
1774 
1775   SmallVector<Value *, 4> Ops;
1776   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
1777     Ops.push_back(CGF.EmitScalarExpr(E->getArg(i)));
1778   }
1779 
1780   // AArch64 scalar builtins are not overloaded, they do not have an extra
1781   // argument that specifies the vector type, need to handle each case.
1782   switch (BuiltinID) {
1783   default: break;
1784   case AArch64::BI__builtin_neon_vdups_lane_f32:
1785   case AArch64::BI__builtin_neon_vdupd_lane_f64:
1786   case AArch64::BI__builtin_neon_vdups_laneq_f32:
1787   case AArch64::BI__builtin_neon_vdupd_laneq_f64: {
1788     return CGF.Builder.CreateExtractElement(Ops[0], Ops[1], "vdup_lane");
1789   }
1790   case AArch64::BI__builtin_neon_vdupb_lane_i8:
1791   case AArch64::BI__builtin_neon_vduph_lane_i16:
1792   case AArch64::BI__builtin_neon_vdups_lane_i32:
1793   case AArch64::BI__builtin_neon_vdupd_lane_i64:
1794   case AArch64::BI__builtin_neon_vdupb_laneq_i8:
1795   case AArch64::BI__builtin_neon_vduph_laneq_i16:
1796   case AArch64::BI__builtin_neon_vdups_laneq_i32:
1797   case AArch64::BI__builtin_neon_vdupd_laneq_i64: {
1798     // The backend treats Neon scalar types as v1ix types
1799     // So we want to dup lane from any vector to v1ix vector
1800     // with shufflevector
1801     s = "vdup_lane";
1802     Value* SV = llvm::ConstantVector::getSplat(1, cast<ConstantInt>(Ops[1]));
1803     Value *Result = CGF.Builder.CreateShuffleVector(Ops[0], Ops[0], SV, s);
1804     llvm::Type *Ty = CGF.ConvertType(E->getCallReturnType());
1805     // AArch64 intrinsic one-element vector type cast to
1806     // scalar type expected by the builtin
1807     return CGF.Builder.CreateBitCast(Result, Ty, s);
1808   }
1809   case AArch64::BI__builtin_neon_vqdmlalh_lane_s16 :
1810   case AArch64::BI__builtin_neon_vqdmlalh_laneq_s16 :
1811   case AArch64::BI__builtin_neon_vqdmlals_lane_s32 :
1812   case AArch64::BI__builtin_neon_vqdmlals_laneq_s32 :
1813   case AArch64::BI__builtin_neon_vqdmlslh_lane_s16 :
1814   case AArch64::BI__builtin_neon_vqdmlslh_laneq_s16 :
1815   case AArch64::BI__builtin_neon_vqdmlsls_lane_s32 :
1816   case AArch64::BI__builtin_neon_vqdmlsls_laneq_s32 : {
1817     Int = Intrinsic::arm_neon_vqadds;
1818     if (BuiltinID == AArch64::BI__builtin_neon_vqdmlslh_lane_s16 ||
1819         BuiltinID == AArch64::BI__builtin_neon_vqdmlslh_laneq_s16 ||
1820         BuiltinID == AArch64::BI__builtin_neon_vqdmlsls_lane_s32 ||
1821         BuiltinID == AArch64::BI__builtin_neon_vqdmlsls_laneq_s32) {
1822       Int = Intrinsic::arm_neon_vqsubs;
1823     }
1824     // create vqdmull call with b * c[i]
1825     llvm::Type *Ty = CGF.ConvertType(E->getArg(1)->getType());
1826     llvm::VectorType *OpVTy = llvm::VectorType::get(Ty, 1);
1827     Ty = CGF.ConvertType(E->getArg(0)->getType());
1828     llvm::VectorType *ResVTy = llvm::VectorType::get(Ty, 1);
1829     Value *F = CGF.CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, ResVTy);
1830     Value *V = UndefValue::get(OpVTy);
1831     llvm::Constant *CI = ConstantInt::get(CGF.Int32Ty, 0);
1832     SmallVector<Value *, 2> MulOps;
1833     MulOps.push_back(Ops[1]);
1834     MulOps.push_back(Ops[2]);
1835     MulOps[0] = CGF.Builder.CreateInsertElement(V, MulOps[0], CI);
1836     MulOps[1] = CGF.Builder.CreateExtractElement(MulOps[1], Ops[3], "extract");
1837     MulOps[1] = CGF.Builder.CreateInsertElement(V, MulOps[1], CI);
1838     Value *MulRes = CGF.Builder.CreateCall2(F, MulOps[0], MulOps[1]);
1839     // create vqadds call with a +/- vqdmull result
1840     F = CGF.CGM.getIntrinsic(Int, ResVTy);
1841     SmallVector<Value *, 2> AddOps;
1842     AddOps.push_back(Ops[0]);
1843     AddOps.push_back(MulRes);
1844     V = UndefValue::get(ResVTy);
1845     AddOps[0] = CGF.Builder.CreateInsertElement(V, AddOps[0], CI);
1846     Value *AddRes = CGF.Builder.CreateCall2(F, AddOps[0], AddOps[1]);
1847     return CGF.Builder.CreateBitCast(AddRes, Ty);
1848   }
1849   case AArch64::BI__builtin_neon_vfmas_lane_f32:
1850   case AArch64::BI__builtin_neon_vfmas_laneq_f32:
1851   case AArch64::BI__builtin_neon_vfmad_lane_f64:
1852   case AArch64::BI__builtin_neon_vfmad_laneq_f64: {
1853     llvm::Type *Ty = CGF.ConvertType(E->getCallReturnType());
1854     Value *F = CGF.CGM.getIntrinsic(Intrinsic::fma, Ty);
1855     Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
1856     return CGF.Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
1857   }
1858   // Scalar Floating-point Multiply Extended
1859   case AArch64::BI__builtin_neon_vmulxs_f32:
1860   case AArch64::BI__builtin_neon_vmulxd_f64: {
1861     Int = Intrinsic::aarch64_neon_vmulx;
1862     llvm::Type *Ty = CGF.ConvertType(E->getCallReturnType());
1863     return CGF.EmitNeonCall(CGF.CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
1864   }
1865   case AArch64::BI__builtin_neon_vmul_n_f64: {
1866     // v1f64 vmul_n_f64  should be mapped to Neon scalar mul lane
1867     llvm::Type *VTy = GetNeonType(&CGF,
1868       NeonTypeFlags(NeonTypeFlags::Float64, false, false));
1869     Ops[0] = CGF.Builder.CreateBitCast(Ops[0], VTy);
1870     llvm::Value *Idx = llvm::ConstantInt::get(CGF.Int32Ty, 0);
1871     Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], Idx, "extract");
1872     Value *Result = CGF.Builder.CreateFMul(Ops[0], Ops[1]);
1873     return CGF.Builder.CreateBitCast(Result, VTy);
1874   }
1875   case AArch64::BI__builtin_neon_vget_lane_i8:
1876   case AArch64::BI__builtin_neon_vget_lane_i16:
1877   case AArch64::BI__builtin_neon_vget_lane_i32:
1878   case AArch64::BI__builtin_neon_vget_lane_i64:
1879   case AArch64::BI__builtin_neon_vget_lane_f32:
1880   case AArch64::BI__builtin_neon_vget_lane_f64:
1881   case AArch64::BI__builtin_neon_vgetq_lane_i8:
1882   case AArch64::BI__builtin_neon_vgetq_lane_i16:
1883   case AArch64::BI__builtin_neon_vgetq_lane_i32:
1884   case AArch64::BI__builtin_neon_vgetq_lane_i64:
1885   case AArch64::BI__builtin_neon_vgetq_lane_f32:
1886   case AArch64::BI__builtin_neon_vgetq_lane_f64:
1887     return CGF.EmitARMBuiltinExpr(ARM::BI__builtin_neon_vget_lane_i8, E);
1888   case AArch64::BI__builtin_neon_vset_lane_i8:
1889   case AArch64::BI__builtin_neon_vset_lane_i16:
1890   case AArch64::BI__builtin_neon_vset_lane_i32:
1891   case AArch64::BI__builtin_neon_vset_lane_i64:
1892   case AArch64::BI__builtin_neon_vset_lane_f32:
1893   case AArch64::BI__builtin_neon_vset_lane_f64:
1894   case AArch64::BI__builtin_neon_vsetq_lane_i8:
1895   case AArch64::BI__builtin_neon_vsetq_lane_i16:
1896   case AArch64::BI__builtin_neon_vsetq_lane_i32:
1897   case AArch64::BI__builtin_neon_vsetq_lane_i64:
1898   case AArch64::BI__builtin_neon_vsetq_lane_f32:
1899   case AArch64::BI__builtin_neon_vsetq_lane_f64:
1900     return CGF.EmitARMBuiltinExpr(ARM::BI__builtin_neon_vset_lane_i8, E);
1901   // Crypto
1902   case AArch64::BI__builtin_neon_vsha1h_u32:
1903     Int = Intrinsic::arm_neon_sha1h;
1904     s = "sha1h"; IntTypes = VectorRet; break;
1905   case AArch64::BI__builtin_neon_vsha1cq_u32:
1906     Int = Intrinsic::aarch64_neon_sha1c;
1907     s = "sha1c"; break;
1908   case AArch64::BI__builtin_neon_vsha1pq_u32:
1909     Int = Intrinsic::aarch64_neon_sha1p;
1910     s = "sha1p"; break;
1911   case AArch64::BI__builtin_neon_vsha1mq_u32:
1912     Int = Intrinsic::aarch64_neon_sha1m;
1913     s = "sha1m"; break;
1914   // Scalar Add
1915   case AArch64::BI__builtin_neon_vaddd_s64:
1916     Int = Intrinsic::aarch64_neon_vaddds;
1917     s = "vaddds"; break;
1918   case AArch64::BI__builtin_neon_vaddd_u64:
1919     Int = Intrinsic::aarch64_neon_vadddu;
1920     s = "vadddu"; break;
1921   // Scalar Sub
1922   case AArch64::BI__builtin_neon_vsubd_s64:
1923     Int = Intrinsic::aarch64_neon_vsubds;
1924     s = "vsubds"; break;
1925   case AArch64::BI__builtin_neon_vsubd_u64:
1926     Int = Intrinsic::aarch64_neon_vsubdu;
1927     s = "vsubdu"; break;
1928   // Scalar Saturating Add
1929   case AArch64::BI__builtin_neon_vqaddb_s8:
1930   case AArch64::BI__builtin_neon_vqaddh_s16:
1931   case AArch64::BI__builtin_neon_vqadds_s32:
1932   case AArch64::BI__builtin_neon_vqaddd_s64:
1933     Int = Intrinsic::arm_neon_vqadds;
1934     s = "vqadds"; IntTypes = VectorRet; break;
1935   case AArch64::BI__builtin_neon_vqaddb_u8:
1936   case AArch64::BI__builtin_neon_vqaddh_u16:
1937   case AArch64::BI__builtin_neon_vqadds_u32:
1938   case AArch64::BI__builtin_neon_vqaddd_u64:
1939     Int = Intrinsic::arm_neon_vqaddu;
1940     s = "vqaddu"; IntTypes = VectorRet; break;
1941   // Scalar Saturating Sub
1942   case AArch64::BI__builtin_neon_vqsubb_s8:
1943   case AArch64::BI__builtin_neon_vqsubh_s16:
1944   case AArch64::BI__builtin_neon_vqsubs_s32:
1945   case AArch64::BI__builtin_neon_vqsubd_s64:
1946     Int = Intrinsic::arm_neon_vqsubs;
1947     s = "vqsubs"; IntTypes = VectorRet; break;
1948   case AArch64::BI__builtin_neon_vqsubb_u8:
1949   case AArch64::BI__builtin_neon_vqsubh_u16:
1950   case AArch64::BI__builtin_neon_vqsubs_u32:
1951   case AArch64::BI__builtin_neon_vqsubd_u64:
1952     Int = Intrinsic::arm_neon_vqsubu;
1953     s = "vqsubu"; IntTypes = VectorRet; break;
1954   // Scalar Shift Left
1955   case AArch64::BI__builtin_neon_vshld_s64:
1956     Int = Intrinsic::aarch64_neon_vshlds;
1957     s = "vshlds"; break;
1958   case AArch64::BI__builtin_neon_vshld_u64:
1959     Int = Intrinsic::aarch64_neon_vshldu;
1960     s = "vshldu"; break;
1961   // Scalar Saturating Shift Left
1962   case AArch64::BI__builtin_neon_vqshlb_s8:
1963   case AArch64::BI__builtin_neon_vqshlh_s16:
1964   case AArch64::BI__builtin_neon_vqshls_s32:
1965   case AArch64::BI__builtin_neon_vqshld_s64:
1966     Int = Intrinsic::aarch64_neon_vqshls;
1967     s = "vqshls"; IntTypes = VectorRet; break;
1968   case AArch64::BI__builtin_neon_vqshlb_u8:
1969   case AArch64::BI__builtin_neon_vqshlh_u16:
1970   case AArch64::BI__builtin_neon_vqshls_u32:
1971   case AArch64::BI__builtin_neon_vqshld_u64:
1972     Int = Intrinsic::aarch64_neon_vqshlu;
1973     s = "vqshlu"; IntTypes = VectorRet; break;
1974   // Scalar Rouding Shift Left
1975   case AArch64::BI__builtin_neon_vrshld_s64:
1976     Int = Intrinsic::aarch64_neon_vrshlds;
1977     s = "vrshlds"; break;
1978   case AArch64::BI__builtin_neon_vrshld_u64:
1979     Int = Intrinsic::aarch64_neon_vrshldu;
1980     s = "vrshldu"; break;
1981   // Scalar Saturating Rouding Shift Left
1982   case AArch64::BI__builtin_neon_vqrshlb_s8:
1983   case AArch64::BI__builtin_neon_vqrshlh_s16:
1984   case AArch64::BI__builtin_neon_vqrshls_s32:
1985   case AArch64::BI__builtin_neon_vqrshld_s64:
1986     Int = Intrinsic::aarch64_neon_vqrshls;
1987     s = "vqrshls"; IntTypes = VectorRet; break;
1988   case AArch64::BI__builtin_neon_vqrshlb_u8:
1989   case AArch64::BI__builtin_neon_vqrshlh_u16:
1990   case AArch64::BI__builtin_neon_vqrshls_u32:
1991   case AArch64::BI__builtin_neon_vqrshld_u64:
1992     Int = Intrinsic::aarch64_neon_vqrshlu;
1993     s = "vqrshlu"; IntTypes = VectorRet; break;
1994   // Scalar Reduce Pairwise Add
1995   case AArch64::BI__builtin_neon_vpaddd_s64:
1996   case AArch64::BI__builtin_neon_vpaddd_u64:
1997     Int = Intrinsic::aarch64_neon_vpadd;
1998     s = "vpadd"; break;
1999   case AArch64::BI__builtin_neon_vaddv_f32:
2000   case AArch64::BI__builtin_neon_vaddvq_f32:
2001   case AArch64::BI__builtin_neon_vaddvq_f64:
2002   case AArch64::BI__builtin_neon_vpadds_f32:
2003   case AArch64::BI__builtin_neon_vpaddd_f64:
2004     Int = Intrinsic::aarch64_neon_vpfadd;
2005     s = "vpfadd"; IntTypes = ScalarRet | VectorCastArg0; break;
2006   // Scalar Reduce Pairwise Floating Point Max
2007   case AArch64::BI__builtin_neon_vmaxv_f32:
2008   case AArch64::BI__builtin_neon_vpmaxs_f32:
2009   case AArch64::BI__builtin_neon_vmaxvq_f64:
2010   case AArch64::BI__builtin_neon_vpmaxqd_f64:
2011     Int = Intrinsic::aarch64_neon_vpmax;
2012     s = "vpmax"; IntTypes = ScalarRet | VectorCastArg0; break;
2013   // Scalar Reduce Pairwise Floating Point Min
2014   case AArch64::BI__builtin_neon_vminv_f32:
2015   case AArch64::BI__builtin_neon_vpmins_f32:
2016   case AArch64::BI__builtin_neon_vminvq_f64:
2017   case AArch64::BI__builtin_neon_vpminqd_f64:
2018     Int = Intrinsic::aarch64_neon_vpmin;
2019     s = "vpmin"; IntTypes = ScalarRet | VectorCastArg0; break;
2020   // Scalar Reduce Pairwise Floating Point Maxnm
2021   case AArch64::BI__builtin_neon_vmaxnmv_f32:
2022   case AArch64::BI__builtin_neon_vpmaxnms_f32:
2023   case AArch64::BI__builtin_neon_vmaxnmvq_f64:
2024   case AArch64::BI__builtin_neon_vpmaxnmqd_f64:
2025     Int = Intrinsic::aarch64_neon_vpfmaxnm;
2026     s = "vpfmaxnm"; IntTypes = ScalarRet | VectorCastArg0; break;
2027   // Scalar Reduce Pairwise Floating Point Minnm
2028   case AArch64::BI__builtin_neon_vminnmv_f32:
2029   case AArch64::BI__builtin_neon_vpminnms_f32:
2030   case AArch64::BI__builtin_neon_vminnmvq_f64:
2031   case AArch64::BI__builtin_neon_vpminnmqd_f64:
2032     Int = Intrinsic::aarch64_neon_vpfminnm;
2033     s = "vpfminnm"; IntTypes = ScalarRet | VectorCastArg0; break;
2034   // The followings are intrinsics with scalar results generated AcrossVec vectors
2035   case AArch64::BI__builtin_neon_vaddlv_s8:
2036   case AArch64::BI__builtin_neon_vaddlv_s16:
2037   case AArch64::BI__builtin_neon_vaddlv_s32:
2038   case AArch64::BI__builtin_neon_vaddlvq_s8:
2039   case AArch64::BI__builtin_neon_vaddlvq_s16:
2040   case AArch64::BI__builtin_neon_vaddlvq_s32:
2041     Int = Intrinsic::aarch64_neon_saddlv;
2042     s = "saddlv"; IntTypes = VectorRet | VectorCastArg1; break;
2043   case AArch64::BI__builtin_neon_vaddlv_u8:
2044   case AArch64::BI__builtin_neon_vaddlv_u16:
2045   case AArch64::BI__builtin_neon_vaddlv_u32:
2046   case AArch64::BI__builtin_neon_vaddlvq_u8:
2047   case AArch64::BI__builtin_neon_vaddlvq_u16:
2048   case AArch64::BI__builtin_neon_vaddlvq_u32:
2049     Int = Intrinsic::aarch64_neon_uaddlv;
2050     s = "uaddlv"; IntTypes = VectorRet | VectorCastArg1; break;
2051   case AArch64::BI__builtin_neon_vmaxv_s8:
2052   case AArch64::BI__builtin_neon_vmaxv_s16:
2053   case AArch64::BI__builtin_neon_vmaxv_s32:
2054   case AArch64::BI__builtin_neon_vmaxvq_s8:
2055   case AArch64::BI__builtin_neon_vmaxvq_s16:
2056   case AArch64::BI__builtin_neon_vmaxvq_s32:
2057     Int = Intrinsic::aarch64_neon_smaxv;
2058     s = "smaxv"; IntTypes = VectorRet | VectorCastArg1; break;
2059   case AArch64::BI__builtin_neon_vmaxv_u8:
2060   case AArch64::BI__builtin_neon_vmaxv_u16:
2061   case AArch64::BI__builtin_neon_vmaxv_u32:
2062   case AArch64::BI__builtin_neon_vmaxvq_u8:
2063   case AArch64::BI__builtin_neon_vmaxvq_u16:
2064   case AArch64::BI__builtin_neon_vmaxvq_u32:
2065     Int = Intrinsic::aarch64_neon_umaxv;
2066     s = "umaxv"; IntTypes = VectorRet | VectorCastArg1; break;
2067   case AArch64::BI__builtin_neon_vminv_s8:
2068   case AArch64::BI__builtin_neon_vminv_s16:
2069   case AArch64::BI__builtin_neon_vminv_s32:
2070   case AArch64::BI__builtin_neon_vminvq_s8:
2071   case AArch64::BI__builtin_neon_vminvq_s16:
2072   case AArch64::BI__builtin_neon_vminvq_s32:
2073     Int = Intrinsic::aarch64_neon_sminv;
2074     s = "sminv"; IntTypes = VectorRet | VectorCastArg1; break;
2075   case AArch64::BI__builtin_neon_vminv_u8:
2076   case AArch64::BI__builtin_neon_vminv_u16:
2077   case AArch64::BI__builtin_neon_vminv_u32:
2078   case AArch64::BI__builtin_neon_vminvq_u8:
2079   case AArch64::BI__builtin_neon_vminvq_u16:
2080   case AArch64::BI__builtin_neon_vminvq_u32:
2081     Int = Intrinsic::aarch64_neon_uminv;
2082     s = "uminv"; IntTypes = VectorRet | VectorCastArg1; break;
2083   case AArch64::BI__builtin_neon_vaddv_s8:
2084   case AArch64::BI__builtin_neon_vaddv_s16:
2085   case AArch64::BI__builtin_neon_vaddv_s32:
2086   case AArch64::BI__builtin_neon_vaddvq_s8:
2087   case AArch64::BI__builtin_neon_vaddvq_s16:
2088   case AArch64::BI__builtin_neon_vaddvq_s32:
2089   case AArch64::BI__builtin_neon_vaddvq_s64:
2090   case AArch64::BI__builtin_neon_vaddv_u8:
2091   case AArch64::BI__builtin_neon_vaddv_u16:
2092   case AArch64::BI__builtin_neon_vaddv_u32:
2093   case AArch64::BI__builtin_neon_vaddvq_u8:
2094   case AArch64::BI__builtin_neon_vaddvq_u16:
2095   case AArch64::BI__builtin_neon_vaddvq_u32:
2096   case AArch64::BI__builtin_neon_vaddvq_u64:
2097     Int = Intrinsic::aarch64_neon_vaddv;
2098     s = "vaddv"; IntTypes = VectorRet | VectorCastArg1; break;
2099   case AArch64::BI__builtin_neon_vmaxvq_f32:
2100     Int = Intrinsic::aarch64_neon_vmaxv;
2101     s = "vmaxv"; break;
2102   case AArch64::BI__builtin_neon_vminvq_f32:
2103     Int = Intrinsic::aarch64_neon_vminv;
2104     s = "vminv"; break;
2105   case AArch64::BI__builtin_neon_vmaxnmvq_f32:
2106     Int = Intrinsic::aarch64_neon_vmaxnmv;
2107     s = "vmaxnmv"; break;
2108   case AArch64::BI__builtin_neon_vminnmvq_f32:
2109     Int = Intrinsic::aarch64_neon_vminnmv;
2110      s = "vminnmv"; break;
2111   // Scalar Integer Saturating Doubling Multiply Half High
2112   case AArch64::BI__builtin_neon_vqdmulhh_s16:
2113   case AArch64::BI__builtin_neon_vqdmulhs_s32:
2114     Int = Intrinsic::arm_neon_vqdmulh;
2115     s = "vqdmulh"; IntTypes = VectorRet; break;
2116   // Scalar Integer Saturating Rounding Doubling Multiply Half High
2117   case AArch64::BI__builtin_neon_vqrdmulhh_s16:
2118   case AArch64::BI__builtin_neon_vqrdmulhs_s32:
2119     Int = Intrinsic::arm_neon_vqrdmulh;
2120     s = "vqrdmulh"; IntTypes = VectorRet; break;
2121   // Scalar Floating-point Reciprocal Step
2122   case AArch64::BI__builtin_neon_vrecpss_f32:
2123   case AArch64::BI__builtin_neon_vrecpsd_f64:
2124     Int = Intrinsic::aarch64_neon_vrecps;
2125     s = "vrecps"; IntTypes = ScalarRet; break;
2126   // Scalar Floating-point Reciprocal Square Root Step
2127   case AArch64::BI__builtin_neon_vrsqrtss_f32:
2128   case AArch64::BI__builtin_neon_vrsqrtsd_f64:
2129     Int = Intrinsic::aarch64_neon_vrsqrts;
2130     s = "vrsqrts"; IntTypes = ScalarRet; break;
2131   // Scalar Signed Integer Convert To Floating-point
2132   case AArch64::BI__builtin_neon_vcvts_f32_s32:
2133   case AArch64::BI__builtin_neon_vcvtd_f64_s64:
2134     Int = Intrinsic::aarch64_neon_vcvtint2fps;
2135     s = "vcvtf"; IntTypes = ScalarRet | VectorGetArg0; break;
2136   // Scalar Unsigned Integer Convert To Floating-point
2137   case AArch64::BI__builtin_neon_vcvts_f32_u32:
2138   case AArch64::BI__builtin_neon_vcvtd_f64_u64:
2139     Int = Intrinsic::aarch64_neon_vcvtint2fpu;
2140     s = "vcvtf"; IntTypes = ScalarRet | VectorGetArg0; break;
2141   // Scalar Floating-point Converts
2142   case AArch64::BI__builtin_neon_vcvtxd_f32_f64:
2143     Int = Intrinsic::aarch64_neon_fcvtxn;
2144     s = "vcvtxn"; break;
2145   case AArch64::BI__builtin_neon_vcvtas_s32_f32:
2146   case AArch64::BI__builtin_neon_vcvtad_s64_f64:
2147     Int = Intrinsic::aarch64_neon_fcvtas;
2148     s = "vcvtas"; IntTypes = VectorRet | ScalarArg1; break;
2149   case AArch64::BI__builtin_neon_vcvtas_u32_f32:
2150   case AArch64::BI__builtin_neon_vcvtad_u64_f64:
2151     Int = Intrinsic::aarch64_neon_fcvtau;
2152     s = "vcvtau"; IntTypes = VectorRet | ScalarArg1; break;
2153   case AArch64::BI__builtin_neon_vcvtms_s32_f32:
2154   case AArch64::BI__builtin_neon_vcvtmd_s64_f64:
2155     Int = Intrinsic::aarch64_neon_fcvtms;
2156     s = "vcvtms"; IntTypes = VectorRet | ScalarArg1; break;
2157   case AArch64::BI__builtin_neon_vcvtms_u32_f32:
2158   case AArch64::BI__builtin_neon_vcvtmd_u64_f64:
2159     Int = Intrinsic::aarch64_neon_fcvtmu;
2160     s = "vcvtmu"; IntTypes = VectorRet | ScalarArg1; break;
2161   case AArch64::BI__builtin_neon_vcvtns_s32_f32:
2162   case AArch64::BI__builtin_neon_vcvtnd_s64_f64:
2163     Int = Intrinsic::aarch64_neon_fcvtns;
2164     s = "vcvtns"; IntTypes = VectorRet | ScalarArg1; break;
2165   case AArch64::BI__builtin_neon_vcvtns_u32_f32:
2166   case AArch64::BI__builtin_neon_vcvtnd_u64_f64:
2167     Int = Intrinsic::aarch64_neon_fcvtnu;
2168     s = "vcvtnu"; IntTypes = VectorRet | ScalarArg1; break;
2169   case AArch64::BI__builtin_neon_vcvtps_s32_f32:
2170   case AArch64::BI__builtin_neon_vcvtpd_s64_f64:
2171     Int = Intrinsic::aarch64_neon_fcvtps;
2172     s = "vcvtps"; IntTypes = VectorRet | ScalarArg1; break;
2173   case AArch64::BI__builtin_neon_vcvtps_u32_f32:
2174   case AArch64::BI__builtin_neon_vcvtpd_u64_f64:
2175     Int = Intrinsic::aarch64_neon_fcvtpu;
2176     s = "vcvtpu"; IntTypes = VectorRet | ScalarArg1; break;
2177   case AArch64::BI__builtin_neon_vcvts_s32_f32:
2178   case AArch64::BI__builtin_neon_vcvtd_s64_f64:
2179     Int = Intrinsic::aarch64_neon_fcvtzs;
2180     s = "vcvtzs"; IntTypes = VectorRet | ScalarArg1; break;
2181   case AArch64::BI__builtin_neon_vcvts_u32_f32:
2182   case AArch64::BI__builtin_neon_vcvtd_u64_f64:
2183     Int = Intrinsic::aarch64_neon_fcvtzu;
2184     s = "vcvtzu"; IntTypes = VectorRet | ScalarArg1; break;
2185   // Scalar Floating-point Reciprocal Estimate
2186   case AArch64::BI__builtin_neon_vrecpes_f32:
2187   case AArch64::BI__builtin_neon_vrecped_f64:
2188     Int = Intrinsic::aarch64_neon_vrecpe;
2189     s = "vrecpe"; IntTypes = ScalarRet; break;
2190   // Scalar Floating-point Reciprocal Exponent
2191   case AArch64::BI__builtin_neon_vrecpxs_f32:
2192   case AArch64::BI__builtin_neon_vrecpxd_f64:
2193     Int = Intrinsic::aarch64_neon_vrecpx;
2194     s = "vrecpx"; IntTypes = ScalarRet; break;
2195   // Scalar Floating-point Reciprocal Square Root Estimate
2196   case AArch64::BI__builtin_neon_vrsqrtes_f32:
2197   case AArch64::BI__builtin_neon_vrsqrted_f64:
2198     Int = Intrinsic::aarch64_neon_vrsqrte;
2199     s = "vrsqrte"; IntTypes = ScalarRet; break;
2200   // Scalar Compare Equal
2201   case AArch64::BI__builtin_neon_vceqd_s64:
2202   case AArch64::BI__builtin_neon_vceqd_u64:
2203     Int = Intrinsic::aarch64_neon_vceq; s = "vceq";
2204     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2205   // Scalar Compare Equal To Zero
2206   case AArch64::BI__builtin_neon_vceqzd_s64:
2207   case AArch64::BI__builtin_neon_vceqzd_u64:
2208     Int = Intrinsic::aarch64_neon_vceq; s = "vceq";
2209     // Add implicit zero operand.
2210     Ops.push_back(llvm::Constant::getNullValue(Ops[0]->getType()));
2211     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2212   // Scalar Compare Greater Than or Equal
2213   case AArch64::BI__builtin_neon_vcged_s64:
2214     Int = Intrinsic::aarch64_neon_vcge; s = "vcge";
2215     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2216   case AArch64::BI__builtin_neon_vcged_u64:
2217     Int = Intrinsic::aarch64_neon_vchs; s = "vcge";
2218     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2219   // Scalar Compare Greater Than or Equal To Zero
2220   case AArch64::BI__builtin_neon_vcgezd_s64:
2221     Int = Intrinsic::aarch64_neon_vcge; s = "vcge";
2222     // Add implicit zero operand.
2223     Ops.push_back(llvm::Constant::getNullValue(Ops[0]->getType()));
2224     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2225   // Scalar Compare Greater Than
2226   case AArch64::BI__builtin_neon_vcgtd_s64:
2227     Int = Intrinsic::aarch64_neon_vcgt; s = "vcgt";
2228     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2229   case AArch64::BI__builtin_neon_vcgtd_u64:
2230     Int = Intrinsic::aarch64_neon_vchi; s = "vcgt";
2231     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2232   // Scalar Compare Greater Than Zero
2233   case AArch64::BI__builtin_neon_vcgtzd_s64:
2234     Int = Intrinsic::aarch64_neon_vcgt; s = "vcgt";
2235     // Add implicit zero operand.
2236     Ops.push_back(llvm::Constant::getNullValue(Ops[0]->getType()));
2237     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2238   // Scalar Compare Less Than or Equal
2239   case AArch64::BI__builtin_neon_vcled_s64:
2240     Int = Intrinsic::aarch64_neon_vcge; s = "vcge";
2241     std::swap(Ops[0], Ops[1]);
2242     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2243   case AArch64::BI__builtin_neon_vcled_u64:
2244     Int = Intrinsic::aarch64_neon_vchs; s = "vchs";
2245     std::swap(Ops[0], Ops[1]);
2246     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2247   // Scalar Compare Less Than or Equal To Zero
2248   case AArch64::BI__builtin_neon_vclezd_s64:
2249     Int = Intrinsic::aarch64_neon_vclez; s = "vcle";
2250     // Add implicit zero operand.
2251     Ops.push_back(llvm::Constant::getNullValue(Ops[0]->getType()));
2252     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2253   // Scalar Compare Less Than
2254   case AArch64::BI__builtin_neon_vcltd_s64:
2255     Int = Intrinsic::aarch64_neon_vcgt; s = "vcgt";
2256     std::swap(Ops[0], Ops[1]);
2257     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2258   case AArch64::BI__builtin_neon_vcltd_u64:
2259     Int = Intrinsic::aarch64_neon_vchi; s = "vchi";
2260     std::swap(Ops[0], Ops[1]);
2261     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2262   // Scalar Compare Less Than Zero
2263   case AArch64::BI__builtin_neon_vcltzd_s64:
2264     Int = Intrinsic::aarch64_neon_vcltz; s = "vclt";
2265     // Add implicit zero operand.
2266     Ops.push_back(llvm::Constant::getNullValue(Ops[0]->getType()));
2267     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2268   // Scalar Floating-point Compare Equal
2269   case AArch64::BI__builtin_neon_vceqs_f32:
2270   case AArch64::BI__builtin_neon_vceqd_f64:
2271     Int = Intrinsic::aarch64_neon_fceq; s = "vceq";
2272     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2273   // Scalar Floating-point Compare Equal To Zero
2274   case AArch64::BI__builtin_neon_vceqzs_f32:
2275   case AArch64::BI__builtin_neon_vceqzd_f64:
2276     Int = Intrinsic::aarch64_neon_fceq; s = "vceq";
2277     // Add implicit zero operand.
2278     Ops.push_back(llvm::Constant::getNullValue(CGF.FloatTy));
2279     IntTypes = VectorRet | ScalarArg0 | ScalarFpCmpzArg1; break;
2280   // Scalar Floating-point Compare Greater Than Or Equal
2281   case AArch64::BI__builtin_neon_vcges_f32:
2282   case AArch64::BI__builtin_neon_vcged_f64:
2283     Int = Intrinsic::aarch64_neon_fcge; s = "vcge";
2284     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2285   // Scalar Floating-point Compare Greater Than Or Equal To Zero
2286   case AArch64::BI__builtin_neon_vcgezs_f32:
2287   case AArch64::BI__builtin_neon_vcgezd_f64:
2288     Int = Intrinsic::aarch64_neon_fcge; s = "vcge";
2289     // Add implicit zero operand.
2290     Ops.push_back(llvm::Constant::getNullValue(CGF.FloatTy));
2291     IntTypes = VectorRet | ScalarArg0 | ScalarFpCmpzArg1; break;
2292   // Scalar Floating-point Compare Greather Than
2293   case AArch64::BI__builtin_neon_vcgts_f32:
2294   case AArch64::BI__builtin_neon_vcgtd_f64:
2295     Int = Intrinsic::aarch64_neon_fcgt; s = "vcgt";
2296     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2297   // Scalar Floating-point Compare Greather Than Zero
2298   case AArch64::BI__builtin_neon_vcgtzs_f32:
2299   case AArch64::BI__builtin_neon_vcgtzd_f64:
2300     Int = Intrinsic::aarch64_neon_fcgt; s = "vcgt";
2301     // Add implicit zero operand.
2302     Ops.push_back(llvm::Constant::getNullValue(CGF.FloatTy));
2303     IntTypes = VectorRet | ScalarArg0 | ScalarFpCmpzArg1; break;
2304   // Scalar Floating-point Compare Less Than or Equal
2305   case AArch64::BI__builtin_neon_vcles_f32:
2306   case AArch64::BI__builtin_neon_vcled_f64:
2307     Int = Intrinsic::aarch64_neon_fcge; s = "vcge";
2308     std::swap(Ops[0], Ops[1]);
2309     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2310   // Scalar Floating-point Compare Less Than Or Equal To Zero
2311   case AArch64::BI__builtin_neon_vclezs_f32:
2312   case AArch64::BI__builtin_neon_vclezd_f64:
2313     Int = Intrinsic::aarch64_neon_fclez; s = "vcle";
2314     // Add implicit zero operand.
2315     Ops.push_back(llvm::Constant::getNullValue(CGF.FloatTy));
2316     IntTypes = VectorRet | ScalarArg0 | ScalarFpCmpzArg1; break;
2317   // Scalar Floating-point Compare Less Than Zero
2318   case AArch64::BI__builtin_neon_vclts_f32:
2319   case AArch64::BI__builtin_neon_vcltd_f64:
2320     Int = Intrinsic::aarch64_neon_fcgt; s = "vcgt";
2321     std::swap(Ops[0], Ops[1]);
2322     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2323   // Scalar Floating-point Compare Less Than Zero
2324   case AArch64::BI__builtin_neon_vcltzs_f32:
2325   case AArch64::BI__builtin_neon_vcltzd_f64:
2326     Int = Intrinsic::aarch64_neon_fcltz; s = "vclt";
2327     // Add implicit zero operand.
2328     Ops.push_back(llvm::Constant::getNullValue(CGF.FloatTy));
2329     IntTypes = VectorRet | ScalarArg0 | ScalarFpCmpzArg1; break;
2330   // Scalar Floating-point Absolute Compare Greater Than Or Equal
2331   case AArch64::BI__builtin_neon_vcages_f32:
2332   case AArch64::BI__builtin_neon_vcaged_f64:
2333     Int = Intrinsic::aarch64_neon_fcage; s = "vcage";
2334     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2335   // Scalar Floating-point Absolute Compare Greater Than
2336   case AArch64::BI__builtin_neon_vcagts_f32:
2337   case AArch64::BI__builtin_neon_vcagtd_f64:
2338     Int = Intrinsic::aarch64_neon_fcagt; s = "vcagt";
2339     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2340   // Scalar Floating-point Absolute Compare Less Than Or Equal
2341   case AArch64::BI__builtin_neon_vcales_f32:
2342   case AArch64::BI__builtin_neon_vcaled_f64:
2343     Int = Intrinsic::aarch64_neon_fcage; s = "vcage";
2344     std::swap(Ops[0], Ops[1]);
2345     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2346   // Scalar Floating-point Absolute Compare Less Than
2347   case AArch64::BI__builtin_neon_vcalts_f32:
2348   case AArch64::BI__builtin_neon_vcaltd_f64:
2349     Int = Intrinsic::aarch64_neon_fcagt; s = "vcalt";
2350     std::swap(Ops[0], Ops[1]);
2351     IntTypes = VectorRet | ScalarArg0 | ScalarArg1; break;
2352   // Scalar Compare Bitwise Test Bits
2353   case AArch64::BI__builtin_neon_vtstd_s64:
2354   case AArch64::BI__builtin_neon_vtstd_u64:
2355     Int = Intrinsic::aarch64_neon_vtstd; s = "vtst";
2356     IntTypes = VectorRet | VectorGetArg0 | VectorGetArg1; break;
2357   // Scalar Absolute Value
2358   case AArch64::BI__builtin_neon_vabsd_s64:
2359     Int = Intrinsic::aarch64_neon_vabs;
2360     s = "vabs"; break;
2361   // Scalar Absolute Difference
2362   case AArch64::BI__builtin_neon_vabds_f32:
2363   case AArch64::BI__builtin_neon_vabdd_f64:
2364     Int = Intrinsic::aarch64_neon_vabd;
2365     s = "vabd"; IntTypes = ScalarRet; break;
2366   // Scalar Signed Saturating Absolute Value
2367   case AArch64::BI__builtin_neon_vqabsb_s8:
2368   case AArch64::BI__builtin_neon_vqabsh_s16:
2369   case AArch64::BI__builtin_neon_vqabss_s32:
2370   case AArch64::BI__builtin_neon_vqabsd_s64:
2371     Int = Intrinsic::arm_neon_vqabs;
2372     s = "vqabs"; IntTypes = VectorRet; break;
2373   // Scalar Negate
2374   case AArch64::BI__builtin_neon_vnegd_s64:
2375     Int = Intrinsic::aarch64_neon_vneg;
2376     s = "vneg"; break;
2377   // Scalar Signed Saturating Negate
2378   case AArch64::BI__builtin_neon_vqnegb_s8:
2379   case AArch64::BI__builtin_neon_vqnegh_s16:
2380   case AArch64::BI__builtin_neon_vqnegs_s32:
2381   case AArch64::BI__builtin_neon_vqnegd_s64:
2382     Int = Intrinsic::arm_neon_vqneg;
2383     s = "vqneg"; IntTypes = VectorRet; break;
2384   // Scalar Signed Saturating Accumulated of Unsigned Value
2385   case AArch64::BI__builtin_neon_vuqaddb_s8:
2386   case AArch64::BI__builtin_neon_vuqaddh_s16:
2387   case AArch64::BI__builtin_neon_vuqadds_s32:
2388   case AArch64::BI__builtin_neon_vuqaddd_s64:
2389     Int = Intrinsic::aarch64_neon_vuqadd;
2390     s = "vuqadd"; IntTypes = VectorRet; break;
2391   // Scalar Unsigned Saturating Accumulated of Signed Value
2392   case AArch64::BI__builtin_neon_vsqaddb_u8:
2393   case AArch64::BI__builtin_neon_vsqaddh_u16:
2394   case AArch64::BI__builtin_neon_vsqadds_u32:
2395   case AArch64::BI__builtin_neon_vsqaddd_u64:
2396     Int = Intrinsic::aarch64_neon_vsqadd;
2397     s = "vsqadd"; IntTypes = VectorRet; break;
2398   // Signed Saturating Doubling Multiply-Add Long
2399   case AArch64::BI__builtin_neon_vqdmlalh_s16:
2400   case AArch64::BI__builtin_neon_vqdmlals_s32:
2401     Int = Intrinsic::aarch64_neon_vqdmlal;
2402     s = "vqdmlal"; IntTypes = VectorRet; break;
2403   // Signed Saturating Doubling Multiply-Subtract Long
2404   case AArch64::BI__builtin_neon_vqdmlslh_s16:
2405   case AArch64::BI__builtin_neon_vqdmlsls_s32:
2406     Int = Intrinsic::aarch64_neon_vqdmlsl;
2407     s = "vqdmlsl"; IntTypes = VectorRet; break;
2408   // Signed Saturating Doubling Multiply Long
2409   case AArch64::BI__builtin_neon_vqdmullh_s16:
2410   case AArch64::BI__builtin_neon_vqdmulls_s32:
2411     Int = Intrinsic::arm_neon_vqdmull;
2412     s = "vqdmull"; IntTypes = VectorRet; break;
2413   // Scalar Signed Saturating Extract Unsigned Narrow
2414   case AArch64::BI__builtin_neon_vqmovunh_s16:
2415   case AArch64::BI__builtin_neon_vqmovuns_s32:
2416   case AArch64::BI__builtin_neon_vqmovund_s64:
2417     Int = Intrinsic::arm_neon_vqmovnsu;
2418     s = "vqmovun"; IntTypes = VectorRet; break;
2419   // Scalar Signed Saturating Extract Narrow
2420   case AArch64::BI__builtin_neon_vqmovnh_s16:
2421   case AArch64::BI__builtin_neon_vqmovns_s32:
2422   case AArch64::BI__builtin_neon_vqmovnd_s64:
2423     Int = Intrinsic::arm_neon_vqmovns;
2424     s = "vqmovn"; IntTypes = VectorRet; break;
2425   // Scalar Unsigned Saturating Extract Narrow
2426   case AArch64::BI__builtin_neon_vqmovnh_u16:
2427   case AArch64::BI__builtin_neon_vqmovns_u32:
2428   case AArch64::BI__builtin_neon_vqmovnd_u64:
2429     Int = Intrinsic::arm_neon_vqmovnu;
2430     s = "vqmovn"; IntTypes = VectorRet; break;
2431   // Scalar Signed Shift Right (Immediate)
2432   case AArch64::BI__builtin_neon_vshrd_n_s64:
2433     Int = Intrinsic::aarch64_neon_vshrds_n;
2434     s = "vsshr"; break;
2435   // Scalar Unsigned Shift Right (Immediate)
2436   case AArch64::BI__builtin_neon_vshrd_n_u64:
2437     Int = Intrinsic::aarch64_neon_vshrdu_n;
2438     s = "vushr"; break;
2439   // Scalar Signed Rounding Shift Right (Immediate)
2440   case AArch64::BI__builtin_neon_vrshrd_n_s64:
2441     Int = Intrinsic::aarch64_neon_vsrshr;
2442     s = "vsrshr"; IntTypes = VectorRet; break;
2443   // Scalar Unsigned Rounding Shift Right (Immediate)
2444   case AArch64::BI__builtin_neon_vrshrd_n_u64:
2445     Int = Intrinsic::aarch64_neon_vurshr;
2446     s = "vurshr"; IntTypes = VectorRet; break;
2447   // Scalar Signed Shift Right and Accumulate (Immediate)
2448   case AArch64::BI__builtin_neon_vsrad_n_s64:
2449     Int = Intrinsic::aarch64_neon_vsrads_n;
2450     s = "vssra"; break;
2451   // Scalar Unsigned Shift Right and Accumulate (Immediate)
2452   case AArch64::BI__builtin_neon_vsrad_n_u64:
2453     Int = Intrinsic::aarch64_neon_vsradu_n;
2454     s = "vusra"; break;
2455   // Scalar Signed Rounding Shift Right and Accumulate (Immediate)
2456   case AArch64::BI__builtin_neon_vrsrad_n_s64:
2457     Int = Intrinsic::aarch64_neon_vrsrads_n;
2458     s = "vsrsra"; break;
2459   // Scalar Unsigned Rounding Shift Right and Accumulate (Immediate)
2460   case AArch64::BI__builtin_neon_vrsrad_n_u64:
2461     Int = Intrinsic::aarch64_neon_vrsradu_n;
2462     s = "vursra"; break;
2463   // Scalar Signed/Unsigned Shift Left (Immediate)
2464   case AArch64::BI__builtin_neon_vshld_n_s64:
2465   case AArch64::BI__builtin_neon_vshld_n_u64:
2466     Int = Intrinsic::aarch64_neon_vshld_n;
2467     s = "vshl"; break;
2468   // Signed Saturating Shift Left (Immediate)
2469   case AArch64::BI__builtin_neon_vqshlb_n_s8:
2470   case AArch64::BI__builtin_neon_vqshlh_n_s16:
2471   case AArch64::BI__builtin_neon_vqshls_n_s32:
2472   case AArch64::BI__builtin_neon_vqshld_n_s64:
2473     Int = Intrinsic::aarch64_neon_vqshls_n;
2474     s = "vsqshl"; IntTypes = VectorRet; break;
2475   // Unsigned Saturating Shift Left (Immediate)
2476   case AArch64::BI__builtin_neon_vqshlb_n_u8:
2477   case AArch64::BI__builtin_neon_vqshlh_n_u16:
2478   case AArch64::BI__builtin_neon_vqshls_n_u32:
2479   case AArch64::BI__builtin_neon_vqshld_n_u64:
2480     Int = Intrinsic::aarch64_neon_vqshlu_n;
2481     s = "vuqshl"; IntTypes = VectorRet; break;
2482   // Signed Saturating Shift Left Unsigned (Immediate)
2483   case AArch64::BI__builtin_neon_vqshlub_n_s8:
2484   case AArch64::BI__builtin_neon_vqshluh_n_s16:
2485   case AArch64::BI__builtin_neon_vqshlus_n_s32:
2486   case AArch64::BI__builtin_neon_vqshlud_n_s64:
2487     Int = Intrinsic::aarch64_neon_vsqshlu;
2488     s = "vsqshlu"; IntTypes = VectorRet; break;
2489   // Shift Right And Insert (Immediate)
2490   case AArch64::BI__builtin_neon_vsrid_n_s64:
2491   case AArch64::BI__builtin_neon_vsrid_n_u64:
2492     Int = Intrinsic::aarch64_neon_vsri;
2493     s = "vsri"; IntTypes = VectorRet; break;
2494   // Shift Left And Insert (Immediate)
2495   case AArch64::BI__builtin_neon_vslid_n_s64:
2496   case AArch64::BI__builtin_neon_vslid_n_u64:
2497     Int = Intrinsic::aarch64_neon_vsli;
2498     s = "vsli"; IntTypes = VectorRet; break;
2499   // Signed Saturating Shift Right Narrow (Immediate)
2500   case AArch64::BI__builtin_neon_vqshrnh_n_s16:
2501   case AArch64::BI__builtin_neon_vqshrns_n_s32:
2502   case AArch64::BI__builtin_neon_vqshrnd_n_s64:
2503     Int = Intrinsic::aarch64_neon_vsqshrn;
2504     s = "vsqshrn"; IntTypes = VectorRet; break;
2505   // Unsigned Saturating Shift Right Narrow (Immediate)
2506   case AArch64::BI__builtin_neon_vqshrnh_n_u16:
2507   case AArch64::BI__builtin_neon_vqshrns_n_u32:
2508   case AArch64::BI__builtin_neon_vqshrnd_n_u64:
2509     Int = Intrinsic::aarch64_neon_vuqshrn;
2510     s = "vuqshrn"; IntTypes = VectorRet; break;
2511   // Signed Saturating Rounded Shift Right Narrow (Immediate)
2512   case AArch64::BI__builtin_neon_vqrshrnh_n_s16:
2513   case AArch64::BI__builtin_neon_vqrshrns_n_s32:
2514   case AArch64::BI__builtin_neon_vqrshrnd_n_s64:
2515     Int = Intrinsic::aarch64_neon_vsqrshrn;
2516     s = "vsqrshrn"; IntTypes = VectorRet; break;
2517   // Unsigned Saturating Rounded Shift Right Narrow (Immediate)
2518   case AArch64::BI__builtin_neon_vqrshrnh_n_u16:
2519   case AArch64::BI__builtin_neon_vqrshrns_n_u32:
2520   case AArch64::BI__builtin_neon_vqrshrnd_n_u64:
2521     Int = Intrinsic::aarch64_neon_vuqrshrn;
2522     s = "vuqrshrn"; IntTypes = VectorRet; break;
2523   // Signed Saturating Shift Right Unsigned Narrow (Immediate)
2524   case AArch64::BI__builtin_neon_vqshrunh_n_s16:
2525   case AArch64::BI__builtin_neon_vqshruns_n_s32:
2526   case AArch64::BI__builtin_neon_vqshrund_n_s64:
2527     Int = Intrinsic::aarch64_neon_vsqshrun;
2528     s = "vsqshrun"; IntTypes = VectorRet; break;
2529   // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate)
2530   case AArch64::BI__builtin_neon_vqrshrunh_n_s16:
2531   case AArch64::BI__builtin_neon_vqrshruns_n_s32:
2532   case AArch64::BI__builtin_neon_vqrshrund_n_s64:
2533     Int = Intrinsic::aarch64_neon_vsqrshrun;
2534     s = "vsqrshrun"; IntTypes = VectorRet; break;
2535   // Scalar Signed Fixed-point Convert To Floating-Point (Immediate)
2536   case AArch64::BI__builtin_neon_vcvts_n_f32_s32:
2537   case AArch64::BI__builtin_neon_vcvtd_n_f64_s64:
2538     Int = Intrinsic::aarch64_neon_vcvtfxs2fp_n;
2539     s = "vcvtf"; IntTypes = ScalarRet | VectorGetArg0; break;
2540   // Scalar Unsigned Fixed-point Convert To Floating-Point (Immediate)
2541   case AArch64::BI__builtin_neon_vcvts_n_f32_u32:
2542   case AArch64::BI__builtin_neon_vcvtd_n_f64_u64:
2543     Int = Intrinsic::aarch64_neon_vcvtfxu2fp_n;
2544     s = "vcvtf"; IntTypes = ScalarRet | VectorGetArg0; break;
2545   // Scalar Floating-point Convert To Signed Fixed-point (Immediate)
2546   case AArch64::BI__builtin_neon_vcvts_n_s32_f32:
2547   case AArch64::BI__builtin_neon_vcvtd_n_s64_f64:
2548     Int = Intrinsic::aarch64_neon_vcvtfp2fxs_n;
2549     s = "fcvtzs"; IntTypes = VectorRet | ScalarArg0; break;
2550   // Scalar Floating-point Convert To Unsigned Fixed-point (Immediate)
2551   case AArch64::BI__builtin_neon_vcvts_n_u32_f32:
2552   case AArch64::BI__builtin_neon_vcvtd_n_u64_f64:
2553     Int = Intrinsic::aarch64_neon_vcvtfp2fxu_n;
2554     s = "fcvtzu"; IntTypes = VectorRet | ScalarArg0; break;
2555   case AArch64::BI__builtin_neon_vmull_p64:
2556     Int = Intrinsic::aarch64_neon_vmull_p64;
2557     s = "vmull"; break;
2558   }
2559 
2560   if (!Int)
2561     return 0;
2562 
2563   // Determine the type(s) of this overloaded AArch64 intrinsic.
2564   Function *F = 0;
2565   SmallVector<llvm::Type *, 3> Tys;
2566 
2567   // Return type.
2568   if (IntTypes & (ScalarRet | VectorRet)) {
2569      llvm::Type *Ty = CGF.ConvertType(E->getCallReturnType());
2570     if (IntTypes & ScalarRet) {
2571       // Scalar return value.
2572       Tys.push_back(Ty);
2573     } else if (IntTypes & VectorRet) {
2574       // Convert the scalar return type to one-vector element type.
2575       Tys.push_back(llvm::VectorType::get(Ty, 1));
2576     }
2577   }
2578 
2579   // Arguments.
2580   if (IntTypes & (ScalarArg0 | VectorGetArg0 | VectorCastArg0)) {
2581     const Expr *Arg = E->getArg(0);
2582     llvm::Type *Ty = CGF.ConvertType(Arg->getType());
2583     if (IntTypes & ScalarArg0) {
2584       // Scalar argument.
2585       Tys.push_back(Ty);
2586     } else if (IntTypes & VectorGetArg0) {
2587       // Convert the scalar argument to one-vector element type.
2588       Tys.push_back(llvm::VectorType::get(Ty, 1));
2589     } else if (IntTypes & VectorCastArg0) {
2590       // Cast the argument to vector type.
2591       Tys.push_back(cast<llvm::VectorType>(Ty));
2592     }
2593   }
2594 
2595   // The only intrinsics that require a 2nd argument are the compare intrinsics.
2596   // However, the builtins don't always have a 2nd argument (e.g.,
2597   // floating-point compare to zero), so we inspect the first argument to
2598   // determine the type.
2599   if (IntTypes & (ScalarArg1 | VectorGetArg1 | VectorCastArg1)) {
2600     const Expr *Arg = E->getArg(0);
2601     llvm::Type *Ty = CGF.ConvertType(Arg->getType());
2602     if (IntTypes & ScalarArg1) {
2603       // Scalar argument.
2604       Tys.push_back(Ty);
2605     } else if (IntTypes & VectorGetArg1) {
2606       // Convert the scalar argument to one-vector element type.
2607       Tys.push_back(llvm::VectorType::get(Ty, 1));
2608     } else if (IntTypes & VectorCastArg1) {
2609       // Cast the argument to a vector type.
2610       Tys.push_back(cast<llvm::VectorType>(Ty));
2611     }
2612   } else if (IntTypes & ScalarFpCmpzArg1) {
2613     // Floating-point zero argument.
2614     Tys.push_back(CGF.FloatTy);
2615   }
2616 
2617   if (IntTypes)
2618      F = CGF.CGM.getIntrinsic(Int, Tys);
2619   else
2620      F = CGF.CGM.getIntrinsic(Int);
2621 
2622   Value *Result = CGF.EmitNeonCall(F, Ops, s);
2623   llvm::Type *ResultType = CGF.ConvertType(E->getType());
2624   // AArch64 intrinsic one-element vector type cast to
2625   // scalar type expected by the builtin
2626   return CGF.Builder.CreateBitCast(Result, ResultType, s);
2627 }
2628 
2629 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
2630     Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
2631     const CmpInst::Predicate Ip, const Twine &Name) {
2632   llvm::Type *OTy = ((llvm::User *)Op)->getOperand(0)->getType();
2633   if (OTy->isPointerTy())
2634     OTy = Ty;
2635   Op = Builder.CreateBitCast(Op, OTy);
2636   if (((llvm::VectorType *)OTy)->getElementType()->isFloatingPointTy()) {
2637     Op = Builder.CreateFCmp(Fp, Op, ConstantAggregateZero::get(OTy));
2638   } else {
2639     Op = Builder.CreateICmp(Ip, Op, ConstantAggregateZero::get(OTy));
2640   }
2641   return Builder.CreateSExt(Op, Ty, Name);
2642 }
2643 
2644 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
2645                                  Value *ExtOp, Value *IndexOp,
2646                                  llvm::Type *ResTy, unsigned IntID,
2647                                  const char *Name) {
2648   SmallVector<Value *, 2> TblOps;
2649   if (ExtOp)
2650     TblOps.push_back(ExtOp);
2651 
2652   // Build a vector containing sequential number like (0, 1, 2, ..., 15)
2653   SmallVector<Constant*, 16> Indices;
2654   llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
2655   for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
2656     Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i));
2657     Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1));
2658   }
2659   Value *SV = llvm::ConstantVector::get(Indices);
2660 
2661   int PairPos = 0, End = Ops.size() - 1;
2662   while (PairPos < End) {
2663     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
2664                                                      Ops[PairPos+1], SV, Name));
2665     PairPos += 2;
2666   }
2667 
2668   // If there's an odd number of 64-bit lookup table, fill the high 64-bit
2669   // of the 128-bit lookup table with zero.
2670   if (PairPos == End) {
2671     Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
2672     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
2673                                                      ZeroTbl, SV, Name));
2674   }
2675 
2676   TblTy = llvm::VectorType::get(TblTy->getElementType(),
2677                                 2*TblTy->getNumElements());
2678   llvm::Type *Tys[2] = { ResTy, TblTy };
2679 
2680   Function *TblF;
2681   TblOps.push_back(IndexOp);
2682   TblF = CGF.CGM.getIntrinsic(IntID, Tys);
2683 
2684   return CGF.EmitNeonCall(TblF, TblOps, Name);
2685 }
2686 
2687 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF,
2688                                         unsigned BuiltinID,
2689                                         const CallExpr *E) {
2690   unsigned int Int = 0;
2691   const char *s = NULL;
2692 
2693   unsigned TblPos;
2694   switch (BuiltinID) {
2695   default:
2696     return 0;
2697   case AArch64::BI__builtin_neon_vtbl1_v:
2698   case AArch64::BI__builtin_neon_vqtbl1_v:
2699   case AArch64::BI__builtin_neon_vqtbl1q_v:
2700   case AArch64::BI__builtin_neon_vtbl2_v:
2701   case AArch64::BI__builtin_neon_vqtbl2_v:
2702   case AArch64::BI__builtin_neon_vqtbl2q_v:
2703   case AArch64::BI__builtin_neon_vtbl3_v:
2704   case AArch64::BI__builtin_neon_vqtbl3_v:
2705   case AArch64::BI__builtin_neon_vqtbl3q_v:
2706   case AArch64::BI__builtin_neon_vtbl4_v:
2707   case AArch64::BI__builtin_neon_vqtbl4_v:
2708   case AArch64::BI__builtin_neon_vqtbl4q_v:
2709     TblPos = 0;
2710     break;
2711   case AArch64::BI__builtin_neon_vtbx1_v:
2712   case AArch64::BI__builtin_neon_vqtbx1_v:
2713   case AArch64::BI__builtin_neon_vqtbx1q_v:
2714   case AArch64::BI__builtin_neon_vtbx2_v:
2715   case AArch64::BI__builtin_neon_vqtbx2_v:
2716   case AArch64::BI__builtin_neon_vqtbx2q_v:
2717   case AArch64::BI__builtin_neon_vtbx3_v:
2718   case AArch64::BI__builtin_neon_vqtbx3_v:
2719   case AArch64::BI__builtin_neon_vqtbx3q_v:
2720   case AArch64::BI__builtin_neon_vtbx4_v:
2721   case AArch64::BI__builtin_neon_vqtbx4_v:
2722   case AArch64::BI__builtin_neon_vqtbx4q_v:
2723     TblPos = 1;
2724     break;
2725   }
2726 
2727   assert(E->getNumArgs() >= 3);
2728 
2729   // Get the last argument, which specifies the vector type.
2730   llvm::APSInt Result;
2731   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
2732   if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
2733     return 0;
2734 
2735   // Determine the type of this overloaded NEON intrinsic.
2736   NeonTypeFlags Type(Result.getZExtValue());
2737   llvm::VectorType *VTy = GetNeonType(&CGF, Type);
2738   llvm::Type *Ty = VTy;
2739   if (!Ty)
2740     return 0;
2741 
2742   SmallVector<Value *, 4> Ops;
2743   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
2744     Ops.push_back(CGF.EmitScalarExpr(E->getArg(i)));
2745   }
2746 
2747   Arg = E->getArg(TblPos);
2748   llvm::Type *TblTy = CGF.ConvertType(Arg->getType());
2749   llvm::VectorType *VTblTy = cast<llvm::VectorType>(TblTy);
2750   llvm::Type *Tys[2] = { Ty, VTblTy };
2751   unsigned nElts = VTy->getNumElements();
2752 
2753   // AArch64 scalar builtins are not overloaded, they do not have an extra
2754   // argument that specifies the vector type, need to handle each case.
2755   SmallVector<Value *, 2> TblOps;
2756   switch (BuiltinID) {
2757   case AArch64::BI__builtin_neon_vtbl1_v: {
2758     TblOps.push_back(Ops[0]);
2759     return packTBLDVectorList(CGF, TblOps, 0, Ops[1], Ty,
2760                               Intrinsic::aarch64_neon_vtbl1, "vtbl1");
2761   }
2762   case AArch64::BI__builtin_neon_vtbl2_v: {
2763     TblOps.push_back(Ops[0]);
2764     TblOps.push_back(Ops[1]);
2765     return packTBLDVectorList(CGF, TblOps, 0, Ops[2], Ty,
2766                               Intrinsic::aarch64_neon_vtbl1, "vtbl1");
2767   }
2768   case AArch64::BI__builtin_neon_vtbl3_v: {
2769     TblOps.push_back(Ops[0]);
2770     TblOps.push_back(Ops[1]);
2771     TblOps.push_back(Ops[2]);
2772     return packTBLDVectorList(CGF, TblOps, 0, Ops[3], Ty,
2773                               Intrinsic::aarch64_neon_vtbl2, "vtbl2");
2774   }
2775   case AArch64::BI__builtin_neon_vtbl4_v: {
2776     TblOps.push_back(Ops[0]);
2777     TblOps.push_back(Ops[1]);
2778     TblOps.push_back(Ops[2]);
2779     TblOps.push_back(Ops[3]);
2780     return packTBLDVectorList(CGF, TblOps, 0, Ops[4], Ty,
2781                               Intrinsic::aarch64_neon_vtbl2, "vtbl2");
2782   }
2783   case AArch64::BI__builtin_neon_vtbx1_v: {
2784     TblOps.push_back(Ops[1]);
2785     Value *TblRes = packTBLDVectorList(CGF, TblOps, 0, Ops[2], Ty,
2786                                     Intrinsic::aarch64_neon_vtbl1, "vtbl1");
2787 
2788     llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8);
2789     Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight);
2790     Value *CmpRes = CGF.Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
2791     CmpRes = CGF.Builder.CreateSExt(CmpRes, Ty);
2792 
2793     SmallVector<Value *, 4> BslOps;
2794     BslOps.push_back(CmpRes);
2795     BslOps.push_back(Ops[0]);
2796     BslOps.push_back(TblRes);
2797     Function *BslF = CGF.CGM.getIntrinsic(Intrinsic::arm_neon_vbsl, Ty);
2798     return CGF.EmitNeonCall(BslF, BslOps, "vbsl");
2799   }
2800   case AArch64::BI__builtin_neon_vtbx2_v: {
2801     TblOps.push_back(Ops[1]);
2802     TblOps.push_back(Ops[2]);
2803     return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty,
2804                               Intrinsic::aarch64_neon_vtbx1, "vtbx1");
2805   }
2806   case AArch64::BI__builtin_neon_vtbx3_v: {
2807     TblOps.push_back(Ops[1]);
2808     TblOps.push_back(Ops[2]);
2809     TblOps.push_back(Ops[3]);
2810     Value *TblRes = packTBLDVectorList(CGF, TblOps, 0, Ops[4], Ty,
2811                                        Intrinsic::aarch64_neon_vtbl2, "vtbl2");
2812 
2813     llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24);
2814     Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour);
2815     Value *CmpRes = CGF.Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
2816                                            TwentyFourV);
2817     CmpRes = CGF.Builder.CreateSExt(CmpRes, Ty);
2818 
2819     SmallVector<Value *, 4> BslOps;
2820     BslOps.push_back(CmpRes);
2821     BslOps.push_back(Ops[0]);
2822     BslOps.push_back(TblRes);
2823     Function *BslF = CGF.CGM.getIntrinsic(Intrinsic::arm_neon_vbsl, Ty);
2824     return CGF.EmitNeonCall(BslF, BslOps, "vbsl");
2825   }
2826   case AArch64::BI__builtin_neon_vtbx4_v: {
2827     TblOps.push_back(Ops[1]);
2828     TblOps.push_back(Ops[2]);
2829     TblOps.push_back(Ops[3]);
2830     TblOps.push_back(Ops[4]);
2831     return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty,
2832                               Intrinsic::aarch64_neon_vtbx2, "vtbx2");
2833   }
2834   case AArch64::BI__builtin_neon_vqtbl1_v:
2835   case AArch64::BI__builtin_neon_vqtbl1q_v:
2836     Int = Intrinsic::aarch64_neon_vtbl1; s = "vtbl1"; break;
2837   case AArch64::BI__builtin_neon_vqtbl2_v:
2838   case AArch64::BI__builtin_neon_vqtbl2q_v: {
2839     Int = Intrinsic::aarch64_neon_vtbl2; s = "vtbl2"; break;
2840   case AArch64::BI__builtin_neon_vqtbl3_v:
2841   case AArch64::BI__builtin_neon_vqtbl3q_v:
2842     Int = Intrinsic::aarch64_neon_vtbl3; s = "vtbl3"; break;
2843   case AArch64::BI__builtin_neon_vqtbl4_v:
2844   case AArch64::BI__builtin_neon_vqtbl4q_v:
2845     Int = Intrinsic::aarch64_neon_vtbl4; s = "vtbl4"; break;
2846   case AArch64::BI__builtin_neon_vqtbx1_v:
2847   case AArch64::BI__builtin_neon_vqtbx1q_v:
2848     Int = Intrinsic::aarch64_neon_vtbx1; s = "vtbx1"; break;
2849   case AArch64::BI__builtin_neon_vqtbx2_v:
2850   case AArch64::BI__builtin_neon_vqtbx2q_v:
2851     Int = Intrinsic::aarch64_neon_vtbx2; s = "vtbx2"; break;
2852   case AArch64::BI__builtin_neon_vqtbx3_v:
2853   case AArch64::BI__builtin_neon_vqtbx3q_v:
2854     Int = Intrinsic::aarch64_neon_vtbx3; s = "vtbx3"; break;
2855   case AArch64::BI__builtin_neon_vqtbx4_v:
2856   case AArch64::BI__builtin_neon_vqtbx4q_v:
2857     Int = Intrinsic::aarch64_neon_vtbx4; s = "vtbx4"; break;
2858   }
2859   }
2860 
2861   if (!Int)
2862     return 0;
2863 
2864   Function *F = CGF.CGM.getIntrinsic(Int, Tys);
2865   return CGF.EmitNeonCall(F, Ops, s);
2866 }
2867 
2868 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
2869                                                const CallExpr *E) {
2870   // Process AArch64 scalar builtins
2871   if (Value *Result = EmitAArch64ScalarBuiltinExpr(*this, BuiltinID, E))
2872     return Result;
2873 
2874   // Process AArch64 table lookup builtins
2875   if (Value *Result = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E))
2876     return Result;
2877 
2878   if (BuiltinID == AArch64::BI__clear_cache) {
2879     assert(E->getNumArgs() == 2 &&
2880            "Variadic __clear_cache slipped through on AArch64");
2881 
2882     const FunctionDecl *FD = E->getDirectCallee();
2883     SmallVector<Value *, 2> Ops;
2884     for (unsigned i = 0; i < E->getNumArgs(); i++)
2885       Ops.push_back(EmitScalarExpr(E->getArg(i)));
2886     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
2887     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
2888     StringRef Name = FD->getName();
2889     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
2890   }
2891 
2892   SmallVector<Value *, 4> Ops;
2893   llvm::Value *Align = 0; // Alignment for load/store
2894 
2895   if (BuiltinID == AArch64::BI__builtin_neon_vldrq_p128) {
2896    Value *Op = EmitScalarExpr(E->getArg(0));
2897    unsigned addressSpace =
2898      cast<llvm::PointerType>(Op->getType())->getAddressSpace();
2899    llvm::Type *Ty = llvm::Type::getFP128PtrTy(getLLVMContext(), addressSpace);
2900    Op = Builder.CreateBitCast(Op, Ty);
2901    Op = Builder.CreateLoad(Op);
2902    Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
2903    return Builder.CreateBitCast(Op, Ty);
2904   }
2905   if (BuiltinID == AArch64::BI__builtin_neon_vstrq_p128) {
2906     Value *Op0 = EmitScalarExpr(E->getArg(0));
2907     unsigned addressSpace =
2908       cast<llvm::PointerType>(Op0->getType())->getAddressSpace();
2909     llvm::Type *PTy = llvm::Type::getFP128PtrTy(getLLVMContext(), addressSpace);
2910     Op0 = Builder.CreateBitCast(Op0, PTy);
2911     Value *Op1 = EmitScalarExpr(E->getArg(1));
2912     llvm::Type *Ty = llvm::Type::getFP128Ty(getLLVMContext());
2913     Op1 = Builder.CreateBitCast(Op1, Ty);
2914     return Builder.CreateStore(Op1, Op0);
2915   }
2916   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
2917     if (i == 0) {
2918       switch (BuiltinID) {
2919       case AArch64::BI__builtin_neon_vst1_x2_v:
2920       case AArch64::BI__builtin_neon_vst1q_x2_v:
2921       case AArch64::BI__builtin_neon_vst1_x3_v:
2922       case AArch64::BI__builtin_neon_vst1q_x3_v:
2923       case AArch64::BI__builtin_neon_vst1_x4_v:
2924       case AArch64::BI__builtin_neon_vst1q_x4_v:
2925       // Handle ld1/st1 lane in this function a little different from ARM.
2926       case AArch64::BI__builtin_neon_vld1_lane_v:
2927       case AArch64::BI__builtin_neon_vld1q_lane_v:
2928       case AArch64::BI__builtin_neon_vst1_lane_v:
2929       case AArch64::BI__builtin_neon_vst1q_lane_v:
2930         // Get the alignment for the argument in addition to the value;
2931         // we'll use it later.
2932         std::pair<llvm::Value *, unsigned> Src =
2933             EmitPointerWithAlignment(E->getArg(0));
2934         Ops.push_back(Src.first);
2935         Align = Builder.getInt32(Src.second);
2936         continue;
2937       }
2938     }
2939     if (i == 1) {
2940       switch (BuiltinID) {
2941       case AArch64::BI__builtin_neon_vld1_x2_v:
2942       case AArch64::BI__builtin_neon_vld1q_x2_v:
2943       case AArch64::BI__builtin_neon_vld1_x3_v:
2944       case AArch64::BI__builtin_neon_vld1q_x3_v:
2945       case AArch64::BI__builtin_neon_vld1_x4_v:
2946       case AArch64::BI__builtin_neon_vld1q_x4_v:
2947       // Handle ld1/st1 dup lane in this function a little different from ARM.
2948       case AArch64::BI__builtin_neon_vld2_dup_v:
2949       case AArch64::BI__builtin_neon_vld2q_dup_v:
2950       case AArch64::BI__builtin_neon_vld3_dup_v:
2951       case AArch64::BI__builtin_neon_vld3q_dup_v:
2952       case AArch64::BI__builtin_neon_vld4_dup_v:
2953       case AArch64::BI__builtin_neon_vld4q_dup_v:
2954       case AArch64::BI__builtin_neon_vld2_lane_v:
2955       case AArch64::BI__builtin_neon_vld2q_lane_v:
2956         // Get the alignment for the argument in addition to the value;
2957         // we'll use it later.
2958         std::pair<llvm::Value *, unsigned> Src =
2959             EmitPointerWithAlignment(E->getArg(1));
2960         Ops.push_back(Src.first);
2961         Align = Builder.getInt32(Src.second);
2962         continue;
2963       }
2964     }
2965     Ops.push_back(EmitScalarExpr(E->getArg(i)));
2966   }
2967 
2968   // Get the last argument, which specifies the vector type.
2969   llvm::APSInt Result;
2970   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
2971   if (!Arg->isIntegerConstantExpr(Result, getContext()))
2972     return 0;
2973 
2974   // Determine the type of this overloaded NEON intrinsic.
2975   NeonTypeFlags Type(Result.getZExtValue());
2976   bool usgn = Type.isUnsigned();
2977   bool quad = Type.isQuad();
2978 
2979   llvm::VectorType *VTy = GetNeonType(this, Type);
2980   llvm::Type *Ty = VTy;
2981   if (!Ty)
2982     return 0;
2983 
2984   unsigned Int;
2985   switch (BuiltinID) {
2986   default:
2987     return 0;
2988 
2989   // AArch64 builtins mapping to legacy ARM v7 builtins.
2990   // FIXME: the mapped builtins listed correspond to what has been tested
2991   // in aarch64-neon-intrinsics.c so far.
2992   case AArch64::BI__builtin_neon_vuzp_v:
2993     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vuzp_v, E);
2994   case AArch64::BI__builtin_neon_vuzpq_v:
2995     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vuzpq_v, E);
2996   case AArch64::BI__builtin_neon_vzip_v:
2997     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vzip_v, E);
2998   case AArch64::BI__builtin_neon_vzipq_v:
2999     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vzipq_v, E);
3000   case AArch64::BI__builtin_neon_vtrn_v:
3001     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vtrn_v, E);
3002   case AArch64::BI__builtin_neon_vtrnq_v:
3003     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vtrnq_v, E);
3004   case AArch64::BI__builtin_neon_vext_v:
3005     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vext_v, E);
3006   case AArch64::BI__builtin_neon_vextq_v:
3007     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vextq_v, E);
3008   case AArch64::BI__builtin_neon_vmul_v:
3009     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmul_v, E);
3010   case AArch64::BI__builtin_neon_vmulq_v:
3011     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmulq_v, E);
3012   case AArch64::BI__builtin_neon_vabd_v:
3013     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vabd_v, E);
3014   case AArch64::BI__builtin_neon_vabdq_v:
3015     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vabdq_v, E);
3016   case AArch64::BI__builtin_neon_vfma_v:
3017     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vfma_v, E);
3018   case AArch64::BI__builtin_neon_vfmaq_v:
3019     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vfmaq_v, E);
3020   case AArch64::BI__builtin_neon_vbsl_v:
3021     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vbsl_v, E);
3022   case AArch64::BI__builtin_neon_vbslq_v:
3023     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vbslq_v, E);
3024   case AArch64::BI__builtin_neon_vrsqrts_v:
3025     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrsqrts_v, E);
3026   case AArch64::BI__builtin_neon_vrsqrtsq_v:
3027     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrsqrtsq_v, E);
3028   case AArch64::BI__builtin_neon_vrecps_v:
3029     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrecps_v, E);
3030   case AArch64::BI__builtin_neon_vrecpsq_v:
3031     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrecpsq_v, E);
3032   case AArch64::BI__builtin_neon_vcale_v:
3033     if (VTy->getVectorNumElements() == 1) {
3034       std::swap(Ops[0], Ops[1]);
3035     } else {
3036       return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcale_v, E);
3037     }
3038   case AArch64::BI__builtin_neon_vcage_v:
3039     if (VTy->getVectorNumElements() == 1) {
3040       // Determine the types of this overloaded AArch64 intrinsic
3041       SmallVector<llvm::Type *, 3> Tys;
3042       Tys.push_back(VTy);
3043       VTy = llvm::VectorType::get(DoubleTy, 1);
3044       Tys.push_back(VTy);
3045       Tys.push_back(VTy);
3046       Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_vcage, Tys);
3047       return EmitNeonCall(F, Ops, "vcage");
3048     }
3049     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcage_v, E);
3050   case AArch64::BI__builtin_neon_vcaleq_v:
3051     std::swap(Ops[0], Ops[1]);
3052   case AArch64::BI__builtin_neon_vcageq_v: {
3053     Function *F;
3054     if (VTy->getElementType()->isIntegerTy(64))
3055       F = CGM.getIntrinsic(Intrinsic::aarch64_neon_vacgeq);
3056     else
3057       F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgeq);
3058     return EmitNeonCall(F, Ops, "vcage");
3059   }
3060   case AArch64::BI__builtin_neon_vcalt_v:
3061     if (VTy->getVectorNumElements() == 1) {
3062       std::swap(Ops[0], Ops[1]);
3063     } else {
3064       return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcalt_v, E);
3065     }
3066   case AArch64::BI__builtin_neon_vcagt_v:
3067     if (VTy->getVectorNumElements() == 1) {
3068       // Determine the types of this overloaded AArch64 intrinsic
3069       SmallVector<llvm::Type *, 3> Tys;
3070       Tys.push_back(VTy);
3071       VTy = llvm::VectorType::get(DoubleTy, 1);
3072       Tys.push_back(VTy);
3073       Tys.push_back(VTy);
3074       Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_vcagt, Tys);
3075       return EmitNeonCall(F, Ops, "vcagt");
3076     }
3077     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcagt_v, E);
3078   case AArch64::BI__builtin_neon_vcaltq_v:
3079     std::swap(Ops[0], Ops[1]);
3080   case AArch64::BI__builtin_neon_vcagtq_v: {
3081     Function *F;
3082     if (VTy->getElementType()->isIntegerTy(64))
3083       F = CGM.getIntrinsic(Intrinsic::aarch64_neon_vacgtq);
3084     else
3085       F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtq);
3086     return EmitNeonCall(F, Ops, "vcagt");
3087   }
3088   case AArch64::BI__builtin_neon_vtst_v:
3089     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vtst_v, E);
3090   case AArch64::BI__builtin_neon_vtstq_v:
3091     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vtstq_v, E);
3092   case AArch64::BI__builtin_neon_vhadd_v:
3093     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vhadd_v, E);
3094   case AArch64::BI__builtin_neon_vhaddq_v:
3095     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vhaddq_v, E);
3096   case AArch64::BI__builtin_neon_vhsub_v:
3097     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vhsub_v, E);
3098   case AArch64::BI__builtin_neon_vhsubq_v:
3099     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vhsubq_v, E);
3100   case AArch64::BI__builtin_neon_vrhadd_v:
3101     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrhadd_v, E);
3102   case AArch64::BI__builtin_neon_vrhaddq_v:
3103     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrhaddq_v, E);
3104   case AArch64::BI__builtin_neon_vqadd_v:
3105     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqadd_v, E);
3106   case AArch64::BI__builtin_neon_vqaddq_v:
3107     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqaddq_v, E);
3108   case AArch64::BI__builtin_neon_vqsub_v:
3109     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqsub_v, E);
3110   case AArch64::BI__builtin_neon_vqsubq_v:
3111     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqsubq_v, E);
3112   case AArch64::BI__builtin_neon_vshl_v:
3113     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshl_v, E);
3114   case AArch64::BI__builtin_neon_vshlq_v:
3115     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshlq_v, E);
3116   case AArch64::BI__builtin_neon_vqshl_v:
3117     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqshl_v, E);
3118   case AArch64::BI__builtin_neon_vqshlq_v:
3119     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqshlq_v, E);
3120   case AArch64::BI__builtin_neon_vrshl_v:
3121     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrshl_v, E);
3122   case AArch64::BI__builtin_neon_vrshlq_v:
3123     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrshlq_v, E);
3124   case AArch64::BI__builtin_neon_vqrshl_v:
3125     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqrshl_v, E);
3126   case AArch64::BI__builtin_neon_vqrshlq_v:
3127     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqrshlq_v, E);
3128   case AArch64::BI__builtin_neon_vaddhn_v:
3129     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vaddhn_v, E);
3130   case AArch64::BI__builtin_neon_vraddhn_v:
3131     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vraddhn_v, E);
3132   case AArch64::BI__builtin_neon_vsubhn_v:
3133     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vsubhn_v, E);
3134   case AArch64::BI__builtin_neon_vrsubhn_v:
3135     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrsubhn_v, E);
3136   case AArch64::BI__builtin_neon_vmull_v:
3137     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmull_v, E);
3138   case AArch64::BI__builtin_neon_vqdmull_v:
3139     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqdmull_v, E);
3140   case AArch64::BI__builtin_neon_vqdmlal_v:
3141     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqdmlal_v, E);
3142   case AArch64::BI__builtin_neon_vqdmlsl_v:
3143     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqdmlsl_v, E);
3144   case AArch64::BI__builtin_neon_vmax_v:
3145     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmax_v, E);
3146   case AArch64::BI__builtin_neon_vmaxq_v:
3147     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmaxq_v, E);
3148   case AArch64::BI__builtin_neon_vmin_v:
3149     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmin_v, E);
3150   case AArch64::BI__builtin_neon_vminq_v:
3151     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vminq_v, E);
3152   case AArch64::BI__builtin_neon_vpmax_v:
3153     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vpmax_v, E);
3154   case AArch64::BI__builtin_neon_vpmin_v:
3155     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vpmin_v, E);
3156   case AArch64::BI__builtin_neon_vpadd_v:
3157     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vpadd_v, E);
3158   case AArch64::BI__builtin_neon_vqdmulh_v:
3159     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqdmulh_v, E);
3160   case AArch64::BI__builtin_neon_vqdmulhq_v:
3161     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqdmulhq_v, E);
3162   case AArch64::BI__builtin_neon_vqrdmulh_v:
3163     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqrdmulh_v, E);
3164   case AArch64::BI__builtin_neon_vqrdmulhq_v:
3165     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqrdmulhq_v, E);
3166 
3167   // Shift by immediate
3168   case AArch64::BI__builtin_neon_vshr_n_v:
3169     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshr_n_v, E);
3170   case AArch64::BI__builtin_neon_vshrq_n_v:
3171     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshrq_n_v, E);
3172   case AArch64::BI__builtin_neon_vrshr_n_v:
3173   case AArch64::BI__builtin_neon_vrshrq_n_v:
3174     Int = usgn ? Intrinsic::aarch64_neon_vurshr
3175                : Intrinsic::aarch64_neon_vsrshr;
3176     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n");
3177   case AArch64::BI__builtin_neon_vsra_n_v:
3178     if (VTy->getElementType()->isIntegerTy(64)) {
3179       Int = usgn ? Intrinsic::aarch64_neon_vsradu_n
3180                  : Intrinsic::aarch64_neon_vsrads_n;
3181       return EmitNeonCall(CGM.getIntrinsic(Int), Ops, "vsra_n");
3182     }
3183     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vsra_n_v, E);
3184   case AArch64::BI__builtin_neon_vsraq_n_v:
3185     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vsraq_n_v, E);
3186   case AArch64::BI__builtin_neon_vrsra_n_v:
3187     if (VTy->getElementType()->isIntegerTy(64)) {
3188       Int = usgn ? Intrinsic::aarch64_neon_vrsradu_n
3189                  : Intrinsic::aarch64_neon_vrsrads_n;
3190       return EmitNeonCall(CGM.getIntrinsic(Int), Ops, "vrsra_n");
3191     }
3192     // fall through
3193   case AArch64::BI__builtin_neon_vrsraq_n_v: {
3194     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3195     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3196     Int = usgn ? Intrinsic::aarch64_neon_vurshr
3197                : Intrinsic::aarch64_neon_vsrshr;
3198     Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
3199     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
3200   }
3201   case AArch64::BI__builtin_neon_vshl_n_v:
3202     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshl_n_v, E);
3203   case AArch64::BI__builtin_neon_vshlq_n_v:
3204     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vshlq_n_v, E);
3205   case AArch64::BI__builtin_neon_vqshl_n_v:
3206     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqshl_n_v, E);
3207   case AArch64::BI__builtin_neon_vqshlq_n_v:
3208     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqshlq_n_v, E);
3209   case AArch64::BI__builtin_neon_vqshlu_n_v:
3210   case AArch64::BI__builtin_neon_vqshluq_n_v:
3211     Int = Intrinsic::aarch64_neon_vsqshlu;
3212     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n");
3213   case AArch64::BI__builtin_neon_vsri_n_v:
3214   case AArch64::BI__builtin_neon_vsriq_n_v:
3215     Int = Intrinsic::aarch64_neon_vsri;
3216     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsri_n");
3217   case AArch64::BI__builtin_neon_vsli_n_v:
3218   case AArch64::BI__builtin_neon_vsliq_n_v:
3219     Int = Intrinsic::aarch64_neon_vsli;
3220     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsli_n");
3221   case AArch64::BI__builtin_neon_vshll_n_v: {
3222     llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
3223     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3224     if (usgn)
3225       Ops[0] = Builder.CreateZExt(Ops[0], VTy);
3226     else
3227       Ops[0] = Builder.CreateSExt(Ops[0], VTy);
3228     Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
3229     return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
3230   }
3231   case AArch64::BI__builtin_neon_vshrn_n_v: {
3232     llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
3233     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3234     Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
3235     if (usgn)
3236       Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
3237     else
3238       Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
3239     return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
3240   }
3241   case AArch64::BI__builtin_neon_vqshrun_n_v:
3242     Int = Intrinsic::aarch64_neon_vsqshrun;
3243     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
3244   case AArch64::BI__builtin_neon_vrshrn_n_v:
3245     Int = Intrinsic::aarch64_neon_vrshrn;
3246     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
3247   case AArch64::BI__builtin_neon_vqrshrun_n_v:
3248     Int = Intrinsic::aarch64_neon_vsqrshrun;
3249     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
3250   case AArch64::BI__builtin_neon_vqshrn_n_v:
3251     Int = usgn ? Intrinsic::aarch64_neon_vuqshrn
3252                : Intrinsic::aarch64_neon_vsqshrn;
3253     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
3254   case AArch64::BI__builtin_neon_vqrshrn_n_v:
3255     Int = usgn ? Intrinsic::aarch64_neon_vuqrshrn
3256                : Intrinsic::aarch64_neon_vsqrshrn;
3257     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
3258 
3259   // Convert
3260   case AArch64::BI__builtin_neon_vmovl_v:
3261     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmovl_v, E);
3262   case AArch64::BI__builtin_neon_vcvt_n_f32_v:
3263     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_n_f32_v, E);
3264   case AArch64::BI__builtin_neon_vcvtq_n_f32_v:
3265     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvtq_n_f32_v, E);
3266   case AArch64::BI__builtin_neon_vcvt_n_f64_v:
3267   case AArch64::BI__builtin_neon_vcvtq_n_f64_v: {
3268     llvm::Type *FloatTy =
3269         GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
3270     llvm::Type *Tys[2] = { FloatTy, Ty };
3271     Int = usgn ? Intrinsic::arm_neon_vcvtfxu2fp
3272                : Intrinsic::arm_neon_vcvtfxs2fp;
3273     Function *F = CGM.getIntrinsic(Int, Tys);
3274     return EmitNeonCall(F, Ops, "vcvt_n");
3275   }
3276   case AArch64::BI__builtin_neon_vcvt_n_s32_v:
3277     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_n_s32_v, E);
3278   case AArch64::BI__builtin_neon_vcvtq_n_s32_v:
3279     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvtq_n_s32_v, E);
3280   case AArch64::BI__builtin_neon_vcvt_n_u32_v:
3281     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_n_u32_v, E);
3282   case AArch64::BI__builtin_neon_vcvtq_n_u32_v:
3283     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvtq_n_u32_v, E);
3284   case AArch64::BI__builtin_neon_vcvt_n_s64_v:
3285   case AArch64::BI__builtin_neon_vcvt_n_u64_v:
3286   case AArch64::BI__builtin_neon_vcvtq_n_s64_v:
3287   case AArch64::BI__builtin_neon_vcvtq_n_u64_v: {
3288     llvm::Type *FloatTy =
3289         GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
3290     llvm::Type *Tys[2] = { Ty, FloatTy };
3291     Int = usgn ? Intrinsic::arm_neon_vcvtfp2fxu
3292                : Intrinsic::arm_neon_vcvtfp2fxs;
3293     Function *F = CGM.getIntrinsic(Int, Tys);
3294     return EmitNeonCall(F, Ops, "vcvt_n");
3295   }
3296 
3297   // Load/Store
3298   case AArch64::BI__builtin_neon_vld1_v:
3299     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld1_v, E);
3300   case AArch64::BI__builtin_neon_vld1q_v:
3301     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld1q_v, E);
3302   case AArch64::BI__builtin_neon_vld2_v:
3303     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld2_v, E);
3304   case AArch64::BI__builtin_neon_vld2q_v:
3305     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld2q_v, E);
3306   case AArch64::BI__builtin_neon_vld3_v:
3307     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld3_v, E);
3308   case AArch64::BI__builtin_neon_vld3q_v:
3309     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld3q_v, E);
3310   case AArch64::BI__builtin_neon_vld4_v:
3311     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld4_v, E);
3312   case AArch64::BI__builtin_neon_vld4q_v:
3313     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld4q_v, E);
3314   case AArch64::BI__builtin_neon_vst1_v:
3315     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst1_v, E);
3316   case AArch64::BI__builtin_neon_vst1q_v:
3317     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst1q_v, E);
3318   case AArch64::BI__builtin_neon_vst2_v:
3319     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst2_v, E);
3320   case AArch64::BI__builtin_neon_vst2q_v:
3321     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst2q_v, E);
3322   case AArch64::BI__builtin_neon_vst3_v:
3323     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst3_v, E);
3324   case AArch64::BI__builtin_neon_vst3q_v:
3325     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst3q_v, E);
3326   case AArch64::BI__builtin_neon_vst4_v:
3327     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst4_v, E);
3328   case AArch64::BI__builtin_neon_vst4q_v:
3329     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst4q_v, E);
3330   case AArch64::BI__builtin_neon_vld1_x2_v:
3331   case AArch64::BI__builtin_neon_vld1q_x2_v:
3332   case AArch64::BI__builtin_neon_vld1_x3_v:
3333   case AArch64::BI__builtin_neon_vld1q_x3_v:
3334   case AArch64::BI__builtin_neon_vld1_x4_v:
3335   case AArch64::BI__builtin_neon_vld1q_x4_v: {
3336     unsigned Int;
3337     switch (BuiltinID) {
3338     case AArch64::BI__builtin_neon_vld1_x2_v:
3339     case AArch64::BI__builtin_neon_vld1q_x2_v:
3340       Int = Intrinsic::aarch64_neon_vld1x2;
3341       break;
3342     case AArch64::BI__builtin_neon_vld1_x3_v:
3343     case AArch64::BI__builtin_neon_vld1q_x3_v:
3344       Int = Intrinsic::aarch64_neon_vld1x3;
3345       break;
3346     case AArch64::BI__builtin_neon_vld1_x4_v:
3347     case AArch64::BI__builtin_neon_vld1q_x4_v:
3348       Int = Intrinsic::aarch64_neon_vld1x4;
3349       break;
3350     }
3351     Function *F = CGM.getIntrinsic(Int, Ty);
3352     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld1xN");
3353     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3354     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3355     return Builder.CreateStore(Ops[1], Ops[0]);
3356   }
3357   case AArch64::BI__builtin_neon_vst1_x2_v:
3358   case AArch64::BI__builtin_neon_vst1q_x2_v:
3359   case AArch64::BI__builtin_neon_vst1_x3_v:
3360   case AArch64::BI__builtin_neon_vst1q_x3_v:
3361   case AArch64::BI__builtin_neon_vst1_x4_v:
3362   case AArch64::BI__builtin_neon_vst1q_x4_v: {
3363     Ops.push_back(Align);
3364     unsigned Int;
3365     switch (BuiltinID) {
3366     case AArch64::BI__builtin_neon_vst1_x2_v:
3367     case AArch64::BI__builtin_neon_vst1q_x2_v:
3368       Int = Intrinsic::aarch64_neon_vst1x2;
3369       break;
3370     case AArch64::BI__builtin_neon_vst1_x3_v:
3371     case AArch64::BI__builtin_neon_vst1q_x3_v:
3372       Int = Intrinsic::aarch64_neon_vst1x3;
3373       break;
3374     case AArch64::BI__builtin_neon_vst1_x4_v:
3375     case AArch64::BI__builtin_neon_vst1q_x4_v:
3376       Int = Intrinsic::aarch64_neon_vst1x4;
3377       break;
3378     }
3379     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "");
3380   }
3381   case AArch64::BI__builtin_neon_vld1_lane_v:
3382   case AArch64::BI__builtin_neon_vld1q_lane_v: {
3383     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3384     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
3385     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3386     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
3387     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
3388     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
3389   }
3390   case AArch64::BI__builtin_neon_vld2_lane_v:
3391     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld2q_lane_v, E);
3392   case AArch64::BI__builtin_neon_vld2q_lane_v:
3393     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld2q_lane_v, E);
3394   case AArch64::BI__builtin_neon_vld3_lane_v:
3395     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld3_lane_v, E);
3396   case AArch64::BI__builtin_neon_vld3q_lane_v:
3397     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld3q_lane_v, E);
3398   case AArch64::BI__builtin_neon_vld4_lane_v:
3399     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld4_lane_v, E);
3400   case AArch64::BI__builtin_neon_vld4q_lane_v:
3401     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld4q_lane_v, E);
3402   case AArch64::BI__builtin_neon_vst1_lane_v:
3403   case AArch64::BI__builtin_neon_vst1q_lane_v: {
3404     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3405     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
3406     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3407     StoreInst *St =
3408         Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty));
3409     St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
3410     return St;
3411   }
3412   case AArch64::BI__builtin_neon_vst2_lane_v:
3413     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst2_lane_v, E);
3414   case AArch64::BI__builtin_neon_vst2q_lane_v:
3415     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst2q_lane_v, E);
3416   case AArch64::BI__builtin_neon_vst3_lane_v:
3417     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst3_lane_v, E);
3418   case AArch64::BI__builtin_neon_vst3q_lane_v:
3419     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst3q_lane_v, E);
3420   case AArch64::BI__builtin_neon_vst4_lane_v:
3421     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst4_lane_v, E);
3422   case AArch64::BI__builtin_neon_vst4q_lane_v:
3423     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vst4q_lane_v, E);
3424   case AArch64::BI__builtin_neon_vld1_dup_v:
3425     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld1_dup_v, E);
3426   case AArch64::BI__builtin_neon_vld1q_dup_v:
3427     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vld1q_dup_v, E);
3428   case AArch64::BI__builtin_neon_vld2_dup_v:
3429   case AArch64::BI__builtin_neon_vld2q_dup_v:
3430   case AArch64::BI__builtin_neon_vld3_dup_v:
3431   case AArch64::BI__builtin_neon_vld3q_dup_v:
3432   case AArch64::BI__builtin_neon_vld4_dup_v:
3433   case AArch64::BI__builtin_neon_vld4q_dup_v: {
3434     // Handle 64-bit x 1 elements as a special-case.  There is no "dup" needed.
3435     if (VTy->getElementType()->getPrimitiveSizeInBits() == 64 &&
3436         VTy->getNumElements() == 1) {
3437       switch (BuiltinID) {
3438       case AArch64::BI__builtin_neon_vld2_dup_v:
3439         Int = Intrinsic::arm_neon_vld2;
3440         break;
3441       case AArch64::BI__builtin_neon_vld3_dup_v:
3442         Int = Intrinsic::arm_neon_vld3;
3443         break;
3444       case AArch64::BI__builtin_neon_vld4_dup_v:
3445         Int = Intrinsic::arm_neon_vld4;
3446         break;
3447       default:
3448         llvm_unreachable("unknown vld_dup intrinsic?");
3449       }
3450       Function *F = CGM.getIntrinsic(Int, Ty);
3451       Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
3452       Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3453       Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3454       return Builder.CreateStore(Ops[1], Ops[0]);
3455     }
3456     switch (BuiltinID) {
3457     case AArch64::BI__builtin_neon_vld2_dup_v:
3458     case AArch64::BI__builtin_neon_vld2q_dup_v:
3459       Int = Intrinsic::arm_neon_vld2lane;
3460       break;
3461     case AArch64::BI__builtin_neon_vld3_dup_v:
3462     case AArch64::BI__builtin_neon_vld3q_dup_v:
3463       Int = Intrinsic::arm_neon_vld3lane;
3464       break;
3465     case AArch64::BI__builtin_neon_vld4_dup_v:
3466     case AArch64::BI__builtin_neon_vld4q_dup_v:
3467       Int = Intrinsic::arm_neon_vld4lane;
3468       break;
3469     }
3470     Function *F = CGM.getIntrinsic(Int, Ty);
3471     llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
3472 
3473     SmallVector<Value *, 6> Args;
3474     Args.push_back(Ops[1]);
3475     Args.append(STy->getNumElements(), UndefValue::get(Ty));
3476 
3477     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
3478     Args.push_back(CI);
3479     Args.push_back(Align);
3480 
3481     Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
3482     // splat lane 0 to all elts in each vector of the result.
3483     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3484       Value *Val = Builder.CreateExtractValue(Ops[1], i);
3485       Value *Elt = Builder.CreateBitCast(Val, Ty);
3486       Elt = EmitNeonSplat(Elt, CI);
3487       Elt = Builder.CreateBitCast(Elt, Val->getType());
3488       Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
3489     }
3490     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3491     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3492     return Builder.CreateStore(Ops[1], Ops[0]);
3493   }
3494 
3495   // Crypto
3496   case AArch64::BI__builtin_neon_vaeseq_v:
3497     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_aese, Ty),
3498                         Ops, "aese");
3499   case AArch64::BI__builtin_neon_vaesdq_v:
3500     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_aesd, Ty),
3501                         Ops, "aesd");
3502   case AArch64::BI__builtin_neon_vaesmcq_v:
3503     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_aesmc, Ty),
3504                         Ops, "aesmc");
3505   case AArch64::BI__builtin_neon_vaesimcq_v:
3506     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_aesimc, Ty),
3507                         Ops, "aesimc");
3508   case AArch64::BI__builtin_neon_vsha1su1q_v:
3509     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1su1, Ty),
3510                         Ops, "sha1su1");
3511   case AArch64::BI__builtin_neon_vsha256su0q_v:
3512     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha256su0, Ty),
3513                         Ops, "sha256su0");
3514   case AArch64::BI__builtin_neon_vsha1su0q_v:
3515     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1su0, Ty),
3516                         Ops, "sha1su0");
3517   case AArch64::BI__builtin_neon_vsha256hq_v:
3518     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha256h, Ty),
3519                         Ops, "sha256h");
3520   case AArch64::BI__builtin_neon_vsha256h2q_v:
3521     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha256h2, Ty),
3522                         Ops, "sha256h2");
3523   case AArch64::BI__builtin_neon_vsha256su1q_v:
3524     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha256su1, Ty),
3525                         Ops, "sha256su1");
3526   case AArch64::BI__builtin_neon_vmul_lane_v:
3527   case AArch64::BI__builtin_neon_vmul_laneq_v: {
3528     // v1f64 vmul_lane should be mapped to Neon scalar mul lane
3529     bool Quad = false;
3530     if (BuiltinID == AArch64::BI__builtin_neon_vmul_laneq_v)
3531       Quad = true;
3532     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
3533     llvm::Type *VTy = GetNeonType(this,
3534       NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
3535     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
3536     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
3537     Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
3538     return Builder.CreateBitCast(Result, Ty);
3539   }
3540 
3541   // AArch64-only builtins
3542   case AArch64::BI__builtin_neon_vfmaq_laneq_v: {
3543     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3544     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3545     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3546 
3547     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
3548     Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
3549     return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
3550   }
3551   case AArch64::BI__builtin_neon_vfmaq_lane_v: {
3552     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3553     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3554     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3555 
3556     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3557     llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
3558                                             VTy->getNumElements() / 2);
3559     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
3560     Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
3561                                                cast<ConstantInt>(Ops[3]));
3562     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
3563 
3564     return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
3565   }
3566   case AArch64::BI__builtin_neon_vfma_lane_v: {
3567     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3568     // v1f64 fma should be mapped to Neon scalar f64 fma
3569     if (VTy && VTy->getElementType() == DoubleTy) {
3570       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
3571       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
3572       llvm::Type *VTy = GetNeonType(this,
3573         NeonTypeFlags(NeonTypeFlags::Float64, false, false));
3574       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
3575       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
3576       Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
3577       Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
3578       return Builder.CreateBitCast(Result, Ty);
3579     }
3580     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3581     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3582     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3583 
3584     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
3585     Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
3586     return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
3587   }
3588   case AArch64::BI__builtin_neon_vfma_laneq_v: {
3589     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3590     // v1f64 fma should be mapped to Neon scalar f64 fma
3591     if (VTy && VTy->getElementType() == DoubleTy) {
3592       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
3593       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
3594       llvm::Type *VTy = GetNeonType(this,
3595         NeonTypeFlags(NeonTypeFlags::Float64, false, true));
3596       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
3597       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
3598       Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
3599       Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
3600       return Builder.CreateBitCast(Result, Ty);
3601     }
3602     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3603     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3604     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3605 
3606     llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
3607                                             VTy->getNumElements() * 2);
3608     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
3609     Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
3610                                                cast<ConstantInt>(Ops[3]));
3611     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
3612 
3613     return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
3614   }
3615   case AArch64::BI__builtin_neon_vfms_v:
3616   case AArch64::BI__builtin_neon_vfmsq_v: {
3617     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3618     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3619     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3620     Ops[1] = Builder.CreateFNeg(Ops[1]);
3621     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
3622 
3623     // LLVM's fma intrinsic puts the accumulator in the last position, but the
3624     // AArch64 intrinsic has it first.
3625     return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
3626   }
3627   case AArch64::BI__builtin_neon_vmaxnm_v:
3628   case AArch64::BI__builtin_neon_vmaxnmq_v: {
3629     Int = Intrinsic::aarch64_neon_vmaxnm;
3630     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
3631   }
3632   case AArch64::BI__builtin_neon_vminnm_v:
3633   case AArch64::BI__builtin_neon_vminnmq_v: {
3634     Int = Intrinsic::aarch64_neon_vminnm;
3635     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
3636   }
3637   case AArch64::BI__builtin_neon_vpmaxnm_v:
3638   case AArch64::BI__builtin_neon_vpmaxnmq_v: {
3639     Int = Intrinsic::aarch64_neon_vpmaxnm;
3640     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
3641   }
3642   case AArch64::BI__builtin_neon_vpminnm_v:
3643   case AArch64::BI__builtin_neon_vpminnmq_v: {
3644     Int = Intrinsic::aarch64_neon_vpminnm;
3645     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
3646   }
3647   case AArch64::BI__builtin_neon_vpmaxq_v: {
3648     Int = usgn ? Intrinsic::arm_neon_vpmaxu : Intrinsic::arm_neon_vpmaxs;
3649     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
3650   }
3651   case AArch64::BI__builtin_neon_vpminq_v: {
3652     Int = usgn ? Intrinsic::arm_neon_vpminu : Intrinsic::arm_neon_vpmins;
3653     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
3654   }
3655   case AArch64::BI__builtin_neon_vpaddq_v: {
3656     Int = Intrinsic::arm_neon_vpadd;
3657     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpadd");
3658   }
3659   case AArch64::BI__builtin_neon_vmulx_v:
3660   case AArch64::BI__builtin_neon_vmulxq_v: {
3661     Int = Intrinsic::aarch64_neon_vmulx;
3662     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
3663   }
3664   case AArch64::BI__builtin_neon_vpaddl_v:
3665   case AArch64::BI__builtin_neon_vpaddlq_v:
3666     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vpaddl_v, E);
3667   case AArch64::BI__builtin_neon_vpadal_v:
3668   case AArch64::BI__builtin_neon_vpadalq_v:
3669     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vpadal_v, E);
3670   case AArch64::BI__builtin_neon_vqabs_v:
3671   case AArch64::BI__builtin_neon_vqabsq_v:
3672     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqabs_v, E);
3673   case AArch64::BI__builtin_neon_vqneg_v:
3674   case AArch64::BI__builtin_neon_vqnegq_v:
3675     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqneg_v, E);
3676   case AArch64::BI__builtin_neon_vabs_v:
3677   case AArch64::BI__builtin_neon_vabsq_v: {
3678     if (VTy->getElementType()->isFloatingPointTy()) {
3679       return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
3680     }
3681     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vabs_v, E);
3682   }
3683   case AArch64::BI__builtin_neon_vsqadd_v:
3684   case AArch64::BI__builtin_neon_vsqaddq_v: {
3685     Int = Intrinsic::aarch64_neon_usqadd;
3686     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
3687   }
3688   case AArch64::BI__builtin_neon_vuqadd_v:
3689   case AArch64::BI__builtin_neon_vuqaddq_v: {
3690     Int = Intrinsic::aarch64_neon_suqadd;
3691     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
3692   }
3693   case AArch64::BI__builtin_neon_vcls_v:
3694   case AArch64::BI__builtin_neon_vclsq_v:
3695     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcls_v, E);
3696   case AArch64::BI__builtin_neon_vclz_v:
3697   case AArch64::BI__builtin_neon_vclzq_v:
3698     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vclz_v, E);
3699   case AArch64::BI__builtin_neon_vcnt_v:
3700   case AArch64::BI__builtin_neon_vcntq_v:
3701     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcnt_v, E);
3702   case AArch64::BI__builtin_neon_vrbit_v:
3703   case AArch64::BI__builtin_neon_vrbitq_v:
3704     Int = Intrinsic::aarch64_neon_rbit;
3705     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
3706   case AArch64::BI__builtin_neon_vmovn_v:
3707     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vmovn_v, E);
3708   case AArch64::BI__builtin_neon_vqmovun_v:
3709     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqmovun_v, E);
3710   case AArch64::BI__builtin_neon_vqmovn_v:
3711     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vqmovn_v, E);
3712   case AArch64::BI__builtin_neon_vcvt_f16_v:
3713     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_f16_v, E);
3714   case AArch64::BI__builtin_neon_vcvt_f32_f16:
3715     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_f32_f16, E);
3716   case AArch64::BI__builtin_neon_vcvt_f32_f64: {
3717     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3718     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, false));
3719     return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
3720   }
3721   case AArch64::BI__builtin_neon_vcvtx_f32_v: {
3722     llvm::Type *EltTy = FloatTy;
3723     llvm::Type *ResTy = llvm::VectorType::get(EltTy, 2);
3724     llvm::Type *Tys[2] = { ResTy, Ty };
3725     Int = Intrinsic::aarch64_neon_vcvtxn;
3726     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtx_f32_f64");
3727   }
3728   case AArch64::BI__builtin_neon_vcvt_f64_f32: {
3729     llvm::Type *OpTy =
3730         GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, false));
3731     Ops[0] = Builder.CreateBitCast(Ops[0], OpTy);
3732     return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
3733   }
3734   case AArch64::BI__builtin_neon_vcvt_f64_v:
3735   case AArch64::BI__builtin_neon_vcvtq_f64_v: {
3736     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3737     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
3738     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
3739                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
3740   }
3741   case AArch64::BI__builtin_neon_vrndn_v:
3742   case AArch64::BI__builtin_neon_vrndnq_v: {
3743     Int = Intrinsic::aarch64_neon_frintn;
3744     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
3745   }
3746   case AArch64::BI__builtin_neon_vrnda_v:
3747   case AArch64::BI__builtin_neon_vrndaq_v: {
3748     Int = Intrinsic::round;
3749     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
3750   }
3751   case AArch64::BI__builtin_neon_vrndp_v:
3752   case AArch64::BI__builtin_neon_vrndpq_v: {
3753     Int = Intrinsic::ceil;
3754     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
3755   }
3756   case AArch64::BI__builtin_neon_vrndm_v:
3757   case AArch64::BI__builtin_neon_vrndmq_v: {
3758     Int = Intrinsic::floor;
3759     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
3760   }
3761   case AArch64::BI__builtin_neon_vrndx_v:
3762   case AArch64::BI__builtin_neon_vrndxq_v: {
3763     Int = Intrinsic::rint;
3764     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
3765   }
3766   case AArch64::BI__builtin_neon_vrnd_v:
3767   case AArch64::BI__builtin_neon_vrndq_v: {
3768     Int = Intrinsic::trunc;
3769     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd");
3770   }
3771   case AArch64::BI__builtin_neon_vrndi_v:
3772   case AArch64::BI__builtin_neon_vrndiq_v: {
3773     Int = Intrinsic::nearbyint;
3774     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
3775   }
3776   case AArch64::BI__builtin_neon_vcvt_s32_v:
3777   case AArch64::BI__builtin_neon_vcvt_u32_v:
3778   case AArch64::BI__builtin_neon_vcvtq_s32_v:
3779   case AArch64::BI__builtin_neon_vcvtq_u32_v:
3780     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvtq_u32_v, E);
3781   case AArch64::BI__builtin_neon_vcvt_s64_v:
3782   case AArch64::BI__builtin_neon_vcvt_u64_v:
3783   case AArch64::BI__builtin_neon_vcvtq_s64_v:
3784   case AArch64::BI__builtin_neon_vcvtq_u64_v: {
3785     llvm::Type *DoubleTy =
3786         GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
3787     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
3788     return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
3789                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
3790   }
3791   case AArch64::BI__builtin_neon_vcvtn_s32_v:
3792   case AArch64::BI__builtin_neon_vcvtnq_s32_v: {
3793     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3794     llvm::Type *Tys[2] = { Ty, OpTy };
3795     Int = Intrinsic::arm_neon_vcvtns;
3796     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtns_f32");
3797   }
3798   case AArch64::BI__builtin_neon_vcvtn_s64_v:
3799   case AArch64::BI__builtin_neon_vcvtnq_s64_v: {
3800     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3801     llvm::Type *Tys[2] = { Ty, OpTy };
3802     Int = Intrinsic::arm_neon_vcvtns;
3803     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtns_f64");
3804   }
3805   case AArch64::BI__builtin_neon_vcvtn_u32_v:
3806   case AArch64::BI__builtin_neon_vcvtnq_u32_v: {
3807     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3808     llvm::Type *Tys[2] = { Ty, OpTy };
3809     Int = Intrinsic::arm_neon_vcvtnu;
3810     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtnu_f32");
3811   }
3812   case AArch64::BI__builtin_neon_vcvtn_u64_v:
3813   case AArch64::BI__builtin_neon_vcvtnq_u64_v: {
3814     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3815     llvm::Type *Tys[2] = { Ty, OpTy };
3816     Int = Intrinsic::arm_neon_vcvtnu;
3817     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtnu_f64");
3818   }
3819   case AArch64::BI__builtin_neon_vcvtp_s32_v:
3820   case AArch64::BI__builtin_neon_vcvtpq_s32_v: {
3821     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3822     llvm::Type *Tys[2] = { Ty, OpTy };
3823     Int = Intrinsic::arm_neon_vcvtps;
3824     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtps_f32");
3825   }
3826   case AArch64::BI__builtin_neon_vcvtp_s64_v:
3827   case AArch64::BI__builtin_neon_vcvtpq_s64_v: {
3828     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3829     llvm::Type *Tys[2] = { Ty, OpTy };
3830     Int = Intrinsic::arm_neon_vcvtps;
3831     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtps_f64");
3832   }
3833   case AArch64::BI__builtin_neon_vcvtp_u32_v:
3834   case AArch64::BI__builtin_neon_vcvtpq_u32_v: {
3835     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3836     llvm::Type *Tys[2] = { Ty, OpTy };
3837     Int = Intrinsic::arm_neon_vcvtpu;
3838     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtpu_f32");
3839   }
3840   case AArch64::BI__builtin_neon_vcvtp_u64_v:
3841   case AArch64::BI__builtin_neon_vcvtpq_u64_v: {
3842     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3843     llvm::Type *Tys[2] = { Ty, OpTy };
3844     Int = Intrinsic::arm_neon_vcvtpu;
3845     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtpu_f64");
3846   }
3847   case AArch64::BI__builtin_neon_vcvtm_s32_v:
3848   case AArch64::BI__builtin_neon_vcvtmq_s32_v: {
3849     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3850     llvm::Type *Tys[2] = { Ty, OpTy };
3851     Int = Intrinsic::arm_neon_vcvtms;
3852     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtms_f32");
3853   }
3854   case AArch64::BI__builtin_neon_vcvtm_s64_v:
3855   case AArch64::BI__builtin_neon_vcvtmq_s64_v: {
3856     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3857     llvm::Type *Tys[2] = { Ty, OpTy };
3858     Int = Intrinsic::arm_neon_vcvtms;
3859     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtms_f64");
3860   }
3861   case AArch64::BI__builtin_neon_vcvtm_u32_v:
3862   case AArch64::BI__builtin_neon_vcvtmq_u32_v: {
3863     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3864     llvm::Type *Tys[2] = { Ty, OpTy };
3865     Int = Intrinsic::arm_neon_vcvtmu;
3866     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtmu_f32");
3867   }
3868   case AArch64::BI__builtin_neon_vcvtm_u64_v:
3869   case AArch64::BI__builtin_neon_vcvtmq_u64_v: {
3870     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3871     llvm::Type *Tys[2] = { Ty, OpTy };
3872     Int = Intrinsic::arm_neon_vcvtmu;
3873     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtmu_f64");
3874   }
3875   case AArch64::BI__builtin_neon_vcvta_s32_v:
3876   case AArch64::BI__builtin_neon_vcvtaq_s32_v: {
3877     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3878     llvm::Type *Tys[2] = { Ty, OpTy };
3879     Int = Intrinsic::arm_neon_vcvtas;
3880     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtas_f32");
3881   }
3882   case AArch64::BI__builtin_neon_vcvta_s64_v:
3883   case AArch64::BI__builtin_neon_vcvtaq_s64_v: {
3884     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3885     llvm::Type *Tys[2] = { Ty, OpTy };
3886     Int = Intrinsic::arm_neon_vcvtas;
3887     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtas_f64");
3888   }
3889   case AArch64::BI__builtin_neon_vcvta_u32_v:
3890   case AArch64::BI__builtin_neon_vcvtaq_u32_v: {
3891     llvm::Type *OpTy = llvm::VectorType::get(FloatTy, VTy->getNumElements());
3892     llvm::Type *Tys[2] = { Ty, OpTy };
3893     Int = Intrinsic::arm_neon_vcvtau;
3894     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtau_f32");
3895   }
3896   case AArch64::BI__builtin_neon_vcvta_u64_v:
3897   case AArch64::BI__builtin_neon_vcvtaq_u64_v: {
3898     llvm::Type *OpTy = llvm::VectorType::get(DoubleTy, VTy->getNumElements());
3899     llvm::Type *Tys[2] = { Ty, OpTy };
3900     Int = Intrinsic::arm_neon_vcvtau;
3901     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtau_f64");
3902   }
3903   case AArch64::BI__builtin_neon_vrecpe_v:
3904   case AArch64::BI__builtin_neon_vrecpeq_v:
3905     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrecpe_v, E);
3906   case AArch64::BI__builtin_neon_vrsqrte_v:
3907   case AArch64::BI__builtin_neon_vrsqrteq_v:
3908     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vrsqrte_v, E);
3909   case AArch64::BI__builtin_neon_vsqrt_v:
3910   case AArch64::BI__builtin_neon_vsqrtq_v: {
3911     Int = Intrinsic::sqrt;
3912     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
3913   }
3914   case AArch64::BI__builtin_neon_vcvt_f32_v:
3915   case AArch64::BI__builtin_neon_vcvtq_f32_v:
3916     return EmitARMBuiltinExpr(ARM::BI__builtin_neon_vcvt_f32_v, E);
3917   case AArch64::BI__builtin_neon_vceqz_v:
3918   case AArch64::BI__builtin_neon_vceqzq_v:
3919     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
3920                                          ICmpInst::ICMP_EQ, "vceqz");
3921   case AArch64::BI__builtin_neon_vcgez_v:
3922   case AArch64::BI__builtin_neon_vcgezq_v:
3923     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
3924                                          ICmpInst::ICMP_SGE, "vcgez");
3925   case AArch64::BI__builtin_neon_vclez_v:
3926   case AArch64::BI__builtin_neon_vclezq_v:
3927     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
3928                                          ICmpInst::ICMP_SLE, "vclez");
3929   case AArch64::BI__builtin_neon_vcgtz_v:
3930   case AArch64::BI__builtin_neon_vcgtzq_v:
3931     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
3932                                          ICmpInst::ICMP_SGT, "vcgtz");
3933   case AArch64::BI__builtin_neon_vcltz_v:
3934   case AArch64::BI__builtin_neon_vcltzq_v:
3935     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
3936                                          ICmpInst::ICMP_SLT, "vcltz");
3937   }
3938 }
3939 
3940 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
3941                                            const CallExpr *E) {
3942   if (BuiltinID == ARM::BI__clear_cache) {
3943     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
3944     const FunctionDecl *FD = E->getDirectCallee();
3945     SmallVector<Value*, 2> Ops;
3946     for (unsigned i = 0; i < 2; i++)
3947       Ops.push_back(EmitScalarExpr(E->getArg(i)));
3948     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
3949     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
3950     StringRef Name = FD->getName();
3951     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
3952   }
3953 
3954   if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
3955       (BuiltinID == ARM::BI__builtin_arm_ldrex &&
3956        getContext().getTypeSize(E->getType()) == 64)) {
3957     Function *F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
3958 
3959     Value *LdPtr = EmitScalarExpr(E->getArg(0));
3960     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
3961                                     "ldrexd");
3962 
3963     Value *Val0 = Builder.CreateExtractValue(Val, 1);
3964     Value *Val1 = Builder.CreateExtractValue(Val, 0);
3965     Val0 = Builder.CreateZExt(Val0, Int64Ty);
3966     Val1 = Builder.CreateZExt(Val1, Int64Ty);
3967 
3968     Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
3969     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
3970     Val = Builder.CreateOr(Val, Val1);
3971     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
3972   }
3973 
3974   if (BuiltinID == ARM::BI__builtin_arm_ldrex) {
3975     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
3976 
3977     QualType Ty = E->getType();
3978     llvm::Type *RealResTy = ConvertType(Ty);
3979     llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(),
3980                                                   getContext().getTypeSize(Ty));
3981     LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo());
3982 
3983     Function *F = CGM.getIntrinsic(Intrinsic::arm_ldrex, LoadAddr->getType());
3984     Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
3985 
3986     if (RealResTy->isPointerTy())
3987       return Builder.CreateIntToPtr(Val, RealResTy);
3988     else {
3989       Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
3990       return Builder.CreateBitCast(Val, RealResTy);
3991     }
3992   }
3993 
3994   if (BuiltinID == ARM::BI__builtin_arm_strexd ||
3995       (BuiltinID == ARM::BI__builtin_arm_strex &&
3996        getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
3997     Function *F = CGM.getIntrinsic(Intrinsic::arm_strexd);
3998     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL);
3999 
4000     Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
4001     Value *Val = EmitScalarExpr(E->getArg(0));
4002     Builder.CreateStore(Val, Tmp);
4003 
4004     Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
4005     Val = Builder.CreateLoad(LdPtr);
4006 
4007     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
4008     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
4009     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
4010     return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd");
4011   }
4012 
4013   if (BuiltinID == ARM::BI__builtin_arm_strex) {
4014     Value *StoreVal = EmitScalarExpr(E->getArg(0));
4015     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
4016 
4017     QualType Ty = E->getArg(0)->getType();
4018     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
4019                                                  getContext().getTypeSize(Ty));
4020     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
4021 
4022     if (StoreVal->getType()->isPointerTy())
4023       StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
4024     else {
4025       StoreVal = Builder.CreateBitCast(StoreVal, StoreTy);
4026       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
4027     }
4028 
4029     Function *F = CGM.getIntrinsic(Intrinsic::arm_strex, StoreAddr->getType());
4030     return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex");
4031   }
4032 
4033   if (BuiltinID == ARM::BI__builtin_arm_clrex) {
4034     Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
4035     return Builder.CreateCall(F);
4036   }
4037 
4038   if (BuiltinID == ARM::BI__builtin_arm_sevl) {
4039     Function *F = CGM.getIntrinsic(Intrinsic::arm_sevl);
4040     return Builder.CreateCall(F);
4041   }
4042 
4043   // CRC32
4044   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4045   switch (BuiltinID) {
4046   case ARM::BI__builtin_arm_crc32b:
4047     CRCIntrinsicID = Intrinsic::arm_crc32b; break;
4048   case ARM::BI__builtin_arm_crc32cb:
4049     CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
4050   case ARM::BI__builtin_arm_crc32h:
4051     CRCIntrinsicID = Intrinsic::arm_crc32h; break;
4052   case ARM::BI__builtin_arm_crc32ch:
4053     CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
4054   case ARM::BI__builtin_arm_crc32w:
4055   case ARM::BI__builtin_arm_crc32d:
4056     CRCIntrinsicID = Intrinsic::arm_crc32w; break;
4057   case ARM::BI__builtin_arm_crc32cw:
4058   case ARM::BI__builtin_arm_crc32cd:
4059     CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
4060   }
4061 
4062   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4063     Value *Arg0 = EmitScalarExpr(E->getArg(0));
4064     Value *Arg1 = EmitScalarExpr(E->getArg(1));
4065 
4066     // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
4067     // intrinsics, hence we need different codegen for these cases.
4068     if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
4069         BuiltinID == ARM::BI__builtin_arm_crc32cd) {
4070       Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
4071       Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
4072       Value *Arg1b = Builder.CreateLShr(Arg1, C1);
4073       Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
4074 
4075       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4076       Value *Res = Builder.CreateCall2(F, Arg0, Arg1a);
4077       return Builder.CreateCall2(F, Res, Arg1b);
4078     } else {
4079       Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
4080 
4081       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4082       return Builder.CreateCall2(F, Arg0, Arg1);
4083     }
4084   }
4085 
4086   SmallVector<Value*, 4> Ops;
4087   llvm::Value *Align = 0;
4088   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
4089     if (i == 0) {
4090       switch (BuiltinID) {
4091       case ARM::BI__builtin_neon_vld1_v:
4092       case ARM::BI__builtin_neon_vld1q_v:
4093       case ARM::BI__builtin_neon_vld1q_lane_v:
4094       case ARM::BI__builtin_neon_vld1_lane_v:
4095       case ARM::BI__builtin_neon_vld1_dup_v:
4096       case ARM::BI__builtin_neon_vld1q_dup_v:
4097       case ARM::BI__builtin_neon_vst1_v:
4098       case ARM::BI__builtin_neon_vst1q_v:
4099       case ARM::BI__builtin_neon_vst1q_lane_v:
4100       case ARM::BI__builtin_neon_vst1_lane_v:
4101       case ARM::BI__builtin_neon_vst2_v:
4102       case ARM::BI__builtin_neon_vst2q_v:
4103       case ARM::BI__builtin_neon_vst2_lane_v:
4104       case ARM::BI__builtin_neon_vst2q_lane_v:
4105       case ARM::BI__builtin_neon_vst3_v:
4106       case ARM::BI__builtin_neon_vst3q_v:
4107       case ARM::BI__builtin_neon_vst3_lane_v:
4108       case ARM::BI__builtin_neon_vst3q_lane_v:
4109       case ARM::BI__builtin_neon_vst4_v:
4110       case ARM::BI__builtin_neon_vst4q_v:
4111       case ARM::BI__builtin_neon_vst4_lane_v:
4112       case ARM::BI__builtin_neon_vst4q_lane_v:
4113         // Get the alignment for the argument in addition to the value;
4114         // we'll use it later.
4115         std::pair<llvm::Value*, unsigned> Src =
4116             EmitPointerWithAlignment(E->getArg(0));
4117         Ops.push_back(Src.first);
4118         Align = Builder.getInt32(Src.second);
4119         continue;
4120       }
4121     }
4122     if (i == 1) {
4123       switch (BuiltinID) {
4124       case ARM::BI__builtin_neon_vld2_v:
4125       case ARM::BI__builtin_neon_vld2q_v:
4126       case ARM::BI__builtin_neon_vld3_v:
4127       case ARM::BI__builtin_neon_vld3q_v:
4128       case ARM::BI__builtin_neon_vld4_v:
4129       case ARM::BI__builtin_neon_vld4q_v:
4130       case ARM::BI__builtin_neon_vld2_lane_v:
4131       case ARM::BI__builtin_neon_vld2q_lane_v:
4132       case ARM::BI__builtin_neon_vld3_lane_v:
4133       case ARM::BI__builtin_neon_vld3q_lane_v:
4134       case ARM::BI__builtin_neon_vld4_lane_v:
4135       case ARM::BI__builtin_neon_vld4q_lane_v:
4136       case ARM::BI__builtin_neon_vld2_dup_v:
4137       case ARM::BI__builtin_neon_vld3_dup_v:
4138       case ARM::BI__builtin_neon_vld4_dup_v:
4139         // Get the alignment for the argument in addition to the value;
4140         // we'll use it later.
4141         std::pair<llvm::Value*, unsigned> Src =
4142             EmitPointerWithAlignment(E->getArg(1));
4143         Ops.push_back(Src.first);
4144         Align = Builder.getInt32(Src.second);
4145         continue;
4146       }
4147     }
4148     Ops.push_back(EmitScalarExpr(E->getArg(i)));
4149   }
4150 
4151   // vget_lane and vset_lane are not overloaded and do not have an extra
4152   // argument that specifies the vector type.
4153   switch (BuiltinID) {
4154   default: break;
4155   case ARM::BI__builtin_neon_vget_lane_i8:
4156   case ARM::BI__builtin_neon_vget_lane_i16:
4157   case ARM::BI__builtin_neon_vget_lane_i32:
4158   case ARM::BI__builtin_neon_vget_lane_i64:
4159   case ARM::BI__builtin_neon_vget_lane_f32:
4160   case ARM::BI__builtin_neon_vgetq_lane_i8:
4161   case ARM::BI__builtin_neon_vgetq_lane_i16:
4162   case ARM::BI__builtin_neon_vgetq_lane_i32:
4163   case ARM::BI__builtin_neon_vgetq_lane_i64:
4164   case ARM::BI__builtin_neon_vgetq_lane_f32:
4165     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4166                                         "vget_lane");
4167   case ARM::BI__builtin_neon_vset_lane_i8:
4168   case ARM::BI__builtin_neon_vset_lane_i16:
4169   case ARM::BI__builtin_neon_vset_lane_i32:
4170   case ARM::BI__builtin_neon_vset_lane_i64:
4171   case ARM::BI__builtin_neon_vset_lane_f32:
4172   case ARM::BI__builtin_neon_vsetq_lane_i8:
4173   case ARM::BI__builtin_neon_vsetq_lane_i16:
4174   case ARM::BI__builtin_neon_vsetq_lane_i32:
4175   case ARM::BI__builtin_neon_vsetq_lane_i64:
4176   case ARM::BI__builtin_neon_vsetq_lane_f32:
4177     Ops.push_back(EmitScalarExpr(E->getArg(2)));
4178     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4179   }
4180 
4181   // Get the last argument, which specifies the vector type.
4182   llvm::APSInt Result;
4183   const Expr *Arg = E->getArg(E->getNumArgs()-1);
4184   if (!Arg->isIntegerConstantExpr(Result, getContext()))
4185     return 0;
4186 
4187   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
4188       BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
4189     // Determine the overloaded type of this builtin.
4190     llvm::Type *Ty;
4191     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
4192       Ty = FloatTy;
4193     else
4194       Ty = DoubleTy;
4195 
4196     // Determine whether this is an unsigned conversion or not.
4197     bool usgn = Result.getZExtValue() == 1;
4198     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
4199 
4200     // Call the appropriate intrinsic.
4201     Function *F = CGM.getIntrinsic(Int, Ty);
4202     return Builder.CreateCall(F, Ops, "vcvtr");
4203   }
4204 
4205   // Determine the type of this overloaded NEON intrinsic.
4206   NeonTypeFlags Type(Result.getZExtValue());
4207   bool usgn = Type.isUnsigned();
4208   bool quad = Type.isQuad();
4209   bool rightShift = false;
4210 
4211   llvm::VectorType *VTy = GetNeonType(this, Type);
4212   llvm::Type *Ty = VTy;
4213   if (!Ty)
4214     return 0;
4215 
4216   unsigned Int;
4217   switch (BuiltinID) {
4218   default: return 0;
4219   case ARM::BI__builtin_neon_vbsl_v:
4220   case ARM::BI__builtin_neon_vbslq_v:
4221     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vbsl, Ty),
4222                         Ops, "vbsl");
4223   case ARM::BI__builtin_neon_vabd_v:
4224   case ARM::BI__builtin_neon_vabdq_v:
4225     Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
4226     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
4227   case ARM::BI__builtin_neon_vabs_v:
4228   case ARM::BI__builtin_neon_vabsq_v:
4229     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vabs, Ty),
4230                         Ops, "vabs");
4231   case ARM::BI__builtin_neon_vaddhn_v: {
4232     llvm::VectorType *SrcTy =
4233         llvm::VectorType::getExtendedElementVectorType(VTy);
4234 
4235     // %sum = add <4 x i32> %lhs, %rhs
4236     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4237     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4238     Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
4239 
4240     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4241     Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
4242                                        SrcTy->getScalarSizeInBits() / 2);
4243     ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
4244     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
4245 
4246     // %res = trunc <4 x i32> %high to <4 x i16>
4247     return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
4248   }
4249   case ARM::BI__builtin_neon_vcale_v:
4250     std::swap(Ops[0], Ops[1]);
4251   case ARM::BI__builtin_neon_vcage_v: {
4252     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacged);
4253     return EmitNeonCall(F, Ops, "vcage");
4254   }
4255   case ARM::BI__builtin_neon_vcaleq_v:
4256     std::swap(Ops[0], Ops[1]);
4257   case ARM::BI__builtin_neon_vcageq_v: {
4258     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgeq);
4259     return EmitNeonCall(F, Ops, "vcage");
4260   }
4261   case ARM::BI__builtin_neon_vcalt_v:
4262     std::swap(Ops[0], Ops[1]);
4263   case ARM::BI__builtin_neon_vcagt_v: {
4264     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtd);
4265     return EmitNeonCall(F, Ops, "vcagt");
4266   }
4267   case ARM::BI__builtin_neon_vcaltq_v:
4268     std::swap(Ops[0], Ops[1]);
4269   case ARM::BI__builtin_neon_vcagtq_v: {
4270     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtq);
4271     return EmitNeonCall(F, Ops, "vcagt");
4272   }
4273   case ARM::BI__builtin_neon_vcls_v:
4274   case ARM::BI__builtin_neon_vclsq_v: {
4275     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcls, Ty);
4276     return EmitNeonCall(F, Ops, "vcls");
4277   }
4278   case ARM::BI__builtin_neon_vclz_v:
4279   case ARM::BI__builtin_neon_vclzq_v: {
4280     // Generate target-independent intrinsic; also need to add second argument
4281     // for whether or not clz of zero is undefined; on ARM it isn't.
4282     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ty);
4283     Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
4284     return EmitNeonCall(F, Ops, "vclz");
4285   }
4286   case ARM::BI__builtin_neon_vcnt_v:
4287   case ARM::BI__builtin_neon_vcntq_v: {
4288     // generate target-independent intrinsic
4289     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, Ty);
4290     return EmitNeonCall(F, Ops, "vctpop");
4291   }
4292   case ARM::BI__builtin_neon_vcvt_f16_v: {
4293     assert(Type.getEltType() == NeonTypeFlags::Float16 && !quad &&
4294            "unexpected vcvt_f16_v builtin");
4295     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf);
4296     return EmitNeonCall(F, Ops, "vcvt");
4297   }
4298   case ARM::BI__builtin_neon_vcvt_f32_f16: {
4299     assert(Type.getEltType() == NeonTypeFlags::Float16 && !quad &&
4300            "unexpected vcvt_f32_f16 builtin");
4301     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp);
4302     return EmitNeonCall(F, Ops, "vcvt");
4303   }
4304   case ARM::BI__builtin_neon_vcvt_f32_v:
4305   case ARM::BI__builtin_neon_vcvtq_f32_v:
4306     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4307     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
4308     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4309                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4310   case ARM::BI__builtin_neon_vcvt_s32_v:
4311   case ARM::BI__builtin_neon_vcvt_u32_v:
4312   case ARM::BI__builtin_neon_vcvtq_s32_v:
4313   case ARM::BI__builtin_neon_vcvtq_u32_v: {
4314     llvm::Type *FloatTy =
4315       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
4316     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
4317     return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
4318                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
4319   }
4320   case ARM::BI__builtin_neon_vcvt_n_f32_v:
4321   case ARM::BI__builtin_neon_vcvtq_n_f32_v: {
4322     llvm::Type *FloatTy =
4323       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
4324     llvm::Type *Tys[2] = { FloatTy, Ty };
4325     Int = usgn ? Intrinsic::arm_neon_vcvtfxu2fp
4326                : Intrinsic::arm_neon_vcvtfxs2fp;
4327     Function *F = CGM.getIntrinsic(Int, Tys);
4328     return EmitNeonCall(F, Ops, "vcvt_n");
4329   }
4330   case ARM::BI__builtin_neon_vcvt_n_s32_v:
4331   case ARM::BI__builtin_neon_vcvt_n_u32_v:
4332   case ARM::BI__builtin_neon_vcvtq_n_s32_v:
4333   case ARM::BI__builtin_neon_vcvtq_n_u32_v: {
4334     llvm::Type *FloatTy =
4335       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
4336     llvm::Type *Tys[2] = { Ty, FloatTy };
4337     Int = usgn ? Intrinsic::arm_neon_vcvtfp2fxu
4338                : Intrinsic::arm_neon_vcvtfp2fxs;
4339     Function *F = CGM.getIntrinsic(Int, Tys);
4340     return EmitNeonCall(F, Ops, "vcvt_n");
4341   }
4342   case ARM::BI__builtin_neon_vext_v:
4343   case ARM::BI__builtin_neon_vextq_v: {
4344     int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
4345     SmallVector<Constant*, 16> Indices;
4346     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4347       Indices.push_back(ConstantInt::get(Int32Ty, i+CV));
4348 
4349     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4350     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4351     Value *SV = llvm::ConstantVector::get(Indices);
4352     return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
4353   }
4354   case ARM::BI__builtin_neon_vhadd_v:
4355   case ARM::BI__builtin_neon_vhaddq_v:
4356     Int = usgn ? Intrinsic::arm_neon_vhaddu : Intrinsic::arm_neon_vhadds;
4357     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vhadd");
4358   case ARM::BI__builtin_neon_vhsub_v:
4359   case ARM::BI__builtin_neon_vhsubq_v:
4360     Int = usgn ? Intrinsic::arm_neon_vhsubu : Intrinsic::arm_neon_vhsubs;
4361     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vhsub");
4362   case ARM::BI__builtin_neon_vld1_v:
4363   case ARM::BI__builtin_neon_vld1q_v:
4364     Ops.push_back(Align);
4365     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty),
4366                         Ops, "vld1");
4367   case ARM::BI__builtin_neon_vld1q_lane_v:
4368     // Handle 64-bit integer elements as a special case.  Use shuffles of
4369     // one-element vectors to avoid poor code for i64 in the backend.
4370     if (VTy->getElementType()->isIntegerTy(64)) {
4371       // Extract the other lane.
4372       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4373       int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
4374       Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
4375       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4376       // Load the value as a one-element vector.
4377       Ty = llvm::VectorType::get(VTy->getElementType(), 1);
4378       Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
4379       Value *Ld = Builder.CreateCall2(F, Ops[0], Align);
4380       // Combine them.
4381       SmallVector<Constant*, 2> Indices;
4382       Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
4383       Indices.push_back(ConstantInt::get(Int32Ty, Lane));
4384       SV = llvm::ConstantVector::get(Indices);
4385       return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
4386     }
4387     // fall through
4388   case ARM::BI__builtin_neon_vld1_lane_v: {
4389     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4390     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
4391     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4392     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
4393     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
4394     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
4395   }
4396   case ARM::BI__builtin_neon_vld1_dup_v:
4397   case ARM::BI__builtin_neon_vld1q_dup_v: {
4398     Value *V = UndefValue::get(Ty);
4399     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
4400     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4401     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
4402     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
4403     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
4404     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
4405     return EmitNeonSplat(Ops[0], CI);
4406   }
4407   case ARM::BI__builtin_neon_vld2_v:
4408   case ARM::BI__builtin_neon_vld2q_v: {
4409     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2, Ty);
4410     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld2");
4411     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4412     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4413     return Builder.CreateStore(Ops[1], Ops[0]);
4414   }
4415   case ARM::BI__builtin_neon_vld3_v:
4416   case ARM::BI__builtin_neon_vld3q_v: {
4417     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3, Ty);
4418     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld3");
4419     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4420     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4421     return Builder.CreateStore(Ops[1], Ops[0]);
4422   }
4423   case ARM::BI__builtin_neon_vld4_v:
4424   case ARM::BI__builtin_neon_vld4q_v: {
4425     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4, Ty);
4426     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld4");
4427     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4428     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4429     return Builder.CreateStore(Ops[1], Ops[0]);
4430   }
4431   case ARM::BI__builtin_neon_vld2_lane_v:
4432   case ARM::BI__builtin_neon_vld2q_lane_v: {
4433     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2lane, Ty);
4434     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4435     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
4436     Ops.push_back(Align);
4437     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
4438     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4439     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4440     return Builder.CreateStore(Ops[1], Ops[0]);
4441   }
4442   case ARM::BI__builtin_neon_vld3_lane_v:
4443   case ARM::BI__builtin_neon_vld3q_lane_v: {
4444     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3lane, Ty);
4445     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4446     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
4447     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
4448     Ops.push_back(Align);
4449     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
4450     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4451     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4452     return Builder.CreateStore(Ops[1], Ops[0]);
4453   }
4454   case ARM::BI__builtin_neon_vld4_lane_v:
4455   case ARM::BI__builtin_neon_vld4q_lane_v: {
4456     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4lane, Ty);
4457     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4458     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
4459     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
4460     Ops[5] = Builder.CreateBitCast(Ops[5], Ty);
4461     Ops.push_back(Align);
4462     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
4463     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4464     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4465     return Builder.CreateStore(Ops[1], Ops[0]);
4466   }
4467   case ARM::BI__builtin_neon_vld2_dup_v:
4468   case ARM::BI__builtin_neon_vld3_dup_v:
4469   case ARM::BI__builtin_neon_vld4_dup_v: {
4470     // Handle 64-bit elements as a special-case.  There is no "dup" needed.
4471     if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
4472       switch (BuiltinID) {
4473       case ARM::BI__builtin_neon_vld2_dup_v:
4474         Int = Intrinsic::arm_neon_vld2;
4475         break;
4476       case ARM::BI__builtin_neon_vld3_dup_v:
4477         Int = Intrinsic::arm_neon_vld3;
4478         break;
4479       case ARM::BI__builtin_neon_vld4_dup_v:
4480         Int = Intrinsic::arm_neon_vld4;
4481         break;
4482       default: llvm_unreachable("unknown vld_dup intrinsic?");
4483       }
4484       Function *F = CGM.getIntrinsic(Int, Ty);
4485       Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
4486       Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4487       Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4488       return Builder.CreateStore(Ops[1], Ops[0]);
4489     }
4490     switch (BuiltinID) {
4491     case ARM::BI__builtin_neon_vld2_dup_v:
4492       Int = Intrinsic::arm_neon_vld2lane;
4493       break;
4494     case ARM::BI__builtin_neon_vld3_dup_v:
4495       Int = Intrinsic::arm_neon_vld3lane;
4496       break;
4497     case ARM::BI__builtin_neon_vld4_dup_v:
4498       Int = Intrinsic::arm_neon_vld4lane;
4499       break;
4500     default: llvm_unreachable("unknown vld_dup intrinsic?");
4501     }
4502     Function *F = CGM.getIntrinsic(Int, Ty);
4503     llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
4504 
4505     SmallVector<Value*, 6> Args;
4506     Args.push_back(Ops[1]);
4507     Args.append(STy->getNumElements(), UndefValue::get(Ty));
4508 
4509     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
4510     Args.push_back(CI);
4511     Args.push_back(Align);
4512 
4513     Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
4514     // splat lane 0 to all elts in each vector of the result.
4515     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
4516       Value *Val = Builder.CreateExtractValue(Ops[1], i);
4517       Value *Elt = Builder.CreateBitCast(Val, Ty);
4518       Elt = EmitNeonSplat(Elt, CI);
4519       Elt = Builder.CreateBitCast(Elt, Val->getType());
4520       Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
4521     }
4522     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4523     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4524     return Builder.CreateStore(Ops[1], Ops[0]);
4525   }
4526   case ARM::BI__builtin_neon_vmax_v:
4527   case ARM::BI__builtin_neon_vmaxq_v:
4528     Int = usgn ? Intrinsic::arm_neon_vmaxu : Intrinsic::arm_neon_vmaxs;
4529     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
4530   case ARM::BI__builtin_neon_vmin_v:
4531   case ARM::BI__builtin_neon_vminq_v:
4532     Int = usgn ? Intrinsic::arm_neon_vminu : Intrinsic::arm_neon_vmins;
4533     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
4534   case ARM::BI__builtin_neon_vmovl_v: {
4535     llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
4536     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
4537     if (usgn)
4538       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
4539     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
4540   }
4541   case ARM::BI__builtin_neon_vmovn_v: {
4542     llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4543     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
4544     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
4545   }
4546   case ARM::BI__builtin_neon_vmul_v:
4547   case ARM::BI__builtin_neon_vmulq_v:
4548     assert(Type.isPoly() && "vmul builtin only supported for polynomial types");
4549     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmulp, Ty),
4550                         Ops, "vmul");
4551   case ARM::BI__builtin_neon_vmull_v:
4552     // FIXME: the integer vmull operations could be emitted in terms of pure
4553     // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
4554     // hoisting the exts outside loops. Until global ISel comes along that can
4555     // see through such movement this leads to bad CodeGen. So we need an
4556     // intrinsic for now.
4557     Int = usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
4558     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
4559     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
4560   case ARM::BI__builtin_neon_vfma_v:
4561   case ARM::BI__builtin_neon_vfmaq_v: {
4562     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4563     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4564     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4565     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4566 
4567     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
4568     return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
4569   }
4570   case ARM::BI__builtin_neon_vpadal_v:
4571   case ARM::BI__builtin_neon_vpadalq_v: {
4572     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
4573     // The source operand type has twice as many elements of half the size.
4574     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4575     llvm::Type *EltTy =
4576       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4577     llvm::Type *NarrowTy =
4578       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4579     llvm::Type *Tys[2] = { Ty, NarrowTy };
4580     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpadal");
4581   }
4582   case ARM::BI__builtin_neon_vpadd_v:
4583     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vpadd, Ty),
4584                         Ops, "vpadd");
4585   case ARM::BI__builtin_neon_vpaddl_v:
4586   case ARM::BI__builtin_neon_vpaddlq_v: {
4587     Int = usgn ? Intrinsic::arm_neon_vpaddlu : Intrinsic::arm_neon_vpaddls;
4588     // The source operand type has twice as many elements of half the size.
4589     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4590     llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
4591     llvm::Type *NarrowTy =
4592       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4593     llvm::Type *Tys[2] = { Ty, NarrowTy };
4594     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
4595   }
4596   case ARM::BI__builtin_neon_vpmax_v:
4597     Int = usgn ? Intrinsic::arm_neon_vpmaxu : Intrinsic::arm_neon_vpmaxs;
4598     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
4599   case ARM::BI__builtin_neon_vpmin_v:
4600     Int = usgn ? Intrinsic::arm_neon_vpminu : Intrinsic::arm_neon_vpmins;
4601     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
4602   case ARM::BI__builtin_neon_vqabs_v:
4603   case ARM::BI__builtin_neon_vqabsq_v:
4604     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqabs, Ty),
4605                         Ops, "vqabs");
4606   case ARM::BI__builtin_neon_vqadd_v:
4607   case ARM::BI__builtin_neon_vqaddq_v:
4608     Int = usgn ? Intrinsic::arm_neon_vqaddu : Intrinsic::arm_neon_vqadds;
4609     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqadd");
4610   case ARM::BI__builtin_neon_vqdmlal_v: {
4611     SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
4612     Value *Mul = EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, Ty),
4613                               MulOps, "vqdmlal");
4614 
4615     SmallVector<Value *, 2> AddOps;
4616     AddOps.push_back(Ops[0]);
4617     AddOps.push_back(Mul);
4618     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqadds, Ty),
4619                         AddOps, "vqdmlal");
4620   }
4621   case ARM::BI__builtin_neon_vqdmlsl_v: {
4622     SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
4623     Value *Mul = EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, Ty),
4624                               MulOps, "vqdmlsl");
4625 
4626     SmallVector<Value *, 2> SubOps;
4627     SubOps.push_back(Ops[0]);
4628     SubOps.push_back(Mul);
4629     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqsubs, Ty),
4630                         SubOps, "vqdmlsl");
4631   }
4632   case ARM::BI__builtin_neon_vqdmulh_v:
4633   case ARM::BI__builtin_neon_vqdmulhq_v:
4634     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmulh, Ty),
4635                         Ops, "vqdmulh");
4636   case ARM::BI__builtin_neon_vqdmull_v:
4637     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, Ty),
4638                         Ops, "vqdmull");
4639   case ARM::BI__builtin_neon_vqmovn_v:
4640     Int = usgn ? Intrinsic::arm_neon_vqmovnu : Intrinsic::arm_neon_vqmovns;
4641     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqmovn");
4642   case ARM::BI__builtin_neon_vqmovun_v:
4643     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqmovnsu, Ty),
4644                         Ops, "vqdmull");
4645   case ARM::BI__builtin_neon_vqneg_v:
4646   case ARM::BI__builtin_neon_vqnegq_v:
4647     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqneg, Ty),
4648                         Ops, "vqneg");
4649   case ARM::BI__builtin_neon_vqrdmulh_v:
4650   case ARM::BI__builtin_neon_vqrdmulhq_v:
4651     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrdmulh, Ty),
4652                         Ops, "vqrdmulh");
4653   case ARM::BI__builtin_neon_vqrshl_v:
4654   case ARM::BI__builtin_neon_vqrshlq_v:
4655     Int = usgn ? Intrinsic::arm_neon_vqrshiftu : Intrinsic::arm_neon_vqrshifts;
4656     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshl");
4657   case ARM::BI__builtin_neon_vqrshrn_n_v:
4658     Int =
4659       usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
4660     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
4661                         1, true);
4662   case ARM::BI__builtin_neon_vqrshrun_n_v:
4663     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
4664                         Ops, "vqrshrun_n", 1, true);
4665   case ARM::BI__builtin_neon_vqshl_v:
4666   case ARM::BI__builtin_neon_vqshlq_v:
4667     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
4668     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl");
4669   case ARM::BI__builtin_neon_vqshl_n_v:
4670   case ARM::BI__builtin_neon_vqshlq_n_v:
4671     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
4672     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
4673                         1, false);
4674   case ARM::BI__builtin_neon_vqshlu_n_v:
4675   case ARM::BI__builtin_neon_vqshluq_n_v:
4676     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftsu, Ty),
4677                         Ops, "vqshlu", 1, false);
4678   case ARM::BI__builtin_neon_vqshrn_n_v:
4679     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
4680     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
4681                         1, true);
4682   case ARM::BI__builtin_neon_vqshrun_n_v:
4683     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
4684                         Ops, "vqshrun_n", 1, true);
4685   case ARM::BI__builtin_neon_vqsub_v:
4686   case ARM::BI__builtin_neon_vqsubq_v:
4687     Int = usgn ? Intrinsic::arm_neon_vqsubu : Intrinsic::arm_neon_vqsubs;
4688     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqsub");
4689   case ARM::BI__builtin_neon_vraddhn_v:
4690     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vraddhn, Ty),
4691                         Ops, "vraddhn");
4692   case ARM::BI__builtin_neon_vrecpe_v:
4693   case ARM::BI__builtin_neon_vrecpeq_v:
4694     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
4695                         Ops, "vrecpe");
4696   case ARM::BI__builtin_neon_vrecps_v:
4697   case ARM::BI__builtin_neon_vrecpsq_v:
4698     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecps, Ty),
4699                         Ops, "vrecps");
4700   case ARM::BI__builtin_neon_vrhadd_v:
4701   case ARM::BI__builtin_neon_vrhaddq_v:
4702     Int = usgn ? Intrinsic::arm_neon_vrhaddu : Intrinsic::arm_neon_vrhadds;
4703     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrhadd");
4704   case ARM::BI__builtin_neon_vrshl_v:
4705   case ARM::BI__builtin_neon_vrshlq_v:
4706     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
4707     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshl");
4708   case ARM::BI__builtin_neon_vrshrn_n_v:
4709     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
4710                         Ops, "vrshrn_n", 1, true);
4711   case ARM::BI__builtin_neon_vrshr_n_v:
4712   case ARM::BI__builtin_neon_vrshrq_n_v:
4713     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
4714     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
4715   case ARM::BI__builtin_neon_vrsqrte_v:
4716   case ARM::BI__builtin_neon_vrsqrteq_v:
4717     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsqrte, Ty),
4718                         Ops, "vrsqrte");
4719   case ARM::BI__builtin_neon_vrsqrts_v:
4720   case ARM::BI__builtin_neon_vrsqrtsq_v:
4721     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsqrts, Ty),
4722                         Ops, "vrsqrts");
4723   case ARM::BI__builtin_neon_vrsra_n_v:
4724   case ARM::BI__builtin_neon_vrsraq_n_v:
4725     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4726     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4727     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
4728     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
4729     Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
4730     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
4731   case ARM::BI__builtin_neon_vrsubhn_v:
4732     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsubhn, Ty),
4733                         Ops, "vrsubhn");
4734   case ARM::BI__builtin_neon_vshl_v:
4735   case ARM::BI__builtin_neon_vshlq_v:
4736     Int = usgn ? Intrinsic::arm_neon_vshiftu : Intrinsic::arm_neon_vshifts;
4737     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vshl");
4738   case ARM::BI__builtin_neon_vshll_n_v:
4739     Int = usgn ? Intrinsic::arm_neon_vshiftlu : Intrinsic::arm_neon_vshiftls;
4740     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vshll", 1);
4741   case ARM::BI__builtin_neon_vshl_n_v:
4742   case ARM::BI__builtin_neon_vshlq_n_v:
4743     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
4744     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
4745                              "vshl_n");
4746   case ARM::BI__builtin_neon_vshrn_n_v:
4747     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftn, Ty),
4748                         Ops, "vshrn_n", 1, true);
4749   case ARM::BI__builtin_neon_vshr_n_v:
4750   case ARM::BI__builtin_neon_vshrq_n_v:
4751     return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, usgn, "vshr_n");
4752   case ARM::BI__builtin_neon_vsri_n_v:
4753   case ARM::BI__builtin_neon_vsriq_n_v:
4754     rightShift = true;
4755   case ARM::BI__builtin_neon_vsli_n_v:
4756   case ARM::BI__builtin_neon_vsliq_n_v:
4757     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
4758     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
4759                         Ops, "vsli_n");
4760   case ARM::BI__builtin_neon_vsra_n_v:
4761   case ARM::BI__builtin_neon_vsraq_n_v:
4762     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4763     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
4764     return Builder.CreateAdd(Ops[0], Ops[1]);
4765   case ARM::BI__builtin_neon_vst1_v:
4766   case ARM::BI__builtin_neon_vst1q_v:
4767     Ops.push_back(Align);
4768     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, Ty),
4769                         Ops, "");
4770   case ARM::BI__builtin_neon_vst1q_lane_v:
4771     // Handle 64-bit integer elements as a special case.  Use a shuffle to get
4772     // a one-element vector and avoid poor code for i64 in the backend.
4773     if (VTy->getElementType()->isIntegerTy(64)) {
4774       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4775       Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
4776       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4777       Ops[2] = Align;
4778       return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
4779                                                  Ops[1]->getType()), Ops);
4780     }
4781     // fall through
4782   case ARM::BI__builtin_neon_vst1_lane_v: {
4783     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4784     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
4785     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4786     StoreInst *St = Builder.CreateStore(Ops[1],
4787                                         Builder.CreateBitCast(Ops[0], Ty));
4788     St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
4789     return St;
4790   }
4791   case ARM::BI__builtin_neon_vst2_v:
4792   case ARM::BI__builtin_neon_vst2q_v:
4793     Ops.push_back(Align);
4794     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2, Ty),
4795                         Ops, "");
4796   case ARM::BI__builtin_neon_vst2_lane_v:
4797   case ARM::BI__builtin_neon_vst2q_lane_v:
4798     Ops.push_back(Align);
4799     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2lane, Ty),
4800                         Ops, "");
4801   case ARM::BI__builtin_neon_vst3_v:
4802   case ARM::BI__builtin_neon_vst3q_v:
4803     Ops.push_back(Align);
4804     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3, Ty),
4805                         Ops, "");
4806   case ARM::BI__builtin_neon_vst3_lane_v:
4807   case ARM::BI__builtin_neon_vst3q_lane_v:
4808     Ops.push_back(Align);
4809     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3lane, Ty),
4810                         Ops, "");
4811   case ARM::BI__builtin_neon_vst4_v:
4812   case ARM::BI__builtin_neon_vst4q_v:
4813     Ops.push_back(Align);
4814     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4, Ty),
4815                         Ops, "");
4816   case ARM::BI__builtin_neon_vst4_lane_v:
4817   case ARM::BI__builtin_neon_vst4q_lane_v:
4818     Ops.push_back(Align);
4819     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4lane, Ty),
4820                         Ops, "");
4821   case ARM::BI__builtin_neon_vsubhn_v: {
4822     llvm::VectorType *SrcTy =
4823         llvm::VectorType::getExtendedElementVectorType(VTy);
4824 
4825     // %sum = add <4 x i32> %lhs, %rhs
4826     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4827     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4828     Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
4829 
4830     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4831     Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
4832                                        SrcTy->getScalarSizeInBits() / 2);
4833     ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
4834     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
4835 
4836     // %res = trunc <4 x i32> %high to <4 x i16>
4837     return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
4838   }
4839   case ARM::BI__builtin_neon_vtbl1_v:
4840     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
4841                         Ops, "vtbl1");
4842   case ARM::BI__builtin_neon_vtbl2_v:
4843     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
4844                         Ops, "vtbl2");
4845   case ARM::BI__builtin_neon_vtbl3_v:
4846     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
4847                         Ops, "vtbl3");
4848   case ARM::BI__builtin_neon_vtbl4_v:
4849     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
4850                         Ops, "vtbl4");
4851   case ARM::BI__builtin_neon_vtbx1_v:
4852     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
4853                         Ops, "vtbx1");
4854   case ARM::BI__builtin_neon_vtbx2_v:
4855     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
4856                         Ops, "vtbx2");
4857   case ARM::BI__builtin_neon_vtbx3_v:
4858     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
4859                         Ops, "vtbx3");
4860   case ARM::BI__builtin_neon_vtbx4_v:
4861     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
4862                         Ops, "vtbx4");
4863   case ARM::BI__builtin_neon_vtst_v:
4864   case ARM::BI__builtin_neon_vtstq_v: {
4865     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4866     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4867     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4868     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4869                                 ConstantAggregateZero::get(Ty));
4870     return Builder.CreateSExt(Ops[0], Ty, "vtst");
4871   }
4872   case ARM::BI__builtin_neon_vtrn_v:
4873   case ARM::BI__builtin_neon_vtrnq_v: {
4874     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4875     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4876     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4877     Value *SV = 0;
4878 
4879     for (unsigned vi = 0; vi != 2; ++vi) {
4880       SmallVector<Constant*, 16> Indices;
4881       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4882         Indices.push_back(Builder.getInt32(i+vi));
4883         Indices.push_back(Builder.getInt32(i+e+vi));
4884       }
4885       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
4886       SV = llvm::ConstantVector::get(Indices);
4887       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
4888       SV = Builder.CreateStore(SV, Addr);
4889     }
4890     return SV;
4891   }
4892   case ARM::BI__builtin_neon_vuzp_v:
4893   case ARM::BI__builtin_neon_vuzpq_v: {
4894     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4895     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4896     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4897     Value *SV = 0;
4898 
4899     for (unsigned vi = 0; vi != 2; ++vi) {
4900       SmallVector<Constant*, 16> Indices;
4901       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4902         Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
4903 
4904       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
4905       SV = llvm::ConstantVector::get(Indices);
4906       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
4907       SV = Builder.CreateStore(SV, Addr);
4908     }
4909     return SV;
4910   }
4911   case ARM::BI__builtin_neon_vzip_v:
4912   case ARM::BI__builtin_neon_vzipq_v: {
4913     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4914     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4915     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4916     Value *SV = 0;
4917 
4918     for (unsigned vi = 0; vi != 2; ++vi) {
4919       SmallVector<Constant*, 16> Indices;
4920       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4921         Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
4922         Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
4923       }
4924       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
4925       SV = llvm::ConstantVector::get(Indices);
4926       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
4927       SV = Builder.CreateStore(SV, Addr);
4928     }
4929     return SV;
4930   }
4931   }
4932 }
4933 
4934 llvm::Value *CodeGenFunction::
4935 BuildVector(ArrayRef<llvm::Value*> Ops) {
4936   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
4937          "Not a power-of-two sized vector!");
4938   bool AllConstants = true;
4939   for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
4940     AllConstants &= isa<Constant>(Ops[i]);
4941 
4942   // If this is a constant vector, create a ConstantVector.
4943   if (AllConstants) {
4944     SmallVector<llvm::Constant*, 16> CstOps;
4945     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
4946       CstOps.push_back(cast<Constant>(Ops[i]));
4947     return llvm::ConstantVector::get(CstOps);
4948   }
4949 
4950   // Otherwise, insertelement the values to build the vector.
4951   Value *Result =
4952     llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
4953 
4954   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
4955     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
4956 
4957   return Result;
4958 }
4959 
4960 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
4961                                            const CallExpr *E) {
4962   SmallVector<Value*, 4> Ops;
4963 
4964   // Find out if any arguments are required to be integer constant expressions.
4965   unsigned ICEArguments = 0;
4966   ASTContext::GetBuiltinTypeError Error;
4967   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
4968   assert(Error == ASTContext::GE_None && "Should not codegen an error");
4969 
4970   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
4971     // If this is a normal argument, just emit it as a scalar.
4972     if ((ICEArguments & (1 << i)) == 0) {
4973       Ops.push_back(EmitScalarExpr(E->getArg(i)));
4974       continue;
4975     }
4976 
4977     // If this is required to be a constant, constant fold it so that we know
4978     // that the generated intrinsic gets a ConstantInt.
4979     llvm::APSInt Result;
4980     bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
4981     assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
4982     Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
4983   }
4984 
4985   switch (BuiltinID) {
4986   default: return 0;
4987   case X86::BI__builtin_ia32_vec_init_v8qi:
4988   case X86::BI__builtin_ia32_vec_init_v4hi:
4989   case X86::BI__builtin_ia32_vec_init_v2si:
4990     return Builder.CreateBitCast(BuildVector(Ops),
4991                                  llvm::Type::getX86_MMXTy(getLLVMContext()));
4992   case X86::BI__builtin_ia32_vec_ext_v2si:
4993     return Builder.CreateExtractElement(Ops[0],
4994                                   llvm::ConstantInt::get(Ops[1]->getType(), 0));
4995   case X86::BI__builtin_ia32_ldmxcsr: {
4996     Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
4997     Builder.CreateStore(Ops[0], Tmp);
4998     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
4999                               Builder.CreateBitCast(Tmp, Int8PtrTy));
5000   }
5001   case X86::BI__builtin_ia32_stmxcsr: {
5002     Value *Tmp = CreateMemTemp(E->getType());
5003     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
5004                        Builder.CreateBitCast(Tmp, Int8PtrTy));
5005     return Builder.CreateLoad(Tmp, "stmxcsr");
5006   }
5007   case X86::BI__builtin_ia32_storehps:
5008   case X86::BI__builtin_ia32_storelps: {
5009     llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
5010     llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
5011 
5012     // cast val v2i64
5013     Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
5014 
5015     // extract (0, 1)
5016     unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
5017     llvm::Value *Idx = llvm::ConstantInt::get(Int32Ty, Index);
5018     Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract");
5019 
5020     // cast pointer to i64 & store
5021     Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
5022     return Builder.CreateStore(Ops[1], Ops[0]);
5023   }
5024   case X86::BI__builtin_ia32_palignr: {
5025     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5026 
5027     // If palignr is shifting the pair of input vectors less than 9 bytes,
5028     // emit a shuffle instruction.
5029     if (shiftVal <= 8) {
5030       SmallVector<llvm::Constant*, 8> Indices;
5031       for (unsigned i = 0; i != 8; ++i)
5032         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
5033 
5034       Value* SV = llvm::ConstantVector::get(Indices);
5035       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5036     }
5037 
5038     // If palignr is shifting the pair of input vectors more than 8 but less
5039     // than 16 bytes, emit a logical right shift of the destination.
5040     if (shiftVal < 16) {
5041       // MMX has these as 1 x i64 vectors for some odd optimization reasons.
5042       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
5043 
5044       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5045       Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
5046 
5047       // create i32 constant
5048       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
5049       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5050     }
5051 
5052     // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
5053     return llvm::Constant::getNullValue(ConvertType(E->getType()));
5054   }
5055   case X86::BI__builtin_ia32_palignr128: {
5056     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5057 
5058     // If palignr is shifting the pair of input vectors less than 17 bytes,
5059     // emit a shuffle instruction.
5060     if (shiftVal <= 16) {
5061       SmallVector<llvm::Constant*, 16> Indices;
5062       for (unsigned i = 0; i != 16; ++i)
5063         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
5064 
5065       Value* SV = llvm::ConstantVector::get(Indices);
5066       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5067     }
5068 
5069     // If palignr is shifting the pair of input vectors more than 16 but less
5070     // than 32 bytes, emit a logical right shift of the destination.
5071     if (shiftVal < 32) {
5072       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
5073 
5074       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5075       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5076 
5077       // create i32 constant
5078       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
5079       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5080     }
5081 
5082     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
5083     return llvm::Constant::getNullValue(ConvertType(E->getType()));
5084   }
5085   case X86::BI__builtin_ia32_palignr256: {
5086     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5087 
5088     // If palignr is shifting the pair of input vectors less than 17 bytes,
5089     // emit a shuffle instruction.
5090     if (shiftVal <= 16) {
5091       SmallVector<llvm::Constant*, 32> Indices;
5092       // 256-bit palignr operates on 128-bit lanes so we need to handle that
5093       for (unsigned l = 0; l != 2; ++l) {
5094         unsigned LaneStart = l * 16;
5095         unsigned LaneEnd = (l+1) * 16;
5096         for (unsigned i = 0; i != 16; ++i) {
5097           unsigned Idx = shiftVal + i + LaneStart;
5098           if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand
5099           Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx));
5100         }
5101       }
5102 
5103       Value* SV = llvm::ConstantVector::get(Indices);
5104       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5105     }
5106 
5107     // If palignr is shifting the pair of input vectors more than 16 but less
5108     // than 32 bytes, emit a logical right shift of the destination.
5109     if (shiftVal < 32) {
5110       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4);
5111 
5112       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5113       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5114 
5115       // create i32 constant
5116       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq);
5117       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5118     }
5119 
5120     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
5121     return llvm::Constant::getNullValue(ConvertType(E->getType()));
5122   }
5123   case X86::BI__builtin_ia32_movntps:
5124   case X86::BI__builtin_ia32_movntps256:
5125   case X86::BI__builtin_ia32_movntpd:
5126   case X86::BI__builtin_ia32_movntpd256:
5127   case X86::BI__builtin_ia32_movntdq:
5128   case X86::BI__builtin_ia32_movntdq256:
5129   case X86::BI__builtin_ia32_movnti:
5130   case X86::BI__builtin_ia32_movnti64: {
5131     llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(),
5132                                            Builder.getInt32(1));
5133 
5134     // Convert the type of the pointer to a pointer to the stored type.
5135     Value *BC = Builder.CreateBitCast(Ops[0],
5136                                 llvm::PointerType::getUnqual(Ops[1]->getType()),
5137                                       "cast");
5138     StoreInst *SI = Builder.CreateStore(Ops[1], BC);
5139     SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
5140 
5141     // If the operand is an integer, we can't assume alignment. Otherwise,
5142     // assume natural alignment.
5143     QualType ArgTy = E->getArg(1)->getType();
5144     unsigned Align;
5145     if (ArgTy->isIntegerType())
5146       Align = 1;
5147     else
5148       Align = getContext().getTypeSizeInChars(ArgTy).getQuantity();
5149     SI->setAlignment(Align);
5150     return SI;
5151   }
5152   // 3DNow!
5153   case X86::BI__builtin_ia32_pswapdsf:
5154   case X86::BI__builtin_ia32_pswapdsi: {
5155     const char *name = 0;
5156     Intrinsic::ID ID = Intrinsic::not_intrinsic;
5157     switch(BuiltinID) {
5158     default: llvm_unreachable("Unsupported intrinsic!");
5159     case X86::BI__builtin_ia32_pswapdsf:
5160     case X86::BI__builtin_ia32_pswapdsi:
5161       name = "pswapd";
5162       ID = Intrinsic::x86_3dnowa_pswapd;
5163       break;
5164     }
5165     llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
5166     Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
5167     llvm::Function *F = CGM.getIntrinsic(ID);
5168     return Builder.CreateCall(F, Ops, name);
5169   }
5170   case X86::BI__builtin_ia32_rdrand16_step:
5171   case X86::BI__builtin_ia32_rdrand32_step:
5172   case X86::BI__builtin_ia32_rdrand64_step:
5173   case X86::BI__builtin_ia32_rdseed16_step:
5174   case X86::BI__builtin_ia32_rdseed32_step:
5175   case X86::BI__builtin_ia32_rdseed64_step: {
5176     Intrinsic::ID ID;
5177     switch (BuiltinID) {
5178     default: llvm_unreachable("Unsupported intrinsic!");
5179     case X86::BI__builtin_ia32_rdrand16_step:
5180       ID = Intrinsic::x86_rdrand_16;
5181       break;
5182     case X86::BI__builtin_ia32_rdrand32_step:
5183       ID = Intrinsic::x86_rdrand_32;
5184       break;
5185     case X86::BI__builtin_ia32_rdrand64_step:
5186       ID = Intrinsic::x86_rdrand_64;
5187       break;
5188     case X86::BI__builtin_ia32_rdseed16_step:
5189       ID = Intrinsic::x86_rdseed_16;
5190       break;
5191     case X86::BI__builtin_ia32_rdseed32_step:
5192       ID = Intrinsic::x86_rdseed_32;
5193       break;
5194     case X86::BI__builtin_ia32_rdseed64_step:
5195       ID = Intrinsic::x86_rdseed_64;
5196       break;
5197     }
5198 
5199     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
5200     Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
5201     return Builder.CreateExtractValue(Call, 1);
5202   }
5203   // AVX2 broadcast
5204   case X86::BI__builtin_ia32_vbroadcastsi256: {
5205     Value *VecTmp = CreateMemTemp(E->getArg(0)->getType());
5206     Builder.CreateStore(Ops[0], VecTmp);
5207     Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128);
5208     return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy));
5209   }
5210   }
5211 }
5212 
5213 
5214 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
5215                                            const CallExpr *E) {
5216   SmallVector<Value*, 4> Ops;
5217 
5218   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
5219     Ops.push_back(EmitScalarExpr(E->getArg(i)));
5220 
5221   Intrinsic::ID ID = Intrinsic::not_intrinsic;
5222 
5223   switch (BuiltinID) {
5224   default: return 0;
5225 
5226   // vec_ld, vec_lvsl, vec_lvsr
5227   case PPC::BI__builtin_altivec_lvx:
5228   case PPC::BI__builtin_altivec_lvxl:
5229   case PPC::BI__builtin_altivec_lvebx:
5230   case PPC::BI__builtin_altivec_lvehx:
5231   case PPC::BI__builtin_altivec_lvewx:
5232   case PPC::BI__builtin_altivec_lvsl:
5233   case PPC::BI__builtin_altivec_lvsr:
5234   {
5235     Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
5236 
5237     Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
5238     Ops.pop_back();
5239 
5240     switch (BuiltinID) {
5241     default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
5242     case PPC::BI__builtin_altivec_lvx:
5243       ID = Intrinsic::ppc_altivec_lvx;
5244       break;
5245     case PPC::BI__builtin_altivec_lvxl:
5246       ID = Intrinsic::ppc_altivec_lvxl;
5247       break;
5248     case PPC::BI__builtin_altivec_lvebx:
5249       ID = Intrinsic::ppc_altivec_lvebx;
5250       break;
5251     case PPC::BI__builtin_altivec_lvehx:
5252       ID = Intrinsic::ppc_altivec_lvehx;
5253       break;
5254     case PPC::BI__builtin_altivec_lvewx:
5255       ID = Intrinsic::ppc_altivec_lvewx;
5256       break;
5257     case PPC::BI__builtin_altivec_lvsl:
5258       ID = Intrinsic::ppc_altivec_lvsl;
5259       break;
5260     case PPC::BI__builtin_altivec_lvsr:
5261       ID = Intrinsic::ppc_altivec_lvsr;
5262       break;
5263     }
5264     llvm::Function *F = CGM.getIntrinsic(ID);
5265     return Builder.CreateCall(F, Ops, "");
5266   }
5267 
5268   // vec_st
5269   case PPC::BI__builtin_altivec_stvx:
5270   case PPC::BI__builtin_altivec_stvxl:
5271   case PPC::BI__builtin_altivec_stvebx:
5272   case PPC::BI__builtin_altivec_stvehx:
5273   case PPC::BI__builtin_altivec_stvewx:
5274   {
5275     Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
5276     Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
5277     Ops.pop_back();
5278 
5279     switch (BuiltinID) {
5280     default: llvm_unreachable("Unsupported st intrinsic!");
5281     case PPC::BI__builtin_altivec_stvx:
5282       ID = Intrinsic::ppc_altivec_stvx;
5283       break;
5284     case PPC::BI__builtin_altivec_stvxl:
5285       ID = Intrinsic::ppc_altivec_stvxl;
5286       break;
5287     case PPC::BI__builtin_altivec_stvebx:
5288       ID = Intrinsic::ppc_altivec_stvebx;
5289       break;
5290     case PPC::BI__builtin_altivec_stvehx:
5291       ID = Intrinsic::ppc_altivec_stvehx;
5292       break;
5293     case PPC::BI__builtin_altivec_stvewx:
5294       ID = Intrinsic::ppc_altivec_stvewx;
5295       break;
5296     }
5297     llvm::Function *F = CGM.getIntrinsic(ID);
5298     return Builder.CreateCall(F, Ops, "");
5299   }
5300   }
5301 }
5302