1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit Builtin calls as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "TargetInfo.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "CGObjCRuntime.h"
18 #include "clang/Basic/TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
20 #include "clang/AST/Decl.h"
21 #include "clang/Basic/TargetBuiltins.h"
22 #include "llvm/Intrinsics.h"
23 #include "llvm/Target/TargetData.h"
24 
25 using namespace clang;
26 using namespace CodeGen;
27 using namespace llvm;
28 
29 /// getBuiltinLibFunction - Given a builtin id for a function like
30 /// "__builtin_fabsf", return a Function* for "fabsf".
31 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
32                                                   unsigned BuiltinID) {
33   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
34 
35   // Get the name, skip over the __builtin_ prefix (if necessary).
36   StringRef Name;
37   GlobalDecl D(FD);
38 
39   // If the builtin has been declared explicitly with an assembler label,
40   // use the mangled name. This differs from the plain label on platforms
41   // that prefix labels.
42   if (FD->hasAttr<AsmLabelAttr>())
43     Name = getMangledName(D);
44   else
45     Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
46 
47   llvm::FunctionType *Ty =
48     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
49 
50   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
51 }
52 
53 /// Emit the conversions required to turn the given value into an
54 /// integer of the given size.
55 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
56                         QualType T, llvm::IntegerType *IntType) {
57   V = CGF.EmitToMemory(V, T);
58 
59   if (V->getType()->isPointerTy())
60     return CGF.Builder.CreatePtrToInt(V, IntType);
61 
62   assert(V->getType() == IntType);
63   return V;
64 }
65 
66 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
67                           QualType T, llvm::Type *ResultType) {
68   V = CGF.EmitFromMemory(V, T);
69 
70   if (ResultType->isPointerTy())
71     return CGF.Builder.CreateIntToPtr(V, ResultType);
72 
73   assert(V->getType() == ResultType);
74   return V;
75 }
76 
77 /// Utility to insert an atomic instruction based on Instrinsic::ID
78 /// and the expression node.
79 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
80                                llvm::AtomicRMWInst::BinOp Kind,
81                                const CallExpr *E) {
82   QualType T = E->getType();
83   assert(E->getArg(0)->getType()->isPointerType());
84   assert(CGF.getContext().hasSameUnqualifiedType(T,
85                                   E->getArg(0)->getType()->getPointeeType()));
86   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
87 
88   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
89   unsigned AddrSpace =
90     cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
91 
92   llvm::IntegerType *IntType =
93     llvm::IntegerType::get(CGF.getLLVMContext(),
94                            CGF.getContext().getTypeSize(T));
95   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
96 
97   llvm::Value *Args[2];
98   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
99   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
100   llvm::Type *ValueType = Args[1]->getType();
101   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
102 
103   llvm::Value *Result =
104       CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
105                                   llvm::SequentiallyConsistent);
106   Result = EmitFromInt(CGF, Result, T, ValueType);
107   return RValue::get(Result);
108 }
109 
110 /// Utility to insert an atomic instruction based Instrinsic::ID and
111 /// the expression node, where the return value is the result of the
112 /// operation.
113 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
114                                    llvm::AtomicRMWInst::BinOp Kind,
115                                    const CallExpr *E,
116                                    Instruction::BinaryOps Op) {
117   QualType T = E->getType();
118   assert(E->getArg(0)->getType()->isPointerType());
119   assert(CGF.getContext().hasSameUnqualifiedType(T,
120                                   E->getArg(0)->getType()->getPointeeType()));
121   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
122 
123   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
124   unsigned AddrSpace =
125     cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
126 
127   llvm::IntegerType *IntType =
128     llvm::IntegerType::get(CGF.getLLVMContext(),
129                            CGF.getContext().getTypeSize(T));
130   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
131 
132   llvm::Value *Args[2];
133   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
134   llvm::Type *ValueType = Args[1]->getType();
135   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
136   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
137 
138   llvm::Value *Result =
139       CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
140                                   llvm::SequentiallyConsistent);
141   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
142   Result = EmitFromInt(CGF, Result, T, ValueType);
143   return RValue::get(Result);
144 }
145 
146 /// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy,
147 /// which must be a scalar floating point type.
148 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
149   const BuiltinType *ValTyP = ValTy->getAs<BuiltinType>();
150   assert(ValTyP && "isn't scalar fp type!");
151 
152   StringRef FnName;
153   switch (ValTyP->getKind()) {
154   default: llvm_unreachable("Isn't a scalar fp type!");
155   case BuiltinType::Float:      FnName = "fabsf"; break;
156   case BuiltinType::Double:     FnName = "fabs"; break;
157   case BuiltinType::LongDouble: FnName = "fabsl"; break;
158   }
159 
160   // The prototype is something that takes and returns whatever V's type is.
161   llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(),
162                                                    false);
163   llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction(FT, FnName);
164 
165   return CGF.Builder.CreateCall(Fn, V, "abs");
166 }
167 
168 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn,
169                               const CallExpr *E, llvm::Value *calleeValue) {
170   return CGF.EmitCall(E->getCallee()->getType(), calleeValue,
171                       ReturnValueSlot(), E->arg_begin(), E->arg_end(), Fn);
172 }
173 
174 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
175                                         unsigned BuiltinID, const CallExpr *E) {
176   // See if we can constant fold this builtin.  If so, don't emit it at all.
177   Expr::EvalResult Result;
178   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
179       !Result.hasSideEffects()) {
180     if (Result.Val.isInt())
181       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
182                                                 Result.Val.getInt()));
183     if (Result.Val.isFloat())
184       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
185                                                Result.Val.getFloat()));
186   }
187 
188   switch (BuiltinID) {
189   default: break;  // Handle intrinsics and libm functions below.
190   case Builtin::BI__builtin___CFStringMakeConstantString:
191   case Builtin::BI__builtin___NSStringMakeConstantString:
192     return RValue::get(CGM.EmitConstantExpr(E, E->getType(), 0));
193   case Builtin::BI__builtin_stdarg_start:
194   case Builtin::BI__builtin_va_start:
195   case Builtin::BI__builtin_va_end: {
196     Value *ArgValue = EmitVAListRef(E->getArg(0));
197     llvm::Type *DestType = Int8PtrTy;
198     if (ArgValue->getType() != DestType)
199       ArgValue = Builder.CreateBitCast(ArgValue, DestType,
200                                        ArgValue->getName().data());
201 
202     Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ?
203       Intrinsic::vaend : Intrinsic::vastart;
204     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue));
205   }
206   case Builtin::BI__builtin_va_copy: {
207     Value *DstPtr = EmitVAListRef(E->getArg(0));
208     Value *SrcPtr = EmitVAListRef(E->getArg(1));
209 
210     llvm::Type *Type = Int8PtrTy;
211 
212     DstPtr = Builder.CreateBitCast(DstPtr, Type);
213     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
214     return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy),
215                                            DstPtr, SrcPtr));
216   }
217   case Builtin::BI__builtin_abs:
218   case Builtin::BI__builtin_labs:
219   case Builtin::BI__builtin_llabs: {
220     Value *ArgValue = EmitScalarExpr(E->getArg(0));
221 
222     Value *NegOp = Builder.CreateNeg(ArgValue, "neg");
223     Value *CmpResult =
224     Builder.CreateICmpSGE(ArgValue,
225                           llvm::Constant::getNullValue(ArgValue->getType()),
226                                                             "abscond");
227     Value *Result =
228       Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs");
229 
230     return RValue::get(Result);
231   }
232   case Builtin::BI__builtin_ctzs:
233   case Builtin::BI__builtin_ctz:
234   case Builtin::BI__builtin_ctzl:
235   case Builtin::BI__builtin_ctzll: {
236     Value *ArgValue = EmitScalarExpr(E->getArg(0));
237 
238     llvm::Type *ArgType = ArgValue->getType();
239     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
240 
241     llvm::Type *ResultType = ConvertType(E->getType());
242     Value *ZeroUndef = Builder.getInt1(Target.isCLZForZeroUndef());
243     Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
244     if (Result->getType() != ResultType)
245       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
246                                      "cast");
247     return RValue::get(Result);
248   }
249   case Builtin::BI__builtin_clzs:
250   case Builtin::BI__builtin_clz:
251   case Builtin::BI__builtin_clzl:
252   case Builtin::BI__builtin_clzll: {
253     Value *ArgValue = EmitScalarExpr(E->getArg(0));
254 
255     llvm::Type *ArgType = ArgValue->getType();
256     Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
257 
258     llvm::Type *ResultType = ConvertType(E->getType());
259     Value *ZeroUndef = Builder.getInt1(Target.isCLZForZeroUndef());
260     Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
261     if (Result->getType() != ResultType)
262       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
263                                      "cast");
264     return RValue::get(Result);
265   }
266   case Builtin::BI__builtin_ffs:
267   case Builtin::BI__builtin_ffsl:
268   case Builtin::BI__builtin_ffsll: {
269     // ffs(x) -> x ? cttz(x) + 1 : 0
270     Value *ArgValue = EmitScalarExpr(E->getArg(0));
271 
272     llvm::Type *ArgType = ArgValue->getType();
273     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
274 
275     llvm::Type *ResultType = ConvertType(E->getType());
276     Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue,
277                                                        Builder.getTrue()),
278                                    llvm::ConstantInt::get(ArgType, 1));
279     Value *Zero = llvm::Constant::getNullValue(ArgType);
280     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
281     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
282     if (Result->getType() != ResultType)
283       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
284                                      "cast");
285     return RValue::get(Result);
286   }
287   case Builtin::BI__builtin_parity:
288   case Builtin::BI__builtin_parityl:
289   case Builtin::BI__builtin_parityll: {
290     // parity(x) -> ctpop(x) & 1
291     Value *ArgValue = EmitScalarExpr(E->getArg(0));
292 
293     llvm::Type *ArgType = ArgValue->getType();
294     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
295 
296     llvm::Type *ResultType = ConvertType(E->getType());
297     Value *Tmp = Builder.CreateCall(F, ArgValue);
298     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
299     if (Result->getType() != ResultType)
300       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
301                                      "cast");
302     return RValue::get(Result);
303   }
304   case Builtin::BI__builtin_popcount:
305   case Builtin::BI__builtin_popcountl:
306   case Builtin::BI__builtin_popcountll: {
307     Value *ArgValue = EmitScalarExpr(E->getArg(0));
308 
309     llvm::Type *ArgType = ArgValue->getType();
310     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
311 
312     llvm::Type *ResultType = ConvertType(E->getType());
313     Value *Result = Builder.CreateCall(F, ArgValue);
314     if (Result->getType() != ResultType)
315       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
316                                      "cast");
317     return RValue::get(Result);
318   }
319   case Builtin::BI__builtin_expect: {
320     Value *ArgValue = EmitScalarExpr(E->getArg(0));
321     llvm::Type *ArgType = ArgValue->getType();
322 
323     Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
324     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
325 
326     Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue,
327                                         "expval");
328     return RValue::get(Result);
329   }
330   case Builtin::BI__builtin_bswap32:
331   case Builtin::BI__builtin_bswap64: {
332     Value *ArgValue = EmitScalarExpr(E->getArg(0));
333     llvm::Type *ArgType = ArgValue->getType();
334     Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType);
335     return RValue::get(Builder.CreateCall(F, ArgValue));
336   }
337   case Builtin::BI__builtin_object_size: {
338     // We rely on constant folding to deal with expressions with side effects.
339     assert(!E->getArg(0)->HasSideEffects(getContext()) &&
340            "should have been constant folded");
341 
342     // We pass this builtin onto the optimizer so that it can
343     // figure out the object size in more complex cases.
344     llvm::Type *ResType = ConvertType(E->getType());
345 
346     // LLVM only supports 0 and 2, make sure that we pass along that
347     // as a boolean.
348     Value *Ty = EmitScalarExpr(E->getArg(1));
349     ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
350     assert(CI);
351     uint64_t val = CI->getZExtValue();
352     CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);
353 
354     Value *F = CGM.getIntrinsic(Intrinsic::objectsize, ResType);
355     return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI));
356   }
357   case Builtin::BI__builtin_prefetch: {
358     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
359     // FIXME: Technically these constants should of type 'int', yes?
360     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
361       llvm::ConstantInt::get(Int32Ty, 0);
362     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
363       llvm::ConstantInt::get(Int32Ty, 3);
364     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
365     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
366     return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data));
367   }
368   case Builtin::BI__builtin_readcyclecounter: {
369     Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
370     return RValue::get(Builder.CreateCall(F));
371   }
372   case Builtin::BI__builtin_trap: {
373     Value *F = CGM.getIntrinsic(Intrinsic::trap);
374     return RValue::get(Builder.CreateCall(F));
375   }
376   case Builtin::BI__builtin_unreachable: {
377     if (CatchUndefined)
378       EmitBranch(getTrapBB());
379     else
380       Builder.CreateUnreachable();
381 
382     // We do need to preserve an insertion point.
383     EmitBlock(createBasicBlock("unreachable.cont"));
384 
385     return RValue::get(0);
386   }
387 
388   case Builtin::BI__builtin_powi:
389   case Builtin::BI__builtin_powif:
390   case Builtin::BI__builtin_powil: {
391     Value *Base = EmitScalarExpr(E->getArg(0));
392     Value *Exponent = EmitScalarExpr(E->getArg(1));
393     llvm::Type *ArgType = Base->getType();
394     Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
395     return RValue::get(Builder.CreateCall2(F, Base, Exponent));
396   }
397 
398   case Builtin::BI__builtin_isgreater:
399   case Builtin::BI__builtin_isgreaterequal:
400   case Builtin::BI__builtin_isless:
401   case Builtin::BI__builtin_islessequal:
402   case Builtin::BI__builtin_islessgreater:
403   case Builtin::BI__builtin_isunordered: {
404     // Ordered comparisons: we know the arguments to these are matching scalar
405     // floating point values.
406     Value *LHS = EmitScalarExpr(E->getArg(0));
407     Value *RHS = EmitScalarExpr(E->getArg(1));
408 
409     switch (BuiltinID) {
410     default: llvm_unreachable("Unknown ordered comparison");
411     case Builtin::BI__builtin_isgreater:
412       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
413       break;
414     case Builtin::BI__builtin_isgreaterequal:
415       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
416       break;
417     case Builtin::BI__builtin_isless:
418       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
419       break;
420     case Builtin::BI__builtin_islessequal:
421       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
422       break;
423     case Builtin::BI__builtin_islessgreater:
424       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
425       break;
426     case Builtin::BI__builtin_isunordered:
427       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
428       break;
429     }
430     // ZExt bool to int type.
431     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
432   }
433   case Builtin::BI__builtin_isnan: {
434     Value *V = EmitScalarExpr(E->getArg(0));
435     V = Builder.CreateFCmpUNO(V, V, "cmp");
436     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
437   }
438 
439   case Builtin::BI__builtin_isinf: {
440     // isinf(x) --> fabs(x) == infinity
441     Value *V = EmitScalarExpr(E->getArg(0));
442     V = EmitFAbs(*this, V, E->getArg(0)->getType());
443 
444     V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf");
445     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
446   }
447 
448   // TODO: BI__builtin_isinf_sign
449   //   isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0
450 
451   case Builtin::BI__builtin_isnormal: {
452     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
453     Value *V = EmitScalarExpr(E->getArg(0));
454     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
455 
456     Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
457     Value *IsLessThanInf =
458       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
459     APFloat Smallest = APFloat::getSmallestNormalized(
460                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
461     Value *IsNormal =
462       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
463                             "isnormal");
464     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
465     V = Builder.CreateAnd(V, IsNormal, "and");
466     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
467   }
468 
469   case Builtin::BI__builtin_isfinite: {
470     // isfinite(x) --> x == x && fabs(x) != infinity;
471     Value *V = EmitScalarExpr(E->getArg(0));
472     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
473 
474     Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
475     Value *IsNotInf =
476       Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
477 
478     V = Builder.CreateAnd(Eq, IsNotInf, "and");
479     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
480   }
481 
482   case Builtin::BI__builtin_fpclassify: {
483     Value *V = EmitScalarExpr(E->getArg(5));
484     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
485 
486     // Create Result
487     BasicBlock *Begin = Builder.GetInsertBlock();
488     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
489     Builder.SetInsertPoint(End);
490     PHINode *Result =
491       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
492                         "fpclassify_result");
493 
494     // if (V==0) return FP_ZERO
495     Builder.SetInsertPoint(Begin);
496     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
497                                           "iszero");
498     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
499     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
500     Builder.CreateCondBr(IsZero, End, NotZero);
501     Result->addIncoming(ZeroLiteral, Begin);
502 
503     // if (V != V) return FP_NAN
504     Builder.SetInsertPoint(NotZero);
505     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
506     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
507     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
508     Builder.CreateCondBr(IsNan, End, NotNan);
509     Result->addIncoming(NanLiteral, NotZero);
510 
511     // if (fabs(V) == infinity) return FP_INFINITY
512     Builder.SetInsertPoint(NotNan);
513     Value *VAbs = EmitFAbs(*this, V, E->getArg(5)->getType());
514     Value *IsInf =
515       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
516                             "isinf");
517     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
518     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
519     Builder.CreateCondBr(IsInf, End, NotInf);
520     Result->addIncoming(InfLiteral, NotNan);
521 
522     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
523     Builder.SetInsertPoint(NotInf);
524     APFloat Smallest = APFloat::getSmallestNormalized(
525         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
526     Value *IsNormal =
527       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
528                             "isnormal");
529     Value *NormalResult =
530       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
531                            EmitScalarExpr(E->getArg(3)));
532     Builder.CreateBr(End);
533     Result->addIncoming(NormalResult, NotInf);
534 
535     // return Result
536     Builder.SetInsertPoint(End);
537     return RValue::get(Result);
538   }
539 
540   case Builtin::BIalloca:
541   case Builtin::BI__builtin_alloca: {
542     Value *Size = EmitScalarExpr(E->getArg(0));
543     return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size));
544   }
545   case Builtin::BIbzero:
546   case Builtin::BI__builtin_bzero: {
547     Value *Address = EmitScalarExpr(E->getArg(0));
548     Value *SizeVal = EmitScalarExpr(E->getArg(1));
549     unsigned Align = GetPointeeAlignment(E->getArg(0));
550     Builder.CreateMemSet(Address, Builder.getInt8(0), SizeVal, Align, false);
551     return RValue::get(Address);
552   }
553   case Builtin::BImemcpy:
554   case Builtin::BI__builtin_memcpy: {
555     Value *Address = EmitScalarExpr(E->getArg(0));
556     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
557     Value *SizeVal = EmitScalarExpr(E->getArg(2));
558     unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
559                               GetPointeeAlignment(E->getArg(1)));
560     Builder.CreateMemCpy(Address, SrcAddr, SizeVal, Align, false);
561     return RValue::get(Address);
562   }
563 
564   case Builtin::BI__builtin___memcpy_chk: {
565     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
566     llvm::APSInt Size, DstSize;
567     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
568         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
569       break;
570     if (Size.ugt(DstSize))
571       break;
572     Value *Dest = EmitScalarExpr(E->getArg(0));
573     Value *Src = EmitScalarExpr(E->getArg(1));
574     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
575     unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
576                               GetPointeeAlignment(E->getArg(1)));
577     Builder.CreateMemCpy(Dest, Src, SizeVal, Align, false);
578     return RValue::get(Dest);
579   }
580 
581   case Builtin::BI__builtin_objc_memmove_collectable: {
582     Value *Address = EmitScalarExpr(E->getArg(0));
583     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
584     Value *SizeVal = EmitScalarExpr(E->getArg(2));
585     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
586                                                   Address, SrcAddr, SizeVal);
587     return RValue::get(Address);
588   }
589 
590   case Builtin::BI__builtin___memmove_chk: {
591     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
592     llvm::APSInt Size, DstSize;
593     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
594         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
595       break;
596     if (Size.ugt(DstSize))
597       break;
598     Value *Dest = EmitScalarExpr(E->getArg(0));
599     Value *Src = EmitScalarExpr(E->getArg(1));
600     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
601     unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
602                               GetPointeeAlignment(E->getArg(1)));
603     Builder.CreateMemMove(Dest, Src, SizeVal, Align, false);
604     return RValue::get(Dest);
605   }
606 
607   case Builtin::BImemmove:
608   case Builtin::BI__builtin_memmove: {
609     Value *Address = EmitScalarExpr(E->getArg(0));
610     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
611     Value *SizeVal = EmitScalarExpr(E->getArg(2));
612     unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
613                               GetPointeeAlignment(E->getArg(1)));
614     Builder.CreateMemMove(Address, SrcAddr, SizeVal, Align, false);
615     return RValue::get(Address);
616   }
617   case Builtin::BImemset:
618   case Builtin::BI__builtin_memset: {
619     Value *Address = EmitScalarExpr(E->getArg(0));
620     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
621                                          Builder.getInt8Ty());
622     Value *SizeVal = EmitScalarExpr(E->getArg(2));
623     unsigned Align = GetPointeeAlignment(E->getArg(0));
624     Builder.CreateMemSet(Address, ByteVal, SizeVal, Align, false);
625     return RValue::get(Address);
626   }
627   case Builtin::BI__builtin___memset_chk: {
628     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
629     llvm::APSInt Size, DstSize;
630     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
631         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
632       break;
633     if (Size.ugt(DstSize))
634       break;
635     Value *Address = EmitScalarExpr(E->getArg(0));
636     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
637                                          Builder.getInt8Ty());
638     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
639     unsigned Align = GetPointeeAlignment(E->getArg(0));
640     Builder.CreateMemSet(Address, ByteVal, SizeVal, Align, false);
641 
642     return RValue::get(Address);
643   }
644   case Builtin::BI__builtin_dwarf_cfa: {
645     // The offset in bytes from the first argument to the CFA.
646     //
647     // Why on earth is this in the frontend?  Is there any reason at
648     // all that the backend can't reasonably determine this while
649     // lowering llvm.eh.dwarf.cfa()?
650     //
651     // TODO: If there's a satisfactory reason, add a target hook for
652     // this instead of hard-coding 0, which is correct for most targets.
653     int32_t Offset = 0;
654 
655     Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
656     return RValue::get(Builder.CreateCall(F,
657                                       llvm::ConstantInt::get(Int32Ty, Offset)));
658   }
659   case Builtin::BI__builtin_return_address: {
660     Value *Depth = EmitScalarExpr(E->getArg(0));
661     Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
662     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
663     return RValue::get(Builder.CreateCall(F, Depth));
664   }
665   case Builtin::BI__builtin_frame_address: {
666     Value *Depth = EmitScalarExpr(E->getArg(0));
667     Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
668     Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
669     return RValue::get(Builder.CreateCall(F, Depth));
670   }
671   case Builtin::BI__builtin_extract_return_addr: {
672     Value *Address = EmitScalarExpr(E->getArg(0));
673     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
674     return RValue::get(Result);
675   }
676   case Builtin::BI__builtin_frob_return_addr: {
677     Value *Address = EmitScalarExpr(E->getArg(0));
678     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
679     return RValue::get(Result);
680   }
681   case Builtin::BI__builtin_dwarf_sp_column: {
682     llvm::IntegerType *Ty
683       = cast<llvm::IntegerType>(ConvertType(E->getType()));
684     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
685     if (Column == -1) {
686       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
687       return RValue::get(llvm::UndefValue::get(Ty));
688     }
689     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
690   }
691   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
692     Value *Address = EmitScalarExpr(E->getArg(0));
693     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
694       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
695     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
696   }
697   case Builtin::BI__builtin_eh_return: {
698     Value *Int = EmitScalarExpr(E->getArg(0));
699     Value *Ptr = EmitScalarExpr(E->getArg(1));
700 
701     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
702     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
703            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
704     Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
705                                   ? Intrinsic::eh_return_i32
706                                   : Intrinsic::eh_return_i64);
707     Builder.CreateCall2(F, Int, Ptr);
708     Builder.CreateUnreachable();
709 
710     // We do need to preserve an insertion point.
711     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
712 
713     return RValue::get(0);
714   }
715   case Builtin::BI__builtin_unwind_init: {
716     Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
717     return RValue::get(Builder.CreateCall(F));
718   }
719   case Builtin::BI__builtin_extend_pointer: {
720     // Extends a pointer to the size of an _Unwind_Word, which is
721     // uint64_t on all platforms.  Generally this gets poked into a
722     // register and eventually used as an address, so if the
723     // addressing registers are wider than pointers and the platform
724     // doesn't implicitly ignore high-order bits when doing
725     // addressing, we need to make sure we zext / sext based on
726     // the platform's expectations.
727     //
728     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
729 
730     // Cast the pointer to intptr_t.
731     Value *Ptr = EmitScalarExpr(E->getArg(0));
732     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
733 
734     // If that's 64 bits, we're done.
735     if (IntPtrTy->getBitWidth() == 64)
736       return RValue::get(Result);
737 
738     // Otherwise, ask the codegen data what to do.
739     if (getTargetHooks().extendPointerWithSExt())
740       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
741     else
742       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
743   }
744   case Builtin::BI__builtin_setjmp: {
745     // Buffer is a void**.
746     Value *Buf = EmitScalarExpr(E->getArg(0));
747 
748     // Store the frame pointer to the setjmp buffer.
749     Value *FrameAddr =
750       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
751                          ConstantInt::get(Int32Ty, 0));
752     Builder.CreateStore(FrameAddr, Buf);
753 
754     // Store the stack pointer to the setjmp buffer.
755     Value *StackAddr =
756       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
757     Value *StackSaveSlot =
758       Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2));
759     Builder.CreateStore(StackAddr, StackSaveSlot);
760 
761     // Call LLVM's EH setjmp, which is lightweight.
762     Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
763     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
764     return RValue::get(Builder.CreateCall(F, Buf));
765   }
766   case Builtin::BI__builtin_longjmp: {
767     Value *Buf = EmitScalarExpr(E->getArg(0));
768     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
769 
770     // Call LLVM's EH longjmp, which is lightweight.
771     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
772 
773     // longjmp doesn't return; mark this as unreachable.
774     Builder.CreateUnreachable();
775 
776     // We do need to preserve an insertion point.
777     EmitBlock(createBasicBlock("longjmp.cont"));
778 
779     return RValue::get(0);
780   }
781   case Builtin::BI__sync_fetch_and_add:
782   case Builtin::BI__sync_fetch_and_sub:
783   case Builtin::BI__sync_fetch_and_or:
784   case Builtin::BI__sync_fetch_and_and:
785   case Builtin::BI__sync_fetch_and_xor:
786   case Builtin::BI__sync_add_and_fetch:
787   case Builtin::BI__sync_sub_and_fetch:
788   case Builtin::BI__sync_and_and_fetch:
789   case Builtin::BI__sync_or_and_fetch:
790   case Builtin::BI__sync_xor_and_fetch:
791   case Builtin::BI__sync_val_compare_and_swap:
792   case Builtin::BI__sync_bool_compare_and_swap:
793   case Builtin::BI__sync_lock_test_and_set:
794   case Builtin::BI__sync_lock_release:
795   case Builtin::BI__sync_swap:
796     llvm_unreachable("Shouldn't make it through sema");
797   case Builtin::BI__sync_fetch_and_add_1:
798   case Builtin::BI__sync_fetch_and_add_2:
799   case Builtin::BI__sync_fetch_and_add_4:
800   case Builtin::BI__sync_fetch_and_add_8:
801   case Builtin::BI__sync_fetch_and_add_16:
802     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
803   case Builtin::BI__sync_fetch_and_sub_1:
804   case Builtin::BI__sync_fetch_and_sub_2:
805   case Builtin::BI__sync_fetch_and_sub_4:
806   case Builtin::BI__sync_fetch_and_sub_8:
807   case Builtin::BI__sync_fetch_and_sub_16:
808     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
809   case Builtin::BI__sync_fetch_and_or_1:
810   case Builtin::BI__sync_fetch_and_or_2:
811   case Builtin::BI__sync_fetch_and_or_4:
812   case Builtin::BI__sync_fetch_and_or_8:
813   case Builtin::BI__sync_fetch_and_or_16:
814     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
815   case Builtin::BI__sync_fetch_and_and_1:
816   case Builtin::BI__sync_fetch_and_and_2:
817   case Builtin::BI__sync_fetch_and_and_4:
818   case Builtin::BI__sync_fetch_and_and_8:
819   case Builtin::BI__sync_fetch_and_and_16:
820     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
821   case Builtin::BI__sync_fetch_and_xor_1:
822   case Builtin::BI__sync_fetch_and_xor_2:
823   case Builtin::BI__sync_fetch_and_xor_4:
824   case Builtin::BI__sync_fetch_and_xor_8:
825   case Builtin::BI__sync_fetch_and_xor_16:
826     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
827 
828   // Clang extensions: not overloaded yet.
829   case Builtin::BI__sync_fetch_and_min:
830     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
831   case Builtin::BI__sync_fetch_and_max:
832     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
833   case Builtin::BI__sync_fetch_and_umin:
834     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
835   case Builtin::BI__sync_fetch_and_umax:
836     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
837 
838   case Builtin::BI__sync_add_and_fetch_1:
839   case Builtin::BI__sync_add_and_fetch_2:
840   case Builtin::BI__sync_add_and_fetch_4:
841   case Builtin::BI__sync_add_and_fetch_8:
842   case Builtin::BI__sync_add_and_fetch_16:
843     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
844                                 llvm::Instruction::Add);
845   case Builtin::BI__sync_sub_and_fetch_1:
846   case Builtin::BI__sync_sub_and_fetch_2:
847   case Builtin::BI__sync_sub_and_fetch_4:
848   case Builtin::BI__sync_sub_and_fetch_8:
849   case Builtin::BI__sync_sub_and_fetch_16:
850     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
851                                 llvm::Instruction::Sub);
852   case Builtin::BI__sync_and_and_fetch_1:
853   case Builtin::BI__sync_and_and_fetch_2:
854   case Builtin::BI__sync_and_and_fetch_4:
855   case Builtin::BI__sync_and_and_fetch_8:
856   case Builtin::BI__sync_and_and_fetch_16:
857     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
858                                 llvm::Instruction::And);
859   case Builtin::BI__sync_or_and_fetch_1:
860   case Builtin::BI__sync_or_and_fetch_2:
861   case Builtin::BI__sync_or_and_fetch_4:
862   case Builtin::BI__sync_or_and_fetch_8:
863   case Builtin::BI__sync_or_and_fetch_16:
864     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
865                                 llvm::Instruction::Or);
866   case Builtin::BI__sync_xor_and_fetch_1:
867   case Builtin::BI__sync_xor_and_fetch_2:
868   case Builtin::BI__sync_xor_and_fetch_4:
869   case Builtin::BI__sync_xor_and_fetch_8:
870   case Builtin::BI__sync_xor_and_fetch_16:
871     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
872                                 llvm::Instruction::Xor);
873 
874   case Builtin::BI__sync_val_compare_and_swap_1:
875   case Builtin::BI__sync_val_compare_and_swap_2:
876   case Builtin::BI__sync_val_compare_and_swap_4:
877   case Builtin::BI__sync_val_compare_and_swap_8:
878   case Builtin::BI__sync_val_compare_and_swap_16: {
879     QualType T = E->getType();
880     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
881     unsigned AddrSpace =
882       cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
883 
884     llvm::IntegerType *IntType =
885       llvm::IntegerType::get(getLLVMContext(),
886                              getContext().getTypeSize(T));
887     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
888 
889     Value *Args[3];
890     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
891     Args[1] = EmitScalarExpr(E->getArg(1));
892     llvm::Type *ValueType = Args[1]->getType();
893     Args[1] = EmitToInt(*this, Args[1], T, IntType);
894     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
895 
896     Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
897                                                 llvm::SequentiallyConsistent);
898     Result = EmitFromInt(*this, Result, T, ValueType);
899     return RValue::get(Result);
900   }
901 
902   case Builtin::BI__sync_bool_compare_and_swap_1:
903   case Builtin::BI__sync_bool_compare_and_swap_2:
904   case Builtin::BI__sync_bool_compare_and_swap_4:
905   case Builtin::BI__sync_bool_compare_and_swap_8:
906   case Builtin::BI__sync_bool_compare_and_swap_16: {
907     QualType T = E->getArg(1)->getType();
908     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
909     unsigned AddrSpace =
910       cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
911 
912     llvm::IntegerType *IntType =
913       llvm::IntegerType::get(getLLVMContext(),
914                              getContext().getTypeSize(T));
915     llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
916 
917     Value *Args[3];
918     Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
919     Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType);
920     Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
921 
922     Value *OldVal = Args[1];
923     Value *PrevVal = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
924                                                  llvm::SequentiallyConsistent);
925     Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal);
926     // zext bool to int.
927     Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
928     return RValue::get(Result);
929   }
930 
931   case Builtin::BI__sync_swap_1:
932   case Builtin::BI__sync_swap_2:
933   case Builtin::BI__sync_swap_4:
934   case Builtin::BI__sync_swap_8:
935   case Builtin::BI__sync_swap_16:
936     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
937 
938   case Builtin::BI__sync_lock_test_and_set_1:
939   case Builtin::BI__sync_lock_test_and_set_2:
940   case Builtin::BI__sync_lock_test_and_set_4:
941   case Builtin::BI__sync_lock_test_and_set_8:
942   case Builtin::BI__sync_lock_test_and_set_16:
943     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
944 
945   case Builtin::BI__sync_lock_release_1:
946   case Builtin::BI__sync_lock_release_2:
947   case Builtin::BI__sync_lock_release_4:
948   case Builtin::BI__sync_lock_release_8:
949   case Builtin::BI__sync_lock_release_16: {
950     Value *Ptr = EmitScalarExpr(E->getArg(0));
951     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
952     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
953     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
954                                              StoreSize.getQuantity() * 8);
955     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
956     llvm::StoreInst *Store =
957       Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
958     Store->setAlignment(StoreSize.getQuantity());
959     Store->setAtomic(llvm::Release);
960     return RValue::get(0);
961   }
962 
963   case Builtin::BI__sync_synchronize: {
964     // We assume this is supposed to correspond to a C++0x-style
965     // sequentially-consistent fence (i.e. this is only usable for
966     // synchonization, not device I/O or anything like that). This intrinsic
967     // is really badly designed in the sense that in theory, there isn't
968     // any way to safely use it... but in practice, it mostly works
969     // to use it with non-atomic loads and stores to get acquire/release
970     // semantics.
971     Builder.CreateFence(llvm::SequentiallyConsistent);
972     return RValue::get(0);
973   }
974 
975   case Builtin::BI__c11_atomic_is_lock_free:
976   case Builtin::BI__atomic_is_lock_free: {
977     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
978     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
979     // _Atomic(T) is always properly-aligned.
980     const char *LibCallName = "__atomic_is_lock_free";
981     CallArgList Args;
982     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
983              getContext().getSizeType());
984     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
985       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
986                getContext().VoidPtrTy);
987     else
988       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
989                getContext().VoidPtrTy);
990     const CGFunctionInfo &FuncInfo =
991         CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args,
992                                                FunctionType::ExtInfo(),
993                                                RequiredArgs::All);
994     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
995     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
996     return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
997   }
998 
999   case Builtin::BI__atomic_test_and_set: {
1000     // Look at the argument type to determine whether this is a volatile
1001     // operation. The parameter type is always volatile.
1002     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1003     bool Volatile =
1004         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1005 
1006     Value *Ptr = EmitScalarExpr(E->getArg(0));
1007     unsigned AddrSpace =
1008         cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
1009     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1010     Value *NewVal = Builder.getInt8(1);
1011     Value *Order = EmitScalarExpr(E->getArg(1));
1012     if (isa<llvm::ConstantInt>(Order)) {
1013       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1014       AtomicRMWInst *Result = 0;
1015       switch (ord) {
1016       case 0:  // memory_order_relaxed
1017       default: // invalid order
1018         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1019                                          Ptr, NewVal,
1020                                          llvm::Monotonic);
1021         break;
1022       case 1:  // memory_order_consume
1023       case 2:  // memory_order_acquire
1024         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1025                                          Ptr, NewVal,
1026                                          llvm::Acquire);
1027         break;
1028       case 3:  // memory_order_release
1029         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1030                                          Ptr, NewVal,
1031                                          llvm::Release);
1032         break;
1033       case 4:  // memory_order_acq_rel
1034         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1035                                          Ptr, NewVal,
1036                                          llvm::AcquireRelease);
1037         break;
1038       case 5:  // memory_order_seq_cst
1039         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1040                                          Ptr, NewVal,
1041                                          llvm::SequentiallyConsistent);
1042         break;
1043       }
1044       Result->setVolatile(Volatile);
1045       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1046     }
1047 
1048     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1049 
1050     llvm::BasicBlock *BBs[5] = {
1051       createBasicBlock("monotonic", CurFn),
1052       createBasicBlock("acquire", CurFn),
1053       createBasicBlock("release", CurFn),
1054       createBasicBlock("acqrel", CurFn),
1055       createBasicBlock("seqcst", CurFn)
1056     };
1057     llvm::AtomicOrdering Orders[5] = {
1058       llvm::Monotonic, llvm::Acquire, llvm::Release,
1059       llvm::AcquireRelease, llvm::SequentiallyConsistent
1060     };
1061 
1062     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1063     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1064 
1065     Builder.SetInsertPoint(ContBB);
1066     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
1067 
1068     for (unsigned i = 0; i < 5; ++i) {
1069       Builder.SetInsertPoint(BBs[i]);
1070       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1071                                                    Ptr, NewVal, Orders[i]);
1072       RMW->setVolatile(Volatile);
1073       Result->addIncoming(RMW, BBs[i]);
1074       Builder.CreateBr(ContBB);
1075     }
1076 
1077     SI->addCase(Builder.getInt32(0), BBs[0]);
1078     SI->addCase(Builder.getInt32(1), BBs[1]);
1079     SI->addCase(Builder.getInt32(2), BBs[1]);
1080     SI->addCase(Builder.getInt32(3), BBs[2]);
1081     SI->addCase(Builder.getInt32(4), BBs[3]);
1082     SI->addCase(Builder.getInt32(5), BBs[4]);
1083 
1084     Builder.SetInsertPoint(ContBB);
1085     return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1086   }
1087 
1088   case Builtin::BI__atomic_clear: {
1089     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1090     bool Volatile =
1091         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1092 
1093     Value *Ptr = EmitScalarExpr(E->getArg(0));
1094     unsigned AddrSpace =
1095         cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
1096     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1097     Value *NewVal = Builder.getInt8(0);
1098     Value *Order = EmitScalarExpr(E->getArg(1));
1099     if (isa<llvm::ConstantInt>(Order)) {
1100       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1101       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1102       Store->setAlignment(1);
1103       switch (ord) {
1104       case 0:  // memory_order_relaxed
1105       default: // invalid order
1106         Store->setOrdering(llvm::Monotonic);
1107         break;
1108       case 3:  // memory_order_release
1109         Store->setOrdering(llvm::Release);
1110         break;
1111       case 5:  // memory_order_seq_cst
1112         Store->setOrdering(llvm::SequentiallyConsistent);
1113         break;
1114       }
1115       return RValue::get(0);
1116     }
1117 
1118     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1119 
1120     llvm::BasicBlock *BBs[3] = {
1121       createBasicBlock("monotonic", CurFn),
1122       createBasicBlock("release", CurFn),
1123       createBasicBlock("seqcst", CurFn)
1124     };
1125     llvm::AtomicOrdering Orders[3] = {
1126       llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent
1127     };
1128 
1129     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1130     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1131 
1132     for (unsigned i = 0; i < 3; ++i) {
1133       Builder.SetInsertPoint(BBs[i]);
1134       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1135       Store->setAlignment(1);
1136       Store->setOrdering(Orders[i]);
1137       Builder.CreateBr(ContBB);
1138     }
1139 
1140     SI->addCase(Builder.getInt32(0), BBs[0]);
1141     SI->addCase(Builder.getInt32(3), BBs[1]);
1142     SI->addCase(Builder.getInt32(5), BBs[2]);
1143 
1144     Builder.SetInsertPoint(ContBB);
1145     return RValue::get(0);
1146   }
1147 
1148   case Builtin::BI__atomic_thread_fence:
1149   case Builtin::BI__atomic_signal_fence:
1150   case Builtin::BI__c11_atomic_thread_fence:
1151   case Builtin::BI__c11_atomic_signal_fence: {
1152     llvm::SynchronizationScope Scope;
1153     if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1154         BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1155       Scope = llvm::SingleThread;
1156     else
1157       Scope = llvm::CrossThread;
1158     Value *Order = EmitScalarExpr(E->getArg(0));
1159     if (isa<llvm::ConstantInt>(Order)) {
1160       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1161       switch (ord) {
1162       case 0:  // memory_order_relaxed
1163       default: // invalid order
1164         break;
1165       case 1:  // memory_order_consume
1166       case 2:  // memory_order_acquire
1167         Builder.CreateFence(llvm::Acquire, Scope);
1168         break;
1169       case 3:  // memory_order_release
1170         Builder.CreateFence(llvm::Release, Scope);
1171         break;
1172       case 4:  // memory_order_acq_rel
1173         Builder.CreateFence(llvm::AcquireRelease, Scope);
1174         break;
1175       case 5:  // memory_order_seq_cst
1176         Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1177         break;
1178       }
1179       return RValue::get(0);
1180     }
1181 
1182     llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1183     AcquireBB = createBasicBlock("acquire", CurFn);
1184     ReleaseBB = createBasicBlock("release", CurFn);
1185     AcqRelBB = createBasicBlock("acqrel", CurFn);
1186     SeqCstBB = createBasicBlock("seqcst", CurFn);
1187     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1188 
1189     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1190     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
1191 
1192     Builder.SetInsertPoint(AcquireBB);
1193     Builder.CreateFence(llvm::Acquire, Scope);
1194     Builder.CreateBr(ContBB);
1195     SI->addCase(Builder.getInt32(1), AcquireBB);
1196     SI->addCase(Builder.getInt32(2), AcquireBB);
1197 
1198     Builder.SetInsertPoint(ReleaseBB);
1199     Builder.CreateFence(llvm::Release, Scope);
1200     Builder.CreateBr(ContBB);
1201     SI->addCase(Builder.getInt32(3), ReleaseBB);
1202 
1203     Builder.SetInsertPoint(AcqRelBB);
1204     Builder.CreateFence(llvm::AcquireRelease, Scope);
1205     Builder.CreateBr(ContBB);
1206     SI->addCase(Builder.getInt32(4), AcqRelBB);
1207 
1208     Builder.SetInsertPoint(SeqCstBB);
1209     Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1210     Builder.CreateBr(ContBB);
1211     SI->addCase(Builder.getInt32(5), SeqCstBB);
1212 
1213     Builder.SetInsertPoint(ContBB);
1214     return RValue::get(0);
1215   }
1216 
1217     // Library functions with special handling.
1218   case Builtin::BIsqrt:
1219   case Builtin::BIsqrtf:
1220   case Builtin::BIsqrtl: {
1221     // TODO: there is currently no set of optimizer flags
1222     // sufficient for us to rewrite sqrt to @llvm.sqrt.
1223     // -fmath-errno=0 is not good enough; we need finiteness.
1224     // We could probably precondition the call with an ult
1225     // against 0, but is that worth the complexity?
1226     break;
1227   }
1228 
1229   case Builtin::BIpow:
1230   case Builtin::BIpowf:
1231   case Builtin::BIpowl: {
1232     // Rewrite sqrt to intrinsic if allowed.
1233     if (!FD->hasAttr<ConstAttr>())
1234       break;
1235     Value *Base = EmitScalarExpr(E->getArg(0));
1236     Value *Exponent = EmitScalarExpr(E->getArg(1));
1237     llvm::Type *ArgType = Base->getType();
1238     Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
1239     return RValue::get(Builder.CreateCall2(F, Base, Exponent));
1240   }
1241 
1242   case Builtin::BIfma:
1243   case Builtin::BIfmaf:
1244   case Builtin::BIfmal:
1245   case Builtin::BI__builtin_fma:
1246   case Builtin::BI__builtin_fmaf:
1247   case Builtin::BI__builtin_fmal: {
1248     // Rewrite fma to intrinsic.
1249     Value *FirstArg = EmitScalarExpr(E->getArg(0));
1250     llvm::Type *ArgType = FirstArg->getType();
1251     Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
1252     return RValue::get(Builder.CreateCall3(F, FirstArg,
1253                                               EmitScalarExpr(E->getArg(1)),
1254                                               EmitScalarExpr(E->getArg(2))));
1255   }
1256 
1257   case Builtin::BI__builtin_signbit:
1258   case Builtin::BI__builtin_signbitf:
1259   case Builtin::BI__builtin_signbitl: {
1260     LLVMContext &C = CGM.getLLVMContext();
1261 
1262     Value *Arg = EmitScalarExpr(E->getArg(0));
1263     llvm::Type *ArgTy = Arg->getType();
1264     if (ArgTy->isPPC_FP128Ty())
1265       break; // FIXME: I'm not sure what the right implementation is here.
1266     int ArgWidth = ArgTy->getPrimitiveSizeInBits();
1267     llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
1268     Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy);
1269     Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy);
1270     Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp);
1271     return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
1272   }
1273   case Builtin::BI__builtin_annotation: {
1274     llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1275     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1276                                       AnnVal->getType());
1277 
1278     // Get the annotation string, go through casts. Sema requires this to be a
1279     // non-wide string literal, potentially casted, so the cast<> is safe.
1280     const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
1281     llvm::StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1282     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
1283   }
1284   }
1285 
1286   // If this is an alias for a lib function (e.g. __builtin_sin), emit
1287   // the call using the normal call path, but using the unmangled
1288   // version of the function name.
1289   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
1290     return emitLibraryCall(*this, FD, E,
1291                            CGM.getBuiltinLibFunction(FD, BuiltinID));
1292 
1293   // If this is a predefined lib function (e.g. malloc), emit the call
1294   // using exactly the normal call path.
1295   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
1296     return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee()));
1297 
1298   // See if we have a target specific intrinsic.
1299   const char *Name = getContext().BuiltinInfo.GetName(BuiltinID);
1300   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
1301   if (const char *Prefix =
1302       llvm::Triple::getArchTypePrefix(Target.getTriple().getArch()))
1303     IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name);
1304 
1305   if (IntrinsicID != Intrinsic::not_intrinsic) {
1306     SmallVector<Value*, 16> Args;
1307 
1308     // Find out if any arguments are required to be integer constant
1309     // expressions.
1310     unsigned ICEArguments = 0;
1311     ASTContext::GetBuiltinTypeError Error;
1312     getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
1313     assert(Error == ASTContext::GE_None && "Should not codegen an error");
1314 
1315     Function *F = CGM.getIntrinsic(IntrinsicID);
1316     llvm::FunctionType *FTy = F->getFunctionType();
1317 
1318     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
1319       Value *ArgValue;
1320       // If this is a normal argument, just emit it as a scalar.
1321       if ((ICEArguments & (1 << i)) == 0) {
1322         ArgValue = EmitScalarExpr(E->getArg(i));
1323       } else {
1324         // If this is required to be a constant, constant fold it so that we
1325         // know that the generated intrinsic gets a ConstantInt.
1326         llvm::APSInt Result;
1327         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
1328         assert(IsConst && "Constant arg isn't actually constant?");
1329         (void)IsConst;
1330         ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
1331       }
1332 
1333       // If the intrinsic arg type is different from the builtin arg type
1334       // we need to do a bit cast.
1335       llvm::Type *PTy = FTy->getParamType(i);
1336       if (PTy != ArgValue->getType()) {
1337         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
1338                "Must be able to losslessly bit cast to param");
1339         ArgValue = Builder.CreateBitCast(ArgValue, PTy);
1340       }
1341 
1342       Args.push_back(ArgValue);
1343     }
1344 
1345     Value *V = Builder.CreateCall(F, Args);
1346     QualType BuiltinRetType = E->getType();
1347 
1348     llvm::Type *RetTy = VoidTy;
1349     if (!BuiltinRetType->isVoidType())
1350       RetTy = ConvertType(BuiltinRetType);
1351 
1352     if (RetTy != V->getType()) {
1353       assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
1354              "Must be able to losslessly bit cast result type");
1355       V = Builder.CreateBitCast(V, RetTy);
1356     }
1357 
1358     return RValue::get(V);
1359   }
1360 
1361   // See if we have a target specific builtin that needs to be lowered.
1362   if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
1363     return RValue::get(V);
1364 
1365   ErrorUnsupported(E, "builtin function");
1366 
1367   // Unknown builtin, for now just dump it out and return undef.
1368   if (hasAggregateLLVMType(E->getType()))
1369     return RValue::getAggregate(CreateMemTemp(E->getType()));
1370   return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
1371 }
1372 
1373 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
1374                                               const CallExpr *E) {
1375   switch (Target.getTriple().getArch()) {
1376   case llvm::Triple::arm:
1377   case llvm::Triple::thumb:
1378     return EmitARMBuiltinExpr(BuiltinID, E);
1379   case llvm::Triple::x86:
1380   case llvm::Triple::x86_64:
1381     return EmitX86BuiltinExpr(BuiltinID, E);
1382   case llvm::Triple::ppc:
1383   case llvm::Triple::ppc64:
1384     return EmitPPCBuiltinExpr(BuiltinID, E);
1385   default:
1386     return 0;
1387   }
1388 }
1389 
1390 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
1391                                      NeonTypeFlags TypeFlags) {
1392   int IsQuad = TypeFlags.isQuad();
1393   switch (TypeFlags.getEltType()) {
1394   case NeonTypeFlags::Int8:
1395   case NeonTypeFlags::Poly8:
1396     return llvm::VectorType::get(CGF->Int8Ty, 8 << IsQuad);
1397   case NeonTypeFlags::Int16:
1398   case NeonTypeFlags::Poly16:
1399   case NeonTypeFlags::Float16:
1400     return llvm::VectorType::get(CGF->Int16Ty, 4 << IsQuad);
1401   case NeonTypeFlags::Int32:
1402     return llvm::VectorType::get(CGF->Int32Ty, 2 << IsQuad);
1403   case NeonTypeFlags::Int64:
1404     return llvm::VectorType::get(CGF->Int64Ty, 1 << IsQuad);
1405   case NeonTypeFlags::Float32:
1406     return llvm::VectorType::get(CGF->FloatTy, 2 << IsQuad);
1407   }
1408   llvm_unreachable("Invalid NeonTypeFlags element type!");
1409 }
1410 
1411 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
1412   unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
1413   Value* SV = llvm::ConstantVector::getSplat(nElts, C);
1414   return Builder.CreateShuffleVector(V, V, SV, "lane");
1415 }
1416 
1417 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
1418                                      const char *name,
1419                                      unsigned shift, bool rightshift) {
1420   unsigned j = 0;
1421   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1422        ai != ae; ++ai, ++j)
1423     if (shift > 0 && shift == j)
1424       Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
1425     else
1426       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
1427 
1428   return Builder.CreateCall(F, Ops, name);
1429 }
1430 
1431 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
1432                                             bool neg) {
1433   int SV = cast<ConstantInt>(V)->getSExtValue();
1434 
1435   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
1436   llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
1437   return llvm::ConstantVector::getSplat(VTy->getNumElements(), C);
1438 }
1439 
1440 /// GetPointeeAlignment - Given an expression with a pointer type, find the
1441 /// alignment of the type referenced by the pointer.  Skip over implicit
1442 /// casts.
1443 unsigned CodeGenFunction::GetPointeeAlignment(const Expr *Addr) {
1444   unsigned Align = 1;
1445   // Check if the type is a pointer.  The implicit cast operand might not be.
1446   while (Addr->getType()->isPointerType()) {
1447     QualType PtTy = Addr->getType()->getPointeeType();
1448 
1449     // Can't get alignment of incomplete types.
1450     if (!PtTy->isIncompleteType()) {
1451       unsigned NewA = getContext().getTypeAlignInChars(PtTy).getQuantity();
1452       if (NewA > Align)
1453         Align = NewA;
1454     }
1455 
1456     // If the address is an implicit cast, repeat with the cast operand.
1457     if (const ImplicitCastExpr *CastAddr = dyn_cast<ImplicitCastExpr>(Addr)) {
1458       Addr = CastAddr->getSubExpr();
1459       continue;
1460     }
1461     break;
1462   }
1463   return Align;
1464 }
1465 
1466 /// GetPointeeAlignmentValue - Given an expression with a pointer type, find
1467 /// the alignment of the type referenced by the pointer.  Skip over implicit
1468 /// casts.  Return the alignment as an llvm::Value.
1469 Value *CodeGenFunction::GetPointeeAlignmentValue(const Expr *Addr) {
1470   return llvm::ConstantInt::get(Int32Ty, GetPointeeAlignment(Addr));
1471 }
1472 
1473 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
1474                                            const CallExpr *E) {
1475   if (BuiltinID == ARM::BI__clear_cache) {
1476     const FunctionDecl *FD = E->getDirectCallee();
1477     // Oddly people write this call without args on occasion and gcc accepts
1478     // it - it's also marked as varargs in the description file.
1479     SmallVector<Value*, 2> Ops;
1480     for (unsigned i = 0; i < E->getNumArgs(); i++)
1481       Ops.push_back(EmitScalarExpr(E->getArg(i)));
1482     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
1483     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
1484     StringRef Name = FD->getName();
1485     return Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
1486   }
1487 
1488   if (BuiltinID == ARM::BI__builtin_arm_ldrexd) {
1489     Function *F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
1490 
1491     Value *LdPtr = EmitScalarExpr(E->getArg(0));
1492     Value *Val = Builder.CreateCall(F, LdPtr, "ldrexd");
1493 
1494     Value *Val0 = Builder.CreateExtractValue(Val, 1);
1495     Value *Val1 = Builder.CreateExtractValue(Val, 0);
1496     Val0 = Builder.CreateZExt(Val0, Int64Ty);
1497     Val1 = Builder.CreateZExt(Val1, Int64Ty);
1498 
1499     Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
1500     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
1501     return Builder.CreateOr(Val, Val1);
1502   }
1503 
1504   if (BuiltinID == ARM::BI__builtin_arm_strexd) {
1505     Function *F = CGM.getIntrinsic(Intrinsic::arm_strexd);
1506     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL);
1507 
1508     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
1509     Value *Tmp = Builder.CreateAlloca(Int64Ty, One);
1510     Value *Val = EmitScalarExpr(E->getArg(0));
1511     Builder.CreateStore(Val, Tmp);
1512 
1513     Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
1514     Val = Builder.CreateLoad(LdPtr);
1515 
1516     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
1517     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
1518     Value *StPtr = EmitScalarExpr(E->getArg(1));
1519     return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd");
1520   }
1521 
1522   SmallVector<Value*, 4> Ops;
1523   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++)
1524     Ops.push_back(EmitScalarExpr(E->getArg(i)));
1525 
1526   // vget_lane and vset_lane are not overloaded and do not have an extra
1527   // argument that specifies the vector type.
1528   switch (BuiltinID) {
1529   default: break;
1530   case ARM::BI__builtin_neon_vget_lane_i8:
1531   case ARM::BI__builtin_neon_vget_lane_i16:
1532   case ARM::BI__builtin_neon_vget_lane_i32:
1533   case ARM::BI__builtin_neon_vget_lane_i64:
1534   case ARM::BI__builtin_neon_vget_lane_f32:
1535   case ARM::BI__builtin_neon_vgetq_lane_i8:
1536   case ARM::BI__builtin_neon_vgetq_lane_i16:
1537   case ARM::BI__builtin_neon_vgetq_lane_i32:
1538   case ARM::BI__builtin_neon_vgetq_lane_i64:
1539   case ARM::BI__builtin_neon_vgetq_lane_f32:
1540     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
1541                                         "vget_lane");
1542   case ARM::BI__builtin_neon_vset_lane_i8:
1543   case ARM::BI__builtin_neon_vset_lane_i16:
1544   case ARM::BI__builtin_neon_vset_lane_i32:
1545   case ARM::BI__builtin_neon_vset_lane_i64:
1546   case ARM::BI__builtin_neon_vset_lane_f32:
1547   case ARM::BI__builtin_neon_vsetq_lane_i8:
1548   case ARM::BI__builtin_neon_vsetq_lane_i16:
1549   case ARM::BI__builtin_neon_vsetq_lane_i32:
1550   case ARM::BI__builtin_neon_vsetq_lane_i64:
1551   case ARM::BI__builtin_neon_vsetq_lane_f32:
1552     Ops.push_back(EmitScalarExpr(E->getArg(2)));
1553     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
1554   }
1555 
1556   // Get the last argument, which specifies the vector type.
1557   llvm::APSInt Result;
1558   const Expr *Arg = E->getArg(E->getNumArgs()-1);
1559   if (!Arg->isIntegerConstantExpr(Result, getContext()))
1560     return 0;
1561 
1562   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
1563       BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
1564     // Determine the overloaded type of this builtin.
1565     llvm::Type *Ty;
1566     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
1567       Ty = FloatTy;
1568     else
1569       Ty = DoubleTy;
1570 
1571     // Determine whether this is an unsigned conversion or not.
1572     bool usgn = Result.getZExtValue() == 1;
1573     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
1574 
1575     // Call the appropriate intrinsic.
1576     Function *F = CGM.getIntrinsic(Int, Ty);
1577     return Builder.CreateCall(F, Ops, "vcvtr");
1578   }
1579 
1580   // Determine the type of this overloaded NEON intrinsic.
1581   NeonTypeFlags Type(Result.getZExtValue());
1582   bool usgn = Type.isUnsigned();
1583   bool quad = Type.isQuad();
1584   bool rightShift = false;
1585 
1586   llvm::VectorType *VTy = GetNeonType(this, Type);
1587   llvm::Type *Ty = VTy;
1588   if (!Ty)
1589     return 0;
1590 
1591   unsigned Int;
1592   switch (BuiltinID) {
1593   default: return 0;
1594   case ARM::BI__builtin_neon_vabd_v:
1595   case ARM::BI__builtin_neon_vabdq_v:
1596     Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
1597     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
1598   case ARM::BI__builtin_neon_vabs_v:
1599   case ARM::BI__builtin_neon_vabsq_v:
1600     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vabs, Ty),
1601                         Ops, "vabs");
1602   case ARM::BI__builtin_neon_vaddhn_v:
1603     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vaddhn, Ty),
1604                         Ops, "vaddhn");
1605   case ARM::BI__builtin_neon_vcale_v:
1606     std::swap(Ops[0], Ops[1]);
1607   case ARM::BI__builtin_neon_vcage_v: {
1608     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacged);
1609     return EmitNeonCall(F, Ops, "vcage");
1610   }
1611   case ARM::BI__builtin_neon_vcaleq_v:
1612     std::swap(Ops[0], Ops[1]);
1613   case ARM::BI__builtin_neon_vcageq_v: {
1614     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgeq);
1615     return EmitNeonCall(F, Ops, "vcage");
1616   }
1617   case ARM::BI__builtin_neon_vcalt_v:
1618     std::swap(Ops[0], Ops[1]);
1619   case ARM::BI__builtin_neon_vcagt_v: {
1620     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtd);
1621     return EmitNeonCall(F, Ops, "vcagt");
1622   }
1623   case ARM::BI__builtin_neon_vcaltq_v:
1624     std::swap(Ops[0], Ops[1]);
1625   case ARM::BI__builtin_neon_vcagtq_v: {
1626     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtq);
1627     return EmitNeonCall(F, Ops, "vcagt");
1628   }
1629   case ARM::BI__builtin_neon_vcls_v:
1630   case ARM::BI__builtin_neon_vclsq_v: {
1631     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcls, Ty);
1632     return EmitNeonCall(F, Ops, "vcls");
1633   }
1634   case ARM::BI__builtin_neon_vclz_v:
1635   case ARM::BI__builtin_neon_vclzq_v: {
1636     // Generate target-independent intrinsic; also need to add second argument
1637     // for whether or not clz of zero is undefined; on ARM it isn't.
1638     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ty);
1639     Ops.push_back(Builder.getInt1(Target.isCLZForZeroUndef()));
1640     return EmitNeonCall(F, Ops, "vclz");
1641   }
1642   case ARM::BI__builtin_neon_vcnt_v:
1643   case ARM::BI__builtin_neon_vcntq_v: {
1644     // generate target-independent intrinsic
1645     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, Ty);
1646     return EmitNeonCall(F, Ops, "vctpop");
1647   }
1648   case ARM::BI__builtin_neon_vcvt_f16_v: {
1649     assert(Type.getEltType() == NeonTypeFlags::Float16 && !quad &&
1650            "unexpected vcvt_f16_v builtin");
1651     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf);
1652     return EmitNeonCall(F, Ops, "vcvt");
1653   }
1654   case ARM::BI__builtin_neon_vcvt_f32_f16: {
1655     assert(Type.getEltType() == NeonTypeFlags::Float16 && !quad &&
1656            "unexpected vcvt_f32_f16 builtin");
1657     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp);
1658     return EmitNeonCall(F, Ops, "vcvt");
1659   }
1660   case ARM::BI__builtin_neon_vcvt_f32_v:
1661   case ARM::BI__builtin_neon_vcvtq_f32_v:
1662     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1663     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
1664     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
1665                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
1666   case ARM::BI__builtin_neon_vcvt_s32_v:
1667   case ARM::BI__builtin_neon_vcvt_u32_v:
1668   case ARM::BI__builtin_neon_vcvtq_s32_v:
1669   case ARM::BI__builtin_neon_vcvtq_u32_v: {
1670     llvm::Type *FloatTy =
1671       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
1672     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
1673     return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
1674                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
1675   }
1676   case ARM::BI__builtin_neon_vcvt_n_f32_v:
1677   case ARM::BI__builtin_neon_vcvtq_n_f32_v: {
1678     llvm::Type *FloatTy =
1679       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
1680     llvm::Type *Tys[2] = { FloatTy, Ty };
1681     Int = usgn ? Intrinsic::arm_neon_vcvtfxu2fp
1682                : Intrinsic::arm_neon_vcvtfxs2fp;
1683     Function *F = CGM.getIntrinsic(Int, Tys);
1684     return EmitNeonCall(F, Ops, "vcvt_n");
1685   }
1686   case ARM::BI__builtin_neon_vcvt_n_s32_v:
1687   case ARM::BI__builtin_neon_vcvt_n_u32_v:
1688   case ARM::BI__builtin_neon_vcvtq_n_s32_v:
1689   case ARM::BI__builtin_neon_vcvtq_n_u32_v: {
1690     llvm::Type *FloatTy =
1691       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
1692     llvm::Type *Tys[2] = { Ty, FloatTy };
1693     Int = usgn ? Intrinsic::arm_neon_vcvtfp2fxu
1694                : Intrinsic::arm_neon_vcvtfp2fxs;
1695     Function *F = CGM.getIntrinsic(Int, Tys);
1696     return EmitNeonCall(F, Ops, "vcvt_n");
1697   }
1698   case ARM::BI__builtin_neon_vext_v:
1699   case ARM::BI__builtin_neon_vextq_v: {
1700     int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
1701     SmallVector<Constant*, 16> Indices;
1702     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
1703       Indices.push_back(ConstantInt::get(Int32Ty, i+CV));
1704 
1705     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1706     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1707     Value *SV = llvm::ConstantVector::get(Indices);
1708     return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
1709   }
1710   case ARM::BI__builtin_neon_vhadd_v:
1711   case ARM::BI__builtin_neon_vhaddq_v:
1712     Int = usgn ? Intrinsic::arm_neon_vhaddu : Intrinsic::arm_neon_vhadds;
1713     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vhadd");
1714   case ARM::BI__builtin_neon_vhsub_v:
1715   case ARM::BI__builtin_neon_vhsubq_v:
1716     Int = usgn ? Intrinsic::arm_neon_vhsubu : Intrinsic::arm_neon_vhsubs;
1717     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vhsub");
1718   case ARM::BI__builtin_neon_vld1_v:
1719   case ARM::BI__builtin_neon_vld1q_v:
1720     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
1721     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty),
1722                         Ops, "vld1");
1723   case ARM::BI__builtin_neon_vld1_lane_v:
1724   case ARM::BI__builtin_neon_vld1q_lane_v: {
1725     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
1726     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
1727     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1728     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
1729     Value *Align = GetPointeeAlignmentValue(E->getArg(0));
1730     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
1731     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
1732   }
1733   case ARM::BI__builtin_neon_vld1_dup_v:
1734   case ARM::BI__builtin_neon_vld1q_dup_v: {
1735     Value *V = UndefValue::get(Ty);
1736     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
1737     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1738     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
1739     Value *Align = GetPointeeAlignmentValue(E->getArg(0));
1740     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
1741     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
1742     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
1743     return EmitNeonSplat(Ops[0], CI);
1744   }
1745   case ARM::BI__builtin_neon_vld2_v:
1746   case ARM::BI__builtin_neon_vld2q_v: {
1747     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2, Ty);
1748     Value *Align = GetPointeeAlignmentValue(E->getArg(1));
1749     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld2");
1750     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1751     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1752     return Builder.CreateStore(Ops[1], Ops[0]);
1753   }
1754   case ARM::BI__builtin_neon_vld3_v:
1755   case ARM::BI__builtin_neon_vld3q_v: {
1756     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3, Ty);
1757     Value *Align = GetPointeeAlignmentValue(E->getArg(1));
1758     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld3");
1759     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1760     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1761     return Builder.CreateStore(Ops[1], Ops[0]);
1762   }
1763   case ARM::BI__builtin_neon_vld4_v:
1764   case ARM::BI__builtin_neon_vld4q_v: {
1765     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4, Ty);
1766     Value *Align = GetPointeeAlignmentValue(E->getArg(1));
1767     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld4");
1768     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1769     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1770     return Builder.CreateStore(Ops[1], Ops[0]);
1771   }
1772   case ARM::BI__builtin_neon_vld2_lane_v:
1773   case ARM::BI__builtin_neon_vld2q_lane_v: {
1774     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2lane, Ty);
1775     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1776     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
1777     Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
1778     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
1779     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1780     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1781     return Builder.CreateStore(Ops[1], Ops[0]);
1782   }
1783   case ARM::BI__builtin_neon_vld3_lane_v:
1784   case ARM::BI__builtin_neon_vld3q_lane_v: {
1785     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3lane, Ty);
1786     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1787     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
1788     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
1789     Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
1790     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
1791     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1792     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1793     return Builder.CreateStore(Ops[1], Ops[0]);
1794   }
1795   case ARM::BI__builtin_neon_vld4_lane_v:
1796   case ARM::BI__builtin_neon_vld4q_lane_v: {
1797     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4lane, Ty);
1798     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
1799     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
1800     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
1801     Ops[5] = Builder.CreateBitCast(Ops[5], Ty);
1802     Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
1803     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
1804     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1805     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1806     return Builder.CreateStore(Ops[1], Ops[0]);
1807   }
1808   case ARM::BI__builtin_neon_vld2_dup_v:
1809   case ARM::BI__builtin_neon_vld3_dup_v:
1810   case ARM::BI__builtin_neon_vld4_dup_v: {
1811     // Handle 64-bit elements as a special-case.  There is no "dup" needed.
1812     if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
1813       switch (BuiltinID) {
1814       case ARM::BI__builtin_neon_vld2_dup_v:
1815         Int = Intrinsic::arm_neon_vld2;
1816         break;
1817       case ARM::BI__builtin_neon_vld3_dup_v:
1818         Int = Intrinsic::arm_neon_vld3;
1819         break;
1820       case ARM::BI__builtin_neon_vld4_dup_v:
1821         Int = Intrinsic::arm_neon_vld4;
1822         break;
1823       default: llvm_unreachable("unknown vld_dup intrinsic?");
1824       }
1825       Function *F = CGM.getIntrinsic(Int, Ty);
1826       Value *Align = GetPointeeAlignmentValue(E->getArg(1));
1827       Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
1828       Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1829       Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1830       return Builder.CreateStore(Ops[1], Ops[0]);
1831     }
1832     switch (BuiltinID) {
1833     case ARM::BI__builtin_neon_vld2_dup_v:
1834       Int = Intrinsic::arm_neon_vld2lane;
1835       break;
1836     case ARM::BI__builtin_neon_vld3_dup_v:
1837       Int = Intrinsic::arm_neon_vld3lane;
1838       break;
1839     case ARM::BI__builtin_neon_vld4_dup_v:
1840       Int = Intrinsic::arm_neon_vld4lane;
1841       break;
1842     default: llvm_unreachable("unknown vld_dup intrinsic?");
1843     }
1844     Function *F = CGM.getIntrinsic(Int, Ty);
1845     llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
1846 
1847     SmallVector<Value*, 6> Args;
1848     Args.push_back(Ops[1]);
1849     Args.append(STy->getNumElements(), UndefValue::get(Ty));
1850 
1851     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
1852     Args.push_back(CI);
1853     Args.push_back(GetPointeeAlignmentValue(E->getArg(1)));
1854 
1855     Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
1856     // splat lane 0 to all elts in each vector of the result.
1857     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1858       Value *Val = Builder.CreateExtractValue(Ops[1], i);
1859       Value *Elt = Builder.CreateBitCast(Val, Ty);
1860       Elt = EmitNeonSplat(Elt, CI);
1861       Elt = Builder.CreateBitCast(Elt, Val->getType());
1862       Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
1863     }
1864     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
1865     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
1866     return Builder.CreateStore(Ops[1], Ops[0]);
1867   }
1868   case ARM::BI__builtin_neon_vmax_v:
1869   case ARM::BI__builtin_neon_vmaxq_v:
1870     Int = usgn ? Intrinsic::arm_neon_vmaxu : Intrinsic::arm_neon_vmaxs;
1871     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
1872   case ARM::BI__builtin_neon_vmin_v:
1873   case ARM::BI__builtin_neon_vminq_v:
1874     Int = usgn ? Intrinsic::arm_neon_vminu : Intrinsic::arm_neon_vmins;
1875     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
1876   case ARM::BI__builtin_neon_vmovl_v: {
1877     llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
1878     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
1879     if (usgn)
1880       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
1881     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
1882   }
1883   case ARM::BI__builtin_neon_vmovn_v: {
1884     llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
1885     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
1886     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
1887   }
1888   case ARM::BI__builtin_neon_vmul_v:
1889   case ARM::BI__builtin_neon_vmulq_v:
1890     assert(Type.isPoly() && "vmul builtin only supported for polynomial types");
1891     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmulp, Ty),
1892                         Ops, "vmul");
1893   case ARM::BI__builtin_neon_vmull_v:
1894     Int = usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
1895     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
1896     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
1897   case ARM::BI__builtin_neon_vpadal_v:
1898   case ARM::BI__builtin_neon_vpadalq_v: {
1899     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
1900     // The source operand type has twice as many elements of half the size.
1901     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1902     llvm::Type *EltTy =
1903       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1904     llvm::Type *NarrowTy =
1905       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
1906     llvm::Type *Tys[2] = { Ty, NarrowTy };
1907     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpadal");
1908   }
1909   case ARM::BI__builtin_neon_vpadd_v:
1910     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vpadd, Ty),
1911                         Ops, "vpadd");
1912   case ARM::BI__builtin_neon_vpaddl_v:
1913   case ARM::BI__builtin_neon_vpaddlq_v: {
1914     Int = usgn ? Intrinsic::arm_neon_vpaddlu : Intrinsic::arm_neon_vpaddls;
1915     // The source operand type has twice as many elements of half the size.
1916     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
1917     llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
1918     llvm::Type *NarrowTy =
1919       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
1920     llvm::Type *Tys[2] = { Ty, NarrowTy };
1921     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
1922   }
1923   case ARM::BI__builtin_neon_vpmax_v:
1924     Int = usgn ? Intrinsic::arm_neon_vpmaxu : Intrinsic::arm_neon_vpmaxs;
1925     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
1926   case ARM::BI__builtin_neon_vpmin_v:
1927     Int = usgn ? Intrinsic::arm_neon_vpminu : Intrinsic::arm_neon_vpmins;
1928     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
1929   case ARM::BI__builtin_neon_vqabs_v:
1930   case ARM::BI__builtin_neon_vqabsq_v:
1931     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqabs, Ty),
1932                         Ops, "vqabs");
1933   case ARM::BI__builtin_neon_vqadd_v:
1934   case ARM::BI__builtin_neon_vqaddq_v:
1935     Int = usgn ? Intrinsic::arm_neon_vqaddu : Intrinsic::arm_neon_vqadds;
1936     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqadd");
1937   case ARM::BI__builtin_neon_vqdmlal_v:
1938     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmlal, Ty),
1939                         Ops, "vqdmlal");
1940   case ARM::BI__builtin_neon_vqdmlsl_v:
1941     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmlsl, Ty),
1942                         Ops, "vqdmlsl");
1943   case ARM::BI__builtin_neon_vqdmulh_v:
1944   case ARM::BI__builtin_neon_vqdmulhq_v:
1945     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmulh, Ty),
1946                         Ops, "vqdmulh");
1947   case ARM::BI__builtin_neon_vqdmull_v:
1948     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, Ty),
1949                         Ops, "vqdmull");
1950   case ARM::BI__builtin_neon_vqmovn_v:
1951     Int = usgn ? Intrinsic::arm_neon_vqmovnu : Intrinsic::arm_neon_vqmovns;
1952     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqmovn");
1953   case ARM::BI__builtin_neon_vqmovun_v:
1954     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqmovnsu, Ty),
1955                         Ops, "vqdmull");
1956   case ARM::BI__builtin_neon_vqneg_v:
1957   case ARM::BI__builtin_neon_vqnegq_v:
1958     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqneg, Ty),
1959                         Ops, "vqneg");
1960   case ARM::BI__builtin_neon_vqrdmulh_v:
1961   case ARM::BI__builtin_neon_vqrdmulhq_v:
1962     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrdmulh, Ty),
1963                         Ops, "vqrdmulh");
1964   case ARM::BI__builtin_neon_vqrshl_v:
1965   case ARM::BI__builtin_neon_vqrshlq_v:
1966     Int = usgn ? Intrinsic::arm_neon_vqrshiftu : Intrinsic::arm_neon_vqrshifts;
1967     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshl");
1968   case ARM::BI__builtin_neon_vqrshrn_n_v:
1969     Int = usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
1970     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
1971                         1, true);
1972   case ARM::BI__builtin_neon_vqrshrun_n_v:
1973     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
1974                         Ops, "vqrshrun_n", 1, true);
1975   case ARM::BI__builtin_neon_vqshl_v:
1976   case ARM::BI__builtin_neon_vqshlq_v:
1977     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
1978     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl");
1979   case ARM::BI__builtin_neon_vqshl_n_v:
1980   case ARM::BI__builtin_neon_vqshlq_n_v:
1981     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
1982     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
1983                         1, false);
1984   case ARM::BI__builtin_neon_vqshlu_n_v:
1985   case ARM::BI__builtin_neon_vqshluq_n_v:
1986     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftsu, Ty),
1987                         Ops, "vqshlu", 1, false);
1988   case ARM::BI__builtin_neon_vqshrn_n_v:
1989     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
1990     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
1991                         1, true);
1992   case ARM::BI__builtin_neon_vqshrun_n_v:
1993     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
1994                         Ops, "vqshrun_n", 1, true);
1995   case ARM::BI__builtin_neon_vqsub_v:
1996   case ARM::BI__builtin_neon_vqsubq_v:
1997     Int = usgn ? Intrinsic::arm_neon_vqsubu : Intrinsic::arm_neon_vqsubs;
1998     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqsub");
1999   case ARM::BI__builtin_neon_vraddhn_v:
2000     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vraddhn, Ty),
2001                         Ops, "vraddhn");
2002   case ARM::BI__builtin_neon_vrecpe_v:
2003   case ARM::BI__builtin_neon_vrecpeq_v:
2004     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
2005                         Ops, "vrecpe");
2006   case ARM::BI__builtin_neon_vrecps_v:
2007   case ARM::BI__builtin_neon_vrecpsq_v:
2008     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecps, Ty),
2009                         Ops, "vrecps");
2010   case ARM::BI__builtin_neon_vrhadd_v:
2011   case ARM::BI__builtin_neon_vrhaddq_v:
2012     Int = usgn ? Intrinsic::arm_neon_vrhaddu : Intrinsic::arm_neon_vrhadds;
2013     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrhadd");
2014   case ARM::BI__builtin_neon_vrshl_v:
2015   case ARM::BI__builtin_neon_vrshlq_v:
2016     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2017     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshl");
2018   case ARM::BI__builtin_neon_vrshrn_n_v:
2019     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
2020                         Ops, "vrshrn_n", 1, true);
2021   case ARM::BI__builtin_neon_vrshr_n_v:
2022   case ARM::BI__builtin_neon_vrshrq_n_v:
2023     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2024     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
2025   case ARM::BI__builtin_neon_vrsqrte_v:
2026   case ARM::BI__builtin_neon_vrsqrteq_v:
2027     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsqrte, Ty),
2028                         Ops, "vrsqrte");
2029   case ARM::BI__builtin_neon_vrsqrts_v:
2030   case ARM::BI__builtin_neon_vrsqrtsq_v:
2031     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsqrts, Ty),
2032                         Ops, "vrsqrts");
2033   case ARM::BI__builtin_neon_vrsra_n_v:
2034   case ARM::BI__builtin_neon_vrsraq_n_v:
2035     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2036     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2037     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
2038     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
2039     Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
2040     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
2041   case ARM::BI__builtin_neon_vrsubhn_v:
2042     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsubhn, Ty),
2043                         Ops, "vrsubhn");
2044   case ARM::BI__builtin_neon_vshl_v:
2045   case ARM::BI__builtin_neon_vshlq_v:
2046     Int = usgn ? Intrinsic::arm_neon_vshiftu : Intrinsic::arm_neon_vshifts;
2047     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vshl");
2048   case ARM::BI__builtin_neon_vshll_n_v:
2049     Int = usgn ? Intrinsic::arm_neon_vshiftlu : Intrinsic::arm_neon_vshiftls;
2050     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vshll", 1);
2051   case ARM::BI__builtin_neon_vshl_n_v:
2052   case ARM::BI__builtin_neon_vshlq_n_v:
2053     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
2054     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], "vshl_n");
2055   case ARM::BI__builtin_neon_vshrn_n_v:
2056     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftn, Ty),
2057                         Ops, "vshrn_n", 1, true);
2058   case ARM::BI__builtin_neon_vshr_n_v:
2059   case ARM::BI__builtin_neon_vshrq_n_v:
2060     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2061     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
2062     if (usgn)
2063       return Builder.CreateLShr(Ops[0], Ops[1], "vshr_n");
2064     else
2065       return Builder.CreateAShr(Ops[0], Ops[1], "vshr_n");
2066   case ARM::BI__builtin_neon_vsri_n_v:
2067   case ARM::BI__builtin_neon_vsriq_n_v:
2068     rightShift = true;
2069   case ARM::BI__builtin_neon_vsli_n_v:
2070   case ARM::BI__builtin_neon_vsliq_n_v:
2071     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
2072     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
2073                         Ops, "vsli_n");
2074   case ARM::BI__builtin_neon_vsra_n_v:
2075   case ARM::BI__builtin_neon_vsraq_n_v:
2076     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2077     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2078     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, false);
2079     if (usgn)
2080       Ops[1] = Builder.CreateLShr(Ops[1], Ops[2], "vsra_n");
2081     else
2082       Ops[1] = Builder.CreateAShr(Ops[1], Ops[2], "vsra_n");
2083     return Builder.CreateAdd(Ops[0], Ops[1]);
2084   case ARM::BI__builtin_neon_vst1_v:
2085   case ARM::BI__builtin_neon_vst1q_v:
2086     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2087     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, Ty),
2088                         Ops, "");
2089   case ARM::BI__builtin_neon_vst1_lane_v:
2090   case ARM::BI__builtin_neon_vst1q_lane_v: {
2091     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2092     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
2093     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
2094     StoreInst *St = Builder.CreateStore(Ops[1],
2095                                         Builder.CreateBitCast(Ops[0], Ty));
2096     Value *Align = GetPointeeAlignmentValue(E->getArg(0));
2097     St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
2098     return St;
2099   }
2100   case ARM::BI__builtin_neon_vst2_v:
2101   case ARM::BI__builtin_neon_vst2q_v:
2102     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2103     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2, Ty),
2104                         Ops, "");
2105   case ARM::BI__builtin_neon_vst2_lane_v:
2106   case ARM::BI__builtin_neon_vst2q_lane_v:
2107     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2108     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2lane, Ty),
2109                         Ops, "");
2110   case ARM::BI__builtin_neon_vst3_v:
2111   case ARM::BI__builtin_neon_vst3q_v:
2112     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2113     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3, Ty),
2114                         Ops, "");
2115   case ARM::BI__builtin_neon_vst3_lane_v:
2116   case ARM::BI__builtin_neon_vst3q_lane_v:
2117     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2118     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3lane, Ty),
2119                         Ops, "");
2120   case ARM::BI__builtin_neon_vst4_v:
2121   case ARM::BI__builtin_neon_vst4q_v:
2122     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2123     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4, Ty),
2124                         Ops, "");
2125   case ARM::BI__builtin_neon_vst4_lane_v:
2126   case ARM::BI__builtin_neon_vst4q_lane_v:
2127     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
2128     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4lane, Ty),
2129                         Ops, "");
2130   case ARM::BI__builtin_neon_vsubhn_v:
2131     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vsubhn, Ty),
2132                         Ops, "vsubhn");
2133   case ARM::BI__builtin_neon_vtbl1_v:
2134     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
2135                         Ops, "vtbl1");
2136   case ARM::BI__builtin_neon_vtbl2_v:
2137     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
2138                         Ops, "vtbl2");
2139   case ARM::BI__builtin_neon_vtbl3_v:
2140     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
2141                         Ops, "vtbl3");
2142   case ARM::BI__builtin_neon_vtbl4_v:
2143     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
2144                         Ops, "vtbl4");
2145   case ARM::BI__builtin_neon_vtbx1_v:
2146     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
2147                         Ops, "vtbx1");
2148   case ARM::BI__builtin_neon_vtbx2_v:
2149     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
2150                         Ops, "vtbx2");
2151   case ARM::BI__builtin_neon_vtbx3_v:
2152     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
2153                         Ops, "vtbx3");
2154   case ARM::BI__builtin_neon_vtbx4_v:
2155     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
2156                         Ops, "vtbx4");
2157   case ARM::BI__builtin_neon_vtst_v:
2158   case ARM::BI__builtin_neon_vtstq_v: {
2159     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2160     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2161     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
2162     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
2163                                 ConstantAggregateZero::get(Ty));
2164     return Builder.CreateSExt(Ops[0], Ty, "vtst");
2165   }
2166   case ARM::BI__builtin_neon_vtrn_v:
2167   case ARM::BI__builtin_neon_vtrnq_v: {
2168     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2169     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2170     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2171     Value *SV = 0;
2172 
2173     for (unsigned vi = 0; vi != 2; ++vi) {
2174       SmallVector<Constant*, 16> Indices;
2175       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2176         Indices.push_back(Builder.getInt32(i+vi));
2177         Indices.push_back(Builder.getInt32(i+e+vi));
2178       }
2179       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2180       SV = llvm::ConstantVector::get(Indices);
2181       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
2182       SV = Builder.CreateStore(SV, Addr);
2183     }
2184     return SV;
2185   }
2186   case ARM::BI__builtin_neon_vuzp_v:
2187   case ARM::BI__builtin_neon_vuzpq_v: {
2188     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2189     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2190     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2191     Value *SV = 0;
2192 
2193     for (unsigned vi = 0; vi != 2; ++vi) {
2194       SmallVector<Constant*, 16> Indices;
2195       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
2196         Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
2197 
2198       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2199       SV = llvm::ConstantVector::get(Indices);
2200       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
2201       SV = Builder.CreateStore(SV, Addr);
2202     }
2203     return SV;
2204   }
2205   case ARM::BI__builtin_neon_vzip_v:
2206   case ARM::BI__builtin_neon_vzipq_v: {
2207     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2208     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2209     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2210     Value *SV = 0;
2211 
2212     for (unsigned vi = 0; vi != 2; ++vi) {
2213       SmallVector<Constant*, 16> Indices;
2214       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2215         Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
2216         Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
2217       }
2218       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2219       SV = llvm::ConstantVector::get(Indices);
2220       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
2221       SV = Builder.CreateStore(SV, Addr);
2222     }
2223     return SV;
2224   }
2225   }
2226 }
2227 
2228 llvm::Value *CodeGenFunction::
2229 BuildVector(ArrayRef<llvm::Value*> Ops) {
2230   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
2231          "Not a power-of-two sized vector!");
2232   bool AllConstants = true;
2233   for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
2234     AllConstants &= isa<Constant>(Ops[i]);
2235 
2236   // If this is a constant vector, create a ConstantVector.
2237   if (AllConstants) {
2238     SmallVector<llvm::Constant*, 16> CstOps;
2239     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2240       CstOps.push_back(cast<Constant>(Ops[i]));
2241     return llvm::ConstantVector::get(CstOps);
2242   }
2243 
2244   // Otherwise, insertelement the values to build the vector.
2245   Value *Result =
2246     llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
2247 
2248   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2249     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
2250 
2251   return Result;
2252 }
2253 
2254 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
2255                                            const CallExpr *E) {
2256   SmallVector<Value*, 4> Ops;
2257 
2258   // Find out if any arguments are required to be integer constant expressions.
2259   unsigned ICEArguments = 0;
2260   ASTContext::GetBuiltinTypeError Error;
2261   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2262   assert(Error == ASTContext::GE_None && "Should not codegen an error");
2263 
2264   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
2265     // If this is a normal argument, just emit it as a scalar.
2266     if ((ICEArguments & (1 << i)) == 0) {
2267       Ops.push_back(EmitScalarExpr(E->getArg(i)));
2268       continue;
2269     }
2270 
2271     // If this is required to be a constant, constant fold it so that we know
2272     // that the generated intrinsic gets a ConstantInt.
2273     llvm::APSInt Result;
2274     bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
2275     assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
2276     Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
2277   }
2278 
2279   switch (BuiltinID) {
2280   default: return 0;
2281   case X86::BI__builtin_ia32_vec_init_v8qi:
2282   case X86::BI__builtin_ia32_vec_init_v4hi:
2283   case X86::BI__builtin_ia32_vec_init_v2si:
2284     return Builder.CreateBitCast(BuildVector(Ops),
2285                                  llvm::Type::getX86_MMXTy(getLLVMContext()));
2286   case X86::BI__builtin_ia32_vec_ext_v2si:
2287     return Builder.CreateExtractElement(Ops[0],
2288                                   llvm::ConstantInt::get(Ops[1]->getType(), 0));
2289   case X86::BI__builtin_ia32_ldmxcsr: {
2290     llvm::Type *PtrTy = Int8PtrTy;
2291     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
2292     Value *Tmp = Builder.CreateAlloca(Int32Ty, One);
2293     Builder.CreateStore(Ops[0], Tmp);
2294     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
2295                               Builder.CreateBitCast(Tmp, PtrTy));
2296   }
2297   case X86::BI__builtin_ia32_stmxcsr: {
2298     llvm::Type *PtrTy = Int8PtrTy;
2299     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
2300     Value *Tmp = Builder.CreateAlloca(Int32Ty, One);
2301     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
2302                        Builder.CreateBitCast(Tmp, PtrTy));
2303     return Builder.CreateLoad(Tmp, "stmxcsr");
2304   }
2305   case X86::BI__builtin_ia32_storehps:
2306   case X86::BI__builtin_ia32_storelps: {
2307     llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
2308     llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
2309 
2310     // cast val v2i64
2311     Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
2312 
2313     // extract (0, 1)
2314     unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
2315     llvm::Value *Idx = llvm::ConstantInt::get(Int32Ty, Index);
2316     Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract");
2317 
2318     // cast pointer to i64 & store
2319     Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
2320     return Builder.CreateStore(Ops[1], Ops[0]);
2321   }
2322   case X86::BI__builtin_ia32_palignr: {
2323     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
2324 
2325     // If palignr is shifting the pair of input vectors less than 9 bytes,
2326     // emit a shuffle instruction.
2327     if (shiftVal <= 8) {
2328       SmallVector<llvm::Constant*, 8> Indices;
2329       for (unsigned i = 0; i != 8; ++i)
2330         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
2331 
2332       Value* SV = llvm::ConstantVector::get(Indices);
2333       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
2334     }
2335 
2336     // If palignr is shifting the pair of input vectors more than 8 but less
2337     // than 16 bytes, emit a logical right shift of the destination.
2338     if (shiftVal < 16) {
2339       // MMX has these as 1 x i64 vectors for some odd optimization reasons.
2340       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
2341 
2342       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
2343       Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
2344 
2345       // create i32 constant
2346       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
2347       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
2348     }
2349 
2350     // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
2351     return llvm::Constant::getNullValue(ConvertType(E->getType()));
2352   }
2353   case X86::BI__builtin_ia32_palignr128: {
2354     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
2355 
2356     // If palignr is shifting the pair of input vectors less than 17 bytes,
2357     // emit a shuffle instruction.
2358     if (shiftVal <= 16) {
2359       SmallVector<llvm::Constant*, 16> Indices;
2360       for (unsigned i = 0; i != 16; ++i)
2361         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
2362 
2363       Value* SV = llvm::ConstantVector::get(Indices);
2364       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
2365     }
2366 
2367     // If palignr is shifting the pair of input vectors more than 16 but less
2368     // than 32 bytes, emit a logical right shift of the destination.
2369     if (shiftVal < 32) {
2370       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
2371 
2372       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
2373       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
2374 
2375       // create i32 constant
2376       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
2377       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
2378     }
2379 
2380     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
2381     return llvm::Constant::getNullValue(ConvertType(E->getType()));
2382   }
2383   case X86::BI__builtin_ia32_palignr256: {
2384     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
2385 
2386     // If palignr is shifting the pair of input vectors less than 17 bytes,
2387     // emit a shuffle instruction.
2388     if (shiftVal <= 16) {
2389       SmallVector<llvm::Constant*, 32> Indices;
2390       // 256-bit palignr operates on 128-bit lanes so we need to handle that
2391       for (unsigned l = 0; l != 2; ++l) {
2392         unsigned LaneStart = l * 16;
2393         unsigned LaneEnd = (l+1) * 16;
2394         for (unsigned i = 0; i != 16; ++i) {
2395           unsigned Idx = shiftVal + i + LaneStart;
2396           if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand
2397           Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx));
2398         }
2399       }
2400 
2401       Value* SV = llvm::ConstantVector::get(Indices);
2402       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
2403     }
2404 
2405     // If palignr is shifting the pair of input vectors more than 16 but less
2406     // than 32 bytes, emit a logical right shift of the destination.
2407     if (shiftVal < 32) {
2408       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4);
2409 
2410       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
2411       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
2412 
2413       // create i32 constant
2414       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq);
2415       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
2416     }
2417 
2418     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
2419     return llvm::Constant::getNullValue(ConvertType(E->getType()));
2420   }
2421   case X86::BI__builtin_ia32_movntps:
2422   case X86::BI__builtin_ia32_movntps256:
2423   case X86::BI__builtin_ia32_movntpd:
2424   case X86::BI__builtin_ia32_movntpd256:
2425   case X86::BI__builtin_ia32_movntdq:
2426   case X86::BI__builtin_ia32_movntdq256:
2427   case X86::BI__builtin_ia32_movnti: {
2428     llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(),
2429                                            Builder.getInt32(1));
2430 
2431     // Convert the type of the pointer to a pointer to the stored type.
2432     Value *BC = Builder.CreateBitCast(Ops[0],
2433                                 llvm::PointerType::getUnqual(Ops[1]->getType()),
2434                                       "cast");
2435     StoreInst *SI = Builder.CreateStore(Ops[1], BC);
2436     SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
2437     SI->setAlignment(16);
2438     return SI;
2439   }
2440   // 3DNow!
2441   case X86::BI__builtin_ia32_pswapdsf:
2442   case X86::BI__builtin_ia32_pswapdsi: {
2443     const char *name = 0;
2444     Intrinsic::ID ID = Intrinsic::not_intrinsic;
2445     switch(BuiltinID) {
2446     default: llvm_unreachable("Unsupported intrinsic!");
2447     case X86::BI__builtin_ia32_pswapdsf:
2448     case X86::BI__builtin_ia32_pswapdsi:
2449       name = "pswapd";
2450       ID = Intrinsic::x86_3dnowa_pswapd;
2451       break;
2452     }
2453     llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
2454     Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
2455     llvm::Function *F = CGM.getIntrinsic(ID);
2456     return Builder.CreateCall(F, Ops, name);
2457   }
2458   case X86::BI__builtin_ia32_rdrand16_step:
2459   case X86::BI__builtin_ia32_rdrand32_step:
2460   case X86::BI__builtin_ia32_rdrand64_step: {
2461     Intrinsic::ID ID;
2462     switch (BuiltinID) {
2463     default: llvm_unreachable("Unsupported intrinsic!");
2464     case X86::BI__builtin_ia32_rdrand16_step:
2465       ID = Intrinsic::x86_rdrand_16;
2466       break;
2467     case X86::BI__builtin_ia32_rdrand32_step:
2468       ID = Intrinsic::x86_rdrand_32;
2469       break;
2470     case X86::BI__builtin_ia32_rdrand64_step:
2471       ID = Intrinsic::x86_rdrand_64;
2472       break;
2473     }
2474 
2475     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
2476     Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
2477     return Builder.CreateExtractValue(Call, 1);
2478   }
2479   }
2480 }
2481 
2482 
2483 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
2484                                            const CallExpr *E) {
2485   SmallVector<Value*, 4> Ops;
2486 
2487   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
2488     Ops.push_back(EmitScalarExpr(E->getArg(i)));
2489 
2490   Intrinsic::ID ID = Intrinsic::not_intrinsic;
2491 
2492   switch (BuiltinID) {
2493   default: return 0;
2494 
2495   // vec_ld, vec_lvsl, vec_lvsr
2496   case PPC::BI__builtin_altivec_lvx:
2497   case PPC::BI__builtin_altivec_lvxl:
2498   case PPC::BI__builtin_altivec_lvebx:
2499   case PPC::BI__builtin_altivec_lvehx:
2500   case PPC::BI__builtin_altivec_lvewx:
2501   case PPC::BI__builtin_altivec_lvsl:
2502   case PPC::BI__builtin_altivec_lvsr:
2503   {
2504     Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
2505 
2506     Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
2507     Ops.pop_back();
2508 
2509     switch (BuiltinID) {
2510     default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
2511     case PPC::BI__builtin_altivec_lvx:
2512       ID = Intrinsic::ppc_altivec_lvx;
2513       break;
2514     case PPC::BI__builtin_altivec_lvxl:
2515       ID = Intrinsic::ppc_altivec_lvxl;
2516       break;
2517     case PPC::BI__builtin_altivec_lvebx:
2518       ID = Intrinsic::ppc_altivec_lvebx;
2519       break;
2520     case PPC::BI__builtin_altivec_lvehx:
2521       ID = Intrinsic::ppc_altivec_lvehx;
2522       break;
2523     case PPC::BI__builtin_altivec_lvewx:
2524       ID = Intrinsic::ppc_altivec_lvewx;
2525       break;
2526     case PPC::BI__builtin_altivec_lvsl:
2527       ID = Intrinsic::ppc_altivec_lvsl;
2528       break;
2529     case PPC::BI__builtin_altivec_lvsr:
2530       ID = Intrinsic::ppc_altivec_lvsr;
2531       break;
2532     }
2533     llvm::Function *F = CGM.getIntrinsic(ID);
2534     return Builder.CreateCall(F, Ops, "");
2535   }
2536 
2537   // vec_st
2538   case PPC::BI__builtin_altivec_stvx:
2539   case PPC::BI__builtin_altivec_stvxl:
2540   case PPC::BI__builtin_altivec_stvebx:
2541   case PPC::BI__builtin_altivec_stvehx:
2542   case PPC::BI__builtin_altivec_stvewx:
2543   {
2544     Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
2545     Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
2546     Ops.pop_back();
2547 
2548     switch (BuiltinID) {
2549     default: llvm_unreachable("Unsupported st intrinsic!");
2550     case PPC::BI__builtin_altivec_stvx:
2551       ID = Intrinsic::ppc_altivec_stvx;
2552       break;
2553     case PPC::BI__builtin_altivec_stvxl:
2554       ID = Intrinsic::ppc_altivec_stvxl;
2555       break;
2556     case PPC::BI__builtin_altivec_stvebx:
2557       ID = Intrinsic::ppc_altivec_stvebx;
2558       break;
2559     case PPC::BI__builtin_altivec_stvehx:
2560       ID = Intrinsic::ppc_altivec_stvehx;
2561       break;
2562     case PPC::BI__builtin_altivec_stvewx:
2563       ID = Intrinsic::ppc_altivec_stvewx;
2564       break;
2565     }
2566     llvm::Function *F = CGM.getIntrinsic(ID);
2567     return Builder.CreateCall(F, Ops, "");
2568   }
2569   }
2570 }
2571