1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit Builtin calls as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCXXABI.h"
15 #include "CGObjCRuntime.h"
16 #include "CGOpenCLRuntime.h"
17 #include "CGRecordLayout.h"
18 #include "CodeGenFunction.h"
19 #include "CodeGenModule.h"
20 #include "ConstantEmitter.h"
21 #include "TargetInfo.h"
22 #include "clang/AST/ASTContext.h"
23 #include "clang/AST/Decl.h"
24 #include "clang/Analysis/Analyses/OSLog.h"
25 #include "clang/Basic/TargetBuiltins.h"
26 #include "clang/Basic/TargetInfo.h"
27 #include "clang/CodeGen/CGFunctionInfo.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/InlineAsm.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/IR/MDBuilder.h"
34 #include "llvm/Support/ConvertUTF.h"
35 #include "llvm/Support/ScopedPrinter.h"
36 #include "llvm/Support/TargetParser.h"
37 #include <sstream>
38 
39 using namespace clang;
40 using namespace CodeGen;
41 using namespace llvm;
42 
43 static
44 int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
45   return std::min(High, std::max(Low, Value));
46 }
47 
48 /// getBuiltinLibFunction - Given a builtin id for a function like
49 /// "__builtin_fabsf", return a Function* for "fabsf".
50 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
51                                                      unsigned BuiltinID) {
52   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
53 
54   // Get the name, skip over the __builtin_ prefix (if necessary).
55   StringRef Name;
56   GlobalDecl D(FD);
57 
58   // If the builtin has been declared explicitly with an assembler label,
59   // use the mangled name. This differs from the plain label on platforms
60   // that prefix labels.
61   if (FD->hasAttr<AsmLabelAttr>())
62     Name = getMangledName(D);
63   else
64     Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
65 
66   llvm::FunctionType *Ty =
67     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
68 
69   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
70 }
71 
72 /// Emit the conversions required to turn the given value into an
73 /// integer of the given size.
74 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
75                         QualType T, llvm::IntegerType *IntType) {
76   V = CGF.EmitToMemory(V, T);
77 
78   if (V->getType()->isPointerTy())
79     return CGF.Builder.CreatePtrToInt(V, IntType);
80 
81   assert(V->getType() == IntType);
82   return V;
83 }
84 
85 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
86                           QualType T, llvm::Type *ResultType) {
87   V = CGF.EmitFromMemory(V, T);
88 
89   if (ResultType->isPointerTy())
90     return CGF.Builder.CreateIntToPtr(V, ResultType);
91 
92   assert(V->getType() == ResultType);
93   return V;
94 }
95 
96 /// Utility to insert an atomic instruction based on Instrinsic::ID
97 /// and the expression node.
98 static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF,
99                                     llvm::AtomicRMWInst::BinOp Kind,
100                                     const CallExpr *E) {
101   QualType T = E->getType();
102   assert(E->getArg(0)->getType()->isPointerType());
103   assert(CGF.getContext().hasSameUnqualifiedType(T,
104                                   E->getArg(0)->getType()->getPointeeType()));
105   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
106 
107   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
108   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
109 
110   llvm::IntegerType *IntType =
111     llvm::IntegerType::get(CGF.getLLVMContext(),
112                            CGF.getContext().getTypeSize(T));
113   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
114 
115   llvm::Value *Args[2];
116   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
117   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
118   llvm::Type *ValueType = Args[1]->getType();
119   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
120 
121   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
122       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
123   return EmitFromInt(CGF, Result, T, ValueType);
124 }
125 
126 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
127   Value *Val = CGF.EmitScalarExpr(E->getArg(0));
128   Value *Address = CGF.EmitScalarExpr(E->getArg(1));
129 
130   // Convert the type of the pointer to a pointer to the stored type.
131   Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
132   Value *BC = CGF.Builder.CreateBitCast(
133       Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
134   LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
135   LV.setNontemporal(true);
136   CGF.EmitStoreOfScalar(Val, LV, false);
137   return nullptr;
138 }
139 
140 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
141   Value *Address = CGF.EmitScalarExpr(E->getArg(0));
142 
143   LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
144   LV.setNontemporal(true);
145   return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
146 }
147 
148 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
149                                llvm::AtomicRMWInst::BinOp Kind,
150                                const CallExpr *E) {
151   return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
152 }
153 
154 /// Utility to insert an atomic instruction based Instrinsic::ID and
155 /// the expression node, where the return value is the result of the
156 /// operation.
157 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
158                                    llvm::AtomicRMWInst::BinOp Kind,
159                                    const CallExpr *E,
160                                    Instruction::BinaryOps Op,
161                                    bool Invert = false) {
162   QualType T = E->getType();
163   assert(E->getArg(0)->getType()->isPointerType());
164   assert(CGF.getContext().hasSameUnqualifiedType(T,
165                                   E->getArg(0)->getType()->getPointeeType()));
166   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
167 
168   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
169   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
170 
171   llvm::IntegerType *IntType =
172     llvm::IntegerType::get(CGF.getLLVMContext(),
173                            CGF.getContext().getTypeSize(T));
174   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
175 
176   llvm::Value *Args[2];
177   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
178   llvm::Type *ValueType = Args[1]->getType();
179   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
180   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
181 
182   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
183       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
184   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
185   if (Invert)
186     Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
187                                      llvm::ConstantInt::get(IntType, -1));
188   Result = EmitFromInt(CGF, Result, T, ValueType);
189   return RValue::get(Result);
190 }
191 
192 /// Utility to insert an atomic cmpxchg instruction.
193 ///
194 /// @param CGF The current codegen function.
195 /// @param E   Builtin call expression to convert to cmpxchg.
196 ///            arg0 - address to operate on
197 ///            arg1 - value to compare with
198 ///            arg2 - new value
199 /// @param ReturnBool Specifies whether to return success flag of
200 ///                   cmpxchg result or the old value.
201 ///
202 /// @returns result of cmpxchg, according to ReturnBool
203 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
204                                      bool ReturnBool) {
205   QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
206   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
207   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
208 
209   llvm::IntegerType *IntType = llvm::IntegerType::get(
210       CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
211   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
212 
213   Value *Args[3];
214   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
215   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
216   llvm::Type *ValueType = Args[1]->getType();
217   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
218   Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
219 
220   Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
221       Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
222       llvm::AtomicOrdering::SequentiallyConsistent);
223   if (ReturnBool)
224     // Extract boolean success flag and zext it to int.
225     return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
226                                   CGF.ConvertType(E->getType()));
227   else
228     // Extract old value and emit it using the same type as compare value.
229     return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
230                        ValueType);
231 }
232 
233 // Emit a simple mangled intrinsic that has 1 argument and a return type
234 // matching the argument type.
235 static Value *emitUnaryBuiltin(CodeGenFunction &CGF,
236                                const CallExpr *E,
237                                unsigned IntrinsicID) {
238   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
239 
240   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
241   return CGF.Builder.CreateCall(F, Src0);
242 }
243 
244 // Emit an intrinsic that has 2 operands of the same type as its result.
245 static Value *emitBinaryBuiltin(CodeGenFunction &CGF,
246                                 const CallExpr *E,
247                                 unsigned IntrinsicID) {
248   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
249   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
250 
251   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
252   return CGF.Builder.CreateCall(F, { Src0, Src1 });
253 }
254 
255 // Emit an intrinsic that has 3 operands of the same type as its result.
256 static Value *emitTernaryBuiltin(CodeGenFunction &CGF,
257                                  const CallExpr *E,
258                                  unsigned IntrinsicID) {
259   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
260   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
261   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
262 
263   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
264   return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
265 }
266 
267 // Emit an intrinsic that has 1 float or double operand, and 1 integer.
268 static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
269                                const CallExpr *E,
270                                unsigned IntrinsicID) {
271   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
272   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
273 
274   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
275   return CGF.Builder.CreateCall(F, {Src0, Src1});
276 }
277 
278 /// EmitFAbs - Emit a call to @llvm.fabs().
279 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
280   Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
281   llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
282   Call->setDoesNotAccessMemory();
283   return Call;
284 }
285 
286 /// Emit the computation of the sign bit for a floating point value. Returns
287 /// the i1 sign bit value.
288 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
289   LLVMContext &C = CGF.CGM.getLLVMContext();
290 
291   llvm::Type *Ty = V->getType();
292   int Width = Ty->getPrimitiveSizeInBits();
293   llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
294   V = CGF.Builder.CreateBitCast(V, IntTy);
295   if (Ty->isPPC_FP128Ty()) {
296     // We want the sign bit of the higher-order double. The bitcast we just
297     // did works as if the double-double was stored to memory and then
298     // read as an i128. The "store" will put the higher-order double in the
299     // lower address in both little- and big-Endian modes, but the "load"
300     // will treat those bits as a different part of the i128: the low bits in
301     // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
302     // we need to shift the high bits down to the low before truncating.
303     Width >>= 1;
304     if (CGF.getTarget().isBigEndian()) {
305       Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
306       V = CGF.Builder.CreateLShr(V, ShiftCst);
307     }
308     // We are truncating value in order to extract the higher-order
309     // double, which we will be using to extract the sign from.
310     IntTy = llvm::IntegerType::get(C, Width);
311     V = CGF.Builder.CreateTrunc(V, IntTy);
312   }
313   Value *Zero = llvm::Constant::getNullValue(IntTy);
314   return CGF.Builder.CreateICmpSLT(V, Zero);
315 }
316 
317 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
318                               const CallExpr *E, llvm::Constant *calleeValue) {
319   CGCallee callee = CGCallee::forDirect(calleeValue, FD);
320   return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
321 }
322 
323 /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
324 /// depending on IntrinsicID.
325 ///
326 /// \arg CGF The current codegen function.
327 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
328 /// \arg X The first argument to the llvm.*.with.overflow.*.
329 /// \arg Y The second argument to the llvm.*.with.overflow.*.
330 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
331 /// \returns The result (i.e. sum/product) returned by the intrinsic.
332 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
333                                           const llvm::Intrinsic::ID IntrinsicID,
334                                           llvm::Value *X, llvm::Value *Y,
335                                           llvm::Value *&Carry) {
336   // Make sure we have integers of the same width.
337   assert(X->getType() == Y->getType() &&
338          "Arguments must be the same type. (Did you forget to make sure both "
339          "arguments have the same integer width?)");
340 
341   llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
342   llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
343   Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
344   return CGF.Builder.CreateExtractValue(Tmp, 0);
345 }
346 
347 static Value *emitRangedBuiltin(CodeGenFunction &CGF,
348                                 unsigned IntrinsicID,
349                                 int low, int high) {
350     llvm::MDBuilder MDHelper(CGF.getLLVMContext());
351     llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
352     Value *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
353     llvm::Instruction *Call = CGF.Builder.CreateCall(F);
354     Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
355     return Call;
356 }
357 
358 namespace {
359   struct WidthAndSignedness {
360     unsigned Width;
361     bool Signed;
362   };
363 }
364 
365 static WidthAndSignedness
366 getIntegerWidthAndSignedness(const clang::ASTContext &context,
367                              const clang::QualType Type) {
368   assert(Type->isIntegerType() && "Given type is not an integer.");
369   unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width;
370   bool Signed = Type->isSignedIntegerType();
371   return {Width, Signed};
372 }
373 
374 // Given one or more integer types, this function produces an integer type that
375 // encompasses them: any value in one of the given types could be expressed in
376 // the encompassing type.
377 static struct WidthAndSignedness
378 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
379   assert(Types.size() > 0 && "Empty list of types.");
380 
381   // If any of the given types is signed, we must return a signed type.
382   bool Signed = false;
383   for (const auto &Type : Types) {
384     Signed |= Type.Signed;
385   }
386 
387   // The encompassing type must have a width greater than or equal to the width
388   // of the specified types.  Additionally, if the encompassing type is signed,
389   // its width must be strictly greater than the width of any unsigned types
390   // given.
391   unsigned Width = 0;
392   for (const auto &Type : Types) {
393     unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
394     if (Width < MinWidth) {
395       Width = MinWidth;
396     }
397   }
398 
399   return {Width, Signed};
400 }
401 
402 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
403   llvm::Type *DestType = Int8PtrTy;
404   if (ArgValue->getType() != DestType)
405     ArgValue =
406         Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
407 
408   Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
409   return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
410 }
411 
412 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
413 /// __builtin_object_size(p, @p To) is correct
414 static bool areBOSTypesCompatible(int From, int To) {
415   // Note: Our __builtin_object_size implementation currently treats Type=0 and
416   // Type=2 identically. Encoding this implementation detail here may make
417   // improving __builtin_object_size difficult in the future, so it's omitted.
418   return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
419 }
420 
421 static llvm::Value *
422 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
423   return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
424 }
425 
426 llvm::Value *
427 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
428                                                  llvm::IntegerType *ResType,
429                                                  llvm::Value *EmittedE) {
430   uint64_t ObjectSize;
431   if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
432     return emitBuiltinObjectSize(E, Type, ResType, EmittedE);
433   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
434 }
435 
436 /// Returns a Value corresponding to the size of the given expression.
437 /// This Value may be either of the following:
438 ///   - A llvm::Argument (if E is a param with the pass_object_size attribute on
439 ///     it)
440 ///   - A call to the @llvm.objectsize intrinsic
441 ///
442 /// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
443 /// and we wouldn't otherwise try to reference a pass_object_size parameter,
444 /// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
445 llvm::Value *
446 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
447                                        llvm::IntegerType *ResType,
448                                        llvm::Value *EmittedE) {
449   // We need to reference an argument if the pointer is a parameter with the
450   // pass_object_size attribute.
451   if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
452     auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
453     auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
454     if (Param != nullptr && PS != nullptr &&
455         areBOSTypesCompatible(PS->getType(), Type)) {
456       auto Iter = SizeArguments.find(Param);
457       assert(Iter != SizeArguments.end());
458 
459       const ImplicitParamDecl *D = Iter->second;
460       auto DIter = LocalDeclMap.find(D);
461       assert(DIter != LocalDeclMap.end());
462 
463       return EmitLoadOfScalar(DIter->second, /*volatile=*/false,
464                               getContext().getSizeType(), E->getLocStart());
465     }
466   }
467 
468   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
469   // evaluate E for side-effects. In either case, we shouldn't lower to
470   // @llvm.objectsize.
471   if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
472     return getDefaultBuiltinObjectSizeResult(Type, ResType);
473 
474   Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
475   assert(Ptr->getType()->isPointerTy() &&
476          "Non-pointer passed to __builtin_object_size?");
477 
478   Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
479 
480   // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
481   Value *Min = Builder.getInt1((Type & 2) != 0);
482   // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
483   Value *NullIsUnknown = Builder.getTrue();
484   return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
485 }
486 
487 namespace {
488 /// A struct to generically desribe a bit test intrinsic.
489 struct BitTest {
490   enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
491   enum InterlockingKind : uint8_t {
492     Unlocked,
493     Sequential,
494     Acquire,
495     Release,
496     NoFence
497   };
498 
499   ActionKind Action;
500   InterlockingKind Interlocking;
501   bool Is64Bit;
502 
503   static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
504 };
505 } // namespace
506 
507 BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
508   switch (BuiltinID) {
509     // Main portable variants.
510   case Builtin::BI_bittest:
511     return {TestOnly, Unlocked, false};
512   case Builtin::BI_bittestandcomplement:
513     return {Complement, Unlocked, false};
514   case Builtin::BI_bittestandreset:
515     return {Reset, Unlocked, false};
516   case Builtin::BI_bittestandset:
517     return {Set, Unlocked, false};
518   case Builtin::BI_interlockedbittestandreset:
519     return {Reset, Sequential, false};
520   case Builtin::BI_interlockedbittestandset:
521     return {Set, Sequential, false};
522 
523     // X86-specific 64-bit variants.
524   case Builtin::BI_bittest64:
525     return {TestOnly, Unlocked, true};
526   case Builtin::BI_bittestandcomplement64:
527     return {Complement, Unlocked, true};
528   case Builtin::BI_bittestandreset64:
529     return {Reset, Unlocked, true};
530   case Builtin::BI_bittestandset64:
531     return {Set, Unlocked, true};
532   case Builtin::BI_interlockedbittestandreset64:
533     return {Reset, Sequential, true};
534   case Builtin::BI_interlockedbittestandset64:
535     return {Set, Sequential, true};
536 
537     // ARM/AArch64-specific ordering variants.
538   case Builtin::BI_interlockedbittestandset_acq:
539     return {Set, Acquire, false};
540   case Builtin::BI_interlockedbittestandset_rel:
541     return {Set, Release, false};
542   case Builtin::BI_interlockedbittestandset_nf:
543     return {Set, NoFence, false};
544   case Builtin::BI_interlockedbittestandreset_acq:
545     return {Reset, Acquire, false};
546   case Builtin::BI_interlockedbittestandreset_rel:
547     return {Reset, Release, false};
548   case Builtin::BI_interlockedbittestandreset_nf:
549     return {Reset, NoFence, false};
550   }
551   llvm_unreachable("expected only bittest intrinsics");
552 }
553 
554 static char bitActionToX86BTCode(BitTest::ActionKind A) {
555   switch (A) {
556   case BitTest::TestOnly:   return '\0';
557   case BitTest::Complement: return 'c';
558   case BitTest::Reset:      return 'r';
559   case BitTest::Set:        return 's';
560   }
561   llvm_unreachable("invalid action");
562 }
563 
564 static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
565                                             BitTest BT,
566                                             const CallExpr *E, Value *BitBase,
567                                             Value *BitPos) {
568   char Action = bitActionToX86BTCode(BT.Action);
569   char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
570 
571   // Build the assembly.
572   SmallString<64> Asm;
573   raw_svector_ostream AsmOS(Asm);
574   if (BT.Interlocking != BitTest::Unlocked)
575     AsmOS << "lock ";
576   AsmOS << "bt";
577   if (Action)
578     AsmOS << Action;
579   AsmOS << SizeSuffix << " $2, ($1)\n\tsetc ${0:b}";
580 
581   // Build the constraints. FIXME: We should support immediates when possible.
582   std::string Constraints = "=r,r,r,~{cc},~{flags},~{fpsr}";
583   llvm::IntegerType *IntType = llvm::IntegerType::get(
584       CGF.getLLVMContext(),
585       CGF.getContext().getTypeSize(E->getArg(1)->getType()));
586   llvm::Type *IntPtrType = IntType->getPointerTo();
587   llvm::FunctionType *FTy =
588       llvm::FunctionType::get(CGF.Int8Ty, {IntPtrType, IntType}, false);
589 
590   llvm::InlineAsm *IA =
591       llvm::InlineAsm::get(FTy, Asm, Constraints, /*SideEffects=*/true);
592   return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
593 }
594 
595 static llvm::AtomicOrdering
596 getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
597   switch (I) {
598   case BitTest::Unlocked:   return llvm::AtomicOrdering::NotAtomic;
599   case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
600   case BitTest::Acquire:    return llvm::AtomicOrdering::Acquire;
601   case BitTest::Release:    return llvm::AtomicOrdering::Release;
602   case BitTest::NoFence:    return llvm::AtomicOrdering::Monotonic;
603   }
604   llvm_unreachable("invalid interlocking");
605 }
606 
607 /// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
608 /// bits and a bit position and read and optionally modify the bit at that
609 /// position. The position index can be arbitrarily large, i.e. it can be larger
610 /// than 31 or 63, so we need an indexed load in the general case.
611 static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
612                                          unsigned BuiltinID,
613                                          const CallExpr *E) {
614   Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
615   Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
616 
617   BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
618 
619   // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
620   // indexing operation internally. Use them if possible.
621   llvm::Triple::ArchType Arch = CGF.getTarget().getTriple().getArch();
622   if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64)
623     return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
624 
625   // Otherwise, use generic code to load one byte and test the bit. Use all but
626   // the bottom three bits as the array index, and the bottom three bits to form
627   // a mask.
628   // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
629   Value *ByteIndex = CGF.Builder.CreateAShr(
630       BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
631   Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
632   Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
633                                                  ByteIndex, "bittest.byteaddr"),
634                    CharUnits::One());
635   Value *PosLow =
636       CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
637                             llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
638 
639   // The updating instructions will need a mask.
640   Value *Mask = nullptr;
641   if (BT.Action != BitTest::TestOnly) {
642     Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
643                                  "bittest.mask");
644   }
645 
646   // Check the action and ordering of the interlocked intrinsics.
647   llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
648 
649   Value *OldByte = nullptr;
650   if (Ordering != llvm::AtomicOrdering::NotAtomic) {
651     // Emit a combined atomicrmw load/store operation for the interlocked
652     // intrinsics.
653     llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
654     if (BT.Action == BitTest::Reset) {
655       Mask = CGF.Builder.CreateNot(Mask);
656       RMWOp = llvm::AtomicRMWInst::And;
657     }
658     OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
659                                           Ordering);
660   } else {
661     // Emit a plain load for the non-interlocked intrinsics.
662     OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
663     Value *NewByte = nullptr;
664     switch (BT.Action) {
665     case BitTest::TestOnly:
666       // Don't store anything.
667       break;
668     case BitTest::Complement:
669       NewByte = CGF.Builder.CreateXor(OldByte, Mask);
670       break;
671     case BitTest::Reset:
672       NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
673       break;
674     case BitTest::Set:
675       NewByte = CGF.Builder.CreateOr(OldByte, Mask);
676       break;
677     }
678     if (NewByte)
679       CGF.Builder.CreateStore(NewByte, ByteAddr);
680   }
681 
682   // However we loaded the old byte, either by plain load or atomicrmw, shift
683   // the bit into the low position and mask it to 0 or 1.
684   Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
685   return CGF.Builder.CreateAnd(
686       ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
687 }
688 
689 namespace {
690 enum class MSVCSetJmpKind {
691   _setjmpex,
692   _setjmp3,
693   _setjmp
694 };
695 }
696 
697 /// MSVC handles setjmp a bit differently on different platforms. On every
698 /// architecture except 32-bit x86, the frame address is passed. On x86, extra
699 /// parameters can be passed as variadic arguments, but we always pass none.
700 static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
701                                const CallExpr *E) {
702   llvm::Value *Arg1 = nullptr;
703   llvm::Type *Arg1Ty = nullptr;
704   StringRef Name;
705   bool IsVarArg = false;
706   if (SJKind == MSVCSetJmpKind::_setjmp3) {
707     Name = "_setjmp3";
708     Arg1Ty = CGF.Int32Ty;
709     Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
710     IsVarArg = true;
711   } else {
712     Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
713     Arg1Ty = CGF.Int8PtrTy;
714     Arg1 = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(Intrinsic::frameaddress),
715                                   llvm::ConstantInt::get(CGF.Int32Ty, 0));
716   }
717 
718   // Mark the call site and declaration with ReturnsTwice.
719   llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
720   llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
721       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
722       llvm::Attribute::ReturnsTwice);
723   llvm::Constant *SetJmpFn = CGF.CGM.CreateRuntimeFunction(
724       llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
725       ReturnsTwiceAttr, /*Local=*/true);
726 
727   llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
728       CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
729   llvm::Value *Args[] = {Buf, Arg1};
730   llvm::CallSite CS = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
731   CS.setAttributes(ReturnsTwiceAttr);
732   return RValue::get(CS.getInstruction());
733 }
734 
735 // Many of MSVC builtins are on both x64 and ARM; to avoid repeating code, we
736 // handle them here.
737 enum class CodeGenFunction::MSVCIntrin {
738   _BitScanForward,
739   _BitScanReverse,
740   _InterlockedAnd,
741   _InterlockedDecrement,
742   _InterlockedExchange,
743   _InterlockedExchangeAdd,
744   _InterlockedExchangeSub,
745   _InterlockedIncrement,
746   _InterlockedOr,
747   _InterlockedXor,
748   __fastfail,
749 };
750 
751 Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
752                                             const CallExpr *E) {
753   switch (BuiltinID) {
754   case MSVCIntrin::_BitScanForward:
755   case MSVCIntrin::_BitScanReverse: {
756     Value *ArgValue = EmitScalarExpr(E->getArg(1));
757 
758     llvm::Type *ArgType = ArgValue->getType();
759     llvm::Type *IndexType =
760       EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType();
761     llvm::Type *ResultType = ConvertType(E->getType());
762 
763     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
764     Value *ResZero = llvm::Constant::getNullValue(ResultType);
765     Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
766 
767     BasicBlock *Begin = Builder.GetInsertBlock();
768     BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
769     Builder.SetInsertPoint(End);
770     PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
771 
772     Builder.SetInsertPoint(Begin);
773     Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
774     BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
775     Builder.CreateCondBr(IsZero, End, NotZero);
776     Result->addIncoming(ResZero, Begin);
777 
778     Builder.SetInsertPoint(NotZero);
779     Address IndexAddress = EmitPointerWithAlignment(E->getArg(0));
780 
781     if (BuiltinID == MSVCIntrin::_BitScanForward) {
782       Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
783       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
784       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
785       Builder.CreateStore(ZeroCount, IndexAddress, false);
786     } else {
787       unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
788       Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
789 
790       Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
791       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
792       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
793       Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
794       Builder.CreateStore(Index, IndexAddress, false);
795     }
796     Builder.CreateBr(End);
797     Result->addIncoming(ResOne, NotZero);
798 
799     Builder.SetInsertPoint(End);
800     return Result;
801   }
802   case MSVCIntrin::_InterlockedAnd:
803     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
804   case MSVCIntrin::_InterlockedExchange:
805     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
806   case MSVCIntrin::_InterlockedExchangeAdd:
807     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
808   case MSVCIntrin::_InterlockedExchangeSub:
809     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
810   case MSVCIntrin::_InterlockedOr:
811     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
812   case MSVCIntrin::_InterlockedXor:
813     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
814 
815   case MSVCIntrin::_InterlockedDecrement: {
816     llvm::Type *IntTy = ConvertType(E->getType());
817     AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
818       AtomicRMWInst::Sub,
819       EmitScalarExpr(E->getArg(0)),
820       ConstantInt::get(IntTy, 1),
821       llvm::AtomicOrdering::SequentiallyConsistent);
822     return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
823   }
824   case MSVCIntrin::_InterlockedIncrement: {
825     llvm::Type *IntTy = ConvertType(E->getType());
826     AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
827       AtomicRMWInst::Add,
828       EmitScalarExpr(E->getArg(0)),
829       ConstantInt::get(IntTy, 1),
830       llvm::AtomicOrdering::SequentiallyConsistent);
831     return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
832   }
833 
834   case MSVCIntrin::__fastfail: {
835     // Request immediate process termination from the kernel. The instruction
836     // sequences to do this are documented on MSDN:
837     // https://msdn.microsoft.com/en-us/library/dn774154.aspx
838     llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
839     StringRef Asm, Constraints;
840     switch (ISA) {
841     default:
842       ErrorUnsupported(E, "__fastfail call for this architecture");
843       break;
844     case llvm::Triple::x86:
845     case llvm::Triple::x86_64:
846       Asm = "int $$0x29";
847       Constraints = "{cx}";
848       break;
849     case llvm::Triple::thumb:
850       Asm = "udf #251";
851       Constraints = "{r0}";
852       break;
853     }
854     llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
855     llvm::InlineAsm *IA =
856         llvm::InlineAsm::get(FTy, Asm, Constraints, /*SideEffects=*/true);
857     llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
858         getLLVMContext(), llvm::AttributeList::FunctionIndex,
859         llvm::Attribute::NoReturn);
860     CallSite CS = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
861     CS.setAttributes(NoReturnAttr);
862     return CS.getInstruction();
863   }
864   }
865   llvm_unreachable("Incorrect MSVC intrinsic!");
866 }
867 
868 namespace {
869 // ARC cleanup for __builtin_os_log_format
870 struct CallObjCArcUse final : EHScopeStack::Cleanup {
871   CallObjCArcUse(llvm::Value *object) : object(object) {}
872   llvm::Value *object;
873 
874   void Emit(CodeGenFunction &CGF, Flags flags) override {
875     CGF.EmitARCIntrinsicUse(object);
876   }
877 };
878 }
879 
880 Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
881                                                  BuiltinCheckKind Kind) {
882   assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
883           && "Unsupported builtin check kind");
884 
885   Value *ArgValue = EmitScalarExpr(E);
886   if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
887     return ArgValue;
888 
889   SanitizerScope SanScope(this);
890   Value *Cond = Builder.CreateICmpNE(
891       ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
892   EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
893             SanitizerHandler::InvalidBuiltin,
894             {EmitCheckSourceLocation(E->getExprLoc()),
895              llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
896             None);
897   return ArgValue;
898 }
899 
900 /// Get the argument type for arguments to os_log_helper.
901 static CanQualType getOSLogArgType(ASTContext &C, int Size) {
902   QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
903   return C.getCanonicalType(UnsignedTy);
904 }
905 
906 llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
907     const analyze_os_log::OSLogBufferLayout &Layout,
908     CharUnits BufferAlignment) {
909   ASTContext &Ctx = getContext();
910 
911   llvm::SmallString<64> Name;
912   {
913     raw_svector_ostream OS(Name);
914     OS << "__os_log_helper";
915     OS << "_" << BufferAlignment.getQuantity();
916     OS << "_" << int(Layout.getSummaryByte());
917     OS << "_" << int(Layout.getNumArgsByte());
918     for (const auto &Item : Layout.Items)
919       OS << "_" << int(Item.getSizeByte()) << "_"
920          << int(Item.getDescriptorByte());
921   }
922 
923   if (llvm::Function *F = CGM.getModule().getFunction(Name))
924     return F;
925 
926   llvm::SmallVector<ImplicitParamDecl, 4> Params;
927   Params.emplace_back(Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"),
928                       Ctx.VoidPtrTy, ImplicitParamDecl::Other);
929 
930   for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
931     char Size = Layout.Items[I].getSizeByte();
932     if (!Size)
933       continue;
934 
935     Params.emplace_back(
936         Ctx, nullptr, SourceLocation(),
937         &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)),
938         getOSLogArgType(Ctx, Size), ImplicitParamDecl::Other);
939   }
940 
941   FunctionArgList Args;
942   for (auto &P : Params)
943     Args.push_back(&P);
944 
945   // The helper function has linkonce_odr linkage to enable the linker to merge
946   // identical functions. To ensure the merging always happens, 'noinline' is
947   // attached to the function when compiling with -Oz.
948   const CGFunctionInfo &FI =
949       CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
950   llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
951   llvm::Function *Fn = llvm::Function::Create(
952       FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
953   Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
954   CGM.SetLLVMFunctionAttributes(nullptr, FI, Fn);
955   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
956 
957   // Attach 'noinline' at -Oz.
958   if (CGM.getCodeGenOpts().OptimizeSize == 2)
959     Fn->addFnAttr(llvm::Attribute::NoInline);
960 
961   auto NL = ApplyDebugLocation::CreateEmpty(*this);
962   IdentifierInfo *II = &Ctx.Idents.get(Name);
963   FunctionDecl *FD = FunctionDecl::Create(
964       Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
965       Ctx.VoidTy, nullptr, SC_PrivateExtern, false, false);
966 
967   StartFunction(FD, Ctx.VoidTy, Fn, FI, Args);
968 
969   // Create a scope with an artificial location for the body of this function.
970   auto AL = ApplyDebugLocation::CreateArtificial(*this);
971 
972   CharUnits Offset;
973   Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(&Params[0]), "buf"),
974                   BufferAlignment);
975   Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
976                       Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
977   Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
978                       Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
979 
980   unsigned I = 1;
981   for (const auto &Item : Layout.Items) {
982     Builder.CreateStore(
983         Builder.getInt8(Item.getDescriptorByte()),
984         Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
985     Builder.CreateStore(
986         Builder.getInt8(Item.getSizeByte()),
987         Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
988 
989     CharUnits Size = Item.size();
990     if (!Size.getQuantity())
991       continue;
992 
993     Address Arg = GetAddrOfLocalVar(&Params[I]);
994     Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
995     Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
996                                  "argDataCast");
997     Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
998     Offset += Size;
999     ++I;
1000   }
1001 
1002   FinishFunction();
1003 
1004   return Fn;
1005 }
1006 
1007 RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
1008   assert(E.getNumArgs() >= 2 &&
1009          "__builtin_os_log_format takes at least 2 arguments");
1010   ASTContext &Ctx = getContext();
1011   analyze_os_log::OSLogBufferLayout Layout;
1012   analyze_os_log::computeOSLogBufferLayout(Ctx, &E, Layout);
1013   Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
1014   llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
1015 
1016   // Ignore argument 1, the format string. It is not currently used.
1017   CallArgList Args;
1018   Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
1019 
1020   for (const auto &Item : Layout.Items) {
1021     int Size = Item.getSizeByte();
1022     if (!Size)
1023       continue;
1024 
1025     llvm::Value *ArgVal;
1026 
1027     if (const Expr *TheExpr = Item.getExpr()) {
1028       ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
1029 
1030       // Check if this is a retainable type.
1031       if (TheExpr->getType()->isObjCRetainableType()) {
1032         assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
1033                "Only scalar can be a ObjC retainable type");
1034         // Check if the object is constant, if not, save it in
1035         // RetainableOperands.
1036         if (!isa<Constant>(ArgVal))
1037           RetainableOperands.push_back(ArgVal);
1038       }
1039     } else {
1040       ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1041     }
1042 
1043     unsigned ArgValSize =
1044         CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
1045     llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
1046                                                      ArgValSize);
1047     ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
1048     CanQualType ArgTy = getOSLogArgType(Ctx, Size);
1049     // If ArgVal has type x86_fp80, zero-extend ArgVal.
1050     ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
1051     Args.add(RValue::get(ArgVal), ArgTy);
1052   }
1053 
1054   const CGFunctionInfo &FI =
1055       CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
1056   llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
1057       Layout, BufAddr.getAlignment());
1058   EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
1059 
1060   // Push a clang.arc.use cleanup for each object in RetainableOperands. The
1061   // cleanup will cause the use to appear after the final log call, keeping
1062   // the object valid while it’s held in the log buffer.  Note that if there’s
1063   // a release cleanup on the object, it will already be active; since
1064   // cleanups are emitted in reverse order, the use will occur before the
1065   // object is released.
1066   if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
1067       CGM.getCodeGenOpts().OptimizationLevel != 0)
1068     for (llvm::Value *Object : RetainableOperands)
1069       pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), Object);
1070 
1071   return RValue::get(BufAddr.getPointer());
1072 }
1073 
1074 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
1075 static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
1076                                        WidthAndSignedness Op1Info,
1077                                        WidthAndSignedness Op2Info,
1078                                        WidthAndSignedness ResultInfo) {
1079   return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1080          Op1Info.Width == Op2Info.Width && Op1Info.Width >= ResultInfo.Width &&
1081          Op1Info.Signed != Op2Info.Signed;
1082 }
1083 
1084 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
1085 /// the generic checked-binop irgen.
1086 static RValue
1087 EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
1088                              WidthAndSignedness Op1Info, const clang::Expr *Op2,
1089                              WidthAndSignedness Op2Info,
1090                              const clang::Expr *ResultArg, QualType ResultQTy,
1091                              WidthAndSignedness ResultInfo) {
1092   assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
1093                                     Op2Info, ResultInfo) &&
1094          "Not a mixed-sign multipliction we can specialize");
1095 
1096   // Emit the signed and unsigned operands.
1097   const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1098   const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1099   llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
1100   llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
1101 
1102   llvm::Type *OpTy = Signed->getType();
1103   llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1104   Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
1105   llvm::Type *ResTy = ResultPtr.getElementType();
1106 
1107   // Take the absolute value of the signed operand.
1108   llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
1109   llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
1110   llvm::Value *AbsSigned =
1111       CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
1112 
1113   // Perform a checked unsigned multiplication.
1114   llvm::Value *UnsignedOverflow;
1115   llvm::Value *UnsignedResult =
1116       EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
1117                             Unsigned, UnsignedOverflow);
1118 
1119   llvm::Value *Overflow, *Result;
1120   if (ResultInfo.Signed) {
1121     // Signed overflow occurs if the result is greater than INT_MAX or lesser
1122     // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
1123     auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width)
1124                       .zextOrSelf(Op1Info.Width);
1125     llvm::Value *MaxResult =
1126         CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
1127                               CGF.Builder.CreateZExt(IsNegative, OpTy));
1128     llvm::Value *SignedOverflow =
1129         CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
1130     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
1131 
1132     // Prepare the signed result (possibly by negating it).
1133     llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
1134     llvm::Value *SignedResult =
1135         CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
1136     Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
1137   } else {
1138     // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
1139     llvm::Value *Underflow = CGF.Builder.CreateAnd(
1140         IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
1141     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
1142     if (ResultInfo.Width < Op1Info.Width) {
1143       auto IntMax =
1144           llvm::APInt::getMaxValue(ResultInfo.Width).zext(Op1Info.Width);
1145       llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
1146           UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
1147       Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
1148     }
1149 
1150     // Negate the product if it would be negative in infinite precision.
1151     Result = CGF.Builder.CreateSelect(
1152         IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
1153 
1154     Result = CGF.Builder.CreateTrunc(Result, ResTy);
1155   }
1156   assert(Overflow && Result && "Missing overflow or result");
1157 
1158   bool isVolatile =
1159       ResultArg->getType()->getPointeeType().isVolatileQualified();
1160   CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
1161                           isVolatile);
1162   return RValue::get(Overflow);
1163 }
1164 
1165 static llvm::Value *dumpRecord(CodeGenFunction &CGF, QualType RType,
1166                                Value *&RecordPtr, CharUnits Align, Value *Func,
1167                                int Lvl) {
1168   const auto *RT = RType->getAs<RecordType>();
1169   ASTContext &Context = CGF.getContext();
1170   RecordDecl *RD = RT->getDecl()->getDefinition();
1171   ASTContext &Ctx = RD->getASTContext();
1172   const ASTRecordLayout &RL = Ctx.getASTRecordLayout(RD);
1173   std::string Pad = std::string(Lvl * 4, ' ');
1174 
1175   Value *GString =
1176       CGF.Builder.CreateGlobalStringPtr(RType.getAsString() + " {\n");
1177   Value *Res = CGF.Builder.CreateCall(Func, {GString});
1178 
1179   static llvm::DenseMap<QualType, const char *> Types;
1180   if (Types.empty()) {
1181     Types[Context.CharTy] = "%c";
1182     Types[Context.BoolTy] = "%d";
1183     Types[Context.SignedCharTy] = "%hhd";
1184     Types[Context.UnsignedCharTy] = "%hhu";
1185     Types[Context.IntTy] = "%d";
1186     Types[Context.UnsignedIntTy] = "%u";
1187     Types[Context.LongTy] = "%ld";
1188     Types[Context.UnsignedLongTy] = "%lu";
1189     Types[Context.LongLongTy] = "%lld";
1190     Types[Context.UnsignedLongLongTy] = "%llu";
1191     Types[Context.ShortTy] = "%hd";
1192     Types[Context.UnsignedShortTy] = "%hu";
1193     Types[Context.VoidPtrTy] = "%p";
1194     Types[Context.FloatTy] = "%f";
1195     Types[Context.DoubleTy] = "%f";
1196     Types[Context.LongDoubleTy] = "%Lf";
1197     Types[Context.getPointerType(Context.CharTy)] = "%s";
1198     Types[Context.getPointerType(Context.getConstType(Context.CharTy))] = "%s";
1199   }
1200 
1201   for (const auto *FD : RD->fields()) {
1202     uint64_t Off = RL.getFieldOffset(FD->getFieldIndex());
1203     Off = Ctx.toCharUnitsFromBits(Off).getQuantity();
1204 
1205     Value *FieldPtr = RecordPtr;
1206     if (RD->isUnion())
1207       FieldPtr = CGF.Builder.CreatePointerCast(
1208           FieldPtr, CGF.ConvertType(Context.getPointerType(FD->getType())));
1209     else
1210       FieldPtr = CGF.Builder.CreateStructGEP(CGF.ConvertType(RType), FieldPtr,
1211                                              FD->getFieldIndex());
1212 
1213     GString = CGF.Builder.CreateGlobalStringPtr(
1214         llvm::Twine(Pad)
1215             .concat(FD->getType().getAsString())
1216             .concat(llvm::Twine(' '))
1217             .concat(FD->getNameAsString())
1218             .concat(" : ")
1219             .str());
1220     Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1221     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1222 
1223     QualType CanonicalType =
1224         FD->getType().getUnqualifiedType().getCanonicalType();
1225 
1226     // We check whether we are in a recursive type
1227     if (CanonicalType->isRecordType()) {
1228       Value *TmpRes =
1229           dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
1230       Res = CGF.Builder.CreateAdd(TmpRes, Res);
1231       continue;
1232     }
1233 
1234     // We try to determine the best format to print the current field
1235     llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1236                              ? Types[Context.VoidPtrTy]
1237                              : Types[CanonicalType];
1238 
1239     Address FieldAddress = Address(FieldPtr, Align);
1240     FieldPtr = CGF.Builder.CreateLoad(FieldAddress);
1241 
1242     // FIXME Need to handle bitfield here
1243     GString = CGF.Builder.CreateGlobalStringPtr(
1244         Format.concat(llvm::Twine('\n')).str());
1245     TmpRes = CGF.Builder.CreateCall(Func, {GString, FieldPtr});
1246     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1247   }
1248 
1249   GString = CGF.Builder.CreateGlobalStringPtr(Pad + "}\n");
1250   Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1251   Res = CGF.Builder.CreateAdd(Res, TmpRes);
1252   return Res;
1253 }
1254 
1255 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
1256                                         unsigned BuiltinID, const CallExpr *E,
1257                                         ReturnValueSlot ReturnValue) {
1258   // See if we can constant fold this builtin.  If so, don't emit it at all.
1259   Expr::EvalResult Result;
1260   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
1261       !Result.hasSideEffects()) {
1262     if (Result.Val.isInt())
1263       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1264                                                 Result.Val.getInt()));
1265     if (Result.Val.isFloat())
1266       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1267                                                Result.Val.getFloat()));
1268   }
1269 
1270   // There are LLVM math intrinsics/instructions corresponding to math library
1271   // functions except the LLVM op will never set errno while the math library
1272   // might. Also, math builtins have the same semantics as their math library
1273   // twins. Thus, we can transform math library and builtin calls to their
1274   // LLVM counterparts if the call is marked 'const' (known to never set errno).
1275   if (FD->hasAttr<ConstAttr>()) {
1276     switch (BuiltinID) {
1277     case Builtin::BIceil:
1278     case Builtin::BIceilf:
1279     case Builtin::BIceill:
1280     case Builtin::BI__builtin_ceil:
1281     case Builtin::BI__builtin_ceilf:
1282     case Builtin::BI__builtin_ceill:
1283       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::ceil));
1284 
1285     case Builtin::BIcopysign:
1286     case Builtin::BIcopysignf:
1287     case Builtin::BIcopysignl:
1288     case Builtin::BI__builtin_copysign:
1289     case Builtin::BI__builtin_copysignf:
1290     case Builtin::BI__builtin_copysignl:
1291     case Builtin::BI__builtin_copysignf128:
1292       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
1293 
1294     case Builtin::BIcos:
1295     case Builtin::BIcosf:
1296     case Builtin::BIcosl:
1297     case Builtin::BI__builtin_cos:
1298     case Builtin::BI__builtin_cosf:
1299     case Builtin::BI__builtin_cosl:
1300       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::cos));
1301 
1302     case Builtin::BIexp:
1303     case Builtin::BIexpf:
1304     case Builtin::BIexpl:
1305     case Builtin::BI__builtin_exp:
1306     case Builtin::BI__builtin_expf:
1307     case Builtin::BI__builtin_expl:
1308       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp));
1309 
1310     case Builtin::BIexp2:
1311     case Builtin::BIexp2f:
1312     case Builtin::BIexp2l:
1313     case Builtin::BI__builtin_exp2:
1314     case Builtin::BI__builtin_exp2f:
1315     case Builtin::BI__builtin_exp2l:
1316       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::exp2));
1317 
1318     case Builtin::BIfabs:
1319     case Builtin::BIfabsf:
1320     case Builtin::BIfabsl:
1321     case Builtin::BI__builtin_fabs:
1322     case Builtin::BI__builtin_fabsf:
1323     case Builtin::BI__builtin_fabsl:
1324     case Builtin::BI__builtin_fabsf128:
1325       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
1326 
1327     case Builtin::BIfloor:
1328     case Builtin::BIfloorf:
1329     case Builtin::BIfloorl:
1330     case Builtin::BI__builtin_floor:
1331     case Builtin::BI__builtin_floorf:
1332     case Builtin::BI__builtin_floorl:
1333       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::floor));
1334 
1335     case Builtin::BIfma:
1336     case Builtin::BIfmaf:
1337     case Builtin::BIfmal:
1338     case Builtin::BI__builtin_fma:
1339     case Builtin::BI__builtin_fmaf:
1340     case Builtin::BI__builtin_fmal:
1341       return RValue::get(emitTernaryBuiltin(*this, E, Intrinsic::fma));
1342 
1343     case Builtin::BIfmax:
1344     case Builtin::BIfmaxf:
1345     case Builtin::BIfmaxl:
1346     case Builtin::BI__builtin_fmax:
1347     case Builtin::BI__builtin_fmaxf:
1348     case Builtin::BI__builtin_fmaxl:
1349       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::maxnum));
1350 
1351     case Builtin::BIfmin:
1352     case Builtin::BIfminf:
1353     case Builtin::BIfminl:
1354     case Builtin::BI__builtin_fmin:
1355     case Builtin::BI__builtin_fminf:
1356     case Builtin::BI__builtin_fminl:
1357       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::minnum));
1358 
1359     // fmod() is a special-case. It maps to the frem instruction rather than an
1360     // LLVM intrinsic.
1361     case Builtin::BIfmod:
1362     case Builtin::BIfmodf:
1363     case Builtin::BIfmodl:
1364     case Builtin::BI__builtin_fmod:
1365     case Builtin::BI__builtin_fmodf:
1366     case Builtin::BI__builtin_fmodl: {
1367       Value *Arg1 = EmitScalarExpr(E->getArg(0));
1368       Value *Arg2 = EmitScalarExpr(E->getArg(1));
1369       return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
1370     }
1371 
1372     case Builtin::BIlog:
1373     case Builtin::BIlogf:
1374     case Builtin::BIlogl:
1375     case Builtin::BI__builtin_log:
1376     case Builtin::BI__builtin_logf:
1377     case Builtin::BI__builtin_logl:
1378       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log));
1379 
1380     case Builtin::BIlog10:
1381     case Builtin::BIlog10f:
1382     case Builtin::BIlog10l:
1383     case Builtin::BI__builtin_log10:
1384     case Builtin::BI__builtin_log10f:
1385     case Builtin::BI__builtin_log10l:
1386       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log10));
1387 
1388     case Builtin::BIlog2:
1389     case Builtin::BIlog2f:
1390     case Builtin::BIlog2l:
1391     case Builtin::BI__builtin_log2:
1392     case Builtin::BI__builtin_log2f:
1393     case Builtin::BI__builtin_log2l:
1394       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::log2));
1395 
1396     case Builtin::BInearbyint:
1397     case Builtin::BInearbyintf:
1398     case Builtin::BInearbyintl:
1399     case Builtin::BI__builtin_nearbyint:
1400     case Builtin::BI__builtin_nearbyintf:
1401     case Builtin::BI__builtin_nearbyintl:
1402       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::nearbyint));
1403 
1404     case Builtin::BIpow:
1405     case Builtin::BIpowf:
1406     case Builtin::BIpowl:
1407     case Builtin::BI__builtin_pow:
1408     case Builtin::BI__builtin_powf:
1409     case Builtin::BI__builtin_powl:
1410       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::pow));
1411 
1412     case Builtin::BIrint:
1413     case Builtin::BIrintf:
1414     case Builtin::BIrintl:
1415     case Builtin::BI__builtin_rint:
1416     case Builtin::BI__builtin_rintf:
1417     case Builtin::BI__builtin_rintl:
1418       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::rint));
1419 
1420     case Builtin::BIround:
1421     case Builtin::BIroundf:
1422     case Builtin::BIroundl:
1423     case Builtin::BI__builtin_round:
1424     case Builtin::BI__builtin_roundf:
1425     case Builtin::BI__builtin_roundl:
1426       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::round));
1427 
1428     case Builtin::BIsin:
1429     case Builtin::BIsinf:
1430     case Builtin::BIsinl:
1431     case Builtin::BI__builtin_sin:
1432     case Builtin::BI__builtin_sinf:
1433     case Builtin::BI__builtin_sinl:
1434       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::sin));
1435 
1436     case Builtin::BIsqrt:
1437     case Builtin::BIsqrtf:
1438     case Builtin::BIsqrtl:
1439     case Builtin::BI__builtin_sqrt:
1440     case Builtin::BI__builtin_sqrtf:
1441     case Builtin::BI__builtin_sqrtl:
1442       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::sqrt));
1443 
1444     case Builtin::BItrunc:
1445     case Builtin::BItruncf:
1446     case Builtin::BItruncl:
1447     case Builtin::BI__builtin_trunc:
1448     case Builtin::BI__builtin_truncf:
1449     case Builtin::BI__builtin_truncl:
1450       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::trunc));
1451 
1452     default:
1453       break;
1454     }
1455   }
1456 
1457   switch (BuiltinID) {
1458   default: break;
1459   case Builtin::BI__builtin___CFStringMakeConstantString:
1460   case Builtin::BI__builtin___NSStringMakeConstantString:
1461     return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
1462   case Builtin::BI__builtin_stdarg_start:
1463   case Builtin::BI__builtin_va_start:
1464   case Builtin::BI__va_start:
1465   case Builtin::BI__builtin_va_end:
1466     return RValue::get(
1467         EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
1468                            ? EmitScalarExpr(E->getArg(0))
1469                            : EmitVAListRef(E->getArg(0)).getPointer(),
1470                        BuiltinID != Builtin::BI__builtin_va_end));
1471   case Builtin::BI__builtin_va_copy: {
1472     Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
1473     Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
1474 
1475     llvm::Type *Type = Int8PtrTy;
1476 
1477     DstPtr = Builder.CreateBitCast(DstPtr, Type);
1478     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
1479     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
1480                                           {DstPtr, SrcPtr}));
1481   }
1482   case Builtin::BI__builtin_abs:
1483   case Builtin::BI__builtin_labs:
1484   case Builtin::BI__builtin_llabs: {
1485     // X < 0 ? -X : X
1486     // The negation has 'nsw' because abs of INT_MIN is undefined.
1487     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1488     Value *NegOp = Builder.CreateNSWNeg(ArgValue, "neg");
1489     Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
1490     Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
1491     Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue, "abs");
1492     return RValue::get(Result);
1493   }
1494   case Builtin::BI__builtin_conj:
1495   case Builtin::BI__builtin_conjf:
1496   case Builtin::BI__builtin_conjl: {
1497     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1498     Value *Real = ComplexVal.first;
1499     Value *Imag = ComplexVal.second;
1500     Value *Zero =
1501       Imag->getType()->isFPOrFPVectorTy()
1502         ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
1503         : llvm::Constant::getNullValue(Imag->getType());
1504 
1505     Imag = Builder.CreateFSub(Zero, Imag, "sub");
1506     return RValue::getComplex(std::make_pair(Real, Imag));
1507   }
1508   case Builtin::BI__builtin_creal:
1509   case Builtin::BI__builtin_crealf:
1510   case Builtin::BI__builtin_creall:
1511   case Builtin::BIcreal:
1512   case Builtin::BIcrealf:
1513   case Builtin::BIcreall: {
1514     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1515     return RValue::get(ComplexVal.first);
1516   }
1517 
1518   case Builtin::BI__builtin_dump_struct: {
1519     Value *Func = EmitScalarExpr(E->getArg(1)->IgnoreImpCasts());
1520     CharUnits Arg0Align = EmitPointerWithAlignment(E->getArg(0)).getAlignment();
1521 
1522     const Expr *Arg0 = E->getArg(0)->IgnoreImpCasts();
1523     QualType Arg0Type = Arg0->getType()->getPointeeType();
1524 
1525     Value *RecordPtr = EmitScalarExpr(Arg0);
1526     Value *Res = dumpRecord(*this, Arg0Type, RecordPtr, Arg0Align, Func, 0);
1527     return RValue::get(Res);
1528   }
1529 
1530   case Builtin::BI__builtin_cimag:
1531   case Builtin::BI__builtin_cimagf:
1532   case Builtin::BI__builtin_cimagl:
1533   case Builtin::BIcimag:
1534   case Builtin::BIcimagf:
1535   case Builtin::BIcimagl: {
1536     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
1537     return RValue::get(ComplexVal.second);
1538   }
1539 
1540   case Builtin::BI__builtin_ctzs:
1541   case Builtin::BI__builtin_ctz:
1542   case Builtin::BI__builtin_ctzl:
1543   case Builtin::BI__builtin_ctzll: {
1544     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
1545 
1546     llvm::Type *ArgType = ArgValue->getType();
1547     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1548 
1549     llvm::Type *ResultType = ConvertType(E->getType());
1550     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1551     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1552     if (Result->getType() != ResultType)
1553       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1554                                      "cast");
1555     return RValue::get(Result);
1556   }
1557   case Builtin::BI__builtin_clzs:
1558   case Builtin::BI__builtin_clz:
1559   case Builtin::BI__builtin_clzl:
1560   case Builtin::BI__builtin_clzll: {
1561     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
1562 
1563     llvm::Type *ArgType = ArgValue->getType();
1564     Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1565 
1566     llvm::Type *ResultType = ConvertType(E->getType());
1567     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1568     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1569     if (Result->getType() != ResultType)
1570       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1571                                      "cast");
1572     return RValue::get(Result);
1573   }
1574   case Builtin::BI__builtin_ffs:
1575   case Builtin::BI__builtin_ffsl:
1576   case Builtin::BI__builtin_ffsll: {
1577     // ffs(x) -> x ? cttz(x) + 1 : 0
1578     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1579 
1580     llvm::Type *ArgType = ArgValue->getType();
1581     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1582 
1583     llvm::Type *ResultType = ConvertType(E->getType());
1584     Value *Tmp =
1585         Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
1586                           llvm::ConstantInt::get(ArgType, 1));
1587     Value *Zero = llvm::Constant::getNullValue(ArgType);
1588     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
1589     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
1590     if (Result->getType() != ResultType)
1591       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1592                                      "cast");
1593     return RValue::get(Result);
1594   }
1595   case Builtin::BI__builtin_parity:
1596   case Builtin::BI__builtin_parityl:
1597   case Builtin::BI__builtin_parityll: {
1598     // parity(x) -> ctpop(x) & 1
1599     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1600 
1601     llvm::Type *ArgType = ArgValue->getType();
1602     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1603 
1604     llvm::Type *ResultType = ConvertType(E->getType());
1605     Value *Tmp = Builder.CreateCall(F, ArgValue);
1606     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
1607     if (Result->getType() != ResultType)
1608       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1609                                      "cast");
1610     return RValue::get(Result);
1611   }
1612   case Builtin::BI__popcnt16:
1613   case Builtin::BI__popcnt:
1614   case Builtin::BI__popcnt64:
1615   case Builtin::BI__builtin_popcount:
1616   case Builtin::BI__builtin_popcountl:
1617   case Builtin::BI__builtin_popcountll: {
1618     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1619 
1620     llvm::Type *ArgType = ArgValue->getType();
1621     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1622 
1623     llvm::Type *ResultType = ConvertType(E->getType());
1624     Value *Result = Builder.CreateCall(F, ArgValue);
1625     if (Result->getType() != ResultType)
1626       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
1627                                      "cast");
1628     return RValue::get(Result);
1629   }
1630   case Builtin::BI_rotr8:
1631   case Builtin::BI_rotr16:
1632   case Builtin::BI_rotr:
1633   case Builtin::BI_lrotr:
1634   case Builtin::BI_rotr64: {
1635     Value *Val = EmitScalarExpr(E->getArg(0));
1636     Value *Shift = EmitScalarExpr(E->getArg(1));
1637 
1638     llvm::Type *ArgType = Val->getType();
1639     Shift = Builder.CreateIntCast(Shift, ArgType, false);
1640     unsigned ArgWidth = ArgType->getIntegerBitWidth();
1641     Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1642 
1643     Value *RightShiftAmt = Builder.CreateAnd(Shift, Mask);
1644     Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1645     Value *LeftShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1646     Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1647     Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1648     return RValue::get(Result);
1649   }
1650   case Builtin::BI_rotl8:
1651   case Builtin::BI_rotl16:
1652   case Builtin::BI_rotl:
1653   case Builtin::BI_lrotl:
1654   case Builtin::BI_rotl64: {
1655     Value *Val = EmitScalarExpr(E->getArg(0));
1656     Value *Shift = EmitScalarExpr(E->getArg(1));
1657 
1658     llvm::Type *ArgType = Val->getType();
1659     Shift = Builder.CreateIntCast(Shift, ArgType, false);
1660     unsigned ArgWidth = ArgType->getIntegerBitWidth();
1661     Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1662 
1663     Value *LeftShiftAmt = Builder.CreateAnd(Shift, Mask);
1664     Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1665     Value *RightShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1666     Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1667     Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1668     return RValue::get(Result);
1669   }
1670   case Builtin::BI__builtin_unpredictable: {
1671     // Always return the argument of __builtin_unpredictable. LLVM does not
1672     // handle this builtin. Metadata for this builtin should be added directly
1673     // to instructions such as branches or switches that use it.
1674     return RValue::get(EmitScalarExpr(E->getArg(0)));
1675   }
1676   case Builtin::BI__builtin_expect: {
1677     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1678     llvm::Type *ArgType = ArgValue->getType();
1679 
1680     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
1681     // Don't generate llvm.expect on -O0 as the backend won't use it for
1682     // anything.
1683     // Note, we still IRGen ExpectedValue because it could have side-effects.
1684     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1685       return RValue::get(ArgValue);
1686 
1687     Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
1688     Value *Result =
1689         Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
1690     return RValue::get(Result);
1691   }
1692   case Builtin::BI__builtin_assume_aligned: {
1693     Value *PtrValue = EmitScalarExpr(E->getArg(0));
1694     Value *OffsetValue =
1695       (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
1696 
1697     Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
1698     ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
1699     unsigned Alignment = (unsigned) AlignmentCI->getZExtValue();
1700 
1701     EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
1702     return RValue::get(PtrValue);
1703   }
1704   case Builtin::BI__assume:
1705   case Builtin::BI__builtin_assume: {
1706     if (E->getArg(0)->HasSideEffects(getContext()))
1707       return RValue::get(nullptr);
1708 
1709     Value *ArgValue = EmitScalarExpr(E->getArg(0));
1710     Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
1711     return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
1712   }
1713   case Builtin::BI__builtin_bswap16:
1714   case Builtin::BI__builtin_bswap32:
1715   case Builtin::BI__builtin_bswap64: {
1716     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
1717   }
1718   case Builtin::BI__builtin_bitreverse8:
1719   case Builtin::BI__builtin_bitreverse16:
1720   case Builtin::BI__builtin_bitreverse32:
1721   case Builtin::BI__builtin_bitreverse64: {
1722     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
1723   }
1724   case Builtin::BI__builtin_object_size: {
1725     unsigned Type =
1726         E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
1727     auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
1728 
1729     // We pass this builtin onto the optimizer so that it can figure out the
1730     // object size in more complex cases.
1731     return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
1732                                              /*EmittedE=*/nullptr));
1733   }
1734   case Builtin::BI__builtin_prefetch: {
1735     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
1736     // FIXME: Technically these constants should of type 'int', yes?
1737     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
1738       llvm::ConstantInt::get(Int32Ty, 0);
1739     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
1740       llvm::ConstantInt::get(Int32Ty, 3);
1741     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
1742     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
1743     return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
1744   }
1745   case Builtin::BI__builtin_readcyclecounter: {
1746     Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
1747     return RValue::get(Builder.CreateCall(F));
1748   }
1749   case Builtin::BI__builtin___clear_cache: {
1750     Value *Begin = EmitScalarExpr(E->getArg(0));
1751     Value *End = EmitScalarExpr(E->getArg(1));
1752     Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
1753     return RValue::get(Builder.CreateCall(F, {Begin, End}));
1754   }
1755   case Builtin::BI__builtin_trap:
1756     return RValue::get(EmitTrapCall(Intrinsic::trap));
1757   case Builtin::BI__debugbreak:
1758     return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
1759   case Builtin::BI__builtin_unreachable: {
1760     EmitUnreachable(E->getExprLoc());
1761 
1762     // We do need to preserve an insertion point.
1763     EmitBlock(createBasicBlock("unreachable.cont"));
1764 
1765     return RValue::get(nullptr);
1766   }
1767 
1768   case Builtin::BI__builtin_powi:
1769   case Builtin::BI__builtin_powif:
1770   case Builtin::BI__builtin_powil: {
1771     Value *Base = EmitScalarExpr(E->getArg(0));
1772     Value *Exponent = EmitScalarExpr(E->getArg(1));
1773     llvm::Type *ArgType = Base->getType();
1774     Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
1775     return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
1776   }
1777 
1778   case Builtin::BI__builtin_isgreater:
1779   case Builtin::BI__builtin_isgreaterequal:
1780   case Builtin::BI__builtin_isless:
1781   case Builtin::BI__builtin_islessequal:
1782   case Builtin::BI__builtin_islessgreater:
1783   case Builtin::BI__builtin_isunordered: {
1784     // Ordered comparisons: we know the arguments to these are matching scalar
1785     // floating point values.
1786     Value *LHS = EmitScalarExpr(E->getArg(0));
1787     Value *RHS = EmitScalarExpr(E->getArg(1));
1788 
1789     switch (BuiltinID) {
1790     default: llvm_unreachable("Unknown ordered comparison");
1791     case Builtin::BI__builtin_isgreater:
1792       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
1793       break;
1794     case Builtin::BI__builtin_isgreaterequal:
1795       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
1796       break;
1797     case Builtin::BI__builtin_isless:
1798       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
1799       break;
1800     case Builtin::BI__builtin_islessequal:
1801       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
1802       break;
1803     case Builtin::BI__builtin_islessgreater:
1804       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
1805       break;
1806     case Builtin::BI__builtin_isunordered:
1807       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
1808       break;
1809     }
1810     // ZExt bool to int type.
1811     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
1812   }
1813   case Builtin::BI__builtin_isnan: {
1814     Value *V = EmitScalarExpr(E->getArg(0));
1815     V = Builder.CreateFCmpUNO(V, V, "cmp");
1816     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1817   }
1818 
1819   case Builtin::BIfinite:
1820   case Builtin::BI__finite:
1821   case Builtin::BIfinitef:
1822   case Builtin::BI__finitef:
1823   case Builtin::BIfinitel:
1824   case Builtin::BI__finitel:
1825   case Builtin::BI__builtin_isinf:
1826   case Builtin::BI__builtin_isfinite: {
1827     // isinf(x)    --> fabs(x) == infinity
1828     // isfinite(x) --> fabs(x) != infinity
1829     // x != NaN via the ordered compare in either case.
1830     Value *V = EmitScalarExpr(E->getArg(0));
1831     Value *Fabs = EmitFAbs(*this, V);
1832     Constant *Infinity = ConstantFP::getInfinity(V->getType());
1833     CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1834                                   ? CmpInst::FCMP_OEQ
1835                                   : CmpInst::FCMP_ONE;
1836     Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
1837     return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
1838   }
1839 
1840   case Builtin::BI__builtin_isinf_sign: {
1841     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
1842     Value *Arg = EmitScalarExpr(E->getArg(0));
1843     Value *AbsArg = EmitFAbs(*this, Arg);
1844     Value *IsInf = Builder.CreateFCmpOEQ(
1845         AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
1846     Value *IsNeg = EmitSignBit(*this, Arg);
1847 
1848     llvm::Type *IntTy = ConvertType(E->getType());
1849     Value *Zero = Constant::getNullValue(IntTy);
1850     Value *One = ConstantInt::get(IntTy, 1);
1851     Value *NegativeOne = ConstantInt::get(IntTy, -1);
1852     Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
1853     Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
1854     return RValue::get(Result);
1855   }
1856 
1857   case Builtin::BI__builtin_isnormal: {
1858     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
1859     Value *V = EmitScalarExpr(E->getArg(0));
1860     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
1861 
1862     Value *Abs = EmitFAbs(*this, V);
1863     Value *IsLessThanInf =
1864       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
1865     APFloat Smallest = APFloat::getSmallestNormalized(
1866                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
1867     Value *IsNormal =
1868       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1869                             "isnormal");
1870     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
1871     V = Builder.CreateAnd(V, IsNormal, "and");
1872     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1873   }
1874 
1875   case Builtin::BI__builtin_fpclassify: {
1876     Value *V = EmitScalarExpr(E->getArg(5));
1877     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
1878 
1879     // Create Result
1880     BasicBlock *Begin = Builder.GetInsertBlock();
1881     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
1882     Builder.SetInsertPoint(End);
1883     PHINode *Result =
1884       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
1885                         "fpclassify_result");
1886 
1887     // if (V==0) return FP_ZERO
1888     Builder.SetInsertPoint(Begin);
1889     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1890                                           "iszero");
1891     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
1892     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
1893     Builder.CreateCondBr(IsZero, End, NotZero);
1894     Result->addIncoming(ZeroLiteral, Begin);
1895 
1896     // if (V != V) return FP_NAN
1897     Builder.SetInsertPoint(NotZero);
1898     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
1899     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
1900     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
1901     Builder.CreateCondBr(IsNan, End, NotNan);
1902     Result->addIncoming(NanLiteral, NotZero);
1903 
1904     // if (fabs(V) == infinity) return FP_INFINITY
1905     Builder.SetInsertPoint(NotNan);
1906     Value *VAbs = EmitFAbs(*this, V);
1907     Value *IsInf =
1908       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1909                             "isinf");
1910     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
1911     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
1912     Builder.CreateCondBr(IsInf, End, NotInf);
1913     Result->addIncoming(InfLiteral, NotNan);
1914 
1915     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
1916     Builder.SetInsertPoint(NotInf);
1917     APFloat Smallest = APFloat::getSmallestNormalized(
1918         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
1919     Value *IsNormal =
1920       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1921                             "isnormal");
1922     Value *NormalResult =
1923       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
1924                            EmitScalarExpr(E->getArg(3)));
1925     Builder.CreateBr(End);
1926     Result->addIncoming(NormalResult, NotInf);
1927 
1928     // return Result
1929     Builder.SetInsertPoint(End);
1930     return RValue::get(Result);
1931   }
1932 
1933   case Builtin::BIalloca:
1934   case Builtin::BI_alloca:
1935   case Builtin::BI__builtin_alloca: {
1936     Value *Size = EmitScalarExpr(E->getArg(0));
1937     const TargetInfo &TI = getContext().getTargetInfo();
1938     // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
1939     unsigned SuitableAlignmentInBytes =
1940         CGM.getContext()
1941             .toCharUnitsFromBits(TI.getSuitableAlign())
1942             .getQuantity();
1943     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1944     AI->setAlignment(SuitableAlignmentInBytes);
1945     return RValue::get(AI);
1946   }
1947 
1948   case Builtin::BI__builtin_alloca_with_align: {
1949     Value *Size = EmitScalarExpr(E->getArg(0));
1950     Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
1951     auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1952     unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1953     unsigned AlignmentInBytes =
1954         CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1955     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1956     AI->setAlignment(AlignmentInBytes);
1957     return RValue::get(AI);
1958   }
1959 
1960   case Builtin::BIbzero:
1961   case Builtin::BI__builtin_bzero: {
1962     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1963     Value *SizeVal = EmitScalarExpr(E->getArg(1));
1964     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1965                         E->getArg(0)->getExprLoc(), FD, 0);
1966     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
1967     return RValue::get(nullptr);
1968   }
1969   case Builtin::BImemcpy:
1970   case Builtin::BI__builtin_memcpy: {
1971     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1972     Address Src = EmitPointerWithAlignment(E->getArg(1));
1973     Value *SizeVal = EmitScalarExpr(E->getArg(2));
1974     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1975                         E->getArg(0)->getExprLoc(), FD, 0);
1976     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1977                         E->getArg(1)->getExprLoc(), FD, 1);
1978     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1979     return RValue::get(Dest.getPointer());
1980   }
1981 
1982   case Builtin::BI__builtin_char_memchr:
1983     BuiltinID = Builtin::BI__builtin_memchr;
1984     break;
1985 
1986   case Builtin::BI__builtin___memcpy_chk: {
1987     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
1988     llvm::APSInt Size, DstSize;
1989     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1990         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1991       break;
1992     if (Size.ugt(DstSize))
1993       break;
1994     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1995     Address Src = EmitPointerWithAlignment(E->getArg(1));
1996     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1997     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1998     return RValue::get(Dest.getPointer());
1999   }
2000 
2001   case Builtin::BI__builtin_objc_memmove_collectable: {
2002     Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
2003     Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
2004     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2005     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
2006                                                   DestAddr, SrcAddr, SizeVal);
2007     return RValue::get(DestAddr.getPointer());
2008   }
2009 
2010   case Builtin::BI__builtin___memmove_chk: {
2011     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
2012     llvm::APSInt Size, DstSize;
2013     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
2014         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
2015       break;
2016     if (Size.ugt(DstSize))
2017       break;
2018     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2019     Address Src = EmitPointerWithAlignment(E->getArg(1));
2020     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2021     Builder.CreateMemMove(Dest, Src, SizeVal, false);
2022     return RValue::get(Dest.getPointer());
2023   }
2024 
2025   case Builtin::BImemmove:
2026   case Builtin::BI__builtin_memmove: {
2027     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2028     Address Src = EmitPointerWithAlignment(E->getArg(1));
2029     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2030     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2031                         E->getArg(0)->getExprLoc(), FD, 0);
2032     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
2033                         E->getArg(1)->getExprLoc(), FD, 1);
2034     Builder.CreateMemMove(Dest, Src, SizeVal, false);
2035     return RValue::get(Dest.getPointer());
2036   }
2037   case Builtin::BImemset:
2038   case Builtin::BI__builtin_memset: {
2039     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2040     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
2041                                          Builder.getInt8Ty());
2042     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2043     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2044                         E->getArg(0)->getExprLoc(), FD, 0);
2045     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
2046     return RValue::get(Dest.getPointer());
2047   }
2048   case Builtin::BI__builtin___memset_chk: {
2049     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
2050     llvm::APSInt Size, DstSize;
2051     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
2052         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
2053       break;
2054     if (Size.ugt(DstSize))
2055       break;
2056     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2057     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
2058                                          Builder.getInt8Ty());
2059     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2060     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
2061     return RValue::get(Dest.getPointer());
2062   }
2063   case Builtin::BI__builtin_wmemcmp: {
2064     // The MSVC runtime library does not provide a definition of wmemcmp, so we
2065     // need an inline implementation.
2066     if (!getTarget().getTriple().isOSMSVCRT())
2067       break;
2068 
2069     llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
2070 
2071     Value *Dst = EmitScalarExpr(E->getArg(0));
2072     Value *Src = EmitScalarExpr(E->getArg(1));
2073     Value *Size = EmitScalarExpr(E->getArg(2));
2074 
2075     BasicBlock *Entry = Builder.GetInsertBlock();
2076     BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
2077     BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
2078     BasicBlock *Next = createBasicBlock("wmemcmp.next");
2079     BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
2080     Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
2081     Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
2082 
2083     EmitBlock(CmpGT);
2084     PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
2085     DstPhi->addIncoming(Dst, Entry);
2086     PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
2087     SrcPhi->addIncoming(Src, Entry);
2088     PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
2089     SizePhi->addIncoming(Size, Entry);
2090     CharUnits WCharAlign =
2091         getContext().getTypeAlignInChars(getContext().WCharTy);
2092     Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
2093     Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
2094     Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
2095     Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
2096 
2097     EmitBlock(CmpLT);
2098     Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
2099     Builder.CreateCondBr(DstLtSrc, Exit, Next);
2100 
2101     EmitBlock(Next);
2102     Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
2103     Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
2104     Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
2105     Value *NextSizeEq0 =
2106         Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
2107     Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
2108     DstPhi->addIncoming(NextDst, Next);
2109     SrcPhi->addIncoming(NextSrc, Next);
2110     SizePhi->addIncoming(NextSize, Next);
2111 
2112     EmitBlock(Exit);
2113     PHINode *Ret = Builder.CreatePHI(IntTy, 4);
2114     Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
2115     Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
2116     Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
2117     Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
2118     return RValue::get(Ret);
2119   }
2120   case Builtin::BI__builtin_dwarf_cfa: {
2121     // The offset in bytes from the first argument to the CFA.
2122     //
2123     // Why on earth is this in the frontend?  Is there any reason at
2124     // all that the backend can't reasonably determine this while
2125     // lowering llvm.eh.dwarf.cfa()?
2126     //
2127     // TODO: If there's a satisfactory reason, add a target hook for
2128     // this instead of hard-coding 0, which is correct for most targets.
2129     int32_t Offset = 0;
2130 
2131     Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
2132     return RValue::get(Builder.CreateCall(F,
2133                                       llvm::ConstantInt::get(Int32Ty, Offset)));
2134   }
2135   case Builtin::BI__builtin_return_address: {
2136     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
2137                                                    getContext().UnsignedIntTy);
2138     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2139     return RValue::get(Builder.CreateCall(F, Depth));
2140   }
2141   case Builtin::BI_ReturnAddress: {
2142     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2143     return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
2144   }
2145   case Builtin::BI__builtin_frame_address: {
2146     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
2147                                                    getContext().UnsignedIntTy);
2148     Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
2149     return RValue::get(Builder.CreateCall(F, Depth));
2150   }
2151   case Builtin::BI__builtin_extract_return_addr: {
2152     Value *Address = EmitScalarExpr(E->getArg(0));
2153     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
2154     return RValue::get(Result);
2155   }
2156   case Builtin::BI__builtin_frob_return_addr: {
2157     Value *Address = EmitScalarExpr(E->getArg(0));
2158     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
2159     return RValue::get(Result);
2160   }
2161   case Builtin::BI__builtin_dwarf_sp_column: {
2162     llvm::IntegerType *Ty
2163       = cast<llvm::IntegerType>(ConvertType(E->getType()));
2164     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
2165     if (Column == -1) {
2166       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
2167       return RValue::get(llvm::UndefValue::get(Ty));
2168     }
2169     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
2170   }
2171   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
2172     Value *Address = EmitScalarExpr(E->getArg(0));
2173     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
2174       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
2175     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
2176   }
2177   case Builtin::BI__builtin_eh_return: {
2178     Value *Int = EmitScalarExpr(E->getArg(0));
2179     Value *Ptr = EmitScalarExpr(E->getArg(1));
2180 
2181     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
2182     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
2183            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
2184     Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
2185                                   ? Intrinsic::eh_return_i32
2186                                   : Intrinsic::eh_return_i64);
2187     Builder.CreateCall(F, {Int, Ptr});
2188     Builder.CreateUnreachable();
2189 
2190     // We do need to preserve an insertion point.
2191     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
2192 
2193     return RValue::get(nullptr);
2194   }
2195   case Builtin::BI__builtin_unwind_init: {
2196     Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
2197     return RValue::get(Builder.CreateCall(F));
2198   }
2199   case Builtin::BI__builtin_extend_pointer: {
2200     // Extends a pointer to the size of an _Unwind_Word, which is
2201     // uint64_t on all platforms.  Generally this gets poked into a
2202     // register and eventually used as an address, so if the
2203     // addressing registers are wider than pointers and the platform
2204     // doesn't implicitly ignore high-order bits when doing
2205     // addressing, we need to make sure we zext / sext based on
2206     // the platform's expectations.
2207     //
2208     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
2209 
2210     // Cast the pointer to intptr_t.
2211     Value *Ptr = EmitScalarExpr(E->getArg(0));
2212     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
2213 
2214     // If that's 64 bits, we're done.
2215     if (IntPtrTy->getBitWidth() == 64)
2216       return RValue::get(Result);
2217 
2218     // Otherwise, ask the codegen data what to do.
2219     if (getTargetHooks().extendPointerWithSExt())
2220       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
2221     else
2222       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
2223   }
2224   case Builtin::BI__builtin_setjmp: {
2225     // Buffer is a void**.
2226     Address Buf = EmitPointerWithAlignment(E->getArg(0));
2227 
2228     // Store the frame pointer to the setjmp buffer.
2229     Value *FrameAddr =
2230       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2231                          ConstantInt::get(Int32Ty, 0));
2232     Builder.CreateStore(FrameAddr, Buf);
2233 
2234     // Store the stack pointer to the setjmp buffer.
2235     Value *StackAddr =
2236         Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
2237     Address StackSaveSlot =
2238       Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
2239     Builder.CreateStore(StackAddr, StackSaveSlot);
2240 
2241     // Call LLVM's EH setjmp, which is lightweight.
2242     Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
2243     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2244     return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
2245   }
2246   case Builtin::BI__builtin_longjmp: {
2247     Value *Buf = EmitScalarExpr(E->getArg(0));
2248     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2249 
2250     // Call LLVM's EH longjmp, which is lightweight.
2251     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
2252 
2253     // longjmp doesn't return; mark this as unreachable.
2254     Builder.CreateUnreachable();
2255 
2256     // We do need to preserve an insertion point.
2257     EmitBlock(createBasicBlock("longjmp.cont"));
2258 
2259     return RValue::get(nullptr);
2260   }
2261   case Builtin::BI__sync_fetch_and_add:
2262   case Builtin::BI__sync_fetch_and_sub:
2263   case Builtin::BI__sync_fetch_and_or:
2264   case Builtin::BI__sync_fetch_and_and:
2265   case Builtin::BI__sync_fetch_and_xor:
2266   case Builtin::BI__sync_fetch_and_nand:
2267   case Builtin::BI__sync_add_and_fetch:
2268   case Builtin::BI__sync_sub_and_fetch:
2269   case Builtin::BI__sync_and_and_fetch:
2270   case Builtin::BI__sync_or_and_fetch:
2271   case Builtin::BI__sync_xor_and_fetch:
2272   case Builtin::BI__sync_nand_and_fetch:
2273   case Builtin::BI__sync_val_compare_and_swap:
2274   case Builtin::BI__sync_bool_compare_and_swap:
2275   case Builtin::BI__sync_lock_test_and_set:
2276   case Builtin::BI__sync_lock_release:
2277   case Builtin::BI__sync_swap:
2278     llvm_unreachable("Shouldn't make it through sema");
2279   case Builtin::BI__sync_fetch_and_add_1:
2280   case Builtin::BI__sync_fetch_and_add_2:
2281   case Builtin::BI__sync_fetch_and_add_4:
2282   case Builtin::BI__sync_fetch_and_add_8:
2283   case Builtin::BI__sync_fetch_and_add_16:
2284     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
2285   case Builtin::BI__sync_fetch_and_sub_1:
2286   case Builtin::BI__sync_fetch_and_sub_2:
2287   case Builtin::BI__sync_fetch_and_sub_4:
2288   case Builtin::BI__sync_fetch_and_sub_8:
2289   case Builtin::BI__sync_fetch_and_sub_16:
2290     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
2291   case Builtin::BI__sync_fetch_and_or_1:
2292   case Builtin::BI__sync_fetch_and_or_2:
2293   case Builtin::BI__sync_fetch_and_or_4:
2294   case Builtin::BI__sync_fetch_and_or_8:
2295   case Builtin::BI__sync_fetch_and_or_16:
2296     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
2297   case Builtin::BI__sync_fetch_and_and_1:
2298   case Builtin::BI__sync_fetch_and_and_2:
2299   case Builtin::BI__sync_fetch_and_and_4:
2300   case Builtin::BI__sync_fetch_and_and_8:
2301   case Builtin::BI__sync_fetch_and_and_16:
2302     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
2303   case Builtin::BI__sync_fetch_and_xor_1:
2304   case Builtin::BI__sync_fetch_and_xor_2:
2305   case Builtin::BI__sync_fetch_and_xor_4:
2306   case Builtin::BI__sync_fetch_and_xor_8:
2307   case Builtin::BI__sync_fetch_and_xor_16:
2308     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
2309   case Builtin::BI__sync_fetch_and_nand_1:
2310   case Builtin::BI__sync_fetch_and_nand_2:
2311   case Builtin::BI__sync_fetch_and_nand_4:
2312   case Builtin::BI__sync_fetch_and_nand_8:
2313   case Builtin::BI__sync_fetch_and_nand_16:
2314     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
2315 
2316   // Clang extensions: not overloaded yet.
2317   case Builtin::BI__sync_fetch_and_min:
2318     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
2319   case Builtin::BI__sync_fetch_and_max:
2320     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
2321   case Builtin::BI__sync_fetch_and_umin:
2322     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
2323   case Builtin::BI__sync_fetch_and_umax:
2324     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
2325 
2326   case Builtin::BI__sync_add_and_fetch_1:
2327   case Builtin::BI__sync_add_and_fetch_2:
2328   case Builtin::BI__sync_add_and_fetch_4:
2329   case Builtin::BI__sync_add_and_fetch_8:
2330   case Builtin::BI__sync_add_and_fetch_16:
2331     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
2332                                 llvm::Instruction::Add);
2333   case Builtin::BI__sync_sub_and_fetch_1:
2334   case Builtin::BI__sync_sub_and_fetch_2:
2335   case Builtin::BI__sync_sub_and_fetch_4:
2336   case Builtin::BI__sync_sub_and_fetch_8:
2337   case Builtin::BI__sync_sub_and_fetch_16:
2338     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
2339                                 llvm::Instruction::Sub);
2340   case Builtin::BI__sync_and_and_fetch_1:
2341   case Builtin::BI__sync_and_and_fetch_2:
2342   case Builtin::BI__sync_and_and_fetch_4:
2343   case Builtin::BI__sync_and_and_fetch_8:
2344   case Builtin::BI__sync_and_and_fetch_16:
2345     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
2346                                 llvm::Instruction::And);
2347   case Builtin::BI__sync_or_and_fetch_1:
2348   case Builtin::BI__sync_or_and_fetch_2:
2349   case Builtin::BI__sync_or_and_fetch_4:
2350   case Builtin::BI__sync_or_and_fetch_8:
2351   case Builtin::BI__sync_or_and_fetch_16:
2352     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
2353                                 llvm::Instruction::Or);
2354   case Builtin::BI__sync_xor_and_fetch_1:
2355   case Builtin::BI__sync_xor_and_fetch_2:
2356   case Builtin::BI__sync_xor_and_fetch_4:
2357   case Builtin::BI__sync_xor_and_fetch_8:
2358   case Builtin::BI__sync_xor_and_fetch_16:
2359     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
2360                                 llvm::Instruction::Xor);
2361   case Builtin::BI__sync_nand_and_fetch_1:
2362   case Builtin::BI__sync_nand_and_fetch_2:
2363   case Builtin::BI__sync_nand_and_fetch_4:
2364   case Builtin::BI__sync_nand_and_fetch_8:
2365   case Builtin::BI__sync_nand_and_fetch_16:
2366     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
2367                                 llvm::Instruction::And, true);
2368 
2369   case Builtin::BI__sync_val_compare_and_swap_1:
2370   case Builtin::BI__sync_val_compare_and_swap_2:
2371   case Builtin::BI__sync_val_compare_and_swap_4:
2372   case Builtin::BI__sync_val_compare_and_swap_8:
2373   case Builtin::BI__sync_val_compare_and_swap_16:
2374     return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
2375 
2376   case Builtin::BI__sync_bool_compare_and_swap_1:
2377   case Builtin::BI__sync_bool_compare_and_swap_2:
2378   case Builtin::BI__sync_bool_compare_and_swap_4:
2379   case Builtin::BI__sync_bool_compare_and_swap_8:
2380   case Builtin::BI__sync_bool_compare_and_swap_16:
2381     return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
2382 
2383   case Builtin::BI__sync_swap_1:
2384   case Builtin::BI__sync_swap_2:
2385   case Builtin::BI__sync_swap_4:
2386   case Builtin::BI__sync_swap_8:
2387   case Builtin::BI__sync_swap_16:
2388     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
2389 
2390   case Builtin::BI__sync_lock_test_and_set_1:
2391   case Builtin::BI__sync_lock_test_and_set_2:
2392   case Builtin::BI__sync_lock_test_and_set_4:
2393   case Builtin::BI__sync_lock_test_and_set_8:
2394   case Builtin::BI__sync_lock_test_and_set_16:
2395     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
2396 
2397   case Builtin::BI__sync_lock_release_1:
2398   case Builtin::BI__sync_lock_release_2:
2399   case Builtin::BI__sync_lock_release_4:
2400   case Builtin::BI__sync_lock_release_8:
2401   case Builtin::BI__sync_lock_release_16: {
2402     Value *Ptr = EmitScalarExpr(E->getArg(0));
2403     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
2404     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
2405     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
2406                                              StoreSize.getQuantity() * 8);
2407     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
2408     llvm::StoreInst *Store =
2409       Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
2410                                  StoreSize);
2411     Store->setAtomic(llvm::AtomicOrdering::Release);
2412     return RValue::get(nullptr);
2413   }
2414 
2415   case Builtin::BI__sync_synchronize: {
2416     // We assume this is supposed to correspond to a C++0x-style
2417     // sequentially-consistent fence (i.e. this is only usable for
2418     // synchronization, not device I/O or anything like that). This intrinsic
2419     // is really badly designed in the sense that in theory, there isn't
2420     // any way to safely use it... but in practice, it mostly works
2421     // to use it with non-atomic loads and stores to get acquire/release
2422     // semantics.
2423     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
2424     return RValue::get(nullptr);
2425   }
2426 
2427   case Builtin::BI__builtin_nontemporal_load:
2428     return RValue::get(EmitNontemporalLoad(*this, E));
2429   case Builtin::BI__builtin_nontemporal_store:
2430     return RValue::get(EmitNontemporalStore(*this, E));
2431   case Builtin::BI__c11_atomic_is_lock_free:
2432   case Builtin::BI__atomic_is_lock_free: {
2433     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
2434     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
2435     // _Atomic(T) is always properly-aligned.
2436     const char *LibCallName = "__atomic_is_lock_free";
2437     CallArgList Args;
2438     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
2439              getContext().getSizeType());
2440     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
2441       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
2442                getContext().VoidPtrTy);
2443     else
2444       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
2445                getContext().VoidPtrTy);
2446     const CGFunctionInfo &FuncInfo =
2447         CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
2448     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
2449     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
2450     return EmitCall(FuncInfo, CGCallee::forDirect(Func),
2451                     ReturnValueSlot(), Args);
2452   }
2453 
2454   case Builtin::BI__atomic_test_and_set: {
2455     // Look at the argument type to determine whether this is a volatile
2456     // operation. The parameter type is always volatile.
2457     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
2458     bool Volatile =
2459         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
2460 
2461     Value *Ptr = EmitScalarExpr(E->getArg(0));
2462     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
2463     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2464     Value *NewVal = Builder.getInt8(1);
2465     Value *Order = EmitScalarExpr(E->getArg(1));
2466     if (isa<llvm::ConstantInt>(Order)) {
2467       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2468       AtomicRMWInst *Result = nullptr;
2469       switch (ord) {
2470       case 0:  // memory_order_relaxed
2471       default: // invalid order
2472         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2473                                          llvm::AtomicOrdering::Monotonic);
2474         break;
2475       case 1: // memory_order_consume
2476       case 2: // memory_order_acquire
2477         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2478                                          llvm::AtomicOrdering::Acquire);
2479         break;
2480       case 3: // memory_order_release
2481         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2482                                          llvm::AtomicOrdering::Release);
2483         break;
2484       case 4: // memory_order_acq_rel
2485 
2486         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2487                                          llvm::AtomicOrdering::AcquireRelease);
2488         break;
2489       case 5: // memory_order_seq_cst
2490         Result = Builder.CreateAtomicRMW(
2491             llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2492             llvm::AtomicOrdering::SequentiallyConsistent);
2493         break;
2494       }
2495       Result->setVolatile(Volatile);
2496       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
2497     }
2498 
2499     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2500 
2501     llvm::BasicBlock *BBs[5] = {
2502       createBasicBlock("monotonic", CurFn),
2503       createBasicBlock("acquire", CurFn),
2504       createBasicBlock("release", CurFn),
2505       createBasicBlock("acqrel", CurFn),
2506       createBasicBlock("seqcst", CurFn)
2507     };
2508     llvm::AtomicOrdering Orders[5] = {
2509         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
2510         llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
2511         llvm::AtomicOrdering::SequentiallyConsistent};
2512 
2513     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2514     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2515 
2516     Builder.SetInsertPoint(ContBB);
2517     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
2518 
2519     for (unsigned i = 0; i < 5; ++i) {
2520       Builder.SetInsertPoint(BBs[i]);
2521       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
2522                                                    Ptr, NewVal, Orders[i]);
2523       RMW->setVolatile(Volatile);
2524       Result->addIncoming(RMW, BBs[i]);
2525       Builder.CreateBr(ContBB);
2526     }
2527 
2528     SI->addCase(Builder.getInt32(0), BBs[0]);
2529     SI->addCase(Builder.getInt32(1), BBs[1]);
2530     SI->addCase(Builder.getInt32(2), BBs[1]);
2531     SI->addCase(Builder.getInt32(3), BBs[2]);
2532     SI->addCase(Builder.getInt32(4), BBs[3]);
2533     SI->addCase(Builder.getInt32(5), BBs[4]);
2534 
2535     Builder.SetInsertPoint(ContBB);
2536     return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
2537   }
2538 
2539   case Builtin::BI__atomic_clear: {
2540     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
2541     bool Volatile =
2542         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
2543 
2544     Address Ptr = EmitPointerWithAlignment(E->getArg(0));
2545     unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
2546     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2547     Value *NewVal = Builder.getInt8(0);
2548     Value *Order = EmitScalarExpr(E->getArg(1));
2549     if (isa<llvm::ConstantInt>(Order)) {
2550       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2551       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2552       switch (ord) {
2553       case 0:  // memory_order_relaxed
2554       default: // invalid order
2555         Store->setOrdering(llvm::AtomicOrdering::Monotonic);
2556         break;
2557       case 3:  // memory_order_release
2558         Store->setOrdering(llvm::AtomicOrdering::Release);
2559         break;
2560       case 5:  // memory_order_seq_cst
2561         Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
2562         break;
2563       }
2564       return RValue::get(nullptr);
2565     }
2566 
2567     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2568 
2569     llvm::BasicBlock *BBs[3] = {
2570       createBasicBlock("monotonic", CurFn),
2571       createBasicBlock("release", CurFn),
2572       createBasicBlock("seqcst", CurFn)
2573     };
2574     llvm::AtomicOrdering Orders[3] = {
2575         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
2576         llvm::AtomicOrdering::SequentiallyConsistent};
2577 
2578     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2579     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2580 
2581     for (unsigned i = 0; i < 3; ++i) {
2582       Builder.SetInsertPoint(BBs[i]);
2583       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2584       Store->setOrdering(Orders[i]);
2585       Builder.CreateBr(ContBB);
2586     }
2587 
2588     SI->addCase(Builder.getInt32(0), BBs[0]);
2589     SI->addCase(Builder.getInt32(3), BBs[1]);
2590     SI->addCase(Builder.getInt32(5), BBs[2]);
2591 
2592     Builder.SetInsertPoint(ContBB);
2593     return RValue::get(nullptr);
2594   }
2595 
2596   case Builtin::BI__atomic_thread_fence:
2597   case Builtin::BI__atomic_signal_fence:
2598   case Builtin::BI__c11_atomic_thread_fence:
2599   case Builtin::BI__c11_atomic_signal_fence: {
2600     llvm::SyncScope::ID SSID;
2601     if (BuiltinID == Builtin::BI__atomic_signal_fence ||
2602         BuiltinID == Builtin::BI__c11_atomic_signal_fence)
2603       SSID = llvm::SyncScope::SingleThread;
2604     else
2605       SSID = llvm::SyncScope::System;
2606     Value *Order = EmitScalarExpr(E->getArg(0));
2607     if (isa<llvm::ConstantInt>(Order)) {
2608       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2609       switch (ord) {
2610       case 0:  // memory_order_relaxed
2611       default: // invalid order
2612         break;
2613       case 1:  // memory_order_consume
2614       case 2:  // memory_order_acquire
2615         Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2616         break;
2617       case 3:  // memory_order_release
2618         Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2619         break;
2620       case 4:  // memory_order_acq_rel
2621         Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2622         break;
2623       case 5:  // memory_order_seq_cst
2624         Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2625         break;
2626       }
2627       return RValue::get(nullptr);
2628     }
2629 
2630     llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
2631     AcquireBB = createBasicBlock("acquire", CurFn);
2632     ReleaseBB = createBasicBlock("release", CurFn);
2633     AcqRelBB = createBasicBlock("acqrel", CurFn);
2634     SeqCstBB = createBasicBlock("seqcst", CurFn);
2635     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2636 
2637     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2638     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
2639 
2640     Builder.SetInsertPoint(AcquireBB);
2641     Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2642     Builder.CreateBr(ContBB);
2643     SI->addCase(Builder.getInt32(1), AcquireBB);
2644     SI->addCase(Builder.getInt32(2), AcquireBB);
2645 
2646     Builder.SetInsertPoint(ReleaseBB);
2647     Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2648     Builder.CreateBr(ContBB);
2649     SI->addCase(Builder.getInt32(3), ReleaseBB);
2650 
2651     Builder.SetInsertPoint(AcqRelBB);
2652     Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2653     Builder.CreateBr(ContBB);
2654     SI->addCase(Builder.getInt32(4), AcqRelBB);
2655 
2656     Builder.SetInsertPoint(SeqCstBB);
2657     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2658     Builder.CreateBr(ContBB);
2659     SI->addCase(Builder.getInt32(5), SeqCstBB);
2660 
2661     Builder.SetInsertPoint(ContBB);
2662     return RValue::get(nullptr);
2663   }
2664 
2665   case Builtin::BI__builtin_signbit:
2666   case Builtin::BI__builtin_signbitf:
2667   case Builtin::BI__builtin_signbitl: {
2668     return RValue::get(
2669         Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
2670                            ConvertType(E->getType())));
2671   }
2672   case Builtin::BI__annotation: {
2673     // Re-encode each wide string to UTF8 and make an MDString.
2674     SmallVector<Metadata *, 1> Strings;
2675     for (const Expr *Arg : E->arguments()) {
2676       const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
2677       assert(Str->getCharByteWidth() == 2);
2678       StringRef WideBytes = Str->getBytes();
2679       std::string StrUtf8;
2680       if (!convertUTF16ToUTF8String(
2681               makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
2682         CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
2683         continue;
2684       }
2685       Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
2686     }
2687 
2688     // Build and MDTuple of MDStrings and emit the intrinsic call.
2689     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
2690     MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
2691     Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
2692     return RValue::getIgnored();
2693   }
2694   case Builtin::BI__builtin_annotation: {
2695     llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
2696     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
2697                                       AnnVal->getType());
2698 
2699     // Get the annotation string, go through casts. Sema requires this to be a
2700     // non-wide string literal, potentially casted, so the cast<> is safe.
2701     const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
2702     StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
2703     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
2704   }
2705   case Builtin::BI__builtin_addcb:
2706   case Builtin::BI__builtin_addcs:
2707   case Builtin::BI__builtin_addc:
2708   case Builtin::BI__builtin_addcl:
2709   case Builtin::BI__builtin_addcll:
2710   case Builtin::BI__builtin_subcb:
2711   case Builtin::BI__builtin_subcs:
2712   case Builtin::BI__builtin_subc:
2713   case Builtin::BI__builtin_subcl:
2714   case Builtin::BI__builtin_subcll: {
2715 
2716     // We translate all of these builtins from expressions of the form:
2717     //   int x = ..., y = ..., carryin = ..., carryout, result;
2718     //   result = __builtin_addc(x, y, carryin, &carryout);
2719     //
2720     // to LLVM IR of the form:
2721     //
2722     //   %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
2723     //   %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
2724     //   %carry1 = extractvalue {i32, i1} %tmp1, 1
2725     //   %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
2726     //                                                       i32 %carryin)
2727     //   %result = extractvalue {i32, i1} %tmp2, 0
2728     //   %carry2 = extractvalue {i32, i1} %tmp2, 1
2729     //   %tmp3 = or i1 %carry1, %carry2
2730     //   %tmp4 = zext i1 %tmp3 to i32
2731     //   store i32 %tmp4, i32* %carryout
2732 
2733     // Scalarize our inputs.
2734     llvm::Value *X = EmitScalarExpr(E->getArg(0));
2735     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2736     llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
2737     Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
2738 
2739     // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
2740     llvm::Intrinsic::ID IntrinsicId;
2741     switch (BuiltinID) {
2742     default: llvm_unreachable("Unknown multiprecision builtin id.");
2743     case Builtin::BI__builtin_addcb:
2744     case Builtin::BI__builtin_addcs:
2745     case Builtin::BI__builtin_addc:
2746     case Builtin::BI__builtin_addcl:
2747     case Builtin::BI__builtin_addcll:
2748       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2749       break;
2750     case Builtin::BI__builtin_subcb:
2751     case Builtin::BI__builtin_subcs:
2752     case Builtin::BI__builtin_subc:
2753     case Builtin::BI__builtin_subcl:
2754     case Builtin::BI__builtin_subcll:
2755       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2756       break;
2757     }
2758 
2759     // Construct our resulting LLVM IR expression.
2760     llvm::Value *Carry1;
2761     llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
2762                                               X, Y, Carry1);
2763     llvm::Value *Carry2;
2764     llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
2765                                               Sum1, Carryin, Carry2);
2766     llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
2767                                                X->getType());
2768     Builder.CreateStore(CarryOut, CarryOutPtr);
2769     return RValue::get(Sum2);
2770   }
2771 
2772   case Builtin::BI__builtin_add_overflow:
2773   case Builtin::BI__builtin_sub_overflow:
2774   case Builtin::BI__builtin_mul_overflow: {
2775     const clang::Expr *LeftArg = E->getArg(0);
2776     const clang::Expr *RightArg = E->getArg(1);
2777     const clang::Expr *ResultArg = E->getArg(2);
2778 
2779     clang::QualType ResultQTy =
2780         ResultArg->getType()->castAs<PointerType>()->getPointeeType();
2781 
2782     WidthAndSignedness LeftInfo =
2783         getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
2784     WidthAndSignedness RightInfo =
2785         getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
2786     WidthAndSignedness ResultInfo =
2787         getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
2788 
2789     // Handle mixed-sign multiplication as a special case, because adding
2790     // runtime or backend support for our generic irgen would be too expensive.
2791     if (isSpecialMixedSignMultiply(BuiltinID, LeftInfo, RightInfo, ResultInfo))
2792       return EmitCheckedMixedSignMultiply(*this, LeftArg, LeftInfo, RightArg,
2793                                           RightInfo, ResultArg, ResultQTy,
2794                                           ResultInfo);
2795 
2796     WidthAndSignedness EncompassingInfo =
2797         EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
2798 
2799     llvm::Type *EncompassingLLVMTy =
2800         llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
2801 
2802     llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
2803 
2804     llvm::Intrinsic::ID IntrinsicId;
2805     switch (BuiltinID) {
2806     default:
2807       llvm_unreachable("Unknown overflow builtin id.");
2808     case Builtin::BI__builtin_add_overflow:
2809       IntrinsicId = EncompassingInfo.Signed
2810                         ? llvm::Intrinsic::sadd_with_overflow
2811                         : llvm::Intrinsic::uadd_with_overflow;
2812       break;
2813     case Builtin::BI__builtin_sub_overflow:
2814       IntrinsicId = EncompassingInfo.Signed
2815                         ? llvm::Intrinsic::ssub_with_overflow
2816                         : llvm::Intrinsic::usub_with_overflow;
2817       break;
2818     case Builtin::BI__builtin_mul_overflow:
2819       IntrinsicId = EncompassingInfo.Signed
2820                         ? llvm::Intrinsic::smul_with_overflow
2821                         : llvm::Intrinsic::umul_with_overflow;
2822       break;
2823     }
2824 
2825     llvm::Value *Left = EmitScalarExpr(LeftArg);
2826     llvm::Value *Right = EmitScalarExpr(RightArg);
2827     Address ResultPtr = EmitPointerWithAlignment(ResultArg);
2828 
2829     // Extend each operand to the encompassing type.
2830     Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
2831     Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
2832 
2833     // Perform the operation on the extended values.
2834     llvm::Value *Overflow, *Result;
2835     Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
2836 
2837     if (EncompassingInfo.Width > ResultInfo.Width) {
2838       // The encompassing type is wider than the result type, so we need to
2839       // truncate it.
2840       llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
2841 
2842       // To see if the truncation caused an overflow, we will extend
2843       // the result and then compare it to the original result.
2844       llvm::Value *ResultTruncExt = Builder.CreateIntCast(
2845           ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2846       llvm::Value *TruncationOverflow =
2847           Builder.CreateICmpNE(Result, ResultTruncExt);
2848 
2849       Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
2850       Result = ResultTrunc;
2851     }
2852 
2853     // Finally, store the result using the pointer.
2854     bool isVolatile =
2855       ResultArg->getType()->getPointeeType().isVolatileQualified();
2856     Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2857 
2858     return RValue::get(Overflow);
2859   }
2860 
2861   case Builtin::BI__builtin_uadd_overflow:
2862   case Builtin::BI__builtin_uaddl_overflow:
2863   case Builtin::BI__builtin_uaddll_overflow:
2864   case Builtin::BI__builtin_usub_overflow:
2865   case Builtin::BI__builtin_usubl_overflow:
2866   case Builtin::BI__builtin_usubll_overflow:
2867   case Builtin::BI__builtin_umul_overflow:
2868   case Builtin::BI__builtin_umull_overflow:
2869   case Builtin::BI__builtin_umulll_overflow:
2870   case Builtin::BI__builtin_sadd_overflow:
2871   case Builtin::BI__builtin_saddl_overflow:
2872   case Builtin::BI__builtin_saddll_overflow:
2873   case Builtin::BI__builtin_ssub_overflow:
2874   case Builtin::BI__builtin_ssubl_overflow:
2875   case Builtin::BI__builtin_ssubll_overflow:
2876   case Builtin::BI__builtin_smul_overflow:
2877   case Builtin::BI__builtin_smull_overflow:
2878   case Builtin::BI__builtin_smulll_overflow: {
2879 
2880     // We translate all of these builtins directly to the relevant llvm IR node.
2881 
2882     // Scalarize our inputs.
2883     llvm::Value *X = EmitScalarExpr(E->getArg(0));
2884     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2885     Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
2886 
2887     // Decide which of the overflow intrinsics we are lowering to:
2888     llvm::Intrinsic::ID IntrinsicId;
2889     switch (BuiltinID) {
2890     default: llvm_unreachable("Unknown overflow builtin id.");
2891     case Builtin::BI__builtin_uadd_overflow:
2892     case Builtin::BI__builtin_uaddl_overflow:
2893     case Builtin::BI__builtin_uaddll_overflow:
2894       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2895       break;
2896     case Builtin::BI__builtin_usub_overflow:
2897     case Builtin::BI__builtin_usubl_overflow:
2898     case Builtin::BI__builtin_usubll_overflow:
2899       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2900       break;
2901     case Builtin::BI__builtin_umul_overflow:
2902     case Builtin::BI__builtin_umull_overflow:
2903     case Builtin::BI__builtin_umulll_overflow:
2904       IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2905       break;
2906     case Builtin::BI__builtin_sadd_overflow:
2907     case Builtin::BI__builtin_saddl_overflow:
2908     case Builtin::BI__builtin_saddll_overflow:
2909       IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2910       break;
2911     case Builtin::BI__builtin_ssub_overflow:
2912     case Builtin::BI__builtin_ssubl_overflow:
2913     case Builtin::BI__builtin_ssubll_overflow:
2914       IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2915       break;
2916     case Builtin::BI__builtin_smul_overflow:
2917     case Builtin::BI__builtin_smull_overflow:
2918     case Builtin::BI__builtin_smulll_overflow:
2919       IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2920       break;
2921     }
2922 
2923 
2924     llvm::Value *Carry;
2925     llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
2926     Builder.CreateStore(Sum, SumOutPtr);
2927 
2928     return RValue::get(Carry);
2929   }
2930   case Builtin::BI__builtin_addressof:
2931     return RValue::get(EmitLValue(E->getArg(0)).getPointer());
2932   case Builtin::BI__builtin_operator_new:
2933     return EmitBuiltinNewDeleteCall(
2934         E->getCallee()->getType()->castAs<FunctionProtoType>(), E, false);
2935   case Builtin::BI__builtin_operator_delete:
2936     return EmitBuiltinNewDeleteCall(
2937         E->getCallee()->getType()->castAs<FunctionProtoType>(), E, true);
2938 
2939   case Builtin::BI__noop:
2940     // __noop always evaluates to an integer literal zero.
2941     return RValue::get(ConstantInt::get(IntTy, 0));
2942   case Builtin::BI__builtin_call_with_static_chain: {
2943     const CallExpr *Call = cast<CallExpr>(E->getArg(0));
2944     const Expr *Chain = E->getArg(1);
2945     return EmitCall(Call->getCallee()->getType(),
2946                     EmitCallee(Call->getCallee()), Call, ReturnValue,
2947                     EmitScalarExpr(Chain));
2948   }
2949   case Builtin::BI_InterlockedExchange8:
2950   case Builtin::BI_InterlockedExchange16:
2951   case Builtin::BI_InterlockedExchange:
2952   case Builtin::BI_InterlockedExchangePointer:
2953     return RValue::get(
2954         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2955   case Builtin::BI_InterlockedCompareExchangePointer: {
2956     llvm::Type *RTy;
2957     llvm::IntegerType *IntType =
2958       IntegerType::get(getLLVMContext(),
2959                        getContext().getTypeSize(E->getType()));
2960     llvm::Type *IntPtrType = IntType->getPointerTo();
2961 
2962     llvm::Value *Destination =
2963       Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
2964 
2965     llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
2966     RTy = Exchange->getType();
2967     Exchange = Builder.CreatePtrToInt(Exchange, IntType);
2968 
2969     llvm::Value *Comparand =
2970       Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
2971 
2972     auto Result =
2973         Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2974                                     AtomicOrdering::SequentiallyConsistent,
2975                                     AtomicOrdering::SequentiallyConsistent);
2976     Result->setVolatile(true);
2977 
2978     return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
2979                                                                          0),
2980                                               RTy));
2981   }
2982   case Builtin::BI_InterlockedCompareExchange8:
2983   case Builtin::BI_InterlockedCompareExchange16:
2984   case Builtin::BI_InterlockedCompareExchange:
2985   case Builtin::BI_InterlockedCompareExchange64: {
2986     AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
2987         EmitScalarExpr(E->getArg(0)),
2988         EmitScalarExpr(E->getArg(2)),
2989         EmitScalarExpr(E->getArg(1)),
2990         AtomicOrdering::SequentiallyConsistent,
2991         AtomicOrdering::SequentiallyConsistent);
2992       CXI->setVolatile(true);
2993       return RValue::get(Builder.CreateExtractValue(CXI, 0));
2994   }
2995   case Builtin::BI_InterlockedIncrement16:
2996   case Builtin::BI_InterlockedIncrement:
2997     return RValue::get(
2998         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2999   case Builtin::BI_InterlockedDecrement16:
3000   case Builtin::BI_InterlockedDecrement:
3001     return RValue::get(
3002         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
3003   case Builtin::BI_InterlockedAnd8:
3004   case Builtin::BI_InterlockedAnd16:
3005   case Builtin::BI_InterlockedAnd:
3006     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
3007   case Builtin::BI_InterlockedExchangeAdd8:
3008   case Builtin::BI_InterlockedExchangeAdd16:
3009   case Builtin::BI_InterlockedExchangeAdd:
3010     return RValue::get(
3011         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
3012   case Builtin::BI_InterlockedExchangeSub8:
3013   case Builtin::BI_InterlockedExchangeSub16:
3014   case Builtin::BI_InterlockedExchangeSub:
3015     return RValue::get(
3016         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
3017   case Builtin::BI_InterlockedOr8:
3018   case Builtin::BI_InterlockedOr16:
3019   case Builtin::BI_InterlockedOr:
3020     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
3021   case Builtin::BI_InterlockedXor8:
3022   case Builtin::BI_InterlockedXor16:
3023   case Builtin::BI_InterlockedXor:
3024     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
3025 
3026   case Builtin::BI_bittest64:
3027   case Builtin::BI_bittest:
3028   case Builtin::BI_bittestandcomplement64:
3029   case Builtin::BI_bittestandcomplement:
3030   case Builtin::BI_bittestandreset64:
3031   case Builtin::BI_bittestandreset:
3032   case Builtin::BI_bittestandset64:
3033   case Builtin::BI_bittestandset:
3034   case Builtin::BI_interlockedbittestandreset:
3035   case Builtin::BI_interlockedbittestandreset64:
3036   case Builtin::BI_interlockedbittestandset64:
3037   case Builtin::BI_interlockedbittestandset:
3038   case Builtin::BI_interlockedbittestandset_acq:
3039   case Builtin::BI_interlockedbittestandset_rel:
3040   case Builtin::BI_interlockedbittestandset_nf:
3041   case Builtin::BI_interlockedbittestandreset_acq:
3042   case Builtin::BI_interlockedbittestandreset_rel:
3043   case Builtin::BI_interlockedbittestandreset_nf:
3044     return RValue::get(EmitBitTestIntrinsic(*this, BuiltinID, E));
3045 
3046   case Builtin::BI__exception_code:
3047   case Builtin::BI_exception_code:
3048     return RValue::get(EmitSEHExceptionCode());
3049   case Builtin::BI__exception_info:
3050   case Builtin::BI_exception_info:
3051     return RValue::get(EmitSEHExceptionInfo());
3052   case Builtin::BI__abnormal_termination:
3053   case Builtin::BI_abnormal_termination:
3054     return RValue::get(EmitSEHAbnormalTermination());
3055   case Builtin::BI_setjmpex:
3056     if (getTarget().getTriple().isOSMSVCRT())
3057       return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmpex, E);
3058     break;
3059   case Builtin::BI_setjmp:
3060     if (getTarget().getTriple().isOSMSVCRT()) {
3061       if (getTarget().getTriple().getArch() == llvm::Triple::x86)
3062         return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmp3, E);
3063       else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
3064         return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmpex, E);
3065       return EmitMSVCRTSetJmp(*this, MSVCSetJmpKind::_setjmp, E);
3066     }
3067     break;
3068 
3069   case Builtin::BI__GetExceptionInfo: {
3070     if (llvm::GlobalVariable *GV =
3071             CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
3072       return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
3073     break;
3074   }
3075 
3076   case Builtin::BI__fastfail:
3077     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
3078 
3079   case Builtin::BI__builtin_coro_size: {
3080     auto & Context = getContext();
3081     auto SizeTy = Context.getSizeType();
3082     auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3083     Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
3084     return RValue::get(Builder.CreateCall(F));
3085   }
3086 
3087   case Builtin::BI__builtin_coro_id:
3088     return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
3089   case Builtin::BI__builtin_coro_promise:
3090     return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
3091   case Builtin::BI__builtin_coro_resume:
3092     return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
3093   case Builtin::BI__builtin_coro_frame:
3094     return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
3095   case Builtin::BI__builtin_coro_noop:
3096     return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
3097   case Builtin::BI__builtin_coro_free:
3098     return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
3099   case Builtin::BI__builtin_coro_destroy:
3100     return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
3101   case Builtin::BI__builtin_coro_done:
3102     return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
3103   case Builtin::BI__builtin_coro_alloc:
3104     return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
3105   case Builtin::BI__builtin_coro_begin:
3106     return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
3107   case Builtin::BI__builtin_coro_end:
3108     return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
3109   case Builtin::BI__builtin_coro_suspend:
3110     return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
3111   case Builtin::BI__builtin_coro_param:
3112     return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
3113 
3114   // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
3115   case Builtin::BIread_pipe:
3116   case Builtin::BIwrite_pipe: {
3117     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3118           *Arg1 = EmitScalarExpr(E->getArg(1));
3119     CGOpenCLRuntime OpenCLRT(CGM);
3120     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3121     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3122 
3123     // Type of the generic packet parameter.
3124     unsigned GenericAS =
3125         getContext().getTargetAddressSpace(LangAS::opencl_generic);
3126     llvm::Type *I8PTy = llvm::PointerType::get(
3127         llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
3128 
3129     // Testing which overloaded version we should generate the call for.
3130     if (2U == E->getNumArgs()) {
3131       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
3132                                                              : "__write_pipe_2";
3133       // Creating a generic function type to be able to call with any builtin or
3134       // user defined type.
3135       llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
3136       llvm::FunctionType *FTy = llvm::FunctionType::get(
3137           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3138       Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
3139       return RValue::get(
3140           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3141                              {Arg0, BCast, PacketSize, PacketAlign}));
3142     } else {
3143       assert(4 == E->getNumArgs() &&
3144              "Illegal number of parameters to pipe function");
3145       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
3146                                                              : "__write_pipe_4";
3147 
3148       llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
3149                               Int32Ty, Int32Ty};
3150       Value *Arg2 = EmitScalarExpr(E->getArg(2)),
3151             *Arg3 = EmitScalarExpr(E->getArg(3));
3152       llvm::FunctionType *FTy = llvm::FunctionType::get(
3153           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3154       Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
3155       // We know the third argument is an integer type, but we may need to cast
3156       // it to i32.
3157       if (Arg2->getType() != Int32Ty)
3158         Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
3159       return RValue::get(Builder.CreateCall(
3160           CGM.CreateRuntimeFunction(FTy, Name),
3161           {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
3162     }
3163   }
3164   // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
3165   // functions
3166   case Builtin::BIreserve_read_pipe:
3167   case Builtin::BIreserve_write_pipe:
3168   case Builtin::BIwork_group_reserve_read_pipe:
3169   case Builtin::BIwork_group_reserve_write_pipe:
3170   case Builtin::BIsub_group_reserve_read_pipe:
3171   case Builtin::BIsub_group_reserve_write_pipe: {
3172     // Composing the mangled name for the function.
3173     const char *Name;
3174     if (BuiltinID == Builtin::BIreserve_read_pipe)
3175       Name = "__reserve_read_pipe";
3176     else if (BuiltinID == Builtin::BIreserve_write_pipe)
3177       Name = "__reserve_write_pipe";
3178     else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
3179       Name = "__work_group_reserve_read_pipe";
3180     else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
3181       Name = "__work_group_reserve_write_pipe";
3182     else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
3183       Name = "__sub_group_reserve_read_pipe";
3184     else
3185       Name = "__sub_group_reserve_write_pipe";
3186 
3187     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3188           *Arg1 = EmitScalarExpr(E->getArg(1));
3189     llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
3190     CGOpenCLRuntime OpenCLRT(CGM);
3191     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3192     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3193 
3194     // Building the generic function prototype.
3195     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
3196     llvm::FunctionType *FTy = llvm::FunctionType::get(
3197         ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3198     // We know the second argument is an integer type, but we may need to cast
3199     // it to i32.
3200     if (Arg1->getType() != Int32Ty)
3201       Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
3202     return RValue::get(
3203         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3204                            {Arg0, Arg1, PacketSize, PacketAlign}));
3205   }
3206   // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
3207   // functions
3208   case Builtin::BIcommit_read_pipe:
3209   case Builtin::BIcommit_write_pipe:
3210   case Builtin::BIwork_group_commit_read_pipe:
3211   case Builtin::BIwork_group_commit_write_pipe:
3212   case Builtin::BIsub_group_commit_read_pipe:
3213   case Builtin::BIsub_group_commit_write_pipe: {
3214     const char *Name;
3215     if (BuiltinID == Builtin::BIcommit_read_pipe)
3216       Name = "__commit_read_pipe";
3217     else if (BuiltinID == Builtin::BIcommit_write_pipe)
3218       Name = "__commit_write_pipe";
3219     else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
3220       Name = "__work_group_commit_read_pipe";
3221     else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
3222       Name = "__work_group_commit_write_pipe";
3223     else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
3224       Name = "__sub_group_commit_read_pipe";
3225     else
3226       Name = "__sub_group_commit_write_pipe";
3227 
3228     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
3229           *Arg1 = EmitScalarExpr(E->getArg(1));
3230     CGOpenCLRuntime OpenCLRT(CGM);
3231     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3232     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3233 
3234     // Building the generic function prototype.
3235     llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
3236     llvm::FunctionType *FTy =
3237         llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
3238                                 llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3239 
3240     return RValue::get(
3241         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3242                            {Arg0, Arg1, PacketSize, PacketAlign}));
3243   }
3244   // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
3245   case Builtin::BIget_pipe_num_packets:
3246   case Builtin::BIget_pipe_max_packets: {
3247     const char *BaseName;
3248     const PipeType *PipeTy = E->getArg(0)->getType()->getAs<PipeType>();
3249     if (BuiltinID == Builtin::BIget_pipe_num_packets)
3250       BaseName = "__get_pipe_num_packets";
3251     else
3252       BaseName = "__get_pipe_max_packets";
3253     auto Name = std::string(BaseName) +
3254                 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
3255 
3256     // Building the generic function prototype.
3257     Value *Arg0 = EmitScalarExpr(E->getArg(0));
3258     CGOpenCLRuntime OpenCLRT(CGM);
3259     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
3260     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
3261     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
3262     llvm::FunctionType *FTy = llvm::FunctionType::get(
3263         Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3264 
3265     return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3266                                           {Arg0, PacketSize, PacketAlign}));
3267   }
3268 
3269   // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
3270   case Builtin::BIto_global:
3271   case Builtin::BIto_local:
3272   case Builtin::BIto_private: {
3273     auto Arg0 = EmitScalarExpr(E->getArg(0));
3274     auto NewArgT = llvm::PointerType::get(Int8Ty,
3275       CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
3276     auto NewRetT = llvm::PointerType::get(Int8Ty,
3277       CGM.getContext().getTargetAddressSpace(
3278         E->getType()->getPointeeType().getAddressSpace()));
3279     auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
3280     llvm::Value *NewArg;
3281     if (Arg0->getType()->getPointerAddressSpace() !=
3282         NewArgT->getPointerAddressSpace())
3283       NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
3284     else
3285       NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
3286     auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
3287     auto NewCall =
3288         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
3289     return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
3290       ConvertType(E->getType())));
3291   }
3292 
3293   // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
3294   // It contains four different overload formats specified in Table 6.13.17.1.
3295   case Builtin::BIenqueue_kernel: {
3296     StringRef Name; // Generated function call name
3297     unsigned NumArgs = E->getNumArgs();
3298 
3299     llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
3300     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3301         getContext().getTargetAddressSpace(LangAS::opencl_generic));
3302 
3303     llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
3304     llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
3305     LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
3306     llvm::Value *Range = NDRangeL.getAddress().getPointer();
3307     llvm::Type *RangeTy = NDRangeL.getAddress().getType();
3308 
3309     if (NumArgs == 4) {
3310       // The most basic form of the call with parameters:
3311       // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
3312       Name = "__enqueue_kernel_basic";
3313       llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
3314                               GenericVoidPtrTy};
3315       llvm::FunctionType *FTy = llvm::FunctionType::get(
3316           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3317 
3318       auto Info =
3319           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
3320       llvm::Value *Kernel =
3321           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3322       llvm::Value *Block =
3323           Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3324 
3325       AttrBuilder B;
3326       B.addAttribute(Attribute::ByVal);
3327       llvm::AttributeList ByValAttrSet =
3328           llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
3329 
3330       auto RTCall =
3331           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
3332                              {Queue, Flags, Range, Kernel, Block});
3333       RTCall->setAttributes(ByValAttrSet);
3334       return RValue::get(RTCall);
3335     }
3336     assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
3337 
3338     // Create a temporary array to hold the sizes of local pointer arguments
3339     // for the block. \p First is the position of the first size argument.
3340     auto CreateArrayForSizeVar = [=](unsigned First) {
3341       auto *AT = llvm::ArrayType::get(SizeTy, NumArgs - First);
3342       auto *Arr = Builder.CreateAlloca(AT);
3343       llvm::Value *Ptr;
3344       // Each of the following arguments specifies the size of the corresponding
3345       // argument passed to the enqueued block.
3346       auto *Zero = llvm::ConstantInt::get(IntTy, 0);
3347       for (unsigned I = First; I < NumArgs; ++I) {
3348         auto *Index = llvm::ConstantInt::get(IntTy, I - First);
3349         auto *GEP = Builder.CreateGEP(Arr, {Zero, Index});
3350         if (I == First)
3351           Ptr = GEP;
3352         auto *V =
3353             Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
3354         Builder.CreateAlignedStore(
3355             V, GEP, CGM.getDataLayout().getPrefTypeAlignment(SizeTy));
3356       }
3357       return Ptr;
3358     };
3359 
3360     // Could have events and/or varargs.
3361     if (E->getArg(3)->getType()->isBlockPointerType()) {
3362       // No events passed, but has variadic arguments.
3363       Name = "__enqueue_kernel_varargs";
3364       auto Info =
3365           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
3366       llvm::Value *Kernel =
3367           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3368       auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3369       auto *PtrToSizeArray = CreateArrayForSizeVar(4);
3370 
3371       // Create a vector of the arguments, as well as a constant value to
3372       // express to the runtime the number of variadic arguments.
3373       std::vector<llvm::Value *> Args = {
3374           Queue,  Flags, Range,
3375           Kernel, Block, ConstantInt::get(IntTy, NumArgs - 4),
3376           PtrToSizeArray};
3377       std::vector<llvm::Type *> ArgTys = {
3378           QueueTy,          IntTy,            RangeTy,
3379           GenericVoidPtrTy, GenericVoidPtrTy, IntTy,
3380           PtrToSizeArray->getType()};
3381 
3382       llvm::FunctionType *FTy = llvm::FunctionType::get(
3383           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3384       return RValue::get(
3385           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3386                              llvm::ArrayRef<llvm::Value *>(Args)));
3387     }
3388     // Any calls now have event arguments passed.
3389     if (NumArgs >= 7) {
3390       llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
3391       llvm::Type *EventPtrTy = EventTy->getPointerTo(
3392           CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
3393 
3394       llvm::Value *NumEvents =
3395           Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
3396       llvm::Value *EventList =
3397           E->getArg(4)->getType()->isArrayType()
3398               ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
3399               : EmitScalarExpr(E->getArg(4));
3400       llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
3401       // Convert to generic address space.
3402       EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
3403       ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy);
3404       auto Info =
3405           CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(6));
3406       llvm::Value *Kernel =
3407           Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3408       llvm::Value *Block =
3409           Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3410 
3411       std::vector<llvm::Type *> ArgTys = {
3412           QueueTy,    Int32Ty,    RangeTy,          Int32Ty,
3413           EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
3414 
3415       std::vector<llvm::Value *> Args = {Queue,     Flags,    Range,  NumEvents,
3416                                          EventList, ClkEvent, Kernel, Block};
3417 
3418       if (NumArgs == 7) {
3419         // Has events but no variadics.
3420         Name = "__enqueue_kernel_basic_events";
3421         llvm::FunctionType *FTy = llvm::FunctionType::get(
3422             Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3423         return RValue::get(
3424             Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3425                                llvm::ArrayRef<llvm::Value *>(Args)));
3426       }
3427       // Has event info and variadics
3428       // Pass the number of variadics to the runtime function too.
3429       Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
3430       ArgTys.push_back(Int32Ty);
3431       Name = "__enqueue_kernel_events_varargs";
3432 
3433       auto *PtrToSizeArray = CreateArrayForSizeVar(7);
3434       Args.push_back(PtrToSizeArray);
3435       ArgTys.push_back(PtrToSizeArray->getType());
3436 
3437       llvm::FunctionType *FTy = llvm::FunctionType::get(
3438           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
3439       return RValue::get(
3440           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3441                              llvm::ArrayRef<llvm::Value *>(Args)));
3442     }
3443     LLVM_FALLTHROUGH;
3444   }
3445   // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
3446   // parameter.
3447   case Builtin::BIget_kernel_work_group_size: {
3448     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3449         getContext().getTargetAddressSpace(LangAS::opencl_generic));
3450     auto Info =
3451         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
3452     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3453     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3454     return RValue::get(Builder.CreateCall(
3455         CGM.CreateRuntimeFunction(
3456             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3457                                     false),
3458             "__get_kernel_work_group_size_impl"),
3459         {Kernel, Arg}));
3460   }
3461   case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
3462     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3463         getContext().getTargetAddressSpace(LangAS::opencl_generic));
3464     auto Info =
3465         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
3466     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3467     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3468     return RValue::get(Builder.CreateCall(
3469         CGM.CreateRuntimeFunction(
3470             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3471                                     false),
3472             "__get_kernel_preferred_work_group_size_multiple_impl"),
3473         {Kernel, Arg}));
3474   }
3475   case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
3476   case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
3477     llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3478         getContext().getTargetAddressSpace(LangAS::opencl_generic));
3479     LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
3480     llvm::Value *NDRange = NDRangeL.getAddress().getPointer();
3481     auto Info =
3482         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
3483     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3484     Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3485     const char *Name =
3486         BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
3487             ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
3488             : "__get_kernel_sub_group_count_for_ndrange_impl";
3489     return RValue::get(Builder.CreateCall(
3490         CGM.CreateRuntimeFunction(
3491             llvm::FunctionType::get(
3492                 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
3493                 false),
3494             Name),
3495         {NDRange, Kernel, Block}));
3496   }
3497 
3498   case Builtin::BI__builtin_store_half:
3499   case Builtin::BI__builtin_store_halff: {
3500     Value *Val = EmitScalarExpr(E->getArg(0));
3501     Address Address = EmitPointerWithAlignment(E->getArg(1));
3502     Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
3503     return RValue::get(Builder.CreateStore(HalfVal, Address));
3504   }
3505   case Builtin::BI__builtin_load_half: {
3506     Address Address = EmitPointerWithAlignment(E->getArg(0));
3507     Value *HalfVal = Builder.CreateLoad(Address);
3508     return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
3509   }
3510   case Builtin::BI__builtin_load_halff: {
3511     Address Address = EmitPointerWithAlignment(E->getArg(0));
3512     Value *HalfVal = Builder.CreateLoad(Address);
3513     return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
3514   }
3515   case Builtin::BIprintf:
3516     if (getTarget().getTriple().isNVPTX())
3517       return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
3518     break;
3519   case Builtin::BI__builtin_canonicalize:
3520   case Builtin::BI__builtin_canonicalizef:
3521   case Builtin::BI__builtin_canonicalizel:
3522     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize));
3523 
3524   case Builtin::BI__builtin_thread_pointer: {
3525     if (!getContext().getTargetInfo().isTLSSupported())
3526       CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
3527     // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
3528     break;
3529   }
3530   case Builtin::BI__builtin_os_log_format:
3531     return emitBuiltinOSLogFormat(*E);
3532 
3533   case Builtin::BI__builtin_os_log_format_buffer_size: {
3534     analyze_os_log::OSLogBufferLayout Layout;
3535     analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout);
3536     return RValue::get(ConstantInt::get(ConvertType(E->getType()),
3537                                         Layout.size().getQuantity()));
3538   }
3539 
3540   case Builtin::BI__xray_customevent: {
3541     if (!ShouldXRayInstrumentFunction())
3542       return RValue::getIgnored();
3543 
3544     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3545             XRayInstrKind::Custom))
3546       return RValue::getIgnored();
3547 
3548     if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3549       if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
3550         return RValue::getIgnored();
3551 
3552     Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
3553     auto FTy = F->getFunctionType();
3554     auto Arg0 = E->getArg(0);
3555     auto Arg0Val = EmitScalarExpr(Arg0);
3556     auto Arg0Ty = Arg0->getType();
3557     auto PTy0 = FTy->getParamType(0);
3558     if (PTy0 != Arg0Val->getType()) {
3559       if (Arg0Ty->isArrayType())
3560         Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
3561       else
3562         Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
3563     }
3564     auto Arg1 = EmitScalarExpr(E->getArg(1));
3565     auto PTy1 = FTy->getParamType(1);
3566     if (PTy1 != Arg1->getType())
3567       Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
3568     return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
3569   }
3570 
3571   case Builtin::BI__xray_typedevent: {
3572     // TODO: There should be a way to always emit events even if the current
3573     // function is not instrumented. Losing events in a stream can cripple
3574     // a trace.
3575     if (!ShouldXRayInstrumentFunction())
3576       return RValue::getIgnored();
3577 
3578     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3579             XRayInstrKind::Typed))
3580       return RValue::getIgnored();
3581 
3582     if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3583       if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
3584         return RValue::getIgnored();
3585 
3586     Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
3587     auto FTy = F->getFunctionType();
3588     auto Arg0 = EmitScalarExpr(E->getArg(0));
3589     auto PTy0 = FTy->getParamType(0);
3590     if (PTy0 != Arg0->getType())
3591       Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
3592     auto Arg1 = E->getArg(1);
3593     auto Arg1Val = EmitScalarExpr(Arg1);
3594     auto Arg1Ty = Arg1->getType();
3595     auto PTy1 = FTy->getParamType(1);
3596     if (PTy1 != Arg1Val->getType()) {
3597       if (Arg1Ty->isArrayType())
3598         Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
3599       else
3600         Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
3601     }
3602     auto Arg2 = EmitScalarExpr(E->getArg(2));
3603     auto PTy2 = FTy->getParamType(2);
3604     if (PTy2 != Arg2->getType())
3605       Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
3606     return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
3607   }
3608 
3609   case Builtin::BI__builtin_ms_va_start:
3610   case Builtin::BI__builtin_ms_va_end:
3611     return RValue::get(
3612         EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
3613                        BuiltinID == Builtin::BI__builtin_ms_va_start));
3614 
3615   case Builtin::BI__builtin_ms_va_copy: {
3616     // Lower this manually. We can't reliably determine whether or not any
3617     // given va_copy() is for a Win64 va_list from the calling convention
3618     // alone, because it's legal to do this from a System V ABI function.
3619     // With opaque pointer types, we won't have enough information in LLVM
3620     // IR to determine this from the argument types, either. Best to do it
3621     // now, while we have enough information.
3622     Address DestAddr = EmitMSVAListRef(E->getArg(0));
3623     Address SrcAddr = EmitMSVAListRef(E->getArg(1));
3624 
3625     llvm::Type *BPP = Int8PtrPtrTy;
3626 
3627     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
3628                        DestAddr.getAlignment());
3629     SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
3630                       SrcAddr.getAlignment());
3631 
3632     Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
3633     return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
3634   }
3635   }
3636 
3637   // If this is an alias for a lib function (e.g. __builtin_sin), emit
3638   // the call using the normal call path, but using the unmangled
3639   // version of the function name.
3640   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
3641     return emitLibraryCall(*this, FD, E,
3642                            CGM.getBuiltinLibFunction(FD, BuiltinID));
3643 
3644   // If this is a predefined lib function (e.g. malloc), emit the call
3645   // using exactly the normal call path.
3646   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
3647     return emitLibraryCall(*this, FD, E,
3648                       cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
3649 
3650   // Check that a call to a target specific builtin has the correct target
3651   // features.
3652   // This is down here to avoid non-target specific builtins, however, if
3653   // generic builtins start to require generic target features then we
3654   // can move this up to the beginning of the function.
3655   checkTargetFeatures(E, FD);
3656 
3657   // See if we have a target specific intrinsic.
3658   const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
3659   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
3660   StringRef Prefix =
3661       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
3662   if (!Prefix.empty()) {
3663     IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
3664     // NOTE we don't need to perform a compatibility flag check here since the
3665     // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
3666     // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
3667     if (IntrinsicID == Intrinsic::not_intrinsic)
3668       IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
3669   }
3670 
3671   if (IntrinsicID != Intrinsic::not_intrinsic) {
3672     SmallVector<Value*, 16> Args;
3673 
3674     // Find out if any arguments are required to be integer constant
3675     // expressions.
3676     unsigned ICEArguments = 0;
3677     ASTContext::GetBuiltinTypeError Error;
3678     getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3679     assert(Error == ASTContext::GE_None && "Should not codegen an error");
3680 
3681     Function *F = CGM.getIntrinsic(IntrinsicID);
3682     llvm::FunctionType *FTy = F->getFunctionType();
3683 
3684     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
3685       Value *ArgValue;
3686       // If this is a normal argument, just emit it as a scalar.
3687       if ((ICEArguments & (1 << i)) == 0) {
3688         ArgValue = EmitScalarExpr(E->getArg(i));
3689       } else {
3690         // If this is required to be a constant, constant fold it so that we
3691         // know that the generated intrinsic gets a ConstantInt.
3692         llvm::APSInt Result;
3693         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
3694         assert(IsConst && "Constant arg isn't actually constant?");
3695         (void)IsConst;
3696         ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
3697       }
3698 
3699       // If the intrinsic arg type is different from the builtin arg type
3700       // we need to do a bit cast.
3701       llvm::Type *PTy = FTy->getParamType(i);
3702       if (PTy != ArgValue->getType()) {
3703         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
3704                "Must be able to losslessly bit cast to param");
3705         ArgValue = Builder.CreateBitCast(ArgValue, PTy);
3706       }
3707 
3708       Args.push_back(ArgValue);
3709     }
3710 
3711     Value *V = Builder.CreateCall(F, Args);
3712     QualType BuiltinRetType = E->getType();
3713 
3714     llvm::Type *RetTy = VoidTy;
3715     if (!BuiltinRetType->isVoidType())
3716       RetTy = ConvertType(BuiltinRetType);
3717 
3718     if (RetTy != V->getType()) {
3719       assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
3720              "Must be able to losslessly bit cast result type");
3721       V = Builder.CreateBitCast(V, RetTy);
3722     }
3723 
3724     return RValue::get(V);
3725   }
3726 
3727   // See if we have a target specific builtin that needs to be lowered.
3728   if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
3729     return RValue::get(V);
3730 
3731   ErrorUnsupported(E, "builtin function");
3732 
3733   // Unknown builtin, for now just dump it out and return undef.
3734   return GetUndefRValue(E->getType());
3735 }
3736 
3737 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
3738                                         unsigned BuiltinID, const CallExpr *E,
3739                                         llvm::Triple::ArchType Arch) {
3740   switch (Arch) {
3741   case llvm::Triple::arm:
3742   case llvm::Triple::armeb:
3743   case llvm::Triple::thumb:
3744   case llvm::Triple::thumbeb:
3745     return CGF->EmitARMBuiltinExpr(BuiltinID, E, Arch);
3746   case llvm::Triple::aarch64:
3747   case llvm::Triple::aarch64_be:
3748     return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
3749   case llvm::Triple::x86:
3750   case llvm::Triple::x86_64:
3751     return CGF->EmitX86BuiltinExpr(BuiltinID, E);
3752   case llvm::Triple::ppc:
3753   case llvm::Triple::ppc64:
3754   case llvm::Triple::ppc64le:
3755     return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
3756   case llvm::Triple::r600:
3757   case llvm::Triple::amdgcn:
3758     return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
3759   case llvm::Triple::systemz:
3760     return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
3761   case llvm::Triple::nvptx:
3762   case llvm::Triple::nvptx64:
3763     return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
3764   case llvm::Triple::wasm32:
3765   case llvm::Triple::wasm64:
3766     return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
3767   case llvm::Triple::hexagon:
3768     return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
3769   default:
3770     return nullptr;
3771   }
3772 }
3773 
3774 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
3775                                               const CallExpr *E) {
3776   if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
3777     assert(getContext().getAuxTargetInfo() && "Missing aux target info");
3778     return EmitTargetArchBuiltinExpr(
3779         this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
3780         getContext().getAuxTargetInfo()->getTriple().getArch());
3781   }
3782 
3783   return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
3784                                    getTarget().getTriple().getArch());
3785 }
3786 
3787 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
3788                                      NeonTypeFlags TypeFlags,
3789                                      bool HasLegalHalfType=true,
3790                                      bool V1Ty=false) {
3791   int IsQuad = TypeFlags.isQuad();
3792   switch (TypeFlags.getEltType()) {
3793   case NeonTypeFlags::Int8:
3794   case NeonTypeFlags::Poly8:
3795     return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
3796   case NeonTypeFlags::Int16:
3797   case NeonTypeFlags::Poly16:
3798     return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3799   case NeonTypeFlags::Float16:
3800     if (HasLegalHalfType)
3801       return llvm::VectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
3802     else
3803       return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3804   case NeonTypeFlags::Int32:
3805     return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
3806   case NeonTypeFlags::Int64:
3807   case NeonTypeFlags::Poly64:
3808     return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
3809   case NeonTypeFlags::Poly128:
3810     // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
3811     // There is a lot of i128 and f128 API missing.
3812     // so we use v16i8 to represent poly128 and get pattern matched.
3813     return llvm::VectorType::get(CGF->Int8Ty, 16);
3814   case NeonTypeFlags::Float32:
3815     return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
3816   case NeonTypeFlags::Float64:
3817     return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
3818   }
3819   llvm_unreachable("Unknown vector element type!");
3820 }
3821 
3822 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
3823                                           NeonTypeFlags IntTypeFlags) {
3824   int IsQuad = IntTypeFlags.isQuad();
3825   switch (IntTypeFlags.getEltType()) {
3826   case NeonTypeFlags::Int16:
3827     return llvm::VectorType::get(CGF->HalfTy, (4 << IsQuad));
3828   case NeonTypeFlags::Int32:
3829     return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
3830   case NeonTypeFlags::Int64:
3831     return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
3832   default:
3833     llvm_unreachable("Type can't be converted to floating-point!");
3834   }
3835 }
3836 
3837 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
3838   unsigned nElts = V->getType()->getVectorNumElements();
3839   Value* SV = llvm::ConstantVector::getSplat(nElts, C);
3840   return Builder.CreateShuffleVector(V, V, SV, "lane");
3841 }
3842 
3843 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
3844                                      const char *name,
3845                                      unsigned shift, bool rightshift) {
3846   unsigned j = 0;
3847   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3848        ai != ae; ++ai, ++j)
3849     if (shift > 0 && shift == j)
3850       Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
3851     else
3852       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
3853 
3854   return Builder.CreateCall(F, Ops, name);
3855 }
3856 
3857 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
3858                                             bool neg) {
3859   int SV = cast<ConstantInt>(V)->getSExtValue();
3860   return ConstantInt::get(Ty, neg ? -SV : SV);
3861 }
3862 
3863 // Right-shift a vector by a constant.
3864 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
3865                                           llvm::Type *Ty, bool usgn,
3866                                           const char *name) {
3867   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3868 
3869   int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
3870   int EltSize = VTy->getScalarSizeInBits();
3871 
3872   Vec = Builder.CreateBitCast(Vec, Ty);
3873 
3874   // lshr/ashr are undefined when the shift amount is equal to the vector
3875   // element size.
3876   if (ShiftAmt == EltSize) {
3877     if (usgn) {
3878       // Right-shifting an unsigned value by its size yields 0.
3879       return llvm::ConstantAggregateZero::get(VTy);
3880     } else {
3881       // Right-shifting a signed value by its size is equivalent
3882       // to a shift of size-1.
3883       --ShiftAmt;
3884       Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
3885     }
3886   }
3887 
3888   Shift = EmitNeonShiftVector(Shift, Ty, false);
3889   if (usgn)
3890     return Builder.CreateLShr(Vec, Shift, name);
3891   else
3892     return Builder.CreateAShr(Vec, Shift, name);
3893 }
3894 
3895 enum {
3896   AddRetType = (1 << 0),
3897   Add1ArgType = (1 << 1),
3898   Add2ArgTypes = (1 << 2),
3899 
3900   VectorizeRetType = (1 << 3),
3901   VectorizeArgTypes = (1 << 4),
3902 
3903   InventFloatType = (1 << 5),
3904   UnsignedAlts = (1 << 6),
3905 
3906   Use64BitVectors = (1 << 7),
3907   Use128BitVectors = (1 << 8),
3908 
3909   Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
3910   VectorRet = AddRetType | VectorizeRetType,
3911   VectorRetGetArgs01 =
3912       AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
3913   FpCmpzModifiers =
3914       AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
3915 };
3916 
3917 namespace {
3918 struct NeonIntrinsicInfo {
3919   const char *NameHint;
3920   unsigned BuiltinID;
3921   unsigned LLVMIntrinsic;
3922   unsigned AltLLVMIntrinsic;
3923   unsigned TypeModifier;
3924 
3925   bool operator<(unsigned RHSBuiltinID) const {
3926     return BuiltinID < RHSBuiltinID;
3927   }
3928   bool operator<(const NeonIntrinsicInfo &TE) const {
3929     return BuiltinID < TE.BuiltinID;
3930   }
3931 };
3932 } // end anonymous namespace
3933 
3934 #define NEONMAP0(NameBase) \
3935   { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
3936 
3937 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
3938   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3939       Intrinsic::LLVMIntrinsic, 0, TypeModifier }
3940 
3941 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
3942   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3943       Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
3944       TypeModifier }
3945 
3946 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
3947   NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3948   NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
3949   NEONMAP1(vabs_v, arm_neon_vabs, 0),
3950   NEONMAP1(vabsq_v, arm_neon_vabs, 0),
3951   NEONMAP0(vaddhn_v),
3952   NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3953   NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3954   NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3955   NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3956   NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
3957   NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
3958   NEONMAP1(vcage_v, arm_neon_vacge, 0),
3959   NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3960   NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3961   NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3962   NEONMAP1(vcale_v, arm_neon_vacge, 0),
3963   NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3964   NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3965   NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3966   NEONMAP0(vceqz_v),
3967   NEONMAP0(vceqzq_v),
3968   NEONMAP0(vcgez_v),
3969   NEONMAP0(vcgezq_v),
3970   NEONMAP0(vcgtz_v),
3971   NEONMAP0(vcgtzq_v),
3972   NEONMAP0(vclez_v),
3973   NEONMAP0(vclezq_v),
3974   NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
3975   NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
3976   NEONMAP0(vcltz_v),
3977   NEONMAP0(vcltzq_v),
3978   NEONMAP1(vclz_v, ctlz, Add1ArgType),
3979   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3980   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3981   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3982   NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3983   NEONMAP0(vcvt_f16_v),
3984   NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3985   NEONMAP0(vcvt_f32_v),
3986   NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3987   NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3988   NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
3989   NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3990   NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3991   NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
3992   NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3993   NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3994   NEONMAP0(vcvt_s16_v),
3995   NEONMAP0(vcvt_s32_v),
3996   NEONMAP0(vcvt_s64_v),
3997   NEONMAP0(vcvt_u16_v),
3998   NEONMAP0(vcvt_u32_v),
3999   NEONMAP0(vcvt_u64_v),
4000   NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
4001   NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
4002   NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
4003   NEONMAP1(vcvta_u16_v, arm_neon_vcvtau, 0),
4004   NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
4005   NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
4006   NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
4007   NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
4008   NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
4009   NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
4010   NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
4011   NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
4012   NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
4013   NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
4014   NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
4015   NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
4016   NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
4017   NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
4018   NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
4019   NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
4020   NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
4021   NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
4022   NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
4023   NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
4024   NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
4025   NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
4026   NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
4027   NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
4028   NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
4029   NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
4030   NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
4031   NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
4032   NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
4033   NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
4034   NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
4035   NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
4036   NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
4037   NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
4038   NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
4039   NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
4040   NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
4041   NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
4042   NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
4043   NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
4044   NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
4045   NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
4046   NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
4047   NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
4048   NEONMAP0(vcvtq_f16_v),
4049   NEONMAP0(vcvtq_f32_v),
4050   NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4051   NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4052   NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4053   NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4054   NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4055   NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4056   NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4057   NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4058   NEONMAP0(vcvtq_s16_v),
4059   NEONMAP0(vcvtq_s32_v),
4060   NEONMAP0(vcvtq_s64_v),
4061   NEONMAP0(vcvtq_u16_v),
4062   NEONMAP0(vcvtq_u32_v),
4063   NEONMAP0(vcvtq_u64_v),
4064   NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
4065   NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
4066   NEONMAP0(vext_v),
4067   NEONMAP0(vextq_v),
4068   NEONMAP0(vfma_v),
4069   NEONMAP0(vfmaq_v),
4070   NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
4071   NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
4072   NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
4073   NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
4074   NEONMAP0(vld1_dup_v),
4075   NEONMAP1(vld1_v, arm_neon_vld1, 0),
4076   NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
4077   NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
4078   NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
4079   NEONMAP0(vld1q_dup_v),
4080   NEONMAP1(vld1q_v, arm_neon_vld1, 0),
4081   NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
4082   NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
4083   NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
4084   NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
4085   NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
4086   NEONMAP1(vld2_v, arm_neon_vld2, 0),
4087   NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
4088   NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
4089   NEONMAP1(vld2q_v, arm_neon_vld2, 0),
4090   NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
4091   NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
4092   NEONMAP1(vld3_v, arm_neon_vld3, 0),
4093   NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
4094   NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
4095   NEONMAP1(vld3q_v, arm_neon_vld3, 0),
4096   NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
4097   NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
4098   NEONMAP1(vld4_v, arm_neon_vld4, 0),
4099   NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
4100   NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
4101   NEONMAP1(vld4q_v, arm_neon_vld4, 0),
4102   NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
4103   NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
4104   NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
4105   NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
4106   NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
4107   NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
4108   NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
4109   NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
4110   NEONMAP0(vmovl_v),
4111   NEONMAP0(vmovn_v),
4112   NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
4113   NEONMAP0(vmull_v),
4114   NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
4115   NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
4116   NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
4117   NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
4118   NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
4119   NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
4120   NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
4121   NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
4122   NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
4123   NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
4124   NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
4125   NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
4126   NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
4127   NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
4128   NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
4129   NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
4130   NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
4131   NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
4132   NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
4133   NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
4134   NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
4135   NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
4136   NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
4137   NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
4138   NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
4139   NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
4140   NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
4141   NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
4142   NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
4143   NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
4144   NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
4145   NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
4146   NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
4147   NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
4148   NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
4149   NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4150   NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4151   NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
4152   NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
4153   NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
4154   NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
4155   NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
4156   NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
4157   NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
4158   NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
4159   NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
4160   NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
4161   NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
4162   NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
4163   NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
4164   NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
4165   NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
4166   NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
4167   NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
4168   NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
4169   NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
4170   NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
4171   NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4172   NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4173   NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
4174   NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
4175   NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
4176   NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
4177   NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
4178   NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
4179   NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
4180   NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
4181   NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
4182   NEONMAP0(vshl_n_v),
4183   NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
4184   NEONMAP0(vshll_n_v),
4185   NEONMAP0(vshlq_n_v),
4186   NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
4187   NEONMAP0(vshr_n_v),
4188   NEONMAP0(vshrn_n_v),
4189   NEONMAP0(vshrq_n_v),
4190   NEONMAP1(vst1_v, arm_neon_vst1, 0),
4191   NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
4192   NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
4193   NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
4194   NEONMAP1(vst1q_v, arm_neon_vst1, 0),
4195   NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
4196   NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
4197   NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
4198   NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
4199   NEONMAP1(vst2_v, arm_neon_vst2, 0),
4200   NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
4201   NEONMAP1(vst2q_v, arm_neon_vst2, 0),
4202   NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
4203   NEONMAP1(vst3_v, arm_neon_vst3, 0),
4204   NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
4205   NEONMAP1(vst3q_v, arm_neon_vst3, 0),
4206   NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
4207   NEONMAP1(vst4_v, arm_neon_vst4, 0),
4208   NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
4209   NEONMAP1(vst4q_v, arm_neon_vst4, 0),
4210   NEONMAP0(vsubhn_v),
4211   NEONMAP0(vtrn_v),
4212   NEONMAP0(vtrnq_v),
4213   NEONMAP0(vtst_v),
4214   NEONMAP0(vtstq_v),
4215   NEONMAP0(vuzp_v),
4216   NEONMAP0(vuzpq_v),
4217   NEONMAP0(vzip_v),
4218   NEONMAP0(vzipq_v)
4219 };
4220 
4221 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
4222   NEONMAP1(vabs_v, aarch64_neon_abs, 0),
4223   NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
4224   NEONMAP0(vaddhn_v),
4225   NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
4226   NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
4227   NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
4228   NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
4229   NEONMAP1(vcage_v, aarch64_neon_facge, 0),
4230   NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
4231   NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
4232   NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
4233   NEONMAP1(vcale_v, aarch64_neon_facge, 0),
4234   NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
4235   NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
4236   NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
4237   NEONMAP0(vceqz_v),
4238   NEONMAP0(vceqzq_v),
4239   NEONMAP0(vcgez_v),
4240   NEONMAP0(vcgezq_v),
4241   NEONMAP0(vcgtz_v),
4242   NEONMAP0(vcgtzq_v),
4243   NEONMAP0(vclez_v),
4244   NEONMAP0(vclezq_v),
4245   NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
4246   NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
4247   NEONMAP0(vcltz_v),
4248   NEONMAP0(vcltzq_v),
4249   NEONMAP1(vclz_v, ctlz, Add1ArgType),
4250   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
4251   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
4252   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
4253   NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
4254   NEONMAP0(vcvt_f16_v),
4255   NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
4256   NEONMAP0(vcvt_f32_v),
4257   NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4258   NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4259   NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4260   NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4261   NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4262   NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4263   NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4264   NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4265   NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4266   NEONMAP0(vcvtq_f16_v),
4267   NEONMAP0(vcvtq_f32_v),
4268   NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4269   NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4270   NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4271   NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4272   NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4273   NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4274   NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4275   NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4276   NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4277   NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
4278   NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4279   NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4280   NEONMAP0(vext_v),
4281   NEONMAP0(vextq_v),
4282   NEONMAP0(vfma_v),
4283   NEONMAP0(vfmaq_v),
4284   NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
4285   NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
4286   NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
4287   NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
4288   NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
4289   NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
4290   NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
4291   NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
4292   NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
4293   NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
4294   NEONMAP0(vmovl_v),
4295   NEONMAP0(vmovn_v),
4296   NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
4297   NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
4298   NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
4299   NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
4300   NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
4301   NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
4302   NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
4303   NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
4304   NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
4305   NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
4306   NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
4307   NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
4308   NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
4309   NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
4310   NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
4311   NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
4312   NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
4313   NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
4314   NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
4315   NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
4316   NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
4317   NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
4318   NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
4319   NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
4320   NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
4321   NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
4322   NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
4323   NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
4324   NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
4325   NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
4326   NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
4327   NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
4328   NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4329   NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4330   NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
4331   NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
4332   NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
4333   NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
4334   NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
4335   NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
4336   NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
4337   NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
4338   NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4339   NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4340   NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
4341   NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
4342   NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
4343   NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
4344   NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
4345   NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
4346   NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
4347   NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
4348   NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
4349   NEONMAP0(vshl_n_v),
4350   NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
4351   NEONMAP0(vshll_n_v),
4352   NEONMAP0(vshlq_n_v),
4353   NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
4354   NEONMAP0(vshr_n_v),
4355   NEONMAP0(vshrn_n_v),
4356   NEONMAP0(vshrq_n_v),
4357   NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
4358   NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
4359   NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
4360   NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
4361   NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
4362   NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
4363   NEONMAP0(vsubhn_v),
4364   NEONMAP0(vtst_v),
4365   NEONMAP0(vtstq_v),
4366 };
4367 
4368 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
4369   NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
4370   NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
4371   NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
4372   NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
4373   NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
4374   NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
4375   NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
4376   NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
4377   NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
4378   NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4379   NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
4380   NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
4381   NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
4382   NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
4383   NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4384   NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4385   NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
4386   NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
4387   NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
4388   NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
4389   NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
4390   NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
4391   NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
4392   NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
4393   NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4394   NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4395   NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4396   NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4397   NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4398   NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4399   NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4400   NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4401   NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4402   NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4403   NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4404   NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4405   NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4406   NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4407   NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4408   NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4409   NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4410   NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4411   NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4412   NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4413   NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4414   NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4415   NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4416   NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4417   NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
4418   NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4419   NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4420   NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4421   NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4422   NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
4423   NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
4424   NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4425   NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4426   NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
4427   NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
4428   NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4429   NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4430   NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4431   NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4432   NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
4433   NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
4434   NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4435   NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
4436   NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
4437   NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
4438   NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
4439   NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
4440   NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
4441   NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4442   NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
4443   NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4444   NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
4445   NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4446   NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
4447   NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4448   NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
4449   NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
4450   NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
4451   NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
4452   NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
4453   NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
4454   NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
4455   NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
4456   NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
4457   NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
4458   NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
4459   NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
4460   NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
4461   NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
4462   NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
4463   NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
4464   NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
4465   NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
4466   NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
4467   NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
4468   NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
4469   NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
4470   NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
4471   NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
4472   NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
4473   NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
4474   NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
4475   NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
4476   NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
4477   NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
4478   NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
4479   NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
4480   NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
4481   NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
4482   NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
4483   NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
4484   NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
4485   NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
4486   NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
4487   NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
4488   NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
4489   NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
4490   NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
4491   NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
4492   NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
4493   NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
4494   NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
4495   NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
4496   NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
4497   NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
4498   NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
4499   NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4500   NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4501   NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4502   NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4503   NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
4504   NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
4505   NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4506   NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4507   NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
4508   NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
4509   NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
4510   NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
4511   NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
4512   NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
4513   NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
4514   NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
4515   NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
4516   NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
4517   NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
4518   NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
4519   NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
4520   NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
4521   NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
4522   NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
4523   NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
4524   NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
4525   NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
4526   NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
4527   NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
4528   NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
4529   NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
4530   NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
4531   NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
4532   NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
4533   NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
4534   NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
4535   NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
4536   NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
4537   NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
4538   NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
4539   NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
4540   NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
4541   NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
4542   NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
4543   NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
4544   NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
4545   NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
4546   NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
4547   NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
4548   NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
4549   NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
4550   NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
4551   NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
4552   NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
4553   NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
4554   NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
4555   NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
4556   NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
4557   NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
4558   NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
4559   NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
4560   NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
4561   // FP16 scalar intrinisics go here.
4562   NEONMAP1(vabdh_f16, aarch64_sisd_fabd, Add1ArgType),
4563   NEONMAP1(vcvtah_s32_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4564   NEONMAP1(vcvtah_s64_f16, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
4565   NEONMAP1(vcvtah_u32_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4566   NEONMAP1(vcvtah_u64_f16, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
4567   NEONMAP1(vcvth_n_f16_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4568   NEONMAP1(vcvth_n_f16_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
4569   NEONMAP1(vcvth_n_f16_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4570   NEONMAP1(vcvth_n_f16_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
4571   NEONMAP1(vcvth_n_s32_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4572   NEONMAP1(vcvth_n_s64_f16, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
4573   NEONMAP1(vcvth_n_u32_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4574   NEONMAP1(vcvth_n_u64_f16, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
4575   NEONMAP1(vcvtmh_s32_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4576   NEONMAP1(vcvtmh_s64_f16, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
4577   NEONMAP1(vcvtmh_u32_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4578   NEONMAP1(vcvtmh_u64_f16, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
4579   NEONMAP1(vcvtnh_s32_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4580   NEONMAP1(vcvtnh_s64_f16, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
4581   NEONMAP1(vcvtnh_u32_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4582   NEONMAP1(vcvtnh_u64_f16, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
4583   NEONMAP1(vcvtph_s32_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4584   NEONMAP1(vcvtph_s64_f16, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
4585   NEONMAP1(vcvtph_u32_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4586   NEONMAP1(vcvtph_u64_f16, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
4587   NEONMAP1(vmulxh_f16, aarch64_neon_fmulx, Add1ArgType),
4588   NEONMAP1(vrecpeh_f16, aarch64_neon_frecpe, Add1ArgType),
4589   NEONMAP1(vrecpxh_f16, aarch64_neon_frecpx, Add1ArgType),
4590   NEONMAP1(vrsqrteh_f16, aarch64_neon_frsqrte, Add1ArgType),
4591   NEONMAP1(vrsqrtsh_f16, aarch64_neon_frsqrts, Add1ArgType),
4592 };
4593 
4594 #undef NEONMAP0
4595 #undef NEONMAP1
4596 #undef NEONMAP2
4597 
4598 static bool NEONSIMDIntrinsicsProvenSorted = false;
4599 
4600 static bool AArch64SIMDIntrinsicsProvenSorted = false;
4601 static bool AArch64SISDIntrinsicsProvenSorted = false;
4602 
4603 
4604 static const NeonIntrinsicInfo *
4605 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap,
4606                        unsigned BuiltinID, bool &MapProvenSorted) {
4607 
4608 #ifndef NDEBUG
4609   if (!MapProvenSorted) {
4610     assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
4611     MapProvenSorted = true;
4612   }
4613 #endif
4614 
4615   const NeonIntrinsicInfo *Builtin =
4616       std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
4617 
4618   if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
4619     return Builtin;
4620 
4621   return nullptr;
4622 }
4623 
4624 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4625                                                    unsigned Modifier,
4626                                                    llvm::Type *ArgType,
4627                                                    const CallExpr *E) {
4628   int VectorSize = 0;
4629   if (Modifier & Use64BitVectors)
4630     VectorSize = 64;
4631   else if (Modifier & Use128BitVectors)
4632     VectorSize = 128;
4633 
4634   // Return type.
4635   SmallVector<llvm::Type *, 3> Tys;
4636   if (Modifier & AddRetType) {
4637     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
4638     if (Modifier & VectorizeRetType)
4639       Ty = llvm::VectorType::get(
4640           Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
4641 
4642     Tys.push_back(Ty);
4643   }
4644 
4645   // Arguments.
4646   if (Modifier & VectorizeArgTypes) {
4647     int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
4648     ArgType = llvm::VectorType::get(ArgType, Elts);
4649   }
4650 
4651   if (Modifier & (Add1ArgType | Add2ArgTypes))
4652     Tys.push_back(ArgType);
4653 
4654   if (Modifier & Add2ArgTypes)
4655     Tys.push_back(ArgType);
4656 
4657   if (Modifier & InventFloatType)
4658     Tys.push_back(FloatTy);
4659 
4660   return CGM.getIntrinsic(IntrinsicID, Tys);
4661 }
4662 
4663 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF,
4664                                             const NeonIntrinsicInfo &SISDInfo,
4665                                             SmallVectorImpl<Value *> &Ops,
4666                                             const CallExpr *E) {
4667   unsigned BuiltinID = SISDInfo.BuiltinID;
4668   unsigned int Int = SISDInfo.LLVMIntrinsic;
4669   unsigned Modifier = SISDInfo.TypeModifier;
4670   const char *s = SISDInfo.NameHint;
4671 
4672   switch (BuiltinID) {
4673   case NEON::BI__builtin_neon_vcled_s64:
4674   case NEON::BI__builtin_neon_vcled_u64:
4675   case NEON::BI__builtin_neon_vcles_f32:
4676   case NEON::BI__builtin_neon_vcled_f64:
4677   case NEON::BI__builtin_neon_vcltd_s64:
4678   case NEON::BI__builtin_neon_vcltd_u64:
4679   case NEON::BI__builtin_neon_vclts_f32:
4680   case NEON::BI__builtin_neon_vcltd_f64:
4681   case NEON::BI__builtin_neon_vcales_f32:
4682   case NEON::BI__builtin_neon_vcaled_f64:
4683   case NEON::BI__builtin_neon_vcalts_f32:
4684   case NEON::BI__builtin_neon_vcaltd_f64:
4685     // Only one direction of comparisons actually exist, cmle is actually a cmge
4686     // with swapped operands. The table gives us the right intrinsic but we
4687     // still need to do the swap.
4688     std::swap(Ops[0], Ops[1]);
4689     break;
4690   }
4691 
4692   assert(Int && "Generic code assumes a valid intrinsic");
4693 
4694   // Determine the type(s) of this overloaded AArch64 intrinsic.
4695   const Expr *Arg = E->getArg(0);
4696   llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
4697   Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
4698 
4699   int j = 0;
4700   ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
4701   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
4702        ai != ae; ++ai, ++j) {
4703     llvm::Type *ArgTy = ai->getType();
4704     if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
4705              ArgTy->getPrimitiveSizeInBits())
4706       continue;
4707 
4708     assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
4709     // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
4710     // it before inserting.
4711     Ops[j] =
4712         CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
4713     Ops[j] =
4714         CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
4715   }
4716 
4717   Value *Result = CGF.EmitNeonCall(F, Ops, s);
4718   llvm::Type *ResultType = CGF.ConvertType(E->getType());
4719   if (ResultType->getPrimitiveSizeInBits() <
4720       Result->getType()->getPrimitiveSizeInBits())
4721     return CGF.Builder.CreateExtractElement(Result, C0);
4722 
4723   return CGF.Builder.CreateBitCast(Result, ResultType, s);
4724 }
4725 
4726 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
4727     unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
4728     const char *NameHint, unsigned Modifier, const CallExpr *E,
4729     SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
4730     llvm::Triple::ArchType Arch) {
4731   // Get the last argument, which specifies the vector type.
4732   llvm::APSInt NeonTypeConst;
4733   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
4734   if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
4735     return nullptr;
4736 
4737   // Determine the type of this overloaded NEON intrinsic.
4738   NeonTypeFlags Type(NeonTypeConst.getZExtValue());
4739   bool Usgn = Type.isUnsigned();
4740   bool Quad = Type.isQuad();
4741   const bool HasLegalHalfType = getTarget().hasLegalHalfType();
4742 
4743   llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType);
4744   llvm::Type *Ty = VTy;
4745   if (!Ty)
4746     return nullptr;
4747 
4748   auto getAlignmentValue32 = [&](Address addr) -> Value* {
4749     return Builder.getInt32(addr.getAlignment().getQuantity());
4750   };
4751 
4752   unsigned Int = LLVMIntrinsic;
4753   if ((Modifier & UnsignedAlts) && !Usgn)
4754     Int = AltLLVMIntrinsic;
4755 
4756   switch (BuiltinID) {
4757   default: break;
4758   case NEON::BI__builtin_neon_vabs_v:
4759   case NEON::BI__builtin_neon_vabsq_v:
4760     if (VTy->getElementType()->isFloatingPointTy())
4761       return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
4762     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
4763   case NEON::BI__builtin_neon_vaddhn_v: {
4764     llvm::VectorType *SrcTy =
4765         llvm::VectorType::getExtendedElementVectorType(VTy);
4766 
4767     // %sum = add <4 x i32> %lhs, %rhs
4768     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4769     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4770     Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
4771 
4772     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
4773     Constant *ShiftAmt =
4774         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4775     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
4776 
4777     // %res = trunc <4 x i32> %high to <4 x i16>
4778     return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
4779   }
4780   case NEON::BI__builtin_neon_vcale_v:
4781   case NEON::BI__builtin_neon_vcaleq_v:
4782   case NEON::BI__builtin_neon_vcalt_v:
4783   case NEON::BI__builtin_neon_vcaltq_v:
4784     std::swap(Ops[0], Ops[1]);
4785     LLVM_FALLTHROUGH;
4786   case NEON::BI__builtin_neon_vcage_v:
4787   case NEON::BI__builtin_neon_vcageq_v:
4788   case NEON::BI__builtin_neon_vcagt_v:
4789   case NEON::BI__builtin_neon_vcagtq_v: {
4790     llvm::Type *Ty;
4791     switch (VTy->getScalarSizeInBits()) {
4792     default: llvm_unreachable("unexpected type");
4793     case 32:
4794       Ty = FloatTy;
4795       break;
4796     case 64:
4797       Ty = DoubleTy;
4798       break;
4799     case 16:
4800       Ty = HalfTy;
4801       break;
4802     }
4803     llvm::Type *VecFlt = llvm::VectorType::get(Ty, VTy->getNumElements());
4804     llvm::Type *Tys[] = { VTy, VecFlt };
4805     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4806     return EmitNeonCall(F, Ops, NameHint);
4807   }
4808   case NEON::BI__builtin_neon_vceqz_v:
4809   case NEON::BI__builtin_neon_vceqzq_v:
4810     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
4811                                          ICmpInst::ICMP_EQ, "vceqz");
4812   case NEON::BI__builtin_neon_vcgez_v:
4813   case NEON::BI__builtin_neon_vcgezq_v:
4814     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
4815                                          ICmpInst::ICMP_SGE, "vcgez");
4816   case NEON::BI__builtin_neon_vclez_v:
4817   case NEON::BI__builtin_neon_vclezq_v:
4818     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
4819                                          ICmpInst::ICMP_SLE, "vclez");
4820   case NEON::BI__builtin_neon_vcgtz_v:
4821   case NEON::BI__builtin_neon_vcgtzq_v:
4822     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
4823                                          ICmpInst::ICMP_SGT, "vcgtz");
4824   case NEON::BI__builtin_neon_vcltz_v:
4825   case NEON::BI__builtin_neon_vcltzq_v:
4826     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
4827                                          ICmpInst::ICMP_SLT, "vcltz");
4828   case NEON::BI__builtin_neon_vclz_v:
4829   case NEON::BI__builtin_neon_vclzq_v:
4830     // We generate target-independent intrinsic, which needs a second argument
4831     // for whether or not clz of zero is undefined; on ARM it isn't.
4832     Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
4833     break;
4834   case NEON::BI__builtin_neon_vcvt_f32_v:
4835   case NEON::BI__builtin_neon_vcvtq_f32_v:
4836     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4837     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
4838                      HasLegalHalfType);
4839     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4840                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4841   case NEON::BI__builtin_neon_vcvt_f16_v:
4842   case NEON::BI__builtin_neon_vcvtq_f16_v:
4843     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4844     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
4845                      HasLegalHalfType);
4846     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4847                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4848   case NEON::BI__builtin_neon_vcvt_n_f16_v:
4849   case NEON::BI__builtin_neon_vcvt_n_f32_v:
4850   case NEON::BI__builtin_neon_vcvt_n_f64_v:
4851   case NEON::BI__builtin_neon_vcvtq_n_f16_v:
4852   case NEON::BI__builtin_neon_vcvtq_n_f32_v:
4853   case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
4854     llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
4855     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
4856     Function *F = CGM.getIntrinsic(Int, Tys);
4857     return EmitNeonCall(F, Ops, "vcvt_n");
4858   }
4859   case NEON::BI__builtin_neon_vcvt_n_s16_v:
4860   case NEON::BI__builtin_neon_vcvt_n_s32_v:
4861   case NEON::BI__builtin_neon_vcvt_n_u16_v:
4862   case NEON::BI__builtin_neon_vcvt_n_u32_v:
4863   case NEON::BI__builtin_neon_vcvt_n_s64_v:
4864   case NEON::BI__builtin_neon_vcvt_n_u64_v:
4865   case NEON::BI__builtin_neon_vcvtq_n_s16_v:
4866   case NEON::BI__builtin_neon_vcvtq_n_s32_v:
4867   case NEON::BI__builtin_neon_vcvtq_n_u16_v:
4868   case NEON::BI__builtin_neon_vcvtq_n_u32_v:
4869   case NEON::BI__builtin_neon_vcvtq_n_s64_v:
4870   case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
4871     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4872     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4873     return EmitNeonCall(F, Ops, "vcvt_n");
4874   }
4875   case NEON::BI__builtin_neon_vcvt_s32_v:
4876   case NEON::BI__builtin_neon_vcvt_u32_v:
4877   case NEON::BI__builtin_neon_vcvt_s64_v:
4878   case NEON::BI__builtin_neon_vcvt_u64_v:
4879   case NEON::BI__builtin_neon_vcvt_s16_v:
4880   case NEON::BI__builtin_neon_vcvt_u16_v:
4881   case NEON::BI__builtin_neon_vcvtq_s32_v:
4882   case NEON::BI__builtin_neon_vcvtq_u32_v:
4883   case NEON::BI__builtin_neon_vcvtq_s64_v:
4884   case NEON::BI__builtin_neon_vcvtq_u64_v:
4885   case NEON::BI__builtin_neon_vcvtq_s16_v:
4886   case NEON::BI__builtin_neon_vcvtq_u16_v: {
4887     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
4888     return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
4889                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
4890   }
4891   case NEON::BI__builtin_neon_vcvta_s16_v:
4892   case NEON::BI__builtin_neon_vcvta_s32_v:
4893   case NEON::BI__builtin_neon_vcvta_s64_v:
4894   case NEON::BI__builtin_neon_vcvta_u16_v:
4895   case NEON::BI__builtin_neon_vcvta_u32_v:
4896   case NEON::BI__builtin_neon_vcvta_u64_v:
4897   case NEON::BI__builtin_neon_vcvtaq_s16_v:
4898   case NEON::BI__builtin_neon_vcvtaq_s32_v:
4899   case NEON::BI__builtin_neon_vcvtaq_s64_v:
4900   case NEON::BI__builtin_neon_vcvtaq_u16_v:
4901   case NEON::BI__builtin_neon_vcvtaq_u32_v:
4902   case NEON::BI__builtin_neon_vcvtaq_u64_v:
4903   case NEON::BI__builtin_neon_vcvtn_s16_v:
4904   case NEON::BI__builtin_neon_vcvtn_s32_v:
4905   case NEON::BI__builtin_neon_vcvtn_s64_v:
4906   case NEON::BI__builtin_neon_vcvtn_u16_v:
4907   case NEON::BI__builtin_neon_vcvtn_u32_v:
4908   case NEON::BI__builtin_neon_vcvtn_u64_v:
4909   case NEON::BI__builtin_neon_vcvtnq_s16_v:
4910   case NEON::BI__builtin_neon_vcvtnq_s32_v:
4911   case NEON::BI__builtin_neon_vcvtnq_s64_v:
4912   case NEON::BI__builtin_neon_vcvtnq_u16_v:
4913   case NEON::BI__builtin_neon_vcvtnq_u32_v:
4914   case NEON::BI__builtin_neon_vcvtnq_u64_v:
4915   case NEON::BI__builtin_neon_vcvtp_s16_v:
4916   case NEON::BI__builtin_neon_vcvtp_s32_v:
4917   case NEON::BI__builtin_neon_vcvtp_s64_v:
4918   case NEON::BI__builtin_neon_vcvtp_u16_v:
4919   case NEON::BI__builtin_neon_vcvtp_u32_v:
4920   case NEON::BI__builtin_neon_vcvtp_u64_v:
4921   case NEON::BI__builtin_neon_vcvtpq_s16_v:
4922   case NEON::BI__builtin_neon_vcvtpq_s32_v:
4923   case NEON::BI__builtin_neon_vcvtpq_s64_v:
4924   case NEON::BI__builtin_neon_vcvtpq_u16_v:
4925   case NEON::BI__builtin_neon_vcvtpq_u32_v:
4926   case NEON::BI__builtin_neon_vcvtpq_u64_v:
4927   case NEON::BI__builtin_neon_vcvtm_s16_v:
4928   case NEON::BI__builtin_neon_vcvtm_s32_v:
4929   case NEON::BI__builtin_neon_vcvtm_s64_v:
4930   case NEON::BI__builtin_neon_vcvtm_u16_v:
4931   case NEON::BI__builtin_neon_vcvtm_u32_v:
4932   case NEON::BI__builtin_neon_vcvtm_u64_v:
4933   case NEON::BI__builtin_neon_vcvtmq_s16_v:
4934   case NEON::BI__builtin_neon_vcvtmq_s32_v:
4935   case NEON::BI__builtin_neon_vcvtmq_s64_v:
4936   case NEON::BI__builtin_neon_vcvtmq_u16_v:
4937   case NEON::BI__builtin_neon_vcvtmq_u32_v:
4938   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
4939     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
4940     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
4941   }
4942   case NEON::BI__builtin_neon_vext_v:
4943   case NEON::BI__builtin_neon_vextq_v: {
4944     int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
4945     SmallVector<uint32_t, 16> Indices;
4946     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4947       Indices.push_back(i+CV);
4948 
4949     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4950     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4951     return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
4952   }
4953   case NEON::BI__builtin_neon_vfma_v:
4954   case NEON::BI__builtin_neon_vfmaq_v: {
4955     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4956     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4957     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4958     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4959 
4960     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
4961     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
4962   }
4963   case NEON::BI__builtin_neon_vld1_v:
4964   case NEON::BI__builtin_neon_vld1q_v: {
4965     llvm::Type *Tys[] = {Ty, Int8PtrTy};
4966     Ops.push_back(getAlignmentValue32(PtrOp0));
4967     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
4968   }
4969   case NEON::BI__builtin_neon_vld1_x2_v:
4970   case NEON::BI__builtin_neon_vld1q_x2_v:
4971   case NEON::BI__builtin_neon_vld1_x3_v:
4972   case NEON::BI__builtin_neon_vld1q_x3_v:
4973   case NEON::BI__builtin_neon_vld1_x4_v:
4974   case NEON::BI__builtin_neon_vld1q_x4_v: {
4975     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
4976     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
4977     llvm::Type *Tys[2] = { VTy, PTy };
4978     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4979     Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
4980     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4981     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4982     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4983   }
4984   case NEON::BI__builtin_neon_vld2_v:
4985   case NEON::BI__builtin_neon_vld2q_v:
4986   case NEON::BI__builtin_neon_vld3_v:
4987   case NEON::BI__builtin_neon_vld3q_v:
4988   case NEON::BI__builtin_neon_vld4_v:
4989   case NEON::BI__builtin_neon_vld4q_v:
4990   case NEON::BI__builtin_neon_vld2_dup_v:
4991   case NEON::BI__builtin_neon_vld2q_dup_v:
4992   case NEON::BI__builtin_neon_vld3_dup_v:
4993   case NEON::BI__builtin_neon_vld3q_dup_v:
4994   case NEON::BI__builtin_neon_vld4_dup_v:
4995   case NEON::BI__builtin_neon_vld4q_dup_v: {
4996     llvm::Type *Tys[] = {Ty, Int8PtrTy};
4997     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4998     Value *Align = getAlignmentValue32(PtrOp1);
4999     Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
5000     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5001     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5002     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5003   }
5004   case NEON::BI__builtin_neon_vld1_dup_v:
5005   case NEON::BI__builtin_neon_vld1q_dup_v: {
5006     Value *V = UndefValue::get(Ty);
5007     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5008     PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
5009     LoadInst *Ld = Builder.CreateLoad(PtrOp0);
5010     llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5011     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
5012     return EmitNeonSplat(Ops[0], CI);
5013   }
5014   case NEON::BI__builtin_neon_vld2_lane_v:
5015   case NEON::BI__builtin_neon_vld2q_lane_v:
5016   case NEON::BI__builtin_neon_vld3_lane_v:
5017   case NEON::BI__builtin_neon_vld3q_lane_v:
5018   case NEON::BI__builtin_neon_vld4_lane_v:
5019   case NEON::BI__builtin_neon_vld4q_lane_v: {
5020     llvm::Type *Tys[] = {Ty, Int8PtrTy};
5021     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5022     for (unsigned I = 2; I < Ops.size() - 1; ++I)
5023       Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
5024     Ops.push_back(getAlignmentValue32(PtrOp1));
5025     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
5026     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5027     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5028     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5029   }
5030   case NEON::BI__builtin_neon_vmovl_v: {
5031     llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
5032     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
5033     if (Usgn)
5034       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
5035     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
5036   }
5037   case NEON::BI__builtin_neon_vmovn_v: {
5038     llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5039     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
5040     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
5041   }
5042   case NEON::BI__builtin_neon_vmull_v:
5043     // FIXME: the integer vmull operations could be emitted in terms of pure
5044     // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
5045     // hoisting the exts outside loops. Until global ISel comes along that can
5046     // see through such movement this leads to bad CodeGen. So we need an
5047     // intrinsic for now.
5048     Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
5049     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
5050     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
5051   case NEON::BI__builtin_neon_vpadal_v:
5052   case NEON::BI__builtin_neon_vpadalq_v: {
5053     // The source operand type has twice as many elements of half the size.
5054     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5055     llvm::Type *EltTy =
5056       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5057     llvm::Type *NarrowTy =
5058       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5059     llvm::Type *Tys[2] = { Ty, NarrowTy };
5060     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
5061   }
5062   case NEON::BI__builtin_neon_vpaddl_v:
5063   case NEON::BI__builtin_neon_vpaddlq_v: {
5064     // The source operand type has twice as many elements of half the size.
5065     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5066     llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5067     llvm::Type *NarrowTy =
5068       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5069     llvm::Type *Tys[2] = { Ty, NarrowTy };
5070     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
5071   }
5072   case NEON::BI__builtin_neon_vqdmlal_v:
5073   case NEON::BI__builtin_neon_vqdmlsl_v: {
5074     SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
5075     Ops[1] =
5076         EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
5077     Ops.resize(2);
5078     return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
5079   }
5080   case NEON::BI__builtin_neon_vqshl_n_v:
5081   case NEON::BI__builtin_neon_vqshlq_n_v:
5082     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
5083                         1, false);
5084   case NEON::BI__builtin_neon_vqshlu_n_v:
5085   case NEON::BI__builtin_neon_vqshluq_n_v:
5086     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
5087                         1, false);
5088   case NEON::BI__builtin_neon_vrecpe_v:
5089   case NEON::BI__builtin_neon_vrecpeq_v:
5090   case NEON::BI__builtin_neon_vrsqrte_v:
5091   case NEON::BI__builtin_neon_vrsqrteq_v:
5092     Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
5093     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
5094 
5095   case NEON::BI__builtin_neon_vrshr_n_v:
5096   case NEON::BI__builtin_neon_vrshrq_n_v:
5097     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
5098                         1, true);
5099   case NEON::BI__builtin_neon_vshl_n_v:
5100   case NEON::BI__builtin_neon_vshlq_n_v:
5101     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
5102     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
5103                              "vshl_n");
5104   case NEON::BI__builtin_neon_vshll_n_v: {
5105     llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
5106     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5107     if (Usgn)
5108       Ops[0] = Builder.CreateZExt(Ops[0], VTy);
5109     else
5110       Ops[0] = Builder.CreateSExt(Ops[0], VTy);
5111     Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
5112     return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
5113   }
5114   case NEON::BI__builtin_neon_vshrn_n_v: {
5115     llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5116     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5117     Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
5118     if (Usgn)
5119       Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
5120     else
5121       Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
5122     return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
5123   }
5124   case NEON::BI__builtin_neon_vshr_n_v:
5125   case NEON::BI__builtin_neon_vshrq_n_v:
5126     return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
5127   case NEON::BI__builtin_neon_vst1_v:
5128   case NEON::BI__builtin_neon_vst1q_v:
5129   case NEON::BI__builtin_neon_vst2_v:
5130   case NEON::BI__builtin_neon_vst2q_v:
5131   case NEON::BI__builtin_neon_vst3_v:
5132   case NEON::BI__builtin_neon_vst3q_v:
5133   case NEON::BI__builtin_neon_vst4_v:
5134   case NEON::BI__builtin_neon_vst4q_v:
5135   case NEON::BI__builtin_neon_vst2_lane_v:
5136   case NEON::BI__builtin_neon_vst2q_lane_v:
5137   case NEON::BI__builtin_neon_vst3_lane_v:
5138   case NEON::BI__builtin_neon_vst3q_lane_v:
5139   case NEON::BI__builtin_neon_vst4_lane_v:
5140   case NEON::BI__builtin_neon_vst4q_lane_v: {
5141     llvm::Type *Tys[] = {Int8PtrTy, Ty};
5142     Ops.push_back(getAlignmentValue32(PtrOp0));
5143     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
5144   }
5145   case NEON::BI__builtin_neon_vst1_x2_v:
5146   case NEON::BI__builtin_neon_vst1q_x2_v:
5147   case NEON::BI__builtin_neon_vst1_x3_v:
5148   case NEON::BI__builtin_neon_vst1q_x3_v:
5149   case NEON::BI__builtin_neon_vst1_x4_v:
5150   case NEON::BI__builtin_neon_vst1q_x4_v: {
5151     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5152     // TODO: Currently in AArch32 mode the pointer operand comes first, whereas
5153     // in AArch64 it comes last. We may want to stick to one or another.
5154     if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be) {
5155       llvm::Type *Tys[2] = { VTy, PTy };
5156       std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
5157       return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
5158     }
5159     llvm::Type *Tys[2] = { PTy, VTy };
5160     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "");
5161   }
5162   case NEON::BI__builtin_neon_vsubhn_v: {
5163     llvm::VectorType *SrcTy =
5164         llvm::VectorType::getExtendedElementVectorType(VTy);
5165 
5166     // %sum = add <4 x i32> %lhs, %rhs
5167     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5168     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
5169     Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
5170 
5171     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
5172     Constant *ShiftAmt =
5173         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
5174     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
5175 
5176     // %res = trunc <4 x i32> %high to <4 x i16>
5177     return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
5178   }
5179   case NEON::BI__builtin_neon_vtrn_v:
5180   case NEON::BI__builtin_neon_vtrnq_v: {
5181     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5182     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5183     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5184     Value *SV = nullptr;
5185 
5186     for (unsigned vi = 0; vi != 2; ++vi) {
5187       SmallVector<uint32_t, 16> Indices;
5188       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5189         Indices.push_back(i+vi);
5190         Indices.push_back(i+e+vi);
5191       }
5192       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5193       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
5194       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5195     }
5196     return SV;
5197   }
5198   case NEON::BI__builtin_neon_vtst_v:
5199   case NEON::BI__builtin_neon_vtstq_v: {
5200     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5201     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5202     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5203     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5204                                 ConstantAggregateZero::get(Ty));
5205     return Builder.CreateSExt(Ops[0], Ty, "vtst");
5206   }
5207   case NEON::BI__builtin_neon_vuzp_v:
5208   case NEON::BI__builtin_neon_vuzpq_v: {
5209     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5210     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5211     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5212     Value *SV = nullptr;
5213 
5214     for (unsigned vi = 0; vi != 2; ++vi) {
5215       SmallVector<uint32_t, 16> Indices;
5216       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5217         Indices.push_back(2*i+vi);
5218 
5219       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5220       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
5221       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5222     }
5223     return SV;
5224   }
5225   case NEON::BI__builtin_neon_vzip_v:
5226   case NEON::BI__builtin_neon_vzipq_v: {
5227     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5228     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5229     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5230     Value *SV = nullptr;
5231 
5232     for (unsigned vi = 0; vi != 2; ++vi) {
5233       SmallVector<uint32_t, 16> Indices;
5234       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5235         Indices.push_back((i + vi*e) >> 1);
5236         Indices.push_back(((i + vi*e) >> 1)+e);
5237       }
5238       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5239       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
5240       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5241     }
5242     return SV;
5243   }
5244   case NEON::BI__builtin_neon_vdot_v:
5245   case NEON::BI__builtin_neon_vdotq_v: {
5246     llvm::Type *InputTy =
5247         llvm::VectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
5248     llvm::Type *Tys[2] = { Ty, InputTy };
5249     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
5250     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot");
5251   }
5252   }
5253 
5254   assert(Int && "Expected valid intrinsic number");
5255 
5256   // Determine the type(s) of this overloaded AArch64 intrinsic.
5257   Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
5258 
5259   Value *Result = EmitNeonCall(F, Ops, NameHint);
5260   llvm::Type *ResultType = ConvertType(E->getType());
5261   // AArch64 intrinsic one-element vector type cast to
5262   // scalar type expected by the builtin
5263   return Builder.CreateBitCast(Result, ResultType, NameHint);
5264 }
5265 
5266 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
5267     Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
5268     const CmpInst::Predicate Ip, const Twine &Name) {
5269   llvm::Type *OTy = Op->getType();
5270 
5271   // FIXME: this is utterly horrific. We should not be looking at previous
5272   // codegen context to find out what needs doing. Unfortunately TableGen
5273   // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
5274   // (etc).
5275   if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
5276     OTy = BI->getOperand(0)->getType();
5277 
5278   Op = Builder.CreateBitCast(Op, OTy);
5279   if (OTy->getScalarType()->isFloatingPointTy()) {
5280     Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
5281   } else {
5282     Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
5283   }
5284   return Builder.CreateSExt(Op, Ty, Name);
5285 }
5286 
5287 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
5288                                  Value *ExtOp, Value *IndexOp,
5289                                  llvm::Type *ResTy, unsigned IntID,
5290                                  const char *Name) {
5291   SmallVector<Value *, 2> TblOps;
5292   if (ExtOp)
5293     TblOps.push_back(ExtOp);
5294 
5295   // Build a vector containing sequential number like (0, 1, 2, ..., 15)
5296   SmallVector<uint32_t, 16> Indices;
5297   llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
5298   for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
5299     Indices.push_back(2*i);
5300     Indices.push_back(2*i+1);
5301   }
5302 
5303   int PairPos = 0, End = Ops.size() - 1;
5304   while (PairPos < End) {
5305     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
5306                                                      Ops[PairPos+1], Indices,
5307                                                      Name));
5308     PairPos += 2;
5309   }
5310 
5311   // If there's an odd number of 64-bit lookup table, fill the high 64-bit
5312   // of the 128-bit lookup table with zero.
5313   if (PairPos == End) {
5314     Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
5315     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
5316                                                      ZeroTbl, Indices, Name));
5317   }
5318 
5319   Function *TblF;
5320   TblOps.push_back(IndexOp);
5321   TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
5322 
5323   return CGF.EmitNeonCall(TblF, TblOps, Name);
5324 }
5325 
5326 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
5327   unsigned Value;
5328   switch (BuiltinID) {
5329   default:
5330     return nullptr;
5331   case ARM::BI__builtin_arm_nop:
5332     Value = 0;
5333     break;
5334   case ARM::BI__builtin_arm_yield:
5335   case ARM::BI__yield:
5336     Value = 1;
5337     break;
5338   case ARM::BI__builtin_arm_wfe:
5339   case ARM::BI__wfe:
5340     Value = 2;
5341     break;
5342   case ARM::BI__builtin_arm_wfi:
5343   case ARM::BI__wfi:
5344     Value = 3;
5345     break;
5346   case ARM::BI__builtin_arm_sev:
5347   case ARM::BI__sev:
5348     Value = 4;
5349     break;
5350   case ARM::BI__builtin_arm_sevl:
5351   case ARM::BI__sevl:
5352     Value = 5;
5353     break;
5354   }
5355 
5356   return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
5357                             llvm::ConstantInt::get(Int32Ty, Value));
5358 }
5359 
5360 // Generates the IR for the read/write special register builtin,
5361 // ValueType is the type of the value that is to be written or read,
5362 // RegisterType is the type of the register being written to or read from.
5363 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
5364                                          const CallExpr *E,
5365                                          llvm::Type *RegisterType,
5366                                          llvm::Type *ValueType,
5367                                          bool IsRead,
5368                                          StringRef SysReg = "") {
5369   // write and register intrinsics only support 32 and 64 bit operations.
5370   assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
5371           && "Unsupported size for register.");
5372 
5373   CodeGen::CGBuilderTy &Builder = CGF.Builder;
5374   CodeGen::CodeGenModule &CGM = CGF.CGM;
5375   LLVMContext &Context = CGM.getLLVMContext();
5376 
5377   if (SysReg.empty()) {
5378     const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
5379     SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
5380   }
5381 
5382   llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
5383   llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5384   llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5385 
5386   llvm::Type *Types[] = { RegisterType };
5387 
5388   bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
5389   assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
5390             && "Can't fit 64-bit value in 32-bit register");
5391 
5392   if (IsRead) {
5393     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
5394     llvm::Value *Call = Builder.CreateCall(F, Metadata);
5395 
5396     if (MixedTypes)
5397       // Read into 64 bit register and then truncate result to 32 bit.
5398       return Builder.CreateTrunc(Call, ValueType);
5399 
5400     if (ValueType->isPointerTy())
5401       // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
5402       return Builder.CreateIntToPtr(Call, ValueType);
5403 
5404     return Call;
5405   }
5406 
5407   llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
5408   llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
5409   if (MixedTypes) {
5410     // Extend 32 bit write value to 64 bit to pass to write.
5411     ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
5412     return Builder.CreateCall(F, { Metadata, ArgValue });
5413   }
5414 
5415   if (ValueType->isPointerTy()) {
5416     // Have VoidPtrTy ArgValue but want to return an i32/i64.
5417     ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
5418     return Builder.CreateCall(F, { Metadata, ArgValue });
5419   }
5420 
5421   return Builder.CreateCall(F, { Metadata, ArgValue });
5422 }
5423 
5424 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
5425 /// argument that specifies the vector type.
5426 static bool HasExtraNeonArgument(unsigned BuiltinID) {
5427   switch (BuiltinID) {
5428   default: break;
5429   case NEON::BI__builtin_neon_vget_lane_i8:
5430   case NEON::BI__builtin_neon_vget_lane_i16:
5431   case NEON::BI__builtin_neon_vget_lane_i32:
5432   case NEON::BI__builtin_neon_vget_lane_i64:
5433   case NEON::BI__builtin_neon_vget_lane_f32:
5434   case NEON::BI__builtin_neon_vgetq_lane_i8:
5435   case NEON::BI__builtin_neon_vgetq_lane_i16:
5436   case NEON::BI__builtin_neon_vgetq_lane_i32:
5437   case NEON::BI__builtin_neon_vgetq_lane_i64:
5438   case NEON::BI__builtin_neon_vgetq_lane_f32:
5439   case NEON::BI__builtin_neon_vset_lane_i8:
5440   case NEON::BI__builtin_neon_vset_lane_i16:
5441   case NEON::BI__builtin_neon_vset_lane_i32:
5442   case NEON::BI__builtin_neon_vset_lane_i64:
5443   case NEON::BI__builtin_neon_vset_lane_f32:
5444   case NEON::BI__builtin_neon_vsetq_lane_i8:
5445   case NEON::BI__builtin_neon_vsetq_lane_i16:
5446   case NEON::BI__builtin_neon_vsetq_lane_i32:
5447   case NEON::BI__builtin_neon_vsetq_lane_i64:
5448   case NEON::BI__builtin_neon_vsetq_lane_f32:
5449   case NEON::BI__builtin_neon_vsha1h_u32:
5450   case NEON::BI__builtin_neon_vsha1cq_u32:
5451   case NEON::BI__builtin_neon_vsha1pq_u32:
5452   case NEON::BI__builtin_neon_vsha1mq_u32:
5453   case clang::ARM::BI_MoveToCoprocessor:
5454   case clang::ARM::BI_MoveToCoprocessor2:
5455     return false;
5456   }
5457   return true;
5458 }
5459 
5460 Value *CodeGenFunction::EmitISOVolatileLoad(const CallExpr *E) {
5461   Value *Ptr = EmitScalarExpr(E->getArg(0));
5462   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
5463   CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy);
5464   llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5465                                            LoadSize.getQuantity() * 8);
5466   Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5467   llvm::LoadInst *Load =
5468     Builder.CreateAlignedLoad(Ptr, LoadSize);
5469   Load->setVolatile(true);
5470   return Load;
5471 }
5472 
5473 Value *CodeGenFunction::EmitISOVolatileStore(const CallExpr *E) {
5474   Value *Ptr = EmitScalarExpr(E->getArg(0));
5475   Value *Value = EmitScalarExpr(E->getArg(1));
5476   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
5477   CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
5478   llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5479                                            StoreSize.getQuantity() * 8);
5480   Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5481   llvm::StoreInst *Store =
5482     Builder.CreateAlignedStore(Value, Ptr,
5483                                StoreSize);
5484   Store->setVolatile(true);
5485   return Store;
5486 }
5487 
5488 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
5489                                            const CallExpr *E,
5490                                            llvm::Triple::ArchType Arch) {
5491   if (auto Hint = GetValueForARMHint(BuiltinID))
5492     return Hint;
5493 
5494   if (BuiltinID == ARM::BI__emit) {
5495     bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
5496     llvm::FunctionType *FTy =
5497         llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
5498 
5499     APSInt Value;
5500     if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext()))
5501       llvm_unreachable("Sema will ensure that the parameter is constant");
5502 
5503     uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
5504 
5505     llvm::InlineAsm *Emit =
5506         IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
5507                                  /*SideEffects=*/true)
5508                 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
5509                                  /*SideEffects=*/true);
5510 
5511     return Builder.CreateCall(Emit);
5512   }
5513 
5514   if (BuiltinID == ARM::BI__builtin_arm_dbg) {
5515     Value *Option = EmitScalarExpr(E->getArg(0));
5516     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
5517   }
5518 
5519   if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
5520     Value *Address = EmitScalarExpr(E->getArg(0));
5521     Value *RW      = EmitScalarExpr(E->getArg(1));
5522     Value *IsData  = EmitScalarExpr(E->getArg(2));
5523 
5524     // Locality is not supported on ARM target
5525     Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
5526 
5527     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
5528     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5529   }
5530 
5531   if (BuiltinID == ARM::BI__builtin_arm_rbit) {
5532     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5533     return Builder.CreateCall(
5534         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
5535   }
5536 
5537   if (BuiltinID == ARM::BI__clear_cache) {
5538     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
5539     const FunctionDecl *FD = E->getDirectCallee();
5540     Value *Ops[2];
5541     for (unsigned i = 0; i < 2; i++)
5542       Ops[i] = EmitScalarExpr(E->getArg(i));
5543     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5544     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5545     StringRef Name = FD->getName();
5546     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
5547   }
5548 
5549   if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
5550       BuiltinID == ARM::BI__builtin_arm_mcrr2) {
5551     Function *F;
5552 
5553     switch (BuiltinID) {
5554     default: llvm_unreachable("unexpected builtin");
5555     case ARM::BI__builtin_arm_mcrr:
5556       F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
5557       break;
5558     case ARM::BI__builtin_arm_mcrr2:
5559       F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
5560       break;
5561     }
5562 
5563     // MCRR{2} instruction has 5 operands but
5564     // the intrinsic has 4 because Rt and Rt2
5565     // are represented as a single unsigned 64
5566     // bit integer in the intrinsic definition
5567     // but internally it's represented as 2 32
5568     // bit integers.
5569 
5570     Value *Coproc = EmitScalarExpr(E->getArg(0));
5571     Value *Opc1 = EmitScalarExpr(E->getArg(1));
5572     Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
5573     Value *CRm = EmitScalarExpr(E->getArg(3));
5574 
5575     Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
5576     Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
5577     Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
5578     Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
5579 
5580     return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
5581   }
5582 
5583   if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
5584       BuiltinID == ARM::BI__builtin_arm_mrrc2) {
5585     Function *F;
5586 
5587     switch (BuiltinID) {
5588     default: llvm_unreachable("unexpected builtin");
5589     case ARM::BI__builtin_arm_mrrc:
5590       F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
5591       break;
5592     case ARM::BI__builtin_arm_mrrc2:
5593       F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
5594       break;
5595     }
5596 
5597     Value *Coproc = EmitScalarExpr(E->getArg(0));
5598     Value *Opc1 = EmitScalarExpr(E->getArg(1));
5599     Value *CRm  = EmitScalarExpr(E->getArg(2));
5600     Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
5601 
5602     // Returns an unsigned 64 bit integer, represented
5603     // as two 32 bit integers.
5604 
5605     Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
5606     Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
5607     Rt = Builder.CreateZExt(Rt, Int64Ty);
5608     Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
5609 
5610     Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
5611     RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
5612     RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
5613 
5614     return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
5615   }
5616 
5617   if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
5618       ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
5619         BuiltinID == ARM::BI__builtin_arm_ldaex) &&
5620        getContext().getTypeSize(E->getType()) == 64) ||
5621       BuiltinID == ARM::BI__ldrexd) {
5622     Function *F;
5623 
5624     switch (BuiltinID) {
5625     default: llvm_unreachable("unexpected builtin");
5626     case ARM::BI__builtin_arm_ldaex:
5627       F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
5628       break;
5629     case ARM::BI__builtin_arm_ldrexd:
5630     case ARM::BI__builtin_arm_ldrex:
5631     case ARM::BI__ldrexd:
5632       F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
5633       break;
5634     }
5635 
5636     Value *LdPtr = EmitScalarExpr(E->getArg(0));
5637     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
5638                                     "ldrexd");
5639 
5640     Value *Val0 = Builder.CreateExtractValue(Val, 1);
5641     Value *Val1 = Builder.CreateExtractValue(Val, 0);
5642     Val0 = Builder.CreateZExt(Val0, Int64Ty);
5643     Val1 = Builder.CreateZExt(Val1, Int64Ty);
5644 
5645     Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
5646     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
5647     Val = Builder.CreateOr(Val, Val1);
5648     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
5649   }
5650 
5651   if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
5652       BuiltinID == ARM::BI__builtin_arm_ldaex) {
5653     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
5654 
5655     QualType Ty = E->getType();
5656     llvm::Type *RealResTy = ConvertType(Ty);
5657     llvm::Type *PtrTy = llvm::IntegerType::get(
5658         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
5659     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
5660 
5661     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
5662                                        ? Intrinsic::arm_ldaex
5663                                        : Intrinsic::arm_ldrex,
5664                                    PtrTy);
5665     Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
5666 
5667     if (RealResTy->isPointerTy())
5668       return Builder.CreateIntToPtr(Val, RealResTy);
5669     else {
5670       llvm::Type *IntResTy = llvm::IntegerType::get(
5671           getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
5672       Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
5673       return Builder.CreateBitCast(Val, RealResTy);
5674     }
5675   }
5676 
5677   if (BuiltinID == ARM::BI__builtin_arm_strexd ||
5678       ((BuiltinID == ARM::BI__builtin_arm_stlex ||
5679         BuiltinID == ARM::BI__builtin_arm_strex) &&
5680        getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
5681     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
5682                                        ? Intrinsic::arm_stlexd
5683                                        : Intrinsic::arm_strexd);
5684     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
5685 
5686     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
5687     Value *Val = EmitScalarExpr(E->getArg(0));
5688     Builder.CreateStore(Val, Tmp);
5689 
5690     Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
5691     Val = Builder.CreateLoad(LdPtr);
5692 
5693     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
5694     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
5695     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
5696     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
5697   }
5698 
5699   if (BuiltinID == ARM::BI__builtin_arm_strex ||
5700       BuiltinID == ARM::BI__builtin_arm_stlex) {
5701     Value *StoreVal = EmitScalarExpr(E->getArg(0));
5702     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
5703 
5704     QualType Ty = E->getArg(0)->getType();
5705     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
5706                                                  getContext().getTypeSize(Ty));
5707     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
5708 
5709     if (StoreVal->getType()->isPointerTy())
5710       StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
5711     else {
5712       llvm::Type *IntTy = llvm::IntegerType::get(
5713           getLLVMContext(),
5714           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
5715       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
5716       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
5717     }
5718 
5719     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
5720                                        ? Intrinsic::arm_stlex
5721                                        : Intrinsic::arm_strex,
5722                                    StoreAddr->getType());
5723     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
5724   }
5725 
5726   switch (BuiltinID) {
5727   case ARM::BI__iso_volatile_load8:
5728   case ARM::BI__iso_volatile_load16:
5729   case ARM::BI__iso_volatile_load32:
5730   case ARM::BI__iso_volatile_load64:
5731     return EmitISOVolatileLoad(E);
5732   case ARM::BI__iso_volatile_store8:
5733   case ARM::BI__iso_volatile_store16:
5734   case ARM::BI__iso_volatile_store32:
5735   case ARM::BI__iso_volatile_store64:
5736     return EmitISOVolatileStore(E);
5737   }
5738 
5739   if (BuiltinID == ARM::BI__builtin_arm_clrex) {
5740     Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
5741     return Builder.CreateCall(F);
5742   }
5743 
5744   // CRC32
5745   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
5746   switch (BuiltinID) {
5747   case ARM::BI__builtin_arm_crc32b:
5748     CRCIntrinsicID = Intrinsic::arm_crc32b; break;
5749   case ARM::BI__builtin_arm_crc32cb:
5750     CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
5751   case ARM::BI__builtin_arm_crc32h:
5752     CRCIntrinsicID = Intrinsic::arm_crc32h; break;
5753   case ARM::BI__builtin_arm_crc32ch:
5754     CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
5755   case ARM::BI__builtin_arm_crc32w:
5756   case ARM::BI__builtin_arm_crc32d:
5757     CRCIntrinsicID = Intrinsic::arm_crc32w; break;
5758   case ARM::BI__builtin_arm_crc32cw:
5759   case ARM::BI__builtin_arm_crc32cd:
5760     CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
5761   }
5762 
5763   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5764     Value *Arg0 = EmitScalarExpr(E->getArg(0));
5765     Value *Arg1 = EmitScalarExpr(E->getArg(1));
5766 
5767     // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
5768     // intrinsics, hence we need different codegen for these cases.
5769     if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
5770         BuiltinID == ARM::BI__builtin_arm_crc32cd) {
5771       Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
5772       Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
5773       Value *Arg1b = Builder.CreateLShr(Arg1, C1);
5774       Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
5775 
5776       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5777       Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
5778       return Builder.CreateCall(F, {Res, Arg1b});
5779     } else {
5780       Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
5781 
5782       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5783       return Builder.CreateCall(F, {Arg0, Arg1});
5784     }
5785   }
5786 
5787   if (BuiltinID == ARM::BI__builtin_arm_rsr ||
5788       BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5789       BuiltinID == ARM::BI__builtin_arm_rsrp ||
5790       BuiltinID == ARM::BI__builtin_arm_wsr ||
5791       BuiltinID == ARM::BI__builtin_arm_wsr64 ||
5792       BuiltinID == ARM::BI__builtin_arm_wsrp) {
5793 
5794     bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
5795                   BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5796                   BuiltinID == ARM::BI__builtin_arm_rsrp;
5797 
5798     bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
5799                             BuiltinID == ARM::BI__builtin_arm_wsrp;
5800 
5801     bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5802                    BuiltinID == ARM::BI__builtin_arm_wsr64;
5803 
5804     llvm::Type *ValueType;
5805     llvm::Type *RegisterType;
5806     if (IsPointerBuiltin) {
5807       ValueType = VoidPtrTy;
5808       RegisterType = Int32Ty;
5809     } else if (Is64Bit) {
5810       ValueType = RegisterType = Int64Ty;
5811     } else {
5812       ValueType = RegisterType = Int32Ty;
5813     }
5814 
5815     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
5816   }
5817 
5818   // Find out if any arguments are required to be integer constant
5819   // expressions.
5820   unsigned ICEArguments = 0;
5821   ASTContext::GetBuiltinTypeError Error;
5822   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5823   assert(Error == ASTContext::GE_None && "Should not codegen an error");
5824 
5825   auto getAlignmentValue32 = [&](Address addr) -> Value* {
5826     return Builder.getInt32(addr.getAlignment().getQuantity());
5827   };
5828 
5829   Address PtrOp0 = Address::invalid();
5830   Address PtrOp1 = Address::invalid();
5831   SmallVector<Value*, 4> Ops;
5832   bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
5833   unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
5834   for (unsigned i = 0, e = NumArgs; i != e; i++) {
5835     if (i == 0) {
5836       switch (BuiltinID) {
5837       case NEON::BI__builtin_neon_vld1_v:
5838       case NEON::BI__builtin_neon_vld1q_v:
5839       case NEON::BI__builtin_neon_vld1q_lane_v:
5840       case NEON::BI__builtin_neon_vld1_lane_v:
5841       case NEON::BI__builtin_neon_vld1_dup_v:
5842       case NEON::BI__builtin_neon_vld1q_dup_v:
5843       case NEON::BI__builtin_neon_vst1_v:
5844       case NEON::BI__builtin_neon_vst1q_v:
5845       case NEON::BI__builtin_neon_vst1q_lane_v:
5846       case NEON::BI__builtin_neon_vst1_lane_v:
5847       case NEON::BI__builtin_neon_vst2_v:
5848       case NEON::BI__builtin_neon_vst2q_v:
5849       case NEON::BI__builtin_neon_vst2_lane_v:
5850       case NEON::BI__builtin_neon_vst2q_lane_v:
5851       case NEON::BI__builtin_neon_vst3_v:
5852       case NEON::BI__builtin_neon_vst3q_v:
5853       case NEON::BI__builtin_neon_vst3_lane_v:
5854       case NEON::BI__builtin_neon_vst3q_lane_v:
5855       case NEON::BI__builtin_neon_vst4_v:
5856       case NEON::BI__builtin_neon_vst4q_v:
5857       case NEON::BI__builtin_neon_vst4_lane_v:
5858       case NEON::BI__builtin_neon_vst4q_lane_v:
5859         // Get the alignment for the argument in addition to the value;
5860         // we'll use it later.
5861         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
5862         Ops.push_back(PtrOp0.getPointer());
5863         continue;
5864       }
5865     }
5866     if (i == 1) {
5867       switch (BuiltinID) {
5868       case NEON::BI__builtin_neon_vld2_v:
5869       case NEON::BI__builtin_neon_vld2q_v:
5870       case NEON::BI__builtin_neon_vld3_v:
5871       case NEON::BI__builtin_neon_vld3q_v:
5872       case NEON::BI__builtin_neon_vld4_v:
5873       case NEON::BI__builtin_neon_vld4q_v:
5874       case NEON::BI__builtin_neon_vld2_lane_v:
5875       case NEON::BI__builtin_neon_vld2q_lane_v:
5876       case NEON::BI__builtin_neon_vld3_lane_v:
5877       case NEON::BI__builtin_neon_vld3q_lane_v:
5878       case NEON::BI__builtin_neon_vld4_lane_v:
5879       case NEON::BI__builtin_neon_vld4q_lane_v:
5880       case NEON::BI__builtin_neon_vld2_dup_v:
5881       case NEON::BI__builtin_neon_vld2q_dup_v:
5882       case NEON::BI__builtin_neon_vld3_dup_v:
5883       case NEON::BI__builtin_neon_vld3q_dup_v:
5884       case NEON::BI__builtin_neon_vld4_dup_v:
5885       case NEON::BI__builtin_neon_vld4q_dup_v:
5886         // Get the alignment for the argument in addition to the value;
5887         // we'll use it later.
5888         PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
5889         Ops.push_back(PtrOp1.getPointer());
5890         continue;
5891       }
5892     }
5893 
5894     if ((ICEArguments & (1 << i)) == 0) {
5895       Ops.push_back(EmitScalarExpr(E->getArg(i)));
5896     } else {
5897       // If this is required to be a constant, constant fold it so that we know
5898       // that the generated intrinsic gets a ConstantInt.
5899       llvm::APSInt Result;
5900       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
5901       assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
5902       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5903     }
5904   }
5905 
5906   switch (BuiltinID) {
5907   default: break;
5908 
5909   case NEON::BI__builtin_neon_vget_lane_i8:
5910   case NEON::BI__builtin_neon_vget_lane_i16:
5911   case NEON::BI__builtin_neon_vget_lane_i32:
5912   case NEON::BI__builtin_neon_vget_lane_i64:
5913   case NEON::BI__builtin_neon_vget_lane_f32:
5914   case NEON::BI__builtin_neon_vgetq_lane_i8:
5915   case NEON::BI__builtin_neon_vgetq_lane_i16:
5916   case NEON::BI__builtin_neon_vgetq_lane_i32:
5917   case NEON::BI__builtin_neon_vgetq_lane_i64:
5918   case NEON::BI__builtin_neon_vgetq_lane_f32:
5919     return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
5920 
5921   case NEON::BI__builtin_neon_vrndns_f32: {
5922     Value *Arg = EmitScalarExpr(E->getArg(0));
5923     llvm::Type *Tys[] = {Arg->getType()};
5924     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
5925     return Builder.CreateCall(F, {Arg}, "vrndn"); }
5926 
5927   case NEON::BI__builtin_neon_vset_lane_i8:
5928   case NEON::BI__builtin_neon_vset_lane_i16:
5929   case NEON::BI__builtin_neon_vset_lane_i32:
5930   case NEON::BI__builtin_neon_vset_lane_i64:
5931   case NEON::BI__builtin_neon_vset_lane_f32:
5932   case NEON::BI__builtin_neon_vsetq_lane_i8:
5933   case NEON::BI__builtin_neon_vsetq_lane_i16:
5934   case NEON::BI__builtin_neon_vsetq_lane_i32:
5935   case NEON::BI__builtin_neon_vsetq_lane_i64:
5936   case NEON::BI__builtin_neon_vsetq_lane_f32:
5937     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5938 
5939   case NEON::BI__builtin_neon_vsha1h_u32:
5940     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
5941                         "vsha1h");
5942   case NEON::BI__builtin_neon_vsha1cq_u32:
5943     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
5944                         "vsha1h");
5945   case NEON::BI__builtin_neon_vsha1pq_u32:
5946     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
5947                         "vsha1h");
5948   case NEON::BI__builtin_neon_vsha1mq_u32:
5949     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
5950                         "vsha1h");
5951 
5952   // The ARM _MoveToCoprocessor builtins put the input register value as
5953   // the first argument, but the LLVM intrinsic expects it as the third one.
5954   case ARM::BI_MoveToCoprocessor:
5955   case ARM::BI_MoveToCoprocessor2: {
5956     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
5957                                    Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
5958     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
5959                                   Ops[3], Ops[4], Ops[5]});
5960   }
5961   case ARM::BI_BitScanForward:
5962   case ARM::BI_BitScanForward64:
5963     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
5964   case ARM::BI_BitScanReverse:
5965   case ARM::BI_BitScanReverse64:
5966     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
5967 
5968   case ARM::BI_InterlockedAnd64:
5969     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
5970   case ARM::BI_InterlockedExchange64:
5971     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
5972   case ARM::BI_InterlockedExchangeAdd64:
5973     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
5974   case ARM::BI_InterlockedExchangeSub64:
5975     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
5976   case ARM::BI_InterlockedOr64:
5977     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
5978   case ARM::BI_InterlockedXor64:
5979     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
5980   case ARM::BI_InterlockedDecrement64:
5981     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
5982   case ARM::BI_InterlockedIncrement64:
5983     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
5984   }
5985 
5986   // Get the last argument, which specifies the vector type.
5987   assert(HasExtraArg);
5988   llvm::APSInt Result;
5989   const Expr *Arg = E->getArg(E->getNumArgs()-1);
5990   if (!Arg->isIntegerConstantExpr(Result, getContext()))
5991     return nullptr;
5992 
5993   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
5994       BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
5995     // Determine the overloaded type of this builtin.
5996     llvm::Type *Ty;
5997     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
5998       Ty = FloatTy;
5999     else
6000       Ty = DoubleTy;
6001 
6002     // Determine whether this is an unsigned conversion or not.
6003     bool usgn = Result.getZExtValue() == 1;
6004     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
6005 
6006     // Call the appropriate intrinsic.
6007     Function *F = CGM.getIntrinsic(Int, Ty);
6008     return Builder.CreateCall(F, Ops, "vcvtr");
6009   }
6010 
6011   // Determine the type of this overloaded NEON intrinsic.
6012   NeonTypeFlags Type(Result.getZExtValue());
6013   bool usgn = Type.isUnsigned();
6014   bool rightShift = false;
6015 
6016   llvm::VectorType *VTy = GetNeonType(this, Type,
6017                                       getTarget().hasLegalHalfType());
6018   llvm::Type *Ty = VTy;
6019   if (!Ty)
6020     return nullptr;
6021 
6022   // Many NEON builtins have identical semantics and uses in ARM and
6023   // AArch64. Emit these in a single function.
6024   auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
6025   const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
6026       IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
6027   if (Builtin)
6028     return EmitCommonNeonBuiltinExpr(
6029         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
6030         Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
6031 
6032   unsigned Int;
6033   switch (BuiltinID) {
6034   default: return nullptr;
6035   case NEON::BI__builtin_neon_vld1q_lane_v:
6036     // Handle 64-bit integer elements as a special case.  Use shuffles of
6037     // one-element vectors to avoid poor code for i64 in the backend.
6038     if (VTy->getElementType()->isIntegerTy(64)) {
6039       // Extract the other lane.
6040       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6041       uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
6042       Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
6043       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6044       // Load the value as a one-element vector.
6045       Ty = llvm::VectorType::get(VTy->getElementType(), 1);
6046       llvm::Type *Tys[] = {Ty, Int8PtrTy};
6047       Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
6048       Value *Align = getAlignmentValue32(PtrOp0);
6049       Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
6050       // Combine them.
6051       uint32_t Indices[] = {1 - Lane, Lane};
6052       SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
6053       return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
6054     }
6055     LLVM_FALLTHROUGH;
6056   case NEON::BI__builtin_neon_vld1_lane_v: {
6057     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6058     PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
6059     Value *Ld = Builder.CreateLoad(PtrOp0);
6060     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
6061   }
6062   case NEON::BI__builtin_neon_vqrshrn_n_v:
6063     Int =
6064       usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
6065     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
6066                         1, true);
6067   case NEON::BI__builtin_neon_vqrshrun_n_v:
6068     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
6069                         Ops, "vqrshrun_n", 1, true);
6070   case NEON::BI__builtin_neon_vqshrn_n_v:
6071     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
6072     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
6073                         1, true);
6074   case NEON::BI__builtin_neon_vqshrun_n_v:
6075     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
6076                         Ops, "vqshrun_n", 1, true);
6077   case NEON::BI__builtin_neon_vrecpe_v:
6078   case NEON::BI__builtin_neon_vrecpeq_v:
6079     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
6080                         Ops, "vrecpe");
6081   case NEON::BI__builtin_neon_vrshrn_n_v:
6082     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
6083                         Ops, "vrshrn_n", 1, true);
6084   case NEON::BI__builtin_neon_vrsra_n_v:
6085   case NEON::BI__builtin_neon_vrsraq_n_v:
6086     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6087     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6088     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
6089     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
6090     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
6091     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
6092   case NEON::BI__builtin_neon_vsri_n_v:
6093   case NEON::BI__builtin_neon_vsriq_n_v:
6094     rightShift = true;
6095     LLVM_FALLTHROUGH;
6096   case NEON::BI__builtin_neon_vsli_n_v:
6097   case NEON::BI__builtin_neon_vsliq_n_v:
6098     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
6099     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
6100                         Ops, "vsli_n");
6101   case NEON::BI__builtin_neon_vsra_n_v:
6102   case NEON::BI__builtin_neon_vsraq_n_v:
6103     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6104     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
6105     return Builder.CreateAdd(Ops[0], Ops[1]);
6106   case NEON::BI__builtin_neon_vst1q_lane_v:
6107     // Handle 64-bit integer elements as a special case.  Use a shuffle to get
6108     // a one-element vector and avoid poor code for i64 in the backend.
6109     if (VTy->getElementType()->isIntegerTy(64)) {
6110       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6111       Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
6112       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6113       Ops[2] = getAlignmentValue32(PtrOp0);
6114       llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
6115       return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
6116                                                  Tys), Ops);
6117     }
6118     LLVM_FALLTHROUGH;
6119   case NEON::BI__builtin_neon_vst1_lane_v: {
6120     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6121     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6122     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6123     auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
6124     return St;
6125   }
6126   case NEON::BI__builtin_neon_vtbl1_v:
6127     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
6128                         Ops, "vtbl1");
6129   case NEON::BI__builtin_neon_vtbl2_v:
6130     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
6131                         Ops, "vtbl2");
6132   case NEON::BI__builtin_neon_vtbl3_v:
6133     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
6134                         Ops, "vtbl3");
6135   case NEON::BI__builtin_neon_vtbl4_v:
6136     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
6137                         Ops, "vtbl4");
6138   case NEON::BI__builtin_neon_vtbx1_v:
6139     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
6140                         Ops, "vtbx1");
6141   case NEON::BI__builtin_neon_vtbx2_v:
6142     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
6143                         Ops, "vtbx2");
6144   case NEON::BI__builtin_neon_vtbx3_v:
6145     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
6146                         Ops, "vtbx3");
6147   case NEON::BI__builtin_neon_vtbx4_v:
6148     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
6149                         Ops, "vtbx4");
6150   }
6151 }
6152 
6153 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
6154                                       const CallExpr *E,
6155                                       SmallVectorImpl<Value *> &Ops,
6156                                       llvm::Triple::ArchType Arch) {
6157   unsigned int Int = 0;
6158   const char *s = nullptr;
6159 
6160   switch (BuiltinID) {
6161   default:
6162     return nullptr;
6163   case NEON::BI__builtin_neon_vtbl1_v:
6164   case NEON::BI__builtin_neon_vqtbl1_v:
6165   case NEON::BI__builtin_neon_vqtbl1q_v:
6166   case NEON::BI__builtin_neon_vtbl2_v:
6167   case NEON::BI__builtin_neon_vqtbl2_v:
6168   case NEON::BI__builtin_neon_vqtbl2q_v:
6169   case NEON::BI__builtin_neon_vtbl3_v:
6170   case NEON::BI__builtin_neon_vqtbl3_v:
6171   case NEON::BI__builtin_neon_vqtbl3q_v:
6172   case NEON::BI__builtin_neon_vtbl4_v:
6173   case NEON::BI__builtin_neon_vqtbl4_v:
6174   case NEON::BI__builtin_neon_vqtbl4q_v:
6175     break;
6176   case NEON::BI__builtin_neon_vtbx1_v:
6177   case NEON::BI__builtin_neon_vqtbx1_v:
6178   case NEON::BI__builtin_neon_vqtbx1q_v:
6179   case NEON::BI__builtin_neon_vtbx2_v:
6180   case NEON::BI__builtin_neon_vqtbx2_v:
6181   case NEON::BI__builtin_neon_vqtbx2q_v:
6182   case NEON::BI__builtin_neon_vtbx3_v:
6183   case NEON::BI__builtin_neon_vqtbx3_v:
6184   case NEON::BI__builtin_neon_vqtbx3q_v:
6185   case NEON::BI__builtin_neon_vtbx4_v:
6186   case NEON::BI__builtin_neon_vqtbx4_v:
6187   case NEON::BI__builtin_neon_vqtbx4q_v:
6188     break;
6189   }
6190 
6191   assert(E->getNumArgs() >= 3);
6192 
6193   // Get the last argument, which specifies the vector type.
6194   llvm::APSInt Result;
6195   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
6196   if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
6197     return nullptr;
6198 
6199   // Determine the type of this overloaded NEON intrinsic.
6200   NeonTypeFlags Type(Result.getZExtValue());
6201   llvm::VectorType *Ty = GetNeonType(&CGF, Type);
6202   if (!Ty)
6203     return nullptr;
6204 
6205   CodeGen::CGBuilderTy &Builder = CGF.Builder;
6206 
6207   // AArch64 scalar builtins are not overloaded, they do not have an extra
6208   // argument that specifies the vector type, need to handle each case.
6209   switch (BuiltinID) {
6210   case NEON::BI__builtin_neon_vtbl1_v: {
6211     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr,
6212                               Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
6213                               "vtbl1");
6214   }
6215   case NEON::BI__builtin_neon_vtbl2_v: {
6216     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr,
6217                               Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
6218                               "vtbl1");
6219   }
6220   case NEON::BI__builtin_neon_vtbl3_v: {
6221     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr,
6222                               Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
6223                               "vtbl2");
6224   }
6225   case NEON::BI__builtin_neon_vtbl4_v: {
6226     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr,
6227                               Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
6228                               "vtbl2");
6229   }
6230   case NEON::BI__builtin_neon_vtbx1_v: {
6231     Value *TblRes =
6232         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2],
6233                            Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
6234 
6235     llvm::Constant *EightV = ConstantInt::get(Ty, 8);
6236     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
6237     CmpRes = Builder.CreateSExt(CmpRes, Ty);
6238 
6239     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6240     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6241     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
6242   }
6243   case NEON::BI__builtin_neon_vtbx2_v: {
6244     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0],
6245                               Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
6246                               "vtbx1");
6247   }
6248   case NEON::BI__builtin_neon_vtbx3_v: {
6249     Value *TblRes =
6250         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4],
6251                            Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
6252 
6253     llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
6254     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
6255                                            TwentyFourV);
6256     CmpRes = Builder.CreateSExt(CmpRes, Ty);
6257 
6258     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6259     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6260     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
6261   }
6262   case NEON::BI__builtin_neon_vtbx4_v: {
6263     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0],
6264                               Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
6265                               "vtbx2");
6266   }
6267   case NEON::BI__builtin_neon_vqtbl1_v:
6268   case NEON::BI__builtin_neon_vqtbl1q_v:
6269     Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
6270   case NEON::BI__builtin_neon_vqtbl2_v:
6271   case NEON::BI__builtin_neon_vqtbl2q_v: {
6272     Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
6273   case NEON::BI__builtin_neon_vqtbl3_v:
6274   case NEON::BI__builtin_neon_vqtbl3q_v:
6275     Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
6276   case NEON::BI__builtin_neon_vqtbl4_v:
6277   case NEON::BI__builtin_neon_vqtbl4q_v:
6278     Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
6279   case NEON::BI__builtin_neon_vqtbx1_v:
6280   case NEON::BI__builtin_neon_vqtbx1q_v:
6281     Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
6282   case NEON::BI__builtin_neon_vqtbx2_v:
6283   case NEON::BI__builtin_neon_vqtbx2q_v:
6284     Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
6285   case NEON::BI__builtin_neon_vqtbx3_v:
6286   case NEON::BI__builtin_neon_vqtbx3q_v:
6287     Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
6288   case NEON::BI__builtin_neon_vqtbx4_v:
6289   case NEON::BI__builtin_neon_vqtbx4q_v:
6290     Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
6291   }
6292   }
6293 
6294   if (!Int)
6295     return nullptr;
6296 
6297   Function *F = CGF.CGM.getIntrinsic(Int, Ty);
6298   return CGF.EmitNeonCall(F, Ops, s);
6299 }
6300 
6301 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
6302   llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
6303   Op = Builder.CreateBitCast(Op, Int16Ty);
6304   Value *V = UndefValue::get(VTy);
6305   llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
6306   Op = Builder.CreateInsertElement(V, Op, CI);
6307   return Op;
6308 }
6309 
6310 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
6311                                                const CallExpr *E,
6312                                                llvm::Triple::ArchType Arch) {
6313   unsigned HintID = static_cast<unsigned>(-1);
6314   switch (BuiltinID) {
6315   default: break;
6316   case AArch64::BI__builtin_arm_nop:
6317     HintID = 0;
6318     break;
6319   case AArch64::BI__builtin_arm_yield:
6320   case AArch64::BI__yield:
6321     HintID = 1;
6322     break;
6323   case AArch64::BI__builtin_arm_wfe:
6324   case AArch64::BI__wfe:
6325     HintID = 2;
6326     break;
6327   case AArch64::BI__builtin_arm_wfi:
6328   case AArch64::BI__wfi:
6329     HintID = 3;
6330     break;
6331   case AArch64::BI__builtin_arm_sev:
6332   case AArch64::BI__sev:
6333     HintID = 4;
6334     break;
6335   case AArch64::BI__builtin_arm_sevl:
6336   case AArch64::BI__sevl:
6337     HintID = 5;
6338     break;
6339   }
6340 
6341   if (HintID != static_cast<unsigned>(-1)) {
6342     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
6343     return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
6344   }
6345 
6346   if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
6347     Value *Address         = EmitScalarExpr(E->getArg(0));
6348     Value *RW              = EmitScalarExpr(E->getArg(1));
6349     Value *CacheLevel      = EmitScalarExpr(E->getArg(2));
6350     Value *RetentionPolicy = EmitScalarExpr(E->getArg(3));
6351     Value *IsData          = EmitScalarExpr(E->getArg(4));
6352 
6353     Value *Locality = nullptr;
6354     if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
6355       // Temporal fetch, needs to convert cache level to locality.
6356       Locality = llvm::ConstantInt::get(Int32Ty,
6357         -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3);
6358     } else {
6359       // Streaming fetch.
6360       Locality = llvm::ConstantInt::get(Int32Ty, 0);
6361     }
6362 
6363     // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
6364     // PLDL3STRM or PLDL2STRM.
6365     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
6366     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
6367   }
6368 
6369   if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
6370     assert((getContext().getTypeSize(E->getType()) == 32) &&
6371            "rbit of unusual size!");
6372     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
6373     return Builder.CreateCall(
6374         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
6375   }
6376   if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
6377     assert((getContext().getTypeSize(E->getType()) == 64) &&
6378            "rbit of unusual size!");
6379     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
6380     return Builder.CreateCall(
6381         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
6382   }
6383 
6384   if (BuiltinID == AArch64::BI__clear_cache) {
6385     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
6386     const FunctionDecl *FD = E->getDirectCallee();
6387     Value *Ops[2];
6388     for (unsigned i = 0; i < 2; i++)
6389       Ops[i] = EmitScalarExpr(E->getArg(i));
6390     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
6391     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
6392     StringRef Name = FD->getName();
6393     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
6394   }
6395 
6396   if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6397       BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
6398       getContext().getTypeSize(E->getType()) == 128) {
6399     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6400                                        ? Intrinsic::aarch64_ldaxp
6401                                        : Intrinsic::aarch64_ldxp);
6402 
6403     Value *LdPtr = EmitScalarExpr(E->getArg(0));
6404     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
6405                                     "ldxp");
6406 
6407     Value *Val0 = Builder.CreateExtractValue(Val, 1);
6408     Value *Val1 = Builder.CreateExtractValue(Val, 0);
6409     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
6410     Val0 = Builder.CreateZExt(Val0, Int128Ty);
6411     Val1 = Builder.CreateZExt(Val1, Int128Ty);
6412 
6413     Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
6414     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
6415     Val = Builder.CreateOr(Val, Val1);
6416     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
6417   } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6418              BuiltinID == AArch64::BI__builtin_arm_ldaex) {
6419     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
6420 
6421     QualType Ty = E->getType();
6422     llvm::Type *RealResTy = ConvertType(Ty);
6423     llvm::Type *PtrTy = llvm::IntegerType::get(
6424         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
6425     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
6426 
6427     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6428                                        ? Intrinsic::aarch64_ldaxr
6429                                        : Intrinsic::aarch64_ldxr,
6430                                    PtrTy);
6431     Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
6432 
6433     if (RealResTy->isPointerTy())
6434       return Builder.CreateIntToPtr(Val, RealResTy);
6435 
6436     llvm::Type *IntResTy = llvm::IntegerType::get(
6437         getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
6438     Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
6439     return Builder.CreateBitCast(Val, RealResTy);
6440   }
6441 
6442   if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
6443        BuiltinID == AArch64::BI__builtin_arm_stlex) &&
6444       getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
6445     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6446                                        ? Intrinsic::aarch64_stlxp
6447                                        : Intrinsic::aarch64_stxp);
6448     llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
6449 
6450     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
6451     EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
6452 
6453     Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
6454     llvm::Value *Val = Builder.CreateLoad(Tmp);
6455 
6456     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
6457     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
6458     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
6459                                          Int8PtrTy);
6460     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
6461   }
6462 
6463   if (BuiltinID == AArch64::BI__builtin_arm_strex ||
6464       BuiltinID == AArch64::BI__builtin_arm_stlex) {
6465     Value *StoreVal = EmitScalarExpr(E->getArg(0));
6466     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
6467 
6468     QualType Ty = E->getArg(0)->getType();
6469     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
6470                                                  getContext().getTypeSize(Ty));
6471     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
6472 
6473     if (StoreVal->getType()->isPointerTy())
6474       StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
6475     else {
6476       llvm::Type *IntTy = llvm::IntegerType::get(
6477           getLLVMContext(),
6478           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
6479       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
6480       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
6481     }
6482 
6483     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6484                                        ? Intrinsic::aarch64_stlxr
6485                                        : Intrinsic::aarch64_stxr,
6486                                    StoreAddr->getType());
6487     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
6488   }
6489 
6490   if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
6491     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
6492     return Builder.CreateCall(F);
6493   }
6494 
6495   // CRC32
6496   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
6497   switch (BuiltinID) {
6498   case AArch64::BI__builtin_arm_crc32b:
6499     CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
6500   case AArch64::BI__builtin_arm_crc32cb:
6501     CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
6502   case AArch64::BI__builtin_arm_crc32h:
6503     CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
6504   case AArch64::BI__builtin_arm_crc32ch:
6505     CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
6506   case AArch64::BI__builtin_arm_crc32w:
6507     CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
6508   case AArch64::BI__builtin_arm_crc32cw:
6509     CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
6510   case AArch64::BI__builtin_arm_crc32d:
6511     CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
6512   case AArch64::BI__builtin_arm_crc32cd:
6513     CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
6514   }
6515 
6516   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
6517     Value *Arg0 = EmitScalarExpr(E->getArg(0));
6518     Value *Arg1 = EmitScalarExpr(E->getArg(1));
6519     Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6520 
6521     llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
6522     Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
6523 
6524     return Builder.CreateCall(F, {Arg0, Arg1});
6525   }
6526 
6527   if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
6528       BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
6529       BuiltinID == AArch64::BI__builtin_arm_rsrp ||
6530       BuiltinID == AArch64::BI__builtin_arm_wsr ||
6531       BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
6532       BuiltinID == AArch64::BI__builtin_arm_wsrp) {
6533 
6534     bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
6535                   BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
6536                   BuiltinID == AArch64::BI__builtin_arm_rsrp;
6537 
6538     bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
6539                             BuiltinID == AArch64::BI__builtin_arm_wsrp;
6540 
6541     bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
6542                    BuiltinID != AArch64::BI__builtin_arm_wsr;
6543 
6544     llvm::Type *ValueType;
6545     llvm::Type *RegisterType = Int64Ty;
6546     if (IsPointerBuiltin) {
6547       ValueType = VoidPtrTy;
6548     } else if (Is64Bit) {
6549       ValueType = Int64Ty;
6550     } else {
6551       ValueType = Int32Ty;
6552     }
6553 
6554     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
6555   }
6556 
6557   // Find out if any arguments are required to be integer constant
6558   // expressions.
6559   unsigned ICEArguments = 0;
6560   ASTContext::GetBuiltinTypeError Error;
6561   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
6562   assert(Error == ASTContext::GE_None && "Should not codegen an error");
6563 
6564   llvm::SmallVector<Value*, 4> Ops;
6565   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
6566     if ((ICEArguments & (1 << i)) == 0) {
6567       Ops.push_back(EmitScalarExpr(E->getArg(i)));
6568     } else {
6569       // If this is required to be a constant, constant fold it so that we know
6570       // that the generated intrinsic gets a ConstantInt.
6571       llvm::APSInt Result;
6572       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
6573       assert(IsConst && "Constant arg isn't actually constant?");
6574       (void)IsConst;
6575       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
6576     }
6577   }
6578 
6579   auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
6580   const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
6581       SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
6582 
6583   if (Builtin) {
6584     Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
6585     Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
6586     assert(Result && "SISD intrinsic should have been handled");
6587     return Result;
6588   }
6589 
6590   llvm::APSInt Result;
6591   const Expr *Arg = E->getArg(E->getNumArgs()-1);
6592   NeonTypeFlags Type(0);
6593   if (Arg->isIntegerConstantExpr(Result, getContext()))
6594     // Determine the type of this overloaded NEON intrinsic.
6595     Type = NeonTypeFlags(Result.getZExtValue());
6596 
6597   bool usgn = Type.isUnsigned();
6598   bool quad = Type.isQuad();
6599 
6600   // Handle non-overloaded intrinsics first.
6601   switch (BuiltinID) {
6602   default: break;
6603   case NEON::BI__builtin_neon_vabsh_f16:
6604     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6605     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
6606   case NEON::BI__builtin_neon_vldrq_p128: {
6607     llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
6608     llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
6609     Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
6610     return Builder.CreateAlignedLoad(Int128Ty, Ptr,
6611                                      CharUnits::fromQuantity(16));
6612   }
6613   case NEON::BI__builtin_neon_vstrq_p128: {
6614     llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
6615     Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
6616     return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
6617   }
6618   case NEON::BI__builtin_neon_vcvts_u32_f32:
6619   case NEON::BI__builtin_neon_vcvtd_u64_f64:
6620     usgn = true;
6621     LLVM_FALLTHROUGH;
6622   case NEON::BI__builtin_neon_vcvts_s32_f32:
6623   case NEON::BI__builtin_neon_vcvtd_s64_f64: {
6624     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6625     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
6626     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
6627     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
6628     Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
6629     if (usgn)
6630       return Builder.CreateFPToUI(Ops[0], InTy);
6631     return Builder.CreateFPToSI(Ops[0], InTy);
6632   }
6633   case NEON::BI__builtin_neon_vcvts_f32_u32:
6634   case NEON::BI__builtin_neon_vcvtd_f64_u64:
6635     usgn = true;
6636     LLVM_FALLTHROUGH;
6637   case NEON::BI__builtin_neon_vcvts_f32_s32:
6638   case NEON::BI__builtin_neon_vcvtd_f64_s64: {
6639     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6640     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
6641     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
6642     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
6643     Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
6644     if (usgn)
6645       return Builder.CreateUIToFP(Ops[0], FTy);
6646     return Builder.CreateSIToFP(Ops[0], FTy);
6647   }
6648   case NEON::BI__builtin_neon_vcvth_f16_u16:
6649   case NEON::BI__builtin_neon_vcvth_f16_u32:
6650   case NEON::BI__builtin_neon_vcvth_f16_u64:
6651     usgn = true;
6652     // FALL THROUGH
6653   case NEON::BI__builtin_neon_vcvth_f16_s16:
6654   case NEON::BI__builtin_neon_vcvth_f16_s32:
6655   case NEON::BI__builtin_neon_vcvth_f16_s64: {
6656     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6657     llvm::Type *FTy = HalfTy;
6658     llvm::Type *InTy;
6659     if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
6660       InTy = Int64Ty;
6661     else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
6662       InTy = Int32Ty;
6663     else
6664       InTy = Int16Ty;
6665     Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
6666     if (usgn)
6667       return Builder.CreateUIToFP(Ops[0], FTy);
6668     return Builder.CreateSIToFP(Ops[0], FTy);
6669   }
6670   case NEON::BI__builtin_neon_vcvth_u16_f16:
6671     usgn = true;
6672     // FALL THROUGH
6673   case NEON::BI__builtin_neon_vcvth_s16_f16: {
6674     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6675     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6676     if (usgn)
6677       return Builder.CreateFPToUI(Ops[0], Int16Ty);
6678     return Builder.CreateFPToSI(Ops[0], Int16Ty);
6679   }
6680   case NEON::BI__builtin_neon_vcvth_u32_f16:
6681     usgn = true;
6682     // FALL THROUGH
6683   case NEON::BI__builtin_neon_vcvth_s32_f16: {
6684     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6685     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6686     if (usgn)
6687       return Builder.CreateFPToUI(Ops[0], Int32Ty);
6688     return Builder.CreateFPToSI(Ops[0], Int32Ty);
6689   }
6690   case NEON::BI__builtin_neon_vcvth_u64_f16:
6691     usgn = true;
6692     // FALL THROUGH
6693   case NEON::BI__builtin_neon_vcvth_s64_f16: {
6694     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6695     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6696     if (usgn)
6697       return Builder.CreateFPToUI(Ops[0], Int64Ty);
6698     return Builder.CreateFPToSI(Ops[0], Int64Ty);
6699   }
6700   case NEON::BI__builtin_neon_vcvtah_u16_f16:
6701   case NEON::BI__builtin_neon_vcvtmh_u16_f16:
6702   case NEON::BI__builtin_neon_vcvtnh_u16_f16:
6703   case NEON::BI__builtin_neon_vcvtph_u16_f16:
6704   case NEON::BI__builtin_neon_vcvtah_s16_f16:
6705   case NEON::BI__builtin_neon_vcvtmh_s16_f16:
6706   case NEON::BI__builtin_neon_vcvtnh_s16_f16:
6707   case NEON::BI__builtin_neon_vcvtph_s16_f16: {
6708     unsigned Int;
6709     llvm::Type* InTy = Int32Ty;
6710     llvm::Type* FTy  = HalfTy;
6711     llvm::Type *Tys[2] = {InTy, FTy};
6712     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6713     switch (BuiltinID) {
6714     default: llvm_unreachable("missing builtin ID in switch!");
6715     case NEON::BI__builtin_neon_vcvtah_u16_f16:
6716       Int = Intrinsic::aarch64_neon_fcvtau; break;
6717     case NEON::BI__builtin_neon_vcvtmh_u16_f16:
6718       Int = Intrinsic::aarch64_neon_fcvtmu; break;
6719     case NEON::BI__builtin_neon_vcvtnh_u16_f16:
6720       Int = Intrinsic::aarch64_neon_fcvtnu; break;
6721     case NEON::BI__builtin_neon_vcvtph_u16_f16:
6722       Int = Intrinsic::aarch64_neon_fcvtpu; break;
6723     case NEON::BI__builtin_neon_vcvtah_s16_f16:
6724       Int = Intrinsic::aarch64_neon_fcvtas; break;
6725     case NEON::BI__builtin_neon_vcvtmh_s16_f16:
6726       Int = Intrinsic::aarch64_neon_fcvtms; break;
6727     case NEON::BI__builtin_neon_vcvtnh_s16_f16:
6728       Int = Intrinsic::aarch64_neon_fcvtns; break;
6729     case NEON::BI__builtin_neon_vcvtph_s16_f16:
6730       Int = Intrinsic::aarch64_neon_fcvtps; break;
6731     }
6732     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
6733     return Builder.CreateTrunc(Ops[0], Int16Ty);
6734   }
6735   case NEON::BI__builtin_neon_vcaleh_f16:
6736   case NEON::BI__builtin_neon_vcalth_f16:
6737   case NEON::BI__builtin_neon_vcageh_f16:
6738   case NEON::BI__builtin_neon_vcagth_f16: {
6739     unsigned Int;
6740     llvm::Type* InTy = Int32Ty;
6741     llvm::Type* FTy  = HalfTy;
6742     llvm::Type *Tys[2] = {InTy, FTy};
6743     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6744     switch (BuiltinID) {
6745     default: llvm_unreachable("missing builtin ID in switch!");
6746     case NEON::BI__builtin_neon_vcageh_f16:
6747       Int = Intrinsic::aarch64_neon_facge; break;
6748     case NEON::BI__builtin_neon_vcagth_f16:
6749       Int = Intrinsic::aarch64_neon_facgt; break;
6750     case NEON::BI__builtin_neon_vcaleh_f16:
6751       Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
6752     case NEON::BI__builtin_neon_vcalth_f16:
6753       Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
6754     }
6755     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
6756     return Builder.CreateTrunc(Ops[0], Int16Ty);
6757   }
6758   case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6759   case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
6760     unsigned Int;
6761     llvm::Type* InTy = Int32Ty;
6762     llvm::Type* FTy  = HalfTy;
6763     llvm::Type *Tys[2] = {InTy, FTy};
6764     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6765     switch (BuiltinID) {
6766     default: llvm_unreachable("missing builtin ID in switch!");
6767     case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6768       Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
6769     case NEON::BI__builtin_neon_vcvth_n_u16_f16:
6770       Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
6771     }
6772     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
6773     return Builder.CreateTrunc(Ops[0], Int16Ty);
6774   }
6775   case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6776   case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
6777     unsigned Int;
6778     llvm::Type* FTy  = HalfTy;
6779     llvm::Type* InTy = Int32Ty;
6780     llvm::Type *Tys[2] = {FTy, InTy};
6781     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6782     switch (BuiltinID) {
6783     default: llvm_unreachable("missing builtin ID in switch!");
6784     case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6785       Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
6786       Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
6787       break;
6788     case NEON::BI__builtin_neon_vcvth_n_f16_u16:
6789       Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
6790       Ops[0] = Builder.CreateZExt(Ops[0], InTy);
6791       break;
6792     }
6793     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
6794   }
6795   case NEON::BI__builtin_neon_vpaddd_s64: {
6796     llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
6797     Value *Vec = EmitScalarExpr(E->getArg(0));
6798     // The vector is v2f64, so make sure it's bitcast to that.
6799     Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
6800     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6801     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6802     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6803     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6804     // Pairwise addition of a v2f64 into a scalar f64.
6805     return Builder.CreateAdd(Op0, Op1, "vpaddd");
6806   }
6807   case NEON::BI__builtin_neon_vpaddd_f64: {
6808     llvm::Type *Ty =
6809       llvm::VectorType::get(DoubleTy, 2);
6810     Value *Vec = EmitScalarExpr(E->getArg(0));
6811     // The vector is v2f64, so make sure it's bitcast to that.
6812     Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
6813     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6814     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6815     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6816     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6817     // Pairwise addition of a v2f64 into a scalar f64.
6818     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
6819   }
6820   case NEON::BI__builtin_neon_vpadds_f32: {
6821     llvm::Type *Ty =
6822       llvm::VectorType::get(FloatTy, 2);
6823     Value *Vec = EmitScalarExpr(E->getArg(0));
6824     // The vector is v2f32, so make sure it's bitcast to that.
6825     Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
6826     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6827     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6828     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
6829     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
6830     // Pairwise addition of a v2f32 into a scalar f32.
6831     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
6832   }
6833   case NEON::BI__builtin_neon_vceqzd_s64:
6834   case NEON::BI__builtin_neon_vceqzd_f64:
6835   case NEON::BI__builtin_neon_vceqzs_f32:
6836   case NEON::BI__builtin_neon_vceqzh_f16:
6837     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6838     return EmitAArch64CompareBuiltinExpr(
6839         Ops[0], ConvertType(E->getCallReturnType(getContext())),
6840         ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz");
6841   case NEON::BI__builtin_neon_vcgezd_s64:
6842   case NEON::BI__builtin_neon_vcgezd_f64:
6843   case NEON::BI__builtin_neon_vcgezs_f32:
6844   case NEON::BI__builtin_neon_vcgezh_f16:
6845     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6846     return EmitAArch64CompareBuiltinExpr(
6847         Ops[0], ConvertType(E->getCallReturnType(getContext())),
6848         ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez");
6849   case NEON::BI__builtin_neon_vclezd_s64:
6850   case NEON::BI__builtin_neon_vclezd_f64:
6851   case NEON::BI__builtin_neon_vclezs_f32:
6852   case NEON::BI__builtin_neon_vclezh_f16:
6853     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6854     return EmitAArch64CompareBuiltinExpr(
6855         Ops[0], ConvertType(E->getCallReturnType(getContext())),
6856         ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez");
6857   case NEON::BI__builtin_neon_vcgtzd_s64:
6858   case NEON::BI__builtin_neon_vcgtzd_f64:
6859   case NEON::BI__builtin_neon_vcgtzs_f32:
6860   case NEON::BI__builtin_neon_vcgtzh_f16:
6861     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6862     return EmitAArch64CompareBuiltinExpr(
6863         Ops[0], ConvertType(E->getCallReturnType(getContext())),
6864         ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz");
6865   case NEON::BI__builtin_neon_vcltzd_s64:
6866   case NEON::BI__builtin_neon_vcltzd_f64:
6867   case NEON::BI__builtin_neon_vcltzs_f32:
6868   case NEON::BI__builtin_neon_vcltzh_f16:
6869     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6870     return EmitAArch64CompareBuiltinExpr(
6871         Ops[0], ConvertType(E->getCallReturnType(getContext())),
6872         ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
6873 
6874   case NEON::BI__builtin_neon_vceqzd_u64: {
6875     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6876     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6877     Ops[0] =
6878         Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
6879     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
6880   }
6881   case NEON::BI__builtin_neon_vceqd_f64:
6882   case NEON::BI__builtin_neon_vcled_f64:
6883   case NEON::BI__builtin_neon_vcltd_f64:
6884   case NEON::BI__builtin_neon_vcged_f64:
6885   case NEON::BI__builtin_neon_vcgtd_f64: {
6886     llvm::CmpInst::Predicate P;
6887     switch (BuiltinID) {
6888     default: llvm_unreachable("missing builtin ID in switch!");
6889     case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
6890     case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
6891     case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
6892     case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
6893     case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
6894     }
6895     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6896     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6897     Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
6898     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6899     return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
6900   }
6901   case NEON::BI__builtin_neon_vceqs_f32:
6902   case NEON::BI__builtin_neon_vcles_f32:
6903   case NEON::BI__builtin_neon_vclts_f32:
6904   case NEON::BI__builtin_neon_vcges_f32:
6905   case NEON::BI__builtin_neon_vcgts_f32: {
6906     llvm::CmpInst::Predicate P;
6907     switch (BuiltinID) {
6908     default: llvm_unreachable("missing builtin ID in switch!");
6909     case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
6910     case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
6911     case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
6912     case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
6913     case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
6914     }
6915     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6916     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
6917     Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
6918     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6919     return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
6920   }
6921   case NEON::BI__builtin_neon_vceqh_f16:
6922   case NEON::BI__builtin_neon_vcleh_f16:
6923   case NEON::BI__builtin_neon_vclth_f16:
6924   case NEON::BI__builtin_neon_vcgeh_f16:
6925   case NEON::BI__builtin_neon_vcgth_f16: {
6926     llvm::CmpInst::Predicate P;
6927     switch (BuiltinID) {
6928     default: llvm_unreachable("missing builtin ID in switch!");
6929     case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ; break;
6930     case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE; break;
6931     case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT; break;
6932     case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE; break;
6933     case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT; break;
6934     }
6935     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6936     Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6937     Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
6938     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6939     return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
6940   }
6941   case NEON::BI__builtin_neon_vceqd_s64:
6942   case NEON::BI__builtin_neon_vceqd_u64:
6943   case NEON::BI__builtin_neon_vcgtd_s64:
6944   case NEON::BI__builtin_neon_vcgtd_u64:
6945   case NEON::BI__builtin_neon_vcltd_s64:
6946   case NEON::BI__builtin_neon_vcltd_u64:
6947   case NEON::BI__builtin_neon_vcged_u64:
6948   case NEON::BI__builtin_neon_vcged_s64:
6949   case NEON::BI__builtin_neon_vcled_u64:
6950   case NEON::BI__builtin_neon_vcled_s64: {
6951     llvm::CmpInst::Predicate P;
6952     switch (BuiltinID) {
6953     default: llvm_unreachable("missing builtin ID in switch!");
6954     case NEON::BI__builtin_neon_vceqd_s64:
6955     case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
6956     case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
6957     case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
6958     case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
6959     case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
6960     case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
6961     case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
6962     case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
6963     case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
6964     }
6965     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6966     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6967     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6968     Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
6969     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
6970   }
6971   case NEON::BI__builtin_neon_vtstd_s64:
6972   case NEON::BI__builtin_neon_vtstd_u64: {
6973     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6974     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6975     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6976     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
6977     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
6978                                 llvm::Constant::getNullValue(Int64Ty));
6979     return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
6980   }
6981   case NEON::BI__builtin_neon_vset_lane_i8:
6982   case NEON::BI__builtin_neon_vset_lane_i16:
6983   case NEON::BI__builtin_neon_vset_lane_i32:
6984   case NEON::BI__builtin_neon_vset_lane_i64:
6985   case NEON::BI__builtin_neon_vset_lane_f32:
6986   case NEON::BI__builtin_neon_vsetq_lane_i8:
6987   case NEON::BI__builtin_neon_vsetq_lane_i16:
6988   case NEON::BI__builtin_neon_vsetq_lane_i32:
6989   case NEON::BI__builtin_neon_vsetq_lane_i64:
6990   case NEON::BI__builtin_neon_vsetq_lane_f32:
6991     Ops.push_back(EmitScalarExpr(E->getArg(2)));
6992     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6993   case NEON::BI__builtin_neon_vset_lane_f64:
6994     // The vector type needs a cast for the v1f64 variant.
6995     Ops[1] = Builder.CreateBitCast(Ops[1],
6996                                    llvm::VectorType::get(DoubleTy, 1));
6997     Ops.push_back(EmitScalarExpr(E->getArg(2)));
6998     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
6999   case NEON::BI__builtin_neon_vsetq_lane_f64:
7000     // The vector type needs a cast for the v2f64 variant.
7001     Ops[1] = Builder.CreateBitCast(Ops[1],
7002         llvm::VectorType::get(DoubleTy, 2));
7003     Ops.push_back(EmitScalarExpr(E->getArg(2)));
7004     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7005 
7006   case NEON::BI__builtin_neon_vget_lane_i8:
7007   case NEON::BI__builtin_neon_vdupb_lane_i8:
7008     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
7009     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7010                                         "vget_lane");
7011   case NEON::BI__builtin_neon_vgetq_lane_i8:
7012   case NEON::BI__builtin_neon_vdupb_laneq_i8:
7013     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
7014     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7015                                         "vgetq_lane");
7016   case NEON::BI__builtin_neon_vget_lane_i16:
7017   case NEON::BI__builtin_neon_vduph_lane_i16:
7018     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
7019     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7020                                         "vget_lane");
7021   case NEON::BI__builtin_neon_vgetq_lane_i16:
7022   case NEON::BI__builtin_neon_vduph_laneq_i16:
7023     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
7024     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7025                                         "vgetq_lane");
7026   case NEON::BI__builtin_neon_vget_lane_i32:
7027   case NEON::BI__builtin_neon_vdups_lane_i32:
7028     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
7029     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7030                                         "vget_lane");
7031   case NEON::BI__builtin_neon_vdups_lane_f32:
7032     Ops[0] = Builder.CreateBitCast(Ops[0],
7033         llvm::VectorType::get(FloatTy, 2));
7034     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7035                                         "vdups_lane");
7036   case NEON::BI__builtin_neon_vgetq_lane_i32:
7037   case NEON::BI__builtin_neon_vdups_laneq_i32:
7038     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
7039     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7040                                         "vgetq_lane");
7041   case NEON::BI__builtin_neon_vget_lane_i64:
7042   case NEON::BI__builtin_neon_vdupd_lane_i64:
7043     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
7044     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7045                                         "vget_lane");
7046   case NEON::BI__builtin_neon_vdupd_lane_f64:
7047     Ops[0] = Builder.CreateBitCast(Ops[0],
7048         llvm::VectorType::get(DoubleTy, 1));
7049     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7050                                         "vdupd_lane");
7051   case NEON::BI__builtin_neon_vgetq_lane_i64:
7052   case NEON::BI__builtin_neon_vdupd_laneq_i64:
7053     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
7054     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7055                                         "vgetq_lane");
7056   case NEON::BI__builtin_neon_vget_lane_f32:
7057     Ops[0] = Builder.CreateBitCast(Ops[0],
7058         llvm::VectorType::get(FloatTy, 2));
7059     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7060                                         "vget_lane");
7061   case NEON::BI__builtin_neon_vget_lane_f64:
7062     Ops[0] = Builder.CreateBitCast(Ops[0],
7063         llvm::VectorType::get(DoubleTy, 1));
7064     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7065                                         "vget_lane");
7066   case NEON::BI__builtin_neon_vgetq_lane_f32:
7067   case NEON::BI__builtin_neon_vdups_laneq_f32:
7068     Ops[0] = Builder.CreateBitCast(Ops[0],
7069         llvm::VectorType::get(FloatTy, 4));
7070     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7071                                         "vgetq_lane");
7072   case NEON::BI__builtin_neon_vgetq_lane_f64:
7073   case NEON::BI__builtin_neon_vdupd_laneq_f64:
7074     Ops[0] = Builder.CreateBitCast(Ops[0],
7075         llvm::VectorType::get(DoubleTy, 2));
7076     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7077                                         "vgetq_lane");
7078   case NEON::BI__builtin_neon_vaddh_f16:
7079     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7080     return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
7081   case NEON::BI__builtin_neon_vsubh_f16:
7082     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7083     return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
7084   case NEON::BI__builtin_neon_vmulh_f16:
7085     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7086     return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
7087   case NEON::BI__builtin_neon_vdivh_f16:
7088     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7089     return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
7090   case NEON::BI__builtin_neon_vfmah_f16: {
7091     Value *F = CGM.getIntrinsic(Intrinsic::fma, HalfTy);
7092     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
7093     return Builder.CreateCall(F,
7094       {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]});
7095   }
7096   case NEON::BI__builtin_neon_vfmsh_f16: {
7097     Value *F = CGM.getIntrinsic(Intrinsic::fma, HalfTy);
7098     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(HalfTy);
7099     Value* Sub = Builder.CreateFSub(Zero, EmitScalarExpr(E->getArg(1)), "vsubh");
7100     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
7101     return Builder.CreateCall(F, {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]});
7102   }
7103   case NEON::BI__builtin_neon_vaddd_s64:
7104   case NEON::BI__builtin_neon_vaddd_u64:
7105     return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
7106   case NEON::BI__builtin_neon_vsubd_s64:
7107   case NEON::BI__builtin_neon_vsubd_u64:
7108     return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
7109   case NEON::BI__builtin_neon_vqdmlalh_s16:
7110   case NEON::BI__builtin_neon_vqdmlslh_s16: {
7111     SmallVector<Value *, 2> ProductOps;
7112     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7113     ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
7114     llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7115     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7116                           ProductOps, "vqdmlXl");
7117     Constant *CI = ConstantInt::get(SizeTy, 0);
7118     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
7119 
7120     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
7121                                         ? Intrinsic::aarch64_neon_sqadd
7122                                         : Intrinsic::aarch64_neon_sqsub;
7123     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
7124   }
7125   case NEON::BI__builtin_neon_vqshlud_n_s64: {
7126     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7127     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7128     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
7129                         Ops, "vqshlu_n");
7130   }
7131   case NEON::BI__builtin_neon_vqshld_n_u64:
7132   case NEON::BI__builtin_neon_vqshld_n_s64: {
7133     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
7134                                    ? Intrinsic::aarch64_neon_uqshl
7135                                    : Intrinsic::aarch64_neon_sqshl;
7136     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7137     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7138     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
7139   }
7140   case NEON::BI__builtin_neon_vrshrd_n_u64:
7141   case NEON::BI__builtin_neon_vrshrd_n_s64: {
7142     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
7143                                    ? Intrinsic::aarch64_neon_urshl
7144                                    : Intrinsic::aarch64_neon_srshl;
7145     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7146     int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
7147     Ops[1] = ConstantInt::get(Int64Ty, -SV);
7148     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
7149   }
7150   case NEON::BI__builtin_neon_vrsrad_n_u64:
7151   case NEON::BI__builtin_neon_vrsrad_n_s64: {
7152     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
7153                                    ? Intrinsic::aarch64_neon_urshl
7154                                    : Intrinsic::aarch64_neon_srshl;
7155     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7156     Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
7157     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
7158                                 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
7159     return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
7160   }
7161   case NEON::BI__builtin_neon_vshld_n_s64:
7162   case NEON::BI__builtin_neon_vshld_n_u64: {
7163     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7164     return Builder.CreateShl(
7165         Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
7166   }
7167   case NEON::BI__builtin_neon_vshrd_n_s64: {
7168     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7169     return Builder.CreateAShr(
7170         Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
7171                                                    Amt->getZExtValue())),
7172         "shrd_n");
7173   }
7174   case NEON::BI__builtin_neon_vshrd_n_u64: {
7175     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
7176     uint64_t ShiftAmt = Amt->getZExtValue();
7177     // Right-shifting an unsigned value by its size yields 0.
7178     if (ShiftAmt == 64)
7179       return ConstantInt::get(Int64Ty, 0);
7180     return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
7181                               "shrd_n");
7182   }
7183   case NEON::BI__builtin_neon_vsrad_n_s64: {
7184     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
7185     Ops[1] = Builder.CreateAShr(
7186         Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
7187                                                    Amt->getZExtValue())),
7188         "shrd_n");
7189     return Builder.CreateAdd(Ops[0], Ops[1]);
7190   }
7191   case NEON::BI__builtin_neon_vsrad_n_u64: {
7192     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
7193     uint64_t ShiftAmt = Amt->getZExtValue();
7194     // Right-shifting an unsigned value by its size yields 0.
7195     // As Op + 0 = Op, return Ops[0] directly.
7196     if (ShiftAmt == 64)
7197       return Ops[0];
7198     Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
7199                                 "shrd_n");
7200     return Builder.CreateAdd(Ops[0], Ops[1]);
7201   }
7202   case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
7203   case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
7204   case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
7205   case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
7206     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
7207                                           "lane");
7208     SmallVector<Value *, 2> ProductOps;
7209     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7210     ProductOps.push_back(vectorWrapScalar16(Ops[2]));
7211     llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7212     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7213                           ProductOps, "vqdmlXl");
7214     Constant *CI = ConstantInt::get(SizeTy, 0);
7215     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
7216     Ops.pop_back();
7217 
7218     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
7219                        BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
7220                           ? Intrinsic::aarch64_neon_sqadd
7221                           : Intrinsic::aarch64_neon_sqsub;
7222     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
7223   }
7224   case NEON::BI__builtin_neon_vqdmlals_s32:
7225   case NEON::BI__builtin_neon_vqdmlsls_s32: {
7226     SmallVector<Value *, 2> ProductOps;
7227     ProductOps.push_back(Ops[1]);
7228     ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
7229     Ops[1] =
7230         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7231                      ProductOps, "vqdmlXl");
7232 
7233     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
7234                                         ? Intrinsic::aarch64_neon_sqadd
7235                                         : Intrinsic::aarch64_neon_sqsub;
7236     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
7237   }
7238   case NEON::BI__builtin_neon_vqdmlals_lane_s32:
7239   case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
7240   case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
7241   case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
7242     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
7243                                           "lane");
7244     SmallVector<Value *, 2> ProductOps;
7245     ProductOps.push_back(Ops[1]);
7246     ProductOps.push_back(Ops[2]);
7247     Ops[1] =
7248         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7249                      ProductOps, "vqdmlXl");
7250     Ops.pop_back();
7251 
7252     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
7253                        BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
7254                           ? Intrinsic::aarch64_neon_sqadd
7255                           : Intrinsic::aarch64_neon_sqsub;
7256     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
7257   }
7258   }
7259 
7260   llvm::VectorType *VTy = GetNeonType(this, Type);
7261   llvm::Type *Ty = VTy;
7262   if (!Ty)
7263     return nullptr;
7264 
7265   // Not all intrinsics handled by the common case work for AArch64 yet, so only
7266   // defer to common code if it's been added to our special map.
7267   Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
7268                                    AArch64SIMDIntrinsicsProvenSorted);
7269 
7270   if (Builtin)
7271     return EmitCommonNeonBuiltinExpr(
7272         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
7273         Builtin->NameHint, Builtin->TypeModifier, E, Ops,
7274         /*never use addresses*/ Address::invalid(), Address::invalid(), Arch);
7275 
7276   if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
7277     return V;
7278 
7279   unsigned Int;
7280   switch (BuiltinID) {
7281   default: return nullptr;
7282   case NEON::BI__builtin_neon_vbsl_v:
7283   case NEON::BI__builtin_neon_vbslq_v: {
7284     llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
7285     Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
7286     Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
7287     Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
7288 
7289     Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
7290     Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
7291     Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
7292     return Builder.CreateBitCast(Ops[0], Ty);
7293   }
7294   case NEON::BI__builtin_neon_vfma_lane_v:
7295   case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
7296     // The ARM builtins (and instructions) have the addend as the first
7297     // operand, but the 'fma' intrinsics have it last. Swap it around here.
7298     Value *Addend = Ops[0];
7299     Value *Multiplicand = Ops[1];
7300     Value *LaneSource = Ops[2];
7301     Ops[0] = Multiplicand;
7302     Ops[1] = LaneSource;
7303     Ops[2] = Addend;
7304 
7305     // Now adjust things to handle the lane access.
7306     llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
7307       llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
7308       VTy;
7309     llvm::Constant *cst = cast<Constant>(Ops[3]);
7310     Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
7311     Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
7312     Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
7313 
7314     Ops.pop_back();
7315     Int = Intrinsic::fma;
7316     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
7317   }
7318   case NEON::BI__builtin_neon_vfma_laneq_v: {
7319     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
7320     // v1f64 fma should be mapped to Neon scalar f64 fma
7321     if (VTy && VTy->getElementType() == DoubleTy) {
7322       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7323       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7324       llvm::Type *VTy = GetNeonType(this,
7325         NeonTypeFlags(NeonTypeFlags::Float64, false, true));
7326       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
7327       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
7328       Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
7329       Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7330       return Builder.CreateBitCast(Result, Ty);
7331     }
7332     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
7333     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7334     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7335 
7336     llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
7337                                             VTy->getNumElements() * 2);
7338     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
7339     Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
7340                                                cast<ConstantInt>(Ops[3]));
7341     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
7342 
7343     return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7344   }
7345   case NEON::BI__builtin_neon_vfmaq_laneq_v: {
7346     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
7347     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7348     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7349 
7350     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7351     Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
7352     return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7353   }
7354   case NEON::BI__builtin_neon_vfmah_lane_f16:
7355   case NEON::BI__builtin_neon_vfmas_lane_f32:
7356   case NEON::BI__builtin_neon_vfmah_laneq_f16:
7357   case NEON::BI__builtin_neon_vfmas_laneq_f32:
7358   case NEON::BI__builtin_neon_vfmad_lane_f64:
7359   case NEON::BI__builtin_neon_vfmad_laneq_f64: {
7360     Ops.push_back(EmitScalarExpr(E->getArg(3)));
7361     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
7362     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
7363     Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
7364     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7365   }
7366   case NEON::BI__builtin_neon_vmull_v:
7367     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7368     Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
7369     if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
7370     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
7371   case NEON::BI__builtin_neon_vmax_v:
7372   case NEON::BI__builtin_neon_vmaxq_v:
7373     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7374     Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
7375     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
7376     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
7377   case NEON::BI__builtin_neon_vmaxh_f16: {
7378     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7379     Int = Intrinsic::aarch64_neon_fmax;
7380     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
7381   }
7382   case NEON::BI__builtin_neon_vmin_v:
7383   case NEON::BI__builtin_neon_vminq_v:
7384     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7385     Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
7386     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
7387     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
7388   case NEON::BI__builtin_neon_vminh_f16: {
7389     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7390     Int = Intrinsic::aarch64_neon_fmin;
7391     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
7392   }
7393   case NEON::BI__builtin_neon_vabd_v:
7394   case NEON::BI__builtin_neon_vabdq_v:
7395     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7396     Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
7397     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
7398     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
7399   case NEON::BI__builtin_neon_vpadal_v:
7400   case NEON::BI__builtin_neon_vpadalq_v: {
7401     unsigned ArgElts = VTy->getNumElements();
7402     llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
7403     unsigned BitWidth = EltTy->getBitWidth();
7404     llvm::Type *ArgTy = llvm::VectorType::get(
7405         llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
7406     llvm::Type* Tys[2] = { VTy, ArgTy };
7407     Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
7408     SmallVector<llvm::Value*, 1> TmpOps;
7409     TmpOps.push_back(Ops[1]);
7410     Function *F = CGM.getIntrinsic(Int, Tys);
7411     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
7412     llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
7413     return Builder.CreateAdd(tmp, addend);
7414   }
7415   case NEON::BI__builtin_neon_vpmin_v:
7416   case NEON::BI__builtin_neon_vpminq_v:
7417     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7418     Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
7419     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
7420     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
7421   case NEON::BI__builtin_neon_vpmax_v:
7422   case NEON::BI__builtin_neon_vpmaxq_v:
7423     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
7424     Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
7425     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
7426     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
7427   case NEON::BI__builtin_neon_vminnm_v:
7428   case NEON::BI__builtin_neon_vminnmq_v:
7429     Int = Intrinsic::aarch64_neon_fminnm;
7430     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
7431   case NEON::BI__builtin_neon_vminnmh_f16:
7432     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7433     Int = Intrinsic::aarch64_neon_fminnm;
7434     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm");
7435   case NEON::BI__builtin_neon_vmaxnm_v:
7436   case NEON::BI__builtin_neon_vmaxnmq_v:
7437     Int = Intrinsic::aarch64_neon_fmaxnm;
7438     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
7439   case NEON::BI__builtin_neon_vmaxnmh_f16:
7440     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7441     Int = Intrinsic::aarch64_neon_fmaxnm;
7442     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm");
7443   case NEON::BI__builtin_neon_vrecpss_f32: {
7444     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7445     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
7446                         Ops, "vrecps");
7447   }
7448   case NEON::BI__builtin_neon_vrecpsd_f64:
7449     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7450     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
7451                         Ops, "vrecps");
7452   case NEON::BI__builtin_neon_vrecpsh_f16:
7453     Ops.push_back(EmitScalarExpr(E->getArg(1)));
7454     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
7455                         Ops, "vrecps");
7456   case NEON::BI__builtin_neon_vqshrun_n_v:
7457     Int = Intrinsic::aarch64_neon_sqshrun;
7458     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
7459   case NEON::BI__builtin_neon_vqrshrun_n_v:
7460     Int = Intrinsic::aarch64_neon_sqrshrun;
7461     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
7462   case NEON::BI__builtin_neon_vqshrn_n_v:
7463     Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
7464     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
7465   case NEON::BI__builtin_neon_vrshrn_n_v:
7466     Int = Intrinsic::aarch64_neon_rshrn;
7467     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
7468   case NEON::BI__builtin_neon_vqrshrn_n_v:
7469     Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
7470     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
7471   case NEON::BI__builtin_neon_vrndah_f16: {
7472     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7473     Int = Intrinsic::round;
7474     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda");
7475   }
7476   case NEON::BI__builtin_neon_vrnda_v:
7477   case NEON::BI__builtin_neon_vrndaq_v: {
7478     Int = Intrinsic::round;
7479     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
7480   }
7481   case NEON::BI__builtin_neon_vrndih_f16: {
7482     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7483     Int = Intrinsic::nearbyint;
7484     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi");
7485   }
7486   case NEON::BI__builtin_neon_vrndi_v:
7487   case NEON::BI__builtin_neon_vrndiq_v: {
7488     Int = Intrinsic::nearbyint;
7489     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
7490   }
7491   case NEON::BI__builtin_neon_vrndmh_f16: {
7492     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7493     Int = Intrinsic::floor;
7494     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm");
7495   }
7496   case NEON::BI__builtin_neon_vrndm_v:
7497   case NEON::BI__builtin_neon_vrndmq_v: {
7498     Int = Intrinsic::floor;
7499     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
7500   }
7501   case NEON::BI__builtin_neon_vrndnh_f16: {
7502     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7503     Int = Intrinsic::aarch64_neon_frintn;
7504     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
7505   }
7506   case NEON::BI__builtin_neon_vrndn_v:
7507   case NEON::BI__builtin_neon_vrndnq_v: {
7508     Int = Intrinsic::aarch64_neon_frintn;
7509     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
7510   }
7511   case NEON::BI__builtin_neon_vrndph_f16: {
7512     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7513     Int = Intrinsic::ceil;
7514     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp");
7515   }
7516   case NEON::BI__builtin_neon_vrndp_v:
7517   case NEON::BI__builtin_neon_vrndpq_v: {
7518     Int = Intrinsic::ceil;
7519     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
7520   }
7521   case NEON::BI__builtin_neon_vrndxh_f16: {
7522     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7523     Int = Intrinsic::rint;
7524     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx");
7525   }
7526   case NEON::BI__builtin_neon_vrndx_v:
7527   case NEON::BI__builtin_neon_vrndxq_v: {
7528     Int = Intrinsic::rint;
7529     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
7530   }
7531   case NEON::BI__builtin_neon_vrndh_f16: {
7532     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7533     Int = Intrinsic::trunc;
7534     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz");
7535   }
7536   case NEON::BI__builtin_neon_vrnd_v:
7537   case NEON::BI__builtin_neon_vrndq_v: {
7538     Int = Intrinsic::trunc;
7539     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
7540   }
7541   case NEON::BI__builtin_neon_vcvt_f64_v:
7542   case NEON::BI__builtin_neon_vcvtq_f64_v:
7543     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7544     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
7545     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
7546                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
7547   case NEON::BI__builtin_neon_vcvt_f64_f32: {
7548     assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
7549            "unexpected vcvt_f64_f32 builtin");
7550     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
7551     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
7552 
7553     return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
7554   }
7555   case NEON::BI__builtin_neon_vcvt_f32_f64: {
7556     assert(Type.getEltType() == NeonTypeFlags::Float32 &&
7557            "unexpected vcvt_f32_f64 builtin");
7558     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
7559     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
7560 
7561     return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
7562   }
7563   case NEON::BI__builtin_neon_vcvt_s32_v:
7564   case NEON::BI__builtin_neon_vcvt_u32_v:
7565   case NEON::BI__builtin_neon_vcvt_s64_v:
7566   case NEON::BI__builtin_neon_vcvt_u64_v:
7567 	case NEON::BI__builtin_neon_vcvt_s16_v:
7568 	case NEON::BI__builtin_neon_vcvt_u16_v:
7569   case NEON::BI__builtin_neon_vcvtq_s32_v:
7570   case NEON::BI__builtin_neon_vcvtq_u32_v:
7571   case NEON::BI__builtin_neon_vcvtq_s64_v:
7572   case NEON::BI__builtin_neon_vcvtq_u64_v:
7573 	case NEON::BI__builtin_neon_vcvtq_s16_v:
7574 	case NEON::BI__builtin_neon_vcvtq_u16_v: {
7575     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
7576     if (usgn)
7577       return Builder.CreateFPToUI(Ops[0], Ty);
7578     return Builder.CreateFPToSI(Ops[0], Ty);
7579   }
7580   case NEON::BI__builtin_neon_vcvta_s16_v:
7581   case NEON::BI__builtin_neon_vcvta_u16_v:
7582   case NEON::BI__builtin_neon_vcvta_s32_v:
7583   case NEON::BI__builtin_neon_vcvtaq_s16_v:
7584   case NEON::BI__builtin_neon_vcvtaq_s32_v:
7585   case NEON::BI__builtin_neon_vcvta_u32_v:
7586   case NEON::BI__builtin_neon_vcvtaq_u16_v:
7587   case NEON::BI__builtin_neon_vcvtaq_u32_v:
7588   case NEON::BI__builtin_neon_vcvta_s64_v:
7589   case NEON::BI__builtin_neon_vcvtaq_s64_v:
7590   case NEON::BI__builtin_neon_vcvta_u64_v:
7591   case NEON::BI__builtin_neon_vcvtaq_u64_v: {
7592     Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
7593     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
7594     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
7595   }
7596   case NEON::BI__builtin_neon_vcvtm_s16_v:
7597   case NEON::BI__builtin_neon_vcvtm_s32_v:
7598   case NEON::BI__builtin_neon_vcvtmq_s16_v:
7599   case NEON::BI__builtin_neon_vcvtmq_s32_v:
7600   case NEON::BI__builtin_neon_vcvtm_u16_v:
7601   case NEON::BI__builtin_neon_vcvtm_u32_v:
7602   case NEON::BI__builtin_neon_vcvtmq_u16_v:
7603   case NEON::BI__builtin_neon_vcvtmq_u32_v:
7604   case NEON::BI__builtin_neon_vcvtm_s64_v:
7605   case NEON::BI__builtin_neon_vcvtmq_s64_v:
7606   case NEON::BI__builtin_neon_vcvtm_u64_v:
7607   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7608     Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
7609     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
7610     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
7611   }
7612   case NEON::BI__builtin_neon_vcvtn_s16_v:
7613   case NEON::BI__builtin_neon_vcvtn_s32_v:
7614   case NEON::BI__builtin_neon_vcvtnq_s16_v:
7615   case NEON::BI__builtin_neon_vcvtnq_s32_v:
7616   case NEON::BI__builtin_neon_vcvtn_u16_v:
7617   case NEON::BI__builtin_neon_vcvtn_u32_v:
7618   case NEON::BI__builtin_neon_vcvtnq_u16_v:
7619   case NEON::BI__builtin_neon_vcvtnq_u32_v:
7620   case NEON::BI__builtin_neon_vcvtn_s64_v:
7621   case NEON::BI__builtin_neon_vcvtnq_s64_v:
7622   case NEON::BI__builtin_neon_vcvtn_u64_v:
7623   case NEON::BI__builtin_neon_vcvtnq_u64_v: {
7624     Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
7625     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
7626     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
7627   }
7628   case NEON::BI__builtin_neon_vcvtp_s16_v:
7629   case NEON::BI__builtin_neon_vcvtp_s32_v:
7630   case NEON::BI__builtin_neon_vcvtpq_s16_v:
7631   case NEON::BI__builtin_neon_vcvtpq_s32_v:
7632   case NEON::BI__builtin_neon_vcvtp_u16_v:
7633   case NEON::BI__builtin_neon_vcvtp_u32_v:
7634   case NEON::BI__builtin_neon_vcvtpq_u16_v:
7635   case NEON::BI__builtin_neon_vcvtpq_u32_v:
7636   case NEON::BI__builtin_neon_vcvtp_s64_v:
7637   case NEON::BI__builtin_neon_vcvtpq_s64_v:
7638   case NEON::BI__builtin_neon_vcvtp_u64_v:
7639   case NEON::BI__builtin_neon_vcvtpq_u64_v: {
7640     Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
7641     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
7642     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
7643   }
7644   case NEON::BI__builtin_neon_vmulx_v:
7645   case NEON::BI__builtin_neon_vmulxq_v: {
7646     Int = Intrinsic::aarch64_neon_fmulx;
7647     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
7648   }
7649   case NEON::BI__builtin_neon_vmulxh_lane_f16:
7650   case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
7651     // vmulx_lane should be mapped to Neon scalar mulx after
7652     // extracting the scalar element
7653     Ops.push_back(EmitScalarExpr(E->getArg(2)));
7654     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
7655     Ops.pop_back();
7656     Int = Intrinsic::aarch64_neon_fmulx;
7657     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx");
7658   }
7659   case NEON::BI__builtin_neon_vmul_lane_v:
7660   case NEON::BI__builtin_neon_vmul_laneq_v: {
7661     // v1f64 vmul_lane should be mapped to Neon scalar mul lane
7662     bool Quad = false;
7663     if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
7664       Quad = true;
7665     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7666     llvm::Type *VTy = GetNeonType(this,
7667       NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
7668     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7669     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
7670     Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
7671     return Builder.CreateBitCast(Result, Ty);
7672   }
7673   case NEON::BI__builtin_neon_vnegd_s64:
7674     return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
7675   case NEON::BI__builtin_neon_vnegh_f16:
7676     return Builder.CreateFNeg(EmitScalarExpr(E->getArg(0)), "vnegh");
7677   case NEON::BI__builtin_neon_vpmaxnm_v:
7678   case NEON::BI__builtin_neon_vpmaxnmq_v: {
7679     Int = Intrinsic::aarch64_neon_fmaxnmp;
7680     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
7681   }
7682   case NEON::BI__builtin_neon_vpminnm_v:
7683   case NEON::BI__builtin_neon_vpminnmq_v: {
7684     Int = Intrinsic::aarch64_neon_fminnmp;
7685     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
7686   }
7687   case NEON::BI__builtin_neon_vsqrth_f16: {
7688     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7689     Int = Intrinsic::sqrt;
7690     return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt");
7691   }
7692   case NEON::BI__builtin_neon_vsqrt_v:
7693   case NEON::BI__builtin_neon_vsqrtq_v: {
7694     Int = Intrinsic::sqrt;
7695     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7696     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
7697   }
7698   case NEON::BI__builtin_neon_vrbit_v:
7699   case NEON::BI__builtin_neon_vrbitq_v: {
7700     Int = Intrinsic::aarch64_neon_rbit;
7701     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
7702   }
7703   case NEON::BI__builtin_neon_vaddv_u8:
7704     // FIXME: These are handled by the AArch64 scalar code.
7705     usgn = true;
7706     LLVM_FALLTHROUGH;
7707   case NEON::BI__builtin_neon_vaddv_s8: {
7708     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7709     Ty = Int32Ty;
7710     VTy = llvm::VectorType::get(Int8Ty, 8);
7711     llvm::Type *Tys[2] = { Ty, VTy };
7712     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7713     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
7714     return Builder.CreateTrunc(Ops[0], Int8Ty);
7715   }
7716   case NEON::BI__builtin_neon_vaddv_u16:
7717     usgn = true;
7718     LLVM_FALLTHROUGH;
7719   case NEON::BI__builtin_neon_vaddv_s16: {
7720     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7721     Ty = Int32Ty;
7722     VTy = llvm::VectorType::get(Int16Ty, 4);
7723     llvm::Type *Tys[2] = { Ty, VTy };
7724     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7725     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
7726     return Builder.CreateTrunc(Ops[0], Int16Ty);
7727   }
7728   case NEON::BI__builtin_neon_vaddvq_u8:
7729     usgn = true;
7730     LLVM_FALLTHROUGH;
7731   case NEON::BI__builtin_neon_vaddvq_s8: {
7732     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7733     Ty = Int32Ty;
7734     VTy = llvm::VectorType::get(Int8Ty, 16);
7735     llvm::Type *Tys[2] = { Ty, VTy };
7736     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7737     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
7738     return Builder.CreateTrunc(Ops[0], Int8Ty);
7739   }
7740   case NEON::BI__builtin_neon_vaddvq_u16:
7741     usgn = true;
7742     LLVM_FALLTHROUGH;
7743   case NEON::BI__builtin_neon_vaddvq_s16: {
7744     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7745     Ty = Int32Ty;
7746     VTy = llvm::VectorType::get(Int16Ty, 8);
7747     llvm::Type *Tys[2] = { Ty, VTy };
7748     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7749     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
7750     return Builder.CreateTrunc(Ops[0], Int16Ty);
7751   }
7752   case NEON::BI__builtin_neon_vmaxv_u8: {
7753     Int = Intrinsic::aarch64_neon_umaxv;
7754     Ty = Int32Ty;
7755     VTy = llvm::VectorType::get(Int8Ty, 8);
7756     llvm::Type *Tys[2] = { Ty, VTy };
7757     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7758     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7759     return Builder.CreateTrunc(Ops[0], Int8Ty);
7760   }
7761   case NEON::BI__builtin_neon_vmaxv_u16: {
7762     Int = Intrinsic::aarch64_neon_umaxv;
7763     Ty = Int32Ty;
7764     VTy = llvm::VectorType::get(Int16Ty, 4);
7765     llvm::Type *Tys[2] = { Ty, VTy };
7766     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7767     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7768     return Builder.CreateTrunc(Ops[0], Int16Ty);
7769   }
7770   case NEON::BI__builtin_neon_vmaxvq_u8: {
7771     Int = Intrinsic::aarch64_neon_umaxv;
7772     Ty = Int32Ty;
7773     VTy = llvm::VectorType::get(Int8Ty, 16);
7774     llvm::Type *Tys[2] = { Ty, VTy };
7775     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7776     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7777     return Builder.CreateTrunc(Ops[0], Int8Ty);
7778   }
7779   case NEON::BI__builtin_neon_vmaxvq_u16: {
7780     Int = Intrinsic::aarch64_neon_umaxv;
7781     Ty = Int32Ty;
7782     VTy = llvm::VectorType::get(Int16Ty, 8);
7783     llvm::Type *Tys[2] = { Ty, VTy };
7784     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7785     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7786     return Builder.CreateTrunc(Ops[0], Int16Ty);
7787   }
7788   case NEON::BI__builtin_neon_vmaxv_s8: {
7789     Int = Intrinsic::aarch64_neon_smaxv;
7790     Ty = Int32Ty;
7791     VTy = llvm::VectorType::get(Int8Ty, 8);
7792     llvm::Type *Tys[2] = { Ty, VTy };
7793     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7794     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7795     return Builder.CreateTrunc(Ops[0], Int8Ty);
7796   }
7797   case NEON::BI__builtin_neon_vmaxv_s16: {
7798     Int = Intrinsic::aarch64_neon_smaxv;
7799     Ty = Int32Ty;
7800     VTy = llvm::VectorType::get(Int16Ty, 4);
7801     llvm::Type *Tys[2] = { Ty, VTy };
7802     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7803     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7804     return Builder.CreateTrunc(Ops[0], Int16Ty);
7805   }
7806   case NEON::BI__builtin_neon_vmaxvq_s8: {
7807     Int = Intrinsic::aarch64_neon_smaxv;
7808     Ty = Int32Ty;
7809     VTy = llvm::VectorType::get(Int8Ty, 16);
7810     llvm::Type *Tys[2] = { Ty, VTy };
7811     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7812     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7813     return Builder.CreateTrunc(Ops[0], Int8Ty);
7814   }
7815   case NEON::BI__builtin_neon_vmaxvq_s16: {
7816     Int = Intrinsic::aarch64_neon_smaxv;
7817     Ty = Int32Ty;
7818     VTy = llvm::VectorType::get(Int16Ty, 8);
7819     llvm::Type *Tys[2] = { Ty, VTy };
7820     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7821     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7822     return Builder.CreateTrunc(Ops[0], Int16Ty);
7823   }
7824   case NEON::BI__builtin_neon_vmaxv_f16: {
7825     Int = Intrinsic::aarch64_neon_fmaxv;
7826     Ty = HalfTy;
7827     VTy = llvm::VectorType::get(HalfTy, 4);
7828     llvm::Type *Tys[2] = { Ty, VTy };
7829     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7830     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7831     return Builder.CreateTrunc(Ops[0], HalfTy);
7832   }
7833   case NEON::BI__builtin_neon_vmaxvq_f16: {
7834     Int = Intrinsic::aarch64_neon_fmaxv;
7835     Ty = HalfTy;
7836     VTy = llvm::VectorType::get(HalfTy, 8);
7837     llvm::Type *Tys[2] = { Ty, VTy };
7838     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7839     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
7840     return Builder.CreateTrunc(Ops[0], HalfTy);
7841   }
7842   case NEON::BI__builtin_neon_vminv_u8: {
7843     Int = Intrinsic::aarch64_neon_uminv;
7844     Ty = Int32Ty;
7845     VTy = llvm::VectorType::get(Int8Ty, 8);
7846     llvm::Type *Tys[2] = { Ty, VTy };
7847     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7848     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7849     return Builder.CreateTrunc(Ops[0], Int8Ty);
7850   }
7851   case NEON::BI__builtin_neon_vminv_u16: {
7852     Int = Intrinsic::aarch64_neon_uminv;
7853     Ty = Int32Ty;
7854     VTy = llvm::VectorType::get(Int16Ty, 4);
7855     llvm::Type *Tys[2] = { Ty, VTy };
7856     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7857     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7858     return Builder.CreateTrunc(Ops[0], Int16Ty);
7859   }
7860   case NEON::BI__builtin_neon_vminvq_u8: {
7861     Int = Intrinsic::aarch64_neon_uminv;
7862     Ty = Int32Ty;
7863     VTy = llvm::VectorType::get(Int8Ty, 16);
7864     llvm::Type *Tys[2] = { Ty, VTy };
7865     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7866     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7867     return Builder.CreateTrunc(Ops[0], Int8Ty);
7868   }
7869   case NEON::BI__builtin_neon_vminvq_u16: {
7870     Int = Intrinsic::aarch64_neon_uminv;
7871     Ty = Int32Ty;
7872     VTy = llvm::VectorType::get(Int16Ty, 8);
7873     llvm::Type *Tys[2] = { Ty, VTy };
7874     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7875     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7876     return Builder.CreateTrunc(Ops[0], Int16Ty);
7877   }
7878   case NEON::BI__builtin_neon_vminv_s8: {
7879     Int = Intrinsic::aarch64_neon_sminv;
7880     Ty = Int32Ty;
7881     VTy = llvm::VectorType::get(Int8Ty, 8);
7882     llvm::Type *Tys[2] = { Ty, VTy };
7883     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7884     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7885     return Builder.CreateTrunc(Ops[0], Int8Ty);
7886   }
7887   case NEON::BI__builtin_neon_vminv_s16: {
7888     Int = Intrinsic::aarch64_neon_sminv;
7889     Ty = Int32Ty;
7890     VTy = llvm::VectorType::get(Int16Ty, 4);
7891     llvm::Type *Tys[2] = { Ty, VTy };
7892     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7893     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7894     return Builder.CreateTrunc(Ops[0], Int16Ty);
7895   }
7896   case NEON::BI__builtin_neon_vminvq_s8: {
7897     Int = Intrinsic::aarch64_neon_sminv;
7898     Ty = Int32Ty;
7899     VTy = llvm::VectorType::get(Int8Ty, 16);
7900     llvm::Type *Tys[2] = { Ty, VTy };
7901     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7902     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7903     return Builder.CreateTrunc(Ops[0], Int8Ty);
7904   }
7905   case NEON::BI__builtin_neon_vminvq_s16: {
7906     Int = Intrinsic::aarch64_neon_sminv;
7907     Ty = Int32Ty;
7908     VTy = llvm::VectorType::get(Int16Ty, 8);
7909     llvm::Type *Tys[2] = { Ty, VTy };
7910     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7911     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7912     return Builder.CreateTrunc(Ops[0], Int16Ty);
7913   }
7914   case NEON::BI__builtin_neon_vminv_f16: {
7915     Int = Intrinsic::aarch64_neon_fminv;
7916     Ty = HalfTy;
7917     VTy = llvm::VectorType::get(HalfTy, 4);
7918     llvm::Type *Tys[2] = { Ty, VTy };
7919     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7920     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7921     return Builder.CreateTrunc(Ops[0], HalfTy);
7922   }
7923   case NEON::BI__builtin_neon_vminvq_f16: {
7924     Int = Intrinsic::aarch64_neon_fminv;
7925     Ty = HalfTy;
7926     VTy = llvm::VectorType::get(HalfTy, 8);
7927     llvm::Type *Tys[2] = { Ty, VTy };
7928     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7929     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
7930     return Builder.CreateTrunc(Ops[0], HalfTy);
7931   }
7932   case NEON::BI__builtin_neon_vmaxnmv_f16: {
7933     Int = Intrinsic::aarch64_neon_fmaxnmv;
7934     Ty = HalfTy;
7935     VTy = llvm::VectorType::get(HalfTy, 4);
7936     llvm::Type *Tys[2] = { Ty, VTy };
7937     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7938     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
7939     return Builder.CreateTrunc(Ops[0], HalfTy);
7940   }
7941   case NEON::BI__builtin_neon_vmaxnmvq_f16: {
7942     Int = Intrinsic::aarch64_neon_fmaxnmv;
7943     Ty = HalfTy;
7944     VTy = llvm::VectorType::get(HalfTy, 8);
7945     llvm::Type *Tys[2] = { Ty, VTy };
7946     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7947     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
7948     return Builder.CreateTrunc(Ops[0], HalfTy);
7949   }
7950   case NEON::BI__builtin_neon_vminnmv_f16: {
7951     Int = Intrinsic::aarch64_neon_fminnmv;
7952     Ty = HalfTy;
7953     VTy = llvm::VectorType::get(HalfTy, 4);
7954     llvm::Type *Tys[2] = { Ty, VTy };
7955     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7956     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
7957     return Builder.CreateTrunc(Ops[0], HalfTy);
7958   }
7959   case NEON::BI__builtin_neon_vminnmvq_f16: {
7960     Int = Intrinsic::aarch64_neon_fminnmv;
7961     Ty = HalfTy;
7962     VTy = llvm::VectorType::get(HalfTy, 8);
7963     llvm::Type *Tys[2] = { Ty, VTy };
7964     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7965     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
7966     return Builder.CreateTrunc(Ops[0], HalfTy);
7967   }
7968   case NEON::BI__builtin_neon_vmul_n_f64: {
7969     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7970     Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
7971     return Builder.CreateFMul(Ops[0], RHS);
7972   }
7973   case NEON::BI__builtin_neon_vaddlv_u8: {
7974     Int = Intrinsic::aarch64_neon_uaddlv;
7975     Ty = Int32Ty;
7976     VTy = llvm::VectorType::get(Int8Ty, 8);
7977     llvm::Type *Tys[2] = { Ty, VTy };
7978     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7979     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7980     return Builder.CreateTrunc(Ops[0], Int16Ty);
7981   }
7982   case NEON::BI__builtin_neon_vaddlv_u16: {
7983     Int = Intrinsic::aarch64_neon_uaddlv;
7984     Ty = Int32Ty;
7985     VTy = llvm::VectorType::get(Int16Ty, 4);
7986     llvm::Type *Tys[2] = { Ty, VTy };
7987     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7988     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7989   }
7990   case NEON::BI__builtin_neon_vaddlvq_u8: {
7991     Int = Intrinsic::aarch64_neon_uaddlv;
7992     Ty = Int32Ty;
7993     VTy = llvm::VectorType::get(Int8Ty, 16);
7994     llvm::Type *Tys[2] = { Ty, VTy };
7995     Ops.push_back(EmitScalarExpr(E->getArg(0)));
7996     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
7997     return Builder.CreateTrunc(Ops[0], Int16Ty);
7998   }
7999   case NEON::BI__builtin_neon_vaddlvq_u16: {
8000     Int = Intrinsic::aarch64_neon_uaddlv;
8001     Ty = Int32Ty;
8002     VTy = llvm::VectorType::get(Int16Ty, 8);
8003     llvm::Type *Tys[2] = { Ty, VTy };
8004     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8005     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
8006   }
8007   case NEON::BI__builtin_neon_vaddlv_s8: {
8008     Int = Intrinsic::aarch64_neon_saddlv;
8009     Ty = Int32Ty;
8010     VTy = llvm::VectorType::get(Int8Ty, 8);
8011     llvm::Type *Tys[2] = { Ty, VTy };
8012     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8013     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
8014     return Builder.CreateTrunc(Ops[0], Int16Ty);
8015   }
8016   case NEON::BI__builtin_neon_vaddlv_s16: {
8017     Int = Intrinsic::aarch64_neon_saddlv;
8018     Ty = Int32Ty;
8019     VTy = llvm::VectorType::get(Int16Ty, 4);
8020     llvm::Type *Tys[2] = { Ty, VTy };
8021     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8022     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
8023   }
8024   case NEON::BI__builtin_neon_vaddlvq_s8: {
8025     Int = Intrinsic::aarch64_neon_saddlv;
8026     Ty = Int32Ty;
8027     VTy = llvm::VectorType::get(Int8Ty, 16);
8028     llvm::Type *Tys[2] = { Ty, VTy };
8029     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8030     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
8031     return Builder.CreateTrunc(Ops[0], Int16Ty);
8032   }
8033   case NEON::BI__builtin_neon_vaddlvq_s16: {
8034     Int = Intrinsic::aarch64_neon_saddlv;
8035     Ty = Int32Ty;
8036     VTy = llvm::VectorType::get(Int16Ty, 8);
8037     llvm::Type *Tys[2] = { Ty, VTy };
8038     Ops.push_back(EmitScalarExpr(E->getArg(0)));
8039     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
8040   }
8041   case NEON::BI__builtin_neon_vsri_n_v:
8042   case NEON::BI__builtin_neon_vsriq_n_v: {
8043     Int = Intrinsic::aarch64_neon_vsri;
8044     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8045     return EmitNeonCall(Intrin, Ops, "vsri_n");
8046   }
8047   case NEON::BI__builtin_neon_vsli_n_v:
8048   case NEON::BI__builtin_neon_vsliq_n_v: {
8049     Int = Intrinsic::aarch64_neon_vsli;
8050     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8051     return EmitNeonCall(Intrin, Ops, "vsli_n");
8052   }
8053   case NEON::BI__builtin_neon_vsra_n_v:
8054   case NEON::BI__builtin_neon_vsraq_n_v:
8055     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8056     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
8057     return Builder.CreateAdd(Ops[0], Ops[1]);
8058   case NEON::BI__builtin_neon_vrsra_n_v:
8059   case NEON::BI__builtin_neon_vrsraq_n_v: {
8060     Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
8061     SmallVector<llvm::Value*,2> TmpOps;
8062     TmpOps.push_back(Ops[1]);
8063     TmpOps.push_back(Ops[2]);
8064     Function* F = CGM.getIntrinsic(Int, Ty);
8065     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
8066     Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
8067     return Builder.CreateAdd(Ops[0], tmp);
8068   }
8069   case NEON::BI__builtin_neon_vld1_v:
8070   case NEON::BI__builtin_neon_vld1q_v: {
8071     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8072     auto Alignment = CharUnits::fromQuantity(
8073         BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
8074     return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
8075   }
8076   case NEON::BI__builtin_neon_vst1_v:
8077   case NEON::BI__builtin_neon_vst1q_v:
8078     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8079     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8080     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8081   case NEON::BI__builtin_neon_vld1_lane_v:
8082   case NEON::BI__builtin_neon_vld1q_lane_v: {
8083     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8084     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8085     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8086     auto Alignment = CharUnits::fromQuantity(
8087         BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
8088     Ops[0] =
8089         Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8090     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
8091   }
8092   case NEON::BI__builtin_neon_vld1_dup_v:
8093   case NEON::BI__builtin_neon_vld1q_dup_v: {
8094     Value *V = UndefValue::get(Ty);
8095     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8096     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8097     auto Alignment = CharUnits::fromQuantity(
8098         BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
8099     Ops[0] =
8100         Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8101     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
8102     Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
8103     return EmitNeonSplat(Ops[0], CI);
8104   }
8105   case NEON::BI__builtin_neon_vst1_lane_v:
8106   case NEON::BI__builtin_neon_vst1q_lane_v:
8107     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8108     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
8109     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8110     return Builder.CreateDefaultAlignedStore(Ops[1],
8111                                              Builder.CreateBitCast(Ops[0], Ty));
8112   case NEON::BI__builtin_neon_vld2_v:
8113   case NEON::BI__builtin_neon_vld2q_v: {
8114     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8115     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8116     llvm::Type *Tys[2] = { VTy, PTy };
8117     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
8118     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
8119     Ops[0] = Builder.CreateBitCast(Ops[0],
8120                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8121     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8122   }
8123   case NEON::BI__builtin_neon_vld3_v:
8124   case NEON::BI__builtin_neon_vld3q_v: {
8125     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8126     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8127     llvm::Type *Tys[2] = { VTy, PTy };
8128     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
8129     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
8130     Ops[0] = Builder.CreateBitCast(Ops[0],
8131                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8132     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8133   }
8134   case NEON::BI__builtin_neon_vld4_v:
8135   case NEON::BI__builtin_neon_vld4q_v: {
8136     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8137     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8138     llvm::Type *Tys[2] = { VTy, PTy };
8139     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
8140     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
8141     Ops[0] = Builder.CreateBitCast(Ops[0],
8142                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8143     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8144   }
8145   case NEON::BI__builtin_neon_vld2_dup_v:
8146   case NEON::BI__builtin_neon_vld2q_dup_v: {
8147     llvm::Type *PTy =
8148       llvm::PointerType::getUnqual(VTy->getElementType());
8149     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8150     llvm::Type *Tys[2] = { VTy, PTy };
8151     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
8152     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
8153     Ops[0] = Builder.CreateBitCast(Ops[0],
8154                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8155     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8156   }
8157   case NEON::BI__builtin_neon_vld3_dup_v:
8158   case NEON::BI__builtin_neon_vld3q_dup_v: {
8159     llvm::Type *PTy =
8160       llvm::PointerType::getUnqual(VTy->getElementType());
8161     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8162     llvm::Type *Tys[2] = { VTy, PTy };
8163     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
8164     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
8165     Ops[0] = Builder.CreateBitCast(Ops[0],
8166                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8167     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8168   }
8169   case NEON::BI__builtin_neon_vld4_dup_v:
8170   case NEON::BI__builtin_neon_vld4q_dup_v: {
8171     llvm::Type *PTy =
8172       llvm::PointerType::getUnqual(VTy->getElementType());
8173     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8174     llvm::Type *Tys[2] = { VTy, PTy };
8175     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
8176     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
8177     Ops[0] = Builder.CreateBitCast(Ops[0],
8178                 llvm::PointerType::getUnqual(Ops[1]->getType()));
8179     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8180   }
8181   case NEON::BI__builtin_neon_vld2_lane_v:
8182   case NEON::BI__builtin_neon_vld2q_lane_v: {
8183     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8184     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
8185     Ops.push_back(Ops[1]);
8186     Ops.erase(Ops.begin()+1);
8187     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8188     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8189     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8190     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
8191     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8192     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8193     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8194   }
8195   case NEON::BI__builtin_neon_vld3_lane_v:
8196   case NEON::BI__builtin_neon_vld3q_lane_v: {
8197     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8198     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
8199     Ops.push_back(Ops[1]);
8200     Ops.erase(Ops.begin()+1);
8201     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8202     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8203     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8204     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8205     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
8206     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8207     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8208     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8209   }
8210   case NEON::BI__builtin_neon_vld4_lane_v:
8211   case NEON::BI__builtin_neon_vld4q_lane_v: {
8212     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8213     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
8214     Ops.push_back(Ops[1]);
8215     Ops.erase(Ops.begin()+1);
8216     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8217     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8218     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8219     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
8220     Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
8221     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
8222     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8223     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8224     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8225   }
8226   case NEON::BI__builtin_neon_vst2_v:
8227   case NEON::BI__builtin_neon_vst2q_v: {
8228     Ops.push_back(Ops[0]);
8229     Ops.erase(Ops.begin());
8230     llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
8231     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
8232                         Ops, "");
8233   }
8234   case NEON::BI__builtin_neon_vst2_lane_v:
8235   case NEON::BI__builtin_neon_vst2q_lane_v: {
8236     Ops.push_back(Ops[0]);
8237     Ops.erase(Ops.begin());
8238     Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
8239     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8240     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
8241                         Ops, "");
8242   }
8243   case NEON::BI__builtin_neon_vst3_v:
8244   case NEON::BI__builtin_neon_vst3q_v: {
8245     Ops.push_back(Ops[0]);
8246     Ops.erase(Ops.begin());
8247     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8248     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
8249                         Ops, "");
8250   }
8251   case NEON::BI__builtin_neon_vst3_lane_v:
8252   case NEON::BI__builtin_neon_vst3q_lane_v: {
8253     Ops.push_back(Ops[0]);
8254     Ops.erase(Ops.begin());
8255     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8256     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8257     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
8258                         Ops, "");
8259   }
8260   case NEON::BI__builtin_neon_vst4_v:
8261   case NEON::BI__builtin_neon_vst4q_v: {
8262     Ops.push_back(Ops[0]);
8263     Ops.erase(Ops.begin());
8264     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8265     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
8266                         Ops, "");
8267   }
8268   case NEON::BI__builtin_neon_vst4_lane_v:
8269   case NEON::BI__builtin_neon_vst4q_lane_v: {
8270     Ops.push_back(Ops[0]);
8271     Ops.erase(Ops.begin());
8272     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8273     llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
8274     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
8275                         Ops, "");
8276   }
8277   case NEON::BI__builtin_neon_vtrn_v:
8278   case NEON::BI__builtin_neon_vtrnq_v: {
8279     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8280     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8281     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8282     Value *SV = nullptr;
8283 
8284     for (unsigned vi = 0; vi != 2; ++vi) {
8285       SmallVector<uint32_t, 16> Indices;
8286       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8287         Indices.push_back(i+vi);
8288         Indices.push_back(i+e+vi);
8289       }
8290       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8291       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
8292       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8293     }
8294     return SV;
8295   }
8296   case NEON::BI__builtin_neon_vuzp_v:
8297   case NEON::BI__builtin_neon_vuzpq_v: {
8298     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8299     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8300     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8301     Value *SV = nullptr;
8302 
8303     for (unsigned vi = 0; vi != 2; ++vi) {
8304       SmallVector<uint32_t, 16> Indices;
8305       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8306         Indices.push_back(2*i+vi);
8307 
8308       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8309       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
8310       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8311     }
8312     return SV;
8313   }
8314   case NEON::BI__builtin_neon_vzip_v:
8315   case NEON::BI__builtin_neon_vzipq_v: {
8316     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8317     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8318     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8319     Value *SV = nullptr;
8320 
8321     for (unsigned vi = 0; vi != 2; ++vi) {
8322       SmallVector<uint32_t, 16> Indices;
8323       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8324         Indices.push_back((i + vi*e) >> 1);
8325         Indices.push_back(((i + vi*e) >> 1)+e);
8326       }
8327       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8328       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
8329       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8330     }
8331     return SV;
8332   }
8333   case NEON::BI__builtin_neon_vqtbl1q_v: {
8334     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
8335                         Ops, "vtbl1");
8336   }
8337   case NEON::BI__builtin_neon_vqtbl2q_v: {
8338     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
8339                         Ops, "vtbl2");
8340   }
8341   case NEON::BI__builtin_neon_vqtbl3q_v: {
8342     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
8343                         Ops, "vtbl3");
8344   }
8345   case NEON::BI__builtin_neon_vqtbl4q_v: {
8346     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
8347                         Ops, "vtbl4");
8348   }
8349   case NEON::BI__builtin_neon_vqtbx1q_v: {
8350     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
8351                         Ops, "vtbx1");
8352   }
8353   case NEON::BI__builtin_neon_vqtbx2q_v: {
8354     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
8355                         Ops, "vtbx2");
8356   }
8357   case NEON::BI__builtin_neon_vqtbx3q_v: {
8358     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
8359                         Ops, "vtbx3");
8360   }
8361   case NEON::BI__builtin_neon_vqtbx4q_v: {
8362     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
8363                         Ops, "vtbx4");
8364   }
8365   case NEON::BI__builtin_neon_vsqadd_v:
8366   case NEON::BI__builtin_neon_vsqaddq_v: {
8367     Int = Intrinsic::aarch64_neon_usqadd;
8368     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
8369   }
8370   case NEON::BI__builtin_neon_vuqadd_v:
8371   case NEON::BI__builtin_neon_vuqaddq_v: {
8372     Int = Intrinsic::aarch64_neon_suqadd;
8373     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
8374   }
8375   case AArch64::BI__iso_volatile_load8:
8376   case AArch64::BI__iso_volatile_load16:
8377   case AArch64::BI__iso_volatile_load32:
8378   case AArch64::BI__iso_volatile_load64:
8379     return EmitISOVolatileLoad(E);
8380   case AArch64::BI__iso_volatile_store8:
8381   case AArch64::BI__iso_volatile_store16:
8382   case AArch64::BI__iso_volatile_store32:
8383   case AArch64::BI__iso_volatile_store64:
8384     return EmitISOVolatileStore(E);
8385   }
8386 }
8387 
8388 llvm::Value *CodeGenFunction::
8389 BuildVector(ArrayRef<llvm::Value*> Ops) {
8390   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
8391          "Not a power-of-two sized vector!");
8392   bool AllConstants = true;
8393   for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
8394     AllConstants &= isa<Constant>(Ops[i]);
8395 
8396   // If this is a constant vector, create a ConstantVector.
8397   if (AllConstants) {
8398     SmallVector<llvm::Constant*, 16> CstOps;
8399     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
8400       CstOps.push_back(cast<Constant>(Ops[i]));
8401     return llvm::ConstantVector::get(CstOps);
8402   }
8403 
8404   // Otherwise, insertelement the values to build the vector.
8405   Value *Result =
8406     llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
8407 
8408   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
8409     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
8410 
8411   return Result;
8412 }
8413 
8414 // Convert the mask from an integer type to a vector of i1.
8415 static Value *getMaskVecValue(CodeGenFunction &CGF, Value *Mask,
8416                               unsigned NumElts) {
8417 
8418   llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(),
8419                          cast<IntegerType>(Mask->getType())->getBitWidth());
8420   Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy);
8421 
8422   // If we have less than 8 elements, then the starting mask was an i8 and
8423   // we need to extract down to the right number of elements.
8424   if (NumElts < 8) {
8425     uint32_t Indices[4];
8426     for (unsigned i = 0; i != NumElts; ++i)
8427       Indices[i] = i;
8428     MaskVec = CGF.Builder.CreateShuffleVector(MaskVec, MaskVec,
8429                                              makeArrayRef(Indices, NumElts),
8430                                              "extract");
8431   }
8432   return MaskVec;
8433 }
8434 
8435 static Value *EmitX86MaskedStore(CodeGenFunction &CGF,
8436                                  ArrayRef<Value *> Ops,
8437                                  unsigned Align) {
8438   // Cast the pointer to right type.
8439   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
8440                                llvm::PointerType::getUnqual(Ops[1]->getType()));
8441 
8442   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
8443                                    Ops[1]->getType()->getVectorNumElements());
8444 
8445   return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Align, MaskVec);
8446 }
8447 
8448 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF,
8449                                 ArrayRef<Value *> Ops, unsigned Align) {
8450   // Cast the pointer to right type.
8451   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
8452                                llvm::PointerType::getUnqual(Ops[1]->getType()));
8453 
8454   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
8455                                    Ops[1]->getType()->getVectorNumElements());
8456 
8457   return CGF.Builder.CreateMaskedLoad(Ptr, Align, MaskVec, Ops[1]);
8458 }
8459 
8460 static Value *EmitX86ExpandLoad(CodeGenFunction &CGF,
8461                                 ArrayRef<Value *> Ops) {
8462   llvm::Type *ResultTy = Ops[1]->getType();
8463   llvm::Type *PtrTy = ResultTy->getVectorElementType();
8464 
8465   // Cast the pointer to element type.
8466   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
8467                                          llvm::PointerType::getUnqual(PtrTy));
8468 
8469   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
8470                                    ResultTy->getVectorNumElements());
8471 
8472   llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_expandload,
8473                                            ResultTy);
8474   return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
8475 }
8476 
8477 static Value *EmitX86CompressStore(CodeGenFunction &CGF,
8478                                    ArrayRef<Value *> Ops) {
8479   llvm::Type *ResultTy = Ops[1]->getType();
8480   llvm::Type *PtrTy = ResultTy->getVectorElementType();
8481 
8482   // Cast the pointer to element type.
8483   Value *Ptr = CGF.Builder.CreateBitCast(Ops[0],
8484                                          llvm::PointerType::getUnqual(PtrTy));
8485 
8486   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
8487                                    ResultTy->getVectorNumElements());
8488 
8489   llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_compressstore,
8490                                            ResultTy);
8491   return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
8492 }
8493 
8494 static Value *EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc,
8495                               unsigned NumElts, ArrayRef<Value *> Ops,
8496                               bool InvertLHS = false) {
8497   Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts);
8498   Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts);
8499 
8500   if (InvertLHS)
8501     LHS = CGF.Builder.CreateNot(LHS);
8502 
8503   return CGF.Builder.CreateBitCast(CGF.Builder.CreateBinOp(Opc, LHS, RHS),
8504                                   CGF.Builder.getIntNTy(std::max(NumElts, 8U)));
8505 }
8506 
8507 static Value *EmitX86Select(CodeGenFunction &CGF,
8508                             Value *Mask, Value *Op0, Value *Op1) {
8509 
8510   // If the mask is all ones just return first argument.
8511   if (const auto *C = dyn_cast<Constant>(Mask))
8512     if (C->isAllOnesValue())
8513       return Op0;
8514 
8515   Mask = getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
8516 
8517   return CGF.Builder.CreateSelect(Mask, Op0, Op1);
8518 }
8519 
8520 static Value *EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp,
8521                                          unsigned NumElts, Value *MaskIn) {
8522   if (MaskIn) {
8523     const auto *C = dyn_cast<Constant>(MaskIn);
8524     if (!C || !C->isAllOnesValue())
8525       Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, MaskIn, NumElts));
8526   }
8527 
8528   if (NumElts < 8) {
8529     uint32_t Indices[8];
8530     for (unsigned i = 0; i != NumElts; ++i)
8531       Indices[i] = i;
8532     for (unsigned i = NumElts; i != 8; ++i)
8533       Indices[i] = i % NumElts + NumElts;
8534     Cmp = CGF.Builder.CreateShuffleVector(
8535         Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
8536   }
8537 
8538   return CGF.Builder.CreateBitCast(Cmp,
8539                                    IntegerType::get(CGF.getLLVMContext(),
8540                                                     std::max(NumElts, 8U)));
8541 }
8542 
8543 static Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC,
8544                                    bool Signed, ArrayRef<Value *> Ops) {
8545   assert((Ops.size() == 2 || Ops.size() == 4) &&
8546          "Unexpected number of arguments");
8547   unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
8548   Value *Cmp;
8549 
8550   if (CC == 3) {
8551     Cmp = Constant::getNullValue(
8552                        llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
8553   } else if (CC == 7) {
8554     Cmp = Constant::getAllOnesValue(
8555                        llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
8556   } else {
8557     ICmpInst::Predicate Pred;
8558     switch (CC) {
8559     default: llvm_unreachable("Unknown condition code");
8560     case 0: Pred = ICmpInst::ICMP_EQ;  break;
8561     case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
8562     case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
8563     case 4: Pred = ICmpInst::ICMP_NE;  break;
8564     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
8565     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
8566     }
8567     Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
8568   }
8569 
8570   Value *MaskIn = nullptr;
8571   if (Ops.size() == 4)
8572     MaskIn = Ops[3];
8573 
8574   return EmitX86MaskedCompareResult(CGF, Cmp, NumElts, MaskIn);
8575 }
8576 
8577 static Value *EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In) {
8578   Value *Zero = Constant::getNullValue(In->getType());
8579   return EmitX86MaskedCompare(CGF, 1, true, { In, Zero });
8580 }
8581 
8582 static Value *EmitX86Abs(CodeGenFunction &CGF, ArrayRef<Value *> Ops) {
8583 
8584   llvm::Type *Ty = Ops[0]->getType();
8585   Value *Zero = llvm::Constant::getNullValue(Ty);
8586   Value *Sub = CGF.Builder.CreateSub(Zero, Ops[0]);
8587   Value *Cmp = CGF.Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero);
8588   Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Sub);
8589   return Res;
8590 }
8591 
8592 static Value *EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred,
8593                             ArrayRef<Value *> Ops) {
8594   Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
8595   Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
8596 
8597   assert(Ops.size() == 2);
8598   return Res;
8599 }
8600 
8601 // Lowers X86 FMA intrinsics to IR.
8602 static Value *EmitX86FMAExpr(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
8603                              unsigned BuiltinID, bool IsAddSub) {
8604 
8605   bool Subtract = false;
8606   Intrinsic::ID IID = Intrinsic::not_intrinsic;
8607   switch (BuiltinID) {
8608   default: break;
8609   case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
8610     Subtract = true;
8611     LLVM_FALLTHROUGH;
8612   case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
8613   case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
8614   case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
8615     IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512; break;
8616   case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
8617     Subtract = true;
8618     LLVM_FALLTHROUGH;
8619   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
8620   case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
8621   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
8622     IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512; break;
8623   case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
8624     Subtract = true;
8625     LLVM_FALLTHROUGH;
8626   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
8627   case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
8628   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
8629     IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
8630     break;
8631   case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
8632     Subtract = true;
8633     LLVM_FALLTHROUGH;
8634   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
8635   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
8636   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
8637     IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
8638     break;
8639   }
8640 
8641   Value *A = Ops[0];
8642   Value *B = Ops[1];
8643   Value *C = Ops[2];
8644 
8645   if (Subtract)
8646     C = CGF.Builder.CreateFNeg(C);
8647 
8648   Value *Res;
8649 
8650   // Only handle in case of _MM_FROUND_CUR_DIRECTION/4 (no rounding).
8651   if (IID != Intrinsic::not_intrinsic &&
8652       cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4) {
8653     Function *Intr = CGF.CGM.getIntrinsic(IID);
8654     Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() });
8655   } else {
8656     llvm::Type *Ty = A->getType();
8657     Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ty);
8658     Res = CGF.Builder.CreateCall(FMA, {A, B, C} );
8659 
8660     if (IsAddSub) {
8661       // Negate even elts in C using a mask.
8662       unsigned NumElts = Ty->getVectorNumElements();
8663       SmallVector<Constant *, 16> NMask;
8664       Constant *Zero = ConstantInt::get(CGF.Builder.getInt1Ty(), 0);
8665       Constant *One = ConstantInt::get(CGF.Builder.getInt1Ty(), 1);
8666       for (unsigned i = 0; i < NumElts; ++i) {
8667         NMask.push_back(i % 2 == 0 ? One : Zero);
8668       }
8669       Value *NegMask = ConstantVector::get(NMask);
8670 
8671       Value *NegC = CGF.Builder.CreateFNeg(C);
8672       Value *FMSub = CGF.Builder.CreateCall(FMA, {A, B, NegC} );
8673       Res = CGF.Builder.CreateSelect(NegMask, FMSub, Res);
8674     }
8675   }
8676 
8677   // Handle any required masking.
8678   Value *MaskFalseVal = nullptr;
8679   switch (BuiltinID) {
8680   case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
8681   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
8682   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
8683   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
8684     MaskFalseVal = Ops[0];
8685     break;
8686   case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
8687   case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
8688   case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
8689   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
8690     MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
8691     break;
8692   case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
8693   case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
8694   case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
8695   case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
8696   case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
8697   case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
8698   case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
8699   case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
8700     MaskFalseVal = Ops[2];
8701     break;
8702   }
8703 
8704   if (MaskFalseVal)
8705     return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal);
8706 
8707   return Res;
8708 }
8709 
8710 static Value *EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned,
8711                            ArrayRef<Value *> Ops) {
8712   llvm::Type *Ty = Ops[0]->getType();
8713   // Arguments have a vXi32 type so cast to vXi64.
8714   Ty = llvm::VectorType::get(CGF.Int64Ty,
8715                              Ty->getPrimitiveSizeInBits() / 64);
8716   Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty);
8717   Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty);
8718 
8719   if (IsSigned) {
8720     // Shift left then arithmetic shift right.
8721     Constant *ShiftAmt = ConstantInt::get(Ty, 32);
8722     LHS = CGF.Builder.CreateShl(LHS, ShiftAmt);
8723     LHS = CGF.Builder.CreateAShr(LHS, ShiftAmt);
8724     RHS = CGF.Builder.CreateShl(RHS, ShiftAmt);
8725     RHS = CGF.Builder.CreateAShr(RHS, ShiftAmt);
8726   } else {
8727     // Clear the upper bits.
8728     Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
8729     LHS = CGF.Builder.CreateAnd(LHS, Mask);
8730     RHS = CGF.Builder.CreateAnd(RHS, Mask);
8731   }
8732 
8733   return CGF.Builder.CreateMul(LHS, RHS);
8734 }
8735 
8736 // Emit a masked pternlog intrinsic. This only exists because the header has to
8737 // use a macro and we aren't able to pass the input argument to a pternlog
8738 // builtin and a select builtin without evaluating it twice.
8739 static Value *EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask,
8740                              ArrayRef<Value *> Ops) {
8741   llvm::Type *Ty = Ops[0]->getType();
8742 
8743   unsigned VecWidth = Ty->getPrimitiveSizeInBits();
8744   unsigned EltWidth = Ty->getScalarSizeInBits();
8745   Intrinsic::ID IID;
8746   if (VecWidth == 128 && EltWidth == 32)
8747     IID = Intrinsic::x86_avx512_pternlog_d_128;
8748   else if (VecWidth == 256 && EltWidth == 32)
8749     IID = Intrinsic::x86_avx512_pternlog_d_256;
8750   else if (VecWidth == 512 && EltWidth == 32)
8751     IID = Intrinsic::x86_avx512_pternlog_d_512;
8752   else if (VecWidth == 128 && EltWidth == 64)
8753     IID = Intrinsic::x86_avx512_pternlog_q_128;
8754   else if (VecWidth == 256 && EltWidth == 64)
8755     IID = Intrinsic::x86_avx512_pternlog_q_256;
8756   else if (VecWidth == 512 && EltWidth == 64)
8757     IID = Intrinsic::x86_avx512_pternlog_q_512;
8758   else
8759     llvm_unreachable("Unexpected intrinsic");
8760 
8761   Value *Ternlog = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IID),
8762                                           Ops.drop_back());
8763   Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
8764   return EmitX86Select(CGF, Ops[4], Ternlog, PassThru);
8765 }
8766 
8767 static Value *EmitX86SExtMask(CodeGenFunction &CGF, Value *Op,
8768                               llvm::Type *DstTy) {
8769   unsigned NumberOfElements = DstTy->getVectorNumElements();
8770   Value *Mask = getMaskVecValue(CGF, Op, NumberOfElements);
8771   return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
8772 }
8773 
8774 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
8775   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
8776   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
8777   return EmitX86CpuIs(CPUStr);
8778 }
8779 
8780 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
8781 
8782   llvm::Type *Int32Ty = Builder.getInt32Ty();
8783 
8784   // Matching the struct layout from the compiler-rt/libgcc structure that is
8785   // filled in:
8786   // unsigned int __cpu_vendor;
8787   // unsigned int __cpu_type;
8788   // unsigned int __cpu_subtype;
8789   // unsigned int __cpu_features[1];
8790   llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
8791                                           llvm::ArrayType::get(Int32Ty, 1));
8792 
8793   // Grab the global __cpu_model.
8794   llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
8795 
8796   // Calculate the index needed to access the correct field based on the
8797   // range. Also adjust the expected value.
8798   unsigned Index;
8799   unsigned Value;
8800   std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
8801 #define X86_VENDOR(ENUM, STRING)                                               \
8802   .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
8803 #define X86_CPU_TYPE_COMPAT_WITH_ALIAS(ARCHNAME, ENUM, STR, ALIAS)             \
8804   .Cases(STR, ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
8805 #define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR)                               \
8806   .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
8807 #define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR)                            \
8808   .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
8809 #include "llvm/Support/X86TargetParser.def"
8810                                .Default({0, 0});
8811   assert(Value != 0 && "Invalid CPUStr passed to CpuIs");
8812 
8813   // Grab the appropriate field from __cpu_model.
8814   llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
8815                          ConstantInt::get(Int32Ty, Index)};
8816   llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
8817   CpuValue = Builder.CreateAlignedLoad(CpuValue, CharUnits::fromQuantity(4));
8818 
8819   // Check the value of the field against the requested value.
8820   return Builder.CreateICmpEQ(CpuValue,
8821                                   llvm::ConstantInt::get(Int32Ty, Value));
8822 }
8823 
8824 Value *CodeGenFunction::EmitX86CpuSupports(const CallExpr *E) {
8825   const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
8826   StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
8827   return EmitX86CpuSupports(FeatureStr);
8828 }
8829 
8830 Value *CodeGenFunction::EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs) {
8831   // Processor features and mapping to processor feature value.
8832 
8833   uint32_t FeaturesMask = 0;
8834 
8835   for (const StringRef &FeatureStr : FeatureStrs) {
8836     unsigned Feature =
8837         StringSwitch<unsigned>(FeatureStr)
8838 #define X86_FEATURE_COMPAT(VAL, ENUM, STR) .Case(STR, VAL)
8839 #include "llvm/Support/X86TargetParser.def"
8840         ;
8841     FeaturesMask |= (1U << Feature);
8842   }
8843 
8844   // Matching the struct layout from the compiler-rt/libgcc structure that is
8845   // filled in:
8846   // unsigned int __cpu_vendor;
8847   // unsigned int __cpu_type;
8848   // unsigned int __cpu_subtype;
8849   // unsigned int __cpu_features[1];
8850   llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
8851                                           llvm::ArrayType::get(Int32Ty, 1));
8852 
8853   // Grab the global __cpu_model.
8854   llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
8855 
8856   // Grab the first (0th) element from the field __cpu_features off of the
8857   // global in the struct STy.
8858   Value *Idxs[] = {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 3),
8859                    ConstantInt::get(Int32Ty, 0)};
8860   Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
8861   Value *Features =
8862       Builder.CreateAlignedLoad(CpuFeatures, CharUnits::fromQuantity(4));
8863 
8864   // Check the value of the bit corresponding to the feature requested.
8865   Value *Bitset = Builder.CreateAnd(
8866       Features, llvm::ConstantInt::get(Int32Ty, FeaturesMask));
8867   return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0));
8868 }
8869 
8870 Value *CodeGenFunction::EmitX86CpuInit() {
8871   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
8872                                                     /*Variadic*/ false);
8873   llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, "__cpu_indicator_init");
8874   return Builder.CreateCall(Func);
8875 }
8876 
8877 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
8878                                            const CallExpr *E) {
8879   if (BuiltinID == X86::BI__builtin_cpu_is)
8880     return EmitX86CpuIs(E);
8881   if (BuiltinID == X86::BI__builtin_cpu_supports)
8882     return EmitX86CpuSupports(E);
8883   if (BuiltinID == X86::BI__builtin_cpu_init)
8884     return EmitX86CpuInit();
8885 
8886   SmallVector<Value*, 4> Ops;
8887 
8888   // Find out if any arguments are required to be integer constant expressions.
8889   unsigned ICEArguments = 0;
8890   ASTContext::GetBuiltinTypeError Error;
8891   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
8892   assert(Error == ASTContext::GE_None && "Should not codegen an error");
8893 
8894   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
8895     // If this is a normal argument, just emit it as a scalar.
8896     if ((ICEArguments & (1 << i)) == 0) {
8897       Ops.push_back(EmitScalarExpr(E->getArg(i)));
8898       continue;
8899     }
8900 
8901     // If this is required to be a constant, constant fold it so that we know
8902     // that the generated intrinsic gets a ConstantInt.
8903     llvm::APSInt Result;
8904     bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
8905     assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
8906     Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
8907   }
8908 
8909   // These exist so that the builtin that takes an immediate can be bounds
8910   // checked by clang to avoid passing bad immediates to the backend. Since
8911   // AVX has a larger immediate than SSE we would need separate builtins to
8912   // do the different bounds checking. Rather than create a clang specific
8913   // SSE only builtin, this implements eight separate builtins to match gcc
8914   // implementation.
8915   auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) {
8916     Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
8917     llvm::Function *F = CGM.getIntrinsic(ID);
8918     return Builder.CreateCall(F, Ops);
8919   };
8920 
8921   // For the vector forms of FP comparisons, translate the builtins directly to
8922   // IR.
8923   // TODO: The builtins could be removed if the SSE header files used vector
8924   // extension comparisons directly (vector ordered/unordered may need
8925   // additional support via __builtin_isnan()).
8926   auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) {
8927     Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
8928     llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
8929     llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
8930     Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
8931     return Builder.CreateBitCast(Sext, FPVecTy);
8932   };
8933 
8934   switch (BuiltinID) {
8935   default: return nullptr;
8936   case X86::BI_mm_prefetch: {
8937     Value *Address = Ops[0];
8938     ConstantInt *C = cast<ConstantInt>(Ops[1]);
8939     Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
8940     Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
8941     Value *Data = ConstantInt::get(Int32Ty, 1);
8942     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
8943     return Builder.CreateCall(F, {Address, RW, Locality, Data});
8944   }
8945   case X86::BI_mm_clflush: {
8946     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
8947                               Ops[0]);
8948   }
8949   case X86::BI_mm_lfence: {
8950     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
8951   }
8952   case X86::BI_mm_mfence: {
8953     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
8954   }
8955   case X86::BI_mm_sfence: {
8956     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
8957   }
8958   case X86::BI_mm_pause: {
8959     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
8960   }
8961   case X86::BI__rdtsc: {
8962     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
8963   }
8964   case X86::BI__builtin_ia32_undef128:
8965   case X86::BI__builtin_ia32_undef256:
8966   case X86::BI__builtin_ia32_undef512:
8967     // The x86 definition of "undef" is not the same as the LLVM definition
8968     // (PR32176). We leave optimizing away an unnecessary zero constant to the
8969     // IR optimizer and backend.
8970     // TODO: If we had a "freeze" IR instruction to generate a fixed undef
8971     // value, we should use that here instead of a zero.
8972     return llvm::Constant::getNullValue(ConvertType(E->getType()));
8973   case X86::BI__builtin_ia32_vec_init_v8qi:
8974   case X86::BI__builtin_ia32_vec_init_v4hi:
8975   case X86::BI__builtin_ia32_vec_init_v2si:
8976     return Builder.CreateBitCast(BuildVector(Ops),
8977                                  llvm::Type::getX86_MMXTy(getLLVMContext()));
8978   case X86::BI__builtin_ia32_vec_ext_v2si:
8979   case X86::BI__builtin_ia32_vec_ext_v16qi:
8980   case X86::BI__builtin_ia32_vec_ext_v8hi:
8981   case X86::BI__builtin_ia32_vec_ext_v4si:
8982   case X86::BI__builtin_ia32_vec_ext_v4sf:
8983   case X86::BI__builtin_ia32_vec_ext_v2di:
8984   case X86::BI__builtin_ia32_vec_ext_v32qi:
8985   case X86::BI__builtin_ia32_vec_ext_v16hi:
8986   case X86::BI__builtin_ia32_vec_ext_v8si:
8987   case X86::BI__builtin_ia32_vec_ext_v4di: {
8988     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
8989     uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
8990     Index &= NumElts - 1;
8991     // These builtins exist so we can ensure the index is an ICE and in range.
8992     // Otherwise we could just do this in the header file.
8993     return Builder.CreateExtractElement(Ops[0], Index);
8994   }
8995   case X86::BI__builtin_ia32_vec_set_v16qi:
8996   case X86::BI__builtin_ia32_vec_set_v8hi:
8997   case X86::BI__builtin_ia32_vec_set_v4si:
8998   case X86::BI__builtin_ia32_vec_set_v2di:
8999   case X86::BI__builtin_ia32_vec_set_v32qi:
9000   case X86::BI__builtin_ia32_vec_set_v16hi:
9001   case X86::BI__builtin_ia32_vec_set_v8si:
9002   case X86::BI__builtin_ia32_vec_set_v4di: {
9003     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9004     unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
9005     Index &= NumElts - 1;
9006     // These builtins exist so we can ensure the index is an ICE and in range.
9007     // Otherwise we could just do this in the header file.
9008     return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
9009   }
9010   case X86::BI_mm_setcsr:
9011   case X86::BI__builtin_ia32_ldmxcsr: {
9012     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
9013     Builder.CreateStore(Ops[0], Tmp);
9014     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
9015                           Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
9016   }
9017   case X86::BI_mm_getcsr:
9018   case X86::BI__builtin_ia32_stmxcsr: {
9019     Address Tmp = CreateMemTemp(E->getType());
9020     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
9021                        Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
9022     return Builder.CreateLoad(Tmp, "stmxcsr");
9023   }
9024   case X86::BI__builtin_ia32_xsave:
9025   case X86::BI__builtin_ia32_xsave64:
9026   case X86::BI__builtin_ia32_xrstor:
9027   case X86::BI__builtin_ia32_xrstor64:
9028   case X86::BI__builtin_ia32_xsaveopt:
9029   case X86::BI__builtin_ia32_xsaveopt64:
9030   case X86::BI__builtin_ia32_xrstors:
9031   case X86::BI__builtin_ia32_xrstors64:
9032   case X86::BI__builtin_ia32_xsavec:
9033   case X86::BI__builtin_ia32_xsavec64:
9034   case X86::BI__builtin_ia32_xsaves:
9035   case X86::BI__builtin_ia32_xsaves64: {
9036     Intrinsic::ID ID;
9037 #define INTRINSIC_X86_XSAVE_ID(NAME) \
9038     case X86::BI__builtin_ia32_##NAME: \
9039       ID = Intrinsic::x86_##NAME; \
9040       break
9041     switch (BuiltinID) {
9042     default: llvm_unreachable("Unsupported intrinsic!");
9043     INTRINSIC_X86_XSAVE_ID(xsave);
9044     INTRINSIC_X86_XSAVE_ID(xsave64);
9045     INTRINSIC_X86_XSAVE_ID(xrstor);
9046     INTRINSIC_X86_XSAVE_ID(xrstor64);
9047     INTRINSIC_X86_XSAVE_ID(xsaveopt);
9048     INTRINSIC_X86_XSAVE_ID(xsaveopt64);
9049     INTRINSIC_X86_XSAVE_ID(xrstors);
9050     INTRINSIC_X86_XSAVE_ID(xrstors64);
9051     INTRINSIC_X86_XSAVE_ID(xsavec);
9052     INTRINSIC_X86_XSAVE_ID(xsavec64);
9053     INTRINSIC_X86_XSAVE_ID(xsaves);
9054     INTRINSIC_X86_XSAVE_ID(xsaves64);
9055     }
9056 #undef INTRINSIC_X86_XSAVE_ID
9057     Value *Mhi = Builder.CreateTrunc(
9058       Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
9059     Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
9060     Ops[1] = Mhi;
9061     Ops.push_back(Mlo);
9062     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
9063   }
9064   case X86::BI__builtin_ia32_storedqudi128_mask:
9065   case X86::BI__builtin_ia32_storedqusi128_mask:
9066   case X86::BI__builtin_ia32_storedquhi128_mask:
9067   case X86::BI__builtin_ia32_storedquqi128_mask:
9068   case X86::BI__builtin_ia32_storeupd128_mask:
9069   case X86::BI__builtin_ia32_storeups128_mask:
9070   case X86::BI__builtin_ia32_storedqudi256_mask:
9071   case X86::BI__builtin_ia32_storedqusi256_mask:
9072   case X86::BI__builtin_ia32_storedquhi256_mask:
9073   case X86::BI__builtin_ia32_storedquqi256_mask:
9074   case X86::BI__builtin_ia32_storeupd256_mask:
9075   case X86::BI__builtin_ia32_storeups256_mask:
9076   case X86::BI__builtin_ia32_storedqudi512_mask:
9077   case X86::BI__builtin_ia32_storedqusi512_mask:
9078   case X86::BI__builtin_ia32_storedquhi512_mask:
9079   case X86::BI__builtin_ia32_storedquqi512_mask:
9080   case X86::BI__builtin_ia32_storeupd512_mask:
9081   case X86::BI__builtin_ia32_storeups512_mask:
9082     return EmitX86MaskedStore(*this, Ops, 1);
9083 
9084   case X86::BI__builtin_ia32_storess128_mask:
9085   case X86::BI__builtin_ia32_storesd128_mask: {
9086     return EmitX86MaskedStore(*this, Ops, 1);
9087   }
9088   case X86::BI__builtin_ia32_vpopcntb_128:
9089   case X86::BI__builtin_ia32_vpopcntd_128:
9090   case X86::BI__builtin_ia32_vpopcntq_128:
9091   case X86::BI__builtin_ia32_vpopcntw_128:
9092   case X86::BI__builtin_ia32_vpopcntb_256:
9093   case X86::BI__builtin_ia32_vpopcntd_256:
9094   case X86::BI__builtin_ia32_vpopcntq_256:
9095   case X86::BI__builtin_ia32_vpopcntw_256:
9096   case X86::BI__builtin_ia32_vpopcntb_512:
9097   case X86::BI__builtin_ia32_vpopcntd_512:
9098   case X86::BI__builtin_ia32_vpopcntq_512:
9099   case X86::BI__builtin_ia32_vpopcntw_512: {
9100     llvm::Type *ResultType = ConvertType(E->getType());
9101     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
9102     return Builder.CreateCall(F, Ops);
9103   }
9104   case X86::BI__builtin_ia32_cvtmask2b128:
9105   case X86::BI__builtin_ia32_cvtmask2b256:
9106   case X86::BI__builtin_ia32_cvtmask2b512:
9107   case X86::BI__builtin_ia32_cvtmask2w128:
9108   case X86::BI__builtin_ia32_cvtmask2w256:
9109   case X86::BI__builtin_ia32_cvtmask2w512:
9110   case X86::BI__builtin_ia32_cvtmask2d128:
9111   case X86::BI__builtin_ia32_cvtmask2d256:
9112   case X86::BI__builtin_ia32_cvtmask2d512:
9113   case X86::BI__builtin_ia32_cvtmask2q128:
9114   case X86::BI__builtin_ia32_cvtmask2q256:
9115   case X86::BI__builtin_ia32_cvtmask2q512:
9116     return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType()));
9117 
9118   case X86::BI__builtin_ia32_cvtb2mask128:
9119   case X86::BI__builtin_ia32_cvtb2mask256:
9120   case X86::BI__builtin_ia32_cvtb2mask512:
9121   case X86::BI__builtin_ia32_cvtw2mask128:
9122   case X86::BI__builtin_ia32_cvtw2mask256:
9123   case X86::BI__builtin_ia32_cvtw2mask512:
9124   case X86::BI__builtin_ia32_cvtd2mask128:
9125   case X86::BI__builtin_ia32_cvtd2mask256:
9126   case X86::BI__builtin_ia32_cvtd2mask512:
9127   case X86::BI__builtin_ia32_cvtq2mask128:
9128   case X86::BI__builtin_ia32_cvtq2mask256:
9129   case X86::BI__builtin_ia32_cvtq2mask512:
9130     return EmitX86ConvertToMask(*this, Ops[0]);
9131 
9132   case X86::BI__builtin_ia32_vfmaddss3:
9133   case X86::BI__builtin_ia32_vfmaddsd3: {
9134     Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
9135     Value *B = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
9136     Value *C = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
9137     Function *FMA = CGM.getIntrinsic(Intrinsic::fma, A->getType());
9138     Value *Res = Builder.CreateCall(FMA, {A, B, C} );
9139     return Builder.CreateInsertElement(Ops[0], Res, (uint64_t)0);
9140   }
9141   case X86::BI__builtin_ia32_vfmaddps:
9142   case X86::BI__builtin_ia32_vfmaddpd:
9143   case X86::BI__builtin_ia32_vfmaddps256:
9144   case X86::BI__builtin_ia32_vfmaddpd256:
9145   case X86::BI__builtin_ia32_vfmaddps512_mask:
9146   case X86::BI__builtin_ia32_vfmaddps512_maskz:
9147   case X86::BI__builtin_ia32_vfmaddps512_mask3:
9148   case X86::BI__builtin_ia32_vfmsubps512_mask3:
9149   case X86::BI__builtin_ia32_vfmaddpd512_mask:
9150   case X86::BI__builtin_ia32_vfmaddpd512_maskz:
9151   case X86::BI__builtin_ia32_vfmaddpd512_mask3:
9152   case X86::BI__builtin_ia32_vfmsubpd512_mask3:
9153     return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/false);
9154   case X86::BI__builtin_ia32_vfmaddsubps:
9155   case X86::BI__builtin_ia32_vfmaddsubpd:
9156   case X86::BI__builtin_ia32_vfmaddsubps256:
9157   case X86::BI__builtin_ia32_vfmaddsubpd256:
9158   case X86::BI__builtin_ia32_vfmaddsubps512_mask:
9159   case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
9160   case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
9161   case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
9162   case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
9163   case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
9164   case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
9165   case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
9166     return EmitX86FMAExpr(*this, Ops, BuiltinID, /*IsAddSub*/true);
9167 
9168   case X86::BI__builtin_ia32_movdqa32store128_mask:
9169   case X86::BI__builtin_ia32_movdqa64store128_mask:
9170   case X86::BI__builtin_ia32_storeaps128_mask:
9171   case X86::BI__builtin_ia32_storeapd128_mask:
9172   case X86::BI__builtin_ia32_movdqa32store256_mask:
9173   case X86::BI__builtin_ia32_movdqa64store256_mask:
9174   case X86::BI__builtin_ia32_storeaps256_mask:
9175   case X86::BI__builtin_ia32_storeapd256_mask:
9176   case X86::BI__builtin_ia32_movdqa32store512_mask:
9177   case X86::BI__builtin_ia32_movdqa64store512_mask:
9178   case X86::BI__builtin_ia32_storeaps512_mask:
9179   case X86::BI__builtin_ia32_storeapd512_mask: {
9180     unsigned Align =
9181       getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity();
9182     return EmitX86MaskedStore(*this, Ops, Align);
9183   }
9184   case X86::BI__builtin_ia32_loadups128_mask:
9185   case X86::BI__builtin_ia32_loadups256_mask:
9186   case X86::BI__builtin_ia32_loadups512_mask:
9187   case X86::BI__builtin_ia32_loadupd128_mask:
9188   case X86::BI__builtin_ia32_loadupd256_mask:
9189   case X86::BI__builtin_ia32_loadupd512_mask:
9190   case X86::BI__builtin_ia32_loaddquqi128_mask:
9191   case X86::BI__builtin_ia32_loaddquqi256_mask:
9192   case X86::BI__builtin_ia32_loaddquqi512_mask:
9193   case X86::BI__builtin_ia32_loaddquhi128_mask:
9194   case X86::BI__builtin_ia32_loaddquhi256_mask:
9195   case X86::BI__builtin_ia32_loaddquhi512_mask:
9196   case X86::BI__builtin_ia32_loaddqusi128_mask:
9197   case X86::BI__builtin_ia32_loaddqusi256_mask:
9198   case X86::BI__builtin_ia32_loaddqusi512_mask:
9199   case X86::BI__builtin_ia32_loaddqudi128_mask:
9200   case X86::BI__builtin_ia32_loaddqudi256_mask:
9201   case X86::BI__builtin_ia32_loaddqudi512_mask:
9202     return EmitX86MaskedLoad(*this, Ops, 1);
9203 
9204   case X86::BI__builtin_ia32_loadss128_mask:
9205   case X86::BI__builtin_ia32_loadsd128_mask:
9206     return EmitX86MaskedLoad(*this, Ops, 1);
9207 
9208   case X86::BI__builtin_ia32_loadaps128_mask:
9209   case X86::BI__builtin_ia32_loadaps256_mask:
9210   case X86::BI__builtin_ia32_loadaps512_mask:
9211   case X86::BI__builtin_ia32_loadapd128_mask:
9212   case X86::BI__builtin_ia32_loadapd256_mask:
9213   case X86::BI__builtin_ia32_loadapd512_mask:
9214   case X86::BI__builtin_ia32_movdqa32load128_mask:
9215   case X86::BI__builtin_ia32_movdqa32load256_mask:
9216   case X86::BI__builtin_ia32_movdqa32load512_mask:
9217   case X86::BI__builtin_ia32_movdqa64load128_mask:
9218   case X86::BI__builtin_ia32_movdqa64load256_mask:
9219   case X86::BI__builtin_ia32_movdqa64load512_mask: {
9220     unsigned Align =
9221       getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity();
9222     return EmitX86MaskedLoad(*this, Ops, Align);
9223   }
9224 
9225   case X86::BI__builtin_ia32_expandloaddf128_mask:
9226   case X86::BI__builtin_ia32_expandloaddf256_mask:
9227   case X86::BI__builtin_ia32_expandloaddf512_mask:
9228   case X86::BI__builtin_ia32_expandloadsf128_mask:
9229   case X86::BI__builtin_ia32_expandloadsf256_mask:
9230   case X86::BI__builtin_ia32_expandloadsf512_mask:
9231   case X86::BI__builtin_ia32_expandloaddi128_mask:
9232   case X86::BI__builtin_ia32_expandloaddi256_mask:
9233   case X86::BI__builtin_ia32_expandloaddi512_mask:
9234   case X86::BI__builtin_ia32_expandloadsi128_mask:
9235   case X86::BI__builtin_ia32_expandloadsi256_mask:
9236   case X86::BI__builtin_ia32_expandloadsi512_mask:
9237   case X86::BI__builtin_ia32_expandloadhi128_mask:
9238   case X86::BI__builtin_ia32_expandloadhi256_mask:
9239   case X86::BI__builtin_ia32_expandloadhi512_mask:
9240   case X86::BI__builtin_ia32_expandloadqi128_mask:
9241   case X86::BI__builtin_ia32_expandloadqi256_mask:
9242   case X86::BI__builtin_ia32_expandloadqi512_mask:
9243     return EmitX86ExpandLoad(*this, Ops);
9244 
9245   case X86::BI__builtin_ia32_compressstoredf128_mask:
9246   case X86::BI__builtin_ia32_compressstoredf256_mask:
9247   case X86::BI__builtin_ia32_compressstoredf512_mask:
9248   case X86::BI__builtin_ia32_compressstoresf128_mask:
9249   case X86::BI__builtin_ia32_compressstoresf256_mask:
9250   case X86::BI__builtin_ia32_compressstoresf512_mask:
9251   case X86::BI__builtin_ia32_compressstoredi128_mask:
9252   case X86::BI__builtin_ia32_compressstoredi256_mask:
9253   case X86::BI__builtin_ia32_compressstoredi512_mask:
9254   case X86::BI__builtin_ia32_compressstoresi128_mask:
9255   case X86::BI__builtin_ia32_compressstoresi256_mask:
9256   case X86::BI__builtin_ia32_compressstoresi512_mask:
9257   case X86::BI__builtin_ia32_compressstorehi128_mask:
9258   case X86::BI__builtin_ia32_compressstorehi256_mask:
9259   case X86::BI__builtin_ia32_compressstorehi512_mask:
9260   case X86::BI__builtin_ia32_compressstoreqi128_mask:
9261   case X86::BI__builtin_ia32_compressstoreqi256_mask:
9262   case X86::BI__builtin_ia32_compressstoreqi512_mask:
9263     return EmitX86CompressStore(*this, Ops);
9264 
9265   case X86::BI__builtin_ia32_storehps:
9266   case X86::BI__builtin_ia32_storelps: {
9267     llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
9268     llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
9269 
9270     // cast val v2i64
9271     Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
9272 
9273     // extract (0, 1)
9274     unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
9275     Ops[1] = Builder.CreateExtractElement(Ops[1], Index, "extract");
9276 
9277     // cast pointer to i64 & store
9278     Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
9279     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9280   }
9281   case X86::BI__builtin_ia32_vextractf128_pd256:
9282   case X86::BI__builtin_ia32_vextractf128_ps256:
9283   case X86::BI__builtin_ia32_vextractf128_si256:
9284   case X86::BI__builtin_ia32_extract128i256:
9285   case X86::BI__builtin_ia32_extractf64x4_mask:
9286   case X86::BI__builtin_ia32_extractf32x4_mask:
9287   case X86::BI__builtin_ia32_extracti64x4_mask:
9288   case X86::BI__builtin_ia32_extracti32x4_mask:
9289   case X86::BI__builtin_ia32_extractf32x8_mask:
9290   case X86::BI__builtin_ia32_extracti32x8_mask:
9291   case X86::BI__builtin_ia32_extractf32x4_256_mask:
9292   case X86::BI__builtin_ia32_extracti32x4_256_mask:
9293   case X86::BI__builtin_ia32_extractf64x2_256_mask:
9294   case X86::BI__builtin_ia32_extracti64x2_256_mask:
9295   case X86::BI__builtin_ia32_extractf64x2_512_mask:
9296   case X86::BI__builtin_ia32_extracti64x2_512_mask: {
9297     llvm::Type *DstTy = ConvertType(E->getType());
9298     unsigned NumElts = DstTy->getVectorNumElements();
9299     unsigned SrcNumElts = Ops[0]->getType()->getVectorNumElements();
9300     unsigned SubVectors = SrcNumElts / NumElts;
9301     unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
9302     assert(llvm::isPowerOf2_32(SubVectors) && "Expected power of 2 subvectors");
9303     Index &= SubVectors - 1; // Remove any extra bits.
9304     Index *= NumElts;
9305 
9306     uint32_t Indices[16];
9307     for (unsigned i = 0; i != NumElts; ++i)
9308       Indices[i] = i + Index;
9309 
9310     Value *Res = Builder.CreateShuffleVector(Ops[0],
9311                                              UndefValue::get(Ops[0]->getType()),
9312                                              makeArrayRef(Indices, NumElts),
9313                                              "extract");
9314 
9315     if (Ops.size() == 4)
9316       Res = EmitX86Select(*this, Ops[3], Res, Ops[2]);
9317 
9318     return Res;
9319   }
9320   case X86::BI__builtin_ia32_vinsertf128_pd256:
9321   case X86::BI__builtin_ia32_vinsertf128_ps256:
9322   case X86::BI__builtin_ia32_vinsertf128_si256:
9323   case X86::BI__builtin_ia32_insert128i256:
9324   case X86::BI__builtin_ia32_insertf64x4:
9325   case X86::BI__builtin_ia32_insertf32x4:
9326   case X86::BI__builtin_ia32_inserti64x4:
9327   case X86::BI__builtin_ia32_inserti32x4:
9328   case X86::BI__builtin_ia32_insertf32x8:
9329   case X86::BI__builtin_ia32_inserti32x8:
9330   case X86::BI__builtin_ia32_insertf32x4_256:
9331   case X86::BI__builtin_ia32_inserti32x4_256:
9332   case X86::BI__builtin_ia32_insertf64x2_256:
9333   case X86::BI__builtin_ia32_inserti64x2_256:
9334   case X86::BI__builtin_ia32_insertf64x2_512:
9335   case X86::BI__builtin_ia32_inserti64x2_512: {
9336     unsigned DstNumElts = Ops[0]->getType()->getVectorNumElements();
9337     unsigned SrcNumElts = Ops[1]->getType()->getVectorNumElements();
9338     unsigned SubVectors = DstNumElts / SrcNumElts;
9339     unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
9340     assert(llvm::isPowerOf2_32(SubVectors) && "Expected power of 2 subvectors");
9341     Index &= SubVectors - 1; // Remove any extra bits.
9342     Index *= SrcNumElts;
9343 
9344     uint32_t Indices[16];
9345     for (unsigned i = 0; i != DstNumElts; ++i)
9346       Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
9347 
9348     Value *Op1 = Builder.CreateShuffleVector(Ops[1],
9349                                              UndefValue::get(Ops[1]->getType()),
9350                                              makeArrayRef(Indices, DstNumElts),
9351                                              "widen");
9352 
9353     for (unsigned i = 0; i != DstNumElts; ++i) {
9354       if (i >= Index && i < (Index + SrcNumElts))
9355         Indices[i] = (i - Index) + DstNumElts;
9356       else
9357         Indices[i] = i;
9358     }
9359 
9360     return Builder.CreateShuffleVector(Ops[0], Op1,
9361                                        makeArrayRef(Indices, DstNumElts),
9362                                        "insert");
9363   }
9364   case X86::BI__builtin_ia32_pmovqd512_mask:
9365   case X86::BI__builtin_ia32_pmovwb512_mask: {
9366     Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
9367     return EmitX86Select(*this, Ops[2], Res, Ops[1]);
9368   }
9369   case X86::BI__builtin_ia32_pmovdb512_mask:
9370   case X86::BI__builtin_ia32_pmovdw512_mask:
9371   case X86::BI__builtin_ia32_pmovqw512_mask: {
9372     if (const auto *C = dyn_cast<Constant>(Ops[2]))
9373       if (C->isAllOnesValue())
9374         return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
9375 
9376     Intrinsic::ID IID;
9377     switch (BuiltinID) {
9378     default: llvm_unreachable("Unsupported intrinsic!");
9379     case X86::BI__builtin_ia32_pmovdb512_mask:
9380       IID = Intrinsic::x86_avx512_mask_pmov_db_512;
9381       break;
9382     case X86::BI__builtin_ia32_pmovdw512_mask:
9383       IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
9384       break;
9385     case X86::BI__builtin_ia32_pmovqw512_mask:
9386       IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
9387       break;
9388     }
9389 
9390     Function *Intr = CGM.getIntrinsic(IID);
9391     return Builder.CreateCall(Intr, Ops);
9392   }
9393   case X86::BI__builtin_ia32_pblendw128:
9394   case X86::BI__builtin_ia32_blendpd:
9395   case X86::BI__builtin_ia32_blendps:
9396   case X86::BI__builtin_ia32_blendpd256:
9397   case X86::BI__builtin_ia32_blendps256:
9398   case X86::BI__builtin_ia32_pblendw256:
9399   case X86::BI__builtin_ia32_pblendd128:
9400   case X86::BI__builtin_ia32_pblendd256: {
9401     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9402     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9403 
9404     uint32_t Indices[16];
9405     // If there are more than 8 elements, the immediate is used twice so make
9406     // sure we handle that.
9407     for (unsigned i = 0; i != NumElts; ++i)
9408       Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
9409 
9410     return Builder.CreateShuffleVector(Ops[0], Ops[1],
9411                                        makeArrayRef(Indices, NumElts),
9412                                        "blend");
9413   }
9414   case X86::BI__builtin_ia32_pshuflw:
9415   case X86::BI__builtin_ia32_pshuflw256:
9416   case X86::BI__builtin_ia32_pshuflw512: {
9417     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9418     llvm::Type *Ty = Ops[0]->getType();
9419     unsigned NumElts = Ty->getVectorNumElements();
9420 
9421     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
9422     Imm = (Imm & 0xff) * 0x01010101;
9423 
9424     uint32_t Indices[32];
9425     for (unsigned l = 0; l != NumElts; l += 8) {
9426       for (unsigned i = 0; i != 4; ++i) {
9427         Indices[l + i] = l + (Imm & 3);
9428         Imm >>= 2;
9429       }
9430       for (unsigned i = 4; i != 8; ++i)
9431         Indices[l + i] = l + i;
9432     }
9433 
9434     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9435                                        makeArrayRef(Indices, NumElts),
9436                                        "pshuflw");
9437   }
9438   case X86::BI__builtin_ia32_pshufhw:
9439   case X86::BI__builtin_ia32_pshufhw256:
9440   case X86::BI__builtin_ia32_pshufhw512: {
9441     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9442     llvm::Type *Ty = Ops[0]->getType();
9443     unsigned NumElts = Ty->getVectorNumElements();
9444 
9445     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
9446     Imm = (Imm & 0xff) * 0x01010101;
9447 
9448     uint32_t Indices[32];
9449     for (unsigned l = 0; l != NumElts; l += 8) {
9450       for (unsigned i = 0; i != 4; ++i)
9451         Indices[l + i] = l + i;
9452       for (unsigned i = 4; i != 8; ++i) {
9453         Indices[l + i] = l + 4 + (Imm & 3);
9454         Imm >>= 2;
9455       }
9456     }
9457 
9458     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9459                                        makeArrayRef(Indices, NumElts),
9460                                        "pshufhw");
9461   }
9462   case X86::BI__builtin_ia32_pshufd:
9463   case X86::BI__builtin_ia32_pshufd256:
9464   case X86::BI__builtin_ia32_pshufd512:
9465   case X86::BI__builtin_ia32_vpermilpd:
9466   case X86::BI__builtin_ia32_vpermilps:
9467   case X86::BI__builtin_ia32_vpermilpd256:
9468   case X86::BI__builtin_ia32_vpermilps256:
9469   case X86::BI__builtin_ia32_vpermilpd512:
9470   case X86::BI__builtin_ia32_vpermilps512: {
9471     uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9472     llvm::Type *Ty = Ops[0]->getType();
9473     unsigned NumElts = Ty->getVectorNumElements();
9474     unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
9475     unsigned NumLaneElts = NumElts / NumLanes;
9476 
9477     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
9478     Imm = (Imm & 0xff) * 0x01010101;
9479 
9480     uint32_t Indices[16];
9481     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
9482       for (unsigned i = 0; i != NumLaneElts; ++i) {
9483         Indices[i + l] = (Imm % NumLaneElts) + l;
9484         Imm /= NumLaneElts;
9485       }
9486     }
9487 
9488     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9489                                        makeArrayRef(Indices, NumElts),
9490                                        "permil");
9491   }
9492   case X86::BI__builtin_ia32_shufpd:
9493   case X86::BI__builtin_ia32_shufpd256:
9494   case X86::BI__builtin_ia32_shufpd512:
9495   case X86::BI__builtin_ia32_shufps:
9496   case X86::BI__builtin_ia32_shufps256:
9497   case X86::BI__builtin_ia32_shufps512: {
9498     uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9499     llvm::Type *Ty = Ops[0]->getType();
9500     unsigned NumElts = Ty->getVectorNumElements();
9501     unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
9502     unsigned NumLaneElts = NumElts / NumLanes;
9503 
9504     // Splat the 8-bits of immediate 4 times to help the loop wrap around.
9505     Imm = (Imm & 0xff) * 0x01010101;
9506 
9507     uint32_t Indices[16];
9508     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
9509       for (unsigned i = 0; i != NumLaneElts; ++i) {
9510         unsigned Index = Imm % NumLaneElts;
9511         Imm /= NumLaneElts;
9512         if (i >= (NumLaneElts / 2))
9513           Index += NumElts;
9514         Indices[l + i] = l + Index;
9515       }
9516     }
9517 
9518     return Builder.CreateShuffleVector(Ops[0], Ops[1],
9519                                        makeArrayRef(Indices, NumElts),
9520                                        "shufp");
9521   }
9522   case X86::BI__builtin_ia32_permdi256:
9523   case X86::BI__builtin_ia32_permdf256:
9524   case X86::BI__builtin_ia32_permdi512:
9525   case X86::BI__builtin_ia32_permdf512: {
9526     unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9527     llvm::Type *Ty = Ops[0]->getType();
9528     unsigned NumElts = Ty->getVectorNumElements();
9529 
9530     // These intrinsics operate on 256-bit lanes of four 64-bit elements.
9531     uint32_t Indices[8];
9532     for (unsigned l = 0; l != NumElts; l += 4)
9533       for (unsigned i = 0; i != 4; ++i)
9534         Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
9535 
9536     return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9537                                        makeArrayRef(Indices, NumElts),
9538                                        "perm");
9539   }
9540   case X86::BI__builtin_ia32_palignr128:
9541   case X86::BI__builtin_ia32_palignr256:
9542   case X86::BI__builtin_ia32_palignr512: {
9543     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
9544 
9545     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9546     assert(NumElts % 16 == 0);
9547 
9548     // If palignr is shifting the pair of vectors more than the size of two
9549     // lanes, emit zero.
9550     if (ShiftVal >= 32)
9551       return llvm::Constant::getNullValue(ConvertType(E->getType()));
9552 
9553     // If palignr is shifting the pair of input vectors more than one lane,
9554     // but less than two lanes, convert to shifting in zeroes.
9555     if (ShiftVal > 16) {
9556       ShiftVal -= 16;
9557       Ops[1] = Ops[0];
9558       Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
9559     }
9560 
9561     uint32_t Indices[64];
9562     // 256-bit palignr operates on 128-bit lanes so we need to handle that
9563     for (unsigned l = 0; l != NumElts; l += 16) {
9564       for (unsigned i = 0; i != 16; ++i) {
9565         unsigned Idx = ShiftVal + i;
9566         if (Idx >= 16)
9567           Idx += NumElts - 16; // End of lane, switch operand.
9568         Indices[l + i] = Idx + l;
9569       }
9570     }
9571 
9572     return Builder.CreateShuffleVector(Ops[1], Ops[0],
9573                                        makeArrayRef(Indices, NumElts),
9574                                        "palignr");
9575   }
9576   case X86::BI__builtin_ia32_alignd128:
9577   case X86::BI__builtin_ia32_alignd256:
9578   case X86::BI__builtin_ia32_alignd512:
9579   case X86::BI__builtin_ia32_alignq128:
9580   case X86::BI__builtin_ia32_alignq256:
9581   case X86::BI__builtin_ia32_alignq512: {
9582     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9583     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
9584 
9585     // Mask the shift amount to width of two vectors.
9586     ShiftVal &= (2 * NumElts) - 1;
9587 
9588     uint32_t Indices[16];
9589     for (unsigned i = 0; i != NumElts; ++i)
9590       Indices[i] = i + ShiftVal;
9591 
9592     return Builder.CreateShuffleVector(Ops[1], Ops[0],
9593                                        makeArrayRef(Indices, NumElts),
9594                                        "valign");
9595   }
9596   case X86::BI__builtin_ia32_shuf_f32x4_256:
9597   case X86::BI__builtin_ia32_shuf_f64x2_256:
9598   case X86::BI__builtin_ia32_shuf_i32x4_256:
9599   case X86::BI__builtin_ia32_shuf_i64x2_256:
9600   case X86::BI__builtin_ia32_shuf_f32x4:
9601   case X86::BI__builtin_ia32_shuf_f64x2:
9602   case X86::BI__builtin_ia32_shuf_i32x4:
9603   case X86::BI__builtin_ia32_shuf_i64x2: {
9604     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9605     llvm::Type *Ty = Ops[0]->getType();
9606     unsigned NumElts = Ty->getVectorNumElements();
9607     unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
9608     unsigned NumLaneElts = NumElts / NumLanes;
9609 
9610     uint32_t Indices[16];
9611     for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
9612       unsigned Index = (Imm % NumLanes) * NumLaneElts;
9613       Imm /= NumLanes; // Discard the bits we just used.
9614       if (l >= (NumElts / 2))
9615         Index += NumElts; // Switch to other source.
9616       for (unsigned i = 0; i != NumLaneElts; ++i) {
9617         Indices[l + i] = Index + i;
9618       }
9619     }
9620 
9621     return Builder.CreateShuffleVector(Ops[0], Ops[1],
9622                                        makeArrayRef(Indices, NumElts),
9623                                        "shuf");
9624   }
9625 
9626   case X86::BI__builtin_ia32_vperm2f128_pd256:
9627   case X86::BI__builtin_ia32_vperm2f128_ps256:
9628   case X86::BI__builtin_ia32_vperm2f128_si256:
9629   case X86::BI__builtin_ia32_permti256: {
9630     unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9631     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9632 
9633     // This takes a very simple approach since there are two lanes and a
9634     // shuffle can have 2 inputs. So we reserve the first input for the first
9635     // lane and the second input for the second lane. This may result in
9636     // duplicate sources, but this can be dealt with in the backend.
9637 
9638     Value *OutOps[2];
9639     uint32_t Indices[8];
9640     for (unsigned l = 0; l != 2; ++l) {
9641       // Determine the source for this lane.
9642       if (Imm & (1 << ((l * 4) + 3)))
9643         OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
9644       else if (Imm & (1 << ((l * 4) + 1)))
9645         OutOps[l] = Ops[1];
9646       else
9647         OutOps[l] = Ops[0];
9648 
9649       for (unsigned i = 0; i != NumElts/2; ++i) {
9650         // Start with ith element of the source for this lane.
9651         unsigned Idx = (l * NumElts) + i;
9652         // If bit 0 of the immediate half is set, switch to the high half of
9653         // the source.
9654         if (Imm & (1 << (l * 4)))
9655           Idx += NumElts/2;
9656         Indices[(l * (NumElts/2)) + i] = Idx;
9657       }
9658     }
9659 
9660     return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
9661                                        makeArrayRef(Indices, NumElts),
9662                                        "vperm");
9663   }
9664 
9665   case X86::BI__builtin_ia32_pslldqi128_byteshift:
9666   case X86::BI__builtin_ia32_pslldqi256_byteshift:
9667   case X86::BI__builtin_ia32_pslldqi512_byteshift: {
9668     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
9669     llvm::Type *ResultType = Ops[0]->getType();
9670     // Builtin type is vXi64 so multiply by 8 to get bytes.
9671     unsigned NumElts = ResultType->getVectorNumElements() * 8;
9672 
9673     // If pslldq is shifting the vector more than 15 bytes, emit zero.
9674     if (ShiftVal >= 16)
9675       return llvm::Constant::getNullValue(ResultType);
9676 
9677     uint32_t Indices[64];
9678     // 256/512-bit pslldq operates on 128-bit lanes so we need to handle that
9679     for (unsigned l = 0; l != NumElts; l += 16) {
9680       for (unsigned i = 0; i != 16; ++i) {
9681         unsigned Idx = NumElts + i - ShiftVal;
9682         if (Idx < NumElts) Idx -= NumElts - 16; // end of lane, switch operand.
9683         Indices[l + i] = Idx + l;
9684       }
9685     }
9686 
9687     llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
9688     Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
9689     Value *Zero = llvm::Constant::getNullValue(VecTy);
9690     Value *SV = Builder.CreateShuffleVector(Zero, Cast,
9691                                             makeArrayRef(Indices, NumElts),
9692                                             "pslldq");
9693     return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast");
9694   }
9695   case X86::BI__builtin_ia32_psrldqi128_byteshift:
9696   case X86::BI__builtin_ia32_psrldqi256_byteshift:
9697   case X86::BI__builtin_ia32_psrldqi512_byteshift: {
9698     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
9699     llvm::Type *ResultType = Ops[0]->getType();
9700     // Builtin type is vXi64 so multiply by 8 to get bytes.
9701     unsigned NumElts = ResultType->getVectorNumElements() * 8;
9702 
9703     // If psrldq is shifting the vector more than 15 bytes, emit zero.
9704     if (ShiftVal >= 16)
9705       return llvm::Constant::getNullValue(ResultType);
9706 
9707     uint32_t Indices[64];
9708     // 256/512-bit psrldq operates on 128-bit lanes so we need to handle that
9709     for (unsigned l = 0; l != NumElts; l += 16) {
9710       for (unsigned i = 0; i != 16; ++i) {
9711         unsigned Idx = i + ShiftVal;
9712         if (Idx >= 16) Idx += NumElts - 16; // end of lane, switch operand.
9713         Indices[l + i] = Idx + l;
9714       }
9715     }
9716 
9717     llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
9718     Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
9719     Value *Zero = llvm::Constant::getNullValue(VecTy);
9720     Value *SV = Builder.CreateShuffleVector(Cast, Zero,
9721                                             makeArrayRef(Indices, NumElts),
9722                                             "psrldq");
9723     return Builder.CreateBitCast(SV, ResultType, "cast");
9724   }
9725   case X86::BI__builtin_ia32_movnti:
9726   case X86::BI__builtin_ia32_movnti64:
9727   case X86::BI__builtin_ia32_movntsd:
9728   case X86::BI__builtin_ia32_movntss: {
9729     llvm::MDNode *Node = llvm::MDNode::get(
9730         getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
9731 
9732     Value *Ptr = Ops[0];
9733     Value *Src = Ops[1];
9734 
9735     // Extract the 0'th element of the source vector.
9736     if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
9737         BuiltinID == X86::BI__builtin_ia32_movntss)
9738       Src = Builder.CreateExtractElement(Src, (uint64_t)0, "extract");
9739 
9740     // Convert the type of the pointer to a pointer to the stored type.
9741     Value *BC = Builder.CreateBitCast(
9742         Ptr, llvm::PointerType::getUnqual(Src->getType()), "cast");
9743 
9744     // Unaligned nontemporal store of the scalar value.
9745     StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
9746     SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
9747     SI->setAlignment(1);
9748     return SI;
9749   }
9750 
9751   case X86::BI__builtin_ia32_selectb_128:
9752   case X86::BI__builtin_ia32_selectb_256:
9753   case X86::BI__builtin_ia32_selectb_512:
9754   case X86::BI__builtin_ia32_selectw_128:
9755   case X86::BI__builtin_ia32_selectw_256:
9756   case X86::BI__builtin_ia32_selectw_512:
9757   case X86::BI__builtin_ia32_selectd_128:
9758   case X86::BI__builtin_ia32_selectd_256:
9759   case X86::BI__builtin_ia32_selectd_512:
9760   case X86::BI__builtin_ia32_selectq_128:
9761   case X86::BI__builtin_ia32_selectq_256:
9762   case X86::BI__builtin_ia32_selectq_512:
9763   case X86::BI__builtin_ia32_selectps_128:
9764   case X86::BI__builtin_ia32_selectps_256:
9765   case X86::BI__builtin_ia32_selectps_512:
9766   case X86::BI__builtin_ia32_selectpd_128:
9767   case X86::BI__builtin_ia32_selectpd_256:
9768   case X86::BI__builtin_ia32_selectpd_512:
9769     return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]);
9770   case X86::BI__builtin_ia32_cmpb128_mask:
9771   case X86::BI__builtin_ia32_cmpb256_mask:
9772   case X86::BI__builtin_ia32_cmpb512_mask:
9773   case X86::BI__builtin_ia32_cmpw128_mask:
9774   case X86::BI__builtin_ia32_cmpw256_mask:
9775   case X86::BI__builtin_ia32_cmpw512_mask:
9776   case X86::BI__builtin_ia32_cmpd128_mask:
9777   case X86::BI__builtin_ia32_cmpd256_mask:
9778   case X86::BI__builtin_ia32_cmpd512_mask:
9779   case X86::BI__builtin_ia32_cmpq128_mask:
9780   case X86::BI__builtin_ia32_cmpq256_mask:
9781   case X86::BI__builtin_ia32_cmpq512_mask: {
9782     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
9783     return EmitX86MaskedCompare(*this, CC, true, Ops);
9784   }
9785   case X86::BI__builtin_ia32_ucmpb128_mask:
9786   case X86::BI__builtin_ia32_ucmpb256_mask:
9787   case X86::BI__builtin_ia32_ucmpb512_mask:
9788   case X86::BI__builtin_ia32_ucmpw128_mask:
9789   case X86::BI__builtin_ia32_ucmpw256_mask:
9790   case X86::BI__builtin_ia32_ucmpw512_mask:
9791   case X86::BI__builtin_ia32_ucmpd128_mask:
9792   case X86::BI__builtin_ia32_ucmpd256_mask:
9793   case X86::BI__builtin_ia32_ucmpd512_mask:
9794   case X86::BI__builtin_ia32_ucmpq128_mask:
9795   case X86::BI__builtin_ia32_ucmpq256_mask:
9796   case X86::BI__builtin_ia32_ucmpq512_mask: {
9797     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
9798     return EmitX86MaskedCompare(*this, CC, false, Ops);
9799   }
9800 
9801   case X86::BI__builtin_ia32_kortestchi:
9802   case X86::BI__builtin_ia32_kortestzhi: {
9803     Value *Or = EmitX86MaskLogic(*this, Instruction::Or, 16, Ops);
9804     Value *C;
9805     if (BuiltinID == X86::BI__builtin_ia32_kortestchi)
9806       C = llvm::Constant::getAllOnesValue(Builder.getInt16Ty());
9807     else
9808       C = llvm::Constant::getNullValue(Builder.getInt16Ty());
9809     Value *Cmp = Builder.CreateICmpEQ(Or, C);
9810     return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
9811   }
9812 
9813   case X86::BI__builtin_ia32_kandhi:
9814     return EmitX86MaskLogic(*this, Instruction::And, 16, Ops);
9815   case X86::BI__builtin_ia32_kandnhi:
9816     return EmitX86MaskLogic(*this, Instruction::And, 16, Ops, true);
9817   case X86::BI__builtin_ia32_korhi:
9818     return EmitX86MaskLogic(*this, Instruction::Or, 16, Ops);
9819   case X86::BI__builtin_ia32_kxnorhi:
9820     return EmitX86MaskLogic(*this, Instruction::Xor, 16, Ops, true);
9821   case X86::BI__builtin_ia32_kxorhi:
9822     return EmitX86MaskLogic(*this, Instruction::Xor, 16, Ops);
9823   case X86::BI__builtin_ia32_knothi: {
9824     Ops[0] = getMaskVecValue(*this, Ops[0], 16);
9825     return Builder.CreateBitCast(Builder.CreateNot(Ops[0]),
9826                                  Builder.getInt16Ty());
9827   }
9828 
9829   case X86::BI__builtin_ia32_kunpckdi:
9830   case X86::BI__builtin_ia32_kunpcksi:
9831   case X86::BI__builtin_ia32_kunpckhi: {
9832     unsigned NumElts = Ops[0]->getType()->getScalarSizeInBits();
9833     Value *LHS = getMaskVecValue(*this, Ops[0], NumElts);
9834     Value *RHS = getMaskVecValue(*this, Ops[1], NumElts);
9835     uint32_t Indices[64];
9836     for (unsigned i = 0; i != NumElts; ++i)
9837       Indices[i] = i;
9838 
9839     // First extract half of each vector. This gives better codegen than
9840     // doing it in a single shuffle.
9841     LHS = Builder.CreateShuffleVector(LHS, LHS,
9842                                       makeArrayRef(Indices, NumElts / 2));
9843     RHS = Builder.CreateShuffleVector(RHS, RHS,
9844                                       makeArrayRef(Indices, NumElts / 2));
9845     // Concat the vectors.
9846     // NOTE: Operands are swapped to match the intrinsic definition.
9847     Value *Res = Builder.CreateShuffleVector(RHS, LHS,
9848                                              makeArrayRef(Indices, NumElts));
9849     return Builder.CreateBitCast(Res, Ops[0]->getType());
9850   }
9851 
9852   case X86::BI__builtin_ia32_vplzcntd_128:
9853   case X86::BI__builtin_ia32_vplzcntd_256:
9854   case X86::BI__builtin_ia32_vplzcntd_512:
9855   case X86::BI__builtin_ia32_vplzcntq_128:
9856   case X86::BI__builtin_ia32_vplzcntq_256:
9857   case X86::BI__builtin_ia32_vplzcntq_512: {
9858     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
9859     return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)});
9860   }
9861   case X86::BI__builtin_ia32_sqrtss:
9862   case X86::BI__builtin_ia32_sqrtsd: {
9863     Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
9864     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
9865     A = Builder.CreateCall(F, {A});
9866     return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
9867   }
9868   case X86::BI__builtin_ia32_sqrtsd_round_mask:
9869   case X86::BI__builtin_ia32_sqrtss_round_mask: {
9870     unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
9871     // Support only if the rounding mode is 4 (AKA CUR_DIRECTION),
9872     // otherwise keep the intrinsic.
9873     if (CC != 4) {
9874       Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtsd_round_mask ?
9875                           Intrinsic::x86_avx512_mask_sqrt_sd :
9876                           Intrinsic::x86_avx512_mask_sqrt_ss;
9877       return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
9878     }
9879     Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
9880     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
9881     Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
9882     int MaskSize = Ops[3]->getType()->getScalarSizeInBits();
9883     llvm::Type *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), MaskSize);
9884     Value *Mask = Builder.CreateBitCast(Ops[3], MaskTy);
9885     Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
9886     A = Builder.CreateSelect(Mask, Builder.CreateCall(F, {A}), Src);
9887     return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
9888   }
9889   case X86::BI__builtin_ia32_sqrtpd256:
9890   case X86::BI__builtin_ia32_sqrtpd:
9891   case X86::BI__builtin_ia32_sqrtps256:
9892   case X86::BI__builtin_ia32_sqrtps: {
9893     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType());
9894     return Builder.CreateCall(F, {Ops[0]});
9895   }
9896   case X86::BI__builtin_ia32_sqrtps512_mask:
9897   case X86::BI__builtin_ia32_sqrtpd512_mask: {
9898     unsigned CC = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
9899     // Support only if the rounding mode is 4 (AKA CUR_DIRECTION),
9900     // otherwise keep the intrinsic.
9901     if (CC != 4) {
9902       Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtps512_mask ?
9903                           Intrinsic::x86_avx512_mask_sqrt_ps_512 :
9904                           Intrinsic::x86_avx512_mask_sqrt_pd_512;
9905       return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
9906     }
9907     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType());
9908     return EmitX86Select(*this, Ops[2], Builder.CreateCall(F, {Ops[0]}),
9909                          Ops[1]);
9910   }
9911   case X86::BI__builtin_ia32_pabsb128:
9912   case X86::BI__builtin_ia32_pabsw128:
9913   case X86::BI__builtin_ia32_pabsd128:
9914   case X86::BI__builtin_ia32_pabsb256:
9915   case X86::BI__builtin_ia32_pabsw256:
9916   case X86::BI__builtin_ia32_pabsd256:
9917   case X86::BI__builtin_ia32_pabsq128:
9918   case X86::BI__builtin_ia32_pabsq256:
9919   case X86::BI__builtin_ia32_pabsb512:
9920   case X86::BI__builtin_ia32_pabsw512:
9921   case X86::BI__builtin_ia32_pabsd512:
9922   case X86::BI__builtin_ia32_pabsq512:
9923     return EmitX86Abs(*this, Ops);
9924 
9925   case X86::BI__builtin_ia32_pmaxsb128:
9926   case X86::BI__builtin_ia32_pmaxsw128:
9927   case X86::BI__builtin_ia32_pmaxsd128:
9928   case X86::BI__builtin_ia32_pmaxsq128:
9929   case X86::BI__builtin_ia32_pmaxsb256:
9930   case X86::BI__builtin_ia32_pmaxsw256:
9931   case X86::BI__builtin_ia32_pmaxsd256:
9932   case X86::BI__builtin_ia32_pmaxsq256:
9933   case X86::BI__builtin_ia32_pmaxsb512:
9934   case X86::BI__builtin_ia32_pmaxsw512:
9935   case X86::BI__builtin_ia32_pmaxsd512:
9936   case X86::BI__builtin_ia32_pmaxsq512:
9937     return EmitX86MinMax(*this, ICmpInst::ICMP_SGT, Ops);
9938   case X86::BI__builtin_ia32_pmaxub128:
9939   case X86::BI__builtin_ia32_pmaxuw128:
9940   case X86::BI__builtin_ia32_pmaxud128:
9941   case X86::BI__builtin_ia32_pmaxuq128:
9942   case X86::BI__builtin_ia32_pmaxub256:
9943   case X86::BI__builtin_ia32_pmaxuw256:
9944   case X86::BI__builtin_ia32_pmaxud256:
9945   case X86::BI__builtin_ia32_pmaxuq256:
9946   case X86::BI__builtin_ia32_pmaxub512:
9947   case X86::BI__builtin_ia32_pmaxuw512:
9948   case X86::BI__builtin_ia32_pmaxud512:
9949   case X86::BI__builtin_ia32_pmaxuq512:
9950     return EmitX86MinMax(*this, ICmpInst::ICMP_UGT, Ops);
9951   case X86::BI__builtin_ia32_pminsb128:
9952   case X86::BI__builtin_ia32_pminsw128:
9953   case X86::BI__builtin_ia32_pminsd128:
9954   case X86::BI__builtin_ia32_pminsq128:
9955   case X86::BI__builtin_ia32_pminsb256:
9956   case X86::BI__builtin_ia32_pminsw256:
9957   case X86::BI__builtin_ia32_pminsd256:
9958   case X86::BI__builtin_ia32_pminsq256:
9959   case X86::BI__builtin_ia32_pminsb512:
9960   case X86::BI__builtin_ia32_pminsw512:
9961   case X86::BI__builtin_ia32_pminsd512:
9962   case X86::BI__builtin_ia32_pminsq512:
9963     return EmitX86MinMax(*this, ICmpInst::ICMP_SLT, Ops);
9964   case X86::BI__builtin_ia32_pminub128:
9965   case X86::BI__builtin_ia32_pminuw128:
9966   case X86::BI__builtin_ia32_pminud128:
9967   case X86::BI__builtin_ia32_pminuq128:
9968   case X86::BI__builtin_ia32_pminub256:
9969   case X86::BI__builtin_ia32_pminuw256:
9970   case X86::BI__builtin_ia32_pminud256:
9971   case X86::BI__builtin_ia32_pminuq256:
9972   case X86::BI__builtin_ia32_pminub512:
9973   case X86::BI__builtin_ia32_pminuw512:
9974   case X86::BI__builtin_ia32_pminud512:
9975   case X86::BI__builtin_ia32_pminuq512:
9976     return EmitX86MinMax(*this, ICmpInst::ICMP_ULT, Ops);
9977 
9978   case X86::BI__builtin_ia32_pmuludq128:
9979   case X86::BI__builtin_ia32_pmuludq256:
9980   case X86::BI__builtin_ia32_pmuludq512:
9981     return EmitX86Muldq(*this, /*IsSigned*/false, Ops);
9982 
9983   case X86::BI__builtin_ia32_pmuldq128:
9984   case X86::BI__builtin_ia32_pmuldq256:
9985   case X86::BI__builtin_ia32_pmuldq512:
9986     return EmitX86Muldq(*this, /*IsSigned*/true, Ops);
9987 
9988   case X86::BI__builtin_ia32_pternlogd512_mask:
9989   case X86::BI__builtin_ia32_pternlogq512_mask:
9990   case X86::BI__builtin_ia32_pternlogd128_mask:
9991   case X86::BI__builtin_ia32_pternlogd256_mask:
9992   case X86::BI__builtin_ia32_pternlogq128_mask:
9993   case X86::BI__builtin_ia32_pternlogq256_mask:
9994     return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops);
9995 
9996   case X86::BI__builtin_ia32_pternlogd512_maskz:
9997   case X86::BI__builtin_ia32_pternlogq512_maskz:
9998   case X86::BI__builtin_ia32_pternlogd128_maskz:
9999   case X86::BI__builtin_ia32_pternlogd256_maskz:
10000   case X86::BI__builtin_ia32_pternlogq128_maskz:
10001   case X86::BI__builtin_ia32_pternlogq256_maskz:
10002     return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops);
10003 
10004   case X86::BI__builtin_ia32_divss_round_mask:
10005   case X86::BI__builtin_ia32_divsd_round_mask: {
10006     Intrinsic::ID ID;
10007     switch (BuiltinID) {
10008     default: llvm_unreachable("Unsupported intrinsic!");
10009     case X86::BI__builtin_ia32_divss_round_mask:
10010       ID = Intrinsic::x86_avx512_mask_div_ss_round; break;
10011     case X86::BI__builtin_ia32_divsd_round_mask:
10012       ID = Intrinsic::x86_avx512_mask_div_sd_round; break;
10013     }
10014     Function *Intr = CGM.getIntrinsic(ID);
10015 
10016     // If round parameter is not _MM_FROUND_CUR_DIRECTION, don't lower.
10017     if (cast<llvm::ConstantInt>(Ops[4])->getZExtValue() != (uint64_t)4)
10018       return Builder.CreateCall(Intr, Ops);
10019 
10020     Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
10021     Value *B = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
10022     Value *C = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
10023     Value *Mask = Ops[3];
10024     Value *Div = Builder.CreateFDiv(A, B);
10025     llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(),
10026                              cast<IntegerType>(Mask->getType())->getBitWidth());
10027     Mask = Builder.CreateBitCast(Mask, MaskTy);
10028     Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
10029     Value *Select = Builder.CreateSelect(Mask, Div, C);
10030     return Builder.CreateInsertElement(Ops[0], Select, (uint64_t)0);
10031   }
10032 
10033   // 3DNow!
10034   case X86::BI__builtin_ia32_pswapdsf:
10035   case X86::BI__builtin_ia32_pswapdsi: {
10036     llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
10037     Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
10038     llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
10039     return Builder.CreateCall(F, Ops, "pswapd");
10040   }
10041   case X86::BI__builtin_ia32_rdrand16_step:
10042   case X86::BI__builtin_ia32_rdrand32_step:
10043   case X86::BI__builtin_ia32_rdrand64_step:
10044   case X86::BI__builtin_ia32_rdseed16_step:
10045   case X86::BI__builtin_ia32_rdseed32_step:
10046   case X86::BI__builtin_ia32_rdseed64_step: {
10047     Intrinsic::ID ID;
10048     switch (BuiltinID) {
10049     default: llvm_unreachable("Unsupported intrinsic!");
10050     case X86::BI__builtin_ia32_rdrand16_step:
10051       ID = Intrinsic::x86_rdrand_16;
10052       break;
10053     case X86::BI__builtin_ia32_rdrand32_step:
10054       ID = Intrinsic::x86_rdrand_32;
10055       break;
10056     case X86::BI__builtin_ia32_rdrand64_step:
10057       ID = Intrinsic::x86_rdrand_64;
10058       break;
10059     case X86::BI__builtin_ia32_rdseed16_step:
10060       ID = Intrinsic::x86_rdseed_16;
10061       break;
10062     case X86::BI__builtin_ia32_rdseed32_step:
10063       ID = Intrinsic::x86_rdseed_32;
10064       break;
10065     case X86::BI__builtin_ia32_rdseed64_step:
10066       ID = Intrinsic::x86_rdseed_64;
10067       break;
10068     }
10069 
10070     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
10071     Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
10072                                       Ops[0]);
10073     return Builder.CreateExtractValue(Call, 1);
10074   }
10075 
10076   case X86::BI__builtin_ia32_fpclassps128_mask:
10077   case X86::BI__builtin_ia32_fpclassps256_mask:
10078   case X86::BI__builtin_ia32_fpclassps512_mask:
10079   case X86::BI__builtin_ia32_fpclasspd128_mask:
10080   case X86::BI__builtin_ia32_fpclasspd256_mask:
10081   case X86::BI__builtin_ia32_fpclasspd512_mask: {
10082     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10083     Value *MaskIn = Ops[2];
10084     Ops.erase(&Ops[2]);
10085 
10086     Intrinsic::ID ID;
10087     switch (BuiltinID) {
10088     default: llvm_unreachable("Unsupported intrinsic!");
10089     case X86::BI__builtin_ia32_fpclassps128_mask:
10090       ID = Intrinsic::x86_avx512_fpclass_ps_128;
10091       break;
10092     case X86::BI__builtin_ia32_fpclassps256_mask:
10093       ID = Intrinsic::x86_avx512_fpclass_ps_256;
10094       break;
10095     case X86::BI__builtin_ia32_fpclassps512_mask:
10096       ID = Intrinsic::x86_avx512_fpclass_ps_512;
10097       break;
10098     case X86::BI__builtin_ia32_fpclasspd128_mask:
10099       ID = Intrinsic::x86_avx512_fpclass_pd_128;
10100       break;
10101     case X86::BI__builtin_ia32_fpclasspd256_mask:
10102       ID = Intrinsic::x86_avx512_fpclass_pd_256;
10103       break;
10104     case X86::BI__builtin_ia32_fpclasspd512_mask:
10105       ID = Intrinsic::x86_avx512_fpclass_pd_512;
10106       break;
10107     }
10108 
10109     Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
10110     return EmitX86MaskedCompareResult(*this, Fpclass, NumElts, MaskIn);
10111   }
10112 
10113   // packed comparison intrinsics
10114   case X86::BI__builtin_ia32_cmpeqps:
10115   case X86::BI__builtin_ia32_cmpeqpd:
10116     return getVectorFCmpIR(CmpInst::FCMP_OEQ);
10117   case X86::BI__builtin_ia32_cmpltps:
10118   case X86::BI__builtin_ia32_cmpltpd:
10119     return getVectorFCmpIR(CmpInst::FCMP_OLT);
10120   case X86::BI__builtin_ia32_cmpleps:
10121   case X86::BI__builtin_ia32_cmplepd:
10122     return getVectorFCmpIR(CmpInst::FCMP_OLE);
10123   case X86::BI__builtin_ia32_cmpunordps:
10124   case X86::BI__builtin_ia32_cmpunordpd:
10125     return getVectorFCmpIR(CmpInst::FCMP_UNO);
10126   case X86::BI__builtin_ia32_cmpneqps:
10127   case X86::BI__builtin_ia32_cmpneqpd:
10128     return getVectorFCmpIR(CmpInst::FCMP_UNE);
10129   case X86::BI__builtin_ia32_cmpnltps:
10130   case X86::BI__builtin_ia32_cmpnltpd:
10131     return getVectorFCmpIR(CmpInst::FCMP_UGE);
10132   case X86::BI__builtin_ia32_cmpnleps:
10133   case X86::BI__builtin_ia32_cmpnlepd:
10134     return getVectorFCmpIR(CmpInst::FCMP_UGT);
10135   case X86::BI__builtin_ia32_cmpordps:
10136   case X86::BI__builtin_ia32_cmpordpd:
10137     return getVectorFCmpIR(CmpInst::FCMP_ORD);
10138   case X86::BI__builtin_ia32_cmpps:
10139   case X86::BI__builtin_ia32_cmpps256:
10140   case X86::BI__builtin_ia32_cmppd:
10141   case X86::BI__builtin_ia32_cmppd256:
10142   case X86::BI__builtin_ia32_cmpps128_mask:
10143   case X86::BI__builtin_ia32_cmpps256_mask:
10144   case X86::BI__builtin_ia32_cmpps512_mask:
10145   case X86::BI__builtin_ia32_cmppd128_mask:
10146   case X86::BI__builtin_ia32_cmppd256_mask:
10147   case X86::BI__builtin_ia32_cmppd512_mask: {
10148     // Lowering vector comparisons to fcmp instructions, while
10149     // ignoring signalling behaviour requested
10150     // ignoring rounding mode requested
10151     // This is is only possible as long as FENV_ACCESS is not implemented.
10152     // See also: https://reviews.llvm.org/D45616
10153 
10154     // The third argument is the comparison condition, and integer in the
10155     // range [0, 31]
10156     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
10157 
10158     // Lowering to IR fcmp instruction.
10159     // Ignoring requested signaling behaviour,
10160     // e.g. both _CMP_GT_OS & _CMP_GT_OQ are translated to FCMP_OGT.
10161     FCmpInst::Predicate Pred;
10162     switch (CC) {
10163     case 0x00: Pred = FCmpInst::FCMP_OEQ; break;
10164     case 0x01: Pred = FCmpInst::FCMP_OLT; break;
10165     case 0x02: Pred = FCmpInst::FCMP_OLE; break;
10166     case 0x03: Pred = FCmpInst::FCMP_UNO; break;
10167     case 0x04: Pred = FCmpInst::FCMP_UNE; break;
10168     case 0x05: Pred = FCmpInst::FCMP_UGE; break;
10169     case 0x06: Pred = FCmpInst::FCMP_UGT; break;
10170     case 0x07: Pred = FCmpInst::FCMP_ORD; break;
10171     case 0x08: Pred = FCmpInst::FCMP_UEQ; break;
10172     case 0x09: Pred = FCmpInst::FCMP_ULT; break;
10173     case 0x0a: Pred = FCmpInst::FCMP_ULE; break;
10174     case 0x0c: Pred = FCmpInst::FCMP_ONE; break;
10175     case 0x0d: Pred = FCmpInst::FCMP_OGE; break;
10176     case 0x0e: Pred = FCmpInst::FCMP_OGT; break;
10177     case 0x10: Pred = FCmpInst::FCMP_OEQ; break;
10178     case 0x11: Pred = FCmpInst::FCMP_OLT; break;
10179     case 0x12: Pred = FCmpInst::FCMP_OLE; break;
10180     case 0x13: Pred = FCmpInst::FCMP_UNO; break;
10181     case 0x14: Pred = FCmpInst::FCMP_UNE; break;
10182     case 0x15: Pred = FCmpInst::FCMP_UGE; break;
10183     case 0x16: Pred = FCmpInst::FCMP_UGT; break;
10184     case 0x17: Pred = FCmpInst::FCMP_ORD; break;
10185     case 0x18: Pred = FCmpInst::FCMP_UEQ; break;
10186     case 0x19: Pred = FCmpInst::FCMP_ULT; break;
10187     case 0x1a: Pred = FCmpInst::FCMP_ULE; break;
10188     case 0x1c: Pred = FCmpInst::FCMP_ONE; break;
10189     case 0x1d: Pred = FCmpInst::FCMP_OGE; break;
10190     case 0x1e: Pred = FCmpInst::FCMP_OGT; break;
10191     // _CMP_TRUE_UQ, _CMP_TRUE_US produce -1,-1... vector
10192     // on any input and _CMP_FALSE_OQ, _CMP_FALSE_OS produce 0, 0...
10193     case 0x0b: // FALSE_OQ
10194     case 0x1b: // FALSE_OS
10195       return llvm::Constant::getNullValue(ConvertType(E->getType()));
10196     case 0x0f: // TRUE_UQ
10197     case 0x1f: // TRUE_US
10198       return llvm::Constant::getAllOnesValue(ConvertType(E->getType()));
10199 
10200     default: llvm_unreachable("Unhandled CC");
10201     }
10202 
10203     // Builtins without the _mask suffix return a vector of integers
10204     // of the same width as the input vectors
10205     switch (BuiltinID) {
10206     case X86::BI__builtin_ia32_cmpps512_mask:
10207     case X86::BI__builtin_ia32_cmppd512_mask:
10208     case X86::BI__builtin_ia32_cmpps128_mask:
10209     case X86::BI__builtin_ia32_cmpps256_mask:
10210     case X86::BI__builtin_ia32_cmppd128_mask:
10211     case X86::BI__builtin_ia32_cmppd256_mask: {
10212       unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10213       Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
10214       return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]);
10215     }
10216     default:
10217       return getVectorFCmpIR(Pred);
10218     }
10219   }
10220 
10221   // SSE scalar comparison intrinsics
10222   case X86::BI__builtin_ia32_cmpeqss:
10223     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
10224   case X86::BI__builtin_ia32_cmpltss:
10225     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
10226   case X86::BI__builtin_ia32_cmpless:
10227     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
10228   case X86::BI__builtin_ia32_cmpunordss:
10229     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
10230   case X86::BI__builtin_ia32_cmpneqss:
10231     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
10232   case X86::BI__builtin_ia32_cmpnltss:
10233     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
10234   case X86::BI__builtin_ia32_cmpnless:
10235     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
10236   case X86::BI__builtin_ia32_cmpordss:
10237     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
10238   case X86::BI__builtin_ia32_cmpeqsd:
10239     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
10240   case X86::BI__builtin_ia32_cmpltsd:
10241     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
10242   case X86::BI__builtin_ia32_cmplesd:
10243     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
10244   case X86::BI__builtin_ia32_cmpunordsd:
10245     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
10246   case X86::BI__builtin_ia32_cmpneqsd:
10247     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
10248   case X86::BI__builtin_ia32_cmpnltsd:
10249     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
10250   case X86::BI__builtin_ia32_cmpnlesd:
10251     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
10252   case X86::BI__builtin_ia32_cmpordsd:
10253     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
10254 
10255   case X86::BI__emul:
10256   case X86::BI__emulu: {
10257     llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
10258     bool isSigned = (BuiltinID == X86::BI__emul);
10259     Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
10260     Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
10261     return Builder.CreateMul(LHS, RHS, "", !isSigned, isSigned);
10262   }
10263   case X86::BI__mulh:
10264   case X86::BI__umulh:
10265   case X86::BI_mul128:
10266   case X86::BI_umul128: {
10267     llvm::Type *ResType = ConvertType(E->getType());
10268     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
10269 
10270     bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
10271     Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
10272     Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
10273 
10274     Value *MulResult, *HigherBits;
10275     if (IsSigned) {
10276       MulResult = Builder.CreateNSWMul(LHS, RHS);
10277       HigherBits = Builder.CreateAShr(MulResult, 64);
10278     } else {
10279       MulResult = Builder.CreateNUWMul(LHS, RHS);
10280       HigherBits = Builder.CreateLShr(MulResult, 64);
10281     }
10282     HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
10283 
10284     if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
10285       return HigherBits;
10286 
10287     Address HighBitsAddress = EmitPointerWithAlignment(E->getArg(2));
10288     Builder.CreateStore(HigherBits, HighBitsAddress);
10289     return Builder.CreateIntCast(MulResult, ResType, IsSigned);
10290   }
10291 
10292   case X86::BI__faststorefence: {
10293     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
10294                                llvm::SyncScope::System);
10295   }
10296   case X86::BI_ReadWriteBarrier:
10297   case X86::BI_ReadBarrier:
10298   case X86::BI_WriteBarrier: {
10299     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
10300                                llvm::SyncScope::SingleThread);
10301   }
10302   case X86::BI_BitScanForward:
10303   case X86::BI_BitScanForward64:
10304     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
10305   case X86::BI_BitScanReverse:
10306   case X86::BI_BitScanReverse64:
10307     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
10308 
10309   case X86::BI_InterlockedAnd64:
10310     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
10311   case X86::BI_InterlockedExchange64:
10312     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
10313   case X86::BI_InterlockedExchangeAdd64:
10314     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
10315   case X86::BI_InterlockedExchangeSub64:
10316     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
10317   case X86::BI_InterlockedOr64:
10318     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
10319   case X86::BI_InterlockedXor64:
10320     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
10321   case X86::BI_InterlockedDecrement64:
10322     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
10323   case X86::BI_InterlockedIncrement64:
10324     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
10325   case X86::BI_InterlockedCompareExchange128: {
10326     // InterlockedCompareExchange128 doesn't directly refer to 128bit ints,
10327     // instead it takes pointers to 64bit ints for Destination and
10328     // ComparandResult, and exchange is taken as two 64bit ints (high & low).
10329     // The previous value is written to ComparandResult, and success is
10330     // returned.
10331 
10332     llvm::Type *Int128Ty = Builder.getInt128Ty();
10333     llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
10334 
10335     Value *Destination =
10336         Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PtrTy);
10337     Value *ExchangeHigh128 =
10338         Builder.CreateZExt(EmitScalarExpr(E->getArg(1)), Int128Ty);
10339     Value *ExchangeLow128 =
10340         Builder.CreateZExt(EmitScalarExpr(E->getArg(2)), Int128Ty);
10341     Address ComparandResult(
10342         Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int128PtrTy),
10343         getContext().toCharUnitsFromBits(128));
10344 
10345     Value *Exchange = Builder.CreateOr(
10346         Builder.CreateShl(ExchangeHigh128, 64, "", false, false),
10347         ExchangeLow128);
10348 
10349     Value *Comparand = Builder.CreateLoad(ComparandResult);
10350 
10351     AtomicCmpXchgInst *CXI =
10352         Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
10353                                     AtomicOrdering::SequentiallyConsistent,
10354                                     AtomicOrdering::SequentiallyConsistent);
10355     CXI->setVolatile(true);
10356 
10357     // Write the result back to the inout pointer.
10358     Builder.CreateStore(Builder.CreateExtractValue(CXI, 0), ComparandResult);
10359 
10360     // Get the success boolean and zero extend it to i8.
10361     Value *Success = Builder.CreateExtractValue(CXI, 1);
10362     return Builder.CreateZExt(Success, ConvertType(E->getType()));
10363   }
10364 
10365   case X86::BI_AddressOfReturnAddress: {
10366     Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);
10367     return Builder.CreateCall(F);
10368   }
10369   case X86::BI__stosb: {
10370     // We treat __stosb as a volatile memset - it may not generate "rep stosb"
10371     // instruction, but it will create a memset that won't be optimized away.
10372     return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1, true);
10373   }
10374   case X86::BI__ud2:
10375     // llvm.trap makes a ud2a instruction on x86.
10376     return EmitTrapCall(Intrinsic::trap);
10377   case X86::BI__int2c: {
10378     // This syscall signals a driver assertion failure in x86 NT kernels.
10379     llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
10380     llvm::InlineAsm *IA =
10381         llvm::InlineAsm::get(FTy, "int $$0x2c", "", /*SideEffects=*/true);
10382     llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
10383         getLLVMContext(), llvm::AttributeList::FunctionIndex,
10384         llvm::Attribute::NoReturn);
10385     CallSite CS = Builder.CreateCall(IA);
10386     CS.setAttributes(NoReturnAttr);
10387     return CS.getInstruction();
10388   }
10389   case X86::BI__readfsbyte:
10390   case X86::BI__readfsword:
10391   case X86::BI__readfsdword:
10392   case X86::BI__readfsqword: {
10393     llvm::Type *IntTy = ConvertType(E->getType());
10394     Value *Ptr = Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)),
10395                                         llvm::PointerType::get(IntTy, 257));
10396     LoadInst *Load = Builder.CreateAlignedLoad(
10397         IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
10398     Load->setVolatile(true);
10399     return Load;
10400   }
10401   case X86::BI__readgsbyte:
10402   case X86::BI__readgsword:
10403   case X86::BI__readgsdword:
10404   case X86::BI__readgsqword: {
10405     llvm::Type *IntTy = ConvertType(E->getType());
10406     Value *Ptr = Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)),
10407                                         llvm::PointerType::get(IntTy, 256));
10408     LoadInst *Load = Builder.CreateAlignedLoad(
10409         IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
10410     Load->setVolatile(true);
10411     return Load;
10412   }
10413   }
10414 }
10415 
10416 
10417 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
10418                                            const CallExpr *E) {
10419   SmallVector<Value*, 4> Ops;
10420 
10421   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
10422     Ops.push_back(EmitScalarExpr(E->getArg(i)));
10423 
10424   Intrinsic::ID ID = Intrinsic::not_intrinsic;
10425 
10426   switch (BuiltinID) {
10427   default: return nullptr;
10428 
10429   // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we
10430   // call __builtin_readcyclecounter.
10431   case PPC::BI__builtin_ppc_get_timebase:
10432     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
10433 
10434   // vec_ld, vec_xl_be, vec_lvsl, vec_lvsr
10435   case PPC::BI__builtin_altivec_lvx:
10436   case PPC::BI__builtin_altivec_lvxl:
10437   case PPC::BI__builtin_altivec_lvebx:
10438   case PPC::BI__builtin_altivec_lvehx:
10439   case PPC::BI__builtin_altivec_lvewx:
10440   case PPC::BI__builtin_altivec_lvsl:
10441   case PPC::BI__builtin_altivec_lvsr:
10442   case PPC::BI__builtin_vsx_lxvd2x:
10443   case PPC::BI__builtin_vsx_lxvw4x:
10444   case PPC::BI__builtin_vsx_lxvd2x_be:
10445   case PPC::BI__builtin_vsx_lxvw4x_be:
10446   case PPC::BI__builtin_vsx_lxvl:
10447   case PPC::BI__builtin_vsx_lxvll:
10448   {
10449     if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
10450        BuiltinID == PPC::BI__builtin_vsx_lxvll){
10451       Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
10452     }else {
10453       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
10454       Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
10455       Ops.pop_back();
10456     }
10457 
10458     switch (BuiltinID) {
10459     default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
10460     case PPC::BI__builtin_altivec_lvx:
10461       ID = Intrinsic::ppc_altivec_lvx;
10462       break;
10463     case PPC::BI__builtin_altivec_lvxl:
10464       ID = Intrinsic::ppc_altivec_lvxl;
10465       break;
10466     case PPC::BI__builtin_altivec_lvebx:
10467       ID = Intrinsic::ppc_altivec_lvebx;
10468       break;
10469     case PPC::BI__builtin_altivec_lvehx:
10470       ID = Intrinsic::ppc_altivec_lvehx;
10471       break;
10472     case PPC::BI__builtin_altivec_lvewx:
10473       ID = Intrinsic::ppc_altivec_lvewx;
10474       break;
10475     case PPC::BI__builtin_altivec_lvsl:
10476       ID = Intrinsic::ppc_altivec_lvsl;
10477       break;
10478     case PPC::BI__builtin_altivec_lvsr:
10479       ID = Intrinsic::ppc_altivec_lvsr;
10480       break;
10481     case PPC::BI__builtin_vsx_lxvd2x:
10482       ID = Intrinsic::ppc_vsx_lxvd2x;
10483       break;
10484     case PPC::BI__builtin_vsx_lxvw4x:
10485       ID = Intrinsic::ppc_vsx_lxvw4x;
10486       break;
10487     case PPC::BI__builtin_vsx_lxvd2x_be:
10488       ID = Intrinsic::ppc_vsx_lxvd2x_be;
10489       break;
10490     case PPC::BI__builtin_vsx_lxvw4x_be:
10491       ID = Intrinsic::ppc_vsx_lxvw4x_be;
10492       break;
10493     case PPC::BI__builtin_vsx_lxvl:
10494       ID = Intrinsic::ppc_vsx_lxvl;
10495       break;
10496     case PPC::BI__builtin_vsx_lxvll:
10497       ID = Intrinsic::ppc_vsx_lxvll;
10498       break;
10499     }
10500     llvm::Function *F = CGM.getIntrinsic(ID);
10501     return Builder.CreateCall(F, Ops, "");
10502   }
10503 
10504   // vec_st, vec_xst_be
10505   case PPC::BI__builtin_altivec_stvx:
10506   case PPC::BI__builtin_altivec_stvxl:
10507   case PPC::BI__builtin_altivec_stvebx:
10508   case PPC::BI__builtin_altivec_stvehx:
10509   case PPC::BI__builtin_altivec_stvewx:
10510   case PPC::BI__builtin_vsx_stxvd2x:
10511   case PPC::BI__builtin_vsx_stxvw4x:
10512   case PPC::BI__builtin_vsx_stxvd2x_be:
10513   case PPC::BI__builtin_vsx_stxvw4x_be:
10514   case PPC::BI__builtin_vsx_stxvl:
10515   case PPC::BI__builtin_vsx_stxvll:
10516   {
10517     if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
10518       BuiltinID == PPC::BI__builtin_vsx_stxvll ){
10519       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
10520     }else {
10521       Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
10522       Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
10523       Ops.pop_back();
10524     }
10525 
10526     switch (BuiltinID) {
10527     default: llvm_unreachable("Unsupported st intrinsic!");
10528     case PPC::BI__builtin_altivec_stvx:
10529       ID = Intrinsic::ppc_altivec_stvx;
10530       break;
10531     case PPC::BI__builtin_altivec_stvxl:
10532       ID = Intrinsic::ppc_altivec_stvxl;
10533       break;
10534     case PPC::BI__builtin_altivec_stvebx:
10535       ID = Intrinsic::ppc_altivec_stvebx;
10536       break;
10537     case PPC::BI__builtin_altivec_stvehx:
10538       ID = Intrinsic::ppc_altivec_stvehx;
10539       break;
10540     case PPC::BI__builtin_altivec_stvewx:
10541       ID = Intrinsic::ppc_altivec_stvewx;
10542       break;
10543     case PPC::BI__builtin_vsx_stxvd2x:
10544       ID = Intrinsic::ppc_vsx_stxvd2x;
10545       break;
10546     case PPC::BI__builtin_vsx_stxvw4x:
10547       ID = Intrinsic::ppc_vsx_stxvw4x;
10548       break;
10549     case PPC::BI__builtin_vsx_stxvd2x_be:
10550       ID = Intrinsic::ppc_vsx_stxvd2x_be;
10551       break;
10552     case PPC::BI__builtin_vsx_stxvw4x_be:
10553       ID = Intrinsic::ppc_vsx_stxvw4x_be;
10554       break;
10555     case PPC::BI__builtin_vsx_stxvl:
10556       ID = Intrinsic::ppc_vsx_stxvl;
10557       break;
10558     case PPC::BI__builtin_vsx_stxvll:
10559       ID = Intrinsic::ppc_vsx_stxvll;
10560       break;
10561     }
10562     llvm::Function *F = CGM.getIntrinsic(ID);
10563     return Builder.CreateCall(F, Ops, "");
10564   }
10565   // Square root
10566   case PPC::BI__builtin_vsx_xvsqrtsp:
10567   case PPC::BI__builtin_vsx_xvsqrtdp: {
10568     llvm::Type *ResultType = ConvertType(E->getType());
10569     Value *X = EmitScalarExpr(E->getArg(0));
10570     ID = Intrinsic::sqrt;
10571     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10572     return Builder.CreateCall(F, X);
10573   }
10574   // Count leading zeros
10575   case PPC::BI__builtin_altivec_vclzb:
10576   case PPC::BI__builtin_altivec_vclzh:
10577   case PPC::BI__builtin_altivec_vclzw:
10578   case PPC::BI__builtin_altivec_vclzd: {
10579     llvm::Type *ResultType = ConvertType(E->getType());
10580     Value *X = EmitScalarExpr(E->getArg(0));
10581     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
10582     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
10583     return Builder.CreateCall(F, {X, Undef});
10584   }
10585   case PPC::BI__builtin_altivec_vctzb:
10586   case PPC::BI__builtin_altivec_vctzh:
10587   case PPC::BI__builtin_altivec_vctzw:
10588   case PPC::BI__builtin_altivec_vctzd: {
10589     llvm::Type *ResultType = ConvertType(E->getType());
10590     Value *X = EmitScalarExpr(E->getArg(0));
10591     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
10592     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
10593     return Builder.CreateCall(F, {X, Undef});
10594   }
10595   case PPC::BI__builtin_altivec_vpopcntb:
10596   case PPC::BI__builtin_altivec_vpopcnth:
10597   case PPC::BI__builtin_altivec_vpopcntw:
10598   case PPC::BI__builtin_altivec_vpopcntd: {
10599     llvm::Type *ResultType = ConvertType(E->getType());
10600     Value *X = EmitScalarExpr(E->getArg(0));
10601     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
10602     return Builder.CreateCall(F, X);
10603   }
10604   // Copy sign
10605   case PPC::BI__builtin_vsx_xvcpsgnsp:
10606   case PPC::BI__builtin_vsx_xvcpsgndp: {
10607     llvm::Type *ResultType = ConvertType(E->getType());
10608     Value *X = EmitScalarExpr(E->getArg(0));
10609     Value *Y = EmitScalarExpr(E->getArg(1));
10610     ID = Intrinsic::copysign;
10611     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10612     return Builder.CreateCall(F, {X, Y});
10613   }
10614   // Rounding/truncation
10615   case PPC::BI__builtin_vsx_xvrspip:
10616   case PPC::BI__builtin_vsx_xvrdpip:
10617   case PPC::BI__builtin_vsx_xvrdpim:
10618   case PPC::BI__builtin_vsx_xvrspim:
10619   case PPC::BI__builtin_vsx_xvrdpi:
10620   case PPC::BI__builtin_vsx_xvrspi:
10621   case PPC::BI__builtin_vsx_xvrdpic:
10622   case PPC::BI__builtin_vsx_xvrspic:
10623   case PPC::BI__builtin_vsx_xvrdpiz:
10624   case PPC::BI__builtin_vsx_xvrspiz: {
10625     llvm::Type *ResultType = ConvertType(E->getType());
10626     Value *X = EmitScalarExpr(E->getArg(0));
10627     if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
10628         BuiltinID == PPC::BI__builtin_vsx_xvrspim)
10629       ID = Intrinsic::floor;
10630     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
10631              BuiltinID == PPC::BI__builtin_vsx_xvrspi)
10632       ID = Intrinsic::round;
10633     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
10634              BuiltinID == PPC::BI__builtin_vsx_xvrspic)
10635       ID = Intrinsic::nearbyint;
10636     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
10637              BuiltinID == PPC::BI__builtin_vsx_xvrspip)
10638       ID = Intrinsic::ceil;
10639     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
10640              BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
10641       ID = Intrinsic::trunc;
10642     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10643     return Builder.CreateCall(F, X);
10644   }
10645 
10646   // Absolute value
10647   case PPC::BI__builtin_vsx_xvabsdp:
10648   case PPC::BI__builtin_vsx_xvabssp: {
10649     llvm::Type *ResultType = ConvertType(E->getType());
10650     Value *X = EmitScalarExpr(E->getArg(0));
10651     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
10652     return Builder.CreateCall(F, X);
10653   }
10654 
10655   // FMA variations
10656   case PPC::BI__builtin_vsx_xvmaddadp:
10657   case PPC::BI__builtin_vsx_xvmaddasp:
10658   case PPC::BI__builtin_vsx_xvnmaddadp:
10659   case PPC::BI__builtin_vsx_xvnmaddasp:
10660   case PPC::BI__builtin_vsx_xvmsubadp:
10661   case PPC::BI__builtin_vsx_xvmsubasp:
10662   case PPC::BI__builtin_vsx_xvnmsubadp:
10663   case PPC::BI__builtin_vsx_xvnmsubasp: {
10664     llvm::Type *ResultType = ConvertType(E->getType());
10665     Value *X = EmitScalarExpr(E->getArg(0));
10666     Value *Y = EmitScalarExpr(E->getArg(1));
10667     Value *Z = EmitScalarExpr(E->getArg(2));
10668     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
10669     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
10670     switch (BuiltinID) {
10671       case PPC::BI__builtin_vsx_xvmaddadp:
10672       case PPC::BI__builtin_vsx_xvmaddasp:
10673         return Builder.CreateCall(F, {X, Y, Z});
10674       case PPC::BI__builtin_vsx_xvnmaddadp:
10675       case PPC::BI__builtin_vsx_xvnmaddasp:
10676         return Builder.CreateFSub(Zero,
10677                                   Builder.CreateCall(F, {X, Y, Z}), "sub");
10678       case PPC::BI__builtin_vsx_xvmsubadp:
10679       case PPC::BI__builtin_vsx_xvmsubasp:
10680         return Builder.CreateCall(F,
10681                                   {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
10682       case PPC::BI__builtin_vsx_xvnmsubadp:
10683       case PPC::BI__builtin_vsx_xvnmsubasp:
10684         Value *FsubRes =
10685           Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
10686         return Builder.CreateFSub(Zero, FsubRes, "sub");
10687     }
10688     llvm_unreachable("Unknown FMA operation");
10689     return nullptr; // Suppress no-return warning
10690   }
10691 
10692   case PPC::BI__builtin_vsx_insertword: {
10693     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
10694 
10695     // Third argument is a compile time constant int. It must be clamped to
10696     // to the range [0, 12].
10697     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10698     assert(ArgCI &&
10699            "Third arg to xxinsertw intrinsic must be constant integer");
10700     const int64_t MaxIndex = 12;
10701     int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex);
10702 
10703     // The builtin semantics don't exactly match the xxinsertw instructions
10704     // semantics (which ppc_vsx_xxinsertw follows). The builtin extracts the
10705     // word from the first argument, and inserts it in the second argument. The
10706     // instruction extracts the word from its second input register and inserts
10707     // it into its first input register, so swap the first and second arguments.
10708     std::swap(Ops[0], Ops[1]);
10709 
10710     // Need to cast the second argument from a vector of unsigned int to a
10711     // vector of long long.
10712     Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
10713 
10714     if (getTarget().isLittleEndian()) {
10715       // Create a shuffle mask of (1, 0)
10716       Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
10717                                    ConstantInt::get(Int32Ty, 0)
10718                                  };
10719       Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10720 
10721       // Reverse the double words in the vector we will extract from.
10722       Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10723       Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask);
10724 
10725       // Reverse the index.
10726       Index = MaxIndex - Index;
10727     }
10728 
10729     // Intrinsic expects the first arg to be a vector of int.
10730     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
10731     Ops[2] = ConstantInt::getSigned(Int32Ty, Index);
10732     return Builder.CreateCall(F, Ops);
10733   }
10734 
10735   case PPC::BI__builtin_vsx_extractuword: {
10736     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
10737 
10738     // Intrinsic expects the first argument to be a vector of doublewords.
10739     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10740 
10741     // The second argument is a compile time constant int that needs to
10742     // be clamped to the range [0, 12].
10743     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
10744     assert(ArgCI &&
10745            "Second Arg to xxextractuw intrinsic must be a constant integer!");
10746     const int64_t MaxIndex = 12;
10747     int64_t Index = clamp(ArgCI->getSExtValue(), 0, MaxIndex);
10748 
10749     if (getTarget().isLittleEndian()) {
10750       // Reverse the index.
10751       Index = MaxIndex - Index;
10752       Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
10753 
10754       // Emit the call, then reverse the double words of the results vector.
10755       Value *Call = Builder.CreateCall(F, Ops);
10756 
10757       // Create a shuffle mask of (1, 0)
10758       Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
10759                                    ConstantInt::get(Int32Ty, 0)
10760                                  };
10761       Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10762 
10763       Value *ShuffleCall = Builder.CreateShuffleVector(Call, Call, ShuffleMask);
10764       return ShuffleCall;
10765     } else {
10766       Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
10767       return Builder.CreateCall(F, Ops);
10768     }
10769   }
10770 
10771   case PPC::BI__builtin_vsx_xxpermdi: {
10772     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10773     assert(ArgCI && "Third arg must be constant integer!");
10774 
10775     unsigned Index = ArgCI->getZExtValue();
10776     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10777     Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
10778 
10779     // Element zero comes from the first input vector and element one comes from
10780     // the second. The element indices within each vector are numbered in big
10781     // endian order so the shuffle mask must be adjusted for this on little
10782     // endian platforms (i.e. index is complemented and source vector reversed).
10783     unsigned ElemIdx0;
10784     unsigned ElemIdx1;
10785     if (getTarget().isLittleEndian()) {
10786       ElemIdx0 = (~Index & 1) + 2;
10787       ElemIdx1 = (~Index & 2) >> 1;
10788     } else { // BigEndian
10789       ElemIdx0 = (Index & 2) >> 1;
10790       ElemIdx1 = 2 + (Index & 1);
10791     }
10792 
10793     Constant *ShuffleElts[2] = {ConstantInt::get(Int32Ty, ElemIdx0),
10794                                 ConstantInt::get(Int32Ty, ElemIdx1)};
10795     Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10796 
10797     Value *ShuffleCall =
10798         Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
10799     QualType BIRetType = E->getType();
10800     auto RetTy = ConvertType(BIRetType);
10801     return Builder.CreateBitCast(ShuffleCall, RetTy);
10802   }
10803 
10804   case PPC::BI__builtin_vsx_xxsldwi: {
10805     ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10806     assert(ArgCI && "Third argument must be a compile time constant");
10807     unsigned Index = ArgCI->getZExtValue() & 0x3;
10808     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
10809     Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int32Ty, 4));
10810 
10811     // Create a shuffle mask
10812     unsigned ElemIdx0;
10813     unsigned ElemIdx1;
10814     unsigned ElemIdx2;
10815     unsigned ElemIdx3;
10816     if (getTarget().isLittleEndian()) {
10817       // Little endian element N comes from element 8+N-Index of the
10818       // concatenated wide vector (of course, using modulo arithmetic on
10819       // the total number of elements).
10820       ElemIdx0 = (8 - Index) % 8;
10821       ElemIdx1 = (9 - Index) % 8;
10822       ElemIdx2 = (10 - Index) % 8;
10823       ElemIdx3 = (11 - Index) % 8;
10824     } else {
10825       // Big endian ElemIdx<N> = Index + N
10826       ElemIdx0 = Index;
10827       ElemIdx1 = Index + 1;
10828       ElemIdx2 = Index + 2;
10829       ElemIdx3 = Index + 3;
10830     }
10831 
10832     Constant *ShuffleElts[4] = {ConstantInt::get(Int32Ty, ElemIdx0),
10833                                 ConstantInt::get(Int32Ty, ElemIdx1),
10834                                 ConstantInt::get(Int32Ty, ElemIdx2),
10835                                 ConstantInt::get(Int32Ty, ElemIdx3)};
10836 
10837     Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10838     Value *ShuffleCall =
10839         Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
10840     QualType BIRetType = E->getType();
10841     auto RetTy = ConvertType(BIRetType);
10842     return Builder.CreateBitCast(ShuffleCall, RetTy);
10843   }
10844   }
10845 }
10846 
10847 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
10848                                               const CallExpr *E) {
10849   switch (BuiltinID) {
10850   case AMDGPU::BI__builtin_amdgcn_div_scale:
10851   case AMDGPU::BI__builtin_amdgcn_div_scalef: {
10852     // Translate from the intrinsics's struct return to the builtin's out
10853     // argument.
10854 
10855     Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3));
10856 
10857     llvm::Value *X = EmitScalarExpr(E->getArg(0));
10858     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
10859     llvm::Value *Z = EmitScalarExpr(E->getArg(2));
10860 
10861     llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
10862                                            X->getType());
10863 
10864     llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z});
10865 
10866     llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
10867     llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
10868 
10869     llvm::Type *RealFlagType
10870       = FlagOutPtr.getPointer()->getType()->getPointerElementType();
10871 
10872     llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
10873     Builder.CreateStore(FlagExt, FlagOutPtr);
10874     return Result;
10875   }
10876   case AMDGPU::BI__builtin_amdgcn_div_fmas:
10877   case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
10878     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
10879     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
10880     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
10881     llvm::Value *Src3 = EmitScalarExpr(E->getArg(3));
10882 
10883     llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
10884                                       Src0->getType());
10885     llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
10886     return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
10887   }
10888 
10889   case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
10890     return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_ds_swizzle);
10891   case AMDGPU::BI__builtin_amdgcn_mov_dpp: {
10892     llvm::SmallVector<llvm::Value *, 5> Args;
10893     for (unsigned I = 0; I != 5; ++I)
10894       Args.push_back(EmitScalarExpr(E->getArg(I)));
10895     Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_mov_dpp,
10896                                     Args[0]->getType());
10897     return Builder.CreateCall(F, Args);
10898   }
10899   case AMDGPU::BI__builtin_amdgcn_div_fixup:
10900   case AMDGPU::BI__builtin_amdgcn_div_fixupf:
10901   case AMDGPU::BI__builtin_amdgcn_div_fixuph:
10902     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup);
10903   case AMDGPU::BI__builtin_amdgcn_trig_preop:
10904   case AMDGPU::BI__builtin_amdgcn_trig_preopf:
10905     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop);
10906   case AMDGPU::BI__builtin_amdgcn_rcp:
10907   case AMDGPU::BI__builtin_amdgcn_rcpf:
10908   case AMDGPU::BI__builtin_amdgcn_rcph:
10909     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp);
10910   case AMDGPU::BI__builtin_amdgcn_rsq:
10911   case AMDGPU::BI__builtin_amdgcn_rsqf:
10912   case AMDGPU::BI__builtin_amdgcn_rsqh:
10913     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq);
10914   case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
10915   case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
10916     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp);
10917   case AMDGPU::BI__builtin_amdgcn_sinf:
10918   case AMDGPU::BI__builtin_amdgcn_sinh:
10919     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin);
10920   case AMDGPU::BI__builtin_amdgcn_cosf:
10921   case AMDGPU::BI__builtin_amdgcn_cosh:
10922     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos);
10923   case AMDGPU::BI__builtin_amdgcn_log_clampf:
10924     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp);
10925   case AMDGPU::BI__builtin_amdgcn_ldexp:
10926   case AMDGPU::BI__builtin_amdgcn_ldexpf:
10927   case AMDGPU::BI__builtin_amdgcn_ldexph:
10928     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp);
10929   case AMDGPU::BI__builtin_amdgcn_frexp_mant:
10930   case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
10931   case AMDGPU::BI__builtin_amdgcn_frexp_manth:
10932     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant);
10933   case AMDGPU::BI__builtin_amdgcn_frexp_exp:
10934   case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
10935     Value *Src0 = EmitScalarExpr(E->getArg(0));
10936     Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
10937                                 { Builder.getInt32Ty(), Src0->getType() });
10938     return Builder.CreateCall(F, Src0);
10939   }
10940   case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
10941     Value *Src0 = EmitScalarExpr(E->getArg(0));
10942     Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
10943                                 { Builder.getInt16Ty(), Src0->getType() });
10944     return Builder.CreateCall(F, Src0);
10945   }
10946   case AMDGPU::BI__builtin_amdgcn_fract:
10947   case AMDGPU::BI__builtin_amdgcn_fractf:
10948   case AMDGPU::BI__builtin_amdgcn_fracth:
10949     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract);
10950   case AMDGPU::BI__builtin_amdgcn_lerp:
10951     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_lerp);
10952   case AMDGPU::BI__builtin_amdgcn_uicmp:
10953   case AMDGPU::BI__builtin_amdgcn_uicmpl:
10954   case AMDGPU::BI__builtin_amdgcn_sicmp:
10955   case AMDGPU::BI__builtin_amdgcn_sicmpl:
10956     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_icmp);
10957   case AMDGPU::BI__builtin_amdgcn_fcmp:
10958   case AMDGPU::BI__builtin_amdgcn_fcmpf:
10959     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fcmp);
10960   case AMDGPU::BI__builtin_amdgcn_class:
10961   case AMDGPU::BI__builtin_amdgcn_classf:
10962   case AMDGPU::BI__builtin_amdgcn_classh:
10963     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class);
10964   case AMDGPU::BI__builtin_amdgcn_fmed3f:
10965   case AMDGPU::BI__builtin_amdgcn_fmed3h:
10966     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fmed3);
10967   case AMDGPU::BI__builtin_amdgcn_read_exec: {
10968     CallInst *CI = cast<CallInst>(
10969       EmitSpecialRegisterBuiltin(*this, E, Int64Ty, Int64Ty, true, "exec"));
10970     CI->setConvergent();
10971     return CI;
10972   }
10973   case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
10974   case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
10975     StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
10976       "exec_lo" : "exec_hi";
10977     CallInst *CI = cast<CallInst>(
10978       EmitSpecialRegisterBuiltin(*this, E, Int32Ty, Int32Ty, true, RegName));
10979     CI->setConvergent();
10980     return CI;
10981   }
10982   case AMDGPU::BI__builtin_amdgcn_ds_faddf:
10983   case AMDGPU::BI__builtin_amdgcn_ds_fminf:
10984   case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
10985     llvm::SmallVector<llvm::Value *, 5> Args;
10986     for (unsigned I = 0; I != 5; ++I)
10987       Args.push_back(EmitScalarExpr(E->getArg(I)));
10988     const llvm::Type *PtrTy = Args[0]->getType();
10989     // check pointer parameter
10990     if (!PtrTy->isPointerTy() ||
10991         E->getArg(0)
10992                 ->getType()
10993                 ->getPointeeType()
10994                 .getQualifiers()
10995                 .getAddressSpace() != LangAS::opencl_local ||
10996         !PtrTy->getPointerElementType()->isFloatTy()) {
10997        CGM.Error(E->getArg(0)->getLocStart(),
10998                 "parameter should have type \"local float*\"");
10999       return nullptr;
11000     }
11001     // check float parameter
11002     if (!Args[1]->getType()->isFloatTy()) {
11003       CGM.Error(E->getArg(1)->getLocStart(),
11004                 "parameter should have type \"float\"");
11005       return nullptr;
11006     }
11007 
11008     Intrinsic::ID ID;
11009     switch (BuiltinID) {
11010     case AMDGPU::BI__builtin_amdgcn_ds_faddf:
11011       ID = Intrinsic::amdgcn_ds_fadd;
11012       break;
11013     case AMDGPU::BI__builtin_amdgcn_ds_fminf:
11014       ID = Intrinsic::amdgcn_ds_fmin;
11015       break;
11016     case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
11017       ID = Intrinsic::amdgcn_ds_fmax;
11018       break;
11019     default:
11020       llvm_unreachable("Unknown BuiltinID");
11021     }
11022     Value *F = CGM.getIntrinsic(ID);
11023     return Builder.CreateCall(F, Args);
11024   }
11025 
11026   // amdgcn workitem
11027   case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
11028     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
11029   case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
11030     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024);
11031   case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
11032     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
11033 
11034   // r600 intrinsics
11035   case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
11036   case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
11037     return emitUnaryBuiltin(*this, E, Intrinsic::r600_recipsqrt_ieee);
11038   case AMDGPU::BI__builtin_r600_read_tidig_x:
11039     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024);
11040   case AMDGPU::BI__builtin_r600_read_tidig_y:
11041     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024);
11042   case AMDGPU::BI__builtin_r600_read_tidig_z:
11043     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024);
11044   default:
11045     return nullptr;
11046   }
11047 }
11048 
11049 /// Handle a SystemZ function in which the final argument is a pointer
11050 /// to an int that receives the post-instruction CC value.  At the LLVM level
11051 /// this is represented as a function that returns a {result, cc} pair.
11052 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF,
11053                                          unsigned IntrinsicID,
11054                                          const CallExpr *E) {
11055   unsigned NumArgs = E->getNumArgs() - 1;
11056   SmallVector<Value *, 8> Args(NumArgs);
11057   for (unsigned I = 0; I < NumArgs; ++I)
11058     Args[I] = CGF.EmitScalarExpr(E->getArg(I));
11059   Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs));
11060   Value *F = CGF.CGM.getIntrinsic(IntrinsicID);
11061   Value *Call = CGF.Builder.CreateCall(F, Args);
11062   Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
11063   CGF.Builder.CreateStore(CC, CCPtr);
11064   return CGF.Builder.CreateExtractValue(Call, 0);
11065 }
11066 
11067 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
11068                                                const CallExpr *E) {
11069   switch (BuiltinID) {
11070   case SystemZ::BI__builtin_tbegin: {
11071     Value *TDB = EmitScalarExpr(E->getArg(0));
11072     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
11073     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
11074     return Builder.CreateCall(F, {TDB, Control});
11075   }
11076   case SystemZ::BI__builtin_tbegin_nofloat: {
11077     Value *TDB = EmitScalarExpr(E->getArg(0));
11078     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
11079     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
11080     return Builder.CreateCall(F, {TDB, Control});
11081   }
11082   case SystemZ::BI__builtin_tbeginc: {
11083     Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
11084     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
11085     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
11086     return Builder.CreateCall(F, {TDB, Control});
11087   }
11088   case SystemZ::BI__builtin_tabort: {
11089     Value *Data = EmitScalarExpr(E->getArg(0));
11090     Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
11091     return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort"));
11092   }
11093   case SystemZ::BI__builtin_non_tx_store: {
11094     Value *Address = EmitScalarExpr(E->getArg(0));
11095     Value *Data = EmitScalarExpr(E->getArg(1));
11096     Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
11097     return Builder.CreateCall(F, {Data, Address});
11098   }
11099 
11100   // Vector builtins.  Note that most vector builtins are mapped automatically
11101   // to target-specific LLVM intrinsics.  The ones handled specially here can
11102   // be represented via standard LLVM IR, which is preferable to enable common
11103   // LLVM optimizations.
11104 
11105   case SystemZ::BI__builtin_s390_vpopctb:
11106   case SystemZ::BI__builtin_s390_vpopcth:
11107   case SystemZ::BI__builtin_s390_vpopctf:
11108   case SystemZ::BI__builtin_s390_vpopctg: {
11109     llvm::Type *ResultType = ConvertType(E->getType());
11110     Value *X = EmitScalarExpr(E->getArg(0));
11111     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
11112     return Builder.CreateCall(F, X);
11113   }
11114 
11115   case SystemZ::BI__builtin_s390_vclzb:
11116   case SystemZ::BI__builtin_s390_vclzh:
11117   case SystemZ::BI__builtin_s390_vclzf:
11118   case SystemZ::BI__builtin_s390_vclzg: {
11119     llvm::Type *ResultType = ConvertType(E->getType());
11120     Value *X = EmitScalarExpr(E->getArg(0));
11121     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
11122     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
11123     return Builder.CreateCall(F, {X, Undef});
11124   }
11125 
11126   case SystemZ::BI__builtin_s390_vctzb:
11127   case SystemZ::BI__builtin_s390_vctzh:
11128   case SystemZ::BI__builtin_s390_vctzf:
11129   case SystemZ::BI__builtin_s390_vctzg: {
11130     llvm::Type *ResultType = ConvertType(E->getType());
11131     Value *X = EmitScalarExpr(E->getArg(0));
11132     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
11133     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
11134     return Builder.CreateCall(F, {X, Undef});
11135   }
11136 
11137   case SystemZ::BI__builtin_s390_vfsqsb:
11138   case SystemZ::BI__builtin_s390_vfsqdb: {
11139     llvm::Type *ResultType = ConvertType(E->getType());
11140     Value *X = EmitScalarExpr(E->getArg(0));
11141     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
11142     return Builder.CreateCall(F, X);
11143   }
11144   case SystemZ::BI__builtin_s390_vfmasb:
11145   case SystemZ::BI__builtin_s390_vfmadb: {
11146     llvm::Type *ResultType = ConvertType(E->getType());
11147     Value *X = EmitScalarExpr(E->getArg(0));
11148     Value *Y = EmitScalarExpr(E->getArg(1));
11149     Value *Z = EmitScalarExpr(E->getArg(2));
11150     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
11151     return Builder.CreateCall(F, {X, Y, Z});
11152   }
11153   case SystemZ::BI__builtin_s390_vfmssb:
11154   case SystemZ::BI__builtin_s390_vfmsdb: {
11155     llvm::Type *ResultType = ConvertType(E->getType());
11156     Value *X = EmitScalarExpr(E->getArg(0));
11157     Value *Y = EmitScalarExpr(E->getArg(1));
11158     Value *Z = EmitScalarExpr(E->getArg(2));
11159     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11160     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
11161     return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
11162   }
11163   case SystemZ::BI__builtin_s390_vfnmasb:
11164   case SystemZ::BI__builtin_s390_vfnmadb: {
11165     llvm::Type *ResultType = ConvertType(E->getType());
11166     Value *X = EmitScalarExpr(E->getArg(0));
11167     Value *Y = EmitScalarExpr(E->getArg(1));
11168     Value *Z = EmitScalarExpr(E->getArg(2));
11169     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11170     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
11171     return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, Z}), "sub");
11172   }
11173   case SystemZ::BI__builtin_s390_vfnmssb:
11174   case SystemZ::BI__builtin_s390_vfnmsdb: {
11175     llvm::Type *ResultType = ConvertType(E->getType());
11176     Value *X = EmitScalarExpr(E->getArg(0));
11177     Value *Y = EmitScalarExpr(E->getArg(1));
11178     Value *Z = EmitScalarExpr(E->getArg(2));
11179     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11180     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
11181     Value *NegZ = Builder.CreateFSub(Zero, Z, "sub");
11182     return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, NegZ}));
11183   }
11184   case SystemZ::BI__builtin_s390_vflpsb:
11185   case SystemZ::BI__builtin_s390_vflpdb: {
11186     llvm::Type *ResultType = ConvertType(E->getType());
11187     Value *X = EmitScalarExpr(E->getArg(0));
11188     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
11189     return Builder.CreateCall(F, X);
11190   }
11191   case SystemZ::BI__builtin_s390_vflnsb:
11192   case SystemZ::BI__builtin_s390_vflndb: {
11193     llvm::Type *ResultType = ConvertType(E->getType());
11194     Value *X = EmitScalarExpr(E->getArg(0));
11195     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11196     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
11197     return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub");
11198   }
11199   case SystemZ::BI__builtin_s390_vfisb:
11200   case SystemZ::BI__builtin_s390_vfidb: {
11201     llvm::Type *ResultType = ConvertType(E->getType());
11202     Value *X = EmitScalarExpr(E->getArg(0));
11203     // Constant-fold the M4 and M5 mask arguments.
11204     llvm::APSInt M4, M5;
11205     bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext());
11206     bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext());
11207     assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?");
11208     (void)IsConstM4; (void)IsConstM5;
11209     // Check whether this instance can be represented via a LLVM standard
11210     // intrinsic.  We only support some combinations of M4 and M5.
11211     Intrinsic::ID ID = Intrinsic::not_intrinsic;
11212     switch (M4.getZExtValue()) {
11213     default: break;
11214     case 0:  // IEEE-inexact exception allowed
11215       switch (M5.getZExtValue()) {
11216       default: break;
11217       case 0: ID = Intrinsic::rint; break;
11218       }
11219       break;
11220     case 4:  // IEEE-inexact exception suppressed
11221       switch (M5.getZExtValue()) {
11222       default: break;
11223       case 0: ID = Intrinsic::nearbyint; break;
11224       case 1: ID = Intrinsic::round; break;
11225       case 5: ID = Intrinsic::trunc; break;
11226       case 6: ID = Intrinsic::ceil; break;
11227       case 7: ID = Intrinsic::floor; break;
11228       }
11229       break;
11230     }
11231     if (ID != Intrinsic::not_intrinsic) {
11232       Function *F = CGM.getIntrinsic(ID, ResultType);
11233       return Builder.CreateCall(F, X);
11234     }
11235     switch (BuiltinID) {
11236       case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb; break;
11237       case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb; break;
11238       default: llvm_unreachable("Unknown BuiltinID");
11239     }
11240     Function *F = CGM.getIntrinsic(ID);
11241     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11242     Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
11243     return Builder.CreateCall(F, {X, M4Value, M5Value});
11244   }
11245   case SystemZ::BI__builtin_s390_vfmaxsb:
11246   case SystemZ::BI__builtin_s390_vfmaxdb: {
11247     llvm::Type *ResultType = ConvertType(E->getType());
11248     Value *X = EmitScalarExpr(E->getArg(0));
11249     Value *Y = EmitScalarExpr(E->getArg(1));
11250     // Constant-fold the M4 mask argument.
11251     llvm::APSInt M4;
11252     bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
11253     assert(IsConstM4 && "Constant arg isn't actually constant?");
11254     (void)IsConstM4;
11255     // Check whether this instance can be represented via a LLVM standard
11256     // intrinsic.  We only support some values of M4.
11257     Intrinsic::ID ID = Intrinsic::not_intrinsic;
11258     switch (M4.getZExtValue()) {
11259     default: break;
11260     case 4: ID = Intrinsic::maxnum; break;
11261     }
11262     if (ID != Intrinsic::not_intrinsic) {
11263       Function *F = CGM.getIntrinsic(ID, ResultType);
11264       return Builder.CreateCall(F, {X, Y});
11265     }
11266     switch (BuiltinID) {
11267       case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb; break;
11268       case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb; break;
11269       default: llvm_unreachable("Unknown BuiltinID");
11270     }
11271     Function *F = CGM.getIntrinsic(ID);
11272     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11273     return Builder.CreateCall(F, {X, Y, M4Value});
11274   }
11275   case SystemZ::BI__builtin_s390_vfminsb:
11276   case SystemZ::BI__builtin_s390_vfmindb: {
11277     llvm::Type *ResultType = ConvertType(E->getType());
11278     Value *X = EmitScalarExpr(E->getArg(0));
11279     Value *Y = EmitScalarExpr(E->getArg(1));
11280     // Constant-fold the M4 mask argument.
11281     llvm::APSInt M4;
11282     bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
11283     assert(IsConstM4 && "Constant arg isn't actually constant?");
11284     (void)IsConstM4;
11285     // Check whether this instance can be represented via a LLVM standard
11286     // intrinsic.  We only support some values of M4.
11287     Intrinsic::ID ID = Intrinsic::not_intrinsic;
11288     switch (M4.getZExtValue()) {
11289     default: break;
11290     case 4: ID = Intrinsic::minnum; break;
11291     }
11292     if (ID != Intrinsic::not_intrinsic) {
11293       Function *F = CGM.getIntrinsic(ID, ResultType);
11294       return Builder.CreateCall(F, {X, Y});
11295     }
11296     switch (BuiltinID) {
11297       case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb; break;
11298       case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb; break;
11299       default: llvm_unreachable("Unknown BuiltinID");
11300     }
11301     Function *F = CGM.getIntrinsic(ID);
11302     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11303     return Builder.CreateCall(F, {X, Y, M4Value});
11304   }
11305 
11306   // Vector intrisincs that output the post-instruction CC value.
11307 
11308 #define INTRINSIC_WITH_CC(NAME) \
11309     case SystemZ::BI__builtin_##NAME: \
11310       return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
11311 
11312   INTRINSIC_WITH_CC(s390_vpkshs);
11313   INTRINSIC_WITH_CC(s390_vpksfs);
11314   INTRINSIC_WITH_CC(s390_vpksgs);
11315 
11316   INTRINSIC_WITH_CC(s390_vpklshs);
11317   INTRINSIC_WITH_CC(s390_vpklsfs);
11318   INTRINSIC_WITH_CC(s390_vpklsgs);
11319 
11320   INTRINSIC_WITH_CC(s390_vceqbs);
11321   INTRINSIC_WITH_CC(s390_vceqhs);
11322   INTRINSIC_WITH_CC(s390_vceqfs);
11323   INTRINSIC_WITH_CC(s390_vceqgs);
11324 
11325   INTRINSIC_WITH_CC(s390_vchbs);
11326   INTRINSIC_WITH_CC(s390_vchhs);
11327   INTRINSIC_WITH_CC(s390_vchfs);
11328   INTRINSIC_WITH_CC(s390_vchgs);
11329 
11330   INTRINSIC_WITH_CC(s390_vchlbs);
11331   INTRINSIC_WITH_CC(s390_vchlhs);
11332   INTRINSIC_WITH_CC(s390_vchlfs);
11333   INTRINSIC_WITH_CC(s390_vchlgs);
11334 
11335   INTRINSIC_WITH_CC(s390_vfaebs);
11336   INTRINSIC_WITH_CC(s390_vfaehs);
11337   INTRINSIC_WITH_CC(s390_vfaefs);
11338 
11339   INTRINSIC_WITH_CC(s390_vfaezbs);
11340   INTRINSIC_WITH_CC(s390_vfaezhs);
11341   INTRINSIC_WITH_CC(s390_vfaezfs);
11342 
11343   INTRINSIC_WITH_CC(s390_vfeebs);
11344   INTRINSIC_WITH_CC(s390_vfeehs);
11345   INTRINSIC_WITH_CC(s390_vfeefs);
11346 
11347   INTRINSIC_WITH_CC(s390_vfeezbs);
11348   INTRINSIC_WITH_CC(s390_vfeezhs);
11349   INTRINSIC_WITH_CC(s390_vfeezfs);
11350 
11351   INTRINSIC_WITH_CC(s390_vfenebs);
11352   INTRINSIC_WITH_CC(s390_vfenehs);
11353   INTRINSIC_WITH_CC(s390_vfenefs);
11354 
11355   INTRINSIC_WITH_CC(s390_vfenezbs);
11356   INTRINSIC_WITH_CC(s390_vfenezhs);
11357   INTRINSIC_WITH_CC(s390_vfenezfs);
11358 
11359   INTRINSIC_WITH_CC(s390_vistrbs);
11360   INTRINSIC_WITH_CC(s390_vistrhs);
11361   INTRINSIC_WITH_CC(s390_vistrfs);
11362 
11363   INTRINSIC_WITH_CC(s390_vstrcbs);
11364   INTRINSIC_WITH_CC(s390_vstrchs);
11365   INTRINSIC_WITH_CC(s390_vstrcfs);
11366 
11367   INTRINSIC_WITH_CC(s390_vstrczbs);
11368   INTRINSIC_WITH_CC(s390_vstrczhs);
11369   INTRINSIC_WITH_CC(s390_vstrczfs);
11370 
11371   INTRINSIC_WITH_CC(s390_vfcesbs);
11372   INTRINSIC_WITH_CC(s390_vfcedbs);
11373   INTRINSIC_WITH_CC(s390_vfchsbs);
11374   INTRINSIC_WITH_CC(s390_vfchdbs);
11375   INTRINSIC_WITH_CC(s390_vfchesbs);
11376   INTRINSIC_WITH_CC(s390_vfchedbs);
11377 
11378   INTRINSIC_WITH_CC(s390_vftcisb);
11379   INTRINSIC_WITH_CC(s390_vftcidb);
11380 
11381 #undef INTRINSIC_WITH_CC
11382 
11383   default:
11384     return nullptr;
11385   }
11386 }
11387 
11388 Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
11389                                              const CallExpr *E) {
11390   auto MakeLdg = [&](unsigned IntrinsicID) {
11391     Value *Ptr = EmitScalarExpr(E->getArg(0));
11392     clang::CharUnits Align =
11393         getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
11394     return Builder.CreateCall(
11395         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
11396                                        Ptr->getType()}),
11397         {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
11398   };
11399   auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
11400     Value *Ptr = EmitScalarExpr(E->getArg(0));
11401     return Builder.CreateCall(
11402         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
11403                                        Ptr->getType()}),
11404         {Ptr, EmitScalarExpr(E->getArg(1))});
11405   };
11406   switch (BuiltinID) {
11407   case NVPTX::BI__nvvm_atom_add_gen_i:
11408   case NVPTX::BI__nvvm_atom_add_gen_l:
11409   case NVPTX::BI__nvvm_atom_add_gen_ll:
11410     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E);
11411 
11412   case NVPTX::BI__nvvm_atom_sub_gen_i:
11413   case NVPTX::BI__nvvm_atom_sub_gen_l:
11414   case NVPTX::BI__nvvm_atom_sub_gen_ll:
11415     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E);
11416 
11417   case NVPTX::BI__nvvm_atom_and_gen_i:
11418   case NVPTX::BI__nvvm_atom_and_gen_l:
11419   case NVPTX::BI__nvvm_atom_and_gen_ll:
11420     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E);
11421 
11422   case NVPTX::BI__nvvm_atom_or_gen_i:
11423   case NVPTX::BI__nvvm_atom_or_gen_l:
11424   case NVPTX::BI__nvvm_atom_or_gen_ll:
11425     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E);
11426 
11427   case NVPTX::BI__nvvm_atom_xor_gen_i:
11428   case NVPTX::BI__nvvm_atom_xor_gen_l:
11429   case NVPTX::BI__nvvm_atom_xor_gen_ll:
11430     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E);
11431 
11432   case NVPTX::BI__nvvm_atom_xchg_gen_i:
11433   case NVPTX::BI__nvvm_atom_xchg_gen_l:
11434   case NVPTX::BI__nvvm_atom_xchg_gen_ll:
11435     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E);
11436 
11437   case NVPTX::BI__nvvm_atom_max_gen_i:
11438   case NVPTX::BI__nvvm_atom_max_gen_l:
11439   case NVPTX::BI__nvvm_atom_max_gen_ll:
11440     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E);
11441 
11442   case NVPTX::BI__nvvm_atom_max_gen_ui:
11443   case NVPTX::BI__nvvm_atom_max_gen_ul:
11444   case NVPTX::BI__nvvm_atom_max_gen_ull:
11445     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E);
11446 
11447   case NVPTX::BI__nvvm_atom_min_gen_i:
11448   case NVPTX::BI__nvvm_atom_min_gen_l:
11449   case NVPTX::BI__nvvm_atom_min_gen_ll:
11450     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E);
11451 
11452   case NVPTX::BI__nvvm_atom_min_gen_ui:
11453   case NVPTX::BI__nvvm_atom_min_gen_ul:
11454   case NVPTX::BI__nvvm_atom_min_gen_ull:
11455     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E);
11456 
11457   case NVPTX::BI__nvvm_atom_cas_gen_i:
11458   case NVPTX::BI__nvvm_atom_cas_gen_l:
11459   case NVPTX::BI__nvvm_atom_cas_gen_ll:
11460     // __nvvm_atom_cas_gen_* should return the old value rather than the
11461     // success flag.
11462     return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false);
11463 
11464   case NVPTX::BI__nvvm_atom_add_gen_f: {
11465     Value *Ptr = EmitScalarExpr(E->getArg(0));
11466     Value *Val = EmitScalarExpr(E->getArg(1));
11467     // atomicrmw only deals with integer arguments so we need to use
11468     // LLVM's nvvm_atomic_load_add_f32 intrinsic for that.
11469     Value *FnALAF32 =
11470         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
11471     return Builder.CreateCall(FnALAF32, {Ptr, Val});
11472   }
11473 
11474   case NVPTX::BI__nvvm_atom_add_gen_d: {
11475     Value *Ptr = EmitScalarExpr(E->getArg(0));
11476     Value *Val = EmitScalarExpr(E->getArg(1));
11477     // atomicrmw only deals with integer arguments, so we need to use
11478     // LLVM's nvvm_atomic_load_add_f64 intrinsic.
11479     Value *FnALAF64 =
11480         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f64, Ptr->getType());
11481     return Builder.CreateCall(FnALAF64, {Ptr, Val});
11482   }
11483 
11484   case NVPTX::BI__nvvm_atom_inc_gen_ui: {
11485     Value *Ptr = EmitScalarExpr(E->getArg(0));
11486     Value *Val = EmitScalarExpr(E->getArg(1));
11487     Value *FnALI32 =
11488         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
11489     return Builder.CreateCall(FnALI32, {Ptr, Val});
11490   }
11491 
11492   case NVPTX::BI__nvvm_atom_dec_gen_ui: {
11493     Value *Ptr = EmitScalarExpr(E->getArg(0));
11494     Value *Val = EmitScalarExpr(E->getArg(1));
11495     Value *FnALD32 =
11496         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
11497     return Builder.CreateCall(FnALD32, {Ptr, Val});
11498   }
11499 
11500   case NVPTX::BI__nvvm_ldg_c:
11501   case NVPTX::BI__nvvm_ldg_c2:
11502   case NVPTX::BI__nvvm_ldg_c4:
11503   case NVPTX::BI__nvvm_ldg_s:
11504   case NVPTX::BI__nvvm_ldg_s2:
11505   case NVPTX::BI__nvvm_ldg_s4:
11506   case NVPTX::BI__nvvm_ldg_i:
11507   case NVPTX::BI__nvvm_ldg_i2:
11508   case NVPTX::BI__nvvm_ldg_i4:
11509   case NVPTX::BI__nvvm_ldg_l:
11510   case NVPTX::BI__nvvm_ldg_ll:
11511   case NVPTX::BI__nvvm_ldg_ll2:
11512   case NVPTX::BI__nvvm_ldg_uc:
11513   case NVPTX::BI__nvvm_ldg_uc2:
11514   case NVPTX::BI__nvvm_ldg_uc4:
11515   case NVPTX::BI__nvvm_ldg_us:
11516   case NVPTX::BI__nvvm_ldg_us2:
11517   case NVPTX::BI__nvvm_ldg_us4:
11518   case NVPTX::BI__nvvm_ldg_ui:
11519   case NVPTX::BI__nvvm_ldg_ui2:
11520   case NVPTX::BI__nvvm_ldg_ui4:
11521   case NVPTX::BI__nvvm_ldg_ul:
11522   case NVPTX::BI__nvvm_ldg_ull:
11523   case NVPTX::BI__nvvm_ldg_ull2:
11524     // PTX Interoperability section 2.2: "For a vector with an even number of
11525     // elements, its alignment is set to number of elements times the alignment
11526     // of its member: n*alignof(t)."
11527     return MakeLdg(Intrinsic::nvvm_ldg_global_i);
11528   case NVPTX::BI__nvvm_ldg_f:
11529   case NVPTX::BI__nvvm_ldg_f2:
11530   case NVPTX::BI__nvvm_ldg_f4:
11531   case NVPTX::BI__nvvm_ldg_d:
11532   case NVPTX::BI__nvvm_ldg_d2:
11533     return MakeLdg(Intrinsic::nvvm_ldg_global_f);
11534 
11535   case NVPTX::BI__nvvm_atom_cta_add_gen_i:
11536   case NVPTX::BI__nvvm_atom_cta_add_gen_l:
11537   case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
11538     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
11539   case NVPTX::BI__nvvm_atom_sys_add_gen_i:
11540   case NVPTX::BI__nvvm_atom_sys_add_gen_l:
11541   case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
11542     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
11543   case NVPTX::BI__nvvm_atom_cta_add_gen_f:
11544   case NVPTX::BI__nvvm_atom_cta_add_gen_d:
11545     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
11546   case NVPTX::BI__nvvm_atom_sys_add_gen_f:
11547   case NVPTX::BI__nvvm_atom_sys_add_gen_d:
11548     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
11549   case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
11550   case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
11551   case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
11552     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
11553   case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
11554   case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
11555   case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
11556     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
11557   case NVPTX::BI__nvvm_atom_cta_max_gen_i:
11558   case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
11559   case NVPTX::BI__nvvm_atom_cta_max_gen_l:
11560   case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
11561   case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
11562   case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
11563     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
11564   case NVPTX::BI__nvvm_atom_sys_max_gen_i:
11565   case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
11566   case NVPTX::BI__nvvm_atom_sys_max_gen_l:
11567   case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
11568   case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
11569   case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
11570     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
11571   case NVPTX::BI__nvvm_atom_cta_min_gen_i:
11572   case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
11573   case NVPTX::BI__nvvm_atom_cta_min_gen_l:
11574   case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
11575   case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
11576   case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
11577     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
11578   case NVPTX::BI__nvvm_atom_sys_min_gen_i:
11579   case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
11580   case NVPTX::BI__nvvm_atom_sys_min_gen_l:
11581   case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
11582   case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
11583   case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
11584     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
11585   case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
11586     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
11587   case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
11588     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
11589   case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
11590     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
11591   case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
11592     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
11593   case NVPTX::BI__nvvm_atom_cta_and_gen_i:
11594   case NVPTX::BI__nvvm_atom_cta_and_gen_l:
11595   case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
11596     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
11597   case NVPTX::BI__nvvm_atom_sys_and_gen_i:
11598   case NVPTX::BI__nvvm_atom_sys_and_gen_l:
11599   case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
11600     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
11601   case NVPTX::BI__nvvm_atom_cta_or_gen_i:
11602   case NVPTX::BI__nvvm_atom_cta_or_gen_l:
11603   case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
11604     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
11605   case NVPTX::BI__nvvm_atom_sys_or_gen_i:
11606   case NVPTX::BI__nvvm_atom_sys_or_gen_l:
11607   case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
11608     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
11609   case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
11610   case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
11611   case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
11612     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
11613   case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
11614   case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
11615   case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
11616     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
11617   case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
11618   case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
11619   case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
11620     Value *Ptr = EmitScalarExpr(E->getArg(0));
11621     return Builder.CreateCall(
11622         CGM.getIntrinsic(
11623             Intrinsic::nvvm_atomic_cas_gen_i_cta,
11624             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
11625         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
11626   }
11627   case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
11628   case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
11629   case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
11630     Value *Ptr = EmitScalarExpr(E->getArg(0));
11631     return Builder.CreateCall(
11632         CGM.getIntrinsic(
11633             Intrinsic::nvvm_atomic_cas_gen_i_sys,
11634             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
11635         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
11636   }
11637   case NVPTX::BI__nvvm_match_all_sync_i32p:
11638   case NVPTX::BI__nvvm_match_all_sync_i64p: {
11639     Value *Mask = EmitScalarExpr(E->getArg(0));
11640     Value *Val = EmitScalarExpr(E->getArg(1));
11641     Address PredOutPtr = EmitPointerWithAlignment(E->getArg(2));
11642     Value *ResultPair = Builder.CreateCall(
11643         CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
11644                              ? Intrinsic::nvvm_match_all_sync_i32p
11645                              : Intrinsic::nvvm_match_all_sync_i64p),
11646         {Mask, Val});
11647     Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
11648                                      PredOutPtr.getElementType());
11649     Builder.CreateStore(Pred, PredOutPtr);
11650     return Builder.CreateExtractValue(ResultPair, 0);
11651   }
11652   case NVPTX::BI__hmma_m16n16k16_ld_a:
11653   case NVPTX::BI__hmma_m16n16k16_ld_b:
11654   case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
11655   case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
11656   case NVPTX::BI__hmma_m32n8k16_ld_a:
11657   case NVPTX::BI__hmma_m32n8k16_ld_b:
11658   case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
11659   case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
11660   case NVPTX::BI__hmma_m8n32k16_ld_a:
11661   case NVPTX::BI__hmma_m8n32k16_ld_b:
11662   case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
11663   case NVPTX::BI__hmma_m8n32k16_ld_c_f32: {
11664     Address Dst = EmitPointerWithAlignment(E->getArg(0));
11665     Value *Src = EmitScalarExpr(E->getArg(1));
11666     Value *Ldm = EmitScalarExpr(E->getArg(2));
11667     llvm::APSInt isColMajorArg;
11668     if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
11669       return nullptr;
11670     bool isColMajor = isColMajorArg.getSExtValue();
11671     unsigned IID;
11672     unsigned NumResults;
11673     switch (BuiltinID) {
11674     case NVPTX::BI__hmma_m16n16k16_ld_a:
11675       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride
11676                        : Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride;
11677       NumResults = 8;
11678       break;
11679     case NVPTX::BI__hmma_m16n16k16_ld_b:
11680       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride
11681                        : Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride;
11682       NumResults = 8;
11683       break;
11684     case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
11685       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride
11686                        : Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride;
11687       NumResults = 4;
11688       break;
11689     case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
11690       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride
11691                        : Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride;
11692       NumResults = 8;
11693       break;
11694     case NVPTX::BI__hmma_m32n8k16_ld_a:
11695       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride
11696                        : Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride;
11697       NumResults = 8;
11698       break;
11699     case NVPTX::BI__hmma_m32n8k16_ld_b:
11700       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride
11701                        : Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride;
11702       NumResults = 8;
11703       break;
11704     case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
11705       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride
11706                        : Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride;
11707       NumResults = 4;
11708       break;
11709     case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
11710       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride
11711                        : Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride;
11712       NumResults = 8;
11713       break;
11714     case NVPTX::BI__hmma_m8n32k16_ld_a:
11715       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride
11716                        : Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride;
11717       NumResults = 8;
11718       break;
11719     case NVPTX::BI__hmma_m8n32k16_ld_b:
11720       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride
11721                        : Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride;
11722       NumResults = 8;
11723       break;
11724     case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
11725       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride
11726                        : Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride;
11727       NumResults = 4;
11728       break;
11729     case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
11730       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride
11731                        : Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride;
11732       NumResults = 8;
11733       break;
11734     default:
11735       llvm_unreachable("Unexpected builtin ID.");
11736     }
11737     Value *Result =
11738         Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
11739 
11740     // Save returned values.
11741     for (unsigned i = 0; i < NumResults; ++i) {
11742       Builder.CreateAlignedStore(
11743           Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
11744                                 Dst.getElementType()),
11745           Builder.CreateGEP(Dst.getPointer(), llvm::ConstantInt::get(IntTy, i)),
11746           CharUnits::fromQuantity(4));
11747     }
11748     return Result;
11749   }
11750 
11751   case NVPTX::BI__hmma_m16n16k16_st_c_f16:
11752   case NVPTX::BI__hmma_m16n16k16_st_c_f32:
11753   case NVPTX::BI__hmma_m32n8k16_st_c_f16:
11754   case NVPTX::BI__hmma_m32n8k16_st_c_f32:
11755   case NVPTX::BI__hmma_m8n32k16_st_c_f16:
11756   case NVPTX::BI__hmma_m8n32k16_st_c_f32: {
11757     Value *Dst = EmitScalarExpr(E->getArg(0));
11758     Address Src = EmitPointerWithAlignment(E->getArg(1));
11759     Value *Ldm = EmitScalarExpr(E->getArg(2));
11760     llvm::APSInt isColMajorArg;
11761     if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
11762       return nullptr;
11763     bool isColMajor = isColMajorArg.getSExtValue();
11764     unsigned IID;
11765     unsigned NumResults = 8;
11766     // PTX Instructions (and LLVM instrinsics) are defined for slice _d_, yet
11767     // for some reason nvcc builtins use _c_.
11768     switch (BuiltinID) {
11769     case NVPTX::BI__hmma_m16n16k16_st_c_f16:
11770       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride
11771                        : Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride;
11772       NumResults = 4;
11773       break;
11774     case NVPTX::BI__hmma_m16n16k16_st_c_f32:
11775       IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride
11776                        : Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride;
11777       break;
11778     case NVPTX::BI__hmma_m32n8k16_st_c_f16:
11779       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride
11780                        : Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride;
11781       NumResults = 4;
11782       break;
11783     case NVPTX::BI__hmma_m32n8k16_st_c_f32:
11784       IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride
11785                        : Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride;
11786       break;
11787     case NVPTX::BI__hmma_m8n32k16_st_c_f16:
11788       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride
11789                        : Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride;
11790       NumResults = 4;
11791       break;
11792     case NVPTX::BI__hmma_m8n32k16_st_c_f32:
11793       IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride
11794                        : Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride;
11795       break;
11796     default:
11797       llvm_unreachable("Unexpected builtin ID.");
11798     }
11799     Function *Intrinsic = CGM.getIntrinsic(IID, Dst->getType());
11800     llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
11801     SmallVector<Value *, 10> Values = {Dst};
11802     for (unsigned i = 0; i < NumResults; ++i) {
11803       Value *V = Builder.CreateAlignedLoad(
11804           Builder.CreateGEP(Src.getPointer(), llvm::ConstantInt::get(IntTy, i)),
11805           CharUnits::fromQuantity(4));
11806       Values.push_back(Builder.CreateBitCast(V, ParamType));
11807     }
11808     Values.push_back(Ldm);
11809     Value *Result = Builder.CreateCall(Intrinsic, Values);
11810     return Result;
11811   }
11812 
11813   // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
11814   // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
11815   case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
11816   case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
11817   case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
11818   case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
11819   case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
11820   case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
11821   case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
11822   case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
11823   case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
11824   case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
11825   case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
11826   case NVPTX::BI__hmma_m8n32k16_mma_f16f32: {
11827     Address Dst = EmitPointerWithAlignment(E->getArg(0));
11828     Address SrcA = EmitPointerWithAlignment(E->getArg(1));
11829     Address SrcB = EmitPointerWithAlignment(E->getArg(2));
11830     Address SrcC = EmitPointerWithAlignment(E->getArg(3));
11831     llvm::APSInt LayoutArg;
11832     if (!E->getArg(4)->isIntegerConstantExpr(LayoutArg, getContext()))
11833       return nullptr;
11834     int Layout = LayoutArg.getSExtValue();
11835     if (Layout < 0 || Layout > 3)
11836       return nullptr;
11837     llvm::APSInt SatfArg;
11838     if (!E->getArg(5)->isIntegerConstantExpr(SatfArg, getContext()))
11839       return nullptr;
11840     bool Satf = SatfArg.getSExtValue();
11841 
11842     // clang-format off
11843 #define MMA_VARIANTS(geom, type) {{                                 \
11844       Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type,             \
11845       Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
11846       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type,             \
11847       Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
11848       Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type,             \
11849       Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
11850       Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type,             \
11851       Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite  \
11852     }}
11853     // clang-format on
11854 
11855     auto getMMAIntrinsic = [Layout, Satf](std::array<unsigned, 8> Variants) {
11856       unsigned Index = Layout * 2 + Satf;
11857       assert(Index < 8);
11858       return Variants[Index];
11859     };
11860     unsigned IID;
11861     unsigned NumEltsC;
11862     unsigned NumEltsD;
11863     switch (BuiltinID) {
11864     case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
11865       IID = getMMAIntrinsic(MMA_VARIANTS(m16n16k16, f16_f16));
11866       NumEltsC = 4;
11867       NumEltsD = 4;
11868       break;
11869     case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
11870       IID = getMMAIntrinsic(MMA_VARIANTS(m16n16k16, f32_f16));
11871       NumEltsC = 4;
11872       NumEltsD = 8;
11873       break;
11874     case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
11875       IID = getMMAIntrinsic(MMA_VARIANTS(m16n16k16, f16_f32));
11876       NumEltsC = 8;
11877       NumEltsD = 4;
11878       break;
11879     case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
11880       IID = getMMAIntrinsic(MMA_VARIANTS(m16n16k16, f32_f32));
11881       NumEltsC = 8;
11882       NumEltsD = 8;
11883       break;
11884     case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
11885       IID = getMMAIntrinsic(MMA_VARIANTS(m32n8k16, f16_f16));
11886       NumEltsC = 4;
11887       NumEltsD = 4;
11888       break;
11889     case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
11890       IID = getMMAIntrinsic(MMA_VARIANTS(m32n8k16, f32_f16));
11891       NumEltsC = 4;
11892       NumEltsD = 8;
11893       break;
11894     case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
11895       IID = getMMAIntrinsic(MMA_VARIANTS(m32n8k16, f16_f32));
11896       NumEltsC = 8;
11897       NumEltsD = 4;
11898       break;
11899     case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
11900       IID = getMMAIntrinsic(MMA_VARIANTS(m32n8k16, f32_f32));
11901       NumEltsC = 8;
11902       NumEltsD = 8;
11903       break;
11904     case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
11905       IID = getMMAIntrinsic(MMA_VARIANTS(m8n32k16, f16_f16));
11906       NumEltsC = 4;
11907       NumEltsD = 4;
11908       break;
11909     case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
11910       IID = getMMAIntrinsic(MMA_VARIANTS(m8n32k16, f32_f16));
11911       NumEltsC = 4;
11912       NumEltsD = 8;
11913       break;
11914     case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
11915       IID = getMMAIntrinsic(MMA_VARIANTS(m8n32k16, f16_f32));
11916       NumEltsC = 8;
11917       NumEltsD = 4;
11918       break;
11919     case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
11920       IID = getMMAIntrinsic(MMA_VARIANTS(m8n32k16, f32_f32));
11921       NumEltsC = 8;
11922       NumEltsD = 8;
11923       break;
11924     default:
11925       llvm_unreachable("Unexpected builtin ID.");
11926     }
11927 #undef MMA_VARIANTS
11928 
11929     SmallVector<Value *, 24> Values;
11930     Function *Intrinsic = CGM.getIntrinsic(IID);
11931     llvm::Type *ABType = Intrinsic->getFunctionType()->getParamType(0);
11932     // Load A
11933     for (unsigned i = 0; i < 8; ++i) {
11934       Value *V = Builder.CreateAlignedLoad(
11935           Builder.CreateGEP(SrcA.getPointer(),
11936                             llvm::ConstantInt::get(IntTy, i)),
11937           CharUnits::fromQuantity(4));
11938       Values.push_back(Builder.CreateBitCast(V, ABType));
11939     }
11940     // Load B
11941     for (unsigned i = 0; i < 8; ++i) {
11942       Value *V = Builder.CreateAlignedLoad(
11943           Builder.CreateGEP(SrcB.getPointer(),
11944                             llvm::ConstantInt::get(IntTy, i)),
11945           CharUnits::fromQuantity(4));
11946       Values.push_back(Builder.CreateBitCast(V, ABType));
11947     }
11948     // Load C
11949     llvm::Type *CType = Intrinsic->getFunctionType()->getParamType(16);
11950     for (unsigned i = 0; i < NumEltsC; ++i) {
11951       Value *V = Builder.CreateAlignedLoad(
11952           Builder.CreateGEP(SrcC.getPointer(),
11953                             llvm::ConstantInt::get(IntTy, i)),
11954           CharUnits::fromQuantity(4));
11955       Values.push_back(Builder.CreateBitCast(V, CType));
11956     }
11957     Value *Result = Builder.CreateCall(Intrinsic, Values);
11958     llvm::Type *DType = Dst.getElementType();
11959     for (unsigned i = 0; i < NumEltsD; ++i)
11960       Builder.CreateAlignedStore(
11961           Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
11962           Builder.CreateGEP(Dst.getPointer(), llvm::ConstantInt::get(IntTy, i)),
11963           CharUnits::fromQuantity(4));
11964     return Result;
11965   }
11966   default:
11967     return nullptr;
11968   }
11969 }
11970 
11971 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
11972                                                    const CallExpr *E) {
11973   switch (BuiltinID) {
11974   case WebAssembly::BI__builtin_wasm_memory_size: {
11975     llvm::Type *ResultType = ConvertType(E->getType());
11976     Value *I = EmitScalarExpr(E->getArg(0));
11977     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
11978     return Builder.CreateCall(Callee, I);
11979   }
11980   case WebAssembly::BI__builtin_wasm_memory_grow: {
11981     llvm::Type *ResultType = ConvertType(E->getType());
11982     Value *Args[] = {
11983       EmitScalarExpr(E->getArg(0)),
11984       EmitScalarExpr(E->getArg(1))
11985     };
11986     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
11987     return Builder.CreateCall(Callee, Args);
11988   }
11989   case WebAssembly::BI__builtin_wasm_mem_size: {
11990     llvm::Type *ResultType = ConvertType(E->getType());
11991     Value *I = EmitScalarExpr(E->getArg(0));
11992     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_mem_size, ResultType);
11993     return Builder.CreateCall(Callee, I);
11994   }
11995   case WebAssembly::BI__builtin_wasm_mem_grow: {
11996     llvm::Type *ResultType = ConvertType(E->getType());
11997     Value *Args[] = {
11998       EmitScalarExpr(E->getArg(0)),
11999       EmitScalarExpr(E->getArg(1))
12000     };
12001     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_mem_grow, ResultType);
12002     return Builder.CreateCall(Callee, Args);
12003   }
12004   case WebAssembly::BI__builtin_wasm_current_memory: {
12005     llvm::Type *ResultType = ConvertType(E->getType());
12006     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType);
12007     return Builder.CreateCall(Callee);
12008   }
12009   case WebAssembly::BI__builtin_wasm_grow_memory: {
12010     Value *X = EmitScalarExpr(E->getArg(0));
12011     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType());
12012     return Builder.CreateCall(Callee, X);
12013   }
12014   case WebAssembly::BI__builtin_wasm_throw: {
12015     Value *Tag = EmitScalarExpr(E->getArg(0));
12016     Value *Obj = EmitScalarExpr(E->getArg(1));
12017     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
12018     return Builder.CreateCall(Callee, {Tag, Obj});
12019   }
12020   case WebAssembly::BI__builtin_wasm_rethrow: {
12021     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow);
12022     return Builder.CreateCall(Callee);
12023   }
12024 
12025   default:
12026     return nullptr;
12027   }
12028 }
12029 
12030 Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
12031                                                const CallExpr *E) {
12032   SmallVector<llvm::Value *, 4> Ops;
12033   Intrinsic::ID ID = Intrinsic::not_intrinsic;
12034 
12035   auto MakeCircLd = [&](unsigned IntID, bool HasImm) {
12036     // The base pointer is passed by address, so it needs to be loaded.
12037     Address BP = EmitPointerWithAlignment(E->getArg(0));
12038     BP = Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
12039                  BP.getAlignment());
12040     llvm::Value *Base = Builder.CreateLoad(BP);
12041     // Operands are Base, Increment, Modifier, Start.
12042     if (HasImm)
12043       Ops = { Base, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)),
12044               EmitScalarExpr(E->getArg(3)) };
12045     else
12046       Ops = { Base, EmitScalarExpr(E->getArg(1)),
12047               EmitScalarExpr(E->getArg(2)) };
12048 
12049     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12050     llvm::Value *NewBase = Builder.CreateExtractValue(Result, 1);
12051     llvm::Value *LV = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
12052                                             NewBase->getType()->getPointerTo());
12053     Address Dest = EmitPointerWithAlignment(E->getArg(0));
12054     // The intrinsic generates two results. The new value for the base pointer
12055     // needs to be stored.
12056     Builder.CreateAlignedStore(NewBase, LV, Dest.getAlignment());
12057     return Builder.CreateExtractValue(Result, 0);
12058   };
12059 
12060   auto MakeCircSt = [&](unsigned IntID, bool HasImm) {
12061     // The base pointer is passed by address, so it needs to be loaded.
12062     Address BP = EmitPointerWithAlignment(E->getArg(0));
12063     BP = Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
12064                  BP.getAlignment());
12065     llvm::Value *Base = Builder.CreateLoad(BP);
12066     // Operands are Base, Increment, Modifier, Value, Start.
12067     if (HasImm)
12068       Ops = { Base, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)),
12069               EmitScalarExpr(E->getArg(3)), EmitScalarExpr(E->getArg(4)) };
12070     else
12071       Ops = { Base, EmitScalarExpr(E->getArg(1)),
12072               EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3)) };
12073 
12074     llvm::Value *NewBase = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12075     llvm::Value *LV = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
12076                                             NewBase->getType()->getPointerTo());
12077     Address Dest = EmitPointerWithAlignment(E->getArg(0));
12078     // The intrinsic generates one result, which is the new value for the base
12079     // pointer. It needs to be stored.
12080     return Builder.CreateAlignedStore(NewBase, LV, Dest.getAlignment());
12081   };
12082 
12083   // Handle the conversion of bit-reverse load intrinsics to bit code.
12084   // The intrinsic call after this function only reads from memory and the
12085   // write to memory is dealt by the store instruction.
12086   auto MakeBrevLd = [&](unsigned IntID, llvm::Type *DestTy) {
12087     // The intrinsic generates one result, which is the new value for the base
12088     // pointer. It needs to be returned. The result of the load instruction is
12089     // passed to intrinsic by address, so the value needs to be stored.
12090     llvm::Value *BaseAddress =
12091         Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int8PtrTy);
12092 
12093     // Expressions like &(*pt++) will be incremented per evaluation.
12094     // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression
12095     // per call.
12096     Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
12097     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy),
12098                        DestAddr.getAlignment());
12099     llvm::Value *DestAddress = DestAddr.getPointer();
12100 
12101     // Operands are Base, Dest, Modifier.
12102     // The intrinsic format in LLVM IR is defined as
12103     // { ValueType, i8* } (i8*, i32).
12104     Ops = {BaseAddress, EmitScalarExpr(E->getArg(2))};
12105 
12106     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12107     // The value needs to be stored as the variable is passed by reference.
12108     llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
12109 
12110     // The store needs to be truncated to fit the destination type.
12111     // While i32 and i64 are natively supported on Hexagon, i8 and i16 needs
12112     // to be handled with stores of respective destination type.
12113     DestVal = Builder.CreateTrunc(DestVal, DestTy);
12114 
12115     llvm::Value *DestForStore =
12116         Builder.CreateBitCast(DestAddress, DestVal->getType()->getPointerTo());
12117     Builder.CreateAlignedStore(DestVal, DestForStore, DestAddr.getAlignment());
12118     // The updated value of the base pointer is returned.
12119     return Builder.CreateExtractValue(Result, 1);
12120   };
12121 
12122   switch (BuiltinID) {
12123   case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
12124   case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B: {
12125     Address Dest = EmitPointerWithAlignment(E->getArg(2));
12126     unsigned Size;
12127     if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vaddcarry) {
12128       Size = 512;
12129       ID = Intrinsic::hexagon_V6_vaddcarry;
12130     } else {
12131       Size = 1024;
12132       ID = Intrinsic::hexagon_V6_vaddcarry_128B;
12133     }
12134     Dest = Builder.CreateBitCast(Dest,
12135         llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
12136     LoadInst *QLd = Builder.CreateLoad(Dest);
12137     Ops = { EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), QLd };
12138     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
12139     llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
12140     llvm::Value *Base = Builder.CreateBitCast(EmitScalarExpr(E->getArg(2)),
12141                                               Vprd->getType()->getPointerTo(0));
12142     Builder.CreateAlignedStore(Vprd, Base, Dest.getAlignment());
12143     return Builder.CreateExtractValue(Result, 0);
12144   }
12145   case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
12146   case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
12147     Address Dest = EmitPointerWithAlignment(E->getArg(2));
12148     unsigned Size;
12149     if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vsubcarry) {
12150       Size = 512;
12151       ID = Intrinsic::hexagon_V6_vsubcarry;
12152     } else {
12153       Size = 1024;
12154       ID = Intrinsic::hexagon_V6_vsubcarry_128B;
12155     }
12156     Dest = Builder.CreateBitCast(Dest,
12157         llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
12158     LoadInst *QLd = Builder.CreateLoad(Dest);
12159     Ops = { EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), QLd };
12160     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
12161     llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
12162     llvm::Value *Base = Builder.CreateBitCast(EmitScalarExpr(E->getArg(2)),
12163                                               Vprd->getType()->getPointerTo(0));
12164     Builder.CreateAlignedStore(Vprd, Base, Dest.getAlignment());
12165     return Builder.CreateExtractValue(Result, 0);
12166   }
12167   case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
12168     return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pci, /*HasImm*/true);
12169   case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
12170     return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pci,  /*HasImm*/true);
12171   case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
12172     return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pci, /*HasImm*/true);
12173   case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
12174     return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pci,  /*HasImm*/true);
12175   case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
12176     return MakeCircLd(Intrinsic::hexagon_L2_loadri_pci,  /*HasImm*/true);
12177   case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
12178     return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pci,  /*HasImm*/true);
12179   case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
12180     return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pcr, /*HasImm*/false);
12181   case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
12182     return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pcr,  /*HasImm*/false);
12183   case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
12184     return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pcr, /*HasImm*/false);
12185   case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
12186     return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pcr,  /*HasImm*/false);
12187   case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
12188     return MakeCircLd(Intrinsic::hexagon_L2_loadri_pcr,  /*HasImm*/false);
12189   case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
12190     return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pcr,  /*HasImm*/false);
12191   case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
12192     return MakeCircSt(Intrinsic::hexagon_S2_storerb_pci, /*HasImm*/true);
12193   case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
12194     return MakeCircSt(Intrinsic::hexagon_S2_storerh_pci, /*HasImm*/true);
12195   case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
12196     return MakeCircSt(Intrinsic::hexagon_S2_storerf_pci, /*HasImm*/true);
12197   case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
12198     return MakeCircSt(Intrinsic::hexagon_S2_storeri_pci, /*HasImm*/true);
12199   case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
12200     return MakeCircSt(Intrinsic::hexagon_S2_storerd_pci, /*HasImm*/true);
12201   case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
12202     return MakeCircSt(Intrinsic::hexagon_S2_storerb_pcr, /*HasImm*/false);
12203   case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
12204     return MakeCircSt(Intrinsic::hexagon_S2_storerh_pcr, /*HasImm*/false);
12205   case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
12206     return MakeCircSt(Intrinsic::hexagon_S2_storerf_pcr, /*HasImm*/false);
12207   case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
12208     return MakeCircSt(Intrinsic::hexagon_S2_storeri_pcr, /*HasImm*/false);
12209   case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
12210     return MakeCircSt(Intrinsic::hexagon_S2_storerd_pcr, /*HasImm*/false);
12211   case Hexagon::BI__builtin_brev_ldub:
12212     return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
12213   case Hexagon::BI__builtin_brev_ldb:
12214     return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
12215   case Hexagon::BI__builtin_brev_lduh:
12216     return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
12217   case Hexagon::BI__builtin_brev_ldh:
12218     return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
12219   case Hexagon::BI__builtin_brev_ldw:
12220     return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
12221   case Hexagon::BI__builtin_brev_ldd:
12222     return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
12223   default:
12224     break;
12225   } // switch
12226 
12227   return nullptr;
12228 }
12229