1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This contains code to emit Builtin calls as LLVM code. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGCXXABI.h" 16 #include "CGObjCRuntime.h" 17 #include "CodeGenModule.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/ASTContext.h" 20 #include "clang/AST/Decl.h" 21 #include "clang/Basic/TargetBuiltins.h" 22 #include "clang/Basic/TargetInfo.h" 23 #include "clang/CodeGen/CGFunctionInfo.h" 24 #include "llvm/ADT/StringExtras.h" 25 #include "llvm/IR/CallSite.h" 26 #include "llvm/IR/DataLayout.h" 27 #include "llvm/IR/InlineAsm.h" 28 #include "llvm/IR/Intrinsics.h" 29 #include <sstream> 30 31 using namespace clang; 32 using namespace CodeGen; 33 using namespace llvm; 34 35 /// getBuiltinLibFunction - Given a builtin id for a function like 36 /// "__builtin_fabsf", return a Function* for "fabsf". 37 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, 38 unsigned BuiltinID) { 39 assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); 40 41 // Get the name, skip over the __builtin_ prefix (if necessary). 42 StringRef Name; 43 GlobalDecl D(FD); 44 45 // If the builtin has been declared explicitly with an assembler label, 46 // use the mangled name. This differs from the plain label on platforms 47 // that prefix labels. 48 if (FD->hasAttr<AsmLabelAttr>()) 49 Name = getMangledName(D); 50 else 51 Name = Context.BuiltinInfo.getName(BuiltinID) + 10; 52 53 llvm::FunctionType *Ty = 54 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); 55 56 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); 57 } 58 59 /// Emit the conversions required to turn the given value into an 60 /// integer of the given size. 61 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, 62 QualType T, llvm::IntegerType *IntType) { 63 V = CGF.EmitToMemory(V, T); 64 65 if (V->getType()->isPointerTy()) 66 return CGF.Builder.CreatePtrToInt(V, IntType); 67 68 assert(V->getType() == IntType); 69 return V; 70 } 71 72 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, 73 QualType T, llvm::Type *ResultType) { 74 V = CGF.EmitFromMemory(V, T); 75 76 if (ResultType->isPointerTy()) 77 return CGF.Builder.CreateIntToPtr(V, ResultType); 78 79 assert(V->getType() == ResultType); 80 return V; 81 } 82 83 /// Utility to insert an atomic instruction based on Instrinsic::ID 84 /// and the expression node. 85 static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF, 86 llvm::AtomicRMWInst::BinOp Kind, 87 const CallExpr *E) { 88 QualType T = E->getType(); 89 assert(E->getArg(0)->getType()->isPointerType()); 90 assert(CGF.getContext().hasSameUnqualifiedType(T, 91 E->getArg(0)->getType()->getPointeeType())); 92 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 93 94 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 95 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 96 97 llvm::IntegerType *IntType = 98 llvm::IntegerType::get(CGF.getLLVMContext(), 99 CGF.getContext().getTypeSize(T)); 100 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 101 102 llvm::Value *Args[2]; 103 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 104 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 105 llvm::Type *ValueType = Args[1]->getType(); 106 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 107 108 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 109 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 110 return EmitFromInt(CGF, Result, T, ValueType); 111 } 112 113 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) { 114 Value *Val = CGF.EmitScalarExpr(E->getArg(0)); 115 Value *Address = CGF.EmitScalarExpr(E->getArg(1)); 116 117 // Convert the type of the pointer to a pointer to the stored type. 118 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType()); 119 Value *BC = CGF.Builder.CreateBitCast( 120 Address, llvm::PointerType::getUnqual(Val->getType()), "cast"); 121 LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType()); 122 LV.setNontemporal(true); 123 CGF.EmitStoreOfScalar(Val, LV, false); 124 return nullptr; 125 } 126 127 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) { 128 Value *Address = CGF.EmitScalarExpr(E->getArg(0)); 129 130 LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType()); 131 LV.setNontemporal(true); 132 return CGF.EmitLoadOfScalar(LV, E->getExprLoc()); 133 } 134 135 static RValue EmitBinaryAtomic(CodeGenFunction &CGF, 136 llvm::AtomicRMWInst::BinOp Kind, 137 const CallExpr *E) { 138 return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E)); 139 } 140 141 /// Utility to insert an atomic instruction based Instrinsic::ID and 142 /// the expression node, where the return value is the result of the 143 /// operation. 144 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, 145 llvm::AtomicRMWInst::BinOp Kind, 146 const CallExpr *E, 147 Instruction::BinaryOps Op, 148 bool Invert = false) { 149 QualType T = E->getType(); 150 assert(E->getArg(0)->getType()->isPointerType()); 151 assert(CGF.getContext().hasSameUnqualifiedType(T, 152 E->getArg(0)->getType()->getPointeeType())); 153 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); 154 155 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 156 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 157 158 llvm::IntegerType *IntType = 159 llvm::IntegerType::get(CGF.getLLVMContext(), 160 CGF.getContext().getTypeSize(T)); 161 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 162 163 llvm::Value *Args[2]; 164 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 165 llvm::Type *ValueType = Args[1]->getType(); 166 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 167 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 168 169 llvm::Value *Result = CGF.Builder.CreateAtomicRMW( 170 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent); 171 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); 172 if (Invert) 173 Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, 174 llvm::ConstantInt::get(IntType, -1)); 175 Result = EmitFromInt(CGF, Result, T, ValueType); 176 return RValue::get(Result); 177 } 178 179 /// @brief Utility to insert an atomic cmpxchg instruction. 180 /// 181 /// @param CGF The current codegen function. 182 /// @param E Builtin call expression to convert to cmpxchg. 183 /// arg0 - address to operate on 184 /// arg1 - value to compare with 185 /// arg2 - new value 186 /// @param ReturnBool Specifies whether to return success flag of 187 /// cmpxchg result or the old value. 188 /// 189 /// @returns result of cmpxchg, according to ReturnBool 190 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, 191 bool ReturnBool) { 192 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType(); 193 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); 194 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); 195 196 llvm::IntegerType *IntType = llvm::IntegerType::get( 197 CGF.getLLVMContext(), CGF.getContext().getTypeSize(T)); 198 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); 199 200 Value *Args[3]; 201 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); 202 Args[1] = CGF.EmitScalarExpr(E->getArg(1)); 203 llvm::Type *ValueType = Args[1]->getType(); 204 Args[1] = EmitToInt(CGF, Args[1], T, IntType); 205 Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType); 206 207 Value *Pair = CGF.Builder.CreateAtomicCmpXchg( 208 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent, 209 llvm::AtomicOrdering::SequentiallyConsistent); 210 if (ReturnBool) 211 // Extract boolean success flag and zext it to int. 212 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1), 213 CGF.ConvertType(E->getType())); 214 else 215 // Extract old value and emit it using the same type as compare value. 216 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T, 217 ValueType); 218 } 219 220 /// EmitFAbs - Emit a call to @llvm.fabs(). 221 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { 222 Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); 223 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); 224 Call->setDoesNotAccessMemory(); 225 return Call; 226 } 227 228 /// Emit the computation of the sign bit for a floating point value. Returns 229 /// the i1 sign bit value. 230 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { 231 LLVMContext &C = CGF.CGM.getLLVMContext(); 232 233 llvm::Type *Ty = V->getType(); 234 int Width = Ty->getPrimitiveSizeInBits(); 235 llvm::Type *IntTy = llvm::IntegerType::get(C, Width); 236 V = CGF.Builder.CreateBitCast(V, IntTy); 237 if (Ty->isPPC_FP128Ty()) { 238 // We want the sign bit of the higher-order double. The bitcast we just 239 // did works as if the double-double was stored to memory and then 240 // read as an i128. The "store" will put the higher-order double in the 241 // lower address in both little- and big-Endian modes, but the "load" 242 // will treat those bits as a different part of the i128: the low bits in 243 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian 244 // we need to shift the high bits down to the low before truncating. 245 Width >>= 1; 246 if (CGF.getTarget().isBigEndian()) { 247 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width); 248 V = CGF.Builder.CreateLShr(V, ShiftCst); 249 } 250 // We are truncating value in order to extract the higher-order 251 // double, which we will be using to extract the sign from. 252 IntTy = llvm::IntegerType::get(C, Width); 253 V = CGF.Builder.CreateTrunc(V, IntTy); 254 } 255 Value *Zero = llvm::Constant::getNullValue(IntTy); 256 return CGF.Builder.CreateICmpSLT(V, Zero); 257 } 258 259 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, 260 const CallExpr *E, llvm::Value *calleeValue) { 261 return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, 262 ReturnValueSlot(), Fn); 263 } 264 265 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* 266 /// depending on IntrinsicID. 267 /// 268 /// \arg CGF The current codegen function. 269 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate. 270 /// \arg X The first argument to the llvm.*.with.overflow.*. 271 /// \arg Y The second argument to the llvm.*.with.overflow.*. 272 /// \arg Carry The carry returned by the llvm.*.with.overflow.*. 273 /// \returns The result (i.e. sum/product) returned by the intrinsic. 274 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, 275 const llvm::Intrinsic::ID IntrinsicID, 276 llvm::Value *X, llvm::Value *Y, 277 llvm::Value *&Carry) { 278 // Make sure we have integers of the same width. 279 assert(X->getType() == Y->getType() && 280 "Arguments must be the same type. (Did you forget to make sure both " 281 "arguments have the same integer width?)"); 282 283 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); 284 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); 285 Carry = CGF.Builder.CreateExtractValue(Tmp, 1); 286 return CGF.Builder.CreateExtractValue(Tmp, 0); 287 } 288 289 // Emit a simple mangled intrinsic that has 1 argument and a return type 290 // matching the argument type. 291 static Value *emitUnaryBuiltin(CodeGenFunction &CGF, 292 const CallExpr *E, 293 unsigned IntrinsicID) { 294 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 295 296 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 297 return CGF.Builder.CreateCall(F, Src0); 298 } 299 300 // Emit an intrinsic that has 3 float or double operands. 301 static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, 302 const CallExpr *E, 303 unsigned IntrinsicID) { 304 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 305 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 306 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); 307 308 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 309 return CGF.Builder.CreateCall(F, {Src0, Src1, Src2}); 310 } 311 312 // Emit an intrinsic that has 1 float or double operand, and 1 integer. 313 static Value *emitFPIntBuiltin(CodeGenFunction &CGF, 314 const CallExpr *E, 315 unsigned IntrinsicID) { 316 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); 317 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); 318 319 Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); 320 return CGF.Builder.CreateCall(F, {Src0, Src1}); 321 } 322 323 namespace { 324 struct WidthAndSignedness { 325 unsigned Width; 326 bool Signed; 327 }; 328 } 329 330 static WidthAndSignedness 331 getIntegerWidthAndSignedness(const clang::ASTContext &context, 332 const clang::QualType Type) { 333 assert(Type->isIntegerType() && "Given type is not an integer."); 334 unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width; 335 bool Signed = Type->isSignedIntegerType(); 336 return {Width, Signed}; 337 } 338 339 // Given one or more integer types, this function produces an integer type that 340 // encompasses them: any value in one of the given types could be expressed in 341 // the encompassing type. 342 static struct WidthAndSignedness 343 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) { 344 assert(Types.size() > 0 && "Empty list of types."); 345 346 // If any of the given types is signed, we must return a signed type. 347 bool Signed = false; 348 for (const auto &Type : Types) { 349 Signed |= Type.Signed; 350 } 351 352 // The encompassing type must have a width greater than or equal to the width 353 // of the specified types. Aditionally, if the encompassing type is signed, 354 // its width must be strictly greater than the width of any unsigned types 355 // given. 356 unsigned Width = 0; 357 for (const auto &Type : Types) { 358 unsigned MinWidth = Type.Width + (Signed && !Type.Signed); 359 if (Width < MinWidth) { 360 Width = MinWidth; 361 } 362 } 363 364 return {Width, Signed}; 365 } 366 367 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) { 368 llvm::Type *DestType = Int8PtrTy; 369 if (ArgValue->getType() != DestType) 370 ArgValue = 371 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data()); 372 373 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend; 374 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue); 375 } 376 377 /// Checks if using the result of __builtin_object_size(p, @p From) in place of 378 /// __builtin_object_size(p, @p To) is correct 379 static bool areBOSTypesCompatible(int From, int To) { 380 // Note: Our __builtin_object_size implementation currently treats Type=0 and 381 // Type=2 identically. Encoding this implementation detail here may make 382 // improving __builtin_object_size difficult in the future, so it's omitted. 383 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2); 384 } 385 386 static llvm::Value * 387 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) { 388 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true); 389 } 390 391 llvm::Value * 392 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type, 393 llvm::IntegerType *ResType) { 394 uint64_t ObjectSize; 395 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type)) 396 return emitBuiltinObjectSize(E, Type, ResType); 397 return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true); 398 } 399 400 /// Returns a Value corresponding to the size of the given expression. 401 /// This Value may be either of the following: 402 /// - A llvm::Argument (if E is a param with the pass_object_size attribute on 403 /// it) 404 /// - A call to the @llvm.objectsize intrinsic 405 llvm::Value * 406 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type, 407 llvm::IntegerType *ResType) { 408 // We need to reference an argument if the pointer is a parameter with the 409 // pass_object_size attribute. 410 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) { 411 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl()); 412 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>(); 413 if (Param != nullptr && PS != nullptr && 414 areBOSTypesCompatible(PS->getType(), Type)) { 415 auto Iter = SizeArguments.find(Param); 416 assert(Iter != SizeArguments.end()); 417 418 const ImplicitParamDecl *D = Iter->second; 419 auto DIter = LocalDeclMap.find(D); 420 assert(DIter != LocalDeclMap.end()); 421 422 return EmitLoadOfScalar(DIter->second, /*volatile=*/false, 423 getContext().getSizeType(), E->getLocStart()); 424 } 425 } 426 427 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't 428 // evaluate E for side-effects. In either case, we shouldn't lower to 429 // @llvm.objectsize. 430 if (Type == 3 || E->HasSideEffects(getContext())) 431 return getDefaultBuiltinObjectSizeResult(Type, ResType); 432 433 // LLVM only supports 0 and 2, make sure that we pass along that 434 // as a boolean. 435 auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1); 436 // FIXME: Get right address space. 437 llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)}; 438 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); 439 return Builder.CreateCall(F, {EmitScalarExpr(E), CI}); 440 } 441 442 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, 443 unsigned BuiltinID, const CallExpr *E, 444 ReturnValueSlot ReturnValue) { 445 // See if we can constant fold this builtin. If so, don't emit it at all. 446 Expr::EvalResult Result; 447 if (E->EvaluateAsRValue(Result, CGM.getContext()) && 448 !Result.hasSideEffects()) { 449 if (Result.Val.isInt()) 450 return RValue::get(llvm::ConstantInt::get(getLLVMContext(), 451 Result.Val.getInt())); 452 if (Result.Val.isFloat()) 453 return RValue::get(llvm::ConstantFP::get(getLLVMContext(), 454 Result.Val.getFloat())); 455 } 456 457 switch (BuiltinID) { 458 default: break; // Handle intrinsics and libm functions below. 459 case Builtin::BI__builtin___CFStringMakeConstantString: 460 case Builtin::BI__builtin___NSStringMakeConstantString: 461 return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); 462 case Builtin::BI__builtin_stdarg_start: 463 case Builtin::BI__builtin_va_start: 464 case Builtin::BI__va_start: 465 case Builtin::BI__builtin_va_end: 466 return RValue::get( 467 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start 468 ? EmitScalarExpr(E->getArg(0)) 469 : EmitVAListRef(E->getArg(0)).getPointer(), 470 BuiltinID != Builtin::BI__builtin_va_end)); 471 case Builtin::BI__builtin_va_copy: { 472 Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); 473 Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); 474 475 llvm::Type *Type = Int8PtrTy; 476 477 DstPtr = Builder.CreateBitCast(DstPtr, Type); 478 SrcPtr = Builder.CreateBitCast(SrcPtr, Type); 479 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), 480 {DstPtr, SrcPtr})); 481 } 482 case Builtin::BI__builtin_abs: 483 case Builtin::BI__builtin_labs: 484 case Builtin::BI__builtin_llabs: { 485 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 486 487 Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); 488 Value *CmpResult = 489 Builder.CreateICmpSGE(ArgValue, 490 llvm::Constant::getNullValue(ArgValue->getType()), 491 "abscond"); 492 Value *Result = 493 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); 494 495 return RValue::get(Result); 496 } 497 case Builtin::BI__builtin_fabs: 498 case Builtin::BI__builtin_fabsf: 499 case Builtin::BI__builtin_fabsl: { 500 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 501 Value *Result = EmitFAbs(*this, Arg1); 502 return RValue::get(Result); 503 } 504 case Builtin::BI__builtin_fmod: 505 case Builtin::BI__builtin_fmodf: 506 case Builtin::BI__builtin_fmodl: { 507 Value *Arg1 = EmitScalarExpr(E->getArg(0)); 508 Value *Arg2 = EmitScalarExpr(E->getArg(1)); 509 Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); 510 return RValue::get(Result); 511 } 512 513 case Builtin::BI__builtin_conj: 514 case Builtin::BI__builtin_conjf: 515 case Builtin::BI__builtin_conjl: { 516 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 517 Value *Real = ComplexVal.first; 518 Value *Imag = ComplexVal.second; 519 Value *Zero = 520 Imag->getType()->isFPOrFPVectorTy() 521 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) 522 : llvm::Constant::getNullValue(Imag->getType()); 523 524 Imag = Builder.CreateFSub(Zero, Imag, "sub"); 525 return RValue::getComplex(std::make_pair(Real, Imag)); 526 } 527 case Builtin::BI__builtin_creal: 528 case Builtin::BI__builtin_crealf: 529 case Builtin::BI__builtin_creall: 530 case Builtin::BIcreal: 531 case Builtin::BIcrealf: 532 case Builtin::BIcreall: { 533 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 534 return RValue::get(ComplexVal.first); 535 } 536 537 case Builtin::BI__builtin_cimag: 538 case Builtin::BI__builtin_cimagf: 539 case Builtin::BI__builtin_cimagl: 540 case Builtin::BIcimag: 541 case Builtin::BIcimagf: 542 case Builtin::BIcimagl: { 543 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); 544 return RValue::get(ComplexVal.second); 545 } 546 547 case Builtin::BI__builtin_ctzs: 548 case Builtin::BI__builtin_ctz: 549 case Builtin::BI__builtin_ctzl: 550 case Builtin::BI__builtin_ctzll: { 551 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 552 553 llvm::Type *ArgType = ArgValue->getType(); 554 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 555 556 llvm::Type *ResultType = ConvertType(E->getType()); 557 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 558 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 559 if (Result->getType() != ResultType) 560 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 561 "cast"); 562 return RValue::get(Result); 563 } 564 case Builtin::BI__builtin_clzs: 565 case Builtin::BI__builtin_clz: 566 case Builtin::BI__builtin_clzl: 567 case Builtin::BI__builtin_clzll: { 568 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 569 570 llvm::Type *ArgType = ArgValue->getType(); 571 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); 572 573 llvm::Type *ResultType = ConvertType(E->getType()); 574 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); 575 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); 576 if (Result->getType() != ResultType) 577 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 578 "cast"); 579 return RValue::get(Result); 580 } 581 case Builtin::BI__builtin_ffs: 582 case Builtin::BI__builtin_ffsl: 583 case Builtin::BI__builtin_ffsll: { 584 // ffs(x) -> x ? cttz(x) + 1 : 0 585 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 586 587 llvm::Type *ArgType = ArgValue->getType(); 588 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); 589 590 llvm::Type *ResultType = ConvertType(E->getType()); 591 Value *Tmp = 592 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), 593 llvm::ConstantInt::get(ArgType, 1)); 594 Value *Zero = llvm::Constant::getNullValue(ArgType); 595 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); 596 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); 597 if (Result->getType() != ResultType) 598 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 599 "cast"); 600 return RValue::get(Result); 601 } 602 case Builtin::BI__builtin_parity: 603 case Builtin::BI__builtin_parityl: 604 case Builtin::BI__builtin_parityll: { 605 // parity(x) -> ctpop(x) & 1 606 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 607 608 llvm::Type *ArgType = ArgValue->getType(); 609 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 610 611 llvm::Type *ResultType = ConvertType(E->getType()); 612 Value *Tmp = Builder.CreateCall(F, ArgValue); 613 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); 614 if (Result->getType() != ResultType) 615 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 616 "cast"); 617 return RValue::get(Result); 618 } 619 case Builtin::BI__builtin_popcount: 620 case Builtin::BI__builtin_popcountl: 621 case Builtin::BI__builtin_popcountll: { 622 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 623 624 llvm::Type *ArgType = ArgValue->getType(); 625 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); 626 627 llvm::Type *ResultType = ConvertType(E->getType()); 628 Value *Result = Builder.CreateCall(F, ArgValue); 629 if (Result->getType() != ResultType) 630 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, 631 "cast"); 632 return RValue::get(Result); 633 } 634 case Builtin::BI__builtin_unpredictable: { 635 // Always return the argument of __builtin_unpredictable. LLVM does not 636 // handle this builtin. Metadata for this builtin should be added directly 637 // to instructions such as branches or switches that use it. 638 return RValue::get(EmitScalarExpr(E->getArg(0))); 639 } 640 case Builtin::BI__builtin_expect: { 641 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 642 llvm::Type *ArgType = ArgValue->getType(); 643 644 Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); 645 // Don't generate llvm.expect on -O0 as the backend won't use it for 646 // anything. 647 // Note, we still IRGen ExpectedValue because it could have side-effects. 648 if (CGM.getCodeGenOpts().OptimizationLevel == 0) 649 return RValue::get(ArgValue); 650 651 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); 652 Value *Result = 653 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); 654 return RValue::get(Result); 655 } 656 case Builtin::BI__builtin_assume_aligned: { 657 Value *PtrValue = EmitScalarExpr(E->getArg(0)); 658 Value *OffsetValue = 659 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; 660 661 Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); 662 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); 663 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); 664 665 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); 666 return RValue::get(PtrValue); 667 } 668 case Builtin::BI__assume: 669 case Builtin::BI__builtin_assume: { 670 if (E->getArg(0)->HasSideEffects(getContext())) 671 return RValue::get(nullptr); 672 673 Value *ArgValue = EmitScalarExpr(E->getArg(0)); 674 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); 675 return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); 676 } 677 case Builtin::BI__builtin_bswap16: 678 case Builtin::BI__builtin_bswap32: 679 case Builtin::BI__builtin_bswap64: { 680 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap)); 681 } 682 case Builtin::BI__builtin_bitreverse8: 683 case Builtin::BI__builtin_bitreverse16: 684 case Builtin::BI__builtin_bitreverse32: 685 case Builtin::BI__builtin_bitreverse64: { 686 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse)); 687 } 688 case Builtin::BI__builtin_object_size: { 689 unsigned Type = 690 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue(); 691 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType())); 692 693 // We pass this builtin onto the optimizer so that it can figure out the 694 // object size in more complex cases. 695 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType)); 696 } 697 case Builtin::BI__builtin_prefetch: { 698 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); 699 // FIXME: Technically these constants should of type 'int', yes? 700 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : 701 llvm::ConstantInt::get(Int32Ty, 0); 702 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : 703 llvm::ConstantInt::get(Int32Ty, 3); 704 Value *Data = llvm::ConstantInt::get(Int32Ty, 1); 705 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 706 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); 707 } 708 case Builtin::BI__builtin_readcyclecounter: { 709 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); 710 return RValue::get(Builder.CreateCall(F)); 711 } 712 case Builtin::BI__builtin___clear_cache: { 713 Value *Begin = EmitScalarExpr(E->getArg(0)); 714 Value *End = EmitScalarExpr(E->getArg(1)); 715 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); 716 return RValue::get(Builder.CreateCall(F, {Begin, End})); 717 } 718 case Builtin::BI__builtin_trap: 719 return RValue::get(EmitTrapCall(Intrinsic::trap)); 720 case Builtin::BI__debugbreak: 721 return RValue::get(EmitTrapCall(Intrinsic::debugtrap)); 722 case Builtin::BI__builtin_unreachable: { 723 if (SanOpts.has(SanitizerKind::Unreachable)) { 724 SanitizerScope SanScope(this); 725 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), 726 SanitizerKind::Unreachable), 727 "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), 728 None); 729 } else 730 Builder.CreateUnreachable(); 731 732 // We do need to preserve an insertion point. 733 EmitBlock(createBasicBlock("unreachable.cont")); 734 735 return RValue::get(nullptr); 736 } 737 738 case Builtin::BI__builtin_powi: 739 case Builtin::BI__builtin_powif: 740 case Builtin::BI__builtin_powil: { 741 Value *Base = EmitScalarExpr(E->getArg(0)); 742 Value *Exponent = EmitScalarExpr(E->getArg(1)); 743 llvm::Type *ArgType = Base->getType(); 744 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); 745 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 746 } 747 748 case Builtin::BI__builtin_isgreater: 749 case Builtin::BI__builtin_isgreaterequal: 750 case Builtin::BI__builtin_isless: 751 case Builtin::BI__builtin_islessequal: 752 case Builtin::BI__builtin_islessgreater: 753 case Builtin::BI__builtin_isunordered: { 754 // Ordered comparisons: we know the arguments to these are matching scalar 755 // floating point values. 756 Value *LHS = EmitScalarExpr(E->getArg(0)); 757 Value *RHS = EmitScalarExpr(E->getArg(1)); 758 759 switch (BuiltinID) { 760 default: llvm_unreachable("Unknown ordered comparison"); 761 case Builtin::BI__builtin_isgreater: 762 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); 763 break; 764 case Builtin::BI__builtin_isgreaterequal: 765 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); 766 break; 767 case Builtin::BI__builtin_isless: 768 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); 769 break; 770 case Builtin::BI__builtin_islessequal: 771 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); 772 break; 773 case Builtin::BI__builtin_islessgreater: 774 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); 775 break; 776 case Builtin::BI__builtin_isunordered: 777 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); 778 break; 779 } 780 // ZExt bool to int type. 781 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); 782 } 783 case Builtin::BI__builtin_isnan: { 784 Value *V = EmitScalarExpr(E->getArg(0)); 785 V = Builder.CreateFCmpUNO(V, V, "cmp"); 786 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 787 } 788 789 case Builtin::BI__builtin_isinf: 790 case Builtin::BI__builtin_isfinite: { 791 // isinf(x) --> fabs(x) == infinity 792 // isfinite(x) --> fabs(x) != infinity 793 // x != NaN via the ordered compare in either case. 794 Value *V = EmitScalarExpr(E->getArg(0)); 795 Value *Fabs = EmitFAbs(*this, V); 796 Constant *Infinity = ConstantFP::getInfinity(V->getType()); 797 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf) 798 ? CmpInst::FCMP_OEQ 799 : CmpInst::FCMP_ONE; 800 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf"); 801 return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType()))); 802 } 803 804 case Builtin::BI__builtin_isinf_sign: { 805 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 806 Value *Arg = EmitScalarExpr(E->getArg(0)); 807 Value *AbsArg = EmitFAbs(*this, Arg); 808 Value *IsInf = Builder.CreateFCmpOEQ( 809 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); 810 Value *IsNeg = EmitSignBit(*this, Arg); 811 812 llvm::Type *IntTy = ConvertType(E->getType()); 813 Value *Zero = Constant::getNullValue(IntTy); 814 Value *One = ConstantInt::get(IntTy, 1); 815 Value *NegativeOne = ConstantInt::get(IntTy, -1); 816 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); 817 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); 818 return RValue::get(Result); 819 } 820 821 case Builtin::BI__builtin_isnormal: { 822 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min 823 Value *V = EmitScalarExpr(E->getArg(0)); 824 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); 825 826 Value *Abs = EmitFAbs(*this, V); 827 Value *IsLessThanInf = 828 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); 829 APFloat Smallest = APFloat::getSmallestNormalized( 830 getContext().getFloatTypeSemantics(E->getArg(0)->getType())); 831 Value *IsNormal = 832 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), 833 "isnormal"); 834 V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); 835 V = Builder.CreateAnd(V, IsNormal, "and"); 836 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); 837 } 838 839 case Builtin::BI__builtin_fpclassify: { 840 Value *V = EmitScalarExpr(E->getArg(5)); 841 llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); 842 843 // Create Result 844 BasicBlock *Begin = Builder.GetInsertBlock(); 845 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); 846 Builder.SetInsertPoint(End); 847 PHINode *Result = 848 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, 849 "fpclassify_result"); 850 851 // if (V==0) return FP_ZERO 852 Builder.SetInsertPoint(Begin); 853 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), 854 "iszero"); 855 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); 856 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); 857 Builder.CreateCondBr(IsZero, End, NotZero); 858 Result->addIncoming(ZeroLiteral, Begin); 859 860 // if (V != V) return FP_NAN 861 Builder.SetInsertPoint(NotZero); 862 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); 863 Value *NanLiteral = EmitScalarExpr(E->getArg(0)); 864 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); 865 Builder.CreateCondBr(IsNan, End, NotNan); 866 Result->addIncoming(NanLiteral, NotZero); 867 868 // if (fabs(V) == infinity) return FP_INFINITY 869 Builder.SetInsertPoint(NotNan); 870 Value *VAbs = EmitFAbs(*this, V); 871 Value *IsInf = 872 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), 873 "isinf"); 874 Value *InfLiteral = EmitScalarExpr(E->getArg(1)); 875 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); 876 Builder.CreateCondBr(IsInf, End, NotInf); 877 Result->addIncoming(InfLiteral, NotNan); 878 879 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL 880 Builder.SetInsertPoint(NotInf); 881 APFloat Smallest = APFloat::getSmallestNormalized( 882 getContext().getFloatTypeSemantics(E->getArg(5)->getType())); 883 Value *IsNormal = 884 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), 885 "isnormal"); 886 Value *NormalResult = 887 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), 888 EmitScalarExpr(E->getArg(3))); 889 Builder.CreateBr(End); 890 Result->addIncoming(NormalResult, NotInf); 891 892 // return Result 893 Builder.SetInsertPoint(End); 894 return RValue::get(Result); 895 } 896 897 case Builtin::BIalloca: 898 case Builtin::BI_alloca: 899 case Builtin::BI__builtin_alloca: { 900 Value *Size = EmitScalarExpr(E->getArg(0)); 901 return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); 902 } 903 case Builtin::BIbzero: 904 case Builtin::BI__builtin_bzero: { 905 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 906 Value *SizeVal = EmitScalarExpr(E->getArg(1)); 907 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 908 E->getArg(0)->getExprLoc(), FD, 0); 909 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); 910 return RValue::get(Dest.getPointer()); 911 } 912 case Builtin::BImemcpy: 913 case Builtin::BI__builtin_memcpy: { 914 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 915 Address Src = EmitPointerWithAlignment(E->getArg(1)); 916 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 917 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 918 E->getArg(0)->getExprLoc(), FD, 0); 919 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 920 E->getArg(1)->getExprLoc(), FD, 1); 921 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 922 return RValue::get(Dest.getPointer()); 923 } 924 925 case Builtin::BI__builtin___memcpy_chk: { 926 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. 927 llvm::APSInt Size, DstSize; 928 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 929 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 930 break; 931 if (Size.ugt(DstSize)) 932 break; 933 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 934 Address Src = EmitPointerWithAlignment(E->getArg(1)); 935 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 936 Builder.CreateMemCpy(Dest, Src, SizeVal, false); 937 return RValue::get(Dest.getPointer()); 938 } 939 940 case Builtin::BI__builtin_objc_memmove_collectable: { 941 Address DestAddr = EmitPointerWithAlignment(E->getArg(0)); 942 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1)); 943 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 944 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 945 DestAddr, SrcAddr, SizeVal); 946 return RValue::get(DestAddr.getPointer()); 947 } 948 949 case Builtin::BI__builtin___memmove_chk: { 950 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. 951 llvm::APSInt Size, DstSize; 952 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 953 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 954 break; 955 if (Size.ugt(DstSize)) 956 break; 957 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 958 Address Src = EmitPointerWithAlignment(E->getArg(1)); 959 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 960 Builder.CreateMemMove(Dest, Src, SizeVal, false); 961 return RValue::get(Dest.getPointer()); 962 } 963 964 case Builtin::BImemmove: 965 case Builtin::BI__builtin_memmove: { 966 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 967 Address Src = EmitPointerWithAlignment(E->getArg(1)); 968 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 969 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 970 E->getArg(0)->getExprLoc(), FD, 0); 971 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(), 972 E->getArg(1)->getExprLoc(), FD, 1); 973 Builder.CreateMemMove(Dest, Src, SizeVal, false); 974 return RValue::get(Dest.getPointer()); 975 } 976 case Builtin::BImemset: 977 case Builtin::BI__builtin_memset: { 978 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 979 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 980 Builder.getInt8Ty()); 981 Value *SizeVal = EmitScalarExpr(E->getArg(2)); 982 EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), 983 E->getArg(0)->getExprLoc(), FD, 0); 984 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 985 return RValue::get(Dest.getPointer()); 986 } 987 case Builtin::BI__builtin___memset_chk: { 988 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. 989 llvm::APSInt Size, DstSize; 990 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || 991 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) 992 break; 993 if (Size.ugt(DstSize)) 994 break; 995 Address Dest = EmitPointerWithAlignment(E->getArg(0)); 996 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), 997 Builder.getInt8Ty()); 998 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); 999 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); 1000 return RValue::get(Dest.getPointer()); 1001 } 1002 case Builtin::BI__builtin_dwarf_cfa: { 1003 // The offset in bytes from the first argument to the CFA. 1004 // 1005 // Why on earth is this in the frontend? Is there any reason at 1006 // all that the backend can't reasonably determine this while 1007 // lowering llvm.eh.dwarf.cfa()? 1008 // 1009 // TODO: If there's a satisfactory reason, add a target hook for 1010 // this instead of hard-coding 0, which is correct for most targets. 1011 int32_t Offset = 0; 1012 1013 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); 1014 return RValue::get(Builder.CreateCall(F, 1015 llvm::ConstantInt::get(Int32Ty, Offset))); 1016 } 1017 case Builtin::BI__builtin_return_address: { 1018 Value *Depth = 1019 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1020 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); 1021 return RValue::get(Builder.CreateCall(F, Depth)); 1022 } 1023 case Builtin::BI__builtin_frame_address: { 1024 Value *Depth = 1025 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this); 1026 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); 1027 return RValue::get(Builder.CreateCall(F, Depth)); 1028 } 1029 case Builtin::BI__builtin_extract_return_addr: { 1030 Value *Address = EmitScalarExpr(E->getArg(0)); 1031 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); 1032 return RValue::get(Result); 1033 } 1034 case Builtin::BI__builtin_frob_return_addr: { 1035 Value *Address = EmitScalarExpr(E->getArg(0)); 1036 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); 1037 return RValue::get(Result); 1038 } 1039 case Builtin::BI__builtin_dwarf_sp_column: { 1040 llvm::IntegerType *Ty 1041 = cast<llvm::IntegerType>(ConvertType(E->getType())); 1042 int Column = getTargetHooks().getDwarfEHStackPointer(CGM); 1043 if (Column == -1) { 1044 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); 1045 return RValue::get(llvm::UndefValue::get(Ty)); 1046 } 1047 return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); 1048 } 1049 case Builtin::BI__builtin_init_dwarf_reg_size_table: { 1050 Value *Address = EmitScalarExpr(E->getArg(0)); 1051 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) 1052 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); 1053 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); 1054 } 1055 case Builtin::BI__builtin_eh_return: { 1056 Value *Int = EmitScalarExpr(E->getArg(0)); 1057 Value *Ptr = EmitScalarExpr(E->getArg(1)); 1058 1059 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); 1060 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && 1061 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); 1062 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 1063 ? Intrinsic::eh_return_i32 1064 : Intrinsic::eh_return_i64); 1065 Builder.CreateCall(F, {Int, Ptr}); 1066 Builder.CreateUnreachable(); 1067 1068 // We do need to preserve an insertion point. 1069 EmitBlock(createBasicBlock("builtin_eh_return.cont")); 1070 1071 return RValue::get(nullptr); 1072 } 1073 case Builtin::BI__builtin_unwind_init: { 1074 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); 1075 return RValue::get(Builder.CreateCall(F)); 1076 } 1077 case Builtin::BI__builtin_extend_pointer: { 1078 // Extends a pointer to the size of an _Unwind_Word, which is 1079 // uint64_t on all platforms. Generally this gets poked into a 1080 // register and eventually used as an address, so if the 1081 // addressing registers are wider than pointers and the platform 1082 // doesn't implicitly ignore high-order bits when doing 1083 // addressing, we need to make sure we zext / sext based on 1084 // the platform's expectations. 1085 // 1086 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html 1087 1088 // Cast the pointer to intptr_t. 1089 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1090 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); 1091 1092 // If that's 64 bits, we're done. 1093 if (IntPtrTy->getBitWidth() == 64) 1094 return RValue::get(Result); 1095 1096 // Otherwise, ask the codegen data what to do. 1097 if (getTargetHooks().extendPointerWithSExt()) 1098 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); 1099 else 1100 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); 1101 } 1102 case Builtin::BI__builtin_setjmp: { 1103 // Buffer is a void**. 1104 Address Buf = EmitPointerWithAlignment(E->getArg(0)); 1105 1106 // Store the frame pointer to the setjmp buffer. 1107 Value *FrameAddr = 1108 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1109 ConstantInt::get(Int32Ty, 0)); 1110 Builder.CreateStore(FrameAddr, Buf); 1111 1112 // Store the stack pointer to the setjmp buffer. 1113 Value *StackAddr = 1114 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); 1115 Address StackSaveSlot = 1116 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize()); 1117 Builder.CreateStore(StackAddr, StackSaveSlot); 1118 1119 // Call LLVM's EH setjmp, which is lightweight. 1120 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); 1121 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1122 return RValue::get(Builder.CreateCall(F, Buf.getPointer())); 1123 } 1124 case Builtin::BI__builtin_longjmp: { 1125 Value *Buf = EmitScalarExpr(E->getArg(0)); 1126 Buf = Builder.CreateBitCast(Buf, Int8PtrTy); 1127 1128 // Call LLVM's EH longjmp, which is lightweight. 1129 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); 1130 1131 // longjmp doesn't return; mark this as unreachable. 1132 Builder.CreateUnreachable(); 1133 1134 // We do need to preserve an insertion point. 1135 EmitBlock(createBasicBlock("longjmp.cont")); 1136 1137 return RValue::get(nullptr); 1138 } 1139 case Builtin::BI__sync_fetch_and_add: 1140 case Builtin::BI__sync_fetch_and_sub: 1141 case Builtin::BI__sync_fetch_and_or: 1142 case Builtin::BI__sync_fetch_and_and: 1143 case Builtin::BI__sync_fetch_and_xor: 1144 case Builtin::BI__sync_fetch_and_nand: 1145 case Builtin::BI__sync_add_and_fetch: 1146 case Builtin::BI__sync_sub_and_fetch: 1147 case Builtin::BI__sync_and_and_fetch: 1148 case Builtin::BI__sync_or_and_fetch: 1149 case Builtin::BI__sync_xor_and_fetch: 1150 case Builtin::BI__sync_nand_and_fetch: 1151 case Builtin::BI__sync_val_compare_and_swap: 1152 case Builtin::BI__sync_bool_compare_and_swap: 1153 case Builtin::BI__sync_lock_test_and_set: 1154 case Builtin::BI__sync_lock_release: 1155 case Builtin::BI__sync_swap: 1156 llvm_unreachable("Shouldn't make it through sema"); 1157 case Builtin::BI__sync_fetch_and_add_1: 1158 case Builtin::BI__sync_fetch_and_add_2: 1159 case Builtin::BI__sync_fetch_and_add_4: 1160 case Builtin::BI__sync_fetch_and_add_8: 1161 case Builtin::BI__sync_fetch_and_add_16: 1162 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); 1163 case Builtin::BI__sync_fetch_and_sub_1: 1164 case Builtin::BI__sync_fetch_and_sub_2: 1165 case Builtin::BI__sync_fetch_and_sub_4: 1166 case Builtin::BI__sync_fetch_and_sub_8: 1167 case Builtin::BI__sync_fetch_and_sub_16: 1168 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); 1169 case Builtin::BI__sync_fetch_and_or_1: 1170 case Builtin::BI__sync_fetch_and_or_2: 1171 case Builtin::BI__sync_fetch_and_or_4: 1172 case Builtin::BI__sync_fetch_and_or_8: 1173 case Builtin::BI__sync_fetch_and_or_16: 1174 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); 1175 case Builtin::BI__sync_fetch_and_and_1: 1176 case Builtin::BI__sync_fetch_and_and_2: 1177 case Builtin::BI__sync_fetch_and_and_4: 1178 case Builtin::BI__sync_fetch_and_and_8: 1179 case Builtin::BI__sync_fetch_and_and_16: 1180 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); 1181 case Builtin::BI__sync_fetch_and_xor_1: 1182 case Builtin::BI__sync_fetch_and_xor_2: 1183 case Builtin::BI__sync_fetch_and_xor_4: 1184 case Builtin::BI__sync_fetch_and_xor_8: 1185 case Builtin::BI__sync_fetch_and_xor_16: 1186 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); 1187 case Builtin::BI__sync_fetch_and_nand_1: 1188 case Builtin::BI__sync_fetch_and_nand_2: 1189 case Builtin::BI__sync_fetch_and_nand_4: 1190 case Builtin::BI__sync_fetch_and_nand_8: 1191 case Builtin::BI__sync_fetch_and_nand_16: 1192 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); 1193 1194 // Clang extensions: not overloaded yet. 1195 case Builtin::BI__sync_fetch_and_min: 1196 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); 1197 case Builtin::BI__sync_fetch_and_max: 1198 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); 1199 case Builtin::BI__sync_fetch_and_umin: 1200 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); 1201 case Builtin::BI__sync_fetch_and_umax: 1202 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); 1203 1204 case Builtin::BI__sync_add_and_fetch_1: 1205 case Builtin::BI__sync_add_and_fetch_2: 1206 case Builtin::BI__sync_add_and_fetch_4: 1207 case Builtin::BI__sync_add_and_fetch_8: 1208 case Builtin::BI__sync_add_and_fetch_16: 1209 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, 1210 llvm::Instruction::Add); 1211 case Builtin::BI__sync_sub_and_fetch_1: 1212 case Builtin::BI__sync_sub_and_fetch_2: 1213 case Builtin::BI__sync_sub_and_fetch_4: 1214 case Builtin::BI__sync_sub_and_fetch_8: 1215 case Builtin::BI__sync_sub_and_fetch_16: 1216 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, 1217 llvm::Instruction::Sub); 1218 case Builtin::BI__sync_and_and_fetch_1: 1219 case Builtin::BI__sync_and_and_fetch_2: 1220 case Builtin::BI__sync_and_and_fetch_4: 1221 case Builtin::BI__sync_and_and_fetch_8: 1222 case Builtin::BI__sync_and_and_fetch_16: 1223 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, 1224 llvm::Instruction::And); 1225 case Builtin::BI__sync_or_and_fetch_1: 1226 case Builtin::BI__sync_or_and_fetch_2: 1227 case Builtin::BI__sync_or_and_fetch_4: 1228 case Builtin::BI__sync_or_and_fetch_8: 1229 case Builtin::BI__sync_or_and_fetch_16: 1230 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, 1231 llvm::Instruction::Or); 1232 case Builtin::BI__sync_xor_and_fetch_1: 1233 case Builtin::BI__sync_xor_and_fetch_2: 1234 case Builtin::BI__sync_xor_and_fetch_4: 1235 case Builtin::BI__sync_xor_and_fetch_8: 1236 case Builtin::BI__sync_xor_and_fetch_16: 1237 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, 1238 llvm::Instruction::Xor); 1239 case Builtin::BI__sync_nand_and_fetch_1: 1240 case Builtin::BI__sync_nand_and_fetch_2: 1241 case Builtin::BI__sync_nand_and_fetch_4: 1242 case Builtin::BI__sync_nand_and_fetch_8: 1243 case Builtin::BI__sync_nand_and_fetch_16: 1244 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, 1245 llvm::Instruction::And, true); 1246 1247 case Builtin::BI__sync_val_compare_and_swap_1: 1248 case Builtin::BI__sync_val_compare_and_swap_2: 1249 case Builtin::BI__sync_val_compare_and_swap_4: 1250 case Builtin::BI__sync_val_compare_and_swap_8: 1251 case Builtin::BI__sync_val_compare_and_swap_16: 1252 return RValue::get(MakeAtomicCmpXchgValue(*this, E, false)); 1253 1254 case Builtin::BI__sync_bool_compare_and_swap_1: 1255 case Builtin::BI__sync_bool_compare_and_swap_2: 1256 case Builtin::BI__sync_bool_compare_and_swap_4: 1257 case Builtin::BI__sync_bool_compare_and_swap_8: 1258 case Builtin::BI__sync_bool_compare_and_swap_16: 1259 return RValue::get(MakeAtomicCmpXchgValue(*this, E, true)); 1260 1261 case Builtin::BI__sync_swap_1: 1262 case Builtin::BI__sync_swap_2: 1263 case Builtin::BI__sync_swap_4: 1264 case Builtin::BI__sync_swap_8: 1265 case Builtin::BI__sync_swap_16: 1266 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1267 1268 case Builtin::BI__sync_lock_test_and_set_1: 1269 case Builtin::BI__sync_lock_test_and_set_2: 1270 case Builtin::BI__sync_lock_test_and_set_4: 1271 case Builtin::BI__sync_lock_test_and_set_8: 1272 case Builtin::BI__sync_lock_test_and_set_16: 1273 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1274 1275 case Builtin::BI__sync_lock_release_1: 1276 case Builtin::BI__sync_lock_release_2: 1277 case Builtin::BI__sync_lock_release_4: 1278 case Builtin::BI__sync_lock_release_8: 1279 case Builtin::BI__sync_lock_release_16: { 1280 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1281 QualType ElTy = E->getArg(0)->getType()->getPointeeType(); 1282 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); 1283 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), 1284 StoreSize.getQuantity() * 8); 1285 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); 1286 llvm::StoreInst *Store = 1287 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr, 1288 StoreSize); 1289 Store->setAtomic(llvm::AtomicOrdering::Release); 1290 return RValue::get(nullptr); 1291 } 1292 1293 case Builtin::BI__sync_synchronize: { 1294 // We assume this is supposed to correspond to a C++0x-style 1295 // sequentially-consistent fence (i.e. this is only usable for 1296 // synchonization, not device I/O or anything like that). This intrinsic 1297 // is really badly designed in the sense that in theory, there isn't 1298 // any way to safely use it... but in practice, it mostly works 1299 // to use it with non-atomic loads and stores to get acquire/release 1300 // semantics. 1301 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); 1302 return RValue::get(nullptr); 1303 } 1304 1305 case Builtin::BI__builtin_nontemporal_load: 1306 return RValue::get(EmitNontemporalLoad(*this, E)); 1307 case Builtin::BI__builtin_nontemporal_store: 1308 return RValue::get(EmitNontemporalStore(*this, E)); 1309 case Builtin::BI__c11_atomic_is_lock_free: 1310 case Builtin::BI__atomic_is_lock_free: { 1311 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the 1312 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since 1313 // _Atomic(T) is always properly-aligned. 1314 const char *LibCallName = "__atomic_is_lock_free"; 1315 CallArgList Args; 1316 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), 1317 getContext().getSizeType()); 1318 if (BuiltinID == Builtin::BI__atomic_is_lock_free) 1319 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), 1320 getContext().VoidPtrTy); 1321 else 1322 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), 1323 getContext().VoidPtrTy); 1324 const CGFunctionInfo &FuncInfo = 1325 CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args); 1326 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 1327 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 1328 return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 1329 } 1330 1331 case Builtin::BI__atomic_test_and_set: { 1332 // Look at the argument type to determine whether this is a volatile 1333 // operation. The parameter type is always volatile. 1334 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1335 bool Volatile = 1336 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1337 1338 Value *Ptr = EmitScalarExpr(E->getArg(0)); 1339 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); 1340 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1341 Value *NewVal = Builder.getInt8(1); 1342 Value *Order = EmitScalarExpr(E->getArg(1)); 1343 if (isa<llvm::ConstantInt>(Order)) { 1344 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1345 AtomicRMWInst *Result = nullptr; 1346 switch (ord) { 1347 case 0: // memory_order_relaxed 1348 default: // invalid order 1349 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1350 llvm::AtomicOrdering::Monotonic); 1351 break; 1352 case 1: // memory_order_consume 1353 case 2: // memory_order_acquire 1354 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1355 llvm::AtomicOrdering::Acquire); 1356 break; 1357 case 3: // memory_order_release 1358 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1359 llvm::AtomicOrdering::Release); 1360 break; 1361 case 4: // memory_order_acq_rel 1362 1363 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1364 llvm::AtomicOrdering::AcquireRelease); 1365 break; 1366 case 5: // memory_order_seq_cst 1367 Result = Builder.CreateAtomicRMW( 1368 llvm::AtomicRMWInst::Xchg, Ptr, NewVal, 1369 llvm::AtomicOrdering::SequentiallyConsistent); 1370 break; 1371 } 1372 Result->setVolatile(Volatile); 1373 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1374 } 1375 1376 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1377 1378 llvm::BasicBlock *BBs[5] = { 1379 createBasicBlock("monotonic", CurFn), 1380 createBasicBlock("acquire", CurFn), 1381 createBasicBlock("release", CurFn), 1382 createBasicBlock("acqrel", CurFn), 1383 createBasicBlock("seqcst", CurFn) 1384 }; 1385 llvm::AtomicOrdering Orders[5] = { 1386 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire, 1387 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease, 1388 llvm::AtomicOrdering::SequentiallyConsistent}; 1389 1390 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1391 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1392 1393 Builder.SetInsertPoint(ContBB); 1394 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); 1395 1396 for (unsigned i = 0; i < 5; ++i) { 1397 Builder.SetInsertPoint(BBs[i]); 1398 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, 1399 Ptr, NewVal, Orders[i]); 1400 RMW->setVolatile(Volatile); 1401 Result->addIncoming(RMW, BBs[i]); 1402 Builder.CreateBr(ContBB); 1403 } 1404 1405 SI->addCase(Builder.getInt32(0), BBs[0]); 1406 SI->addCase(Builder.getInt32(1), BBs[1]); 1407 SI->addCase(Builder.getInt32(2), BBs[1]); 1408 SI->addCase(Builder.getInt32(3), BBs[2]); 1409 SI->addCase(Builder.getInt32(4), BBs[3]); 1410 SI->addCase(Builder.getInt32(5), BBs[4]); 1411 1412 Builder.SetInsertPoint(ContBB); 1413 return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); 1414 } 1415 1416 case Builtin::BI__atomic_clear: { 1417 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); 1418 bool Volatile = 1419 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); 1420 1421 Address Ptr = EmitPointerWithAlignment(E->getArg(0)); 1422 unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace(); 1423 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); 1424 Value *NewVal = Builder.getInt8(0); 1425 Value *Order = EmitScalarExpr(E->getArg(1)); 1426 if (isa<llvm::ConstantInt>(Order)) { 1427 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1428 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1429 switch (ord) { 1430 case 0: // memory_order_relaxed 1431 default: // invalid order 1432 Store->setOrdering(llvm::AtomicOrdering::Monotonic); 1433 break; 1434 case 3: // memory_order_release 1435 Store->setOrdering(llvm::AtomicOrdering::Release); 1436 break; 1437 case 5: // memory_order_seq_cst 1438 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent); 1439 break; 1440 } 1441 return RValue::get(nullptr); 1442 } 1443 1444 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1445 1446 llvm::BasicBlock *BBs[3] = { 1447 createBasicBlock("monotonic", CurFn), 1448 createBasicBlock("release", CurFn), 1449 createBasicBlock("seqcst", CurFn) 1450 }; 1451 llvm::AtomicOrdering Orders[3] = { 1452 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release, 1453 llvm::AtomicOrdering::SequentiallyConsistent}; 1454 1455 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1456 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); 1457 1458 for (unsigned i = 0; i < 3; ++i) { 1459 Builder.SetInsertPoint(BBs[i]); 1460 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); 1461 Store->setOrdering(Orders[i]); 1462 Builder.CreateBr(ContBB); 1463 } 1464 1465 SI->addCase(Builder.getInt32(0), BBs[0]); 1466 SI->addCase(Builder.getInt32(3), BBs[1]); 1467 SI->addCase(Builder.getInt32(5), BBs[2]); 1468 1469 Builder.SetInsertPoint(ContBB); 1470 return RValue::get(nullptr); 1471 } 1472 1473 case Builtin::BI__atomic_thread_fence: 1474 case Builtin::BI__atomic_signal_fence: 1475 case Builtin::BI__c11_atomic_thread_fence: 1476 case Builtin::BI__c11_atomic_signal_fence: { 1477 llvm::SynchronizationScope Scope; 1478 if (BuiltinID == Builtin::BI__atomic_signal_fence || 1479 BuiltinID == Builtin::BI__c11_atomic_signal_fence) 1480 Scope = llvm::SingleThread; 1481 else 1482 Scope = llvm::CrossThread; 1483 Value *Order = EmitScalarExpr(E->getArg(0)); 1484 if (isa<llvm::ConstantInt>(Order)) { 1485 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1486 switch (ord) { 1487 case 0: // memory_order_relaxed 1488 default: // invalid order 1489 break; 1490 case 1: // memory_order_consume 1491 case 2: // memory_order_acquire 1492 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1493 break; 1494 case 3: // memory_order_release 1495 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1496 break; 1497 case 4: // memory_order_acq_rel 1498 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1499 break; 1500 case 5: // memory_order_seq_cst 1501 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, 1502 Scope); 1503 break; 1504 } 1505 return RValue::get(nullptr); 1506 } 1507 1508 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; 1509 AcquireBB = createBasicBlock("acquire", CurFn); 1510 ReleaseBB = createBasicBlock("release", CurFn); 1511 AcqRelBB = createBasicBlock("acqrel", CurFn); 1512 SeqCstBB = createBasicBlock("seqcst", CurFn); 1513 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1514 1515 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1516 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); 1517 1518 Builder.SetInsertPoint(AcquireBB); 1519 Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope); 1520 Builder.CreateBr(ContBB); 1521 SI->addCase(Builder.getInt32(1), AcquireBB); 1522 SI->addCase(Builder.getInt32(2), AcquireBB); 1523 1524 Builder.SetInsertPoint(ReleaseBB); 1525 Builder.CreateFence(llvm::AtomicOrdering::Release, Scope); 1526 Builder.CreateBr(ContBB); 1527 SI->addCase(Builder.getInt32(3), ReleaseBB); 1528 1529 Builder.SetInsertPoint(AcqRelBB); 1530 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope); 1531 Builder.CreateBr(ContBB); 1532 SI->addCase(Builder.getInt32(4), AcqRelBB); 1533 1534 Builder.SetInsertPoint(SeqCstBB); 1535 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1536 Builder.CreateBr(ContBB); 1537 SI->addCase(Builder.getInt32(5), SeqCstBB); 1538 1539 Builder.SetInsertPoint(ContBB); 1540 return RValue::get(nullptr); 1541 } 1542 1543 // Library functions with special handling. 1544 case Builtin::BIsqrt: 1545 case Builtin::BIsqrtf: 1546 case Builtin::BIsqrtl: { 1547 // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only 1548 // in finite- or unsafe-math mode (the intrinsic has different semantics 1549 // for handling negative numbers compared to the library function, so 1550 // -fmath-errno=0 is not enough). 1551 if (!FD->hasAttr<ConstAttr>()) 1552 break; 1553 if (!(CGM.getCodeGenOpts().UnsafeFPMath || 1554 CGM.getCodeGenOpts().NoNaNsFPMath)) 1555 break; 1556 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 1557 llvm::Type *ArgType = Arg0->getType(); 1558 Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); 1559 return RValue::get(Builder.CreateCall(F, Arg0)); 1560 } 1561 1562 case Builtin::BI__builtin_pow: 1563 case Builtin::BI__builtin_powf: 1564 case Builtin::BI__builtin_powl: 1565 case Builtin::BIpow: 1566 case Builtin::BIpowf: 1567 case Builtin::BIpowl: { 1568 // Transform a call to pow* into a @llvm.pow.* intrinsic call. 1569 if (!FD->hasAttr<ConstAttr>()) 1570 break; 1571 Value *Base = EmitScalarExpr(E->getArg(0)); 1572 Value *Exponent = EmitScalarExpr(E->getArg(1)); 1573 llvm::Type *ArgType = Base->getType(); 1574 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); 1575 return RValue::get(Builder.CreateCall(F, {Base, Exponent})); 1576 } 1577 1578 case Builtin::BIfma: 1579 case Builtin::BIfmaf: 1580 case Builtin::BIfmal: 1581 case Builtin::BI__builtin_fma: 1582 case Builtin::BI__builtin_fmaf: 1583 case Builtin::BI__builtin_fmal: { 1584 // Rewrite fma to intrinsic. 1585 Value *FirstArg = EmitScalarExpr(E->getArg(0)); 1586 llvm::Type *ArgType = FirstArg->getType(); 1587 Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); 1588 return RValue::get( 1589 Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), 1590 EmitScalarExpr(E->getArg(2))})); 1591 } 1592 1593 case Builtin::BI__builtin_signbit: 1594 case Builtin::BI__builtin_signbitf: 1595 case Builtin::BI__builtin_signbitl: { 1596 return RValue::get( 1597 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), 1598 ConvertType(E->getType()))); 1599 } 1600 case Builtin::BI__builtin_annotation: { 1601 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); 1602 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, 1603 AnnVal->getType()); 1604 1605 // Get the annotation string, go through casts. Sema requires this to be a 1606 // non-wide string literal, potentially casted, so the cast<> is safe. 1607 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); 1608 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); 1609 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); 1610 } 1611 case Builtin::BI__builtin_addcb: 1612 case Builtin::BI__builtin_addcs: 1613 case Builtin::BI__builtin_addc: 1614 case Builtin::BI__builtin_addcl: 1615 case Builtin::BI__builtin_addcll: 1616 case Builtin::BI__builtin_subcb: 1617 case Builtin::BI__builtin_subcs: 1618 case Builtin::BI__builtin_subc: 1619 case Builtin::BI__builtin_subcl: 1620 case Builtin::BI__builtin_subcll: { 1621 1622 // We translate all of these builtins from expressions of the form: 1623 // int x = ..., y = ..., carryin = ..., carryout, result; 1624 // result = __builtin_addc(x, y, carryin, &carryout); 1625 // 1626 // to LLVM IR of the form: 1627 // 1628 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) 1629 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 1630 // %carry1 = extractvalue {i32, i1} %tmp1, 1 1631 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, 1632 // i32 %carryin) 1633 // %result = extractvalue {i32, i1} %tmp2, 0 1634 // %carry2 = extractvalue {i32, i1} %tmp2, 1 1635 // %tmp3 = or i1 %carry1, %carry2 1636 // %tmp4 = zext i1 %tmp3 to i32 1637 // store i32 %tmp4, i32* %carryout 1638 1639 // Scalarize our inputs. 1640 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1641 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1642 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); 1643 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3)); 1644 1645 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. 1646 llvm::Intrinsic::ID IntrinsicId; 1647 switch (BuiltinID) { 1648 default: llvm_unreachable("Unknown multiprecision builtin id."); 1649 case Builtin::BI__builtin_addcb: 1650 case Builtin::BI__builtin_addcs: 1651 case Builtin::BI__builtin_addc: 1652 case Builtin::BI__builtin_addcl: 1653 case Builtin::BI__builtin_addcll: 1654 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1655 break; 1656 case Builtin::BI__builtin_subcb: 1657 case Builtin::BI__builtin_subcs: 1658 case Builtin::BI__builtin_subc: 1659 case Builtin::BI__builtin_subcl: 1660 case Builtin::BI__builtin_subcll: 1661 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1662 break; 1663 } 1664 1665 // Construct our resulting LLVM IR expression. 1666 llvm::Value *Carry1; 1667 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, 1668 X, Y, Carry1); 1669 llvm::Value *Carry2; 1670 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, 1671 Sum1, Carryin, Carry2); 1672 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), 1673 X->getType()); 1674 Builder.CreateStore(CarryOut, CarryOutPtr); 1675 return RValue::get(Sum2); 1676 } 1677 1678 case Builtin::BI__builtin_add_overflow: 1679 case Builtin::BI__builtin_sub_overflow: 1680 case Builtin::BI__builtin_mul_overflow: { 1681 const clang::Expr *LeftArg = E->getArg(0); 1682 const clang::Expr *RightArg = E->getArg(1); 1683 const clang::Expr *ResultArg = E->getArg(2); 1684 1685 clang::QualType ResultQTy = 1686 ResultArg->getType()->castAs<PointerType>()->getPointeeType(); 1687 1688 WidthAndSignedness LeftInfo = 1689 getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType()); 1690 WidthAndSignedness RightInfo = 1691 getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType()); 1692 WidthAndSignedness ResultInfo = 1693 getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy); 1694 WidthAndSignedness EncompassingInfo = 1695 EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo}); 1696 1697 llvm::Type *EncompassingLLVMTy = 1698 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width); 1699 1700 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy); 1701 1702 llvm::Intrinsic::ID IntrinsicId; 1703 switch (BuiltinID) { 1704 default: 1705 llvm_unreachable("Unknown overflow builtin id."); 1706 case Builtin::BI__builtin_add_overflow: 1707 IntrinsicId = EncompassingInfo.Signed 1708 ? llvm::Intrinsic::sadd_with_overflow 1709 : llvm::Intrinsic::uadd_with_overflow; 1710 break; 1711 case Builtin::BI__builtin_sub_overflow: 1712 IntrinsicId = EncompassingInfo.Signed 1713 ? llvm::Intrinsic::ssub_with_overflow 1714 : llvm::Intrinsic::usub_with_overflow; 1715 break; 1716 case Builtin::BI__builtin_mul_overflow: 1717 IntrinsicId = EncompassingInfo.Signed 1718 ? llvm::Intrinsic::smul_with_overflow 1719 : llvm::Intrinsic::umul_with_overflow; 1720 break; 1721 } 1722 1723 llvm::Value *Left = EmitScalarExpr(LeftArg); 1724 llvm::Value *Right = EmitScalarExpr(RightArg); 1725 Address ResultPtr = EmitPointerWithAlignment(ResultArg); 1726 1727 // Extend each operand to the encompassing type. 1728 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed); 1729 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed); 1730 1731 // Perform the operation on the extended values. 1732 llvm::Value *Overflow, *Result; 1733 Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow); 1734 1735 if (EncompassingInfo.Width > ResultInfo.Width) { 1736 // The encompassing type is wider than the result type, so we need to 1737 // truncate it. 1738 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy); 1739 1740 // To see if the truncation caused an overflow, we will extend 1741 // the result and then compare it to the original result. 1742 llvm::Value *ResultTruncExt = Builder.CreateIntCast( 1743 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed); 1744 llvm::Value *TruncationOverflow = 1745 Builder.CreateICmpNE(Result, ResultTruncExt); 1746 1747 Overflow = Builder.CreateOr(Overflow, TruncationOverflow); 1748 Result = ResultTrunc; 1749 } 1750 1751 // Finally, store the result using the pointer. 1752 bool isVolatile = 1753 ResultArg->getType()->getPointeeType().isVolatileQualified(); 1754 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile); 1755 1756 return RValue::get(Overflow); 1757 } 1758 1759 case Builtin::BI__builtin_uadd_overflow: 1760 case Builtin::BI__builtin_uaddl_overflow: 1761 case Builtin::BI__builtin_uaddll_overflow: 1762 case Builtin::BI__builtin_usub_overflow: 1763 case Builtin::BI__builtin_usubl_overflow: 1764 case Builtin::BI__builtin_usubll_overflow: 1765 case Builtin::BI__builtin_umul_overflow: 1766 case Builtin::BI__builtin_umull_overflow: 1767 case Builtin::BI__builtin_umulll_overflow: 1768 case Builtin::BI__builtin_sadd_overflow: 1769 case Builtin::BI__builtin_saddl_overflow: 1770 case Builtin::BI__builtin_saddll_overflow: 1771 case Builtin::BI__builtin_ssub_overflow: 1772 case Builtin::BI__builtin_ssubl_overflow: 1773 case Builtin::BI__builtin_ssubll_overflow: 1774 case Builtin::BI__builtin_smul_overflow: 1775 case Builtin::BI__builtin_smull_overflow: 1776 case Builtin::BI__builtin_smulll_overflow: { 1777 1778 // We translate all of these builtins directly to the relevant llvm IR node. 1779 1780 // Scalarize our inputs. 1781 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 1782 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 1783 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2)); 1784 1785 // Decide which of the overflow intrinsics we are lowering to: 1786 llvm::Intrinsic::ID IntrinsicId; 1787 switch (BuiltinID) { 1788 default: llvm_unreachable("Unknown overflow builtin id."); 1789 case Builtin::BI__builtin_uadd_overflow: 1790 case Builtin::BI__builtin_uaddl_overflow: 1791 case Builtin::BI__builtin_uaddll_overflow: 1792 IntrinsicId = llvm::Intrinsic::uadd_with_overflow; 1793 break; 1794 case Builtin::BI__builtin_usub_overflow: 1795 case Builtin::BI__builtin_usubl_overflow: 1796 case Builtin::BI__builtin_usubll_overflow: 1797 IntrinsicId = llvm::Intrinsic::usub_with_overflow; 1798 break; 1799 case Builtin::BI__builtin_umul_overflow: 1800 case Builtin::BI__builtin_umull_overflow: 1801 case Builtin::BI__builtin_umulll_overflow: 1802 IntrinsicId = llvm::Intrinsic::umul_with_overflow; 1803 break; 1804 case Builtin::BI__builtin_sadd_overflow: 1805 case Builtin::BI__builtin_saddl_overflow: 1806 case Builtin::BI__builtin_saddll_overflow: 1807 IntrinsicId = llvm::Intrinsic::sadd_with_overflow; 1808 break; 1809 case Builtin::BI__builtin_ssub_overflow: 1810 case Builtin::BI__builtin_ssubl_overflow: 1811 case Builtin::BI__builtin_ssubll_overflow: 1812 IntrinsicId = llvm::Intrinsic::ssub_with_overflow; 1813 break; 1814 case Builtin::BI__builtin_smul_overflow: 1815 case Builtin::BI__builtin_smull_overflow: 1816 case Builtin::BI__builtin_smulll_overflow: 1817 IntrinsicId = llvm::Intrinsic::smul_with_overflow; 1818 break; 1819 } 1820 1821 1822 llvm::Value *Carry; 1823 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); 1824 Builder.CreateStore(Sum, SumOutPtr); 1825 1826 return RValue::get(Carry); 1827 } 1828 case Builtin::BI__builtin_addressof: 1829 return RValue::get(EmitLValue(E->getArg(0)).getPointer()); 1830 case Builtin::BI__builtin_operator_new: 1831 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1832 E->getArg(0), false); 1833 case Builtin::BI__builtin_operator_delete: 1834 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), 1835 E->getArg(0), true); 1836 case Builtin::BI__noop: 1837 // __noop always evaluates to an integer literal zero. 1838 return RValue::get(ConstantInt::get(IntTy, 0)); 1839 case Builtin::BI__builtin_call_with_static_chain: { 1840 const CallExpr *Call = cast<CallExpr>(E->getArg(0)); 1841 const Expr *Chain = E->getArg(1); 1842 return EmitCall(Call->getCallee()->getType(), 1843 EmitScalarExpr(Call->getCallee()), Call, ReturnValue, 1844 Call->getCalleeDecl(), EmitScalarExpr(Chain)); 1845 } 1846 case Builtin::BI_InterlockedExchange: 1847 case Builtin::BI_InterlockedExchangePointer: 1848 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); 1849 case Builtin::BI_InterlockedCompareExchangePointer: { 1850 llvm::Type *RTy; 1851 llvm::IntegerType *IntType = 1852 IntegerType::get(getLLVMContext(), 1853 getContext().getTypeSize(E->getType())); 1854 llvm::Type *IntPtrType = IntType->getPointerTo(); 1855 1856 llvm::Value *Destination = 1857 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); 1858 1859 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); 1860 RTy = Exchange->getType(); 1861 Exchange = Builder.CreatePtrToInt(Exchange, IntType); 1862 1863 llvm::Value *Comparand = 1864 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); 1865 1866 auto Result = 1867 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, 1868 AtomicOrdering::SequentiallyConsistent, 1869 AtomicOrdering::SequentiallyConsistent); 1870 Result->setVolatile(true); 1871 1872 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, 1873 0), 1874 RTy)); 1875 } 1876 case Builtin::BI_InterlockedCompareExchange: { 1877 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( 1878 EmitScalarExpr(E->getArg(0)), 1879 EmitScalarExpr(E->getArg(2)), 1880 EmitScalarExpr(E->getArg(1)), 1881 AtomicOrdering::SequentiallyConsistent, 1882 AtomicOrdering::SequentiallyConsistent); 1883 CXI->setVolatile(true); 1884 return RValue::get(Builder.CreateExtractValue(CXI, 0)); 1885 } 1886 case Builtin::BI_InterlockedIncrement: { 1887 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1888 AtomicRMWInst::Add, 1889 EmitScalarExpr(E->getArg(0)), 1890 ConstantInt::get(Int32Ty, 1), 1891 llvm::AtomicOrdering::SequentiallyConsistent); 1892 RMWI->setVolatile(true); 1893 return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1))); 1894 } 1895 case Builtin::BI_InterlockedDecrement: { 1896 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1897 AtomicRMWInst::Sub, 1898 EmitScalarExpr(E->getArg(0)), 1899 ConstantInt::get(Int32Ty, 1), 1900 llvm::AtomicOrdering::SequentiallyConsistent); 1901 RMWI->setVolatile(true); 1902 return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1))); 1903 } 1904 case Builtin::BI_InterlockedExchangeAdd: { 1905 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( 1906 AtomicRMWInst::Add, 1907 EmitScalarExpr(E->getArg(0)), 1908 EmitScalarExpr(E->getArg(1)), 1909 llvm::AtomicOrdering::SequentiallyConsistent); 1910 RMWI->setVolatile(true); 1911 return RValue::get(RMWI); 1912 } 1913 case Builtin::BI__readfsdword: { 1914 Value *IntToPtr = 1915 Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), 1916 llvm::PointerType::get(CGM.Int32Ty, 257)); 1917 LoadInst *Load = 1918 Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true); 1919 return RValue::get(Load); 1920 } 1921 1922 case Builtin::BI__exception_code: 1923 case Builtin::BI_exception_code: 1924 return RValue::get(EmitSEHExceptionCode()); 1925 case Builtin::BI__exception_info: 1926 case Builtin::BI_exception_info: 1927 return RValue::get(EmitSEHExceptionInfo()); 1928 case Builtin::BI__abnormal_termination: 1929 case Builtin::BI_abnormal_termination: 1930 return RValue::get(EmitSEHAbnormalTermination()); 1931 case Builtin::BI_setjmpex: { 1932 if (getTarget().getTriple().isOSMSVCRT()) { 1933 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1934 llvm::AttributeSet ReturnsTwiceAttr = 1935 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1936 llvm::Attribute::ReturnsTwice); 1937 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( 1938 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1939 "_setjmpex", ReturnsTwiceAttr); 1940 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1941 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1942 llvm::Value *FrameAddr = 1943 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1944 ConstantInt::get(Int32Ty, 0)); 1945 llvm::Value *Args[] = {Buf, FrameAddr}; 1946 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); 1947 CS.setAttributes(ReturnsTwiceAttr); 1948 return RValue::get(CS.getInstruction()); 1949 } 1950 break; 1951 } 1952 case Builtin::BI_setjmp: { 1953 if (getTarget().getTriple().isOSMSVCRT()) { 1954 llvm::AttributeSet ReturnsTwiceAttr = 1955 AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, 1956 llvm::Attribute::ReturnsTwice); 1957 llvm::Value *Buf = Builder.CreateBitOrPointerCast( 1958 EmitScalarExpr(E->getArg(0)), Int8PtrTy); 1959 llvm::CallSite CS; 1960 if (getTarget().getTriple().getArch() == llvm::Triple::x86) { 1961 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; 1962 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( 1963 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), 1964 "_setjmp3", ReturnsTwiceAttr); 1965 llvm::Value *Count = ConstantInt::get(IntTy, 0); 1966 llvm::Value *Args[] = {Buf, Count}; 1967 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); 1968 } else { 1969 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; 1970 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( 1971 llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), 1972 "_setjmp", ReturnsTwiceAttr); 1973 llvm::Value *FrameAddr = 1974 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), 1975 ConstantInt::get(Int32Ty, 0)); 1976 llvm::Value *Args[] = {Buf, FrameAddr}; 1977 CS = EmitRuntimeCallOrInvoke(SetJmp, Args); 1978 } 1979 CS.setAttributes(ReturnsTwiceAttr); 1980 return RValue::get(CS.getInstruction()); 1981 } 1982 break; 1983 } 1984 1985 case Builtin::BI__GetExceptionInfo: { 1986 if (llvm::GlobalVariable *GV = 1987 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) 1988 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); 1989 break; 1990 } 1991 1992 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions 1993 case Builtin::BIread_pipe: 1994 case Builtin::BIwrite_pipe: { 1995 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 1996 *Arg1 = EmitScalarExpr(E->getArg(1)); 1997 1998 // Type of the generic packet parameter. 1999 unsigned GenericAS = 2000 getContext().getTargetAddressSpace(LangAS::opencl_generic); 2001 llvm::Type *I8PTy = llvm::PointerType::get( 2002 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS); 2003 2004 // Testing which overloaded version we should generate the call for. 2005 if (2U == E->getNumArgs()) { 2006 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2" 2007 : "__write_pipe_2"; 2008 // Creating a generic function type to be able to call with any builtin or 2009 // user defined type. 2010 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy}; 2011 llvm::FunctionType *FTy = llvm::FunctionType::get( 2012 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2013 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy); 2014 return RValue::get(Builder.CreateCall( 2015 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, BCast})); 2016 } else { 2017 assert(4 == E->getNumArgs() && 2018 "Illegal number of parameters to pipe function"); 2019 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4" 2020 : "__write_pipe_4"; 2021 2022 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy}; 2023 Value *Arg2 = EmitScalarExpr(E->getArg(2)), 2024 *Arg3 = EmitScalarExpr(E->getArg(3)); 2025 llvm::FunctionType *FTy = llvm::FunctionType::get( 2026 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2027 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy); 2028 // We know the third argument is an integer type, but we may need to cast 2029 // it to i32. 2030 if (Arg2->getType() != Int32Ty) 2031 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty); 2032 return RValue::get(Builder.CreateCall( 2033 CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1, Arg2, BCast})); 2034 } 2035 } 2036 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write 2037 // functions 2038 case Builtin::BIreserve_read_pipe: 2039 case Builtin::BIreserve_write_pipe: 2040 case Builtin::BIwork_group_reserve_read_pipe: 2041 case Builtin::BIwork_group_reserve_write_pipe: 2042 case Builtin::BIsub_group_reserve_read_pipe: 2043 case Builtin::BIsub_group_reserve_write_pipe: { 2044 // Composing the mangled name for the function. 2045 const char *Name; 2046 if (BuiltinID == Builtin::BIreserve_read_pipe) 2047 Name = "__reserve_read_pipe"; 2048 else if (BuiltinID == Builtin::BIreserve_write_pipe) 2049 Name = "__reserve_write_pipe"; 2050 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe) 2051 Name = "__work_group_reserve_read_pipe"; 2052 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe) 2053 Name = "__work_group_reserve_write_pipe"; 2054 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe) 2055 Name = "__sub_group_reserve_read_pipe"; 2056 else 2057 Name = "__sub_group_reserve_write_pipe"; 2058 2059 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2060 *Arg1 = EmitScalarExpr(E->getArg(1)); 2061 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy); 2062 2063 // Building the generic function prototype. 2064 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty}; 2065 llvm::FunctionType *FTy = llvm::FunctionType::get( 2066 ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2067 // We know the second argument is an integer type, but we may need to cast 2068 // it to i32. 2069 if (Arg1->getType() != Int32Ty) 2070 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty); 2071 return RValue::get( 2072 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2073 } 2074 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe commit read and write 2075 // functions 2076 case Builtin::BIcommit_read_pipe: 2077 case Builtin::BIcommit_write_pipe: 2078 case Builtin::BIwork_group_commit_read_pipe: 2079 case Builtin::BIwork_group_commit_write_pipe: 2080 case Builtin::BIsub_group_commit_read_pipe: 2081 case Builtin::BIsub_group_commit_write_pipe: { 2082 const char *Name; 2083 if (BuiltinID == Builtin::BIcommit_read_pipe) 2084 Name = "__commit_read_pipe"; 2085 else if (BuiltinID == Builtin::BIcommit_write_pipe) 2086 Name = "__commit_write_pipe"; 2087 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe) 2088 Name = "__work_group_commit_read_pipe"; 2089 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe) 2090 Name = "__work_group_commit_write_pipe"; 2091 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe) 2092 Name = "__sub_group_commit_read_pipe"; 2093 else 2094 Name = "__sub_group_commit_write_pipe"; 2095 2096 Value *Arg0 = EmitScalarExpr(E->getArg(0)), 2097 *Arg1 = EmitScalarExpr(E->getArg(1)); 2098 2099 // Building the generic function prototype. 2100 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType()}; 2101 llvm::FunctionType *FTy = 2102 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), 2103 llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2104 2105 return RValue::get( 2106 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0, Arg1})); 2107 } 2108 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions 2109 case Builtin::BIget_pipe_num_packets: 2110 case Builtin::BIget_pipe_max_packets: { 2111 const char *Name; 2112 if (BuiltinID == Builtin::BIget_pipe_num_packets) 2113 Name = "__get_pipe_num_packets"; 2114 else 2115 Name = "__get_pipe_max_packets"; 2116 2117 // Building the generic function prototype. 2118 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 2119 llvm::Type *ArgTys[] = {Arg0->getType()}; 2120 llvm::FunctionType *FTy = llvm::FunctionType::get( 2121 Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false); 2122 2123 return RValue::get( 2124 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), {Arg0})); 2125 } 2126 2127 // OpenCL v2.0 s6.13.9 - Address space qualifier functions. 2128 case Builtin::BIto_global: 2129 case Builtin::BIto_local: 2130 case Builtin::BIto_private: { 2131 auto Arg0 = EmitScalarExpr(E->getArg(0)); 2132 auto NewArgT = llvm::PointerType::get(Int8Ty, 2133 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic)); 2134 auto NewRetT = llvm::PointerType::get(Int8Ty, 2135 CGM.getContext().getTargetAddressSpace( 2136 E->getType()->getPointeeType().getAddressSpace())); 2137 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false); 2138 llvm::Value *NewArg; 2139 if (Arg0->getType()->getPointerAddressSpace() != 2140 NewArgT->getPointerAddressSpace()) 2141 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT); 2142 else 2143 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT); 2144 auto NewCall = Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, 2145 E->getDirectCallee()->getName()), {NewArg}); 2146 return RValue::get(Builder.CreateBitOrPointerCast(NewCall, 2147 ConvertType(E->getType()))); 2148 } 2149 2150 case Builtin::BIprintf: 2151 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) 2152 return EmitCUDADevicePrintfCallExpr(E, ReturnValue); 2153 break; 2154 case Builtin::BI__builtin_canonicalize: 2155 case Builtin::BI__builtin_canonicalizef: 2156 case Builtin::BI__builtin_canonicalizel: 2157 return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize)); 2158 } 2159 2160 // If this is an alias for a lib function (e.g. __builtin_sin), emit 2161 // the call using the normal call path, but using the unmangled 2162 // version of the function name. 2163 if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) 2164 return emitLibraryCall(*this, FD, E, 2165 CGM.getBuiltinLibFunction(FD, BuiltinID)); 2166 2167 // If this is a predefined lib function (e.g. malloc), emit the call 2168 // using exactly the normal call path. 2169 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) 2170 return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); 2171 2172 // Check that a call to a target specific builtin has the correct target 2173 // features. 2174 // This is down here to avoid non-target specific builtins, however, if 2175 // generic builtins start to require generic target features then we 2176 // can move this up to the beginning of the function. 2177 checkTargetFeatures(E, FD); 2178 2179 // See if we have a target specific intrinsic. 2180 const char *Name = getContext().BuiltinInfo.getName(BuiltinID); 2181 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; 2182 if (const char *Prefix = 2183 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { 2184 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); 2185 // NOTE we dont need to perform a compatibility flag check here since the 2186 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the 2187 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. 2188 if (IntrinsicID == Intrinsic::not_intrinsic) 2189 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); 2190 } 2191 2192 if (IntrinsicID != Intrinsic::not_intrinsic) { 2193 SmallVector<Value*, 16> Args; 2194 2195 // Find out if any arguments are required to be integer constant 2196 // expressions. 2197 unsigned ICEArguments = 0; 2198 ASTContext::GetBuiltinTypeError Error; 2199 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 2200 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 2201 2202 Function *F = CGM.getIntrinsic(IntrinsicID); 2203 llvm::FunctionType *FTy = F->getFunctionType(); 2204 2205 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { 2206 Value *ArgValue; 2207 // If this is a normal argument, just emit it as a scalar. 2208 if ((ICEArguments & (1 << i)) == 0) { 2209 ArgValue = EmitScalarExpr(E->getArg(i)); 2210 } else { 2211 // If this is required to be a constant, constant fold it so that we 2212 // know that the generated intrinsic gets a ConstantInt. 2213 llvm::APSInt Result; 2214 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); 2215 assert(IsConst && "Constant arg isn't actually constant?"); 2216 (void)IsConst; 2217 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); 2218 } 2219 2220 // If the intrinsic arg type is different from the builtin arg type 2221 // we need to do a bit cast. 2222 llvm::Type *PTy = FTy->getParamType(i); 2223 if (PTy != ArgValue->getType()) { 2224 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && 2225 "Must be able to losslessly bit cast to param"); 2226 ArgValue = Builder.CreateBitCast(ArgValue, PTy); 2227 } 2228 2229 Args.push_back(ArgValue); 2230 } 2231 2232 Value *V = Builder.CreateCall(F, Args); 2233 QualType BuiltinRetType = E->getType(); 2234 2235 llvm::Type *RetTy = VoidTy; 2236 if (!BuiltinRetType->isVoidType()) 2237 RetTy = ConvertType(BuiltinRetType); 2238 2239 if (RetTy != V->getType()) { 2240 assert(V->getType()->canLosslesslyBitCastTo(RetTy) && 2241 "Must be able to losslessly bit cast result type"); 2242 V = Builder.CreateBitCast(V, RetTy); 2243 } 2244 2245 return RValue::get(V); 2246 } 2247 2248 // See if we have a target specific builtin that needs to be lowered. 2249 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) 2250 return RValue::get(V); 2251 2252 ErrorUnsupported(E, "builtin function"); 2253 2254 // Unknown builtin, for now just dump it out and return undef. 2255 return GetUndefRValue(E->getType()); 2256 } 2257 2258 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, 2259 unsigned BuiltinID, const CallExpr *E, 2260 llvm::Triple::ArchType Arch) { 2261 switch (Arch) { 2262 case llvm::Triple::arm: 2263 case llvm::Triple::armeb: 2264 case llvm::Triple::thumb: 2265 case llvm::Triple::thumbeb: 2266 return CGF->EmitARMBuiltinExpr(BuiltinID, E); 2267 case llvm::Triple::aarch64: 2268 case llvm::Triple::aarch64_be: 2269 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E); 2270 case llvm::Triple::x86: 2271 case llvm::Triple::x86_64: 2272 return CGF->EmitX86BuiltinExpr(BuiltinID, E); 2273 case llvm::Triple::ppc: 2274 case llvm::Triple::ppc64: 2275 case llvm::Triple::ppc64le: 2276 return CGF->EmitPPCBuiltinExpr(BuiltinID, E); 2277 case llvm::Triple::r600: 2278 case llvm::Triple::amdgcn: 2279 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E); 2280 case llvm::Triple::systemz: 2281 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E); 2282 case llvm::Triple::nvptx: 2283 case llvm::Triple::nvptx64: 2284 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E); 2285 case llvm::Triple::wasm32: 2286 case llvm::Triple::wasm64: 2287 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E); 2288 default: 2289 return nullptr; 2290 } 2291 } 2292 2293 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, 2294 const CallExpr *E) { 2295 if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) { 2296 assert(getContext().getAuxTargetInfo() && "Missing aux target info"); 2297 return EmitTargetArchBuiltinExpr( 2298 this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E, 2299 getContext().getAuxTargetInfo()->getTriple().getArch()); 2300 } 2301 2302 return EmitTargetArchBuiltinExpr(this, BuiltinID, E, 2303 getTarget().getTriple().getArch()); 2304 } 2305 2306 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, 2307 NeonTypeFlags TypeFlags, 2308 bool V1Ty=false) { 2309 int IsQuad = TypeFlags.isQuad(); 2310 switch (TypeFlags.getEltType()) { 2311 case NeonTypeFlags::Int8: 2312 case NeonTypeFlags::Poly8: 2313 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); 2314 case NeonTypeFlags::Int16: 2315 case NeonTypeFlags::Poly16: 2316 case NeonTypeFlags::Float16: 2317 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); 2318 case NeonTypeFlags::Int32: 2319 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); 2320 case NeonTypeFlags::Int64: 2321 case NeonTypeFlags::Poly64: 2322 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); 2323 case NeonTypeFlags::Poly128: 2324 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. 2325 // There is a lot of i128 and f128 API missing. 2326 // so we use v16i8 to represent poly128 and get pattern matched. 2327 return llvm::VectorType::get(CGF->Int8Ty, 16); 2328 case NeonTypeFlags::Float32: 2329 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); 2330 case NeonTypeFlags::Float64: 2331 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); 2332 } 2333 llvm_unreachable("Unknown vector element type!"); 2334 } 2335 2336 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF, 2337 NeonTypeFlags IntTypeFlags) { 2338 int IsQuad = IntTypeFlags.isQuad(); 2339 switch (IntTypeFlags.getEltType()) { 2340 case NeonTypeFlags::Int32: 2341 return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad)); 2342 case NeonTypeFlags::Int64: 2343 return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad)); 2344 default: 2345 llvm_unreachable("Type can't be converted to floating-point!"); 2346 } 2347 } 2348 2349 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { 2350 unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); 2351 Value* SV = llvm::ConstantVector::getSplat(nElts, C); 2352 return Builder.CreateShuffleVector(V, V, SV, "lane"); 2353 } 2354 2355 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, 2356 const char *name, 2357 unsigned shift, bool rightshift) { 2358 unsigned j = 0; 2359 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 2360 ai != ae; ++ai, ++j) 2361 if (shift > 0 && shift == j) 2362 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); 2363 else 2364 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); 2365 2366 return Builder.CreateCall(F, Ops, name); 2367 } 2368 2369 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 2370 bool neg) { 2371 int SV = cast<ConstantInt>(V)->getSExtValue(); 2372 return ConstantInt::get(Ty, neg ? -SV : SV); 2373 } 2374 2375 // \brief Right-shift a vector by a constant. 2376 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, 2377 llvm::Type *Ty, bool usgn, 2378 const char *name) { 2379 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 2380 2381 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); 2382 int EltSize = VTy->getScalarSizeInBits(); 2383 2384 Vec = Builder.CreateBitCast(Vec, Ty); 2385 2386 // lshr/ashr are undefined when the shift amount is equal to the vector 2387 // element size. 2388 if (ShiftAmt == EltSize) { 2389 if (usgn) { 2390 // Right-shifting an unsigned value by its size yields 0. 2391 return llvm::ConstantAggregateZero::get(VTy); 2392 } else { 2393 // Right-shifting a signed value by its size is equivalent 2394 // to a shift of size-1. 2395 --ShiftAmt; 2396 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); 2397 } 2398 } 2399 2400 Shift = EmitNeonShiftVector(Shift, Ty, false); 2401 if (usgn) 2402 return Builder.CreateLShr(Vec, Shift, name); 2403 else 2404 return Builder.CreateAShr(Vec, Shift, name); 2405 } 2406 2407 enum { 2408 AddRetType = (1 << 0), 2409 Add1ArgType = (1 << 1), 2410 Add2ArgTypes = (1 << 2), 2411 2412 VectorizeRetType = (1 << 3), 2413 VectorizeArgTypes = (1 << 4), 2414 2415 InventFloatType = (1 << 5), 2416 UnsignedAlts = (1 << 6), 2417 2418 Use64BitVectors = (1 << 7), 2419 Use128BitVectors = (1 << 8), 2420 2421 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, 2422 VectorRet = AddRetType | VectorizeRetType, 2423 VectorRetGetArgs01 = 2424 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, 2425 FpCmpzModifiers = 2426 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType 2427 }; 2428 2429 namespace { 2430 struct NeonIntrinsicInfo { 2431 const char *NameHint; 2432 unsigned BuiltinID; 2433 unsigned LLVMIntrinsic; 2434 unsigned AltLLVMIntrinsic; 2435 unsigned TypeModifier; 2436 2437 bool operator<(unsigned RHSBuiltinID) const { 2438 return BuiltinID < RHSBuiltinID; 2439 } 2440 bool operator<(const NeonIntrinsicInfo &TE) const { 2441 return BuiltinID < TE.BuiltinID; 2442 } 2443 }; 2444 } // end anonymous namespace 2445 2446 #define NEONMAP0(NameBase) \ 2447 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 2448 2449 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 2450 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2451 Intrinsic::LLVMIntrinsic, 0, TypeModifier } 2452 2453 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 2454 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 2455 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 2456 TypeModifier } 2457 2458 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { 2459 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2460 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), 2461 NEONMAP1(vabs_v, arm_neon_vabs, 0), 2462 NEONMAP1(vabsq_v, arm_neon_vabs, 0), 2463 NEONMAP0(vaddhn_v), 2464 NEONMAP1(vaesdq_v, arm_neon_aesd, 0), 2465 NEONMAP1(vaeseq_v, arm_neon_aese, 0), 2466 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), 2467 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), 2468 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), 2469 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), 2470 NEONMAP1(vcage_v, arm_neon_vacge, 0), 2471 NEONMAP1(vcageq_v, arm_neon_vacge, 0), 2472 NEONMAP1(vcagt_v, arm_neon_vacgt, 0), 2473 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), 2474 NEONMAP1(vcale_v, arm_neon_vacge, 0), 2475 NEONMAP1(vcaleq_v, arm_neon_vacge, 0), 2476 NEONMAP1(vcalt_v, arm_neon_vacgt, 0), 2477 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), 2478 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), 2479 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), 2480 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2481 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2482 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2483 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2484 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0), 2485 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), 2486 NEONMAP0(vcvt_f32_v), 2487 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2488 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2489 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2490 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2491 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2492 NEONMAP0(vcvt_s32_v), 2493 NEONMAP0(vcvt_s64_v), 2494 NEONMAP0(vcvt_u32_v), 2495 NEONMAP0(vcvt_u64_v), 2496 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), 2497 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), 2498 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), 2499 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), 2500 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), 2501 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), 2502 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), 2503 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), 2504 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), 2505 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), 2506 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), 2507 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), 2508 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), 2509 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), 2510 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), 2511 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), 2512 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), 2513 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), 2514 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), 2515 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), 2516 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), 2517 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), 2518 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), 2519 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), 2520 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), 2521 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), 2522 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), 2523 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), 2524 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), 2525 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), 2526 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), 2527 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), 2528 NEONMAP0(vcvtq_f32_v), 2529 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), 2530 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), 2531 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), 2532 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), 2533 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), 2534 NEONMAP0(vcvtq_s32_v), 2535 NEONMAP0(vcvtq_s64_v), 2536 NEONMAP0(vcvtq_u32_v), 2537 NEONMAP0(vcvtq_u64_v), 2538 NEONMAP0(vext_v), 2539 NEONMAP0(vextq_v), 2540 NEONMAP0(vfma_v), 2541 NEONMAP0(vfmaq_v), 2542 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2543 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), 2544 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2545 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), 2546 NEONMAP0(vld1_dup_v), 2547 NEONMAP1(vld1_v, arm_neon_vld1, 0), 2548 NEONMAP0(vld1q_dup_v), 2549 NEONMAP1(vld1q_v, arm_neon_vld1, 0), 2550 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), 2551 NEONMAP1(vld2_v, arm_neon_vld2, 0), 2552 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), 2553 NEONMAP1(vld2q_v, arm_neon_vld2, 0), 2554 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), 2555 NEONMAP1(vld3_v, arm_neon_vld3, 0), 2556 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), 2557 NEONMAP1(vld3q_v, arm_neon_vld3, 0), 2558 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), 2559 NEONMAP1(vld4_v, arm_neon_vld4, 0), 2560 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), 2561 NEONMAP1(vld4q_v, arm_neon_vld4, 0), 2562 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2563 NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), 2564 NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), 2565 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), 2566 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2567 NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), 2568 NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), 2569 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), 2570 NEONMAP0(vmovl_v), 2571 NEONMAP0(vmovn_v), 2572 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), 2573 NEONMAP0(vmull_v), 2574 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), 2575 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2576 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), 2577 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), 2578 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2579 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), 2580 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), 2581 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), 2582 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), 2583 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), 2584 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), 2585 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2586 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), 2587 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), 2588 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), 2589 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), 2590 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), 2591 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), 2592 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), 2593 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), 2594 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), 2595 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), 2596 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), 2597 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), 2598 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2599 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), 2600 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2601 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2602 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), 2603 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), 2604 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), 2605 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), 2606 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2607 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), 2608 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), 2609 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2610 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), 2611 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), 2612 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), 2613 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2614 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), 2615 NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), 2616 NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), 2617 NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), 2618 NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), 2619 NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), 2620 NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), 2621 NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), 2622 NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), 2623 NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), 2624 NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), 2625 NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), 2626 NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), 2627 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2628 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), 2629 NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2630 NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), 2631 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2632 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), 2633 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), 2634 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), 2635 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), 2636 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), 2637 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), 2638 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), 2639 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), 2640 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), 2641 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), 2642 NEONMAP0(vshl_n_v), 2643 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2644 NEONMAP0(vshll_n_v), 2645 NEONMAP0(vshlq_n_v), 2646 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), 2647 NEONMAP0(vshr_n_v), 2648 NEONMAP0(vshrn_n_v), 2649 NEONMAP0(vshrq_n_v), 2650 NEONMAP1(vst1_v, arm_neon_vst1, 0), 2651 NEONMAP1(vst1q_v, arm_neon_vst1, 0), 2652 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), 2653 NEONMAP1(vst2_v, arm_neon_vst2, 0), 2654 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), 2655 NEONMAP1(vst2q_v, arm_neon_vst2, 0), 2656 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), 2657 NEONMAP1(vst3_v, arm_neon_vst3, 0), 2658 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), 2659 NEONMAP1(vst3q_v, arm_neon_vst3, 0), 2660 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), 2661 NEONMAP1(vst4_v, arm_neon_vst4, 0), 2662 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), 2663 NEONMAP1(vst4q_v, arm_neon_vst4, 0), 2664 NEONMAP0(vsubhn_v), 2665 NEONMAP0(vtrn_v), 2666 NEONMAP0(vtrnq_v), 2667 NEONMAP0(vtst_v), 2668 NEONMAP0(vtstq_v), 2669 NEONMAP0(vuzp_v), 2670 NEONMAP0(vuzpq_v), 2671 NEONMAP0(vzip_v), 2672 NEONMAP0(vzipq_v) 2673 }; 2674 2675 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { 2676 NEONMAP1(vabs_v, aarch64_neon_abs, 0), 2677 NEONMAP1(vabsq_v, aarch64_neon_abs, 0), 2678 NEONMAP0(vaddhn_v), 2679 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), 2680 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), 2681 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), 2682 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), 2683 NEONMAP1(vcage_v, aarch64_neon_facge, 0), 2684 NEONMAP1(vcageq_v, aarch64_neon_facge, 0), 2685 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), 2686 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), 2687 NEONMAP1(vcale_v, aarch64_neon_facge, 0), 2688 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), 2689 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), 2690 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), 2691 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), 2692 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), 2693 NEONMAP1(vclz_v, ctlz, Add1ArgType), 2694 NEONMAP1(vclzq_v, ctlz, Add1ArgType), 2695 NEONMAP1(vcnt_v, ctpop, Add1ArgType), 2696 NEONMAP1(vcntq_v, ctpop, Add1ArgType), 2697 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0), 2698 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), 2699 NEONMAP0(vcvt_f32_v), 2700 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2701 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2702 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2703 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2704 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2705 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2706 NEONMAP0(vcvtq_f32_v), 2707 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2708 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), 2709 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), 2710 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), 2711 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), 2712 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), 2713 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), 2714 NEONMAP0(vext_v), 2715 NEONMAP0(vextq_v), 2716 NEONMAP0(vfma_v), 2717 NEONMAP0(vfmaq_v), 2718 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2719 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), 2720 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2721 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), 2722 NEONMAP0(vmovl_v), 2723 NEONMAP0(vmovn_v), 2724 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), 2725 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), 2726 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), 2727 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2728 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), 2729 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), 2730 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), 2731 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), 2732 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2733 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), 2734 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), 2735 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), 2736 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), 2737 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), 2738 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), 2739 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), 2740 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), 2741 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), 2742 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), 2743 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), 2744 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), 2745 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2746 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), 2747 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), 2748 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2749 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), 2750 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), 2751 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), 2752 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), 2753 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2754 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), 2755 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), 2756 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2757 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), 2758 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), 2759 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), 2760 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2761 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), 2762 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2763 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), 2764 NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2765 NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), 2766 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2767 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), 2768 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), 2769 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), 2770 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), 2771 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), 2772 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), 2773 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), 2774 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), 2775 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), 2776 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), 2777 NEONMAP0(vshl_n_v), 2778 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2779 NEONMAP0(vshll_n_v), 2780 NEONMAP0(vshlq_n_v), 2781 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), 2782 NEONMAP0(vshr_n_v), 2783 NEONMAP0(vshrn_n_v), 2784 NEONMAP0(vshrq_n_v), 2785 NEONMAP0(vsubhn_v), 2786 NEONMAP0(vtst_v), 2787 NEONMAP0(vtstq_v), 2788 }; 2789 2790 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { 2791 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), 2792 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), 2793 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), 2794 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2795 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2796 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), 2797 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), 2798 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2799 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2800 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2801 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), 2802 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), 2803 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), 2804 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), 2805 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2806 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2807 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2808 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2809 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2810 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2811 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), 2812 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), 2813 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), 2814 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), 2815 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2816 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2817 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), 2818 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), 2819 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2820 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2821 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2822 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2823 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2824 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2825 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), 2826 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), 2827 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2828 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2829 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), 2830 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), 2831 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2832 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2833 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), 2834 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), 2835 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), 2836 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), 2837 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), 2838 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), 2839 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), 2840 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2841 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2842 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2843 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2844 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2845 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2846 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2847 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2848 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), 2849 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), 2850 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2851 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2852 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2853 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2854 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2855 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2856 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2857 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2858 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), 2859 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), 2860 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), 2861 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), 2862 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), 2863 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2864 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), 2865 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2866 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), 2867 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2868 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), 2869 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2870 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), 2871 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), 2872 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), 2873 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2874 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), 2875 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), 2876 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), 2877 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2878 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2879 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), 2880 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), 2881 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), 2882 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), 2883 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), 2884 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), 2885 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), 2886 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), 2887 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), 2888 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), 2889 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), 2890 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), 2891 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2892 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2893 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), 2894 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), 2895 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), 2896 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2897 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), 2898 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2899 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), 2900 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), 2901 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), 2902 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), 2903 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), 2904 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2905 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2906 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), 2907 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), 2908 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), 2909 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), 2910 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), 2911 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), 2912 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), 2913 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), 2914 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2915 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2916 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), 2917 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), 2918 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), 2919 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2920 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), 2921 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2922 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2923 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2924 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2925 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), 2926 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), 2927 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2928 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2929 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), 2930 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), 2931 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), 2932 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), 2933 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), 2934 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), 2935 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2936 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), 2937 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), 2938 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), 2939 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), 2940 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2941 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2942 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), 2943 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), 2944 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), 2945 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2946 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), 2947 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2948 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2949 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), 2950 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), 2951 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), 2952 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), 2953 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), 2954 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), 2955 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), 2956 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), 2957 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), 2958 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), 2959 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), 2960 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), 2961 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), 2962 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), 2963 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), 2964 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), 2965 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), 2966 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), 2967 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), 2968 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), 2969 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), 2970 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), 2971 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), 2972 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), 2973 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2974 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), 2975 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), 2976 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), 2977 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), 2978 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), 2979 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2980 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), 2981 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), 2982 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), 2983 }; 2984 2985 #undef NEONMAP0 2986 #undef NEONMAP1 2987 #undef NEONMAP2 2988 2989 static bool NEONSIMDIntrinsicsProvenSorted = false; 2990 2991 static bool AArch64SIMDIntrinsicsProvenSorted = false; 2992 static bool AArch64SISDIntrinsicsProvenSorted = false; 2993 2994 2995 static const NeonIntrinsicInfo * 2996 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, 2997 unsigned BuiltinID, bool &MapProvenSorted) { 2998 2999 #ifndef NDEBUG 3000 if (!MapProvenSorted) { 3001 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap))); 3002 MapProvenSorted = true; 3003 } 3004 #endif 3005 3006 const NeonIntrinsicInfo *Builtin = 3007 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); 3008 3009 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) 3010 return Builtin; 3011 3012 return nullptr; 3013 } 3014 3015 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, 3016 unsigned Modifier, 3017 llvm::Type *ArgType, 3018 const CallExpr *E) { 3019 int VectorSize = 0; 3020 if (Modifier & Use64BitVectors) 3021 VectorSize = 64; 3022 else if (Modifier & Use128BitVectors) 3023 VectorSize = 128; 3024 3025 // Return type. 3026 SmallVector<llvm::Type *, 3> Tys; 3027 if (Modifier & AddRetType) { 3028 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 3029 if (Modifier & VectorizeRetType) 3030 Ty = llvm::VectorType::get( 3031 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); 3032 3033 Tys.push_back(Ty); 3034 } 3035 3036 // Arguments. 3037 if (Modifier & VectorizeArgTypes) { 3038 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; 3039 ArgType = llvm::VectorType::get(ArgType, Elts); 3040 } 3041 3042 if (Modifier & (Add1ArgType | Add2ArgTypes)) 3043 Tys.push_back(ArgType); 3044 3045 if (Modifier & Add2ArgTypes) 3046 Tys.push_back(ArgType); 3047 3048 if (Modifier & InventFloatType) 3049 Tys.push_back(FloatTy); 3050 3051 return CGM.getIntrinsic(IntrinsicID, Tys); 3052 } 3053 3054 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, 3055 const NeonIntrinsicInfo &SISDInfo, 3056 SmallVectorImpl<Value *> &Ops, 3057 const CallExpr *E) { 3058 unsigned BuiltinID = SISDInfo.BuiltinID; 3059 unsigned int Int = SISDInfo.LLVMIntrinsic; 3060 unsigned Modifier = SISDInfo.TypeModifier; 3061 const char *s = SISDInfo.NameHint; 3062 3063 switch (BuiltinID) { 3064 case NEON::BI__builtin_neon_vcled_s64: 3065 case NEON::BI__builtin_neon_vcled_u64: 3066 case NEON::BI__builtin_neon_vcles_f32: 3067 case NEON::BI__builtin_neon_vcled_f64: 3068 case NEON::BI__builtin_neon_vcltd_s64: 3069 case NEON::BI__builtin_neon_vcltd_u64: 3070 case NEON::BI__builtin_neon_vclts_f32: 3071 case NEON::BI__builtin_neon_vcltd_f64: 3072 case NEON::BI__builtin_neon_vcales_f32: 3073 case NEON::BI__builtin_neon_vcaled_f64: 3074 case NEON::BI__builtin_neon_vcalts_f32: 3075 case NEON::BI__builtin_neon_vcaltd_f64: 3076 // Only one direction of comparisons actually exist, cmle is actually a cmge 3077 // with swapped operands. The table gives us the right intrinsic but we 3078 // still need to do the swap. 3079 std::swap(Ops[0], Ops[1]); 3080 break; 3081 } 3082 3083 assert(Int && "Generic code assumes a valid intrinsic"); 3084 3085 // Determine the type(s) of this overloaded AArch64 intrinsic. 3086 const Expr *Arg = E->getArg(0); 3087 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); 3088 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); 3089 3090 int j = 0; 3091 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); 3092 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); 3093 ai != ae; ++ai, ++j) { 3094 llvm::Type *ArgTy = ai->getType(); 3095 if (Ops[j]->getType()->getPrimitiveSizeInBits() == 3096 ArgTy->getPrimitiveSizeInBits()) 3097 continue; 3098 3099 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); 3100 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate 3101 // it before inserting. 3102 Ops[j] = 3103 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); 3104 Ops[j] = 3105 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); 3106 } 3107 3108 Value *Result = CGF.EmitNeonCall(F, Ops, s); 3109 llvm::Type *ResultType = CGF.ConvertType(E->getType()); 3110 if (ResultType->getPrimitiveSizeInBits() < 3111 Result->getType()->getPrimitiveSizeInBits()) 3112 return CGF.Builder.CreateExtractElement(Result, C0); 3113 3114 return CGF.Builder.CreateBitCast(Result, ResultType, s); 3115 } 3116 3117 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( 3118 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, 3119 const char *NameHint, unsigned Modifier, const CallExpr *E, 3120 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { 3121 // Get the last argument, which specifies the vector type. 3122 llvm::APSInt NeonTypeConst; 3123 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 3124 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) 3125 return nullptr; 3126 3127 // Determine the type of this overloaded NEON intrinsic. 3128 NeonTypeFlags Type(NeonTypeConst.getZExtValue()); 3129 bool Usgn = Type.isUnsigned(); 3130 bool Quad = Type.isQuad(); 3131 3132 llvm::VectorType *VTy = GetNeonType(this, Type); 3133 llvm::Type *Ty = VTy; 3134 if (!Ty) 3135 return nullptr; 3136 3137 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3138 return Builder.getInt32(addr.getAlignment().getQuantity()); 3139 }; 3140 3141 unsigned Int = LLVMIntrinsic; 3142 if ((Modifier & UnsignedAlts) && !Usgn) 3143 Int = AltLLVMIntrinsic; 3144 3145 switch (BuiltinID) { 3146 default: break; 3147 case NEON::BI__builtin_neon_vabs_v: 3148 case NEON::BI__builtin_neon_vabsq_v: 3149 if (VTy->getElementType()->isFloatingPointTy()) 3150 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); 3151 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); 3152 case NEON::BI__builtin_neon_vaddhn_v: { 3153 llvm::VectorType *SrcTy = 3154 llvm::VectorType::getExtendedElementVectorType(VTy); 3155 3156 // %sum = add <4 x i32> %lhs, %rhs 3157 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3158 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3159 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); 3160 3161 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3162 Constant *ShiftAmt = 3163 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3164 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); 3165 3166 // %res = trunc <4 x i32> %high to <4 x i16> 3167 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); 3168 } 3169 case NEON::BI__builtin_neon_vcale_v: 3170 case NEON::BI__builtin_neon_vcaleq_v: 3171 case NEON::BI__builtin_neon_vcalt_v: 3172 case NEON::BI__builtin_neon_vcaltq_v: 3173 std::swap(Ops[0], Ops[1]); 3174 case NEON::BI__builtin_neon_vcage_v: 3175 case NEON::BI__builtin_neon_vcageq_v: 3176 case NEON::BI__builtin_neon_vcagt_v: 3177 case NEON::BI__builtin_neon_vcagtq_v: { 3178 llvm::Type *VecFlt = llvm::VectorType::get( 3179 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, 3180 VTy->getNumElements()); 3181 llvm::Type *Tys[] = { VTy, VecFlt }; 3182 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3183 return EmitNeonCall(F, Ops, NameHint); 3184 } 3185 case NEON::BI__builtin_neon_vclz_v: 3186 case NEON::BI__builtin_neon_vclzq_v: 3187 // We generate target-independent intrinsic, which needs a second argument 3188 // for whether or not clz of zero is undefined; on ARM it isn't. 3189 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); 3190 break; 3191 case NEON::BI__builtin_neon_vcvt_f32_v: 3192 case NEON::BI__builtin_neon_vcvtq_f32_v: 3193 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3194 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); 3195 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 3196 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 3197 case NEON::BI__builtin_neon_vcvt_n_f32_v: 3198 case NEON::BI__builtin_neon_vcvt_n_f64_v: 3199 case NEON::BI__builtin_neon_vcvtq_n_f32_v: 3200 case NEON::BI__builtin_neon_vcvtq_n_f64_v: { 3201 llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty }; 3202 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; 3203 Function *F = CGM.getIntrinsic(Int, Tys); 3204 return EmitNeonCall(F, Ops, "vcvt_n"); 3205 } 3206 case NEON::BI__builtin_neon_vcvt_n_s32_v: 3207 case NEON::BI__builtin_neon_vcvt_n_u32_v: 3208 case NEON::BI__builtin_neon_vcvt_n_s64_v: 3209 case NEON::BI__builtin_neon_vcvt_n_u64_v: 3210 case NEON::BI__builtin_neon_vcvtq_n_s32_v: 3211 case NEON::BI__builtin_neon_vcvtq_n_u32_v: 3212 case NEON::BI__builtin_neon_vcvtq_n_s64_v: 3213 case NEON::BI__builtin_neon_vcvtq_n_u64_v: { 3214 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3215 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3216 return EmitNeonCall(F, Ops, "vcvt_n"); 3217 } 3218 case NEON::BI__builtin_neon_vcvt_s32_v: 3219 case NEON::BI__builtin_neon_vcvt_u32_v: 3220 case NEON::BI__builtin_neon_vcvt_s64_v: 3221 case NEON::BI__builtin_neon_vcvt_u64_v: 3222 case NEON::BI__builtin_neon_vcvtq_s32_v: 3223 case NEON::BI__builtin_neon_vcvtq_u32_v: 3224 case NEON::BI__builtin_neon_vcvtq_s64_v: 3225 case NEON::BI__builtin_neon_vcvtq_u64_v: { 3226 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 3227 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 3228 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); 3229 } 3230 case NEON::BI__builtin_neon_vcvta_s32_v: 3231 case NEON::BI__builtin_neon_vcvta_s64_v: 3232 case NEON::BI__builtin_neon_vcvta_u32_v: 3233 case NEON::BI__builtin_neon_vcvta_u64_v: 3234 case NEON::BI__builtin_neon_vcvtaq_s32_v: 3235 case NEON::BI__builtin_neon_vcvtaq_s64_v: 3236 case NEON::BI__builtin_neon_vcvtaq_u32_v: 3237 case NEON::BI__builtin_neon_vcvtaq_u64_v: 3238 case NEON::BI__builtin_neon_vcvtn_s32_v: 3239 case NEON::BI__builtin_neon_vcvtn_s64_v: 3240 case NEON::BI__builtin_neon_vcvtn_u32_v: 3241 case NEON::BI__builtin_neon_vcvtn_u64_v: 3242 case NEON::BI__builtin_neon_vcvtnq_s32_v: 3243 case NEON::BI__builtin_neon_vcvtnq_s64_v: 3244 case NEON::BI__builtin_neon_vcvtnq_u32_v: 3245 case NEON::BI__builtin_neon_vcvtnq_u64_v: 3246 case NEON::BI__builtin_neon_vcvtp_s32_v: 3247 case NEON::BI__builtin_neon_vcvtp_s64_v: 3248 case NEON::BI__builtin_neon_vcvtp_u32_v: 3249 case NEON::BI__builtin_neon_vcvtp_u64_v: 3250 case NEON::BI__builtin_neon_vcvtpq_s32_v: 3251 case NEON::BI__builtin_neon_vcvtpq_s64_v: 3252 case NEON::BI__builtin_neon_vcvtpq_u32_v: 3253 case NEON::BI__builtin_neon_vcvtpq_u64_v: 3254 case NEON::BI__builtin_neon_vcvtm_s32_v: 3255 case NEON::BI__builtin_neon_vcvtm_s64_v: 3256 case NEON::BI__builtin_neon_vcvtm_u32_v: 3257 case NEON::BI__builtin_neon_vcvtm_u64_v: 3258 case NEON::BI__builtin_neon_vcvtmq_s32_v: 3259 case NEON::BI__builtin_neon_vcvtmq_s64_v: 3260 case NEON::BI__builtin_neon_vcvtmq_u32_v: 3261 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 3262 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 3263 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); 3264 } 3265 case NEON::BI__builtin_neon_vext_v: 3266 case NEON::BI__builtin_neon_vextq_v: { 3267 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); 3268 SmallVector<Constant*, 16> Indices; 3269 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3270 Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); 3271 3272 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3273 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3274 Value *SV = llvm::ConstantVector::get(Indices); 3275 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); 3276 } 3277 case NEON::BI__builtin_neon_vfma_v: 3278 case NEON::BI__builtin_neon_vfmaq_v: { 3279 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 3280 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3281 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3282 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3283 3284 // NEON intrinsic puts accumulator first, unlike the LLVM fma. 3285 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 3286 } 3287 case NEON::BI__builtin_neon_vld1_v: 3288 case NEON::BI__builtin_neon_vld1q_v: { 3289 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3290 Ops.push_back(getAlignmentValue32(PtrOp0)); 3291 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); 3292 } 3293 case NEON::BI__builtin_neon_vld2_v: 3294 case NEON::BI__builtin_neon_vld2q_v: 3295 case NEON::BI__builtin_neon_vld3_v: 3296 case NEON::BI__builtin_neon_vld3q_v: 3297 case NEON::BI__builtin_neon_vld4_v: 3298 case NEON::BI__builtin_neon_vld4q_v: { 3299 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3300 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3301 Value *Align = getAlignmentValue32(PtrOp1); 3302 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); 3303 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3304 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3305 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3306 } 3307 case NEON::BI__builtin_neon_vld1_dup_v: 3308 case NEON::BI__builtin_neon_vld1q_dup_v: { 3309 Value *V = UndefValue::get(Ty); 3310 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 3311 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty); 3312 LoadInst *Ld = Builder.CreateLoad(PtrOp0); 3313 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 3314 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); 3315 return EmitNeonSplat(Ops[0], CI); 3316 } 3317 case NEON::BI__builtin_neon_vld2_lane_v: 3318 case NEON::BI__builtin_neon_vld2q_lane_v: 3319 case NEON::BI__builtin_neon_vld3_lane_v: 3320 case NEON::BI__builtin_neon_vld3q_lane_v: 3321 case NEON::BI__builtin_neon_vld4_lane_v: 3322 case NEON::BI__builtin_neon_vld4q_lane_v: { 3323 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 3324 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); 3325 for (unsigned I = 2; I < Ops.size() - 1; ++I) 3326 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); 3327 Ops.push_back(getAlignmentValue32(PtrOp1)); 3328 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); 3329 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 3330 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3331 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 3332 } 3333 case NEON::BI__builtin_neon_vmovl_v: { 3334 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); 3335 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); 3336 if (Usgn) 3337 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); 3338 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); 3339 } 3340 case NEON::BI__builtin_neon_vmovn_v: { 3341 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3342 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); 3343 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); 3344 } 3345 case NEON::BI__builtin_neon_vmull_v: 3346 // FIXME: the integer vmull operations could be emitted in terms of pure 3347 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of 3348 // hoisting the exts outside loops. Until global ISel comes along that can 3349 // see through such movement this leads to bad CodeGen. So we need an 3350 // intrinsic for now. 3351 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; 3352 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; 3353 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 3354 case NEON::BI__builtin_neon_vpadal_v: 3355 case NEON::BI__builtin_neon_vpadalq_v: { 3356 // The source operand type has twice as many elements of half the size. 3357 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3358 llvm::Type *EltTy = 3359 llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3360 llvm::Type *NarrowTy = 3361 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3362 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3363 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); 3364 } 3365 case NEON::BI__builtin_neon_vpaddl_v: 3366 case NEON::BI__builtin_neon_vpaddlq_v: { 3367 // The source operand type has twice as many elements of half the size. 3368 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); 3369 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); 3370 llvm::Type *NarrowTy = 3371 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); 3372 llvm::Type *Tys[2] = { Ty, NarrowTy }; 3373 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); 3374 } 3375 case NEON::BI__builtin_neon_vqdmlal_v: 3376 case NEON::BI__builtin_neon_vqdmlsl_v: { 3377 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); 3378 Ops[1] = 3379 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal"); 3380 Ops.resize(2); 3381 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); 3382 } 3383 case NEON::BI__builtin_neon_vqshl_n_v: 3384 case NEON::BI__builtin_neon_vqshlq_n_v: 3385 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", 3386 1, false); 3387 case NEON::BI__builtin_neon_vqshlu_n_v: 3388 case NEON::BI__builtin_neon_vqshluq_n_v: 3389 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 3390 1, false); 3391 case NEON::BI__builtin_neon_vrecpe_v: 3392 case NEON::BI__builtin_neon_vrecpeq_v: 3393 case NEON::BI__builtin_neon_vrsqrte_v: 3394 case NEON::BI__builtin_neon_vrsqrteq_v: 3395 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; 3396 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); 3397 3398 case NEON::BI__builtin_neon_vrshr_n_v: 3399 case NEON::BI__builtin_neon_vrshrq_n_v: 3400 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 3401 1, true); 3402 case NEON::BI__builtin_neon_vshl_n_v: 3403 case NEON::BI__builtin_neon_vshlq_n_v: 3404 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); 3405 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], 3406 "vshl_n"); 3407 case NEON::BI__builtin_neon_vshll_n_v: { 3408 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); 3409 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3410 if (Usgn) 3411 Ops[0] = Builder.CreateZExt(Ops[0], VTy); 3412 else 3413 Ops[0] = Builder.CreateSExt(Ops[0], VTy); 3414 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); 3415 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); 3416 } 3417 case NEON::BI__builtin_neon_vshrn_n_v: { 3418 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); 3419 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3420 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); 3421 if (Usgn) 3422 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); 3423 else 3424 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); 3425 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); 3426 } 3427 case NEON::BI__builtin_neon_vshr_n_v: 3428 case NEON::BI__builtin_neon_vshrq_n_v: 3429 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); 3430 case NEON::BI__builtin_neon_vst1_v: 3431 case NEON::BI__builtin_neon_vst1q_v: 3432 case NEON::BI__builtin_neon_vst2_v: 3433 case NEON::BI__builtin_neon_vst2q_v: 3434 case NEON::BI__builtin_neon_vst3_v: 3435 case NEON::BI__builtin_neon_vst3q_v: 3436 case NEON::BI__builtin_neon_vst4_v: 3437 case NEON::BI__builtin_neon_vst4q_v: 3438 case NEON::BI__builtin_neon_vst2_lane_v: 3439 case NEON::BI__builtin_neon_vst2q_lane_v: 3440 case NEON::BI__builtin_neon_vst3_lane_v: 3441 case NEON::BI__builtin_neon_vst3q_lane_v: 3442 case NEON::BI__builtin_neon_vst4_lane_v: 3443 case NEON::BI__builtin_neon_vst4q_lane_v: { 3444 llvm::Type *Tys[] = {Int8PtrTy, Ty}; 3445 Ops.push_back(getAlignmentValue32(PtrOp0)); 3446 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 3447 } 3448 case NEON::BI__builtin_neon_vsubhn_v: { 3449 llvm::VectorType *SrcTy = 3450 llvm::VectorType::getExtendedElementVectorType(VTy); 3451 3452 // %sum = add <4 x i32> %lhs, %rhs 3453 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); 3454 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); 3455 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); 3456 3457 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> 3458 Constant *ShiftAmt = 3459 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2); 3460 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); 3461 3462 // %res = trunc <4 x i32> %high to <4 x i16> 3463 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); 3464 } 3465 case NEON::BI__builtin_neon_vtrn_v: 3466 case NEON::BI__builtin_neon_vtrnq_v: { 3467 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3468 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3469 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3470 Value *SV = nullptr; 3471 3472 for (unsigned vi = 0; vi != 2; ++vi) { 3473 SmallVector<Constant*, 16> Indices; 3474 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3475 Indices.push_back(Builder.getInt32(i+vi)); 3476 Indices.push_back(Builder.getInt32(i+e+vi)); 3477 } 3478 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3479 SV = llvm::ConstantVector::get(Indices); 3480 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 3481 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3482 } 3483 return SV; 3484 } 3485 case NEON::BI__builtin_neon_vtst_v: 3486 case NEON::BI__builtin_neon_vtstq_v: { 3487 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 3488 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3489 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 3490 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 3491 ConstantAggregateZero::get(Ty)); 3492 return Builder.CreateSExt(Ops[0], Ty, "vtst"); 3493 } 3494 case NEON::BI__builtin_neon_vuzp_v: 3495 case NEON::BI__builtin_neon_vuzpq_v: { 3496 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3497 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3498 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3499 Value *SV = nullptr; 3500 3501 for (unsigned vi = 0; vi != 2; ++vi) { 3502 SmallVector<Constant*, 16> Indices; 3503 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 3504 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 3505 3506 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3507 SV = llvm::ConstantVector::get(Indices); 3508 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 3509 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3510 } 3511 return SV; 3512 } 3513 case NEON::BI__builtin_neon_vzip_v: 3514 case NEON::BI__builtin_neon_vzipq_v: { 3515 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 3516 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 3517 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 3518 Value *SV = nullptr; 3519 3520 for (unsigned vi = 0; vi != 2; ++vi) { 3521 SmallVector<Constant*, 16> Indices; 3522 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 3523 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 3524 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 3525 } 3526 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 3527 SV = llvm::ConstantVector::get(Indices); 3528 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 3529 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 3530 } 3531 return SV; 3532 } 3533 } 3534 3535 assert(Int && "Expected valid intrinsic number"); 3536 3537 // Determine the type(s) of this overloaded AArch64 intrinsic. 3538 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); 3539 3540 Value *Result = EmitNeonCall(F, Ops, NameHint); 3541 llvm::Type *ResultType = ConvertType(E->getType()); 3542 // AArch64 intrinsic one-element vector type cast to 3543 // scalar type expected by the builtin 3544 return Builder.CreateBitCast(Result, ResultType, NameHint); 3545 } 3546 3547 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr( 3548 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, 3549 const CmpInst::Predicate Ip, const Twine &Name) { 3550 llvm::Type *OTy = Op->getType(); 3551 3552 // FIXME: this is utterly horrific. We should not be looking at previous 3553 // codegen context to find out what needs doing. Unfortunately TableGen 3554 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 3555 // (etc). 3556 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) 3557 OTy = BI->getOperand(0)->getType(); 3558 3559 Op = Builder.CreateBitCast(Op, OTy); 3560 if (OTy->getScalarType()->isFloatingPointTy()) { 3561 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); 3562 } else { 3563 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); 3564 } 3565 return Builder.CreateSExt(Op, Ty, Name); 3566 } 3567 3568 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, 3569 Value *ExtOp, Value *IndexOp, 3570 llvm::Type *ResTy, unsigned IntID, 3571 const char *Name) { 3572 SmallVector<Value *, 2> TblOps; 3573 if (ExtOp) 3574 TblOps.push_back(ExtOp); 3575 3576 // Build a vector containing sequential number like (0, 1, 2, ..., 15) 3577 SmallVector<Constant*, 16> Indices; 3578 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); 3579 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { 3580 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); 3581 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); 3582 } 3583 Value *SV = llvm::ConstantVector::get(Indices); 3584 3585 int PairPos = 0, End = Ops.size() - 1; 3586 while (PairPos < End) { 3587 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3588 Ops[PairPos+1], SV, Name)); 3589 PairPos += 2; 3590 } 3591 3592 // If there's an odd number of 64-bit lookup table, fill the high 64-bit 3593 // of the 128-bit lookup table with zero. 3594 if (PairPos == End) { 3595 Value *ZeroTbl = ConstantAggregateZero::get(TblTy); 3596 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], 3597 ZeroTbl, SV, Name)); 3598 } 3599 3600 Function *TblF; 3601 TblOps.push_back(IndexOp); 3602 TblF = CGF.CGM.getIntrinsic(IntID, ResTy); 3603 3604 return CGF.EmitNeonCall(TblF, TblOps, Name); 3605 } 3606 3607 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) { 3608 unsigned Value; 3609 switch (BuiltinID) { 3610 default: 3611 return nullptr; 3612 case ARM::BI__builtin_arm_nop: 3613 Value = 0; 3614 break; 3615 case ARM::BI__builtin_arm_yield: 3616 case ARM::BI__yield: 3617 Value = 1; 3618 break; 3619 case ARM::BI__builtin_arm_wfe: 3620 case ARM::BI__wfe: 3621 Value = 2; 3622 break; 3623 case ARM::BI__builtin_arm_wfi: 3624 case ARM::BI__wfi: 3625 Value = 3; 3626 break; 3627 case ARM::BI__builtin_arm_sev: 3628 case ARM::BI__sev: 3629 Value = 4; 3630 break; 3631 case ARM::BI__builtin_arm_sevl: 3632 case ARM::BI__sevl: 3633 Value = 5; 3634 break; 3635 } 3636 3637 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint), 3638 llvm::ConstantInt::get(Int32Ty, Value)); 3639 } 3640 3641 // Generates the IR for the read/write special register builtin, 3642 // ValueType is the type of the value that is to be written or read, 3643 // RegisterType is the type of the register being written to or read from. 3644 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, 3645 const CallExpr *E, 3646 llvm::Type *RegisterType, 3647 llvm::Type *ValueType, bool IsRead) { 3648 // write and register intrinsics only support 32 and 64 bit operations. 3649 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64)) 3650 && "Unsupported size for register."); 3651 3652 CodeGen::CGBuilderTy &Builder = CGF.Builder; 3653 CodeGen::CodeGenModule &CGM = CGF.CGM; 3654 LLVMContext &Context = CGM.getLLVMContext(); 3655 3656 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts(); 3657 StringRef SysReg = cast<StringLiteral>(SysRegStrExpr)->getString(); 3658 3659 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; 3660 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); 3661 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); 3662 3663 llvm::Type *Types[] = { RegisterType }; 3664 3665 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32); 3666 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64)) 3667 && "Can't fit 64-bit value in 32-bit register"); 3668 3669 if (IsRead) { 3670 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); 3671 llvm::Value *Call = Builder.CreateCall(F, Metadata); 3672 3673 if (MixedTypes) 3674 // Read into 64 bit register and then truncate result to 32 bit. 3675 return Builder.CreateTrunc(Call, ValueType); 3676 3677 if (ValueType->isPointerTy()) 3678 // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*). 3679 return Builder.CreateIntToPtr(Call, ValueType); 3680 3681 return Call; 3682 } 3683 3684 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); 3685 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1)); 3686 if (MixedTypes) { 3687 // Extend 32 bit write value to 64 bit to pass to write. 3688 ArgValue = Builder.CreateZExt(ArgValue, RegisterType); 3689 return Builder.CreateCall(F, { Metadata, ArgValue }); 3690 } 3691 3692 if (ValueType->isPointerTy()) { 3693 // Have VoidPtrTy ArgValue but want to return an i32/i64. 3694 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType); 3695 return Builder.CreateCall(F, { Metadata, ArgValue }); 3696 } 3697 3698 return Builder.CreateCall(F, { Metadata, ArgValue }); 3699 } 3700 3701 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra 3702 /// argument that specifies the vector type. 3703 static bool HasExtraNeonArgument(unsigned BuiltinID) { 3704 switch (BuiltinID) { 3705 default: break; 3706 case NEON::BI__builtin_neon_vget_lane_i8: 3707 case NEON::BI__builtin_neon_vget_lane_i16: 3708 case NEON::BI__builtin_neon_vget_lane_i32: 3709 case NEON::BI__builtin_neon_vget_lane_i64: 3710 case NEON::BI__builtin_neon_vget_lane_f32: 3711 case NEON::BI__builtin_neon_vgetq_lane_i8: 3712 case NEON::BI__builtin_neon_vgetq_lane_i16: 3713 case NEON::BI__builtin_neon_vgetq_lane_i32: 3714 case NEON::BI__builtin_neon_vgetq_lane_i64: 3715 case NEON::BI__builtin_neon_vgetq_lane_f32: 3716 case NEON::BI__builtin_neon_vset_lane_i8: 3717 case NEON::BI__builtin_neon_vset_lane_i16: 3718 case NEON::BI__builtin_neon_vset_lane_i32: 3719 case NEON::BI__builtin_neon_vset_lane_i64: 3720 case NEON::BI__builtin_neon_vset_lane_f32: 3721 case NEON::BI__builtin_neon_vsetq_lane_i8: 3722 case NEON::BI__builtin_neon_vsetq_lane_i16: 3723 case NEON::BI__builtin_neon_vsetq_lane_i32: 3724 case NEON::BI__builtin_neon_vsetq_lane_i64: 3725 case NEON::BI__builtin_neon_vsetq_lane_f32: 3726 case NEON::BI__builtin_neon_vsha1h_u32: 3727 case NEON::BI__builtin_neon_vsha1cq_u32: 3728 case NEON::BI__builtin_neon_vsha1pq_u32: 3729 case NEON::BI__builtin_neon_vsha1mq_u32: 3730 case ARM::BI_MoveToCoprocessor: 3731 case ARM::BI_MoveToCoprocessor2: 3732 return false; 3733 } 3734 return true; 3735 } 3736 3737 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, 3738 const CallExpr *E) { 3739 if (auto Hint = GetValueForARMHint(BuiltinID)) 3740 return Hint; 3741 3742 if (BuiltinID == ARM::BI__emit) { 3743 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb; 3744 llvm::FunctionType *FTy = 3745 llvm::FunctionType::get(VoidTy, /*Variadic=*/false); 3746 3747 APSInt Value; 3748 if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext())) 3749 llvm_unreachable("Sema will ensure that the parameter is constant"); 3750 3751 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue(); 3752 3753 llvm::InlineAsm *Emit = 3754 IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "", 3755 /*SideEffects=*/true) 3756 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "", 3757 /*SideEffects=*/true); 3758 3759 return Builder.CreateCall(Emit); 3760 } 3761 3762 if (BuiltinID == ARM::BI__builtin_arm_dbg) { 3763 Value *Option = EmitScalarExpr(E->getArg(0)); 3764 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); 3765 } 3766 3767 if (BuiltinID == ARM::BI__builtin_arm_prefetch) { 3768 Value *Address = EmitScalarExpr(E->getArg(0)); 3769 Value *RW = EmitScalarExpr(E->getArg(1)); 3770 Value *IsData = EmitScalarExpr(E->getArg(2)); 3771 3772 // Locality is not supported on ARM target 3773 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); 3774 3775 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 3776 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 3777 } 3778 3779 if (BuiltinID == ARM::BI__builtin_arm_rbit) { 3780 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), 3781 EmitScalarExpr(E->getArg(0)), 3782 "rbit"); 3783 } 3784 3785 if (BuiltinID == ARM::BI__clear_cache) { 3786 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 3787 const FunctionDecl *FD = E->getDirectCallee(); 3788 Value *Ops[2]; 3789 for (unsigned i = 0; i < 2; i++) 3790 Ops[i] = EmitScalarExpr(E->getArg(i)); 3791 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 3792 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 3793 StringRef Name = FD->getName(); 3794 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 3795 } 3796 3797 if (BuiltinID == ARM::BI__builtin_arm_ldrexd || 3798 ((BuiltinID == ARM::BI__builtin_arm_ldrex || 3799 BuiltinID == ARM::BI__builtin_arm_ldaex) && 3800 getContext().getTypeSize(E->getType()) == 64) || 3801 BuiltinID == ARM::BI__ldrexd) { 3802 Function *F; 3803 3804 switch (BuiltinID) { 3805 default: llvm_unreachable("unexpected builtin"); 3806 case ARM::BI__builtin_arm_ldaex: 3807 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); 3808 break; 3809 case ARM::BI__builtin_arm_ldrexd: 3810 case ARM::BI__builtin_arm_ldrex: 3811 case ARM::BI__ldrexd: 3812 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); 3813 break; 3814 } 3815 3816 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 3817 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 3818 "ldrexd"); 3819 3820 Value *Val0 = Builder.CreateExtractValue(Val, 1); 3821 Value *Val1 = Builder.CreateExtractValue(Val, 0); 3822 Val0 = Builder.CreateZExt(Val0, Int64Ty); 3823 Val1 = Builder.CreateZExt(Val1, Int64Ty); 3824 3825 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); 3826 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 3827 Val = Builder.CreateOr(Val, Val1); 3828 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 3829 } 3830 3831 if (BuiltinID == ARM::BI__builtin_arm_ldrex || 3832 BuiltinID == ARM::BI__builtin_arm_ldaex) { 3833 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 3834 3835 QualType Ty = E->getType(); 3836 llvm::Type *RealResTy = ConvertType(Ty); 3837 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 3838 getContext().getTypeSize(Ty)); 3839 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 3840 3841 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex 3842 ? Intrinsic::arm_ldaex 3843 : Intrinsic::arm_ldrex, 3844 LoadAddr->getType()); 3845 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); 3846 3847 if (RealResTy->isPointerTy()) 3848 return Builder.CreateIntToPtr(Val, RealResTy); 3849 else { 3850 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 3851 return Builder.CreateBitCast(Val, RealResTy); 3852 } 3853 } 3854 3855 if (BuiltinID == ARM::BI__builtin_arm_strexd || 3856 ((BuiltinID == ARM::BI__builtin_arm_stlex || 3857 BuiltinID == ARM::BI__builtin_arm_strex) && 3858 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { 3859 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3860 ? Intrinsic::arm_stlexd 3861 : Intrinsic::arm_strexd); 3862 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr); 3863 3864 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 3865 Value *Val = EmitScalarExpr(E->getArg(0)); 3866 Builder.CreateStore(Val, Tmp); 3867 3868 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); 3869 Val = Builder.CreateLoad(LdPtr); 3870 3871 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 3872 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 3873 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); 3874 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); 3875 } 3876 3877 if (BuiltinID == ARM::BI__builtin_arm_strex || 3878 BuiltinID == ARM::BI__builtin_arm_stlex) { 3879 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 3880 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 3881 3882 QualType Ty = E->getArg(0)->getType(); 3883 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 3884 getContext().getTypeSize(Ty)); 3885 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 3886 3887 if (StoreVal->getType()->isPointerTy()) 3888 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); 3889 else { 3890 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 3891 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); 3892 } 3893 3894 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex 3895 ? Intrinsic::arm_stlex 3896 : Intrinsic::arm_strex, 3897 StoreAddr->getType()); 3898 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); 3899 } 3900 3901 if (BuiltinID == ARM::BI__builtin_arm_clrex) { 3902 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); 3903 return Builder.CreateCall(F); 3904 } 3905 3906 // CRC32 3907 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 3908 switch (BuiltinID) { 3909 case ARM::BI__builtin_arm_crc32b: 3910 CRCIntrinsicID = Intrinsic::arm_crc32b; break; 3911 case ARM::BI__builtin_arm_crc32cb: 3912 CRCIntrinsicID = Intrinsic::arm_crc32cb; break; 3913 case ARM::BI__builtin_arm_crc32h: 3914 CRCIntrinsicID = Intrinsic::arm_crc32h; break; 3915 case ARM::BI__builtin_arm_crc32ch: 3916 CRCIntrinsicID = Intrinsic::arm_crc32ch; break; 3917 case ARM::BI__builtin_arm_crc32w: 3918 case ARM::BI__builtin_arm_crc32d: 3919 CRCIntrinsicID = Intrinsic::arm_crc32w; break; 3920 case ARM::BI__builtin_arm_crc32cw: 3921 case ARM::BI__builtin_arm_crc32cd: 3922 CRCIntrinsicID = Intrinsic::arm_crc32cw; break; 3923 } 3924 3925 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 3926 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 3927 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 3928 3929 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w 3930 // intrinsics, hence we need different codegen for these cases. 3931 if (BuiltinID == ARM::BI__builtin_arm_crc32d || 3932 BuiltinID == ARM::BI__builtin_arm_crc32cd) { 3933 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); 3934 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); 3935 Value *Arg1b = Builder.CreateLShr(Arg1, C1); 3936 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); 3937 3938 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3939 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); 3940 return Builder.CreateCall(F, {Res, Arg1b}); 3941 } else { 3942 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); 3943 3944 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 3945 return Builder.CreateCall(F, {Arg0, Arg1}); 3946 } 3947 } 3948 3949 if (BuiltinID == ARM::BI__builtin_arm_rsr || 3950 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3951 BuiltinID == ARM::BI__builtin_arm_rsrp || 3952 BuiltinID == ARM::BI__builtin_arm_wsr || 3953 BuiltinID == ARM::BI__builtin_arm_wsr64 || 3954 BuiltinID == ARM::BI__builtin_arm_wsrp) { 3955 3956 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr || 3957 BuiltinID == ARM::BI__builtin_arm_rsr64 || 3958 BuiltinID == ARM::BI__builtin_arm_rsrp; 3959 3960 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp || 3961 BuiltinID == ARM::BI__builtin_arm_wsrp; 3962 3963 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 || 3964 BuiltinID == ARM::BI__builtin_arm_wsr64; 3965 3966 llvm::Type *ValueType; 3967 llvm::Type *RegisterType; 3968 if (IsPointerBuiltin) { 3969 ValueType = VoidPtrTy; 3970 RegisterType = Int32Ty; 3971 } else if (Is64Bit) { 3972 ValueType = RegisterType = Int64Ty; 3973 } else { 3974 ValueType = RegisterType = Int32Ty; 3975 } 3976 3977 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 3978 } 3979 3980 // Find out if any arguments are required to be integer constant 3981 // expressions. 3982 unsigned ICEArguments = 0; 3983 ASTContext::GetBuiltinTypeError Error; 3984 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 3985 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 3986 3987 auto getAlignmentValue32 = [&](Address addr) -> Value* { 3988 return Builder.getInt32(addr.getAlignment().getQuantity()); 3989 }; 3990 3991 Address PtrOp0 = Address::invalid(); 3992 Address PtrOp1 = Address::invalid(); 3993 SmallVector<Value*, 4> Ops; 3994 bool HasExtraArg = HasExtraNeonArgument(BuiltinID); 3995 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0); 3996 for (unsigned i = 0, e = NumArgs; i != e; i++) { 3997 if (i == 0) { 3998 switch (BuiltinID) { 3999 case NEON::BI__builtin_neon_vld1_v: 4000 case NEON::BI__builtin_neon_vld1q_v: 4001 case NEON::BI__builtin_neon_vld1q_lane_v: 4002 case NEON::BI__builtin_neon_vld1_lane_v: 4003 case NEON::BI__builtin_neon_vld1_dup_v: 4004 case NEON::BI__builtin_neon_vld1q_dup_v: 4005 case NEON::BI__builtin_neon_vst1_v: 4006 case NEON::BI__builtin_neon_vst1q_v: 4007 case NEON::BI__builtin_neon_vst1q_lane_v: 4008 case NEON::BI__builtin_neon_vst1_lane_v: 4009 case NEON::BI__builtin_neon_vst2_v: 4010 case NEON::BI__builtin_neon_vst2q_v: 4011 case NEON::BI__builtin_neon_vst2_lane_v: 4012 case NEON::BI__builtin_neon_vst2q_lane_v: 4013 case NEON::BI__builtin_neon_vst3_v: 4014 case NEON::BI__builtin_neon_vst3q_v: 4015 case NEON::BI__builtin_neon_vst3_lane_v: 4016 case NEON::BI__builtin_neon_vst3q_lane_v: 4017 case NEON::BI__builtin_neon_vst4_v: 4018 case NEON::BI__builtin_neon_vst4q_v: 4019 case NEON::BI__builtin_neon_vst4_lane_v: 4020 case NEON::BI__builtin_neon_vst4q_lane_v: 4021 // Get the alignment for the argument in addition to the value; 4022 // we'll use it later. 4023 PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); 4024 Ops.push_back(PtrOp0.getPointer()); 4025 continue; 4026 } 4027 } 4028 if (i == 1) { 4029 switch (BuiltinID) { 4030 case NEON::BI__builtin_neon_vld2_v: 4031 case NEON::BI__builtin_neon_vld2q_v: 4032 case NEON::BI__builtin_neon_vld3_v: 4033 case NEON::BI__builtin_neon_vld3q_v: 4034 case NEON::BI__builtin_neon_vld4_v: 4035 case NEON::BI__builtin_neon_vld4q_v: 4036 case NEON::BI__builtin_neon_vld2_lane_v: 4037 case NEON::BI__builtin_neon_vld2q_lane_v: 4038 case NEON::BI__builtin_neon_vld3_lane_v: 4039 case NEON::BI__builtin_neon_vld3q_lane_v: 4040 case NEON::BI__builtin_neon_vld4_lane_v: 4041 case NEON::BI__builtin_neon_vld4q_lane_v: 4042 case NEON::BI__builtin_neon_vld2_dup_v: 4043 case NEON::BI__builtin_neon_vld3_dup_v: 4044 case NEON::BI__builtin_neon_vld4_dup_v: 4045 // Get the alignment for the argument in addition to the value; 4046 // we'll use it later. 4047 PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); 4048 Ops.push_back(PtrOp1.getPointer()); 4049 continue; 4050 } 4051 } 4052 4053 if ((ICEArguments & (1 << i)) == 0) { 4054 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4055 } else { 4056 // If this is required to be a constant, constant fold it so that we know 4057 // that the generated intrinsic gets a ConstantInt. 4058 llvm::APSInt Result; 4059 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4060 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 4061 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4062 } 4063 } 4064 4065 switch (BuiltinID) { 4066 default: break; 4067 4068 case NEON::BI__builtin_neon_vget_lane_i8: 4069 case NEON::BI__builtin_neon_vget_lane_i16: 4070 case NEON::BI__builtin_neon_vget_lane_i32: 4071 case NEON::BI__builtin_neon_vget_lane_i64: 4072 case NEON::BI__builtin_neon_vget_lane_f32: 4073 case NEON::BI__builtin_neon_vgetq_lane_i8: 4074 case NEON::BI__builtin_neon_vgetq_lane_i16: 4075 case NEON::BI__builtin_neon_vgetq_lane_i32: 4076 case NEON::BI__builtin_neon_vgetq_lane_i64: 4077 case NEON::BI__builtin_neon_vgetq_lane_f32: 4078 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); 4079 4080 case NEON::BI__builtin_neon_vset_lane_i8: 4081 case NEON::BI__builtin_neon_vset_lane_i16: 4082 case NEON::BI__builtin_neon_vset_lane_i32: 4083 case NEON::BI__builtin_neon_vset_lane_i64: 4084 case NEON::BI__builtin_neon_vset_lane_f32: 4085 case NEON::BI__builtin_neon_vsetq_lane_i8: 4086 case NEON::BI__builtin_neon_vsetq_lane_i16: 4087 case NEON::BI__builtin_neon_vsetq_lane_i32: 4088 case NEON::BI__builtin_neon_vsetq_lane_i64: 4089 case NEON::BI__builtin_neon_vsetq_lane_f32: 4090 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4091 4092 case NEON::BI__builtin_neon_vsha1h_u32: 4093 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, 4094 "vsha1h"); 4095 case NEON::BI__builtin_neon_vsha1cq_u32: 4096 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, 4097 "vsha1h"); 4098 case NEON::BI__builtin_neon_vsha1pq_u32: 4099 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, 4100 "vsha1h"); 4101 case NEON::BI__builtin_neon_vsha1mq_u32: 4102 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, 4103 "vsha1h"); 4104 4105 // The ARM _MoveToCoprocessor builtins put the input register value as 4106 // the first argument, but the LLVM intrinsic expects it as the third one. 4107 case ARM::BI_MoveToCoprocessor: 4108 case ARM::BI_MoveToCoprocessor2: { 4109 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ? 4110 Intrinsic::arm_mcr : Intrinsic::arm_mcr2); 4111 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], 4112 Ops[3], Ops[4], Ops[5]}); 4113 } 4114 } 4115 4116 // Get the last argument, which specifies the vector type. 4117 assert(HasExtraArg); 4118 llvm::APSInt Result; 4119 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4120 if (!Arg->isIntegerConstantExpr(Result, getContext())) 4121 return nullptr; 4122 4123 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || 4124 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { 4125 // Determine the overloaded type of this builtin. 4126 llvm::Type *Ty; 4127 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) 4128 Ty = FloatTy; 4129 else 4130 Ty = DoubleTy; 4131 4132 // Determine whether this is an unsigned conversion or not. 4133 bool usgn = Result.getZExtValue() == 1; 4134 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; 4135 4136 // Call the appropriate intrinsic. 4137 Function *F = CGM.getIntrinsic(Int, Ty); 4138 return Builder.CreateCall(F, Ops, "vcvtr"); 4139 } 4140 4141 // Determine the type of this overloaded NEON intrinsic. 4142 NeonTypeFlags Type(Result.getZExtValue()); 4143 bool usgn = Type.isUnsigned(); 4144 bool rightShift = false; 4145 4146 llvm::VectorType *VTy = GetNeonType(this, Type); 4147 llvm::Type *Ty = VTy; 4148 if (!Ty) 4149 return nullptr; 4150 4151 // Many NEON builtins have identical semantics and uses in ARM and 4152 // AArch64. Emit these in a single function. 4153 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); 4154 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4155 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); 4156 if (Builtin) 4157 return EmitCommonNeonBuiltinExpr( 4158 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 4159 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); 4160 4161 unsigned Int; 4162 switch (BuiltinID) { 4163 default: return nullptr; 4164 case NEON::BI__builtin_neon_vld1q_lane_v: 4165 // Handle 64-bit integer elements as a special case. Use shuffles of 4166 // one-element vectors to avoid poor code for i64 in the backend. 4167 if (VTy->getElementType()->isIntegerTy(64)) { 4168 // Extract the other lane. 4169 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4170 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); 4171 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); 4172 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4173 // Load the value as a one-element vector. 4174 Ty = llvm::VectorType::get(VTy->getElementType(), 1); 4175 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4176 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys); 4177 Value *Align = getAlignmentValue32(PtrOp0); 4178 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); 4179 // Combine them. 4180 uint32_t Indices[] = {1 - Lane, Lane}; 4181 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 4182 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); 4183 } 4184 // fall through 4185 case NEON::BI__builtin_neon_vld1_lane_v: { 4186 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4187 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType()); 4188 Value *Ld = Builder.CreateLoad(PtrOp0); 4189 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); 4190 } 4191 case NEON::BI__builtin_neon_vld2_dup_v: 4192 case NEON::BI__builtin_neon_vld3_dup_v: 4193 case NEON::BI__builtin_neon_vld4_dup_v: { 4194 // Handle 64-bit elements as a special-case. There is no "dup" needed. 4195 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { 4196 switch (BuiltinID) { 4197 case NEON::BI__builtin_neon_vld2_dup_v: 4198 Int = Intrinsic::arm_neon_vld2; 4199 break; 4200 case NEON::BI__builtin_neon_vld3_dup_v: 4201 Int = Intrinsic::arm_neon_vld3; 4202 break; 4203 case NEON::BI__builtin_neon_vld4_dup_v: 4204 Int = Intrinsic::arm_neon_vld4; 4205 break; 4206 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4207 } 4208 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4209 Function *F = CGM.getIntrinsic(Int, Tys); 4210 llvm::Value *Align = getAlignmentValue32(PtrOp1); 4211 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); 4212 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4213 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4214 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4215 } 4216 switch (BuiltinID) { 4217 case NEON::BI__builtin_neon_vld2_dup_v: 4218 Int = Intrinsic::arm_neon_vld2lane; 4219 break; 4220 case NEON::BI__builtin_neon_vld3_dup_v: 4221 Int = Intrinsic::arm_neon_vld3lane; 4222 break; 4223 case NEON::BI__builtin_neon_vld4_dup_v: 4224 Int = Intrinsic::arm_neon_vld4lane; 4225 break; 4226 default: llvm_unreachable("unknown vld_dup intrinsic?"); 4227 } 4228 llvm::Type *Tys[] = {Ty, Int8PtrTy}; 4229 Function *F = CGM.getIntrinsic(Int, Tys); 4230 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); 4231 4232 SmallVector<Value*, 6> Args; 4233 Args.push_back(Ops[1]); 4234 Args.append(STy->getNumElements(), UndefValue::get(Ty)); 4235 4236 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 4237 Args.push_back(CI); 4238 Args.push_back(getAlignmentValue32(PtrOp1)); 4239 4240 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); 4241 // splat lane 0 to all elts in each vector of the result. 4242 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 4243 Value *Val = Builder.CreateExtractValue(Ops[1], i); 4244 Value *Elt = Builder.CreateBitCast(Val, Ty); 4245 Elt = EmitNeonSplat(Elt, CI); 4246 Elt = Builder.CreateBitCast(Elt, Val->getType()); 4247 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); 4248 } 4249 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4250 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4251 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 4252 } 4253 case NEON::BI__builtin_neon_vqrshrn_n_v: 4254 Int = 4255 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; 4256 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", 4257 1, true); 4258 case NEON::BI__builtin_neon_vqrshrun_n_v: 4259 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), 4260 Ops, "vqrshrun_n", 1, true); 4261 case NEON::BI__builtin_neon_vqshrn_n_v: 4262 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; 4263 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", 4264 1, true); 4265 case NEON::BI__builtin_neon_vqshrun_n_v: 4266 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), 4267 Ops, "vqshrun_n", 1, true); 4268 case NEON::BI__builtin_neon_vrecpe_v: 4269 case NEON::BI__builtin_neon_vrecpeq_v: 4270 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), 4271 Ops, "vrecpe"); 4272 case NEON::BI__builtin_neon_vrshrn_n_v: 4273 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), 4274 Ops, "vrshrn_n", 1, true); 4275 case NEON::BI__builtin_neon_vrsra_n_v: 4276 case NEON::BI__builtin_neon_vrsraq_n_v: 4277 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4278 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4279 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); 4280 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; 4281 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); 4282 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); 4283 case NEON::BI__builtin_neon_vsri_n_v: 4284 case NEON::BI__builtin_neon_vsriq_n_v: 4285 rightShift = true; 4286 case NEON::BI__builtin_neon_vsli_n_v: 4287 case NEON::BI__builtin_neon_vsliq_n_v: 4288 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); 4289 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), 4290 Ops, "vsli_n"); 4291 case NEON::BI__builtin_neon_vsra_n_v: 4292 case NEON::BI__builtin_neon_vsraq_n_v: 4293 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 4294 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 4295 return Builder.CreateAdd(Ops[0], Ops[1]); 4296 case NEON::BI__builtin_neon_vst1q_lane_v: 4297 // Handle 64-bit integer elements as a special case. Use a shuffle to get 4298 // a one-element vector and avoid poor code for i64 in the backend. 4299 if (VTy->getElementType()->isIntegerTy(64)) { 4300 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4301 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); 4302 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); 4303 Ops[2] = getAlignmentValue32(PtrOp0); 4304 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; 4305 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, 4306 Tys), Ops); 4307 } 4308 // fall through 4309 case NEON::BI__builtin_neon_vst1_lane_v: { 4310 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 4311 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 4312 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 4313 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); 4314 return St; 4315 } 4316 case NEON::BI__builtin_neon_vtbl1_v: 4317 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), 4318 Ops, "vtbl1"); 4319 case NEON::BI__builtin_neon_vtbl2_v: 4320 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), 4321 Ops, "vtbl2"); 4322 case NEON::BI__builtin_neon_vtbl3_v: 4323 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), 4324 Ops, "vtbl3"); 4325 case NEON::BI__builtin_neon_vtbl4_v: 4326 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), 4327 Ops, "vtbl4"); 4328 case NEON::BI__builtin_neon_vtbx1_v: 4329 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), 4330 Ops, "vtbx1"); 4331 case NEON::BI__builtin_neon_vtbx2_v: 4332 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), 4333 Ops, "vtbx2"); 4334 case NEON::BI__builtin_neon_vtbx3_v: 4335 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), 4336 Ops, "vtbx3"); 4337 case NEON::BI__builtin_neon_vtbx4_v: 4338 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), 4339 Ops, "vtbx4"); 4340 } 4341 } 4342 4343 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, 4344 const CallExpr *E, 4345 SmallVectorImpl<Value *> &Ops) { 4346 unsigned int Int = 0; 4347 const char *s = nullptr; 4348 4349 switch (BuiltinID) { 4350 default: 4351 return nullptr; 4352 case NEON::BI__builtin_neon_vtbl1_v: 4353 case NEON::BI__builtin_neon_vqtbl1_v: 4354 case NEON::BI__builtin_neon_vqtbl1q_v: 4355 case NEON::BI__builtin_neon_vtbl2_v: 4356 case NEON::BI__builtin_neon_vqtbl2_v: 4357 case NEON::BI__builtin_neon_vqtbl2q_v: 4358 case NEON::BI__builtin_neon_vtbl3_v: 4359 case NEON::BI__builtin_neon_vqtbl3_v: 4360 case NEON::BI__builtin_neon_vqtbl3q_v: 4361 case NEON::BI__builtin_neon_vtbl4_v: 4362 case NEON::BI__builtin_neon_vqtbl4_v: 4363 case NEON::BI__builtin_neon_vqtbl4q_v: 4364 break; 4365 case NEON::BI__builtin_neon_vtbx1_v: 4366 case NEON::BI__builtin_neon_vqtbx1_v: 4367 case NEON::BI__builtin_neon_vqtbx1q_v: 4368 case NEON::BI__builtin_neon_vtbx2_v: 4369 case NEON::BI__builtin_neon_vqtbx2_v: 4370 case NEON::BI__builtin_neon_vqtbx2q_v: 4371 case NEON::BI__builtin_neon_vtbx3_v: 4372 case NEON::BI__builtin_neon_vqtbx3_v: 4373 case NEON::BI__builtin_neon_vqtbx3q_v: 4374 case NEON::BI__builtin_neon_vtbx4_v: 4375 case NEON::BI__builtin_neon_vqtbx4_v: 4376 case NEON::BI__builtin_neon_vqtbx4q_v: 4377 break; 4378 } 4379 4380 assert(E->getNumArgs() >= 3); 4381 4382 // Get the last argument, which specifies the vector type. 4383 llvm::APSInt Result; 4384 const Expr *Arg = E->getArg(E->getNumArgs() - 1); 4385 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) 4386 return nullptr; 4387 4388 // Determine the type of this overloaded NEON intrinsic. 4389 NeonTypeFlags Type(Result.getZExtValue()); 4390 llvm::VectorType *Ty = GetNeonType(&CGF, Type); 4391 if (!Ty) 4392 return nullptr; 4393 4394 CodeGen::CGBuilderTy &Builder = CGF.Builder; 4395 4396 // AArch64 scalar builtins are not overloaded, they do not have an extra 4397 // argument that specifies the vector type, need to handle each case. 4398 switch (BuiltinID) { 4399 case NEON::BI__builtin_neon_vtbl1_v: { 4400 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, 4401 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, 4402 "vtbl1"); 4403 } 4404 case NEON::BI__builtin_neon_vtbl2_v: { 4405 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, 4406 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, 4407 "vtbl1"); 4408 } 4409 case NEON::BI__builtin_neon_vtbl3_v: { 4410 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, 4411 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, 4412 "vtbl2"); 4413 } 4414 case NEON::BI__builtin_neon_vtbl4_v: { 4415 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, 4416 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, 4417 "vtbl2"); 4418 } 4419 case NEON::BI__builtin_neon_vtbx1_v: { 4420 Value *TblRes = 4421 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], 4422 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1"); 4423 4424 llvm::Constant *EightV = ConstantInt::get(Ty, 8); 4425 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); 4426 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4427 4428 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4429 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4430 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4431 } 4432 case NEON::BI__builtin_neon_vtbx2_v: { 4433 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], 4434 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, 4435 "vtbx1"); 4436 } 4437 case NEON::BI__builtin_neon_vtbx3_v: { 4438 Value *TblRes = 4439 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], 4440 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2"); 4441 4442 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24); 4443 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], 4444 TwentyFourV); 4445 CmpRes = Builder.CreateSExt(CmpRes, Ty); 4446 4447 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); 4448 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); 4449 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); 4450 } 4451 case NEON::BI__builtin_neon_vtbx4_v: { 4452 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], 4453 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, 4454 "vtbx2"); 4455 } 4456 case NEON::BI__builtin_neon_vqtbl1_v: 4457 case NEON::BI__builtin_neon_vqtbl1q_v: 4458 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; 4459 case NEON::BI__builtin_neon_vqtbl2_v: 4460 case NEON::BI__builtin_neon_vqtbl2q_v: { 4461 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; 4462 case NEON::BI__builtin_neon_vqtbl3_v: 4463 case NEON::BI__builtin_neon_vqtbl3q_v: 4464 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; 4465 case NEON::BI__builtin_neon_vqtbl4_v: 4466 case NEON::BI__builtin_neon_vqtbl4q_v: 4467 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; 4468 case NEON::BI__builtin_neon_vqtbx1_v: 4469 case NEON::BI__builtin_neon_vqtbx1q_v: 4470 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; 4471 case NEON::BI__builtin_neon_vqtbx2_v: 4472 case NEON::BI__builtin_neon_vqtbx2q_v: 4473 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; 4474 case NEON::BI__builtin_neon_vqtbx3_v: 4475 case NEON::BI__builtin_neon_vqtbx3q_v: 4476 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; 4477 case NEON::BI__builtin_neon_vqtbx4_v: 4478 case NEON::BI__builtin_neon_vqtbx4q_v: 4479 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; 4480 } 4481 } 4482 4483 if (!Int) 4484 return nullptr; 4485 4486 Function *F = CGF.CGM.getIntrinsic(Int, Ty); 4487 return CGF.EmitNeonCall(F, Ops, s); 4488 } 4489 4490 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { 4491 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); 4492 Op = Builder.CreateBitCast(Op, Int16Ty); 4493 Value *V = UndefValue::get(VTy); 4494 llvm::Constant *CI = ConstantInt::get(SizeTy, 0); 4495 Op = Builder.CreateInsertElement(V, Op, CI); 4496 return Op; 4497 } 4498 4499 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, 4500 const CallExpr *E) { 4501 unsigned HintID = static_cast<unsigned>(-1); 4502 switch (BuiltinID) { 4503 default: break; 4504 case AArch64::BI__builtin_arm_nop: 4505 HintID = 0; 4506 break; 4507 case AArch64::BI__builtin_arm_yield: 4508 HintID = 1; 4509 break; 4510 case AArch64::BI__builtin_arm_wfe: 4511 HintID = 2; 4512 break; 4513 case AArch64::BI__builtin_arm_wfi: 4514 HintID = 3; 4515 break; 4516 case AArch64::BI__builtin_arm_sev: 4517 HintID = 4; 4518 break; 4519 case AArch64::BI__builtin_arm_sevl: 4520 HintID = 5; 4521 break; 4522 } 4523 4524 if (HintID != static_cast<unsigned>(-1)) { 4525 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); 4526 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); 4527 } 4528 4529 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { 4530 Value *Address = EmitScalarExpr(E->getArg(0)); 4531 Value *RW = EmitScalarExpr(E->getArg(1)); 4532 Value *CacheLevel = EmitScalarExpr(E->getArg(2)); 4533 Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); 4534 Value *IsData = EmitScalarExpr(E->getArg(4)); 4535 4536 Value *Locality = nullptr; 4537 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { 4538 // Temporal fetch, needs to convert cache level to locality. 4539 Locality = llvm::ConstantInt::get(Int32Ty, 4540 -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); 4541 } else { 4542 // Streaming fetch. 4543 Locality = llvm::ConstantInt::get(Int32Ty, 0); 4544 } 4545 4546 // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify 4547 // PLDL3STRM or PLDL2STRM. 4548 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 4549 return Builder.CreateCall(F, {Address, RW, Locality, IsData}); 4550 } 4551 4552 if (BuiltinID == AArch64::BI__builtin_arm_rbit) { 4553 assert((getContext().getTypeSize(E->getType()) == 32) && 4554 "rbit of unusual size!"); 4555 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4556 return Builder.CreateCall( 4557 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4558 } 4559 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { 4560 assert((getContext().getTypeSize(E->getType()) == 64) && 4561 "rbit of unusual size!"); 4562 llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); 4563 return Builder.CreateCall( 4564 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); 4565 } 4566 4567 if (BuiltinID == AArch64::BI__clear_cache) { 4568 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); 4569 const FunctionDecl *FD = E->getDirectCallee(); 4570 Value *Ops[2]; 4571 for (unsigned i = 0; i < 2; i++) 4572 Ops[i] = EmitScalarExpr(E->getArg(i)); 4573 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); 4574 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); 4575 StringRef Name = FD->getName(); 4576 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); 4577 } 4578 4579 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || 4580 BuiltinID == AArch64::BI__builtin_arm_ldaex) && 4581 getContext().getTypeSize(E->getType()) == 128) { 4582 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4583 ? Intrinsic::aarch64_ldaxp 4584 : Intrinsic::aarch64_ldxp); 4585 4586 Value *LdPtr = EmitScalarExpr(E->getArg(0)); 4587 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), 4588 "ldxp"); 4589 4590 Value *Val0 = Builder.CreateExtractValue(Val, 1); 4591 Value *Val1 = Builder.CreateExtractValue(Val, 0); 4592 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); 4593 Val0 = Builder.CreateZExt(Val0, Int128Ty); 4594 Val1 = Builder.CreateZExt(Val1, Int128Ty); 4595 4596 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); 4597 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); 4598 Val = Builder.CreateOr(Val, Val1); 4599 return Builder.CreateBitCast(Val, ConvertType(E->getType())); 4600 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || 4601 BuiltinID == AArch64::BI__builtin_arm_ldaex) { 4602 Value *LoadAddr = EmitScalarExpr(E->getArg(0)); 4603 4604 QualType Ty = E->getType(); 4605 llvm::Type *RealResTy = ConvertType(Ty); 4606 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), 4607 getContext().getTypeSize(Ty)); 4608 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); 4609 4610 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex 4611 ? Intrinsic::aarch64_ldaxr 4612 : Intrinsic::aarch64_ldxr, 4613 LoadAddr->getType()); 4614 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); 4615 4616 if (RealResTy->isPointerTy()) 4617 return Builder.CreateIntToPtr(Val, RealResTy); 4618 4619 Val = Builder.CreateTruncOrBitCast(Val, IntResTy); 4620 return Builder.CreateBitCast(Val, RealResTy); 4621 } 4622 4623 if ((BuiltinID == AArch64::BI__builtin_arm_strex || 4624 BuiltinID == AArch64::BI__builtin_arm_stlex) && 4625 getContext().getTypeSize(E->getArg(0)->getType()) == 128) { 4626 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4627 ? Intrinsic::aarch64_stlxp 4628 : Intrinsic::aarch64_stxp); 4629 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr); 4630 4631 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 4632 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true); 4633 4634 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy)); 4635 llvm::Value *Val = Builder.CreateLoad(Tmp); 4636 4637 Value *Arg0 = Builder.CreateExtractValue(Val, 0); 4638 Value *Arg1 = Builder.CreateExtractValue(Val, 1); 4639 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), 4640 Int8PtrTy); 4641 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); 4642 } 4643 4644 if (BuiltinID == AArch64::BI__builtin_arm_strex || 4645 BuiltinID == AArch64::BI__builtin_arm_stlex) { 4646 Value *StoreVal = EmitScalarExpr(E->getArg(0)); 4647 Value *StoreAddr = EmitScalarExpr(E->getArg(1)); 4648 4649 QualType Ty = E->getArg(0)->getType(); 4650 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), 4651 getContext().getTypeSize(Ty)); 4652 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); 4653 4654 if (StoreVal->getType()->isPointerTy()) 4655 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); 4656 else { 4657 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); 4658 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); 4659 } 4660 4661 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex 4662 ? Intrinsic::aarch64_stlxr 4663 : Intrinsic::aarch64_stxr, 4664 StoreAddr->getType()); 4665 return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); 4666 } 4667 4668 if (BuiltinID == AArch64::BI__builtin_arm_clrex) { 4669 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); 4670 return Builder.CreateCall(F); 4671 } 4672 4673 // CRC32 4674 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; 4675 switch (BuiltinID) { 4676 case AArch64::BI__builtin_arm_crc32b: 4677 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; 4678 case AArch64::BI__builtin_arm_crc32cb: 4679 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; 4680 case AArch64::BI__builtin_arm_crc32h: 4681 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; 4682 case AArch64::BI__builtin_arm_crc32ch: 4683 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; 4684 case AArch64::BI__builtin_arm_crc32w: 4685 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; 4686 case AArch64::BI__builtin_arm_crc32cw: 4687 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; 4688 case AArch64::BI__builtin_arm_crc32d: 4689 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; 4690 case AArch64::BI__builtin_arm_crc32cd: 4691 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; 4692 } 4693 4694 if (CRCIntrinsicID != Intrinsic::not_intrinsic) { 4695 Value *Arg0 = EmitScalarExpr(E->getArg(0)); 4696 Value *Arg1 = EmitScalarExpr(E->getArg(1)); 4697 Function *F = CGM.getIntrinsic(CRCIntrinsicID); 4698 4699 llvm::Type *DataTy = F->getFunctionType()->getParamType(1); 4700 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); 4701 4702 return Builder.CreateCall(F, {Arg0, Arg1}); 4703 } 4704 4705 if (BuiltinID == AArch64::BI__builtin_arm_rsr || 4706 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4707 BuiltinID == AArch64::BI__builtin_arm_rsrp || 4708 BuiltinID == AArch64::BI__builtin_arm_wsr || 4709 BuiltinID == AArch64::BI__builtin_arm_wsr64 || 4710 BuiltinID == AArch64::BI__builtin_arm_wsrp) { 4711 4712 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr || 4713 BuiltinID == AArch64::BI__builtin_arm_rsr64 || 4714 BuiltinID == AArch64::BI__builtin_arm_rsrp; 4715 4716 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp || 4717 BuiltinID == AArch64::BI__builtin_arm_wsrp; 4718 4719 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr && 4720 BuiltinID != AArch64::BI__builtin_arm_wsr; 4721 4722 llvm::Type *ValueType; 4723 llvm::Type *RegisterType = Int64Ty; 4724 if (IsPointerBuiltin) { 4725 ValueType = VoidPtrTy; 4726 } else if (Is64Bit) { 4727 ValueType = Int64Ty; 4728 } else { 4729 ValueType = Int32Ty; 4730 } 4731 4732 return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead); 4733 } 4734 4735 // Find out if any arguments are required to be integer constant 4736 // expressions. 4737 unsigned ICEArguments = 0; 4738 ASTContext::GetBuiltinTypeError Error; 4739 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 4740 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 4741 4742 llvm::SmallVector<Value*, 4> Ops; 4743 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { 4744 if ((ICEArguments & (1 << i)) == 0) { 4745 Ops.push_back(EmitScalarExpr(E->getArg(i))); 4746 } else { 4747 // If this is required to be a constant, constant fold it so that we know 4748 // that the generated intrinsic gets a ConstantInt. 4749 llvm::APSInt Result; 4750 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 4751 assert(IsConst && "Constant arg isn't actually constant?"); 4752 (void)IsConst; 4753 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 4754 } 4755 } 4756 4757 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); 4758 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( 4759 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); 4760 4761 if (Builtin) { 4762 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); 4763 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); 4764 assert(Result && "SISD intrinsic should have been handled"); 4765 return Result; 4766 } 4767 4768 llvm::APSInt Result; 4769 const Expr *Arg = E->getArg(E->getNumArgs()-1); 4770 NeonTypeFlags Type(0); 4771 if (Arg->isIntegerConstantExpr(Result, getContext())) 4772 // Determine the type of this overloaded NEON intrinsic. 4773 Type = NeonTypeFlags(Result.getZExtValue()); 4774 4775 bool usgn = Type.isUnsigned(); 4776 bool quad = Type.isQuad(); 4777 4778 // Handle non-overloaded intrinsics first. 4779 switch (BuiltinID) { 4780 default: break; 4781 case NEON::BI__builtin_neon_vldrq_p128: { 4782 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4783 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); 4784 return Builder.CreateDefaultAlignedLoad(Ptr); 4785 } 4786 case NEON::BI__builtin_neon_vstrq_p128: { 4787 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); 4788 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); 4789 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr); 4790 } 4791 case NEON::BI__builtin_neon_vcvts_u32_f32: 4792 case NEON::BI__builtin_neon_vcvtd_u64_f64: 4793 usgn = true; 4794 // FALL THROUGH 4795 case NEON::BI__builtin_neon_vcvts_s32_f32: 4796 case NEON::BI__builtin_neon_vcvtd_s64_f64: { 4797 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4798 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4799 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4800 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4801 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); 4802 if (usgn) 4803 return Builder.CreateFPToUI(Ops[0], InTy); 4804 return Builder.CreateFPToSI(Ops[0], InTy); 4805 } 4806 case NEON::BI__builtin_neon_vcvts_f32_u32: 4807 case NEON::BI__builtin_neon_vcvtd_f64_u64: 4808 usgn = true; 4809 // FALL THROUGH 4810 case NEON::BI__builtin_neon_vcvts_f32_s32: 4811 case NEON::BI__builtin_neon_vcvtd_f64_s64: { 4812 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4813 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; 4814 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; 4815 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; 4816 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); 4817 if (usgn) 4818 return Builder.CreateUIToFP(Ops[0], FTy); 4819 return Builder.CreateSIToFP(Ops[0], FTy); 4820 } 4821 case NEON::BI__builtin_neon_vpaddd_s64: { 4822 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2); 4823 Value *Vec = EmitScalarExpr(E->getArg(0)); 4824 // The vector is v2f64, so make sure it's bitcast to that. 4825 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); 4826 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4827 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4828 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4829 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4830 // Pairwise addition of a v2f64 into a scalar f64. 4831 return Builder.CreateAdd(Op0, Op1, "vpaddd"); 4832 } 4833 case NEON::BI__builtin_neon_vpaddd_f64: { 4834 llvm::Type *Ty = 4835 llvm::VectorType::get(DoubleTy, 2); 4836 Value *Vec = EmitScalarExpr(E->getArg(0)); 4837 // The vector is v2f64, so make sure it's bitcast to that. 4838 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); 4839 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4840 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4841 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4842 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4843 // Pairwise addition of a v2f64 into a scalar f64. 4844 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4845 } 4846 case NEON::BI__builtin_neon_vpadds_f32: { 4847 llvm::Type *Ty = 4848 llvm::VectorType::get(FloatTy, 2); 4849 Value *Vec = EmitScalarExpr(E->getArg(0)); 4850 // The vector is v2f32, so make sure it's bitcast to that. 4851 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); 4852 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); 4853 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); 4854 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); 4855 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); 4856 // Pairwise addition of a v2f32 into a scalar f32. 4857 return Builder.CreateFAdd(Op0, Op1, "vpaddd"); 4858 } 4859 case NEON::BI__builtin_neon_vceqzd_s64: 4860 case NEON::BI__builtin_neon_vceqzd_f64: 4861 case NEON::BI__builtin_neon_vceqzs_f32: 4862 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4863 return EmitAArch64CompareBuiltinExpr( 4864 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4865 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); 4866 case NEON::BI__builtin_neon_vcgezd_s64: 4867 case NEON::BI__builtin_neon_vcgezd_f64: 4868 case NEON::BI__builtin_neon_vcgezs_f32: 4869 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4870 return EmitAArch64CompareBuiltinExpr( 4871 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4872 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); 4873 case NEON::BI__builtin_neon_vclezd_s64: 4874 case NEON::BI__builtin_neon_vclezd_f64: 4875 case NEON::BI__builtin_neon_vclezs_f32: 4876 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4877 return EmitAArch64CompareBuiltinExpr( 4878 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4879 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); 4880 case NEON::BI__builtin_neon_vcgtzd_s64: 4881 case NEON::BI__builtin_neon_vcgtzd_f64: 4882 case NEON::BI__builtin_neon_vcgtzs_f32: 4883 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4884 return EmitAArch64CompareBuiltinExpr( 4885 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4886 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); 4887 case NEON::BI__builtin_neon_vcltzd_s64: 4888 case NEON::BI__builtin_neon_vcltzd_f64: 4889 case NEON::BI__builtin_neon_vcltzs_f32: 4890 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4891 return EmitAArch64CompareBuiltinExpr( 4892 Ops[0], ConvertType(E->getCallReturnType(getContext())), 4893 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); 4894 4895 case NEON::BI__builtin_neon_vceqzd_u64: { 4896 Ops.push_back(EmitScalarExpr(E->getArg(0))); 4897 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4898 Ops[0] = 4899 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); 4900 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); 4901 } 4902 case NEON::BI__builtin_neon_vceqd_f64: 4903 case NEON::BI__builtin_neon_vcled_f64: 4904 case NEON::BI__builtin_neon_vcltd_f64: 4905 case NEON::BI__builtin_neon_vcged_f64: 4906 case NEON::BI__builtin_neon_vcgtd_f64: { 4907 llvm::CmpInst::Predicate P; 4908 switch (BuiltinID) { 4909 default: llvm_unreachable("missing builtin ID in switch!"); 4910 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; 4911 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; 4912 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; 4913 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; 4914 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; 4915 } 4916 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4917 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 4918 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 4919 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4920 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); 4921 } 4922 case NEON::BI__builtin_neon_vceqs_f32: 4923 case NEON::BI__builtin_neon_vcles_f32: 4924 case NEON::BI__builtin_neon_vclts_f32: 4925 case NEON::BI__builtin_neon_vcges_f32: 4926 case NEON::BI__builtin_neon_vcgts_f32: { 4927 llvm::CmpInst::Predicate P; 4928 switch (BuiltinID) { 4929 default: llvm_unreachable("missing builtin ID in switch!"); 4930 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; 4931 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; 4932 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; 4933 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; 4934 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; 4935 } 4936 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4937 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); 4938 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); 4939 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); 4940 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); 4941 } 4942 case NEON::BI__builtin_neon_vceqd_s64: 4943 case NEON::BI__builtin_neon_vceqd_u64: 4944 case NEON::BI__builtin_neon_vcgtd_s64: 4945 case NEON::BI__builtin_neon_vcgtd_u64: 4946 case NEON::BI__builtin_neon_vcltd_s64: 4947 case NEON::BI__builtin_neon_vcltd_u64: 4948 case NEON::BI__builtin_neon_vcged_u64: 4949 case NEON::BI__builtin_neon_vcged_s64: 4950 case NEON::BI__builtin_neon_vcled_u64: 4951 case NEON::BI__builtin_neon_vcled_s64: { 4952 llvm::CmpInst::Predicate P; 4953 switch (BuiltinID) { 4954 default: llvm_unreachable("missing builtin ID in switch!"); 4955 case NEON::BI__builtin_neon_vceqd_s64: 4956 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; 4957 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; 4958 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; 4959 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; 4960 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; 4961 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; 4962 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; 4963 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; 4964 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; 4965 } 4966 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4967 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4968 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4969 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); 4970 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); 4971 } 4972 case NEON::BI__builtin_neon_vtstd_s64: 4973 case NEON::BI__builtin_neon_vtstd_u64: { 4974 Ops.push_back(EmitScalarExpr(E->getArg(1))); 4975 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); 4976 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 4977 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); 4978 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 4979 llvm::Constant::getNullValue(Int64Ty)); 4980 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); 4981 } 4982 case NEON::BI__builtin_neon_vset_lane_i8: 4983 case NEON::BI__builtin_neon_vset_lane_i16: 4984 case NEON::BI__builtin_neon_vset_lane_i32: 4985 case NEON::BI__builtin_neon_vset_lane_i64: 4986 case NEON::BI__builtin_neon_vset_lane_f32: 4987 case NEON::BI__builtin_neon_vsetq_lane_i8: 4988 case NEON::BI__builtin_neon_vsetq_lane_i16: 4989 case NEON::BI__builtin_neon_vsetq_lane_i32: 4990 case NEON::BI__builtin_neon_vsetq_lane_i64: 4991 case NEON::BI__builtin_neon_vsetq_lane_f32: 4992 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4993 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 4994 case NEON::BI__builtin_neon_vset_lane_f64: 4995 // The vector type needs a cast for the v1f64 variant. 4996 Ops[1] = Builder.CreateBitCast(Ops[1], 4997 llvm::VectorType::get(DoubleTy, 1)); 4998 Ops.push_back(EmitScalarExpr(E->getArg(2))); 4999 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5000 case NEON::BI__builtin_neon_vsetq_lane_f64: 5001 // The vector type needs a cast for the v2f64 variant. 5002 Ops[1] = Builder.CreateBitCast(Ops[1], 5003 llvm::VectorType::get(DoubleTy, 2)); 5004 Ops.push_back(EmitScalarExpr(E->getArg(2))); 5005 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); 5006 5007 case NEON::BI__builtin_neon_vget_lane_i8: 5008 case NEON::BI__builtin_neon_vdupb_lane_i8: 5009 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); 5010 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5011 "vget_lane"); 5012 case NEON::BI__builtin_neon_vgetq_lane_i8: 5013 case NEON::BI__builtin_neon_vdupb_laneq_i8: 5014 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); 5015 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5016 "vgetq_lane"); 5017 case NEON::BI__builtin_neon_vget_lane_i16: 5018 case NEON::BI__builtin_neon_vduph_lane_i16: 5019 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); 5020 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5021 "vget_lane"); 5022 case NEON::BI__builtin_neon_vgetq_lane_i16: 5023 case NEON::BI__builtin_neon_vduph_laneq_i16: 5024 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); 5025 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5026 "vgetq_lane"); 5027 case NEON::BI__builtin_neon_vget_lane_i32: 5028 case NEON::BI__builtin_neon_vdups_lane_i32: 5029 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); 5030 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5031 "vget_lane"); 5032 case NEON::BI__builtin_neon_vdups_lane_f32: 5033 Ops[0] = Builder.CreateBitCast(Ops[0], 5034 llvm::VectorType::get(FloatTy, 2)); 5035 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5036 "vdups_lane"); 5037 case NEON::BI__builtin_neon_vgetq_lane_i32: 5038 case NEON::BI__builtin_neon_vdups_laneq_i32: 5039 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); 5040 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5041 "vgetq_lane"); 5042 case NEON::BI__builtin_neon_vget_lane_i64: 5043 case NEON::BI__builtin_neon_vdupd_lane_i64: 5044 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); 5045 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5046 "vget_lane"); 5047 case NEON::BI__builtin_neon_vdupd_lane_f64: 5048 Ops[0] = Builder.CreateBitCast(Ops[0], 5049 llvm::VectorType::get(DoubleTy, 1)); 5050 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5051 "vdupd_lane"); 5052 case NEON::BI__builtin_neon_vgetq_lane_i64: 5053 case NEON::BI__builtin_neon_vdupd_laneq_i64: 5054 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); 5055 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5056 "vgetq_lane"); 5057 case NEON::BI__builtin_neon_vget_lane_f32: 5058 Ops[0] = Builder.CreateBitCast(Ops[0], 5059 llvm::VectorType::get(FloatTy, 2)); 5060 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5061 "vget_lane"); 5062 case NEON::BI__builtin_neon_vget_lane_f64: 5063 Ops[0] = Builder.CreateBitCast(Ops[0], 5064 llvm::VectorType::get(DoubleTy, 1)); 5065 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5066 "vget_lane"); 5067 case NEON::BI__builtin_neon_vgetq_lane_f32: 5068 case NEON::BI__builtin_neon_vdups_laneq_f32: 5069 Ops[0] = Builder.CreateBitCast(Ops[0], 5070 llvm::VectorType::get(FloatTy, 4)); 5071 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5072 "vgetq_lane"); 5073 case NEON::BI__builtin_neon_vgetq_lane_f64: 5074 case NEON::BI__builtin_neon_vdupd_laneq_f64: 5075 Ops[0] = Builder.CreateBitCast(Ops[0], 5076 llvm::VectorType::get(DoubleTy, 2)); 5077 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), 5078 "vgetq_lane"); 5079 case NEON::BI__builtin_neon_vaddd_s64: 5080 case NEON::BI__builtin_neon_vaddd_u64: 5081 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); 5082 case NEON::BI__builtin_neon_vsubd_s64: 5083 case NEON::BI__builtin_neon_vsubd_u64: 5084 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); 5085 case NEON::BI__builtin_neon_vqdmlalh_s16: 5086 case NEON::BI__builtin_neon_vqdmlslh_s16: { 5087 SmallVector<Value *, 2> ProductOps; 5088 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5089 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); 5090 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5091 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5092 ProductOps, "vqdmlXl"); 5093 Constant *CI = ConstantInt::get(SizeTy, 0); 5094 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5095 5096 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 5097 ? Intrinsic::aarch64_neon_sqadd 5098 : Intrinsic::aarch64_neon_sqsub; 5099 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); 5100 } 5101 case NEON::BI__builtin_neon_vqshlud_n_s64: { 5102 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5103 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5104 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), 5105 Ops, "vqshlu_n"); 5106 } 5107 case NEON::BI__builtin_neon_vqshld_n_u64: 5108 case NEON::BI__builtin_neon_vqshld_n_s64: { 5109 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 5110 ? Intrinsic::aarch64_neon_uqshl 5111 : Intrinsic::aarch64_neon_sqshl; 5112 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5113 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); 5114 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); 5115 } 5116 case NEON::BI__builtin_neon_vrshrd_n_u64: 5117 case NEON::BI__builtin_neon_vrshrd_n_s64: { 5118 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 5119 ? Intrinsic::aarch64_neon_urshl 5120 : Intrinsic::aarch64_neon_srshl; 5121 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5122 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); 5123 Ops[1] = ConstantInt::get(Int64Ty, -SV); 5124 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); 5125 } 5126 case NEON::BI__builtin_neon_vrsrad_n_u64: 5127 case NEON::BI__builtin_neon_vrsrad_n_s64: { 5128 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 5129 ? Intrinsic::aarch64_neon_urshl 5130 : Intrinsic::aarch64_neon_srshl; 5131 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); 5132 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); 5133 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), 5134 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); 5135 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); 5136 } 5137 case NEON::BI__builtin_neon_vshld_n_s64: 5138 case NEON::BI__builtin_neon_vshld_n_u64: { 5139 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5140 return Builder.CreateShl( 5141 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); 5142 } 5143 case NEON::BI__builtin_neon_vshrd_n_s64: { 5144 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5145 return Builder.CreateAShr( 5146 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5147 Amt->getZExtValue())), 5148 "shrd_n"); 5149 } 5150 case NEON::BI__builtin_neon_vshrd_n_u64: { 5151 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); 5152 uint64_t ShiftAmt = Amt->getZExtValue(); 5153 // Right-shifting an unsigned value by its size yields 0. 5154 if (ShiftAmt == 64) 5155 return ConstantInt::get(Int64Ty, 0); 5156 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), 5157 "shrd_n"); 5158 } 5159 case NEON::BI__builtin_neon_vsrad_n_s64: { 5160 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5161 Ops[1] = Builder.CreateAShr( 5162 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), 5163 Amt->getZExtValue())), 5164 "shrd_n"); 5165 return Builder.CreateAdd(Ops[0], Ops[1]); 5166 } 5167 case NEON::BI__builtin_neon_vsrad_n_u64: { 5168 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); 5169 uint64_t ShiftAmt = Amt->getZExtValue(); 5170 // Right-shifting an unsigned value by its size yields 0. 5171 // As Op + 0 = Op, return Ops[0] directly. 5172 if (ShiftAmt == 64) 5173 return Ops[0]; 5174 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), 5175 "shrd_n"); 5176 return Builder.CreateAdd(Ops[0], Ops[1]); 5177 } 5178 case NEON::BI__builtin_neon_vqdmlalh_lane_s16: 5179 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: 5180 case NEON::BI__builtin_neon_vqdmlslh_lane_s16: 5181 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { 5182 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5183 "lane"); 5184 SmallVector<Value *, 2> ProductOps; 5185 ProductOps.push_back(vectorWrapScalar16(Ops[1])); 5186 ProductOps.push_back(vectorWrapScalar16(Ops[2])); 5187 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); 5188 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), 5189 ProductOps, "vqdmlXl"); 5190 Constant *CI = ConstantInt::get(SizeTy, 0); 5191 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); 5192 Ops.pop_back(); 5193 5194 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || 5195 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) 5196 ? Intrinsic::aarch64_neon_sqadd 5197 : Intrinsic::aarch64_neon_sqsub; 5198 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); 5199 } 5200 case NEON::BI__builtin_neon_vqdmlals_s32: 5201 case NEON::BI__builtin_neon_vqdmlsls_s32: { 5202 SmallVector<Value *, 2> ProductOps; 5203 ProductOps.push_back(Ops[1]); 5204 ProductOps.push_back(EmitScalarExpr(E->getArg(2))); 5205 Ops[1] = 5206 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5207 ProductOps, "vqdmlXl"); 5208 5209 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 5210 ? Intrinsic::aarch64_neon_sqadd 5211 : Intrinsic::aarch64_neon_sqsub; 5212 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); 5213 } 5214 case NEON::BI__builtin_neon_vqdmlals_lane_s32: 5215 case NEON::BI__builtin_neon_vqdmlals_laneq_s32: 5216 case NEON::BI__builtin_neon_vqdmlsls_lane_s32: 5217 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { 5218 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), 5219 "lane"); 5220 SmallVector<Value *, 2> ProductOps; 5221 ProductOps.push_back(Ops[1]); 5222 ProductOps.push_back(Ops[2]); 5223 Ops[1] = 5224 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), 5225 ProductOps, "vqdmlXl"); 5226 Ops.pop_back(); 5227 5228 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || 5229 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) 5230 ? Intrinsic::aarch64_neon_sqadd 5231 : Intrinsic::aarch64_neon_sqsub; 5232 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); 5233 } 5234 } 5235 5236 llvm::VectorType *VTy = GetNeonType(this, Type); 5237 llvm::Type *Ty = VTy; 5238 if (!Ty) 5239 return nullptr; 5240 5241 // Not all intrinsics handled by the common case work for AArch64 yet, so only 5242 // defer to common code if it's been added to our special map. 5243 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, 5244 AArch64SIMDIntrinsicsProvenSorted); 5245 5246 if (Builtin) 5247 return EmitCommonNeonBuiltinExpr( 5248 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, 5249 Builtin->NameHint, Builtin->TypeModifier, E, Ops, 5250 /*never use addresses*/ Address::invalid(), Address::invalid()); 5251 5252 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) 5253 return V; 5254 5255 unsigned Int; 5256 switch (BuiltinID) { 5257 default: return nullptr; 5258 case NEON::BI__builtin_neon_vbsl_v: 5259 case NEON::BI__builtin_neon_vbslq_v: { 5260 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); 5261 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); 5262 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); 5263 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); 5264 5265 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); 5266 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); 5267 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); 5268 return Builder.CreateBitCast(Ops[0], Ty); 5269 } 5270 case NEON::BI__builtin_neon_vfma_lane_v: 5271 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types 5272 // The ARM builtins (and instructions) have the addend as the first 5273 // operand, but the 'fma' intrinsics have it last. Swap it around here. 5274 Value *Addend = Ops[0]; 5275 Value *Multiplicand = Ops[1]; 5276 Value *LaneSource = Ops[2]; 5277 Ops[0] = Multiplicand; 5278 Ops[1] = LaneSource; 5279 Ops[2] = Addend; 5280 5281 // Now adjust things to handle the lane access. 5282 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? 5283 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : 5284 VTy; 5285 llvm::Constant *cst = cast<Constant>(Ops[3]); 5286 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); 5287 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); 5288 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); 5289 5290 Ops.pop_back(); 5291 Int = Intrinsic::fma; 5292 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); 5293 } 5294 case NEON::BI__builtin_neon_vfma_laneq_v: { 5295 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); 5296 // v1f64 fma should be mapped to Neon scalar f64 fma 5297 if (VTy && VTy->getElementType() == DoubleTy) { 5298 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5299 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); 5300 llvm::Type *VTy = GetNeonType(this, 5301 NeonTypeFlags(NeonTypeFlags::Float64, false, true)); 5302 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); 5303 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5304 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); 5305 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5306 return Builder.CreateBitCast(Result, Ty); 5307 } 5308 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5309 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5310 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5311 5312 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), 5313 VTy->getNumElements() * 2); 5314 Ops[2] = Builder.CreateBitCast(Ops[2], STy); 5315 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), 5316 cast<ConstantInt>(Ops[3])); 5317 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); 5318 5319 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5320 } 5321 case NEON::BI__builtin_neon_vfmaq_laneq_v: { 5322 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5323 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5324 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5325 5326 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 5327 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); 5328 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); 5329 } 5330 case NEON::BI__builtin_neon_vfmas_lane_f32: 5331 case NEON::BI__builtin_neon_vfmas_laneq_f32: 5332 case NEON::BI__builtin_neon_vfmad_lane_f64: 5333 case NEON::BI__builtin_neon_vfmad_laneq_f64: { 5334 Ops.push_back(EmitScalarExpr(E->getArg(3))); 5335 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); 5336 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); 5337 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); 5338 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); 5339 } 5340 case NEON::BI__builtin_neon_vmull_v: 5341 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5342 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; 5343 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; 5344 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); 5345 case NEON::BI__builtin_neon_vmax_v: 5346 case NEON::BI__builtin_neon_vmaxq_v: 5347 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5348 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; 5349 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; 5350 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); 5351 case NEON::BI__builtin_neon_vmin_v: 5352 case NEON::BI__builtin_neon_vminq_v: 5353 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5354 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; 5355 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; 5356 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); 5357 case NEON::BI__builtin_neon_vabd_v: 5358 case NEON::BI__builtin_neon_vabdq_v: 5359 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5360 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; 5361 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; 5362 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); 5363 case NEON::BI__builtin_neon_vpadal_v: 5364 case NEON::BI__builtin_neon_vpadalq_v: { 5365 unsigned ArgElts = VTy->getNumElements(); 5366 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); 5367 unsigned BitWidth = EltTy->getBitWidth(); 5368 llvm::Type *ArgTy = llvm::VectorType::get( 5369 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); 5370 llvm::Type* Tys[2] = { VTy, ArgTy }; 5371 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; 5372 SmallVector<llvm::Value*, 1> TmpOps; 5373 TmpOps.push_back(Ops[1]); 5374 Function *F = CGM.getIntrinsic(Int, Tys); 5375 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); 5376 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); 5377 return Builder.CreateAdd(tmp, addend); 5378 } 5379 case NEON::BI__builtin_neon_vpmin_v: 5380 case NEON::BI__builtin_neon_vpminq_v: 5381 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5382 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; 5383 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; 5384 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); 5385 case NEON::BI__builtin_neon_vpmax_v: 5386 case NEON::BI__builtin_neon_vpmaxq_v: 5387 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. 5388 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; 5389 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; 5390 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); 5391 case NEON::BI__builtin_neon_vminnm_v: 5392 case NEON::BI__builtin_neon_vminnmq_v: 5393 Int = Intrinsic::aarch64_neon_fminnm; 5394 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); 5395 case NEON::BI__builtin_neon_vmaxnm_v: 5396 case NEON::BI__builtin_neon_vmaxnmq_v: 5397 Int = Intrinsic::aarch64_neon_fmaxnm; 5398 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); 5399 case NEON::BI__builtin_neon_vrecpss_f32: { 5400 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5401 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy), 5402 Ops, "vrecps"); 5403 } 5404 case NEON::BI__builtin_neon_vrecpsd_f64: { 5405 Ops.push_back(EmitScalarExpr(E->getArg(1))); 5406 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy), 5407 Ops, "vrecps"); 5408 } 5409 case NEON::BI__builtin_neon_vqshrun_n_v: 5410 Int = Intrinsic::aarch64_neon_sqshrun; 5411 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); 5412 case NEON::BI__builtin_neon_vqrshrun_n_v: 5413 Int = Intrinsic::aarch64_neon_sqrshrun; 5414 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); 5415 case NEON::BI__builtin_neon_vqshrn_n_v: 5416 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; 5417 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); 5418 case NEON::BI__builtin_neon_vrshrn_n_v: 5419 Int = Intrinsic::aarch64_neon_rshrn; 5420 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); 5421 case NEON::BI__builtin_neon_vqrshrn_n_v: 5422 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; 5423 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); 5424 case NEON::BI__builtin_neon_vrnda_v: 5425 case NEON::BI__builtin_neon_vrndaq_v: { 5426 Int = Intrinsic::round; 5427 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); 5428 } 5429 case NEON::BI__builtin_neon_vrndi_v: 5430 case NEON::BI__builtin_neon_vrndiq_v: { 5431 Int = Intrinsic::nearbyint; 5432 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); 5433 } 5434 case NEON::BI__builtin_neon_vrndm_v: 5435 case NEON::BI__builtin_neon_vrndmq_v: { 5436 Int = Intrinsic::floor; 5437 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); 5438 } 5439 case NEON::BI__builtin_neon_vrndn_v: 5440 case NEON::BI__builtin_neon_vrndnq_v: { 5441 Int = Intrinsic::aarch64_neon_frintn; 5442 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); 5443 } 5444 case NEON::BI__builtin_neon_vrndp_v: 5445 case NEON::BI__builtin_neon_vrndpq_v: { 5446 Int = Intrinsic::ceil; 5447 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); 5448 } 5449 case NEON::BI__builtin_neon_vrndx_v: 5450 case NEON::BI__builtin_neon_vrndxq_v: { 5451 Int = Intrinsic::rint; 5452 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); 5453 } 5454 case NEON::BI__builtin_neon_vrnd_v: 5455 case NEON::BI__builtin_neon_vrndq_v: { 5456 Int = Intrinsic::trunc; 5457 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); 5458 } 5459 case NEON::BI__builtin_neon_vceqz_v: 5460 case NEON::BI__builtin_neon_vceqzq_v: 5461 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, 5462 ICmpInst::ICMP_EQ, "vceqz"); 5463 case NEON::BI__builtin_neon_vcgez_v: 5464 case NEON::BI__builtin_neon_vcgezq_v: 5465 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, 5466 ICmpInst::ICMP_SGE, "vcgez"); 5467 case NEON::BI__builtin_neon_vclez_v: 5468 case NEON::BI__builtin_neon_vclezq_v: 5469 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, 5470 ICmpInst::ICMP_SLE, "vclez"); 5471 case NEON::BI__builtin_neon_vcgtz_v: 5472 case NEON::BI__builtin_neon_vcgtzq_v: 5473 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, 5474 ICmpInst::ICMP_SGT, "vcgtz"); 5475 case NEON::BI__builtin_neon_vcltz_v: 5476 case NEON::BI__builtin_neon_vcltzq_v: 5477 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, 5478 ICmpInst::ICMP_SLT, "vcltz"); 5479 case NEON::BI__builtin_neon_vcvt_f64_v: 5480 case NEON::BI__builtin_neon_vcvtq_f64_v: 5481 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5482 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); 5483 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 5484 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); 5485 case NEON::BI__builtin_neon_vcvt_f64_f32: { 5486 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && 5487 "unexpected vcvt_f64_f32 builtin"); 5488 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); 5489 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5490 5491 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); 5492 } 5493 case NEON::BI__builtin_neon_vcvt_f32_f64: { 5494 assert(Type.getEltType() == NeonTypeFlags::Float32 && 5495 "unexpected vcvt_f32_f64 builtin"); 5496 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); 5497 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); 5498 5499 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); 5500 } 5501 case NEON::BI__builtin_neon_vcvt_s32_v: 5502 case NEON::BI__builtin_neon_vcvt_u32_v: 5503 case NEON::BI__builtin_neon_vcvt_s64_v: 5504 case NEON::BI__builtin_neon_vcvt_u64_v: 5505 case NEON::BI__builtin_neon_vcvtq_s32_v: 5506 case NEON::BI__builtin_neon_vcvtq_u32_v: 5507 case NEON::BI__builtin_neon_vcvtq_s64_v: 5508 case NEON::BI__builtin_neon_vcvtq_u64_v: { 5509 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); 5510 if (usgn) 5511 return Builder.CreateFPToUI(Ops[0], Ty); 5512 return Builder.CreateFPToSI(Ops[0], Ty); 5513 } 5514 case NEON::BI__builtin_neon_vcvta_s32_v: 5515 case NEON::BI__builtin_neon_vcvtaq_s32_v: 5516 case NEON::BI__builtin_neon_vcvta_u32_v: 5517 case NEON::BI__builtin_neon_vcvtaq_u32_v: 5518 case NEON::BI__builtin_neon_vcvta_s64_v: 5519 case NEON::BI__builtin_neon_vcvtaq_s64_v: 5520 case NEON::BI__builtin_neon_vcvta_u64_v: 5521 case NEON::BI__builtin_neon_vcvtaq_u64_v: { 5522 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; 5523 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5524 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); 5525 } 5526 case NEON::BI__builtin_neon_vcvtm_s32_v: 5527 case NEON::BI__builtin_neon_vcvtmq_s32_v: 5528 case NEON::BI__builtin_neon_vcvtm_u32_v: 5529 case NEON::BI__builtin_neon_vcvtmq_u32_v: 5530 case NEON::BI__builtin_neon_vcvtm_s64_v: 5531 case NEON::BI__builtin_neon_vcvtmq_s64_v: 5532 case NEON::BI__builtin_neon_vcvtm_u64_v: 5533 case NEON::BI__builtin_neon_vcvtmq_u64_v: { 5534 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; 5535 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5536 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); 5537 } 5538 case NEON::BI__builtin_neon_vcvtn_s32_v: 5539 case NEON::BI__builtin_neon_vcvtnq_s32_v: 5540 case NEON::BI__builtin_neon_vcvtn_u32_v: 5541 case NEON::BI__builtin_neon_vcvtnq_u32_v: 5542 case NEON::BI__builtin_neon_vcvtn_s64_v: 5543 case NEON::BI__builtin_neon_vcvtnq_s64_v: 5544 case NEON::BI__builtin_neon_vcvtn_u64_v: 5545 case NEON::BI__builtin_neon_vcvtnq_u64_v: { 5546 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; 5547 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5548 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); 5549 } 5550 case NEON::BI__builtin_neon_vcvtp_s32_v: 5551 case NEON::BI__builtin_neon_vcvtpq_s32_v: 5552 case NEON::BI__builtin_neon_vcvtp_u32_v: 5553 case NEON::BI__builtin_neon_vcvtpq_u32_v: 5554 case NEON::BI__builtin_neon_vcvtp_s64_v: 5555 case NEON::BI__builtin_neon_vcvtpq_s64_v: 5556 case NEON::BI__builtin_neon_vcvtp_u64_v: 5557 case NEON::BI__builtin_neon_vcvtpq_u64_v: { 5558 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; 5559 llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) }; 5560 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); 5561 } 5562 case NEON::BI__builtin_neon_vmulx_v: 5563 case NEON::BI__builtin_neon_vmulxq_v: { 5564 Int = Intrinsic::aarch64_neon_fmulx; 5565 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); 5566 } 5567 case NEON::BI__builtin_neon_vmul_lane_v: 5568 case NEON::BI__builtin_neon_vmul_laneq_v: { 5569 // v1f64 vmul_lane should be mapped to Neon scalar mul lane 5570 bool Quad = false; 5571 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) 5572 Quad = true; 5573 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5574 llvm::Type *VTy = GetNeonType(this, 5575 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); 5576 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5577 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); 5578 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); 5579 return Builder.CreateBitCast(Result, Ty); 5580 } 5581 case NEON::BI__builtin_neon_vnegd_s64: 5582 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); 5583 case NEON::BI__builtin_neon_vpmaxnm_v: 5584 case NEON::BI__builtin_neon_vpmaxnmq_v: { 5585 Int = Intrinsic::aarch64_neon_fmaxnmp; 5586 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); 5587 } 5588 case NEON::BI__builtin_neon_vpminnm_v: 5589 case NEON::BI__builtin_neon_vpminnmq_v: { 5590 Int = Intrinsic::aarch64_neon_fminnmp; 5591 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); 5592 } 5593 case NEON::BI__builtin_neon_vsqrt_v: 5594 case NEON::BI__builtin_neon_vsqrtq_v: { 5595 Int = Intrinsic::sqrt; 5596 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5597 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); 5598 } 5599 case NEON::BI__builtin_neon_vrbit_v: 5600 case NEON::BI__builtin_neon_vrbitq_v: { 5601 Int = Intrinsic::aarch64_neon_rbit; 5602 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); 5603 } 5604 case NEON::BI__builtin_neon_vaddv_u8: 5605 // FIXME: These are handled by the AArch64 scalar code. 5606 usgn = true; 5607 // FALLTHROUGH 5608 case NEON::BI__builtin_neon_vaddv_s8: { 5609 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5610 Ty = Int32Ty; 5611 VTy = llvm::VectorType::get(Int8Ty, 8); 5612 llvm::Type *Tys[2] = { Ty, VTy }; 5613 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5614 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5615 return Builder.CreateTrunc(Ops[0], Int8Ty); 5616 } 5617 case NEON::BI__builtin_neon_vaddv_u16: 5618 usgn = true; 5619 // FALLTHROUGH 5620 case NEON::BI__builtin_neon_vaddv_s16: { 5621 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5622 Ty = Int32Ty; 5623 VTy = llvm::VectorType::get(Int16Ty, 4); 5624 llvm::Type *Tys[2] = { Ty, VTy }; 5625 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5626 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5627 return Builder.CreateTrunc(Ops[0], Int16Ty); 5628 } 5629 case NEON::BI__builtin_neon_vaddvq_u8: 5630 usgn = true; 5631 // FALLTHROUGH 5632 case NEON::BI__builtin_neon_vaddvq_s8: { 5633 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5634 Ty = Int32Ty; 5635 VTy = llvm::VectorType::get(Int8Ty, 16); 5636 llvm::Type *Tys[2] = { Ty, VTy }; 5637 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5638 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5639 return Builder.CreateTrunc(Ops[0], Int8Ty); 5640 } 5641 case NEON::BI__builtin_neon_vaddvq_u16: 5642 usgn = true; 5643 // FALLTHROUGH 5644 case NEON::BI__builtin_neon_vaddvq_s16: { 5645 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; 5646 Ty = Int32Ty; 5647 VTy = llvm::VectorType::get(Int16Ty, 8); 5648 llvm::Type *Tys[2] = { Ty, VTy }; 5649 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5650 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); 5651 return Builder.CreateTrunc(Ops[0], Int16Ty); 5652 } 5653 case NEON::BI__builtin_neon_vmaxv_u8: { 5654 Int = Intrinsic::aarch64_neon_umaxv; 5655 Ty = Int32Ty; 5656 VTy = llvm::VectorType::get(Int8Ty, 8); 5657 llvm::Type *Tys[2] = { Ty, VTy }; 5658 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5659 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5660 return Builder.CreateTrunc(Ops[0], Int8Ty); 5661 } 5662 case NEON::BI__builtin_neon_vmaxv_u16: { 5663 Int = Intrinsic::aarch64_neon_umaxv; 5664 Ty = Int32Ty; 5665 VTy = llvm::VectorType::get(Int16Ty, 4); 5666 llvm::Type *Tys[2] = { Ty, VTy }; 5667 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5668 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5669 return Builder.CreateTrunc(Ops[0], Int16Ty); 5670 } 5671 case NEON::BI__builtin_neon_vmaxvq_u8: { 5672 Int = Intrinsic::aarch64_neon_umaxv; 5673 Ty = Int32Ty; 5674 VTy = llvm::VectorType::get(Int8Ty, 16); 5675 llvm::Type *Tys[2] = { Ty, VTy }; 5676 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5677 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5678 return Builder.CreateTrunc(Ops[0], Int8Ty); 5679 } 5680 case NEON::BI__builtin_neon_vmaxvq_u16: { 5681 Int = Intrinsic::aarch64_neon_umaxv; 5682 Ty = Int32Ty; 5683 VTy = llvm::VectorType::get(Int16Ty, 8); 5684 llvm::Type *Tys[2] = { Ty, VTy }; 5685 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5686 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5687 return Builder.CreateTrunc(Ops[0], Int16Ty); 5688 } 5689 case NEON::BI__builtin_neon_vmaxv_s8: { 5690 Int = Intrinsic::aarch64_neon_smaxv; 5691 Ty = Int32Ty; 5692 VTy = llvm::VectorType::get(Int8Ty, 8); 5693 llvm::Type *Tys[2] = { Ty, VTy }; 5694 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5695 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5696 return Builder.CreateTrunc(Ops[0], Int8Ty); 5697 } 5698 case NEON::BI__builtin_neon_vmaxv_s16: { 5699 Int = Intrinsic::aarch64_neon_smaxv; 5700 Ty = Int32Ty; 5701 VTy = llvm::VectorType::get(Int16Ty, 4); 5702 llvm::Type *Tys[2] = { Ty, VTy }; 5703 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5704 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5705 return Builder.CreateTrunc(Ops[0], Int16Ty); 5706 } 5707 case NEON::BI__builtin_neon_vmaxvq_s8: { 5708 Int = Intrinsic::aarch64_neon_smaxv; 5709 Ty = Int32Ty; 5710 VTy = llvm::VectorType::get(Int8Ty, 16); 5711 llvm::Type *Tys[2] = { Ty, VTy }; 5712 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5713 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5714 return Builder.CreateTrunc(Ops[0], Int8Ty); 5715 } 5716 case NEON::BI__builtin_neon_vmaxvq_s16: { 5717 Int = Intrinsic::aarch64_neon_smaxv; 5718 Ty = Int32Ty; 5719 VTy = llvm::VectorType::get(Int16Ty, 8); 5720 llvm::Type *Tys[2] = { Ty, VTy }; 5721 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5722 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); 5723 return Builder.CreateTrunc(Ops[0], Int16Ty); 5724 } 5725 case NEON::BI__builtin_neon_vminv_u8: { 5726 Int = Intrinsic::aarch64_neon_uminv; 5727 Ty = Int32Ty; 5728 VTy = llvm::VectorType::get(Int8Ty, 8); 5729 llvm::Type *Tys[2] = { Ty, VTy }; 5730 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5731 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5732 return Builder.CreateTrunc(Ops[0], Int8Ty); 5733 } 5734 case NEON::BI__builtin_neon_vminv_u16: { 5735 Int = Intrinsic::aarch64_neon_uminv; 5736 Ty = Int32Ty; 5737 VTy = llvm::VectorType::get(Int16Ty, 4); 5738 llvm::Type *Tys[2] = { Ty, VTy }; 5739 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5740 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5741 return Builder.CreateTrunc(Ops[0], Int16Ty); 5742 } 5743 case NEON::BI__builtin_neon_vminvq_u8: { 5744 Int = Intrinsic::aarch64_neon_uminv; 5745 Ty = Int32Ty; 5746 VTy = llvm::VectorType::get(Int8Ty, 16); 5747 llvm::Type *Tys[2] = { Ty, VTy }; 5748 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5749 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5750 return Builder.CreateTrunc(Ops[0], Int8Ty); 5751 } 5752 case NEON::BI__builtin_neon_vminvq_u16: { 5753 Int = Intrinsic::aarch64_neon_uminv; 5754 Ty = Int32Ty; 5755 VTy = llvm::VectorType::get(Int16Ty, 8); 5756 llvm::Type *Tys[2] = { Ty, VTy }; 5757 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5758 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5759 return Builder.CreateTrunc(Ops[0], Int16Ty); 5760 } 5761 case NEON::BI__builtin_neon_vminv_s8: { 5762 Int = Intrinsic::aarch64_neon_sminv; 5763 Ty = Int32Ty; 5764 VTy = llvm::VectorType::get(Int8Ty, 8); 5765 llvm::Type *Tys[2] = { Ty, VTy }; 5766 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5767 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5768 return Builder.CreateTrunc(Ops[0], Int8Ty); 5769 } 5770 case NEON::BI__builtin_neon_vminv_s16: { 5771 Int = Intrinsic::aarch64_neon_sminv; 5772 Ty = Int32Ty; 5773 VTy = llvm::VectorType::get(Int16Ty, 4); 5774 llvm::Type *Tys[2] = { Ty, VTy }; 5775 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5776 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5777 return Builder.CreateTrunc(Ops[0], Int16Ty); 5778 } 5779 case NEON::BI__builtin_neon_vminvq_s8: { 5780 Int = Intrinsic::aarch64_neon_sminv; 5781 Ty = Int32Ty; 5782 VTy = llvm::VectorType::get(Int8Ty, 16); 5783 llvm::Type *Tys[2] = { Ty, VTy }; 5784 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5785 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5786 return Builder.CreateTrunc(Ops[0], Int8Ty); 5787 } 5788 case NEON::BI__builtin_neon_vminvq_s16: { 5789 Int = Intrinsic::aarch64_neon_sminv; 5790 Ty = Int32Ty; 5791 VTy = llvm::VectorType::get(Int16Ty, 8); 5792 llvm::Type *Tys[2] = { Ty, VTy }; 5793 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5794 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); 5795 return Builder.CreateTrunc(Ops[0], Int16Ty); 5796 } 5797 case NEON::BI__builtin_neon_vmul_n_f64: { 5798 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); 5799 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); 5800 return Builder.CreateFMul(Ops[0], RHS); 5801 } 5802 case NEON::BI__builtin_neon_vaddlv_u8: { 5803 Int = Intrinsic::aarch64_neon_uaddlv; 5804 Ty = Int32Ty; 5805 VTy = llvm::VectorType::get(Int8Ty, 8); 5806 llvm::Type *Tys[2] = { Ty, VTy }; 5807 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5808 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5809 return Builder.CreateTrunc(Ops[0], Int16Ty); 5810 } 5811 case NEON::BI__builtin_neon_vaddlv_u16: { 5812 Int = Intrinsic::aarch64_neon_uaddlv; 5813 Ty = Int32Ty; 5814 VTy = llvm::VectorType::get(Int16Ty, 4); 5815 llvm::Type *Tys[2] = { Ty, VTy }; 5816 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5817 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5818 } 5819 case NEON::BI__builtin_neon_vaddlvq_u8: { 5820 Int = Intrinsic::aarch64_neon_uaddlv; 5821 Ty = Int32Ty; 5822 VTy = llvm::VectorType::get(Int8Ty, 16); 5823 llvm::Type *Tys[2] = { Ty, VTy }; 5824 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5825 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5826 return Builder.CreateTrunc(Ops[0], Int16Ty); 5827 } 5828 case NEON::BI__builtin_neon_vaddlvq_u16: { 5829 Int = Intrinsic::aarch64_neon_uaddlv; 5830 Ty = Int32Ty; 5831 VTy = llvm::VectorType::get(Int16Ty, 8); 5832 llvm::Type *Tys[2] = { Ty, VTy }; 5833 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5834 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5835 } 5836 case NEON::BI__builtin_neon_vaddlv_s8: { 5837 Int = Intrinsic::aarch64_neon_saddlv; 5838 Ty = Int32Ty; 5839 VTy = llvm::VectorType::get(Int8Ty, 8); 5840 llvm::Type *Tys[2] = { Ty, VTy }; 5841 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5842 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5843 return Builder.CreateTrunc(Ops[0], Int16Ty); 5844 } 5845 case NEON::BI__builtin_neon_vaddlv_s16: { 5846 Int = Intrinsic::aarch64_neon_saddlv; 5847 Ty = Int32Ty; 5848 VTy = llvm::VectorType::get(Int16Ty, 4); 5849 llvm::Type *Tys[2] = { Ty, VTy }; 5850 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5851 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5852 } 5853 case NEON::BI__builtin_neon_vaddlvq_s8: { 5854 Int = Intrinsic::aarch64_neon_saddlv; 5855 Ty = Int32Ty; 5856 VTy = llvm::VectorType::get(Int8Ty, 16); 5857 llvm::Type *Tys[2] = { Ty, VTy }; 5858 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5859 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5860 return Builder.CreateTrunc(Ops[0], Int16Ty); 5861 } 5862 case NEON::BI__builtin_neon_vaddlvq_s16: { 5863 Int = Intrinsic::aarch64_neon_saddlv; 5864 Ty = Int32Ty; 5865 VTy = llvm::VectorType::get(Int16Ty, 8); 5866 llvm::Type *Tys[2] = { Ty, VTy }; 5867 Ops.push_back(EmitScalarExpr(E->getArg(0))); 5868 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); 5869 } 5870 case NEON::BI__builtin_neon_vsri_n_v: 5871 case NEON::BI__builtin_neon_vsriq_n_v: { 5872 Int = Intrinsic::aarch64_neon_vsri; 5873 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5874 return EmitNeonCall(Intrin, Ops, "vsri_n"); 5875 } 5876 case NEON::BI__builtin_neon_vsli_n_v: 5877 case NEON::BI__builtin_neon_vsliq_n_v: { 5878 Int = Intrinsic::aarch64_neon_vsli; 5879 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); 5880 return EmitNeonCall(Intrin, Ops, "vsli_n"); 5881 } 5882 case NEON::BI__builtin_neon_vsra_n_v: 5883 case NEON::BI__builtin_neon_vsraq_n_v: 5884 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5885 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); 5886 return Builder.CreateAdd(Ops[0], Ops[1]); 5887 case NEON::BI__builtin_neon_vrsra_n_v: 5888 case NEON::BI__builtin_neon_vrsraq_n_v: { 5889 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; 5890 SmallVector<llvm::Value*,2> TmpOps; 5891 TmpOps.push_back(Ops[1]); 5892 TmpOps.push_back(Ops[2]); 5893 Function* F = CGM.getIntrinsic(Int, Ty); 5894 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); 5895 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); 5896 return Builder.CreateAdd(Ops[0], tmp); 5897 } 5898 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence 5899 // of an Align parameter here. 5900 case NEON::BI__builtin_neon_vld1_x2_v: 5901 case NEON::BI__builtin_neon_vld1q_x2_v: 5902 case NEON::BI__builtin_neon_vld1_x3_v: 5903 case NEON::BI__builtin_neon_vld1q_x3_v: 5904 case NEON::BI__builtin_neon_vld1_x4_v: 5905 case NEON::BI__builtin_neon_vld1q_x4_v: { 5906 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5907 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5908 llvm::Type *Tys[2] = { VTy, PTy }; 5909 unsigned Int; 5910 switch (BuiltinID) { 5911 case NEON::BI__builtin_neon_vld1_x2_v: 5912 case NEON::BI__builtin_neon_vld1q_x2_v: 5913 Int = Intrinsic::aarch64_neon_ld1x2; 5914 break; 5915 case NEON::BI__builtin_neon_vld1_x3_v: 5916 case NEON::BI__builtin_neon_vld1q_x3_v: 5917 Int = Intrinsic::aarch64_neon_ld1x3; 5918 break; 5919 case NEON::BI__builtin_neon_vld1_x4_v: 5920 case NEON::BI__builtin_neon_vld1q_x4_v: 5921 Int = Intrinsic::aarch64_neon_ld1x4; 5922 break; 5923 } 5924 Function *F = CGM.getIntrinsic(Int, Tys); 5925 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); 5926 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5927 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5928 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5929 } 5930 case NEON::BI__builtin_neon_vst1_x2_v: 5931 case NEON::BI__builtin_neon_vst1q_x2_v: 5932 case NEON::BI__builtin_neon_vst1_x3_v: 5933 case NEON::BI__builtin_neon_vst1q_x3_v: 5934 case NEON::BI__builtin_neon_vst1_x4_v: 5935 case NEON::BI__builtin_neon_vst1q_x4_v: { 5936 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); 5937 llvm::Type *Tys[2] = { VTy, PTy }; 5938 unsigned Int; 5939 switch (BuiltinID) { 5940 case NEON::BI__builtin_neon_vst1_x2_v: 5941 case NEON::BI__builtin_neon_vst1q_x2_v: 5942 Int = Intrinsic::aarch64_neon_st1x2; 5943 break; 5944 case NEON::BI__builtin_neon_vst1_x3_v: 5945 case NEON::BI__builtin_neon_vst1q_x3_v: 5946 Int = Intrinsic::aarch64_neon_st1x3; 5947 break; 5948 case NEON::BI__builtin_neon_vst1_x4_v: 5949 case NEON::BI__builtin_neon_vst1q_x4_v: 5950 Int = Intrinsic::aarch64_neon_st1x4; 5951 break; 5952 } 5953 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); 5954 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); 5955 } 5956 case NEON::BI__builtin_neon_vld1_v: 5957 case NEON::BI__builtin_neon_vld1q_v: 5958 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5959 return Builder.CreateDefaultAlignedLoad(Ops[0]); 5960 case NEON::BI__builtin_neon_vst1_v: 5961 case NEON::BI__builtin_neon_vst1q_v: 5962 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); 5963 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); 5964 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5965 case NEON::BI__builtin_neon_vld1_lane_v: 5966 case NEON::BI__builtin_neon_vld1q_lane_v: 5967 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5968 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5969 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5970 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5971 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); 5972 case NEON::BI__builtin_neon_vld1_dup_v: 5973 case NEON::BI__builtin_neon_vld1q_dup_v: { 5974 Value *V = UndefValue::get(Ty); 5975 Ty = llvm::PointerType::getUnqual(VTy->getElementType()); 5976 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 5977 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); 5978 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); 5979 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); 5980 return EmitNeonSplat(Ops[0], CI); 5981 } 5982 case NEON::BI__builtin_neon_vst1_lane_v: 5983 case NEON::BI__builtin_neon_vst1q_lane_v: 5984 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 5985 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); 5986 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 5987 return Builder.CreateDefaultAlignedStore(Ops[1], 5988 Builder.CreateBitCast(Ops[0], Ty)); 5989 case NEON::BI__builtin_neon_vld2_v: 5990 case NEON::BI__builtin_neon_vld2q_v: { 5991 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 5992 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 5993 llvm::Type *Tys[2] = { VTy, PTy }; 5994 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); 5995 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 5996 Ops[0] = Builder.CreateBitCast(Ops[0], 5997 llvm::PointerType::getUnqual(Ops[1]->getType())); 5998 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 5999 } 6000 case NEON::BI__builtin_neon_vld3_v: 6001 case NEON::BI__builtin_neon_vld3q_v: { 6002 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6003 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6004 llvm::Type *Tys[2] = { VTy, PTy }; 6005 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); 6006 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6007 Ops[0] = Builder.CreateBitCast(Ops[0], 6008 llvm::PointerType::getUnqual(Ops[1]->getType())); 6009 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6010 } 6011 case NEON::BI__builtin_neon_vld4_v: 6012 case NEON::BI__builtin_neon_vld4q_v: { 6013 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); 6014 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6015 llvm::Type *Tys[2] = { VTy, PTy }; 6016 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); 6017 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6018 Ops[0] = Builder.CreateBitCast(Ops[0], 6019 llvm::PointerType::getUnqual(Ops[1]->getType())); 6020 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6021 } 6022 case NEON::BI__builtin_neon_vld2_dup_v: 6023 case NEON::BI__builtin_neon_vld2q_dup_v: { 6024 llvm::Type *PTy = 6025 llvm::PointerType::getUnqual(VTy->getElementType()); 6026 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6027 llvm::Type *Tys[2] = { VTy, PTy }; 6028 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); 6029 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); 6030 Ops[0] = Builder.CreateBitCast(Ops[0], 6031 llvm::PointerType::getUnqual(Ops[1]->getType())); 6032 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6033 } 6034 case NEON::BI__builtin_neon_vld3_dup_v: 6035 case NEON::BI__builtin_neon_vld3q_dup_v: { 6036 llvm::Type *PTy = 6037 llvm::PointerType::getUnqual(VTy->getElementType()); 6038 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6039 llvm::Type *Tys[2] = { VTy, PTy }; 6040 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); 6041 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); 6042 Ops[0] = Builder.CreateBitCast(Ops[0], 6043 llvm::PointerType::getUnqual(Ops[1]->getType())); 6044 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6045 } 6046 case NEON::BI__builtin_neon_vld4_dup_v: 6047 case NEON::BI__builtin_neon_vld4q_dup_v: { 6048 llvm::Type *PTy = 6049 llvm::PointerType::getUnqual(VTy->getElementType()); 6050 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); 6051 llvm::Type *Tys[2] = { VTy, PTy }; 6052 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); 6053 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); 6054 Ops[0] = Builder.CreateBitCast(Ops[0], 6055 llvm::PointerType::getUnqual(Ops[1]->getType())); 6056 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6057 } 6058 case NEON::BI__builtin_neon_vld2_lane_v: 6059 case NEON::BI__builtin_neon_vld2q_lane_v: { 6060 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6061 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); 6062 Ops.push_back(Ops[1]); 6063 Ops.erase(Ops.begin()+1); 6064 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6065 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6066 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6067 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); 6068 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6069 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6070 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6071 } 6072 case NEON::BI__builtin_neon_vld3_lane_v: 6073 case NEON::BI__builtin_neon_vld3q_lane_v: { 6074 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6075 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); 6076 Ops.push_back(Ops[1]); 6077 Ops.erase(Ops.begin()+1); 6078 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6079 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6080 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6081 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6082 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); 6083 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6084 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6085 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6086 } 6087 case NEON::BI__builtin_neon_vld4_lane_v: 6088 case NEON::BI__builtin_neon_vld4q_lane_v: { 6089 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; 6090 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); 6091 Ops.push_back(Ops[1]); 6092 Ops.erase(Ops.begin()+1); 6093 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6094 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6095 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); 6096 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); 6097 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); 6098 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); 6099 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); 6100 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); 6101 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6102 } 6103 case NEON::BI__builtin_neon_vst2_v: 6104 case NEON::BI__builtin_neon_vst2q_v: { 6105 Ops.push_back(Ops[0]); 6106 Ops.erase(Ops.begin()); 6107 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; 6108 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), 6109 Ops, ""); 6110 } 6111 case NEON::BI__builtin_neon_vst2_lane_v: 6112 case NEON::BI__builtin_neon_vst2q_lane_v: { 6113 Ops.push_back(Ops[0]); 6114 Ops.erase(Ops.begin()); 6115 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); 6116 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6117 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), 6118 Ops, ""); 6119 } 6120 case NEON::BI__builtin_neon_vst3_v: 6121 case NEON::BI__builtin_neon_vst3q_v: { 6122 Ops.push_back(Ops[0]); 6123 Ops.erase(Ops.begin()); 6124 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; 6125 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), 6126 Ops, ""); 6127 } 6128 case NEON::BI__builtin_neon_vst3_lane_v: 6129 case NEON::BI__builtin_neon_vst3q_lane_v: { 6130 Ops.push_back(Ops[0]); 6131 Ops.erase(Ops.begin()); 6132 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); 6133 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6134 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), 6135 Ops, ""); 6136 } 6137 case NEON::BI__builtin_neon_vst4_v: 6138 case NEON::BI__builtin_neon_vst4q_v: { 6139 Ops.push_back(Ops[0]); 6140 Ops.erase(Ops.begin()); 6141 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; 6142 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), 6143 Ops, ""); 6144 } 6145 case NEON::BI__builtin_neon_vst4_lane_v: 6146 case NEON::BI__builtin_neon_vst4q_lane_v: { 6147 Ops.push_back(Ops[0]); 6148 Ops.erase(Ops.begin()); 6149 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); 6150 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; 6151 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), 6152 Ops, ""); 6153 } 6154 case NEON::BI__builtin_neon_vtrn_v: 6155 case NEON::BI__builtin_neon_vtrnq_v: { 6156 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6157 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6158 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6159 Value *SV = nullptr; 6160 6161 for (unsigned vi = 0; vi != 2; ++vi) { 6162 SmallVector<Constant*, 16> Indices; 6163 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6164 Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); 6165 Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); 6166 } 6167 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6168 SV = llvm::ConstantVector::get(Indices); 6169 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); 6170 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6171 } 6172 return SV; 6173 } 6174 case NEON::BI__builtin_neon_vuzp_v: 6175 case NEON::BI__builtin_neon_vuzpq_v: { 6176 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6177 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6178 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6179 Value *SV = nullptr; 6180 6181 for (unsigned vi = 0; vi != 2; ++vi) { 6182 SmallVector<Constant*, 16> Indices; 6183 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) 6184 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); 6185 6186 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6187 SV = llvm::ConstantVector::get(Indices); 6188 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); 6189 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6190 } 6191 return SV; 6192 } 6193 case NEON::BI__builtin_neon_vzip_v: 6194 case NEON::BI__builtin_neon_vzipq_v: { 6195 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); 6196 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); 6197 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); 6198 Value *SV = nullptr; 6199 6200 for (unsigned vi = 0; vi != 2; ++vi) { 6201 SmallVector<Constant*, 16> Indices; 6202 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { 6203 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); 6204 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); 6205 } 6206 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); 6207 SV = llvm::ConstantVector::get(Indices); 6208 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); 6209 SV = Builder.CreateDefaultAlignedStore(SV, Addr); 6210 } 6211 return SV; 6212 } 6213 case NEON::BI__builtin_neon_vqtbl1q_v: { 6214 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), 6215 Ops, "vtbl1"); 6216 } 6217 case NEON::BI__builtin_neon_vqtbl2q_v: { 6218 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), 6219 Ops, "vtbl2"); 6220 } 6221 case NEON::BI__builtin_neon_vqtbl3q_v: { 6222 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), 6223 Ops, "vtbl3"); 6224 } 6225 case NEON::BI__builtin_neon_vqtbl4q_v: { 6226 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), 6227 Ops, "vtbl4"); 6228 } 6229 case NEON::BI__builtin_neon_vqtbx1q_v: { 6230 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), 6231 Ops, "vtbx1"); 6232 } 6233 case NEON::BI__builtin_neon_vqtbx2q_v: { 6234 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), 6235 Ops, "vtbx2"); 6236 } 6237 case NEON::BI__builtin_neon_vqtbx3q_v: { 6238 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), 6239 Ops, "vtbx3"); 6240 } 6241 case NEON::BI__builtin_neon_vqtbx4q_v: { 6242 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), 6243 Ops, "vtbx4"); 6244 } 6245 case NEON::BI__builtin_neon_vsqadd_v: 6246 case NEON::BI__builtin_neon_vsqaddq_v: { 6247 Int = Intrinsic::aarch64_neon_usqadd; 6248 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); 6249 } 6250 case NEON::BI__builtin_neon_vuqadd_v: 6251 case NEON::BI__builtin_neon_vuqaddq_v: { 6252 Int = Intrinsic::aarch64_neon_suqadd; 6253 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); 6254 } 6255 } 6256 } 6257 6258 llvm::Value *CodeGenFunction:: 6259 BuildVector(ArrayRef<llvm::Value*> Ops) { 6260 assert((Ops.size() & (Ops.size() - 1)) == 0 && 6261 "Not a power-of-two sized vector!"); 6262 bool AllConstants = true; 6263 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) 6264 AllConstants &= isa<Constant>(Ops[i]); 6265 6266 // If this is a constant vector, create a ConstantVector. 6267 if (AllConstants) { 6268 SmallVector<llvm::Constant*, 16> CstOps; 6269 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6270 CstOps.push_back(cast<Constant>(Ops[i])); 6271 return llvm::ConstantVector::get(CstOps); 6272 } 6273 6274 // Otherwise, insertelement the values to build the vector. 6275 Value *Result = 6276 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); 6277 6278 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 6279 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); 6280 6281 return Result; 6282 } 6283 6284 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, 6285 const CallExpr *E) { 6286 if (BuiltinID == X86::BI__builtin_ms_va_start || 6287 BuiltinID == X86::BI__builtin_ms_va_end) 6288 return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), 6289 BuiltinID == X86::BI__builtin_ms_va_start); 6290 if (BuiltinID == X86::BI__builtin_ms_va_copy) { 6291 // Lower this manually. We can't reliably determine whether or not any 6292 // given va_copy() is for a Win64 va_list from the calling convention 6293 // alone, because it's legal to do this from a System V ABI function. 6294 // With opaque pointer types, we won't have enough information in LLVM 6295 // IR to determine this from the argument types, either. Best to do it 6296 // now, while we have enough information. 6297 Address DestAddr = EmitMSVAListRef(E->getArg(0)); 6298 Address SrcAddr = EmitMSVAListRef(E->getArg(1)); 6299 6300 llvm::Type *BPP = Int8PtrPtrTy; 6301 6302 DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"), 6303 DestAddr.getAlignment()); 6304 SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"), 6305 SrcAddr.getAlignment()); 6306 6307 Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val"); 6308 return Builder.CreateStore(ArgPtr, DestAddr); 6309 } 6310 6311 SmallVector<Value*, 4> Ops; 6312 6313 // Find out if any arguments are required to be integer constant expressions. 6314 unsigned ICEArguments = 0; 6315 ASTContext::GetBuiltinTypeError Error; 6316 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); 6317 assert(Error == ASTContext::GE_None && "Should not codegen an error"); 6318 6319 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { 6320 // If this is a normal argument, just emit it as a scalar. 6321 if ((ICEArguments & (1 << i)) == 0) { 6322 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6323 continue; 6324 } 6325 6326 // If this is required to be a constant, constant fold it so that we know 6327 // that the generated intrinsic gets a ConstantInt. 6328 llvm::APSInt Result; 6329 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); 6330 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; 6331 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); 6332 } 6333 6334 switch (BuiltinID) { 6335 default: return nullptr; 6336 case X86::BI__builtin_cpu_supports: { 6337 const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts(); 6338 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString(); 6339 6340 // TODO: When/if this becomes more than x86 specific then use a TargetInfo 6341 // based mapping. 6342 // Processor features and mapping to processor feature value. 6343 enum X86Features { 6344 CMOV = 0, 6345 MMX, 6346 POPCNT, 6347 SSE, 6348 SSE2, 6349 SSE3, 6350 SSSE3, 6351 SSE4_1, 6352 SSE4_2, 6353 AVX, 6354 AVX2, 6355 SSE4_A, 6356 FMA4, 6357 XOP, 6358 FMA, 6359 AVX512F, 6360 BMI, 6361 BMI2, 6362 AES, 6363 PCLMUL, 6364 AVX512VL, 6365 AVX512BW, 6366 AVX512DQ, 6367 AVX512CD, 6368 AVX512ER, 6369 AVX512PF, 6370 AVX512VBMI, 6371 AVX512IFMA, 6372 MAX 6373 }; 6374 6375 X86Features Feature = StringSwitch<X86Features>(FeatureStr) 6376 .Case("cmov", X86Features::CMOV) 6377 .Case("mmx", X86Features::MMX) 6378 .Case("popcnt", X86Features::POPCNT) 6379 .Case("sse", X86Features::SSE) 6380 .Case("sse2", X86Features::SSE2) 6381 .Case("sse3", X86Features::SSE3) 6382 .Case("ssse3", X86Features::SSSE3) 6383 .Case("sse4.1", X86Features::SSE4_1) 6384 .Case("sse4.2", X86Features::SSE4_2) 6385 .Case("avx", X86Features::AVX) 6386 .Case("avx2", X86Features::AVX2) 6387 .Case("sse4a", X86Features::SSE4_A) 6388 .Case("fma4", X86Features::FMA4) 6389 .Case("xop", X86Features::XOP) 6390 .Case("fma", X86Features::FMA) 6391 .Case("avx512f", X86Features::AVX512F) 6392 .Case("bmi", X86Features::BMI) 6393 .Case("bmi2", X86Features::BMI2) 6394 .Case("aes", X86Features::AES) 6395 .Case("pclmul", X86Features::PCLMUL) 6396 .Case("avx512vl", X86Features::AVX512VL) 6397 .Case("avx512bw", X86Features::AVX512BW) 6398 .Case("avx512dq", X86Features::AVX512DQ) 6399 .Case("avx512cd", X86Features::AVX512CD) 6400 .Case("avx512er", X86Features::AVX512ER) 6401 .Case("avx512pf", X86Features::AVX512PF) 6402 .Case("avx512vbmi", X86Features::AVX512VBMI) 6403 .Case("avx512ifma", X86Features::AVX512IFMA) 6404 .Default(X86Features::MAX); 6405 assert(Feature != X86Features::MAX && "Invalid feature!"); 6406 6407 // Matching the struct layout from the compiler-rt/libgcc structure that is 6408 // filled in: 6409 // unsigned int __cpu_vendor; 6410 // unsigned int __cpu_type; 6411 // unsigned int __cpu_subtype; 6412 // unsigned int __cpu_features[1]; 6413 llvm::Type *STy = llvm::StructType::get( 6414 Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr); 6415 6416 // Grab the global __cpu_model. 6417 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model"); 6418 6419 // Grab the first (0th) element from the field __cpu_features off of the 6420 // global in the struct STy. 6421 Value *Idxs[] = { 6422 ConstantInt::get(Int32Ty, 0), 6423 ConstantInt::get(Int32Ty, 3), 6424 ConstantInt::get(Int32Ty, 0) 6425 }; 6426 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs); 6427 Value *Features = Builder.CreateAlignedLoad(CpuFeatures, 6428 CharUnits::fromQuantity(4)); 6429 6430 // Check the value of the bit corresponding to the feature requested. 6431 Value *Bitset = Builder.CreateAnd( 6432 Features, llvm::ConstantInt::get(Int32Ty, 1ULL << Feature)); 6433 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); 6434 } 6435 case X86::BI_mm_prefetch: { 6436 Value *Address = Ops[0]; 6437 Value *RW = ConstantInt::get(Int32Ty, 0); 6438 Value *Locality = Ops[1]; 6439 Value *Data = ConstantInt::get(Int32Ty, 1); 6440 Value *F = CGM.getIntrinsic(Intrinsic::prefetch); 6441 return Builder.CreateCall(F, {Address, RW, Locality, Data}); 6442 } 6443 case X86::BI__builtin_ia32_undef128: 6444 case X86::BI__builtin_ia32_undef256: 6445 case X86::BI__builtin_ia32_undef512: 6446 return UndefValue::get(ConvertType(E->getType())); 6447 case X86::BI__builtin_ia32_vec_init_v8qi: 6448 case X86::BI__builtin_ia32_vec_init_v4hi: 6449 case X86::BI__builtin_ia32_vec_init_v2si: 6450 return Builder.CreateBitCast(BuildVector(Ops), 6451 llvm::Type::getX86_MMXTy(getLLVMContext())); 6452 case X86::BI__builtin_ia32_vec_ext_v2si: 6453 return Builder.CreateExtractElement(Ops[0], 6454 llvm::ConstantInt::get(Ops[1]->getType(), 0)); 6455 case X86::BI__builtin_ia32_ldmxcsr: { 6456 Address Tmp = CreateMemTemp(E->getArg(0)->getType()); 6457 Builder.CreateStore(Ops[0], Tmp); 6458 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), 6459 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6460 } 6461 case X86::BI__builtin_ia32_stmxcsr: { 6462 Address Tmp = CreateMemTemp(E->getType()); 6463 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), 6464 Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy)); 6465 return Builder.CreateLoad(Tmp, "stmxcsr"); 6466 } 6467 case X86::BI__builtin_ia32_xsave: 6468 case X86::BI__builtin_ia32_xsave64: 6469 case X86::BI__builtin_ia32_xrstor: 6470 case X86::BI__builtin_ia32_xrstor64: 6471 case X86::BI__builtin_ia32_xsaveopt: 6472 case X86::BI__builtin_ia32_xsaveopt64: 6473 case X86::BI__builtin_ia32_xrstors: 6474 case X86::BI__builtin_ia32_xrstors64: 6475 case X86::BI__builtin_ia32_xsavec: 6476 case X86::BI__builtin_ia32_xsavec64: 6477 case X86::BI__builtin_ia32_xsaves: 6478 case X86::BI__builtin_ia32_xsaves64: { 6479 Intrinsic::ID ID; 6480 #define INTRINSIC_X86_XSAVE_ID(NAME) \ 6481 case X86::BI__builtin_ia32_##NAME: \ 6482 ID = Intrinsic::x86_##NAME; \ 6483 break 6484 switch (BuiltinID) { 6485 default: llvm_unreachable("Unsupported intrinsic!"); 6486 INTRINSIC_X86_XSAVE_ID(xsave); 6487 INTRINSIC_X86_XSAVE_ID(xsave64); 6488 INTRINSIC_X86_XSAVE_ID(xrstor); 6489 INTRINSIC_X86_XSAVE_ID(xrstor64); 6490 INTRINSIC_X86_XSAVE_ID(xsaveopt); 6491 INTRINSIC_X86_XSAVE_ID(xsaveopt64); 6492 INTRINSIC_X86_XSAVE_ID(xrstors); 6493 INTRINSIC_X86_XSAVE_ID(xrstors64); 6494 INTRINSIC_X86_XSAVE_ID(xsavec); 6495 INTRINSIC_X86_XSAVE_ID(xsavec64); 6496 INTRINSIC_X86_XSAVE_ID(xsaves); 6497 INTRINSIC_X86_XSAVE_ID(xsaves64); 6498 } 6499 #undef INTRINSIC_X86_XSAVE_ID 6500 Value *Mhi = Builder.CreateTrunc( 6501 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); 6502 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); 6503 Ops[1] = Mhi; 6504 Ops.push_back(Mlo); 6505 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); 6506 } 6507 case X86::BI__builtin_ia32_storehps: 6508 case X86::BI__builtin_ia32_storelps: { 6509 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); 6510 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); 6511 6512 // cast val v2i64 6513 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); 6514 6515 // extract (0, 1) 6516 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; 6517 llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); 6518 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); 6519 6520 // cast pointer to i64 & store 6521 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); 6522 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); 6523 } 6524 case X86::BI__builtin_ia32_palignr128: 6525 case X86::BI__builtin_ia32_palignr256: { 6526 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); 6527 6528 unsigned NumElts = 6529 cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); 6530 assert(NumElts % 16 == 0); 6531 unsigned NumLanes = NumElts / 16; 6532 unsigned NumLaneElts = NumElts / NumLanes; 6533 6534 // If palignr is shifting the pair of vectors more than the size of two 6535 // lanes, emit zero. 6536 if (ShiftVal >= (2 * NumLaneElts)) 6537 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6538 6539 // If palignr is shifting the pair of input vectors more than one lane, 6540 // but less than two lanes, convert to shifting in zeroes. 6541 if (ShiftVal > NumLaneElts) { 6542 ShiftVal -= NumLaneElts; 6543 Ops[1] = Ops[0]; 6544 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); 6545 } 6546 6547 uint32_t Indices[32]; 6548 // 256-bit palignr operates on 128-bit lanes so we need to handle that 6549 for (unsigned l = 0; l != NumElts; l += NumLaneElts) { 6550 for (unsigned i = 0; i != NumLaneElts; ++i) { 6551 unsigned Idx = ShiftVal + i; 6552 if (Idx >= NumLaneElts) 6553 Idx += NumElts - NumLaneElts; // End of lane, switch operand. 6554 Indices[l + i] = Idx + l; 6555 } 6556 } 6557 6558 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), 6559 makeArrayRef(Indices, NumElts)); 6560 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); 6561 } 6562 case X86::BI__builtin_ia32_pslldqi256: { 6563 // Shift value is in bits so divide by 8. 6564 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6565 6566 // If pslldq is shifting the vector more than 15 bytes, emit zero. 6567 if (shiftVal >= 16) 6568 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6569 6570 uint32_t Indices[32]; 6571 // 256-bit pslldq operates on 128-bit lanes so we need to handle that 6572 for (unsigned l = 0; l != 32; l += 16) { 6573 for (unsigned i = 0; i != 16; ++i) { 6574 unsigned Idx = 32 + i - shiftVal; 6575 if (Idx < 32) Idx -= 16; // end of lane, switch operand. 6576 Indices[l + i] = Idx + l; 6577 } 6578 } 6579 6580 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6581 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6582 Value *Zero = llvm::Constant::getNullValue(VecTy); 6583 6584 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 6585 SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); 6586 llvm::Type *ResultType = ConvertType(E->getType()); 6587 return Builder.CreateBitCast(SV, ResultType, "cast"); 6588 } 6589 case X86::BI__builtin_ia32_psrldqi256: { 6590 // Shift value is in bits so divide by 8. 6591 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; 6592 6593 // If psrldq is shifting the vector more than 15 bytes, emit zero. 6594 if (shiftVal >= 16) 6595 return llvm::Constant::getNullValue(ConvertType(E->getType())); 6596 6597 uint32_t Indices[32]; 6598 // 256-bit psrldq operates on 128-bit lanes so we need to handle that 6599 for (unsigned l = 0; l != 32; l += 16) { 6600 for (unsigned i = 0; i != 16; ++i) { 6601 unsigned Idx = i + shiftVal; 6602 if (Idx >= 16) Idx += 16; // end of lane, switch operand. 6603 Indices[l + i] = Idx + l; 6604 } 6605 } 6606 6607 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); 6608 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); 6609 Value *Zero = llvm::Constant::getNullValue(VecTy); 6610 6611 Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices); 6612 SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); 6613 llvm::Type *ResultType = ConvertType(E->getType()); 6614 return Builder.CreateBitCast(SV, ResultType, "cast"); 6615 } 6616 case X86::BI__builtin_ia32_movntps: 6617 case X86::BI__builtin_ia32_movntps256: 6618 case X86::BI__builtin_ia32_movntpd: 6619 case X86::BI__builtin_ia32_movntpd256: 6620 case X86::BI__builtin_ia32_movntdq: 6621 case X86::BI__builtin_ia32_movntdq256: 6622 case X86::BI__builtin_ia32_movnti: 6623 case X86::BI__builtin_ia32_movnti64: { 6624 llvm::MDNode *Node = llvm::MDNode::get( 6625 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); 6626 6627 // Convert the type of the pointer to a pointer to the stored type. 6628 Value *BC = Builder.CreateBitCast(Ops[0], 6629 llvm::PointerType::getUnqual(Ops[1]->getType()), 6630 "cast"); 6631 StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); 6632 SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); 6633 6634 // If the operand is an integer, we can't assume alignment. Otherwise, 6635 // assume natural alignment. 6636 QualType ArgTy = E->getArg(1)->getType(); 6637 unsigned Align; 6638 if (ArgTy->isIntegerType()) 6639 Align = 1; 6640 else 6641 Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); 6642 SI->setAlignment(Align); 6643 return SI; 6644 } 6645 // 3DNow! 6646 case X86::BI__builtin_ia32_pswapdsf: 6647 case X86::BI__builtin_ia32_pswapdsi: { 6648 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); 6649 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); 6650 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); 6651 return Builder.CreateCall(F, Ops, "pswapd"); 6652 } 6653 case X86::BI__builtin_ia32_rdrand16_step: 6654 case X86::BI__builtin_ia32_rdrand32_step: 6655 case X86::BI__builtin_ia32_rdrand64_step: 6656 case X86::BI__builtin_ia32_rdseed16_step: 6657 case X86::BI__builtin_ia32_rdseed32_step: 6658 case X86::BI__builtin_ia32_rdseed64_step: { 6659 Intrinsic::ID ID; 6660 switch (BuiltinID) { 6661 default: llvm_unreachable("Unsupported intrinsic!"); 6662 case X86::BI__builtin_ia32_rdrand16_step: 6663 ID = Intrinsic::x86_rdrand_16; 6664 break; 6665 case X86::BI__builtin_ia32_rdrand32_step: 6666 ID = Intrinsic::x86_rdrand_32; 6667 break; 6668 case X86::BI__builtin_ia32_rdrand64_step: 6669 ID = Intrinsic::x86_rdrand_64; 6670 break; 6671 case X86::BI__builtin_ia32_rdseed16_step: 6672 ID = Intrinsic::x86_rdseed_16; 6673 break; 6674 case X86::BI__builtin_ia32_rdseed32_step: 6675 ID = Intrinsic::x86_rdseed_32; 6676 break; 6677 case X86::BI__builtin_ia32_rdseed64_step: 6678 ID = Intrinsic::x86_rdseed_64; 6679 break; 6680 } 6681 6682 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); 6683 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0), 6684 Ops[0]); 6685 return Builder.CreateExtractValue(Call, 1); 6686 } 6687 // SSE comparison intrisics 6688 case X86::BI__builtin_ia32_cmpeqps: 6689 case X86::BI__builtin_ia32_cmpltps: 6690 case X86::BI__builtin_ia32_cmpleps: 6691 case X86::BI__builtin_ia32_cmpunordps: 6692 case X86::BI__builtin_ia32_cmpneqps: 6693 case X86::BI__builtin_ia32_cmpnltps: 6694 case X86::BI__builtin_ia32_cmpnleps: 6695 case X86::BI__builtin_ia32_cmpordps: 6696 case X86::BI__builtin_ia32_cmpeqss: 6697 case X86::BI__builtin_ia32_cmpltss: 6698 case X86::BI__builtin_ia32_cmpless: 6699 case X86::BI__builtin_ia32_cmpunordss: 6700 case X86::BI__builtin_ia32_cmpneqss: 6701 case X86::BI__builtin_ia32_cmpnltss: 6702 case X86::BI__builtin_ia32_cmpnless: 6703 case X86::BI__builtin_ia32_cmpordss: 6704 case X86::BI__builtin_ia32_cmpeqpd: 6705 case X86::BI__builtin_ia32_cmpltpd: 6706 case X86::BI__builtin_ia32_cmplepd: 6707 case X86::BI__builtin_ia32_cmpunordpd: 6708 case X86::BI__builtin_ia32_cmpneqpd: 6709 case X86::BI__builtin_ia32_cmpnltpd: 6710 case X86::BI__builtin_ia32_cmpnlepd: 6711 case X86::BI__builtin_ia32_cmpordpd: 6712 case X86::BI__builtin_ia32_cmpeqsd: 6713 case X86::BI__builtin_ia32_cmpltsd: 6714 case X86::BI__builtin_ia32_cmplesd: 6715 case X86::BI__builtin_ia32_cmpunordsd: 6716 case X86::BI__builtin_ia32_cmpneqsd: 6717 case X86::BI__builtin_ia32_cmpnltsd: 6718 case X86::BI__builtin_ia32_cmpnlesd: 6719 case X86::BI__builtin_ia32_cmpordsd: 6720 // These exist so that the builtin that takes an immediate can be bounds 6721 // checked by clang to avoid passing bad immediates to the backend. Since 6722 // AVX has a larger immediate than SSE we would need separate builtins to 6723 // do the different bounds checking. Rather than create a clang specific 6724 // SSE only builtin, this implements eight separate builtins to match gcc 6725 // implementation. 6726 6727 // Choose the immediate. 6728 unsigned Imm; 6729 switch (BuiltinID) { 6730 default: llvm_unreachable("Unsupported intrinsic!"); 6731 case X86::BI__builtin_ia32_cmpeqps: 6732 case X86::BI__builtin_ia32_cmpeqss: 6733 case X86::BI__builtin_ia32_cmpeqpd: 6734 case X86::BI__builtin_ia32_cmpeqsd: 6735 Imm = 0; 6736 break; 6737 case X86::BI__builtin_ia32_cmpltps: 6738 case X86::BI__builtin_ia32_cmpltss: 6739 case X86::BI__builtin_ia32_cmpltpd: 6740 case X86::BI__builtin_ia32_cmpltsd: 6741 Imm = 1; 6742 break; 6743 case X86::BI__builtin_ia32_cmpleps: 6744 case X86::BI__builtin_ia32_cmpless: 6745 case X86::BI__builtin_ia32_cmplepd: 6746 case X86::BI__builtin_ia32_cmplesd: 6747 Imm = 2; 6748 break; 6749 case X86::BI__builtin_ia32_cmpunordps: 6750 case X86::BI__builtin_ia32_cmpunordss: 6751 case X86::BI__builtin_ia32_cmpunordpd: 6752 case X86::BI__builtin_ia32_cmpunordsd: 6753 Imm = 3; 6754 break; 6755 case X86::BI__builtin_ia32_cmpneqps: 6756 case X86::BI__builtin_ia32_cmpneqss: 6757 case X86::BI__builtin_ia32_cmpneqpd: 6758 case X86::BI__builtin_ia32_cmpneqsd: 6759 Imm = 4; 6760 break; 6761 case X86::BI__builtin_ia32_cmpnltps: 6762 case X86::BI__builtin_ia32_cmpnltss: 6763 case X86::BI__builtin_ia32_cmpnltpd: 6764 case X86::BI__builtin_ia32_cmpnltsd: 6765 Imm = 5; 6766 break; 6767 case X86::BI__builtin_ia32_cmpnleps: 6768 case X86::BI__builtin_ia32_cmpnless: 6769 case X86::BI__builtin_ia32_cmpnlepd: 6770 case X86::BI__builtin_ia32_cmpnlesd: 6771 Imm = 6; 6772 break; 6773 case X86::BI__builtin_ia32_cmpordps: 6774 case X86::BI__builtin_ia32_cmpordss: 6775 case X86::BI__builtin_ia32_cmpordpd: 6776 case X86::BI__builtin_ia32_cmpordsd: 6777 Imm = 7; 6778 break; 6779 } 6780 6781 // Choose the intrinsic ID. 6782 const char *name; 6783 Intrinsic::ID ID; 6784 switch (BuiltinID) { 6785 default: llvm_unreachable("Unsupported intrinsic!"); 6786 case X86::BI__builtin_ia32_cmpeqps: 6787 case X86::BI__builtin_ia32_cmpltps: 6788 case X86::BI__builtin_ia32_cmpleps: 6789 case X86::BI__builtin_ia32_cmpunordps: 6790 case X86::BI__builtin_ia32_cmpneqps: 6791 case X86::BI__builtin_ia32_cmpnltps: 6792 case X86::BI__builtin_ia32_cmpnleps: 6793 case X86::BI__builtin_ia32_cmpordps: 6794 name = "cmpps"; 6795 ID = Intrinsic::x86_sse_cmp_ps; 6796 break; 6797 case X86::BI__builtin_ia32_cmpeqss: 6798 case X86::BI__builtin_ia32_cmpltss: 6799 case X86::BI__builtin_ia32_cmpless: 6800 case X86::BI__builtin_ia32_cmpunordss: 6801 case X86::BI__builtin_ia32_cmpneqss: 6802 case X86::BI__builtin_ia32_cmpnltss: 6803 case X86::BI__builtin_ia32_cmpnless: 6804 case X86::BI__builtin_ia32_cmpordss: 6805 name = "cmpss"; 6806 ID = Intrinsic::x86_sse_cmp_ss; 6807 break; 6808 case X86::BI__builtin_ia32_cmpeqpd: 6809 case X86::BI__builtin_ia32_cmpltpd: 6810 case X86::BI__builtin_ia32_cmplepd: 6811 case X86::BI__builtin_ia32_cmpunordpd: 6812 case X86::BI__builtin_ia32_cmpneqpd: 6813 case X86::BI__builtin_ia32_cmpnltpd: 6814 case X86::BI__builtin_ia32_cmpnlepd: 6815 case X86::BI__builtin_ia32_cmpordpd: 6816 name = "cmppd"; 6817 ID = Intrinsic::x86_sse2_cmp_pd; 6818 break; 6819 case X86::BI__builtin_ia32_cmpeqsd: 6820 case X86::BI__builtin_ia32_cmpltsd: 6821 case X86::BI__builtin_ia32_cmplesd: 6822 case X86::BI__builtin_ia32_cmpunordsd: 6823 case X86::BI__builtin_ia32_cmpneqsd: 6824 case X86::BI__builtin_ia32_cmpnltsd: 6825 case X86::BI__builtin_ia32_cmpnlesd: 6826 case X86::BI__builtin_ia32_cmpordsd: 6827 name = "cmpsd"; 6828 ID = Intrinsic::x86_sse2_cmp_sd; 6829 break; 6830 } 6831 6832 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); 6833 llvm::Function *F = CGM.getIntrinsic(ID); 6834 return Builder.CreateCall(F, Ops, name); 6835 } 6836 } 6837 6838 6839 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, 6840 const CallExpr *E) { 6841 SmallVector<Value*, 4> Ops; 6842 6843 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) 6844 Ops.push_back(EmitScalarExpr(E->getArg(i))); 6845 6846 Intrinsic::ID ID = Intrinsic::not_intrinsic; 6847 6848 switch (BuiltinID) { 6849 default: return nullptr; 6850 6851 // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we 6852 // call __builtin_readcyclecounter. 6853 case PPC::BI__builtin_ppc_get_timebase: 6854 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter)); 6855 6856 // vec_ld, vec_lvsl, vec_lvsr 6857 case PPC::BI__builtin_altivec_lvx: 6858 case PPC::BI__builtin_altivec_lvxl: 6859 case PPC::BI__builtin_altivec_lvebx: 6860 case PPC::BI__builtin_altivec_lvehx: 6861 case PPC::BI__builtin_altivec_lvewx: 6862 case PPC::BI__builtin_altivec_lvsl: 6863 case PPC::BI__builtin_altivec_lvsr: 6864 case PPC::BI__builtin_vsx_lxvd2x: 6865 case PPC::BI__builtin_vsx_lxvw4x: 6866 { 6867 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); 6868 6869 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); 6870 Ops.pop_back(); 6871 6872 switch (BuiltinID) { 6873 default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); 6874 case PPC::BI__builtin_altivec_lvx: 6875 ID = Intrinsic::ppc_altivec_lvx; 6876 break; 6877 case PPC::BI__builtin_altivec_lvxl: 6878 ID = Intrinsic::ppc_altivec_lvxl; 6879 break; 6880 case PPC::BI__builtin_altivec_lvebx: 6881 ID = Intrinsic::ppc_altivec_lvebx; 6882 break; 6883 case PPC::BI__builtin_altivec_lvehx: 6884 ID = Intrinsic::ppc_altivec_lvehx; 6885 break; 6886 case PPC::BI__builtin_altivec_lvewx: 6887 ID = Intrinsic::ppc_altivec_lvewx; 6888 break; 6889 case PPC::BI__builtin_altivec_lvsl: 6890 ID = Intrinsic::ppc_altivec_lvsl; 6891 break; 6892 case PPC::BI__builtin_altivec_lvsr: 6893 ID = Intrinsic::ppc_altivec_lvsr; 6894 break; 6895 case PPC::BI__builtin_vsx_lxvd2x: 6896 ID = Intrinsic::ppc_vsx_lxvd2x; 6897 break; 6898 case PPC::BI__builtin_vsx_lxvw4x: 6899 ID = Intrinsic::ppc_vsx_lxvw4x; 6900 break; 6901 } 6902 llvm::Function *F = CGM.getIntrinsic(ID); 6903 return Builder.CreateCall(F, Ops, ""); 6904 } 6905 6906 // vec_st 6907 case PPC::BI__builtin_altivec_stvx: 6908 case PPC::BI__builtin_altivec_stvxl: 6909 case PPC::BI__builtin_altivec_stvebx: 6910 case PPC::BI__builtin_altivec_stvehx: 6911 case PPC::BI__builtin_altivec_stvewx: 6912 case PPC::BI__builtin_vsx_stxvd2x: 6913 case PPC::BI__builtin_vsx_stxvw4x: 6914 { 6915 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); 6916 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); 6917 Ops.pop_back(); 6918 6919 switch (BuiltinID) { 6920 default: llvm_unreachable("Unsupported st intrinsic!"); 6921 case PPC::BI__builtin_altivec_stvx: 6922 ID = Intrinsic::ppc_altivec_stvx; 6923 break; 6924 case PPC::BI__builtin_altivec_stvxl: 6925 ID = Intrinsic::ppc_altivec_stvxl; 6926 break; 6927 case PPC::BI__builtin_altivec_stvebx: 6928 ID = Intrinsic::ppc_altivec_stvebx; 6929 break; 6930 case PPC::BI__builtin_altivec_stvehx: 6931 ID = Intrinsic::ppc_altivec_stvehx; 6932 break; 6933 case PPC::BI__builtin_altivec_stvewx: 6934 ID = Intrinsic::ppc_altivec_stvewx; 6935 break; 6936 case PPC::BI__builtin_vsx_stxvd2x: 6937 ID = Intrinsic::ppc_vsx_stxvd2x; 6938 break; 6939 case PPC::BI__builtin_vsx_stxvw4x: 6940 ID = Intrinsic::ppc_vsx_stxvw4x; 6941 break; 6942 } 6943 llvm::Function *F = CGM.getIntrinsic(ID); 6944 return Builder.CreateCall(F, Ops, ""); 6945 } 6946 // Square root 6947 case PPC::BI__builtin_vsx_xvsqrtsp: 6948 case PPC::BI__builtin_vsx_xvsqrtdp: { 6949 llvm::Type *ResultType = ConvertType(E->getType()); 6950 Value *X = EmitScalarExpr(E->getArg(0)); 6951 ID = Intrinsic::sqrt; 6952 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 6953 return Builder.CreateCall(F, X); 6954 } 6955 // Count leading zeros 6956 case PPC::BI__builtin_altivec_vclzb: 6957 case PPC::BI__builtin_altivec_vclzh: 6958 case PPC::BI__builtin_altivec_vclzw: 6959 case PPC::BI__builtin_altivec_vclzd: { 6960 llvm::Type *ResultType = ConvertType(E->getType()); 6961 Value *X = EmitScalarExpr(E->getArg(0)); 6962 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 6963 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 6964 return Builder.CreateCall(F, {X, Undef}); 6965 } 6966 // Copy sign 6967 case PPC::BI__builtin_vsx_xvcpsgnsp: 6968 case PPC::BI__builtin_vsx_xvcpsgndp: { 6969 llvm::Type *ResultType = ConvertType(E->getType()); 6970 Value *X = EmitScalarExpr(E->getArg(0)); 6971 Value *Y = EmitScalarExpr(E->getArg(1)); 6972 ID = Intrinsic::copysign; 6973 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 6974 return Builder.CreateCall(F, {X, Y}); 6975 } 6976 // Rounding/truncation 6977 case PPC::BI__builtin_vsx_xvrspip: 6978 case PPC::BI__builtin_vsx_xvrdpip: 6979 case PPC::BI__builtin_vsx_xvrdpim: 6980 case PPC::BI__builtin_vsx_xvrspim: 6981 case PPC::BI__builtin_vsx_xvrdpi: 6982 case PPC::BI__builtin_vsx_xvrspi: 6983 case PPC::BI__builtin_vsx_xvrdpic: 6984 case PPC::BI__builtin_vsx_xvrspic: 6985 case PPC::BI__builtin_vsx_xvrdpiz: 6986 case PPC::BI__builtin_vsx_xvrspiz: { 6987 llvm::Type *ResultType = ConvertType(E->getType()); 6988 Value *X = EmitScalarExpr(E->getArg(0)); 6989 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim || 6990 BuiltinID == PPC::BI__builtin_vsx_xvrspim) 6991 ID = Intrinsic::floor; 6992 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi || 6993 BuiltinID == PPC::BI__builtin_vsx_xvrspi) 6994 ID = Intrinsic::round; 6995 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic || 6996 BuiltinID == PPC::BI__builtin_vsx_xvrspic) 6997 ID = Intrinsic::nearbyint; 6998 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip || 6999 BuiltinID == PPC::BI__builtin_vsx_xvrspip) 7000 ID = Intrinsic::ceil; 7001 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz || 7002 BuiltinID == PPC::BI__builtin_vsx_xvrspiz) 7003 ID = Intrinsic::trunc; 7004 llvm::Function *F = CGM.getIntrinsic(ID, ResultType); 7005 return Builder.CreateCall(F, X); 7006 } 7007 7008 // Absolute value 7009 case PPC::BI__builtin_vsx_xvabsdp: 7010 case PPC::BI__builtin_vsx_xvabssp: { 7011 llvm::Type *ResultType = ConvertType(E->getType()); 7012 Value *X = EmitScalarExpr(E->getArg(0)); 7013 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7014 return Builder.CreateCall(F, X); 7015 } 7016 7017 // FMA variations 7018 case PPC::BI__builtin_vsx_xvmaddadp: 7019 case PPC::BI__builtin_vsx_xvmaddasp: 7020 case PPC::BI__builtin_vsx_xvnmaddadp: 7021 case PPC::BI__builtin_vsx_xvnmaddasp: 7022 case PPC::BI__builtin_vsx_xvmsubadp: 7023 case PPC::BI__builtin_vsx_xvmsubasp: 7024 case PPC::BI__builtin_vsx_xvnmsubadp: 7025 case PPC::BI__builtin_vsx_xvnmsubasp: { 7026 llvm::Type *ResultType = ConvertType(E->getType()); 7027 Value *X = EmitScalarExpr(E->getArg(0)); 7028 Value *Y = EmitScalarExpr(E->getArg(1)); 7029 Value *Z = EmitScalarExpr(E->getArg(2)); 7030 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7031 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7032 switch (BuiltinID) { 7033 case PPC::BI__builtin_vsx_xvmaddadp: 7034 case PPC::BI__builtin_vsx_xvmaddasp: 7035 return Builder.CreateCall(F, {X, Y, Z}); 7036 case PPC::BI__builtin_vsx_xvnmaddadp: 7037 case PPC::BI__builtin_vsx_xvnmaddasp: 7038 return Builder.CreateFSub(Zero, 7039 Builder.CreateCall(F, {X, Y, Z}), "sub"); 7040 case PPC::BI__builtin_vsx_xvmsubadp: 7041 case PPC::BI__builtin_vsx_xvmsubasp: 7042 return Builder.CreateCall(F, 7043 {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7044 case PPC::BI__builtin_vsx_xvnmsubadp: 7045 case PPC::BI__builtin_vsx_xvnmsubasp: 7046 Value *FsubRes = 7047 Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7048 return Builder.CreateFSub(Zero, FsubRes, "sub"); 7049 } 7050 llvm_unreachable("Unknown FMA operation"); 7051 return nullptr; // Suppress no-return warning 7052 } 7053 } 7054 } 7055 7056 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, 7057 const CallExpr *E) { 7058 switch (BuiltinID) { 7059 case AMDGPU::BI__builtin_amdgcn_div_scale: 7060 case AMDGPU::BI__builtin_amdgcn_div_scalef: { 7061 // Translate from the intrinsics's struct return to the builtin's out 7062 // argument. 7063 7064 Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3)); 7065 7066 llvm::Value *X = EmitScalarExpr(E->getArg(0)); 7067 llvm::Value *Y = EmitScalarExpr(E->getArg(1)); 7068 llvm::Value *Z = EmitScalarExpr(E->getArg(2)); 7069 7070 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale, 7071 X->getType()); 7072 7073 llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); 7074 7075 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); 7076 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); 7077 7078 llvm::Type *RealFlagType 7079 = FlagOutPtr.getPointer()->getType()->getPointerElementType(); 7080 7081 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); 7082 Builder.CreateStore(FlagExt, FlagOutPtr); 7083 return Result; 7084 } 7085 case AMDGPU::BI__builtin_amdgcn_div_fmas: 7086 case AMDGPU::BI__builtin_amdgcn_div_fmasf: { 7087 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); 7088 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); 7089 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); 7090 llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); 7091 7092 llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas, 7093 Src0->getType()); 7094 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); 7095 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); 7096 } 7097 case AMDGPU::BI__builtin_amdgcn_div_fixup: 7098 case AMDGPU::BI__builtin_amdgcn_div_fixupf: 7099 return emitTernaryFPBuiltin(*this, E, Intrinsic::amdgcn_div_fixup); 7100 case AMDGPU::BI__builtin_amdgcn_trig_preop: 7101 case AMDGPU::BI__builtin_amdgcn_trig_preopf: 7102 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop); 7103 case AMDGPU::BI__builtin_amdgcn_rcp: 7104 case AMDGPU::BI__builtin_amdgcn_rcpf: 7105 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp); 7106 case AMDGPU::BI__builtin_amdgcn_rsq: 7107 case AMDGPU::BI__builtin_amdgcn_rsqf: 7108 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7109 case AMDGPU::BI__builtin_amdgcn_rsq_clamp: 7110 case AMDGPU::BI__builtin_amdgcn_rsq_clampf: 7111 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp); 7112 case AMDGPU::BI__builtin_amdgcn_sinf: 7113 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin); 7114 case AMDGPU::BI__builtin_amdgcn_cosf: 7115 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos); 7116 case AMDGPU::BI__builtin_amdgcn_log_clampf: 7117 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp); 7118 case AMDGPU::BI__builtin_amdgcn_ldexp: 7119 case AMDGPU::BI__builtin_amdgcn_ldexpf: 7120 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7121 case AMDGPU::BI__builtin_amdgcn_frexp_mant: 7122 case AMDGPU::BI__builtin_amdgcn_frexp_mantf: { 7123 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant); 7124 } 7125 case AMDGPU::BI__builtin_amdgcn_frexp_exp: 7126 case AMDGPU::BI__builtin_amdgcn_frexp_expf: { 7127 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_exp); 7128 } 7129 case AMDGPU::BI__builtin_amdgcn_class: 7130 case AMDGPU::BI__builtin_amdgcn_classf: 7131 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class); 7132 7133 // Legacy amdgpu prefix 7134 case AMDGPU::BI__builtin_amdgpu_rsq: 7135 case AMDGPU::BI__builtin_amdgpu_rsqf: { 7136 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7137 return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq); 7138 return emitUnaryBuiltin(*this, E, Intrinsic::r600_rsq); 7139 } 7140 case AMDGPU::BI__builtin_amdgpu_ldexp: 7141 case AMDGPU::BI__builtin_amdgpu_ldexpf: { 7142 if (getTarget().getTriple().getArch() == Triple::amdgcn) 7143 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp); 7144 return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); 7145 } 7146 default: 7147 return nullptr; 7148 } 7149 } 7150 7151 /// Handle a SystemZ function in which the final argument is a pointer 7152 /// to an int that receives the post-instruction CC value. At the LLVM level 7153 /// this is represented as a function that returns a {result, cc} pair. 7154 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, 7155 unsigned IntrinsicID, 7156 const CallExpr *E) { 7157 unsigned NumArgs = E->getNumArgs() - 1; 7158 SmallVector<Value *, 8> Args(NumArgs); 7159 for (unsigned I = 0; I < NumArgs; ++I) 7160 Args[I] = CGF.EmitScalarExpr(E->getArg(I)); 7161 Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs)); 7162 Value *F = CGF.CGM.getIntrinsic(IntrinsicID); 7163 Value *Call = CGF.Builder.CreateCall(F, Args); 7164 Value *CC = CGF.Builder.CreateExtractValue(Call, 1); 7165 CGF.Builder.CreateStore(CC, CCPtr); 7166 return CGF.Builder.CreateExtractValue(Call, 0); 7167 } 7168 7169 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, 7170 const CallExpr *E) { 7171 switch (BuiltinID) { 7172 case SystemZ::BI__builtin_tbegin: { 7173 Value *TDB = EmitScalarExpr(E->getArg(0)); 7174 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7175 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); 7176 return Builder.CreateCall(F, {TDB, Control}); 7177 } 7178 case SystemZ::BI__builtin_tbegin_nofloat: { 7179 Value *TDB = EmitScalarExpr(E->getArg(0)); 7180 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); 7181 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); 7182 return Builder.CreateCall(F, {TDB, Control}); 7183 } 7184 case SystemZ::BI__builtin_tbeginc: { 7185 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); 7186 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); 7187 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); 7188 return Builder.CreateCall(F, {TDB, Control}); 7189 } 7190 case SystemZ::BI__builtin_tabort: { 7191 Value *Data = EmitScalarExpr(E->getArg(0)); 7192 Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); 7193 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); 7194 } 7195 case SystemZ::BI__builtin_non_tx_store: { 7196 Value *Address = EmitScalarExpr(E->getArg(0)); 7197 Value *Data = EmitScalarExpr(E->getArg(1)); 7198 Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); 7199 return Builder.CreateCall(F, {Data, Address}); 7200 } 7201 7202 // Vector builtins. Note that most vector builtins are mapped automatically 7203 // to target-specific LLVM intrinsics. The ones handled specially here can 7204 // be represented via standard LLVM IR, which is preferable to enable common 7205 // LLVM optimizations. 7206 7207 case SystemZ::BI__builtin_s390_vpopctb: 7208 case SystemZ::BI__builtin_s390_vpopcth: 7209 case SystemZ::BI__builtin_s390_vpopctf: 7210 case SystemZ::BI__builtin_s390_vpopctg: { 7211 llvm::Type *ResultType = ConvertType(E->getType()); 7212 Value *X = EmitScalarExpr(E->getArg(0)); 7213 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); 7214 return Builder.CreateCall(F, X); 7215 } 7216 7217 case SystemZ::BI__builtin_s390_vclzb: 7218 case SystemZ::BI__builtin_s390_vclzh: 7219 case SystemZ::BI__builtin_s390_vclzf: 7220 case SystemZ::BI__builtin_s390_vclzg: { 7221 llvm::Type *ResultType = ConvertType(E->getType()); 7222 Value *X = EmitScalarExpr(E->getArg(0)); 7223 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7224 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); 7225 return Builder.CreateCall(F, {X, Undef}); 7226 } 7227 7228 case SystemZ::BI__builtin_s390_vctzb: 7229 case SystemZ::BI__builtin_s390_vctzh: 7230 case SystemZ::BI__builtin_s390_vctzf: 7231 case SystemZ::BI__builtin_s390_vctzg: { 7232 llvm::Type *ResultType = ConvertType(E->getType()); 7233 Value *X = EmitScalarExpr(E->getArg(0)); 7234 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); 7235 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); 7236 return Builder.CreateCall(F, {X, Undef}); 7237 } 7238 7239 case SystemZ::BI__builtin_s390_vfsqdb: { 7240 llvm::Type *ResultType = ConvertType(E->getType()); 7241 Value *X = EmitScalarExpr(E->getArg(0)); 7242 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); 7243 return Builder.CreateCall(F, X); 7244 } 7245 case SystemZ::BI__builtin_s390_vfmadb: { 7246 llvm::Type *ResultType = ConvertType(E->getType()); 7247 Value *X = EmitScalarExpr(E->getArg(0)); 7248 Value *Y = EmitScalarExpr(E->getArg(1)); 7249 Value *Z = EmitScalarExpr(E->getArg(2)); 7250 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7251 return Builder.CreateCall(F, {X, Y, Z}); 7252 } 7253 case SystemZ::BI__builtin_s390_vfmsdb: { 7254 llvm::Type *ResultType = ConvertType(E->getType()); 7255 Value *X = EmitScalarExpr(E->getArg(0)); 7256 Value *Y = EmitScalarExpr(E->getArg(1)); 7257 Value *Z = EmitScalarExpr(E->getArg(2)); 7258 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7259 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); 7260 return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); 7261 } 7262 case SystemZ::BI__builtin_s390_vflpdb: { 7263 llvm::Type *ResultType = ConvertType(E->getType()); 7264 Value *X = EmitScalarExpr(E->getArg(0)); 7265 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7266 return Builder.CreateCall(F, X); 7267 } 7268 case SystemZ::BI__builtin_s390_vflndb: { 7269 llvm::Type *ResultType = ConvertType(E->getType()); 7270 Value *X = EmitScalarExpr(E->getArg(0)); 7271 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); 7272 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); 7273 return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); 7274 } 7275 case SystemZ::BI__builtin_s390_vfidb: { 7276 llvm::Type *ResultType = ConvertType(E->getType()); 7277 Value *X = EmitScalarExpr(E->getArg(0)); 7278 // Constant-fold the M4 and M5 mask arguments. 7279 llvm::APSInt M4, M5; 7280 bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); 7281 bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); 7282 assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); 7283 (void)IsConstM4; (void)IsConstM5; 7284 // Check whether this instance of vfidb can be represented via a LLVM 7285 // standard intrinsic. We only support some combinations of M4 and M5. 7286 Intrinsic::ID ID = Intrinsic::not_intrinsic; 7287 switch (M4.getZExtValue()) { 7288 default: break; 7289 case 0: // IEEE-inexact exception allowed 7290 switch (M5.getZExtValue()) { 7291 default: break; 7292 case 0: ID = Intrinsic::rint; break; 7293 } 7294 break; 7295 case 4: // IEEE-inexact exception suppressed 7296 switch (M5.getZExtValue()) { 7297 default: break; 7298 case 0: ID = Intrinsic::nearbyint; break; 7299 case 1: ID = Intrinsic::round; break; 7300 case 5: ID = Intrinsic::trunc; break; 7301 case 6: ID = Intrinsic::ceil; break; 7302 case 7: ID = Intrinsic::floor; break; 7303 } 7304 break; 7305 } 7306 if (ID != Intrinsic::not_intrinsic) { 7307 Function *F = CGM.getIntrinsic(ID, ResultType); 7308 return Builder.CreateCall(F, X); 7309 } 7310 Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); 7311 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); 7312 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); 7313 return Builder.CreateCall(F, {X, M4Value, M5Value}); 7314 } 7315 7316 // Vector intrisincs that output the post-instruction CC value. 7317 7318 #define INTRINSIC_WITH_CC(NAME) \ 7319 case SystemZ::BI__builtin_##NAME: \ 7320 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 7321 7322 INTRINSIC_WITH_CC(s390_vpkshs); 7323 INTRINSIC_WITH_CC(s390_vpksfs); 7324 INTRINSIC_WITH_CC(s390_vpksgs); 7325 7326 INTRINSIC_WITH_CC(s390_vpklshs); 7327 INTRINSIC_WITH_CC(s390_vpklsfs); 7328 INTRINSIC_WITH_CC(s390_vpklsgs); 7329 7330 INTRINSIC_WITH_CC(s390_vceqbs); 7331 INTRINSIC_WITH_CC(s390_vceqhs); 7332 INTRINSIC_WITH_CC(s390_vceqfs); 7333 INTRINSIC_WITH_CC(s390_vceqgs); 7334 7335 INTRINSIC_WITH_CC(s390_vchbs); 7336 INTRINSIC_WITH_CC(s390_vchhs); 7337 INTRINSIC_WITH_CC(s390_vchfs); 7338 INTRINSIC_WITH_CC(s390_vchgs); 7339 7340 INTRINSIC_WITH_CC(s390_vchlbs); 7341 INTRINSIC_WITH_CC(s390_vchlhs); 7342 INTRINSIC_WITH_CC(s390_vchlfs); 7343 INTRINSIC_WITH_CC(s390_vchlgs); 7344 7345 INTRINSIC_WITH_CC(s390_vfaebs); 7346 INTRINSIC_WITH_CC(s390_vfaehs); 7347 INTRINSIC_WITH_CC(s390_vfaefs); 7348 7349 INTRINSIC_WITH_CC(s390_vfaezbs); 7350 INTRINSIC_WITH_CC(s390_vfaezhs); 7351 INTRINSIC_WITH_CC(s390_vfaezfs); 7352 7353 INTRINSIC_WITH_CC(s390_vfeebs); 7354 INTRINSIC_WITH_CC(s390_vfeehs); 7355 INTRINSIC_WITH_CC(s390_vfeefs); 7356 7357 INTRINSIC_WITH_CC(s390_vfeezbs); 7358 INTRINSIC_WITH_CC(s390_vfeezhs); 7359 INTRINSIC_WITH_CC(s390_vfeezfs); 7360 7361 INTRINSIC_WITH_CC(s390_vfenebs); 7362 INTRINSIC_WITH_CC(s390_vfenehs); 7363 INTRINSIC_WITH_CC(s390_vfenefs); 7364 7365 INTRINSIC_WITH_CC(s390_vfenezbs); 7366 INTRINSIC_WITH_CC(s390_vfenezhs); 7367 INTRINSIC_WITH_CC(s390_vfenezfs); 7368 7369 INTRINSIC_WITH_CC(s390_vistrbs); 7370 INTRINSIC_WITH_CC(s390_vistrhs); 7371 INTRINSIC_WITH_CC(s390_vistrfs); 7372 7373 INTRINSIC_WITH_CC(s390_vstrcbs); 7374 INTRINSIC_WITH_CC(s390_vstrchs); 7375 INTRINSIC_WITH_CC(s390_vstrcfs); 7376 7377 INTRINSIC_WITH_CC(s390_vstrczbs); 7378 INTRINSIC_WITH_CC(s390_vstrczhs); 7379 INTRINSIC_WITH_CC(s390_vstrczfs); 7380 7381 INTRINSIC_WITH_CC(s390_vfcedbs); 7382 INTRINSIC_WITH_CC(s390_vfchdbs); 7383 INTRINSIC_WITH_CC(s390_vfchedbs); 7384 7385 INTRINSIC_WITH_CC(s390_vftcidb); 7386 7387 #undef INTRINSIC_WITH_CC 7388 7389 default: 7390 return nullptr; 7391 } 7392 } 7393 7394 Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, 7395 const CallExpr *E) { 7396 auto MakeLdg = [&](unsigned IntrinsicID) { 7397 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7398 AlignmentSource AlignSource; 7399 clang::CharUnits Align = 7400 getNaturalPointeeTypeAlignment(E->getArg(0)->getType(), &AlignSource); 7401 return Builder.CreateCall( 7402 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(), 7403 Ptr->getType()}), 7404 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())}); 7405 }; 7406 7407 switch (BuiltinID) { 7408 case NVPTX::BI__nvvm_atom_add_gen_i: 7409 case NVPTX::BI__nvvm_atom_add_gen_l: 7410 case NVPTX::BI__nvvm_atom_add_gen_ll: 7411 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E); 7412 7413 case NVPTX::BI__nvvm_atom_sub_gen_i: 7414 case NVPTX::BI__nvvm_atom_sub_gen_l: 7415 case NVPTX::BI__nvvm_atom_sub_gen_ll: 7416 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E); 7417 7418 case NVPTX::BI__nvvm_atom_and_gen_i: 7419 case NVPTX::BI__nvvm_atom_and_gen_l: 7420 case NVPTX::BI__nvvm_atom_and_gen_ll: 7421 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E); 7422 7423 case NVPTX::BI__nvvm_atom_or_gen_i: 7424 case NVPTX::BI__nvvm_atom_or_gen_l: 7425 case NVPTX::BI__nvvm_atom_or_gen_ll: 7426 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E); 7427 7428 case NVPTX::BI__nvvm_atom_xor_gen_i: 7429 case NVPTX::BI__nvvm_atom_xor_gen_l: 7430 case NVPTX::BI__nvvm_atom_xor_gen_ll: 7431 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E); 7432 7433 case NVPTX::BI__nvvm_atom_xchg_gen_i: 7434 case NVPTX::BI__nvvm_atom_xchg_gen_l: 7435 case NVPTX::BI__nvvm_atom_xchg_gen_ll: 7436 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E); 7437 7438 case NVPTX::BI__nvvm_atom_max_gen_i: 7439 case NVPTX::BI__nvvm_atom_max_gen_l: 7440 case NVPTX::BI__nvvm_atom_max_gen_ll: 7441 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E); 7442 7443 case NVPTX::BI__nvvm_atom_max_gen_ui: 7444 case NVPTX::BI__nvvm_atom_max_gen_ul: 7445 case NVPTX::BI__nvvm_atom_max_gen_ull: 7446 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E); 7447 7448 case NVPTX::BI__nvvm_atom_min_gen_i: 7449 case NVPTX::BI__nvvm_atom_min_gen_l: 7450 case NVPTX::BI__nvvm_atom_min_gen_ll: 7451 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E); 7452 7453 case NVPTX::BI__nvvm_atom_min_gen_ui: 7454 case NVPTX::BI__nvvm_atom_min_gen_ul: 7455 case NVPTX::BI__nvvm_atom_min_gen_ull: 7456 return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E); 7457 7458 case NVPTX::BI__nvvm_atom_cas_gen_i: 7459 case NVPTX::BI__nvvm_atom_cas_gen_l: 7460 case NVPTX::BI__nvvm_atom_cas_gen_ll: 7461 // __nvvm_atom_cas_gen_* should return the old value rather than the 7462 // success flag. 7463 return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false); 7464 7465 case NVPTX::BI__nvvm_atom_add_gen_f: { 7466 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7467 Value *Val = EmitScalarExpr(E->getArg(1)); 7468 // atomicrmw only deals with integer arguments so we need to use 7469 // LLVM's nvvm_atomic_load_add_f32 intrinsic for that. 7470 Value *FnALAF32 = 7471 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType()); 7472 return Builder.CreateCall(FnALAF32, {Ptr, Val}); 7473 } 7474 7475 case NVPTX::BI__nvvm_atom_inc_gen_ui: { 7476 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7477 Value *Val = EmitScalarExpr(E->getArg(1)); 7478 Value *FnALI32 = 7479 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType()); 7480 return Builder.CreateCall(FnALI32, {Ptr, Val}); 7481 } 7482 7483 case NVPTX::BI__nvvm_atom_dec_gen_ui: { 7484 Value *Ptr = EmitScalarExpr(E->getArg(0)); 7485 Value *Val = EmitScalarExpr(E->getArg(1)); 7486 Value *FnALD32 = 7487 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType()); 7488 return Builder.CreateCall(FnALD32, {Ptr, Val}); 7489 } 7490 7491 case NVPTX::BI__nvvm_ldg_c: 7492 case NVPTX::BI__nvvm_ldg_c2: 7493 case NVPTX::BI__nvvm_ldg_c4: 7494 case NVPTX::BI__nvvm_ldg_s: 7495 case NVPTX::BI__nvvm_ldg_s2: 7496 case NVPTX::BI__nvvm_ldg_s4: 7497 case NVPTX::BI__nvvm_ldg_i: 7498 case NVPTX::BI__nvvm_ldg_i2: 7499 case NVPTX::BI__nvvm_ldg_i4: 7500 case NVPTX::BI__nvvm_ldg_l: 7501 case NVPTX::BI__nvvm_ldg_ll: 7502 case NVPTX::BI__nvvm_ldg_ll2: 7503 case NVPTX::BI__nvvm_ldg_uc: 7504 case NVPTX::BI__nvvm_ldg_uc2: 7505 case NVPTX::BI__nvvm_ldg_uc4: 7506 case NVPTX::BI__nvvm_ldg_us: 7507 case NVPTX::BI__nvvm_ldg_us2: 7508 case NVPTX::BI__nvvm_ldg_us4: 7509 case NVPTX::BI__nvvm_ldg_ui: 7510 case NVPTX::BI__nvvm_ldg_ui2: 7511 case NVPTX::BI__nvvm_ldg_ui4: 7512 case NVPTX::BI__nvvm_ldg_ul: 7513 case NVPTX::BI__nvvm_ldg_ull: 7514 case NVPTX::BI__nvvm_ldg_ull2: 7515 // PTX Interoperability section 2.2: "For a vector with an even number of 7516 // elements, its alignment is set to number of elements times the alignment 7517 // of its member: n*alignof(t)." 7518 return MakeLdg(Intrinsic::nvvm_ldg_global_i); 7519 case NVPTX::BI__nvvm_ldg_f: 7520 case NVPTX::BI__nvvm_ldg_f2: 7521 case NVPTX::BI__nvvm_ldg_f4: 7522 case NVPTX::BI__nvvm_ldg_d: 7523 case NVPTX::BI__nvvm_ldg_d2: 7524 return MakeLdg(Intrinsic::nvvm_ldg_global_f); 7525 default: 7526 return nullptr; 7527 } 7528 } 7529 7530 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, 7531 const CallExpr *E) { 7532 switch (BuiltinID) { 7533 case WebAssembly::BI__builtin_wasm_current_memory: { 7534 llvm::Type *ResultType = ConvertType(E->getType()); 7535 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType); 7536 return Builder.CreateCall(Callee); 7537 } 7538 case WebAssembly::BI__builtin_wasm_grow_memory: { 7539 Value *X = EmitScalarExpr(E->getArg(0)); 7540 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType()); 7541 return Builder.CreateCall(Callee, X); 7542 } 7543 7544 default: 7545 return nullptr; 7546 } 7547 } 7548